schain Commit - r1074:238914c187b3 · Jicamarca Repository

merge revisado

José Chávez -

r1074:238914c187b3

parent child

Context file:

r1074:238914c187b3

Collapse all files

schainpy/model/data/BLTRheaderIO.py created 644 +404 0

			@@ -0,0 +1,404
		1	'''
		2
		3	$Author: murco $
		4	$Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
		5	'''
		6	import sys
		7	import numpy
		8	import copy
		9	import datetime
		10	from __builtin__ import None
		11
		12	SPEED_OF_LIGHT = 299792458
		13	SPEED_OF_LIGHT = 3e8
		14
		15	FILE_STRUCTURE = numpy.dtype([ #HEADER 48bytes
		16	('FileMgcNumber','<u4'), #0x23020100
		17	('nFDTdataRecors','<u4'), #No Of FDT data records in this file (0 or more)
		18	('RadarUnitId','<u4'),
		19	('SiteName','<s32'), #Null terminated
		20	])
		21
		22	RECORD_STRUCTURE = numpy.dtype([ #RECORD HEADER 180+20N bytes
		23	('RecMgcNumber','<u4'), #0x23030001
		24	('RecCounter','<u4'), #Record counter(0,1, ...)
		25	('Off2StartNxtRec','<u4'), #Offset to start of next record form start of this record
		26	('Off2StartData','<u4'), #Offset to start of data from start of this record
		27	('EpTimeStamp','<i4'), #Epoch time stamp of start of acquisition (seconds)
		28	('msCompTimeStamp','<u4'), #Millisecond component of time stamp (0,...,999)
		29	('ExpTagName','<s32'), #Experiment tag name (null terminated)
		30	('ExpComment','<s32'), #Experiment comment (null terminated)
		31	('SiteLatDegrees','<f4'), #Site latitude (from GPS) in degrees (positive implies North)
		32	('SiteLongDegrees','<f4'), #Site longitude (from GPS) in degrees (positive implies East)
		33	('RTCgpsStatus','<u4'), #RTC GPS engine status (0=SEEK, 1=LOCK, 2=NOT FITTED, 3=UNAVAILABLE)
		34	('TransmitFrec','<u4'), #Transmit frequency (Hz)
		35	('ReceiveFrec','<u4'), #Receive frequency
		36	('FirstOsciFrec','<u4'), #First local oscillator frequency (Hz)
		37	('Polarisation','<u4'), #(0="O", 1="E", 2="linear 1", 3="linear2")
		38	('ReceiverFiltSett','<u4'), #Receiver filter settings (0,1,2,3)
		39	('nModesInUse','<u4'), #Number of modes in use (1 or 2)
		40	('DualModeIndex','<u4'), #Dual Mode index number for these data (0 or 1)
		41	('DualModeRange','<u4'), #Dual Mode range correction for these data (m)
		42	('nDigChannels','<u4'), #Number of digital channels acquired (2*N)
		43	('SampResolution','<u4'), #Sampling resolution (meters)
		44	('nRangeGatesSamp','<u4'), #Number of range gates sampled
		45	('StartRangeSamp','<u4'), #Start range of sampling (meters)
		46	('PRFhz','<u4'), #PRF (Hz)
		47	('Integrations','<u4'), #Integrations
		48	('nDataPointsTrsf','<u4'), #Number of data points transformed
		49	('nReceiveBeams','<u4'), #Number of receive beams stored in file (1 or N)
		50	('nSpectAverages','<u4'), #Number of spectral averages
		51	('FFTwindowingInd','<u4'), #FFT windowing index (0 = no window)
		52	('BeamAngleAzim','<f4'), #Beam steer angle (azimuth) in degrees (clockwise from true North)
		53	('BeamAngleZen','<f4'), #Beam steer angle (zenith) in degrees (0=> vertical)
		54	('AntennaCoord','<f24'), #Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
		55	('RecPhaseCalibr','<f12'), #Receiver phase calibration (degrees) - N values
		56	('RecAmpCalibr','<f12'), #Receiver amplitude calibration (ratio relative to receiver one) - N values
		57	('ReceiverGaindB','<u12'), #Receiver gains in dB - N values
		58	])
		59
		60
		61	class Header(object):
		62
		63	def __init__(self):
		64	raise NotImplementedError
		65
		66
		67	def read(self):
		68
		69	raise NotImplementedError
		70
		71	def write(self):
		72
		73	raise NotImplementedError
		74
		75	def printInfo(self):
		76
		77	message = "#"*50 + "\n"
		78	message += self.__class__.__name__.upper() + "\n"
		79	message += "#"*50 + "\n"
		80
		81	keyList = self.__dict__.keys()
		82	keyList.sort()
		83
		84	for key in keyList:
		85	message += "%s = %s" %(key, self.__dict__[key]) + "\n"
		86
		87	if "size" not in keyList:
		88	attr = getattr(self, "size")
		89
		90	if attr:
		91	message += "%s = %s" %("size", attr) + "\n"
		92
		93	print message
		94
		95	class FileHeader(Header):
		96
		97	FileMgcNumber= None
		98	nFDTdataRecors=None #No Of FDT data records in this file (0 or more)
		99	RadarUnitId= None
		100	SiteName= None
		101
		102	#__LOCALTIME = None
		103
		104	def __init__(self, useLocalTime=True):
		105
		106	self.FileMgcNumber= 0 #0x23020100
		107	self.nFDTdataRecors=0 #No Of FDT data records in this file (0 or more)
		108	self.RadarUnitId= 0
		109	self.SiteName= ""
		110	self.size = 48
		111
		112	#self.useLocalTime = useLocalTime
		113
		114	def read(self, fp):
		115
		116	try:
		117	header = numpy.fromfile(fp, FILE_STRUCTURE,1)
		118	''' numpy.fromfile(file, dtype, count, sep='')
		119	file : file or str
		120	Open file object or filename.
		121
		122	dtype : data-type
		123	Data type of the returned array. For binary files, it is used to determine
		124	the size and byte-order of the items in the file.
		125
		126	count : int
		127	Number of items to read. -1 means all items (i.e., the complete file).
		128
		129	sep : str
		130	Separator between items if file is a text file. Empty (“”) separator means
		131	the file should be treated as binary. Spaces (” ”) in the separator match zero
		132	or more whitespace characters. A separator consisting only of spaces must match
		133	at least one whitespace.
		134
		135	'''
		136
		137	except Exception, e:
		138	print "FileHeader: "
		139	print eBasicHeader
		140	return 0
		141
		142	self.FileMgcNumber= byte(header['FileMgcNumber'][0])
		143	self.nFDTdataRecors=int(header['nFDTdataRecors'][0]) #No Of FDT data records in this file (0 or more)
		144	self.RadarUnitId= int(header['RadarUnitId'][0])
		145	self.SiteName= char(header['SiteName'][0])
		146
		147
		148	if self.size <48:
		149	return 0
		150
		151	return 1
		152
		153	def write(self, fp):
		154
		155	headerTuple = (self.FileMgcNumber,
		156	self.nFDTdataRecors,
		157	self.RadarUnitId,
		158	self.SiteName,
		159	self.size)
		160
		161
		162	header = numpy.array(headerTuple, FILE_STRUCTURE)
		163	# numpy.array(object, dtype=None, copy=True, order=None, subok=False, ndmin=0)
		164	header.tofile(fp)
		165	''' ndarray.tofile(fid, sep, format) Write array to a file as text or binary (default).
		166
		167	fid : file or str
		168	An open file object, or a string containing a filename.
		169
		170	sep : str
		171	Separator between array items for text output. If “” (empty), a binary file is written,
		172	equivalent to file.write(a.tobytes()).
		173
		174	format : str
		175	Format string for text file output. Each entry in the array is formatted to text by
		176	first converting it to the closest Python type, and then using “format” % item.
		177
		178	'''
		179
		180	return 1
		181
		182
		183	class RecordHeader(Header):
		184
		185	RecMgcNumber=None #0x23030001
		186	RecCounter= None
		187	Off2StartNxtRec= None
		188	EpTimeStamp= None
		189	msCompTimeStamp= None
		190	ExpTagName= None
		191	ExpComment=None
		192	SiteLatDegrees=None
		193	SiteLongDegrees= None
		194	RTCgpsStatus= None
		195	TransmitFrec= None
		196	ReceiveFrec= None
		197	FirstOsciFrec= None
		198	Polarisation= None
		199	ReceiverFiltSett= None
		200	nModesInUse= None
		201	DualModeIndex= None
		202	DualModeRange= None
		203	nDigChannels= None
		204	SampResolution= None
		205	nRangeGatesSamp= None
		206	StartRangeSamp= None
		207	PRFhz= None
		208	Integrations= None
		209	nDataPointsTrsf= None
		210	nReceiveBeams= None
		211	nSpectAverages= None
		212	FFTwindowingInd= None
		213	BeamAngleAzim= None
		214	BeamAngleZen= None
		215	AntennaCoord= None
		216	RecPhaseCalibr= None
		217	RecAmpCalibr= None
		218	ReceiverGaindB= None
		219
		220	'''size = None
		221	nSamples = None
		222	nProfiles = None
		223	nChannels = None
		224	adcResolution = None
		225	pciDioBusWidth = None'''
		226
		227	def __init__(self, RecMgcNumber=None, RecCounter= 0, Off2StartNxtRec= 0,
		228	EpTimeStamp= 0, msCompTimeStamp= 0, ExpTagName= None,
		229	ExpComment=None, SiteLatDegrees=0, SiteLongDegrees= 0,
		230	RTCgpsStatus= 0, TransmitFrec= 0, ReceiveFrec= 0,
		231	FirstOsciFrec= 0, Polarisation= 0, ReceiverFiltSett= 0,
		232	nModesInUse= 0, DualModeIndex= 0, DualModeRange= 0,
		233	nDigChannels= 0, SampResolution= 0, nRangeGatesSamp= 0,
		234	StartRangeSamp= 0, PRFhz= 0, Integrations= 0,
		235	nDataPointsTrsf= 0, nReceiveBeams= 0, nSpectAverages= 0,
		236	FFTwindowingInd= 0, BeamAngleAzim= 0, BeamAngleZen= 0,
		237	AntennaCoord= 0, RecPhaseCalibr= 0, RecAmpCalibr= 0,
		238	ReceiverGaindB= 0):
		239
		240	self.RecMgcNumber = RecMgcNumber #0x23030001
		241	self.RecCounter = RecCounter
		242	self.Off2StartNxtRec = Off2StartNxtRec
		243	self.EpTimeStamp = EpTimeStamp
		244	self.msCompTimeStamp = msCompTimeStamp
		245	self.ExpTagName = ExpTagName
		246	self.ExpComment = ExpComment
		247	self.SiteLatDegrees = SiteLatDegrees
		248	self.SiteLongDegrees = SiteLongDegrees
		249	self.RTCgpsStatus = RTCgpsStatus
		250	self.TransmitFrec = TransmitFrec
		251	self.ReceiveFrec = ReceiveFrec
		252	self.FirstOsciFrec = FirstOsciFrec
		253	self.Polarisation = Polarisation
		254	self.ReceiverFiltSett = ReceiverFiltSett
		255	self.nModesInUse = nModesInUse
		256	self.DualModeIndex = DualModeIndex
		257	self.DualModeRange = DualModeRange
		258	self.nDigChannels = nDigChannels
		259	self.SampResolution = SampResolution
		260	self.nRangeGatesSamp = nRangeGatesSamp
		261	self.StartRangeSamp = StartRangeSamp
		262	self.PRFhz = PRFhz
		263	self.Integrations = Integrations
		264	self.nDataPointsTrsf = nDataPointsTrsf
		265	self.nReceiveBeams = nReceiveBeams
		266	self.nSpectAverages = nSpectAverages
		267	self.FFTwindowingInd = FFTwindowingInd
		268	self.BeamAngleAzim = BeamAngleAzim
		269	self.BeamAngleZen = BeamAngleZen
		270	self.AntennaCoord = AntennaCoord
		271	self.RecPhaseCalibr = RecPhaseCalibr
		272	self.RecAmpCalibr = RecAmpCalibr
		273	self.ReceiverGaindB = ReceiverGaindB
		274
		275
		276	def read(self, fp):
		277
		278	startFp = fp.tell() #The method tell() returns the current position of the file read/write pointer within the file.
		279
		280	try:
		281	header = numpy.fromfile(fp,RECORD_STRUCTURE,1)
		282	except Exception, e:
		283	print "System Header: " + e
		284	return 0
		285
		286	self.RecMgcNumber = header['RecMgcNumber'][0] #0x23030001
		287	self.RecCounter = header['RecCounter'][0]
		288	self.Off2StartNxtRec = header['Off2StartNxtRec'][0]
		289	self.EpTimeStamp = header['EpTimeStamp'][0]
		290	self.msCompTimeStamp = header['msCompTimeStamp'][0]
		291	self.ExpTagName = header['ExpTagName'][0]
		292	self.ExpComment = header['ExpComment'][0]
		293	self.SiteLatDegrees = header['SiteLatDegrees'][0]
		294	self.SiteLongDegrees = header['SiteLongDegrees'][0]
		295	self.RTCgpsStatus = header['RTCgpsStatus'][0]
		296	self.TransmitFrec = header['TransmitFrec'][0]
		297	self.ReceiveFrec = header['ReceiveFrec'][0]
		298	self.FirstOsciFrec = header['FirstOsciFrec'][0]
		299	self.Polarisation = header['Polarisation'][0]
		300	self.ReceiverFiltSett = header['ReceiverFiltSett'][0]
		301	self.nModesInUse = header['nModesInUse'][0]
		302	self.DualModeIndex = header['DualModeIndex'][0]
		303	self.DualModeRange = header['DualModeRange'][0]
		304	self.nDigChannels = header['nDigChannels'][0]
		305	self.SampResolution = header['SampResolution'][0]
		306	self.nRangeGatesSamp = header['nRangeGatesSamp'][0]
		307	self.StartRangeSamp = header['StartRangeSamp'][0]
		308	self.PRFhz = header['PRFhz'][0]
		309	self.Integrations = header['Integrations'][0]
		310	self.nDataPointsTrsf = header['nDataPointsTrsf'][0]
		311	self.nReceiveBeams = header['nReceiveBeams'][0]
		312	self.nSpectAverages = header['nSpectAverages'][0]
		313	self.FFTwindowingInd = header['FFTwindowingInd'][0]
		314	self.BeamAngleAzim = header['BeamAngleAzim'][0]
		315	self.BeamAngleZen = header['BeamAngleZen'][0]
		316	self.AntennaCoord = header['AntennaCoord'][0]
		317	self.RecPhaseCalibr = header['RecPhaseCalibr'][0]
		318	self.RecAmpCalibr = header['RecAmpCalibr'][0]
		319	self.ReceiverGaindB = header['ReceiverGaindB'][0]
		320
		321	Self.size = 180+20*3
		322
		323	endFp = self.size + startFp
		324
		325	if fp.tell() > endFp:
		326	sys.stderr.write("Warning %s: Size value read from System Header is lower than it has to be\n" %fp.name)
		327	return 0
		328
		329	if fp.tell() < endFp:
		330	sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" %fp.name)
		331	return 0
		332
		333	return 1
		334
		335	def write(self, fp):
		336
		337	headerTuple = (self.RecMgcNumber,
		338	self.RecCounter,
		339	self.Off2StartNxtRec,
		340	self.EpTimeStamp,
		341	self.msCompTimeStamp,
		342	self.ExpTagName,
		343	self.ExpComment,
		344	self.SiteLatDegrees,
		345	self.SiteLongDegrees,
		346	self.RTCgpsStatus,
		347	self.TransmitFrec,
		348	self.ReceiveFrec,
		349	self.FirstOsciFrec,
		350	self.Polarisation,
		351	self.ReceiverFiltSett,
		352	self.nModesInUse,
		353	self.DualModeIndex,
		354	self.DualModeRange,
		355	self.nDigChannels,
		356	self.SampResolution,
		357	self.nRangeGatesSamp,
		358	self.StartRangeSamp,
		359	self.PRFhz,
		360	self.Integrations,
		361	self.nDataPointsTrsf,
		362	self.nReceiveBeams,
		363	self.nSpectAverages,
		364	self.FFTwindowingInd,
		365	self.BeamAngleAzim,
		366	self.BeamAngleZen,
		367	self.AntennaCoord,
		368	self.RecPhaseCalibr,
		369	self.RecAmpCalibr,
		370	self.ReceiverGaindB)
		371
		372	# self.size,self.nSamples,
		373	# self.nProfiles,
		374	# self.nChannels,
		375	# self.adcResolution,
		376	# self.pciDioBusWidth
		377
		378	header = numpy.array(headerTuple,RECORD_STRUCTURE)
		379	header.tofile(fp)
		380
		381	return 1
		382
		383
		384	def get_dtype_index(numpy_dtype):
		385
		386	index = None
		387
		388	for i in range(len(NUMPY_DTYPE_LIST)):
		389	if numpy_dtype == NUMPY_DTYPE_LIST[i]:
		390	index = i
		391	break
		392
		393	return index
		394
		395	def get_numpy_dtype(index):
		396
		397	#dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
		398
		399	return NUMPY_DTYPE_LIST[index]
		400
		401
		402	def get_dtype_width(index):
		403
		404	return DTYPE_WIDTH[index] No newline at end of file

schainpy/model/io/MIRAtest.py created 644 +321 0

			@@ -0,0 +1,321
		1	import os, sys
		2	import glob
		3	import fnmatch
		4	import datetime
		5	import time
		6	import re
		7	import h5py
		8	import numpy
		9	import matplotlib.pyplot as plt
		10
		11	import pylab as plb
		12	from scipy.optimize import curve_fit
		13	from scipy import asarray as ar,exp
		14	from scipy import stats
		15
		16	from duplicity.path import Path
		17	from numpy.ma.core import getdata
		18
		19	SPEED_OF_LIGHT = 299792458
		20	SPEED_OF_LIGHT = 3e8
		21
		22	try:
		23	from gevent import sleep
		24	except:
		25	from time import sleep
		26
		27	from schainpy.model.data.jrodata import Spectra
		28	#from schainpy.model.data.BLTRheaderIO import FileHeader, RecordHeader
		29	from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
		30	#from schainpy.model.io.jroIO_bltr import BLTRReader
		31	from numpy import imag, shape, NaN
		32
		33
		34	startFp = open('/home/erick/Documents/MIRA35C/20160117/20160117_0000.zspc',"rb")
		35
		36
		37	FILE_HEADER = numpy.dtype([ #HEADER 1024bytes
		38	('Hname',numpy.str_,32), #Original file name
		39	('Htime',numpy.str_,32), #Date and time when the file was created
		40	('Hoper',numpy.str_,64), #Name of operator who created the file
		41	('Hplace',numpy.str_,128), #Place where the measurements was carried out
		42	('Hdescr',numpy.str_,256), #Description of measurements
		43	('Hdummy',numpy.str_,512), #Reserved space
		44	#Main chunk
		45	('Msign','<i4'), #Main chunk signature FZKF or NUIG
		46	('MsizeData','<i4'), #Size of data block main chunk
		47	#Processing DSP parameters
		48	('PPARsign','<i4'), #PPAR signature
		49	('PPARsize','<i4'), #PPAR size of block
		50	('PPARprf','<i4'), #Pulse repetition frequency
		51	('PPARpdr','<i4'), #Pulse duration
		52	('PPARsft','<i4'), #FFT length
		53	('PPARavc','<i4'), #Number of spectral (in-coherent) averages
		54	('PPARihp','<i4'), #Number of lowest range gate for moment estimation
		55	('PPARchg','<i4'), #Count for gates for moment estimation
		56	('PPARpol','<i4'), #switch on/off polarimetric measurements. Should be 1.
		57	#Service DSP parameters
		58	('SPARatt','<i4'), #STC attenuation on the lowest ranges on/off
		59	('SPARtx','<i4'), #OBSOLETE
		60	('SPARaddGain0','<f4'), #OBSOLETE
		61	('SPARaddGain1','<f4'), #OBSOLETE
		62	('SPARwnd','<i4'), #Debug only. It normal mode it is 0.
		63	('SPARpos','<i4'), #Delay between sync pulse and tx pulse for phase corr, ns
		64	('SPARadd','<i4'), #"add to pulse" to compensate for delay between the leading edge of driver pulse and envelope of the RF signal.
		65	('SPARlen','<i4'), #Time for measuring txn pulse phase. OBSOLETE
		66	('SPARcal','<i4'), #OBSOLETE
		67	('SPARnos','<i4'), #OBSOLETE
		68	('SPARof0','<i4'), #detection threshold
		69	('SPARof1','<i4'), #OBSOLETE
		70	('SPARswt','<i4'), #2nd moment estimation threshold
		71	('SPARsum','<i4'), #OBSOLETE
		72	('SPARosc','<i4'), #flag Oscillosgram mode
		73	('SPARtst','<i4'), #OBSOLETE
		74	('SPARcor','<i4'), #OBSOLETE
		75	('SPARofs','<i4'), #OBSOLETE
		76	('SPARhsn','<i4'), #Hildebrand div noise detection on noise gate
		77	('SPARhsa','<f4'), #Hildebrand div noise detection on all gates
		78	('SPARcalibPow_M','<f4'), #OBSOLETE
		79	('SPARcalibSNR_M','<f4'), #OBSOLETE
		80	('SPARcalibPow_S','<f4'), #OBSOLETE
		81	('SPARcalibSNR_S','<f4'), #OBSOLETE
		82	('SPARrawGate1','<i4'), #Lowest range gate for spectra saving Raw_Gate1 >=5
		83	('SPARrawGate2','<i4'), #Number of range gates with atmospheric signal
		84	('SPARraw','<i4'), #flag - IQ or spectra saving on/off
		85	('SPARprc','<i4'),]) #flag - Moment estimation switched on/off
		86
		87
		88
		89	self.Hname= None
		90	self.Htime= None
		91	self.Hoper= None
		92	self.Hplace= None
		93	self.Hdescr= None
		94	self.Hdummy= None
		95
		96	self.Msign=None
		97	self.MsizeData=None
		98
		99	self.PPARsign=None
		100	self.PPARsize=None
		101	self.PPARprf=None
		102	self.PPARpdr=None
		103	self.PPARsft=None
		104	self.PPARavc=None
		105	self.PPARihp=None
		106	self.PPARchg=None
		107	self.PPARpol=None
		108	#Service DSP parameters
		109	self.SPARatt=None
		110	self.SPARtx=None
		111	self.SPARaddGain0=None
		112	self.SPARaddGain1=None
		113	self.SPARwnd=None
		114	self.SPARpos=None
		115	self.SPARadd=None
		116	self.SPARlen=None
		117	self.SPARcal=None
		118	self.SPARnos=None
		119	self.SPARof0=None
		120	self.SPARof1=None
		121	self.SPARswt=None
		122	self.SPARsum=None
		123	self.SPARosc=None
		124	self.SPARtst=None
		125	self.SPARcor=None
		126	self.SPARofs=None
		127	self.SPARhsn=None
		128	self.SPARhsa=None
		129	self.SPARcalibPow_M=None
		130	self.SPARcalibSNR_M=None
		131	self.SPARcalibPow_S=None
		132	self.SPARcalibSNR_S=None
		133	self.SPARrawGate1=None
		134	self.SPARrawGate2=None
		135	self.SPARraw=None
		136	self.SPARprc=None
		137
		138
		139
		140	header = numpy.fromfile(fp, FILE_HEADER,1)
		141	''' numpy.fromfile(file, dtype, count, sep='')
		142	file : file or str
		143	Open file object or filename.
		144
		145	dtype : data-type
		146	Data type of the returned array. For binary files, it is used to determine
		147	the size and byte-order of the items in the file.
		148
		149	count : int
		150	Number of items to read. -1 means all items (i.e., the complete file).
		151
		152	sep : str
		153	Separator between items if file is a text file. Empty ("") separator means
		154	the file should be treated as binary. Spaces (" ") in the separator match zero
		155	or more whitespace characters. A separator consisting only of spaces must match
		156	at least one whitespace.
		157
		158	'''
		159
		160	Hname= str(header['Hname'][0])
		161	Htime= str(header['Htime'][0])
		162	Hoper= str(header['Hoper'][0])
		163	Hplace= str(header['Hplace'][0])
		164	Hdescr= str(header['Hdescr'][0])
		165	Hdummy= str(header['Hdummy'][0])
		166
		167	Msign=header['Msign'][0]
		168	MsizeData=header['MsizeData'][0]
		169
		170	PPARsign=header['PPARsign'][0]
		171	PPARsize=header['PPARsize'][0]
		172	PPARprf=header['PPARprf'][0]
		173	PPARpdr=header['PPARpdr'][0]
		174	PPARsft=header['PPARsft'][0]
		175	PPARavc=header['PPARavc'][0]
		176	PPARihp=header['PPARihp'][0]
		177	PPARchg=header['PPARchg'][0]
		178	PPARpol=header['PPARpol'][0]
		179	#Service DSP parameters
		180	SPARatt=header['SPARatt'][0]
		181	SPARtx=header['SPARtx'][0]
		182	SPARaddGain0=header['SPARaddGain0'][0]
		183	SPARaddGain1=header['SPARaddGain1'][0]
		184	SPARwnd=header['SPARwnd'][0]
		185	SPARpos=header['SPARpos'][0]
		186	SPARadd=header['SPARadd'][0]
		187	SPARlen=header['SPARlen'][0]
		188	SPARcal=header['SPARcal'][0]
		189	SPARnos=header['SPARnos'][0]
		190	SPARof0=header['SPARof0'][0]
		191	SPARof1=header['SPARof1'][0]
		192	SPARswt=header['SPARswt'][0]
		193	SPARsum=header['SPARsum'][0]
		194	SPARosc=header['SPARosc'][0]
		195	SPARtst=header['SPARtst'][0]
		196	SPARcor=header['SPARcor'][0]
		197	SPARofs=header['SPARofs'][0]
		198	SPARhsn=header['SPARhsn'][0]
		199	SPARhsa=header['SPARhsa'][0]
		200	SPARcalibPow_M=header['SPARcalibPow_M'][0]
		201	SPARcalibSNR_M=header['SPARcalibSNR_M'][0]
		202	SPARcalibPow_S=header['SPARcalibPow_S'][0]
		203	SPARcalibSNR_S=header['SPARcalibSNR_S'][0]
		204	SPARrawGate1=header['SPARrawGate1'][0]
		205	SPARrawGate2=header['SPARrawGate2'][0]
		206	SPARraw=header['SPARraw'][0]
		207	SPARprc=header['SPARprc'][0]
		208
		209
		210
		211	SRVI_STRUCTURE = numpy.dtype([
		212	('frame_cnt','<u4'),#
		213	('time_t','<u4'), #
		214	('tpow','<f4'), #
		215	('npw1','<f4'), #
		216	('npw2','<f4'), #
		217	('cpw1','<f4'), #
		218	('pcw2','<f4'), #
		219	('ps_err','<u4'), #
		220	('te_err','<u4'), #
		221	('rc_err','<u4'), #
		222	('grs1','<u4'), #
		223	('grs2','<u4'), #
		224	('azipos','<f4'), #
		225	('azivel','<f4'), #
		226	('elvpos','<f4'), #
		227	('elvvel','<f4'), #
		228	('northAngle','<f4'), #
		229	('microsec','<u4'), #
		230	('azisetvel','<f4'), #
		231	('elvsetpos','<f4'), #
		232	('RadarConst','<f4'),]) #
		233
		234	JUMP_STRUCTURE = numpy.dtype([
		235	('jump','<u140'),#
		236	('SizeOfDataBlock1',numpy.str_,32),#
		237	('jump','<i4'),#
		238	('DataBlockTitleSRVI1',numpy.str_,32),#
		239	('SizeOfSRVI1','<i4'),])#
		240
		241
		242
		243	#frame_cnt=0, time_t= 0, tpow=0, npw1=0, npw2=0,
		244	#cpw1=0, pcw2=0, ps_err=0, te_err=0, rc_err=0, grs1=0,
		245	#grs2=0, azipos=0, azivel=0, elvpos=0, elvvel=0, northangle=0,
		246	#microsec=0, azisetvel=0, elvsetpos=0, RadarConst=0
		247
		248
		249	frame_cnt = frame_cnt
		250	dwell = time_t
		251	tpow = tpow
		252	npw1 = npw1
		253	npw2 = npw2
		254	cpw1 = cpw1
		255	pcw2 = pcw2
		256	ps_err = ps_err
		257	te_err = te_err
		258	rc_err = rc_err
		259	grs1 = grs1
		260	grs2 = grs2
		261	azipos = azipos
		262	azivel = azivel
		263	elvpos = elvpos
		264	elvvel = elvvel
		265	northAngle = northAngle
		266	microsec = microsec
		267	azisetvel = azisetvel
		268	elvsetpos = elvsetpos
		269	RadarConst5 = RadarConst
		270
		271
		272
		273	#print fp
		274	#startFp = open('/home/erick/Documents/Data/huancayo.20161019.22.fdt',"rb") #The method tell() returns the current position of the file read/write pointer within the file.
		275	#startFp = open(fp,"rb") #The method tell() returns the current position of the file read/write pointer within the file.
		276	#RecCounter=0
		277	#Off2StartNxtRec=811248
		278	#print 'OffsetStartHeader ',self.OffsetStartHeader,'RecCounter ', self.RecCounter, 'Off2StartNxtRec ' , self.Off2StartNxtRec
		279	#OffRHeader= self.OffsetStartHeader + self.RecCounter*self.Off2StartNxtRec
		280	#startFp.seek(OffRHeader, os.SEEK_SET)
		281	print 'debe ser 48, RecCounter*811248', self.OffsetStartHeader,self.RecCounter,self.Off2StartNxtRec
		282	print 'Posicion del bloque: ',OffRHeader
		283
		284	header = numpy.fromfile(startFp,SRVI_STRUCTURE,1)
		285
		286	self.frame_cnt = header['frame_cnt'][0]#
		287	self.time_t = header['frame_cnt'][0] #
		288	self.tpow = header['frame_cnt'][0] #
		289	self.npw1 = header['frame_cnt'][0] #
		290	self.npw2 = header['frame_cnt'][0] #
		291	self.cpw1 = header['frame_cnt'][0] #
		292	self.pcw2 = header['frame_cnt'][0] #
		293	self.ps_err = header['frame_cnt'][0] #
		294	self.te_err = header['frame_cnt'][0] #
		295	self.rc_err = header['frame_cnt'][0] #
		296	self.grs1 = header['frame_cnt'][0] #
		297	self.grs2 = header['frame_cnt'][0] #
		298	self.azipos = header['frame_cnt'][0] #
		299	self.azivel = header['frame_cnt'][0] #
		300	self.elvpos = header['frame_cnt'][0] #
		301	self.elvvel = header['frame_cnt'][0] #
		302	self.northAngle = header['frame_cnt'][0] #
		303	self.microsec = header['frame_cnt'][0] #
		304	self.azisetvel = header['frame_cnt'][0] #
		305	self.elvsetpos = header['frame_cnt'][0] #
		306	self.RadarConst = header['frame_cnt'][0] #
		307
		308
		309	self.ipp= 0.5*(SPEED_OF_LIGHT/self.PRFhz)
		310
		311	self.RHsize = 180+20*self.nChannels
		312	self.Datasize= self.nProfilesself.nChannelsself.nHeights24
		313	#print 'Datasize',self.Datasize
		314	endFp = self.OffsetStartHeader + self.RecCounter*self.Off2StartNxtRec
		315
		316	print '=============================================='
		317
		318	print '=============================================='
		319
		320
		321	No newline at end of file

schainpy/model/io/bltrIO_param.py created 644 +362 0

			@@ -0,0 +1,362
		1	'''
		2	Created on Nov 9, 2016
		3
		4	@author: roj- LouVD
		5	'''
		6
		7
		8	import os
		9	import sys
		10	import time
		11	import glob
		12	import datetime
		13
		14	import numpy
		15
		16	from schainpy.model.proc.jroproc_base import ProcessingUnit
		17	from schainpy.model.data.jrodata import Parameters
		18	from schainpy.model.io.jroIO_base import JRODataReader, isNumber
		19
		20	FILE_HEADER_STRUCTURE = numpy.dtype([
		21	('FMN', '<u4'),
		22	('nrec', '<u4'),
		23	('fr_offset', '<u4'),
		24	('id', '<u4'),
		25	('site', 'u1', (32,))
		26	])
		27
		28	REC_HEADER_STRUCTURE = numpy.dtype([
		29	('rmn', '<u4'),
		30	('rcounter', '<u4'),
		31	('nr_offset', '<u4'),
		32	('tr_offset', '<u4'),
		33	('time', '<u4'),
		34	('time_msec', '<u4'),
		35	('tag', 'u1', (32,)),
		36	('comments', 'u1', (32,)),
		37	('lat', '<f4'),
		38	('lon', '<f4'),
		39	('gps_status', '<u4'),
		40	('freq', '<u4'),
		41	('freq0', '<u4'),
		42	('nchan', '<u4'),
		43	('delta_r', '<u4'),
		44	('nranges', '<u4'),
		45	('r0', '<u4'),
		46	('prf', '<u4'),
		47	('ncoh', '<u4'),
		48	('npoints', '<u4'),
		49	('polarization', '<i4'),
		50	('rx_filter', '<u4'),
		51	('nmodes', '<u4'),
		52	('dmode_index', '<u4'),
		53	('dmode_rngcorr', '<u4'),
		54	('nrxs', '<u4'),
		55	('acf_length', '<u4'),
		56	('acf_lags', '<u4'),
		57	('sea_to_atmos', '<f4'),
		58	('sea_notch', '<u4'),
		59	('lh_sea', '<u4'),
		60	('hh_sea', '<u4'),
		61	('nbins_sea', '<u4'),
		62	('min_snr', '<f4'),
		63	('min_cc', '<f4'),
		64	('max_time_diff', '<f4')
		65	])
		66
		67	DATA_STRUCTURE = numpy.dtype([
		68	('range', '<u4'),
		69	('status', '<u4'),
		70	('zonal', '<f4'),
		71	('meridional', '<f4'),
		72	('vertical', '<f4'),
		73	('zonal_a', '<f4'),
		74	('meridional_a', '<f4'),
		75	('corrected_fading', '<f4'), # seconds
		76	('uncorrected_fading', '<f4'), # seconds
		77	('time_diff', '<f4'),
		78	('major_axis', '<f4'),
		79	('axial_ratio', '<f4'),
		80	('orientation', '<f4'),
		81	('sea_power', '<u4'),
		82	('sea_algorithm', '<u4')
		83	])
		84
		85	class BLTRParamReader(JRODataReader, ProcessingUnit):
		86	'''
		87	Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR from *.sswma files
		88	'''
		89
		90	ext = '.sswma'
		91
		92	def __init__(self, **kwargs):
		93
		94	ProcessingUnit.__init__(self , **kwargs)
		95
		96	self.dataOut = Parameters()
		97	self.counter_records = 0
		98	self.flagNoMoreFiles = 0
		99	self.isConfig = False
		100	self.filename = None
		101
		102	def setup(self,
		103	path=None,
		104	startDate=None,
		105	endDate=None,
		106	ext=None,
		107	startTime=datetime.time(0, 0, 0),
		108	endTime=datetime.time(23, 59, 59),
		109	timezone=0,
		110	status_value=0,
		111	**kwargs):
		112
		113	self.path = path
		114	self.startTime = startTime
		115	self.endTime = endTime
		116	self.status_value = status_value
		117
		118	if self.path is None:
		119	raise ValueError, "The path is not valid"
		120
		121	if ext is None:
		122	ext = self.ext
		123
		124	self.search_files(self.path, startDate, endDate, ext)
		125	self.timezone = timezone
		126	self.fileIndex = 0
		127
		128	if not self.fileList:
		129	raise Warning, "There is no files matching these date in the folder: %s. \n Check 'startDate' and 'endDate' "%(path)
		130
		131	self.setNextFile()
		132
		133	def search_files(self, path, startDate, endDate, ext):
		134	'''
		135	Searching for BLTR rawdata file in path
		136	Creating a list of file to proces included in [startDate,endDate]
		137
		138	Input:
		139	path - Path to find BLTR rawdata files
		140	startDate - Select file from this date
		141	enDate - Select file until this date
		142	ext - Extension of the file to read
		143
		144	'''
		145
		146	print 'Searching file in %s ' % (path)
		147	foldercounter = 0
		148	fileList0 = glob.glob1(path, "*%s" % ext)
		149	fileList0.sort()
		150
		151	self.fileList = []
		152	self.dateFileList = []
		153
		154	for thisFile in fileList0:
		155	year = thisFile[-14:-10]
		156	if not isNumber(year):
		157	continue
		158
		159	month = thisFile[-10:-8]
		160	if not isNumber(month):
		161	continue
		162
		163	day = thisFile[-8:-6]
		164	if not isNumber(day):
		165	continue
		166
		167	year, month, day = int(year), int(month), int(day)
		168	dateFile = datetime.date(year, month, day)
		169
		170	if (startDate > dateFile) or (endDate < dateFile):
		171	continue
		172
		173	self.fileList.append(thisFile)
		174	self.dateFileList.append(dateFile)
		175
		176	return
		177
		178	def setNextFile(self):
		179
		180	file_id = self.fileIndex
		181
		182	if file_id == len(self.fileList):
		183	print '\nNo more files in the folder'
		184	print 'Total number of file(s) read : {}'.format(self.fileIndex + 1)
		185	self.flagNoMoreFiles = 1
		186	return 0
		187
		188	print '\n[Setting file] (%s) ...' % self.fileList[file_id]
		189	filename = os.path.join(self.path, self.fileList[file_id])
		190
		191	dirname, name = os.path.split(filename)
		192	self.siteFile = name.split('.')[0] # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
		193	if self.filename is not None:
		194	self.fp.close()
		195	self.filename = filename
		196	self.fp = open(self.filename, 'rb')
		197	self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
		198	self.nrecords = self.header_file['nrec'][0]
		199	self.sizeOfFile = os.path.getsize(self.filename)
		200	self.counter_records = 0
		201	self.flagIsNewFile = 0
		202	self.fileIndex += 1
		203
		204	return 1
		205
		206	def readNextBlock(self):
		207
		208	while True:
		209	if self.counter_records == self.nrecords:
		210	self.flagIsNewFile = 1
		211	if not self.setNextFile():
		212	return 0
		213
		214	self.readBlock()
		215
		216	if (self.datatime.time() < self.startTime) or (self.datatime.time() > self.endTime):
		217	print "[Reading] Record No. %d/%d -> %s [Skipping]" %(
		218	self.counter_records,
		219	self.nrecords,
		220	self.datatime.ctime())
		221	continue
		222	break
		223
		224	print "[Reading] Record No. %d/%d -> %s" %(
		225	self.counter_records,
		226	self.nrecords,
		227	self.datatime.ctime())
		228
		229	return 1
		230
		231	def readBlock(self):
		232
		233	pointer = self.fp.tell()
		234	header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
		235	self.nchannels = header_rec['nchan'][0]/2
		236	self.kchan = header_rec['nrxs'][0]
		237	self.nmodes = header_rec['nmodes'][0]
		238	self.nranges = header_rec['nranges'][0]
		239	self.fp.seek(pointer)
		240	self.height = numpy.empty((self.nmodes, self.nranges))
		241	self.snr = numpy.empty((self.nmodes, self.nchannels, self.nranges))
		242	self.buffer = numpy.empty((self.nmodes, 3, self.nranges))
		243
		244	for mode in range(self.nmodes):
		245	self.readHeader()
		246	data = self.readData()
		247	self.height[mode] = (data[0] - self.correction) / 1000.
		248	self.buffer[mode] = data[1]
		249	self.snr[mode] = data[2]
		250
		251	self.counter_records = self.counter_records + self.nmodes
		252
		253	return
		254
		255	def readHeader(self):
		256	'''
		257	RecordHeader of BLTR rawdata file
		258	'''
		259
		260	header_structure = numpy.dtype(
		261	REC_HEADER_STRUCTURE.descr + [
		262	('antenna_coord', 'f4', (2, self.nchannels)),
		263	('rx_gains', 'u4', (self.nchannels,)),
		264	('rx_analysis', 'u4', (self.nchannels,))
		265	]
		266	)
		267
		268	self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
		269	self.lat = self.header_rec['lat'][0]
		270	self.lon = self.header_rec['lon'][0]
		271	self.delta = self.header_rec['delta_r'][0]
		272	self.correction = self.header_rec['dmode_rngcorr'][0]
		273	self.imode = self.header_rec['dmode_index'][0]
		274	self.antenna = self.header_rec['antenna_coord']
		275	self.rx_gains = self.header_rec['rx_gains']
		276	self.time = self.header_rec['time'][0]
		277	tseconds = self.header_rec['time'][0]
		278	local_t1 = time.localtime(tseconds)
		279	self.year = local_t1.tm_year
		280	self.month = local_t1.tm_mon
		281	self.day = local_t1.tm_mday
		282	self.t = datetime.datetime(self.year, self.month, self.day)
		283	self.datatime = datetime.datetime.utcfromtimestamp(self.time)
		284
		285	def readData(self):
		286	'''
		287	Reading and filtering data block record of BLTR rawdata file, filtering is according to status_value.
		288
		289	Input:
		290	status_value - Array data is set to NAN for values that are not equal to status_value
		291
		292	'''
		293
		294	data_structure = numpy.dtype(
		295	DATA_STRUCTURE.descr + [
		296	('rx_saturation', 'u4', (self.nchannels,)),
		297	('chan_offset', 'u4', (2 * self.nchannels,)),
		298	('rx_amp', 'u4', (self.nchannels,)),
		299	('rx_snr', 'f4', (self.nchannels,)),
		300	('cross_snr', 'f4', (self.kchan,)),
		301	('sea_power_relative', 'f4', (self.kchan,))]
		302	)
		303
		304	data = numpy.fromfile(self.fp, data_structure, self.nranges)
		305
		306	height = data['range']
		307	winds = numpy.array((data['zonal'], data['meridional'], data['vertical']))
		308	snr = data['rx_snr'].T
		309
		310	winds[numpy.where(winds == -9999.)] = numpy.nan
		311	winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
		312	snr[numpy.where(snr == -9999.)] = numpy.nan
		313	snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
		314	snr = numpy.power(10, snr / 10)
		315
		316	return height, winds, snr
		317
		318	def set_output(self):
		319	'''
		320	Storing data from databuffer to dataOut object
		321	'''
		322
		323	self.dataOut.data_SNR = self.snr
		324	self.dataOut.height = self.height
		325	self.dataOut.data_output = self.buffer
		326	self.dataOut.utctimeInit = self.time
		327	self.dataOut.utctime = self.dataOut.utctimeInit
		328	self.dataOut.useLocalTime = False
		329	self.dataOut.paramInterval = 157
		330	self.dataOut.timezone = self.timezone
		331	self.dataOut.site = self.siteFile
		332	self.dataOut.nrecords = self.nrecords/self.nmodes
		333	self.dataOut.sizeOfFile = self.sizeOfFile
		334	self.dataOut.lat = self.lat
		335	self.dataOut.lon = self.lon
		336	self.dataOut.channelList = range(self.nchannels)
		337	self.dataOut.kchan = self.kchan
		338	# self.dataOut.nHeights = self.nranges
		339	self.dataOut.delta = self.delta
		340	self.dataOut.correction = self.correction
		341	self.dataOut.nmodes = self.nmodes
		342	self.dataOut.imode = self.imode
		343	self.dataOut.antenna = self.antenna
		344	self.dataOut.rx_gains = self.rx_gains
		345	self.dataOut.flagNoData = False
		346
		347	def getData(self):
		348	'''
		349	Storing data from databuffer to dataOut object
		350	'''
		351	if self.flagNoMoreFiles:
		352	self.dataOut.flagNoData = True
		353	print 'No file left to process'
		354	return 0
		355
		356	if not self.readNextBlock():
		357	self.dataOut.flagNoData = True
		358	return 0
		359
		360	self.set_output()
		361
		362	return 1

schainpy/model/io/jroIO_bltr.py created 644 +1154 0

This diff has been collapsed as it changes many lines, (1154 lines changed) Show them Hide them
		@@ -0,0 +1,1154
	1	import os, sys
	2	import glob
	3	import fnmatch
	4	import datetime
	5	import time
	6	import re
	7	import h5py
	8	import numpy
	9	import matplotlib.pyplot as plt
	10
	11	import pylab as plb
	12	from scipy.optimize import curve_fit
	13	from scipy import asarray as ar, exp
	14	from scipy import stats
	15
	16	from numpy.ma.core import getdata
	17
	18	SPEED_OF_LIGHT = 299792458
	19	SPEED_OF_LIGHT = 3e8
	20
	21	try:
	22	from gevent import sleep
	23	except:
	24	from time import sleep
	25
	26	from schainpy.model.data.jrodata import Spectra
	27	#from schainpy.model.data.BLTRheaderIO import FileHeader, RecordHeader
	28	from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
	29	#from schainpy.model.io.jroIO_bltr import BLTRReader
	30	from numpy import imag, shape, NaN
	31
	32	from jroIO_base import JRODataReader
	33
	34
	35	class Header(object):
	36
	37	def __init__(self):
	38	raise NotImplementedError
	39
	40
	41	def read(self):
	42
	43	raise NotImplementedError
	44
	45	def write(self):
	46
	47	raise NotImplementedError
	48
	49	def printInfo(self):
	50
	51	message = "#"*50 + "\n"
	52	message += self.__class__.__name__.upper() + "\n"
	53	message += "#"*50 + "\n"
	54
	55	keyList = self.__dict__.keys()
	56	keyList.sort()
	57
	58	for key in keyList:
	59	message += "%s = %s" %(key, self.__dict__[key]) + "\n"
	60
	61	if "size" not in keyList:
	62	attr = getattr(self, "size")
	63
	64	if attr:
	65	message += "%s = %s" %("size", attr) + "\n"
	66
	67	#print message
	68
	69
	70
	71
	72
	73	FILE_STRUCTURE = numpy.dtype([ #HEADER 48bytes
	74	('FileMgcNumber','<u4'), #0x23020100
	75	('nFDTdataRecors','<u4'), #No Of FDT data records in this file (0 or more)
	76	('OffsetStartHeader','<u4'),
	77	('RadarUnitId','<u4'),
	78	('SiteName',numpy.str_,32), #Null terminated
	79	])
	80
	81	class FileHeaderBLTR(Header):
	82
	83	def __init__(self):
	84
	85	self.FileMgcNumber= 0 #0x23020100
	86	self.nFDTdataRecors=0 #No Of FDT data records in this file (0 or more)
	87	self.RadarUnitId= 0
	88	self.OffsetStartHeader=0
	89	self.SiteName= ""
	90	self.size = 48
	91
	92	def FHread(self, fp):
	93	#try:
	94	startFp = open(fp,"rb")
	95
	96	header = numpy.fromfile(startFp, FILE_STRUCTURE,1)
	97
	98	print ' '
	99	print 'puntero file header', startFp.tell()
	100	print ' '
	101
	102
	103	''' numpy.fromfile(file, dtype, count, sep='')
	104	file : file or str
	105	Open file object or filename.
	106
	107	dtype : data-type
	108	Data type of the returned array. For binary files, it is used to determine
	109	the size and byte-order of the items in the file.
	110
	111	count : int
	112	Number of items to read. -1 means all items (i.e., the complete file).
	113
	114	sep : str
	115	Separator between items if file is a text file. Empty ("") separator means
	116	the file should be treated as binary. Spaces (" ") in the separator match zero
	117	or more whitespace characters. A separator consisting only of spaces must match
	118	at least one whitespace.
	119
	120	'''
	121
	122
	123
	124	self.FileMgcNumber= hex(header['FileMgcNumber'][0])
	125	self.nFDTdataRecors=int(header['nFDTdataRecors'][0]) #No Of FDT data records in this file (0 or more)
	126	self.RadarUnitId= int(header['RadarUnitId'][0])
	127	self.OffsetStartHeader= int(header['OffsetStartHeader'][0])
	128	self.SiteName= str(header['SiteName'][0])
	129
	130	#print 'Numero de bloques', self.nFDTdataRecors
	131
	132
	133	if self.size <48:
	134	return 0
	135
	136	return 1
	137
	138
	139	def write(self, fp):
	140
	141	headerTuple = (self.FileMgcNumber,
	142	self.nFDTdataRecors,
	143	self.RadarUnitId,
	144	self.SiteName,
	145	self.size)
	146
	147
	148	header = numpy.array(headerTuple, FILE_STRUCTURE)
	149	# numpy.array(object, dtype=None, copy=True, order=None, subok=False, ndmin=0)
	150	header.tofile(fp)
	151	''' ndarray.tofile(fid, sep, format) Write array to a file as text or binary (default).
	152
	153	fid : file or str
	154	An open file object, or a string containing a filename.
	155
	156	sep : str
	157	Separator between array items for text output. If "" (empty), a binary file is written,
	158	equivalent to file.write(a.tobytes()).
	159
	160	format : str
	161	Format string for text file output. Each entry in the array is formatted to text by
	162	first converting it to the closest Python type, and then using "format" % item.
	163
	164	'''
	165
	166	return 1
	167
	168
	169
	170
	171
	172	RECORD_STRUCTURE = numpy.dtype([ #RECORD HEADER 180+20N bytes
	173	('RecMgcNumber','<u4'), #0x23030001
	174	('RecCounter','<u4'), #Record counter(0,1, ...)
	175	('Off2StartNxtRec','<u4'), #Offset to start of next record form start of this record
	176	('Off2StartData','<u4'), #Offset to start of data from start of this record
	177	('nUtime','<i4'), #Epoch time stamp of start of acquisition (seconds)
	178	('nMilisec','<u4'), #Millisecond component of time stamp (0,...,999)
	179	('ExpTagName',numpy.str_,32), #Experiment tag name (null terminated)
	180	('ExpComment',numpy.str_,32), #Experiment comment (null terminated)
	181	('SiteLatDegrees','<f4'), #Site latitude (from GPS) in degrees (positive implies North)
	182	('SiteLongDegrees','<f4'), #Site longitude (from GPS) in degrees (positive implies East)
	183	('RTCgpsStatus','<u4'), #RTC GPS engine status (0=SEEK, 1=LOCK, 2=NOT FITTED, 3=UNAVAILABLE)
	184	('TransmitFrec','<u4'), #Transmit frequency (Hz)
	185	('ReceiveFrec','<u4'), #Receive frequency
	186	('FirstOsciFrec','<u4'), #First local oscillator frequency (Hz)
	187	('Polarisation','<u4'), #(0="O", 1="E", 2="linear 1", 3="linear2")
	188	('ReceiverFiltSett','<u4'), #Receiver filter settings (0,1,2,3)
	189	('nModesInUse','<u4'), #Number of modes in use (1 or 2)
	190	('DualModeIndex','<u4'), #Dual Mode index number for these data (0 or 1)
	191	('DualModeRange','<u4'), #Dual Mode range correction for these data (m)
	192	('nDigChannels','<u4'), #Number of digital channels acquired (2*N)
	193	('SampResolution','<u4'), #Sampling resolution (meters)
	194	('nHeights','<u4'), #Number of range gates sampled
	195	('StartRangeSamp','<u4'), #Start range of sampling (meters)
	196	('PRFhz','<u4'), #PRF (Hz)
	197	('nCohInt','<u4'), #Integrations
	198	('nProfiles','<u4'), #Number of data points transformed
	199	('nChannels','<u4'), #Number of receive beams stored in file (1 or N)
	200	('nIncohInt','<u4'), #Number of spectral averages
	201	('FFTwindowingInd','<u4'), #FFT windowing index (0 = no window)
	202	('BeamAngleAzim','<f4'), #Beam steer angle (azimuth) in degrees (clockwise from true North)
	203	('BeamAngleZen','<f4'), #Beam steer angle (zenith) in degrees (0=> vertical)
	204	('AntennaCoord0','<f4'), #Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
	205	('AntennaAngl0','<f4'), #Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
	206	('AntennaCoord1','<f4'), #Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
	207	('AntennaAngl1','<f4'), #Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
	208	('AntennaCoord2','<f4'), #Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
	209	('AntennaAngl2','<f4'), #Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
	210	('RecPhaseCalibr0','<f4'), #Receiver phase calibration (degrees) - N values
	211	('RecPhaseCalibr1','<f4'), #Receiver phase calibration (degrees) - N values
	212	('RecPhaseCalibr2','<f4'), #Receiver phase calibration (degrees) - N values
	213	('RecAmpCalibr0','<f4'), #Receiver amplitude calibration (ratio relative to receiver one) - N values
	214	('RecAmpCalibr1','<f4'), #Receiver amplitude calibration (ratio relative to receiver one) - N values
	215	('RecAmpCalibr2','<f4'), #Receiver amplitude calibration (ratio relative to receiver one) - N values
	216	('ReceiverGaindB0','<i4'), #Receiver gains in dB - N values
	217	('ReceiverGaindB1','<i4'), #Receiver gains in dB - N values
	218	('ReceiverGaindB2','<i4'), #Receiver gains in dB - N values
	219	])
	220
	221
	222	class RecordHeaderBLTR(Header):
	223
	224	def __init__(self, RecMgcNumber=None, RecCounter= 0, Off2StartNxtRec= 811248,
	225	nUtime= 0, nMilisec= 0, ExpTagName= None,
	226	ExpComment=None, SiteLatDegrees=0, SiteLongDegrees= 0,
	227	RTCgpsStatus= 0, TransmitFrec= 0, ReceiveFrec= 0,
	228	FirstOsciFrec= 0, Polarisation= 0, ReceiverFiltSett= 0,
	229	nModesInUse= 0, DualModeIndex= 0, DualModeRange= 0,
	230	nDigChannels= 0, SampResolution= 0, nHeights= 0,
	231	StartRangeSamp= 0, PRFhz= 0, nCohInt= 0,
	232	nProfiles= 0, nChannels= 0, nIncohInt= 0,
	233	FFTwindowingInd= 0, BeamAngleAzim= 0, BeamAngleZen= 0,
	234	AntennaCoord0= 0, AntennaCoord1= 0, AntennaCoord2= 0,
	235	RecPhaseCalibr0= 0, RecPhaseCalibr1= 0, RecPhaseCalibr2= 0,
	236	RecAmpCalibr0= 0, RecAmpCalibr1= 0, RecAmpCalibr2= 0,
	237	AntennaAngl0=0, AntennaAngl1=0, AntennaAngl2=0,
	238	ReceiverGaindB0= 0, ReceiverGaindB1= 0, ReceiverGaindB2= 0, Off2StartData=0, OffsetStartHeader=0):
	239
	240	self.RecMgcNumber = RecMgcNumber #0x23030001
	241	self.RecCounter = RecCounter
	242	self.Off2StartNxtRec = Off2StartNxtRec
	243	self.Off2StartData = Off2StartData
	244	self.nUtime = nUtime
	245	self.nMilisec = nMilisec
	246	self.ExpTagName = ExpTagName
	247	self.ExpComment = ExpComment
	248	self.SiteLatDegrees = SiteLatDegrees
	249	self.SiteLongDegrees = SiteLongDegrees
	250	self.RTCgpsStatus = RTCgpsStatus
	251	self.TransmitFrec = TransmitFrec
	252	self.ReceiveFrec = ReceiveFrec
	253	self.FirstOsciFrec = FirstOsciFrec
	254	self.Polarisation = Polarisation
	255	self.ReceiverFiltSett = ReceiverFiltSett
	256	self.nModesInUse = nModesInUse
	257	self.DualModeIndex = DualModeIndex
	258	self.DualModeRange = DualModeRange
	259	self.nDigChannels = nDigChannels
	260	self.SampResolution = SampResolution
	261	self.nHeights = nHeights
	262	self.StartRangeSamp = StartRangeSamp
	263	self.PRFhz = PRFhz
	264	self.nCohInt = nCohInt
	265	self.nProfiles = nProfiles
	266	self.nChannels = nChannels
	267	self.nIncohInt = nIncohInt
	268	self.FFTwindowingInd = FFTwindowingInd
	269	self.BeamAngleAzim = BeamAngleAzim
	270	self.BeamAngleZen = BeamAngleZen
	271	self.AntennaCoord0 = AntennaCoord0
	272	self.AntennaAngl0 = AntennaAngl0
	273	self.AntennaAngl1 = AntennaAngl1
	274	self.AntennaAngl2 = AntennaAngl2
	275	self.AntennaCoord1 = AntennaCoord1
	276	self.AntennaCoord2 = AntennaCoord2
	277	self.RecPhaseCalibr0 = RecPhaseCalibr0
	278	self.RecPhaseCalibr1 = RecPhaseCalibr1
	279	self.RecPhaseCalibr2 = RecPhaseCalibr2
	280	self.RecAmpCalibr0 = RecAmpCalibr0
	281	self.RecAmpCalibr1 = RecAmpCalibr1
	282	self.RecAmpCalibr2 = RecAmpCalibr2
	283	self.ReceiverGaindB0 = ReceiverGaindB0
	284	self.ReceiverGaindB1 = ReceiverGaindB1
	285	self.ReceiverGaindB2 = ReceiverGaindB2
	286	self.OffsetStartHeader = 48
	287
	288
	289
	290	def RHread(self, fp):
	291	#print fp
	292	#startFp = open('/home/erick/Documents/Data/huancayo.20161019.22.fdt',"rb") #The method tell() returns the current position of the file read/write pointer within the file.
	293	startFp = open(fp,"rb") #The method tell() returns the current position of the file read/write pointer within the file.
	294	#RecCounter=0
	295	#Off2StartNxtRec=811248
	296	OffRHeader= self.OffsetStartHeader + self.RecCounter*self.Off2StartNxtRec
	297	print ' '
	298	print 'puntero Record Header', startFp.tell()
	299	print ' '
	300
	301
	302	startFp.seek(OffRHeader, os.SEEK_SET)
	303
	304	print ' '
	305	print 'puntero Record Header con seek', startFp.tell()
	306	print ' '
	307
	308	#print 'Posicion del bloque: ',OffRHeader
	309
	310	header = numpy.fromfile(startFp,RECORD_STRUCTURE,1)
	311
	312	print ' '
	313	print 'puntero Record Header con seek', startFp.tell()
	314	print ' '
	315
	316	print ' '
	317	#
	318	#print 'puntero Record Header despues de seek', header.tell()
	319	print ' '
	320
	321	self.RecMgcNumber = hex(header['RecMgcNumber'][0]) #0x23030001
	322	self.RecCounter = int(header['RecCounter'][0])
	323	self.Off2StartNxtRec = int(header['Off2StartNxtRec'][0])
	324	self.Off2StartData = int(header['Off2StartData'][0])
	325	self.nUtime = header['nUtime'][0]
	326	self.nMilisec = header['nMilisec'][0]
	327	self.ExpTagName = str(header['ExpTagName'][0])
	328	self.ExpComment = str(header['ExpComment'][0])
	329	self.SiteLatDegrees = header['SiteLatDegrees'][0]
	330	self.SiteLongDegrees = header['SiteLongDegrees'][0]
	331	self.RTCgpsStatus = header['RTCgpsStatus'][0]
	332	self.TransmitFrec = header['TransmitFrec'][0]
	333	self.ReceiveFrec = header['ReceiveFrec'][0]
	334	self.FirstOsciFrec = header['FirstOsciFrec'][0]
	335	self.Polarisation = header['Polarisation'][0]
	336	self.ReceiverFiltSett = header['ReceiverFiltSett'][0]
	337	self.nModesInUse = header['nModesInUse'][0]
	338	self.DualModeIndex = header['DualModeIndex'][0]
	339	self.DualModeRange = header['DualModeRange'][0]
	340	self.nDigChannels = header['nDigChannels'][0]
	341	self.SampResolution = header['SampResolution'][0]
	342	self.nHeights = header['nHeights'][0]
	343	self.StartRangeSamp = header['StartRangeSamp'][0]
	344	self.PRFhz = header['PRFhz'][0]
	345	self.nCohInt = header['nCohInt'][0]
	346	self.nProfiles = header['nProfiles'][0]
	347	self.nChannels = header['nChannels'][0]
	348	self.nIncohInt = header['nIncohInt'][0]
	349	self.FFTwindowingInd = header['FFTwindowingInd'][0]
	350	self.BeamAngleAzim = header['BeamAngleAzim'][0]
	351	self.BeamAngleZen = header['BeamAngleZen'][0]
	352	self.AntennaCoord0 = header['AntennaCoord0'][0]
	353	self.AntennaAngl0 = header['AntennaAngl0'][0]
	354	self.AntennaCoord1 = header['AntennaCoord1'][0]
	355	self.AntennaAngl1 = header['AntennaAngl1'][0]
	356	self.AntennaCoord2 = header['AntennaCoord2'][0]
	357	self.AntennaAngl2 = header['AntennaAngl2'][0]
	358	self.RecPhaseCalibr0 = header['RecPhaseCalibr0'][0]
	359	self.RecPhaseCalibr1 = header['RecPhaseCalibr1'][0]
	360	self.RecPhaseCalibr2 = header['RecPhaseCalibr2'][0]
	361	self.RecAmpCalibr0 = header['RecAmpCalibr0'][0]
	362	self.RecAmpCalibr1 = header['RecAmpCalibr1'][0]
	363	self.RecAmpCalibr2 = header['RecAmpCalibr2'][0]
	364	self.ReceiverGaindB0 = header['ReceiverGaindB0'][0]
	365	self.ReceiverGaindB1 = header['ReceiverGaindB1'][0]
	366	self.ReceiverGaindB2 = header['ReceiverGaindB2'][0]
	367
	368	self.ipp= 0.5*(SPEED_OF_LIGHT/self.PRFhz)
	369
	370	self.RHsize = 180+20*self.nChannels
	371	self.Datasize= self.nProfilesself.nChannelsself.nHeights24
	372	#print 'Datasize',self.Datasize
	373	endFp = self.OffsetStartHeader + self.RecCounter*self.Off2StartNxtRec
	374
	375	print '=============================================='
	376	print 'RecMgcNumber ',self.RecMgcNumber
	377	print 'RecCounter ',self.RecCounter
	378	print 'Off2StartNxtRec ',self.Off2StartNxtRec
	379	print 'Off2StartData ',self.Off2StartData
	380	print 'Range Resolution ',self.SampResolution
	381	print 'First Height ',self.StartRangeSamp
	382	print 'PRF (Hz) ',self.PRFhz
	383	print 'Heights (K) ',self.nHeights
	384	print 'Channels (N) ',self.nChannels
	385	print 'Profiles (J) ',self.nProfiles
	386	print 'iCoh ',self.nCohInt
	387	print 'iInCoh ',self.nIncohInt
	388	print 'BeamAngleAzim ',self.BeamAngleAzim
	389	print 'BeamAngleZen ',self.BeamAngleZen
	390
	391	#print 'ModoEnUso ',self.DualModeIndex
	392	#print 'UtcTime ',self.nUtime
	393	#print 'MiliSec ',self.nMilisec
	394	#print 'Exp TagName ',self.ExpTagName
	395	#print 'Exp Comment ',self.ExpComment
	396	#print 'FFT Window Index ',self.FFTwindowingInd
	397	#print 'N Dig. Channels ',self.nDigChannels
	398	print 'Size de bloque ',self.RHsize
	399	print 'DataSize ',self.Datasize
	400	print 'BeamAngleAzim ',self.BeamAngleAzim
	401	#print 'AntennaCoord0 ',self.AntennaCoord0
	402	#print 'AntennaAngl0 ',self.AntennaAngl0
	403	#print 'AntennaCoord1 ',self.AntennaCoord1
	404	#print 'AntennaAngl1 ',self.AntennaAngl1
	405	#print 'AntennaCoord2 ',self.AntennaCoord2
	406	#print 'AntennaAngl2 ',self.AntennaAngl2
	407	print 'RecPhaseCalibr0 ',self.RecPhaseCalibr0
	408	print 'RecPhaseCalibr1 ',self.RecPhaseCalibr1
	409	print 'RecPhaseCalibr2 ',self.RecPhaseCalibr2
	410	print 'RecAmpCalibr0 ',self.RecAmpCalibr0
	411	print 'RecAmpCalibr1 ',self.RecAmpCalibr1
	412	print 'RecAmpCalibr2 ',self.RecAmpCalibr2
	413	print 'ReceiverGaindB0 ',self.ReceiverGaindB0
	414	print 'ReceiverGaindB1 ',self.ReceiverGaindB1
	415	print 'ReceiverGaindB2 ',self.ReceiverGaindB2
	416	print '=============================================='
	417
	418	if OffRHeader > endFp:
	419	sys.stderr.write("Warning %s: Size value read from System Header is lower than it has to be\n" %fp)
	420	return 0
	421
	422	if OffRHeader < endFp:
	423	sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" %fp)
	424	return 0
	425
	426	return 1
	427
	428
	429	class BLTRSpectraReader (ProcessingUnit, FileHeaderBLTR, RecordHeaderBLTR, JRODataReader):
	430
	431	path = None
	432	startDate = None
	433	endDate = None
	434	startTime = None
	435	endTime = None
	436	walk = None
	437	isConfig = False
	438
	439
	440	fileList= None
	441
	442	#metadata
	443	TimeZone= None
	444	Interval= None
	445	heightList= None
	446
	447	#data
	448	data= None
	449	utctime= None
	450
	451
	452
	453	def __init__(self, **kwargs):
	454
	455	#Eliminar de la base la herencia
	456	ProcessingUnit.__init__(self, **kwargs)
	457
	458	#self.isConfig = False
	459
	460	#self.pts2read_SelfSpectra = 0
	461	#self.pts2read_CrossSpectra = 0
	462	#self.pts2read_DCchannels = 0
	463	#self.datablock = None
	464	self.utc = None
	465	self.ext = ".fdt"
	466	self.optchar = "P"
	467	self.fpFile=None
	468	self.fp = None
	469	self.BlockCounter=0
	470	self.dtype = None
	471	self.fileSizeByHeader = None
	472	self.filenameList = []
	473	self.fileSelector = 0
	474	self.Off2StartNxtRec=0
	475	self.RecCounter=0
	476	self.flagNoMoreFiles = 0
	477	self.data_spc=None
	478	self.data_cspc=None
	479	self.data_output=None
	480	self.path = None
	481	self.OffsetStartHeader=0
	482	self.Off2StartData=0
	483	self.ipp = 0
	484	self.nFDTdataRecors=0
	485	self.blocksize = 0
	486	self.dataOut = Spectra()
	487	self.profileIndex = 1 #Always
	488	self.dataOut.flagNoData=False
	489	self.dataOut.nRdPairs = 0
	490	self.dataOut.pairsList = []
	491	self.dataOut.data_spc=None
	492	self.dataOut.noise=[]
	493	self.dataOut.velocityX=[]
	494	self.dataOut.velocityY=[]
	495	self.dataOut.velocityV=[]
	496
	497
	498
	499	def Files2Read(self, fp):
	500	'''
	501	Function that indicates the number of .fdt files that exist in the folder to be read.
	502	It also creates an organized list with the names of the files to read.
	503	'''
	504	#self.__checkPath()
	505
	506	ListaData=os.listdir(fp) #Gets the list of files within the fp address
	507	ListaData=sorted(ListaData) #Sort the list of files from least to largest by names
	508	nFiles=0 #File Counter
	509	FileList=[] #A list is created that will contain the .fdt files
	510	for IndexFile in ListaData :
	511	if '.fdt' in IndexFile:
	512	FileList.append(IndexFile)
	513	nFiles+=1
	514
	515	#print 'Files2Read'
	516	#print 'Existen '+str(nFiles)+' archivos .fdt'
	517
	518	self.filenameList=FileList #List of files from least to largest by names
	519
	520
	521	def run(self, **kwargs):
	522	'''
	523	This method will be the one that will initiate the data entry, will be called constantly.
	524	You should first verify that your Setup () is set up and then continue to acquire
	525	the data to be processed with getData ().
	526	'''
	527	if not self.isConfig:
	528	self.setup(**kwargs)
	529	self.isConfig = True
	530
	531	self.getData()
	532	#print 'running'
	533
	534
	535	def setup(self, path=None,
	536	startDate=None,
	537	endDate=None,
	538	startTime=None,
	539	endTime=None,
	540	walk=True,
	541	timezone='utc',
	542	code = None,
	543	online=False,
	544	ReadMode=None,
	545	**kwargs):
	546
	547	self.isConfig = True
	548
	549	self.path=path
	550	self.startDate=startDate
	551	self.endDate=endDate
	552	self.startTime=startTime
	553	self.endTime=endTime
	554	self.walk=walk
	555	self.ReadMode=int(ReadMode)
	556
	557	pass
	558
	559
	560	def getData(self):
	561	'''
	562	Before starting this function, you should check that there is still an unread file,
	563	If there are still blocks to read or if the data block is empty.
	564
	565	You should call the file "read".
	566
	567	'''
	568
	569	if self.flagNoMoreFiles:
	570	self.dataOut.flagNoData = True
	571	print 'NoData se vuelve true'
	572	return 0
	573
	574	self.fp=self.path
	575	self.Files2Read(self.fp)
	576	self.readFile(self.fp)
	577	self.dataOut.data_spc = self.data_spc
	578	self.dataOut.data_cspc =self.data_cspc
	579	self.dataOut.data_output=self.data_output
	580
	581	print 'self.dataOut.data_output', shape(self.dataOut.data_output)
	582
	583	#self.removeDC()
	584	return self.dataOut.data_spc
	585
	586
	587	def readFile(self,fp):
	588	'''
	589	You must indicate if you are reading in Online or Offline mode and load the
	590	The parameters for this file reading mode.
	591
	592	Then you must do 2 actions:
	593
	594	1. Get the BLTR FileHeader.
	595	2. Start reading the first block.
	596	'''
	597
	598	#The address of the folder is generated the name of the .fdt file that will be read
	599	print "File: ",self.fileSelector+1
	600
	601	if self.fileSelector < len(self.filenameList):
	602
	603	self.fpFile=str(fp)+'/'+str(self.filenameList[self.fileSelector])
	604	#print self.fpFile
	605	fheader = FileHeaderBLTR()
	606	fheader.FHread(self.fpFile) #Bltr FileHeader Reading
	607	self.nFDTdataRecors=fheader.nFDTdataRecors
	608
	609	self.readBlock() #Block reading
	610	else:
	611	print 'readFile FlagNoData becomes true'
	612	self.flagNoMoreFiles=True
	613	self.dataOut.flagNoData = True
	614	return 0
	615
	616	def getVelRange(self, extrapoints=0):
	617	Lambda= SPEED_OF_LIGHT/50000000
	618	PRF = self.dataOut.PRF#1./(self.dataOut.ippSeconds * self.dataOut.nCohInt)
	619	Vmax=-Lambda/(4.(1./PRF)self.dataOut.nCohInt*2.)
	620	deltafreq = PRF / (self.nProfiles)
	621	deltavel = (Vmax*2) / (self.nProfiles)
	622	freqrange = deltafreq*(numpy.arange(self.nProfiles)-self.nProfiles/2.) - deltafreq/2
	623	velrange = deltavel*(numpy.arange(self.nProfiles)-self.nProfiles/2.)
	624	return velrange
	625
	626	def readBlock(self):
	627	'''
	628	It should be checked if the block has data, if it is not passed to the next file.
	629
	630	Then the following is done:
	631
	632	1. Read the RecordHeader
	633	2. Fill the buffer with the current block number.
	634
	635	'''
	636
	637	if self.BlockCounter < self.nFDTdataRecors-2:
	638	print self.nFDTdataRecors, 'CONDICION!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
	639	if self.ReadMode==1:
	640	rheader = RecordHeaderBLTR(RecCounter=self.BlockCounter+1)
	641	elif self.ReadMode==0:
	642	rheader = RecordHeaderBLTR(RecCounter=self.BlockCounter)
	643
	644	rheader.RHread(self.fpFile) #Bltr FileHeader Reading
	645
	646	self.OffsetStartHeader=rheader.OffsetStartHeader
	647	self.RecCounter=rheader.RecCounter
	648	self.Off2StartNxtRec=rheader.Off2StartNxtRec
	649	self.Off2StartData=rheader.Off2StartData
	650	self.nProfiles=rheader.nProfiles
	651	self.nChannels=rheader.nChannels
	652	self.nHeights=rheader.nHeights
	653	self.frequency=rheader.TransmitFrec
	654	self.DualModeIndex=rheader.DualModeIndex
	655
	656	self.pairsList =[(0,1),(0,2),(1,2)]
	657	self.dataOut.pairsList = self.pairsList
	658
	659	self.nRdPairs=len(self.dataOut.pairsList)
	660	self.dataOut.nRdPairs = self.nRdPairs
	661
	662	self.__firstHeigth=rheader.StartRangeSamp
	663	self.__deltaHeigth=rheader.SampResolution
	664	self.dataOut.heightList= self.__firstHeigth + numpy.array(range(self.nHeights))*self.__deltaHeigth
	665	self.dataOut.channelList = range(self.nChannels)
	666	self.dataOut.nProfiles=rheader.nProfiles
	667	self.dataOut.nIncohInt=rheader.nIncohInt
	668	self.dataOut.nCohInt=rheader.nCohInt
	669	self.dataOut.ippSeconds= 1/float(rheader.PRFhz)
	670	self.dataOut.PRF=rheader.PRFhz
	671	self.dataOut.nFFTPoints=rheader.nProfiles
	672	self.dataOut.utctime=rheader.nUtime
	673	self.dataOut.timeZone=0
	674	self.dataOut.normFactor= self.dataOut.nProfilesself.dataOut.nIncohIntself.dataOut.nCohInt
	675	self.dataOut.outputInterval= self.dataOut.ippSeconds * self.dataOut.nCohInt * self.dataOut.nIncohInt * self.nProfiles
	676
	677	self.data_output=numpy.ones([3,rheader.nHeights])*numpy.NaN
	678	print 'self.data_output', shape(self.data_output)
	679	self.dataOut.velocityX=[]
	680	self.dataOut.velocityY=[]
	681	self.dataOut.velocityV=[]
	682
	683	'''Block Reading, the Block Data is received and Reshape is used to give it
	684	shape.
	685	'''
	686
	687	#Procedure to take the pointer to where the date block starts
	688	startDATA = open(self.fpFile,"rb")
	689	OffDATA= self.OffsetStartHeader + self.RecCounter*self.Off2StartNxtRec+self.Off2StartData
	690	startDATA.seek(OffDATA, os.SEEK_SET)
	691
	692	def moving_average(x, N=2):
	693	return numpy.convolve(x, numpy.ones((N,))/N)[(N-1):]
	694
	695	def gaus(xSamples,a,x0,sigma):
	696	return aexp(-(xSamples-x0)2/(2sigma**2))
	697
	698	def Find(x,value):
	699	for index in range(len(x)):
	700	if x[index]==value:
	701	return index
	702
	703	def pol2cart(rho, phi):
	704	x = rho * numpy.cos(phi)
	705	y = rho * numpy.sin(phi)
	706	return(x, y)
	707
	708
	709
	710
	711	if self.DualModeIndex==self.ReadMode:
	712
	713	self.data_fft = numpy.fromfile( startDATA, [('complex','<c8')],self.nProfilesself.nChannelsself.nHeights )
	714
	715	self.data_fft=self.data_fft.astype(numpy.dtype('complex'))
	716
	717	self.data_block=numpy.reshape(self.data_fft,(self.nHeights, self.nChannels, self.nProfiles ))
	718
	719	self.data_block = numpy.transpose(self.data_block, (1,2,0))
	720
	721	copy = self.data_block.copy()
	722	spc = copy * numpy.conjugate(copy)
	723
	724	self.data_spc = numpy.absolute(spc) # valor absoluto o magnitud
	725
	726	factor = self.dataOut.normFactor
	727
	728
	729	z = self.data_spc.copy()#/factor
	730	z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
	731	#zdB = 10*numpy.log10(z)
	732	print ' '
	733	print 'Z: '
	734	print shape(z)
	735	print ' '
	736	print ' '
	737
	738	self.dataOut.data_spc=self.data_spc
	739
	740	self.noise = self.dataOut.getNoise(ymin_index=80, ymax_index=132)#/factor
	741	#noisedB = 10*numpy.log10(self.noise)
	742
	743
	744	ySamples=numpy.ones([3,self.nProfiles])
	745	phase=numpy.ones([3,self.nProfiles])
	746	CSPCSamples=numpy.ones([3,self.nProfiles],dtype=numpy.complex_)
	747	coherence=numpy.ones([3,self.nProfiles])
	748	PhaseSlope=numpy.ones(3)
	749	PhaseInter=numpy.ones(3)
	750
	751	'''**** Getting CrossSpectra ****'''
	752	cspc=self.data_block.copy()
	753	self.data_cspc=self.data_block.copy()
	754
	755	xFrec=self.getVelRange(1)
	756	VelRange=self.getVelRange(1)
	757	self.dataOut.VelRange=VelRange
	758	#print ' '
	759	#print ' '
	760	#print 'xFrec',xFrec
	761	#print ' '
	762	#print ' '
	763	#Height=35
	764	for i in range(self.nRdPairs):
	765
	766	chan_index0 = self.dataOut.pairsList[i][0]
	767	chan_index1 = self.dataOut.pairsList[i][1]
	768
	769	self.data_cspc[i,:,:]=cspc[chan_index0,:,:] * numpy.conjugate(cspc[chan_index1,:,:])
	770
	771
	772	'''Getting Eij and Nij'''
	773	(AntennaX0,AntennaY0)=pol2cart(rheader.AntennaCoord0, rheader.AntennaAngl0*numpy.pi/180)
	774	(AntennaX1,AntennaY1)=pol2cart(rheader.AntennaCoord1, rheader.AntennaAngl1*numpy.pi/180)
	775	(AntennaX2,AntennaY2)=pol2cart(rheader.AntennaCoord2, rheader.AntennaAngl2*numpy.pi/180)
	776
	777	E01=AntennaX0-AntennaX1
	778	N01=AntennaY0-AntennaY1
	779
	780	E02=AntennaX0-AntennaX2
	781	N02=AntennaY0-AntennaY2
	782
	783	E12=AntennaX1-AntennaX2
	784	N12=AntennaY1-AntennaY2
	785
	786	self.ChanDist= numpy.array([[E01, N01],[E02,N02],[E12,N12]])
	787
	788	self.dataOut.ChanDist = self.ChanDist
	789
	790
	791	# for Height in range(self.nHeights):
	792	#
	793	# for i in range(self.nRdPairs):
	794	#
	795	# '''**** Line of Data SPC ****'''
	796	# zline=z[i,:,Height]
	797	#
	798	# '''**** DC is removed ****'''
	799	# DC=Find(zline,numpy.amax(zline))
	800	# zline[DC]=(zline[DC-1]+zline[DC+1])/2
	801	#
	802	#
	803	# '''**** SPC is normalized ****'''
	804	# FactNorm= zline.copy() / numpy.sum(zline.copy())
	805	# FactNorm= FactNorm/numpy.sum(FactNorm)
	806	#
	807	# SmoothSPC=moving_average(FactNorm,N=3)
	808	#
	809	# xSamples = ar(range(len(SmoothSPC)))
	810	# ySamples[i] = SmoothSPC-self.noise[i]
	811	#
	812	# for i in range(self.nRdPairs):
	813	#
	814	# '''**** Line of Data CSPC ****'''
	815	# cspcLine=self.data_cspc[i,:,Height].copy()
	816	#
	817	#
	818	#
	819	# '''**** CSPC is normalized ****'''
	820	# chan_index0 = self.dataOut.pairsList[i][0]
	821	# chan_index1 = self.dataOut.pairsList[i][1]
	822	# CSPCFactor= numpy.sum(ySamples[chan_index0]) * numpy.sum(ySamples[chan_index1])
	823	#
	824	#
	825	# CSPCNorm= cspcLine.copy() / numpy.sqrt(CSPCFactor)
	826	#
	827	#
	828	# CSPCSamples[i] = CSPCNorm-self.noise[i]
	829	# coherence[i] = numpy.abs(CSPCSamples[i]) / numpy.sqrt(CSPCFactor)
	830	#
	831	# '''**** DC is removed ****'''
	832	# DC=Find(coherence[i],numpy.amax(coherence[i]))
	833	# coherence[i][DC]=(coherence[i][DC-1]+coherence[i][DC+1])/2
	834	# coherence[i]= moving_average(coherence[i],N=2)
	835	#
	836	# phase[i] = moving_average( numpy.arctan2(CSPCSamples[i].imag, CSPCSamples[i].real),N=1)#*180/numpy.pi
	837	#
	838	#
	839	# '''**** Getting fij width ****'''
	840	#
	841	# yMean=[]
	842	# yMean2=[]
	843	#
	844	# for j in range(len(ySamples[1])):
	845	# yMean=numpy.append(yMean,numpy.average([ySamples[0,j],ySamples[1,j],ySamples[2,j]]))
	846	#
	847	# '''***** Getting fitting Gaussian ****'''
	848	# meanGauss=sum(xSamples*yMean) / len(xSamples)
	849	# sigma=sum(yMean(xSamples-meanGauss)*2) / len(xSamples)
	850	# #print 'Height',Height,'SNR', meanGauss/sigma**2
	851	#
	852	# if (abs(meanGauss/sigma**2) > 0.0001) :
	853	#
	854	# try:
	855	# popt,pcov = curve_fit(gaus,xSamples,yMean,p0=[1,meanGauss,sigma])
	856	#
	857	# if numpy.amax(popt)>numpy.amax(yMean)*0.3:
	858	# FitGauss=gaus(xSamples,*popt)
	859	#
	860	# else:
	861	# FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean)
	862	# print 'Verificador: Dentro', Height
	863	# except RuntimeError:
	864	#
	865	# try:
	866	# for j in range(len(ySamples[1])):
	867	# yMean2=numpy.append(yMean2,numpy.average([ySamples[1,j],ySamples[2,j]]))
	868	# popt,pcov = curve_fit(gaus,xSamples,yMean2,p0=[1,meanGauss,sigma])
	869	# FitGauss=gaus(xSamples,*popt)
	870	# print 'Verificador: Exepcion1', Height
	871	# except RuntimeError:
	872	#
	873	# try:
	874	# popt,pcov = curve_fit(gaus,xSamples,ySamples[1],p0=[1,meanGauss,sigma])
	875	# FitGauss=gaus(xSamples,*popt)
	876	# print 'Verificador: Exepcion2', Height
	877	# except RuntimeError:
	878	# FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean)
	879	# print 'Verificador: Exepcion3', Height
	880	# else:
	881	# FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean)
	882	# #print 'Verificador: Fuera', Height
	883	#
	884	#
	885	#
	886	# Maximun=numpy.amax(yMean)
	887	# eMinus1=Maximun*numpy.exp(-1)
	888	#
	889	# HWpos=Find(FitGauss,min(FitGauss, key=lambda value:abs(value-eMinus1)))
	890	# HalfWidth= xFrec[HWpos]
	891	# GCpos=Find(FitGauss, numpy.amax(FitGauss))
	892	# Vpos=Find(FactNorm, numpy.amax(FactNorm))
	893	# #Vpos=numpy.sum(FactNorm)/len(FactNorm)
	894	# #Vpos=Find(FactNorm, min(FactNorm, key=lambda value:abs(value- numpy.mean(FactNorm) )))
	895	# #print 'GCpos',GCpos, numpy.amax(FitGauss), 'HWpos',HWpos
	896	# '''**** Getting Fij ****'''
	897	#
	898	# GaussCenter=xFrec[GCpos]
	899	# if (GaussCenter<0 and HalfWidth>0) or (GaussCenter>0 and HalfWidth<0):
	900	# Fij=abs(GaussCenter)+abs(HalfWidth)+0.0000001
	901	# else:
	902	# Fij=abs(GaussCenter-HalfWidth)+0.0000001
	903	#
	904	# '''**** Getting Frecuency range of significant data ****'''
	905	#
	906	# Rangpos=Find(FitGauss,min(FitGauss, key=lambda value:abs(value-Maximun*0.10)))
	907	#
	908	# if Rangpos<GCpos:
	909	# Range=numpy.array([Rangpos,2*GCpos-Rangpos])
	910	# else:
	911	# Range=numpy.array([2*GCpos-Rangpos,Rangpos])
	912	#
	913	# FrecRange=xFrec[Range[0]:Range[1]]
	914	#
	915	# #print 'FrecRange', FrecRange
	916	# '''**** Getting SCPC Slope ****'''
	917	#
	918	# for i in range(self.nRdPairs):
	919	#
	920	# if len(FrecRange)>5 and len(FrecRange)<self.nProfiles*0.5:
	921	# PhaseRange=moving_average(phase[i,Range[0]:Range[1]],N=3)
	922	#
	923	# slope, intercept, r_value, p_value, std_err = stats.linregress(FrecRange,PhaseRange)
	924	# PhaseSlope[i]=slope
	925	# PhaseInter[i]=intercept
	926	# else:
	927	# PhaseSlope[i]=0
	928	# PhaseInter[i]=0
	929	#
	930	# # plt.figure(i+15)
	931	# # plt.title('FASE ( CH%s*CH%s )' %(self.dataOut.pairsList[i][0],self.dataOut.pairsList[i][1]))
	932	# # plt.xlabel('Frecuencia (KHz)')
	933	# # plt.ylabel('Magnitud')
	934	# # #plt.subplot(311+i)
	935	# # plt.plot(FrecRange,PhaseRange,'b')
	936	# # plt.plot(FrecRange,FrecRange*PhaseSlope[i]+PhaseInter[i],'r')
	937	#
	938	# #plt.axis([-0.6, 0.2, -3.2, 3.2])
	939	#
	940	#
	941	# '''Getting constant C'''
	942	# cC=(Fijnumpy.pi)*2
	943	#
	944	# # '''Getting Eij and Nij'''
	945	# # (AntennaX0,AntennaY0)=pol2cart(rheader.AntennaCoord0, rheader.AntennaAngl0*numpy.pi/180)
	946	# # (AntennaX1,AntennaY1)=pol2cart(rheader.AntennaCoord1, rheader.AntennaAngl1*numpy.pi/180)
	947	# # (AntennaX2,AntennaY2)=pol2cart(rheader.AntennaCoord2, rheader.AntennaAngl2*numpy.pi/180)
	948	# #
	949	# # E01=AntennaX0-AntennaX1
	950	# # N01=AntennaY0-AntennaY1
	951	# #
	952	# # E02=AntennaX0-AntennaX2
	953	# # N02=AntennaY0-AntennaY2
	954	# #
	955	# # E12=AntennaX1-AntennaX2
	956	# # N12=AntennaY1-AntennaY2
	957	#
	958	# '''**** Getting constants F and G ****'''
	959	# MijEijNij=numpy.array([[E02,N02], [E12,N12]])
	960	# MijResult0=(-PhaseSlope[1]cC) / (2numpy.pi)
	961	# MijResult1=(-PhaseSlope[2]cC) / (2numpy.pi)
	962	# MijResults=numpy.array([MijResult0,MijResult1])
	963	# (cF,cG) = numpy.linalg.solve(MijEijNij, MijResults)
	964	#
	965	# '''**** Getting constants A, B and H ****'''
	966	# W01=numpy.amax(coherence[0])
	967	# W02=numpy.amax(coherence[1])
	968	# W12=numpy.amax(coherence[2])
	969	#
	970	# WijResult0=((cFE01+cGN01)**2)/cC - numpy.log(W01 / numpy.sqrt(numpy.pi/cC))
	971	# WijResult1=((cFE02+cGN02)**2)/cC - numpy.log(W02 / numpy.sqrt(numpy.pi/cC))
	972	# WijResult2=((cFE12+cGN12)**2)/cC - numpy.log(W12 / numpy.sqrt(numpy.pi/cC))
	973	#
	974	# WijResults=numpy.array([WijResult0, WijResult1, WijResult2])
	975	#
	976	# WijEijNij=numpy.array([ [E012, N012, 2E01N01] , [E022, N022, 2E02N02] , [E122, N122, 2E12N12] ])
	977	# (cA,cB,cH) = numpy.linalg.solve(WijEijNij, WijResults)
	978	#
	979	# VxVy=numpy.array([[cA,cH],[cH,cB]])
	980	#
	981	# VxVyResults=numpy.array([-cF,-cG])
	982	# (Vx,Vy) = numpy.linalg.solve(VxVy, VxVyResults)
	983	# Vzon = Vy
	984	# Vmer = Vx
	985	# Vmag=numpy.sqrt(Vzon2+Vmer2)
	986	# Vang=numpy.arctan2(Vmer,Vzon)
	987	#
	988	# if abs(Vy)<100 and abs(Vy)> 0.:
	989	# self.dataOut.velocityX=numpy.append(self.dataOut.velocityX, Vzon) #Vmag
	990	# #print 'Vmag',Vmag
	991	# else:
	992	# self.dataOut.velocityX=numpy.append(self.dataOut.velocityX, NaN)
	993	#
	994	# if abs(Vx)<100 and abs(Vx) > 0.:
	995	# self.dataOut.velocityY=numpy.append(self.dataOut.velocityY, Vmer) #Vang
	996	# #print 'Vang',Vang
	997	# else:
	998	# self.dataOut.velocityY=numpy.append(self.dataOut.velocityY, NaN)
	999	#
	1000	# if abs(GaussCenter)<2:
	1001	# self.dataOut.velocityV=numpy.append(self.dataOut.velocityV, xFrec[Vpos])
	1002	#
	1003	# else:
	1004	# self.dataOut.velocityV=numpy.append(self.dataOut.velocityV, NaN)
	1005	#
	1006	#
	1007	# # print '********************************************'
	1008	# # print 'HalfWidth ', HalfWidth
	1009	# # print 'Maximun ', Maximun
	1010	# # print 'eMinus1 ', eMinus1
	1011	# # print 'Rangpos ', Rangpos
	1012	# # print 'GaussCenter ',GaussCenter
	1013	# # print 'E01 ',E01
	1014	# # print 'N01 ',N01
	1015	# # print 'E02 ',E02
	1016	# # print 'N02 ',N02
	1017	# # print 'E12 ',E12
	1018	# # print 'N12 ',N12
	1019	# #print 'self.dataOut.velocityX ', self.dataOut.velocityX
	1020	# # print 'Fij ', Fij
	1021	# # print 'cC ', cC
	1022	# # print 'cF ', cF
	1023	# # print 'cG ', cG
	1024	# # print 'cA ', cA
	1025	# # print 'cB ', cB
	1026	# # print 'cH ', cH
	1027	# # print 'Vx ', Vx
	1028	# # print 'Vy ', Vy
	1029	# # print 'Vmag ', Vmag
	1030	# # print 'Vang ', Vang*180/numpy.pi
	1031	# # print 'PhaseSlope ',PhaseSlope[0]
	1032	# # print 'PhaseSlope ',PhaseSlope[1]
	1033	# # print 'PhaseSlope ',PhaseSlope[2]
	1034	# # print '********************************************'
	1035	# #print 'data_output',shape(self.dataOut.velocityX), shape(self.dataOut.velocityY)
	1036	#
	1037	# #print 'self.dataOut.velocityX', len(self.dataOut.velocityX)
	1038	# #print 'self.dataOut.velocityY', len(self.dataOut.velocityY)
	1039	# #print 'self.dataOut.velocityV', self.dataOut.velocityV
	1040	#
	1041	# self.data_output[0]=numpy.array(self.dataOut.velocityX)
	1042	# self.data_output[1]=numpy.array(self.dataOut.velocityY)
	1043	# self.data_output[2]=numpy.array(self.dataOut.velocityV)
	1044	#
	1045	# prin= self.data_output[0][~numpy.isnan(self.data_output[0])]
	1046	# print ' '
	1047	# print 'VmagAverage',numpy.mean(prin)
	1048	# print ' '
	1049	# # plt.figure(5)
	1050	# # plt.subplot(211)
	1051	# # plt.plot(self.dataOut.velocityX,'yo:')
	1052	# # plt.subplot(212)
	1053	# # plt.plot(self.dataOut.velocityY,'yo:')
	1054	#
	1055	# # plt.figure(1)
	1056	# # # plt.subplot(121)
	1057	# # # plt.plot(xFrec,ySamples[0],'k',label='Ch0')
	1058	# # # plt.plot(xFrec,ySamples[1],'g',label='Ch1')
	1059	# # # plt.plot(xFrec,ySamples[2],'r',label='Ch2')
	1060	# # # plt.plot(xFrec,FitGauss,'yo:',label='fit')
	1061	# # # plt.legend()
	1062	# # plt.title('DATOS A ALTURA DE 2850 METROS')
	1063	# #
	1064	# # plt.xlabel('Frecuencia (KHz)')
	1065	# # plt.ylabel('Magnitud')
	1066	# # # plt.subplot(122)
	1067	# # # plt.title('Fit for Time Constant')
	1068	# # #plt.plot(xFrec,zline)
	1069	# # #plt.plot(xFrec,SmoothSPC,'g')
	1070	# # plt.plot(xFrec,FactNorm)
	1071	# # plt.axis([-4, 4, 0, 0.15])
	1072	# # # plt.xlabel('SelfSpectra KHz')
	1073	# #
	1074	# # plt.figure(10)
	1075	# # # plt.subplot(121)
	1076	# # plt.plot(xFrec,ySamples[0],'b',label='Ch0')
	1077	# # plt.plot(xFrec,ySamples[1],'y',label='Ch1')
	1078	# # plt.plot(xFrec,ySamples[2],'r',label='Ch2')
	1079	# # # plt.plot(xFrec,FitGauss,'yo:',label='fit')
	1080	# # plt.legend()
	1081	# # plt.title('SELFSPECTRA EN CANALES')
	1082	# #
	1083	# # plt.xlabel('Frecuencia (KHz)')
	1084	# # plt.ylabel('Magnitud')
	1085	# # # plt.subplot(122)
	1086	# # # plt.title('Fit for Time Constant')
	1087	# # #plt.plot(xFrec,zline)
	1088	# # #plt.plot(xFrec,SmoothSPC,'g')
	1089	# # # plt.plot(xFrec,FactNorm)
	1090	# # # plt.axis([-4, 4, 0, 0.15])
	1091	# # # plt.xlabel('SelfSpectra KHz')
	1092	# #
	1093	# # plt.figure(9)
	1094	# #
	1095	# #
	1096	# # plt.title('DATOS SUAVIZADOS')
	1097	# # plt.xlabel('Frecuencia (KHz)')
	1098	# # plt.ylabel('Magnitud')
	1099	# # plt.plot(xFrec,SmoothSPC,'g')
	1100	# #
	1101	# # #plt.plot(xFrec,FactNorm)
	1102	# # plt.axis([-4, 4, 0, 0.15])
	1103	# # # plt.xlabel('SelfSpectra KHz')
	1104	# # #
	1105	# # plt.figure(2)
	1106	# # # #plt.subplot(121)
	1107	# # plt.plot(xFrec,yMean,'r',label='Mean SelfSpectra')
	1108	# # plt.plot(xFrec,FitGauss,'yo:',label='Ajuste Gaussiano')
	1109	# # # plt.plot(xFrec[Rangpos],FitGauss[Find(FitGauss,min(FitGauss, key=lambda value:abs(value-Maximun*0.1)))],'bo')
	1110	# # # #plt.plot(xFrec,phase)
	1111	# # # plt.xlabel('Suavizado, promediado KHz')
	1112	# # plt.title('SELFSPECTRA PROMEDIADO')
	1113	# # # #plt.subplot(122)
	1114	# # # #plt.plot(xSamples,zline)
	1115	# # plt.xlabel('Frecuencia (KHz)')
	1116	# # plt.ylabel('Magnitud')
	1117	# # plt.legend()
	1118	# # #
	1119	# # # plt.figure(3)
	1120	# # # plt.subplot(311)
	1121	# # # #plt.plot(xFrec,phase[0])
	1122	# # # plt.plot(xFrec,phase[0],'g')
	1123	# # # plt.subplot(312)
	1124	# # # plt.plot(xFrec,phase[1],'g')
	1125	# # # plt.subplot(313)
	1126	# # # plt.plot(xFrec,phase[2],'g')
	1127	# # # #plt.plot(xFrec,phase[2])
	1128	# # #
	1129	# # # plt.figure(4)
	1130	# # #
	1131	# # # plt.plot(xSamples,coherence[0],'b')
	1132	# # # plt.plot(xSamples,coherence[1],'r')
	1133	# # # plt.plot(xSamples,coherence[2],'g')
	1134	# # plt.show()
	1135	# # #
	1136	# # # plt.clf()
	1137	# # # plt.cla()
	1138	# # # plt.close()
	1139	#
	1140	# print ' '
	1141
	1142
	1143
	1144	self.BlockCounter+=2
	1145
	1146	else:
	1147	self.fileSelector+=1
	1148	self.BlockCounter=0
	1149	print "Next File"
	1150
	1151
	1152
	1153
	1154

schainpy/model/io/jroIO_madrigal.py created 644 +632 0

This diff has been collapsed as it changes many lines, (632 lines changed) Show them Hide them
		@@ -0,0 +1,632
	1	'''
	2	Created on Aug 1, 2017
	3
	4	@author: Juan C. Espinoza
	5	'''
	6
	7	import os
	8	import sys
	9	import time
	10	import json
	11	import glob
	12	import datetime
	13
	14	import numpy
	15	import h5py
	16
	17	from schainpy.model.io.jroIO_base import JRODataReader
	18	from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
	19	from schainpy.model.data.jrodata import Parameters
	20	from schainpy.utils import log
	21
	22	try:
	23	import madrigal.cedar
	24	except:
	25	log.warning(
	26	'You should install "madrigal library" module if you want to read/write Madrigal data'
	27	)
	28
	29	DEF_CATALOG = {
	30	'principleInvestigator': 'Marco Milla',
	31	'expPurpose': None,
	32	'cycleTime': None,
	33	'correlativeExp': None,
	34	'sciRemarks': None,
	35	'instRemarks': None
	36	}
	37	DEF_HEADER = {
	38	'kindatDesc': None,
	39	'analyst': 'Jicamarca User',
	40	'comments': None,
	41	'history': None
	42	}
	43	MNEMONICS = {
	44	10: 'jro',
	45	11: 'jbr',
	46	840: 'jul',
	47	13: 'jas',
	48	1000: 'pbr',
	49	1001: 'hbr',
	50	1002: 'obr',
	51	}
	52
	53	UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
	54
	55	def load_json(obj):
	56	'''
	57	Parse json as string instead of unicode
	58	'''
	59
	60	if isinstance(obj, str):
	61	iterable = json.loads(obj)
	62	else:
	63	iterable = obj
	64
	65	if isinstance(iterable, dict):
	66	return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
	67	for k, v in iterable.items()}
	68	elif isinstance(iterable, (list, tuple)):
	69	return [str(v) if isinstance(v, unicode) else v for v in iterable]
	70
	71	return iterable
	72
	73
	74	class MADReader(JRODataReader, ProcessingUnit):
	75
	76	def __init__(self, **kwargs):
	77
	78	ProcessingUnit.__init__(self, **kwargs)
	79
	80	self.dataOut = Parameters()
	81	self.counter_records = 0
	82	self.nrecords = None
	83	self.flagNoMoreFiles = 0
	84	self.isConfig = False
	85	self.filename = None
	86	self.intervals = set()
	87
	88	def setup(self,
	89	path=None,
	90	startDate=None,
	91	endDate=None,
	92	format=None,
	93	startTime=datetime.time(0, 0, 0),
	94	endTime=datetime.time(23, 59, 59),
	95	**kwargs):
	96
	97	self.path = path
	98	self.startDate = startDate
	99	self.endDate = endDate
	100	self.startTime = startTime
	101	self.endTime = endTime
	102	self.datatime = datetime.datetime(1900,1,1)
	103	self.oneDDict = load_json(kwargs.get('oneDDict',
	104	"{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
	105	self.twoDDict = load_json(kwargs.get('twoDDict',
	106	"{\"GDALT\": \"heightList\"}"))
	107	self.ind2DList = load_json(kwargs.get('ind2DList',
	108	"[\"GDALT\"]"))
	109	if self.path is None:
	110	raise ValueError, 'The path is not valid'
	111
	112	if format is None:
	113	raise ValueError, 'The format is not valid choose simple or hdf5'
	114	elif format.lower() in ('simple', 'txt'):
	115	self.ext = '.txt'
	116	elif format.lower() in ('cedar',):
	117	self.ext = '.001'
	118	else:
	119	self.ext = '.hdf5'
	120
	121	self.search_files(self.path)
	122	self.fileId = 0
	123
	124	if not self.fileList:
	125	raise Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path)
	126
	127	self.setNextFile()
	128
	129	def search_files(self, path):
	130	'''
	131	Searching for madrigal files in path
	132	Creating a list of files to procces included in [startDate,endDate]
	133
	134	Input:
	135	path - Path to find files
	136	'''
	137
	138	log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
	139	foldercounter = 0
	140	fileList0 = glob.glob1(path, '*{}'.format(self.ext))
	141	fileList0.sort()
	142
	143	self.fileList = []
	144	self.dateFileList = []
	145
	146	startDate = self.startDate - datetime.timedelta(1)
	147	endDate = self.endDate + datetime.timedelta(1)
	148
	149	for thisFile in fileList0:
	150	year = thisFile[3:7]
	151	if not year.isdigit():
	152	continue
	153
	154	month = thisFile[7:9]
	155	if not month.isdigit():
	156	continue
	157
	158	day = thisFile[9:11]
	159	if not day.isdigit():
	160	continue
	161
	162	year, month, day = int(year), int(month), int(day)
	163	dateFile = datetime.date(year, month, day)
	164
	165	if (startDate > dateFile) or (endDate < dateFile):
	166	continue
	167
	168	self.fileList.append(thisFile)
	169	self.dateFileList.append(dateFile)
	170
	171	return
	172
	173	def parseHeader(self):
	174	'''
	175	'''
	176
	177	self.output = {}
	178	self.version = '2'
	179	s_parameters = None
	180	if self.ext == '.txt':
	181	self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s]
	182	elif self.ext == '.hdf5':
	183	metadata = self.fp['Metadata']
	184	data = self.fp['Data']['Array Layout']
	185	if 'Independent Spatial Parameters' in metadata:
	186	s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']]
	187	self.version = '3'
	188	one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']]
	189	one_d = [1 for s in one]
	190	two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']]
	191	two_d = [2 for s in two]
	192	self.parameters = one + two
	193	self.parameters_d = one_d + two_d
	194
	195	log.success('Parameters found: {}'.format(','.join(self.parameters)),
	196	'MADReader')
	197	if s_parameters:
	198	log.success('Spatial parameters: {}'.format(','.join(s_parameters)),
	199	'MADReader')
	200
	201	for param in self.oneDDict.keys():
	202	if param.lower() not in self.parameters:
	203	log.warning(
	204	'Parameter {} not found will be ignored'.format(
	205	param),
	206	'MADReader')
	207	self.oneDDict.pop(param, None)
	208
	209	for param, value in self.twoDDict.items():
	210	if param.lower() not in self.parameters:
	211	log.warning(
	212	'Parameter {} not found, it will be ignored'.format(
	213	param),
	214	'MADReader')
	215	self.twoDDict.pop(param, None)
	216	continue
	217	if isinstance(value, list):
	218	if value[0] not in self.output:
	219	self.output[value[0]] = []
	220	self.output[value[0]].append(None)
	221
	222	def parseData(self):
	223	'''
	224	'''
	225
	226	if self.ext == '.txt':
	227	self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
	228	self.nrecords = self.data.shape[0]
	229	self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())])
	230	elif self.ext == '.hdf5':
	231	self.data = self.fp['Data']['Array Layout']
	232	self.nrecords = len(self.data['timestamps'].value)
	233	self.ranges = self.data['range'].value
	234
	235	def setNextFile(self):
	236	'''
	237	'''
	238
	239	file_id = self.fileId
	240
	241	if file_id == len(self.fileList):
	242	log.success('No more files', 'MADReader')
	243	self.flagNoMoreFiles = 1
	244	return 0
	245
	246	log.success(
	247	'Opening: {}'.format(self.fileList[file_id]),
	248	'MADReader'
	249	)
	250
	251	filename = os.path.join(self.path, self.fileList[file_id])
	252
	253	if self.filename is not None:
	254	self.fp.close()
	255
	256	self.filename = filename
	257	self.filedate = self.dateFileList[file_id]
	258
	259	if self.ext=='.hdf5':
	260	self.fp = h5py.File(self.filename, 'r')
	261	else:
	262	self.fp = open(self.filename, 'rb')
	263
	264	self.parseHeader()
	265	self.parseData()
	266	self.sizeOfFile = os.path.getsize(self.filename)
	267	self.counter_records = 0
	268	self.flagIsNewFile = 0
	269	self.fileId += 1
	270
	271	return 1
	272
	273	def readNextBlock(self):
	274
	275	while True:
	276	self.flagDiscontinuousBlock = 0
	277	if self.flagIsNewFile:
	278	if not self.setNextFile():
	279	return 0
	280
	281	self.readBlock()
	282
	283	if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
	284	(self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
	285	log.warning(
	286	'Reading Record No. {}/{} -> {} [Skipping]'.format(
	287	self.counter_records,
	288	self.nrecords,
	289	self.datatime.ctime()),
	290	'MADReader')
	291	continue
	292	break
	293
	294	log.log(
	295	'Reading Record No. {}/{} -> {}'.format(
	296	self.counter_records,
	297	self.nrecords,
	298	self.datatime.ctime()),
	299	'MADReader')
	300
	301	return 1
	302
	303	def readBlock(self):
	304	'''
	305	'''
	306	dum = []
	307	if self.ext == '.txt':
	308	dt = self.data[self.counter_records][:6].astype(int)
	309	self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
	310	while True:
	311	dt = self.data[self.counter_records][:6].astype(int)
	312	datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
	313	if datatime == self.datatime:
	314	dum.append(self.data[self.counter_records])
	315	self.counter_records += 1
	316	if self.counter_records == self.nrecords:
	317	self.flagIsNewFile = True
	318	break
	319	continue
	320	self.intervals.add((datatime-self.datatime).seconds)
	321	if datatime.date() > self.datatime.date():
	322	self.flagDiscontinuousBlock = 1
	323	break
	324	elif self.ext == '.hdf5':
	325	datatime = datetime.datetime.utcfromtimestamp(
	326	self.data['timestamps'][self.counter_records])
	327	nHeights = len(self.ranges)
	328	for n, param in enumerate(self.parameters):
	329	if self.parameters_d[n] == 1:
	330	dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records])
	331	else:
	332	if self.version == '2':
	333	dum.append(self.data['2D Parameters'][param][self.counter_records])
	334	else:
	335	tmp = self.data['2D Parameters'][param].value.T
	336	dum.append(tmp[self.counter_records])
	337	self.intervals.add((datatime-self.datatime).seconds)
	338	if datatime.date()>self.datatime.date():
	339	self.flagDiscontinuousBlock = 1
	340	self.datatime = datatime
	341	self.counter_records += 1
	342	if self.counter_records == self.nrecords:
	343	self.flagIsNewFile = True
	344
	345	self.buffer = numpy.array(dum)
	346	return
	347
	348	def set_output(self):
	349	'''
	350	Storing data from buffer to dataOut object
	351	'''
	352
	353	parameters = [None for __ in self.parameters]
	354
	355	for param, attr in self.oneDDict.items():
	356	x = self.parameters.index(param.lower())
	357	setattr(self.dataOut, attr, self.buffer[0][x])
	358
	359	for param, value in self.twoDDict.items():
	360	x = self.parameters.index(param.lower())
	361	if self.ext == '.txt':
	362	y = self.parameters.index(self.ind2DList[0].lower())
	363	ranges = self.buffer[:,y]
	364	if self.ranges.size == ranges.size:
	365	continue
	366	index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
	367	dummy = numpy.zeros(self.ranges.shape) + numpy.nan
	368	dummy[index] = self.buffer[:,x]
	369	else:
	370	dummy = self.buffer[x]
	371
	372	if isinstance(value, str):
	373	if value not in self.ind2DList:
	374	setattr(self.dataOut, value, dummy.reshape(1,-1))
	375	elif isinstance(value, list):
	376	self.output[value[0]][value[1]] = dummy
	377	parameters[value[1]] = param
	378
	379	for key, value in self.output.items():
	380	setattr(self.dataOut, key, numpy.array(value))
	381
	382	self.dataOut.parameters = [s for s in parameters if s]
	383	self.dataOut.heightList = self.ranges
	384	self.dataOut.utctime = (self.datatime - UT1970).total_seconds()
	385	self.dataOut.utctimeInit = self.dataOut.utctime
	386	self.dataOut.paramInterval = min(self.intervals)
	387	self.dataOut.useLocalTime = False
	388	self.dataOut.flagNoData = False
	389	self.dataOut.nrecords = self.nrecords
	390	self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
	391
	392	def getData(self):
	393	'''
	394	Storing data from databuffer to dataOut object
	395	'''
	396	if self.flagNoMoreFiles:
	397	self.dataOut.flagNoData = True
	398	log.error('No file left to process', 'MADReader')
	399	return 0
	400
	401	if not self.readNextBlock():
	402	self.dataOut.flagNoData = True
	403	return 0
	404
	405	self.set_output()
	406
	407	return 1
	408
	409
	410	class MADWriter(Operation):
	411
	412	missing = -32767
	413
	414	def __init__(self, **kwargs):
	415
	416	Operation.__init__(self, **kwargs)
	417	self.dataOut = Parameters()
	418	self.path = None
	419	self.fp = None
	420
	421	def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}',
	422	metadata='{}', format='cedar', **kwargs):
	423	'''
	424	Inputs:
	425	path - path where files will be created
	426	oneDDict - json of one-dimensional parameters in record where keys
	427	are Madrigal codes (integers or mnemonics) and values the corresponding
	428	dataOut attribute e.g: {
	429	'gdlatr': 'lat',
	430	'gdlonr': 'lon',
	431	'gdlat2':'lat',
	432	'glon2':'lon'}
	433	ind2DList - list of independent spatial two-dimensional parameters e.g:
	434	['heighList']
	435	twoDDict - json of two-dimensional parameters in record where keys
	436	are Madrigal codes (integers or mnemonics) and values the corresponding
	437	dataOut attribute if multidimensional array specify as tupple
	438	('attr', pos) e.g: {
	439	'gdalt': 'heightList',
	440	'vn1p2': ('data_output', 0),
	441	'vn2p2': ('data_output', 1),
	442	'vn3': ('data_output', 2),
	443	'snl': ('data_SNR', 'db')
	444	}
	445	metadata - json of madrigal metadata (kinst, kindat, catalog and header)
	446	'''
	447	if not self.isConfig:
	448	self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs)
	449	self.isConfig = True
	450
	451	self.dataOut = dataOut
	452	self.putData()
	453	return
	454
	455	def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
	456	'''
	457	Configure Operation
	458	'''
	459
	460	self.path = path
	461	self.blocks = kwargs.get('blocks', None)
	462	self.counter = 0
	463	self.oneDDict = load_json(oneDDict)
	464	self.twoDDict = load_json(twoDDict)
	465	self.ind2DList = load_json(ind2DList)
	466	meta = load_json(metadata)
	467	self.kinst = meta.get('kinst')
	468	self.kindat = meta.get('kindat')
	469	self.catalog = meta.get('catalog', DEF_CATALOG)
	470	self.header = meta.get('header', DEF_HEADER)
	471	if format == 'cedar':
	472	self.ext = '.dat'
	473	self.extra_args = {}
	474	elif format == 'hdf5':
	475	self.ext = '.hdf5'
	476	self.extra_args = {'ind2DList': self.ind2DList}
	477
	478	self.keys = [k.lower() for k in self.twoDDict]
	479	if 'range' in self.keys:
	480	self.keys.remove('range')
	481	if 'gdalt' in self.keys:
	482	self.keys.remove('gdalt')
	483
	484	def setFile(self):
	485	'''
	486	Create new cedar file object
	487	'''
	488
	489	self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
	490	date = datetime.datetime.fromtimestamp(self.dataOut.utctime)
	491
	492	filename = '{}{}{}'.format(self.mnemonic,
	493	date.strftime('%Y%m%d_%H%M%S'),
	494	self.ext)
	495
	496	self.fullname = os.path.join(self.path, filename)
	497
	498	if os.path.isfile(self.fullname) :
	499	log.warning(
	500	'Destination path {} already exists. Previous file deleted.'.format(
	501	self.fullname),
	502	'MADWriter')
	503	os.remove(self.fullname)
	504
	505	try:
	506	log.success(
	507	'Creating file: {}'.format(self.fullname),
	508	'MADWriter')
	509	self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
	510	except ValueError, e:
	511	log.error(
	512	'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
	513	'MADWriter')
	514	return
	515
	516	return 1
	517
	518	def writeBlock(self):
	519	'''
	520	Add data records to cedar file taking data from oneDDict and twoDDict
	521	attributes.
	522	Allowed parameters in: parcodes.tab
	523	'''
	524
	525	startTime = datetime.datetime.fromtimestamp(self.dataOut.utctime)
	526	endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
	527	heights = self.dataOut.heightList
	528
	529	if self.ext == '.dat':
	530	invalid = numpy.isnan(self.dataOut.data_output)
	531	self.dataOut.data_output[invalid] = self.missing
	532	out = {}
	533	for key, value in self.twoDDict.items():
	534	key = key.lower()
	535	if isinstance(value, str):
	536	if 'db' in value.lower():
	537	tmp = getattr(self.dataOut, value.replace('_db', ''))
	538	SNRavg = numpy.average(tmp, axis=0)
	539	tmp = 10*numpy.log10(SNRavg)
	540	else:
	541	tmp = getattr(self.dataOut, value)
	542	out[key] = tmp.flatten()
	543	elif isinstance(value, (tuple, list)):
	544	attr, x = value
	545	data = getattr(self.dataOut, attr)
	546	out[key] = data[int(x)]
	547
	548	a = numpy.array([out[k] for k in self.keys])
	549	nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])
	550	index = numpy.where(nrows == False)[0]
	551
	552	rec = madrigal.cedar.MadrigalDataRecord(
	553	self.kinst,
	554	self.kindat,
	555	startTime.year,
	556	startTime.month,
	557	startTime.day,
	558	startTime.hour,
	559	startTime.minute,
	560	startTime.second,
	561	startTime.microsecond/10000,
	562	endTime.year,
	563	endTime.month,
	564	endTime.day,
	565	endTime.hour,
	566	endTime.minute,
	567	endTime.second,
	568	endTime.microsecond/10000,
	569	self.oneDDict.keys(),
	570	self.twoDDict.keys(),
	571	len(index),
	572	**self.extra_args
	573	)
	574
	575	# Setting 1d values
	576	for key in self.oneDDict:
	577	rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
	578
	579	# Setting 2d values
	580	nrec = 0
	581	for n in index:
	582	for key in out:
	583	rec.set2D(key, nrec, out[key][n])
	584	nrec += 1
	585
	586	self.fp.append(rec)
	587	if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
	588	self.fp.dump()
	589	if self.counter % 10 == 0 and self.counter > 0:
	590	log.log(
	591	'Writing {} records'.format(
	592	self.counter),
	593	'MADWriter')
	594
	595	def setHeader(self):
	596	'''
	597	Create an add catalog and header to cedar file
	598	'''
	599
	600	log.success('Closing file {}'.format(self.fullname), 'MADWriter')
	601
	602	if self.ext == '.dat':
	603	self.fp.write()
	604	else:
	605	self.fp.dump()
	606	self.fp.close()
	607
	608	header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
	609	header.createCatalog(**self.catalog)
	610	header.createHeader(**self.header)
	611	header.write()
	612
	613	def putData(self):
	614
	615	if self.dataOut.flagNoData:
	616	return 0
	617
	618	if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:
	619	if self.counter > 0:
	620	self.setHeader()
	621	self.counter = 0
	622
	623	if self.counter == 0:
	624	self.setFile()
	625
	626	self.writeBlock()
	627	self.counter += 1
	628
	629	def close(self):
	630
	631	if self.counter > 0:
	632	self.setHeader()

schainpy/model/io/jroIO_mira35c.py created 644 +803 0

This diff has been collapsed as it changes many lines, (803 lines changed) Show them Hide them
		@@ -0,0 +1,803
	1	import os, sys
	2	import glob
	3	import fnmatch
	4	import datetime
	5	import time
	6	import re
	7	import h5py
	8	import numpy
	9	import matplotlib.pyplot as plt
	10
	11	import pylab as plb
	12	from scipy.optimize import curve_fit
	13	from scipy import asarray as ar,exp
	14	from scipy import stats
	15
	16	from numpy.ma.core import getdata
	17
	18	SPEED_OF_LIGHT = 299792458
	19	SPEED_OF_LIGHT = 3e8
	20
	21	try:
	22	from gevent import sleep
	23	except:
	24	from time import sleep
	25
	26	from schainpy.model.data.jrodata import Spectra
	27	#from schainpy.model.data.BLTRheaderIO import FileHeader, RecordHeader
	28	from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
	29	#from schainpy.model.io.jroIO_bltr import BLTRReader
	30	from numpy import imag, shape, NaN, empty
	31
	32
	33
	34	class Header(object):
	35
	36	def __init__(self):
	37	raise NotImplementedError
	38
	39
	40	def read(self):
	41
	42	raise NotImplementedError
	43
	44	def write(self):
	45
	46	raise NotImplementedError
	47
	48	def printInfo(self):
	49
	50	message = "#"*50 + "\n"
	51	message += self.__class__.__name__.upper() + "\n"
	52	message += "#"*50 + "\n"
	53
	54	keyList = self.__dict__.keys()
	55	keyList.sort()
	56
	57	for key in keyList:
	58	message += "%s = %s" %(key, self.__dict__[key]) + "\n"
	59
	60	if "size" not in keyList:
	61	attr = getattr(self, "size")
	62
	63	if attr:
	64	message += "%s = %s" %("size", attr) + "\n"
	65
	66	#print message
	67
	68
	69	FILE_HEADER = numpy.dtype([ #HEADER 1024bytes
	70	('Hname','a32'), #Original file name
	71	('Htime',numpy.str_,32), #Date and time when the file was created
	72	('Hoper',numpy.str_,64), #Name of operator who created the file
	73	('Hplace',numpy.str_,128), #Place where the measurements was carried out
	74	('Hdescr',numpy.str_,256), #Description of measurements
	75	('Hdummy',numpy.str_,512), #Reserved space
	76	#Main chunk 8bytes
	77	('Msign',numpy.str_,4), #Main chunk signature FZKF or NUIG
	78	('MsizeData','<i4'), #Size of data block main chunk
	79	#Processing DSP parameters 36bytes
	80	('PPARsign',numpy.str_,4), #PPAR signature
	81	('PPARsize','<i4'), #PPAR size of block
	82	('PPARprf','<i4'), #Pulse repetition frequency
	83	('PPARpdr','<i4'), #Pulse duration
	84	('PPARsft','<i4'), #FFT length
	85	('PPARavc','<i4'), #Number of spectral (in-coherent) averages
	86	('PPARihp','<i4'), #Number of lowest range gate for moment estimation
	87	('PPARchg','<i4'), #Count for gates for moment estimation
	88	('PPARpol','<i4'), #switch on/off polarimetric measurements. Should be 1.
	89	#Service DSP parameters 112bytes
	90	('SPARatt','<i4'), #STC attenuation on the lowest ranges on/off
	91	('SPARtx','<i4'), #OBSOLETE
	92	('SPARaddGain0','<f4'), #OBSOLETE
	93	('SPARaddGain1','<f4'), #OBSOLETE
	94	('SPARwnd','<i4'), #Debug only. It normal mode it is 0.
	95	('SPARpos','<i4'), #Delay between sync pulse and tx pulse for phase corr, ns
	96	('SPARadd','<i4'), #"add to pulse" to compensate for delay between the leading edge of driver pulse and envelope of the RF signal.
	97	('SPARlen','<i4'), #Time for measuring txn pulse phase. OBSOLETE
	98	('SPARcal','<i4'), #OBSOLETE
	99	('SPARnos','<i4'), #OBSOLETE
	100	('SPARof0','<i4'), #detection threshold
	101	('SPARof1','<i4'), #OBSOLETE
	102	('SPARswt','<i4'), #2nd moment estimation threshold
	103	('SPARsum','<i4'), #OBSOLETE
	104	('SPARosc','<i4'), #flag Oscillosgram mode
	105	('SPARtst','<i4'), #OBSOLETE
	106	('SPARcor','<i4'), #OBSOLETE
	107	('SPARofs','<i4'), #OBSOLETE
	108	('SPARhsn','<i4'), #Hildebrand div noise detection on noise gate
	109	('SPARhsa','<f4'), #Hildebrand div noise detection on all gates
	110	('SPARcalibPow_M','<f4'), #OBSOLETE
	111	('SPARcalibSNR_M','<f4'), #OBSOLETE
	112	('SPARcalibPow_S','<f4'), #OBSOLETE
	113	('SPARcalibSNR_S','<f4'), #OBSOLETE
	114	('SPARrawGate1','<i4'), #Lowest range gate for spectra saving Raw_Gate1 >=5
	115	('SPARrawGate2','<i4'), #Number of range gates with atmospheric signal
	116	('SPARraw','<i4'), #flag - IQ or spectra saving on/off
	117	('SPARprc','<i4'),]) #flag - Moment estimation switched on/off
	118
	119
	120
	121	class FileHeaderMIRA35c(Header):
	122
	123	def __init__(self):
	124
	125	self.Hname= None
	126	self.Htime= None
	127	self.Hoper= None
	128	self.Hplace= None
	129	self.Hdescr= None
	130	self.Hdummy= None
	131
	132	self.Msign=None
	133	self.MsizeData=None
	134
	135	self.PPARsign=None
	136	self.PPARsize=None
	137	self.PPARprf=None
	138	self.PPARpdr=None
	139	self.PPARsft=None
	140	self.PPARavc=None
	141	self.PPARihp=None
	142	self.PPARchg=None
	143	self.PPARpol=None
	144	#Service DSP parameters
	145	self.SPARatt=None
	146	self.SPARtx=None
	147	self.SPARaddGain0=None
	148	self.SPARaddGain1=None
	149	self.SPARwnd=None
	150	self.SPARpos=None
	151	self.SPARadd=None
	152	self.SPARlen=None
	153	self.SPARcal=None
	154	self.SPARnos=None
	155	self.SPARof0=None
	156	self.SPARof1=None
	157	self.SPARswt=None
	158	self.SPARsum=None
	159	self.SPARosc=None
	160	self.SPARtst=None
	161	self.SPARcor=None
	162	self.SPARofs=None
	163	self.SPARhsn=None
	164	self.SPARhsa=None
	165	self.SPARcalibPow_M=None
	166	self.SPARcalibSNR_M=None
	167	self.SPARcalibPow_S=None
	168	self.SPARcalibSNR_S=None
	169	self.SPARrawGate1=None
	170	self.SPARrawGate2=None
	171	self.SPARraw=None
	172	self.SPARprc=None
	173
	174	self.FHsize=1180
	175
	176	def FHread(self, fp):
	177
	178	header = numpy.fromfile(fp, FILE_HEADER,1)
	179	''' numpy.fromfile(file, dtype, count, sep='')
	180	file : file or str
	181	Open file object or filename.
	182
	183	dtype : data-type
	184	Data type of the returned array. For binary files, it is used to determine
	185	the size and byte-order of the items in the file.
	186
	187	count : int
	188	Number of items to read. -1 means all items (i.e., the complete file).
	189
	190	sep : str
	191	Separator between items if file is a text file. Empty ("") separator means
	192	the file should be treated as binary. Spaces (" ") in the separator match zero
	193	or more whitespace characters. A separator consisting only of spaces must match
	194	at least one whitespace.
	195
	196	'''
	197
	198
	199	self.Hname= str(header['Hname'][0])
	200	self.Htime= str(header['Htime'][0])
	201	self.Hoper= str(header['Hoper'][0])
	202	self.Hplace= str(header['Hplace'][0])
	203	self.Hdescr= str(header['Hdescr'][0])
	204	self.Hdummy= str(header['Hdummy'][0])
	205	#1024
	206
	207	self.Msign=str(header['Msign'][0])
	208	self.MsizeData=header['MsizeData'][0]
	209	#8
	210
	211	self.PPARsign=str(header['PPARsign'][0])
	212	self.PPARsize=header['PPARsize'][0]
	213	self.PPARprf=header['PPARprf'][0]
	214	self.PPARpdr=header['PPARpdr'][0]
	215	self.PPARsft=header['PPARsft'][0]
	216	self.PPARavc=header['PPARavc'][0]
	217	self.PPARihp=header['PPARihp'][0]
	218	self.PPARchg=header['PPARchg'][0]
	219	self.PPARpol=header['PPARpol'][0]
	220	#Service DSP parameters
	221	#36
	222
	223	self.SPARatt=header['SPARatt'][0]
	224	self.SPARtx=header['SPARtx'][0]
	225	self.SPARaddGain0=header['SPARaddGain0'][0]
	226	self.SPARaddGain1=header['SPARaddGain1'][0]
	227	self.SPARwnd=header['SPARwnd'][0]
	228	self.SPARpos=header['SPARpos'][0]
	229	self.SPARadd=header['SPARadd'][0]
	230	self.SPARlen=header['SPARlen'][0]
	231	self.SPARcal=header['SPARcal'][0]
	232	self.SPARnos=header['SPARnos'][0]
	233	self.SPARof0=header['SPARof0'][0]
	234	self.SPARof1=header['SPARof1'][0]
	235	self.SPARswt=header['SPARswt'][0]
	236	self.SPARsum=header['SPARsum'][0]
	237	self.SPARosc=header['SPARosc'][0]
	238	self.SPARtst=header['SPARtst'][0]
	239	self.SPARcor=header['SPARcor'][0]
	240	self.SPARofs=header['SPARofs'][0]
	241	self.SPARhsn=header['SPARhsn'][0]
	242	self.SPARhsa=header['SPARhsa'][0]
	243	self.SPARcalibPow_M=header['SPARcalibPow_M'][0]
	244	self.SPARcalibSNR_M=header['SPARcalibSNR_M'][0]
	245	self.SPARcalibPow_S=header['SPARcalibPow_S'][0]
	246	self.SPARcalibSNR_S=header['SPARcalibSNR_S'][0]
	247	self.SPARrawGate1=header['SPARrawGate1'][0]
	248	self.SPARrawGate2=header['SPARrawGate2'][0]
	249	self.SPARraw=header['SPARraw'][0]
	250	self.SPARprc=header['SPARprc'][0]
	251	#112
	252	#1180
	253	#print 'Pointer fp header', fp.tell()
	254	#print ' '
	255	#print 'SPARrawGate'
	256	#print self.SPARrawGate2 - self.SPARrawGate1
	257
	258	#print ' '
	259	#print 'Hname'
	260	#print self.Hname
	261
	262	#print ' '
	263	#print 'Msign'
	264	#print self.Msign
	265
	266	def write(self, fp):
	267
	268	headerTuple = (self.Hname,
	269	self.Htime,
	270	self.Hoper,
	271	self.Hplace,
	272	self.Hdescr,
	273	self.Hdummy)
	274
	275
	276	header = numpy.array(headerTuple, FILE_HEADER)
	277	# numpy.array(object, dtype=None, copy=True, order=None, subok=False, ndmin=0)
	278	header.tofile(fp)
	279	''' ndarray.tofile(fid, sep, format) Write array to a file as text or binary (default).
	280
	281	fid : file or str
	282	An open file object, or a string containing a filename.
	283
	284	sep : str
	285	Separator between array items for text output. If "" (empty), a binary file is written,
	286	equivalent to file.write(a.tobytes()).
	287
	288	format : str
	289	Format string for text file output. Each entry in the array is formatted to text by
	290	first converting it to the closest Python type, and then using "format" % item.
	291
	292	'''
	293
	294	return 1
	295
	296	SRVI_HEADER = numpy.dtype([
	297	('SignatureSRVI1',numpy.str_,4),#
	298	('SizeOfDataBlock1','<i4'),#
	299	('DataBlockTitleSRVI1',numpy.str_,4),#
	300	('SizeOfSRVI1','<i4'),])#
	301
	302	class SRVIHeader(Header):
	303	def __init__(self, SignatureSRVI1=0, SizeOfDataBlock1=0, DataBlockTitleSRVI1=0, SizeOfSRVI1=0):
	304
	305	self.SignatureSRVI1 = SignatureSRVI1
	306	self.SizeOfDataBlock1 = SizeOfDataBlock1
	307	self.DataBlockTitleSRVI1 = DataBlockTitleSRVI1
	308	self.SizeOfSRVI1 = SizeOfSRVI1
	309
	310	self.SRVIHsize=16
	311
	312	def SRVIread(self, fp):
	313
	314	header = numpy.fromfile(fp, SRVI_HEADER,1)
	315
	316	self.SignatureSRVI1 = str(header['SignatureSRVI1'][0])
	317	self.SizeOfDataBlock1 = header['SizeOfDataBlock1'][0]
	318	self.DataBlockTitleSRVI1 = str(header['DataBlockTitleSRVI1'][0])
	319	self.SizeOfSRVI1 = header['SizeOfSRVI1'][0]
	320	#16
	321	print 'Pointer fp SRVIheader', fp.tell()
	322
	323
	324	SRVI_STRUCTURE = numpy.dtype([
	325	('frame_cnt','<u4'),#
	326	('time_t','<u4'), #
	327	('tpow','<f4'), #
	328	('npw1','<f4'), #
	329	('npw2','<f4'), #
	330	('cpw1','<f4'), #
	331	('pcw2','<f4'), #
	332	('ps_err','<u4'), #
	333	('te_err','<u4'), #
	334	('rc_err','<u4'), #
	335	('grs1','<u4'), #
	336	('grs2','<u4'), #
	337	('azipos','<f4'), #
	338	('azivel','<f4'), #
	339	('elvpos','<f4'), #
	340	('elvvel','<f4'), #
	341	('northAngle','<f4'), #
	342	('microsec','<u4'), #
	343	('azisetvel','<f4'), #
	344	('elvsetpos','<f4'), #
	345	('RadarConst','<f4'),]) #
	346
	347
	348
	349
	350	class RecordHeader(Header):
	351
	352
	353	def __init__(self, frame_cnt=0, time_t= 0, tpow=0, npw1=0, npw2=0,
	354	cpw1=0, pcw2=0, ps_err=0, te_err=0, rc_err=0, grs1=0,
	355	grs2=0, azipos=0, azivel=0, elvpos=0, elvvel=0, northangle=0,
	356	microsec=0, azisetvel=0, elvsetpos=0, RadarConst=0 , RecCounter=0, Off2StartNxtRec=0):
	357
	358
	359	self.frame_cnt = frame_cnt
	360	self.dwell = time_t
	361	self.tpow = tpow
	362	self.npw1 = npw1
	363	self.npw2 = npw2
	364	self.cpw1 = cpw1
	365	self.pcw2 = pcw2
	366	self.ps_err = ps_err
	367	self.te_err = te_err
	368	self.rc_err = rc_err
	369	self.grs1 = grs1
	370	self.grs2 = grs2
	371	self.azipos = azipos
	372	self.azivel = azivel
	373	self.elvpos = elvpos
	374	self.elvvel = elvvel
	375	self.northAngle = northangle
	376	self.microsec = microsec
	377	self.azisetvel = azisetvel
	378	self.elvsetpos = elvsetpos
	379	self.RadarConst = RadarConst
	380	self.RHsize=84
	381	self.RecCounter = RecCounter
	382	self.Off2StartNxtRec=Off2StartNxtRec
	383
	384	def RHread(self, fp):
	385
	386	#startFp = open(fp,"rb") #The method tell() returns the current position of the file read/write pointer within the file.
	387
	388	#OffRHeader= 1180 + self.RecCounter*(self.Off2StartNxtRec)
	389	#startFp.seek(OffRHeader, os.SEEK_SET)
	390
	391	#print 'Posicion del bloque: ',OffRHeader
	392
	393	header = numpy.fromfile(fp,SRVI_STRUCTURE,1)
	394
	395	self.frame_cnt = header['frame_cnt'][0]#
	396	self.time_t = header['time_t'][0] #
	397	self.tpow = header['tpow'][0] #
	398	self.npw1 = header['npw1'][0] #
	399	self.npw2 = header['npw2'][0] #
	400	self.cpw1 = header['cpw1'][0] #
	401	self.pcw2 = header['pcw2'][0] #
	402	self.ps_err = header['ps_err'][0] #
	403	self.te_err = header['te_err'][0] #
	404	self.rc_err = header['rc_err'][0] #
	405	self.grs1 = header['grs1'][0] #
	406	self.grs2 = header['grs2'][0] #
	407	self.azipos = header['azipos'][0] #
	408	self.azivel = header['azivel'][0] #
	409	self.elvpos = header['elvpos'][0] #
	410	self.elvvel = header['elvvel'][0] #
	411	self.northAngle = header['northAngle'][0] #
	412	self.microsec = header['microsec'][0] #
	413	self.azisetvel = header['azisetvel'][0] #
	414	self.elvsetpos = header['elvsetpos'][0] #
	415	self.RadarConst = header['RadarConst'][0] #
	416	#84
	417
	418	#print 'Pointer fp RECheader', fp.tell()
	419
	420	#self.ipp= 0.5*(SPEED_OF_LIGHT/self.PRFhz)
	421
	422	#self.RHsize = 180+20*self.nChannels
	423	#self.Datasize= self.nProfilesself.nChannelsself.nHeights24
	424	#print 'Datasize',self.Datasize
	425	#endFp = self.OffsetStartHeader + self.RecCounter*self.Off2StartNxtRec
	426
	427	print '=============================================='
	428
	429	print '=============================================='
	430
	431
	432	return 1
	433
	434	class MIRA35CReader (ProcessingUnit,FileHeaderMIRA35c,SRVIHeader,RecordHeader):
	435
	436	path = None
	437	startDate = None
	438	endDate = None
	439	startTime = None
	440	endTime = None
	441	walk = None
	442	isConfig = False
	443
	444
	445	fileList= None
	446
	447	#metadata
	448	TimeZone= None
	449	Interval= None
	450	heightList= None
	451
	452	#data
	453	data= None
	454	utctime= None
	455
	456
	457
	458	def __init__(self, **kwargs):
	459
	460	#Eliminar de la base la herencia
	461	ProcessingUnit.__init__(self, **kwargs)
	462	self.PointerReader = 0
	463	self.FileHeaderFlag = False
	464	self.utc = None
	465	self.ext = ".zspca"
	466	self.optchar = "P"
	467	self.fpFile=None
	468	self.fp = None
	469	self.BlockCounter=0
	470	self.dtype = None
	471	self.fileSizeByHeader = None
	472	self.filenameList = []
	473	self.fileSelector = 0
	474	self.Off2StartNxtRec=0
	475	self.RecCounter=0
	476	self.flagNoMoreFiles = 0
	477	self.data_spc=None
	478	#self.data_cspc=None
	479	self.data_output=None
	480	self.path = None
	481	self.OffsetStartHeader=0
	482	self.Off2StartData=0
	483	self.ipp = 0
	484	self.nFDTdataRecors=0
	485	self.blocksize = 0
	486	self.dataOut = Spectra()
	487	self.profileIndex = 1 #Always
	488	self.dataOut.flagNoData=False
	489	self.dataOut.nRdPairs = 0
	490	self.dataOut.pairsList = []
	491	self.dataOut.data_spc=None
	492
	493	self.dataOut.normFactor=1
	494	self.nextfileflag = True
	495	self.dataOut.RadarConst = 0
	496	self.dataOut.HSDV = []
	497	self.dataOut.NPW = []
	498	self.dataOut.COFA = []
	499	self.dataOut.noise = 0
	500
	501
	502	def Files2Read(self, fp):
	503	'''
	504	Function that indicates the number of .fdt files that exist in the folder to be read.
	505	It also creates an organized list with the names of the files to read.
	506	'''
	507	#self.__checkPath()
	508
	509	ListaData=os.listdir(fp) #Gets the list of files within the fp address
	510	ListaData=sorted(ListaData) #Sort the list of files from least to largest by names
	511	nFiles=0 #File Counter
	512	FileList=[] #A list is created that will contain the .fdt files
	513	for IndexFile in ListaData :
	514	if '.zspca' in IndexFile and '.gz' not in IndexFile:
	515	FileList.append(IndexFile)
	516	nFiles+=1
	517
	518	#print 'Files2Read'
	519	#print 'Existen '+str(nFiles)+' archivos .fdt'
	520
	521	self.filenameList=FileList #List of files from least to largest by names
	522
	523
	524	def run(self, **kwargs):
	525	'''
	526	This method will be the one that will initiate the data entry, will be called constantly.
	527	You should first verify that your Setup () is set up and then continue to acquire
	528	the data to be processed with getData ().
	529	'''
	530	if not self.isConfig:
	531	self.setup(**kwargs)
	532	self.isConfig = True
	533
	534	self.getData()
	535
	536
	537	def setup(self, path=None,
	538	startDate=None,
	539	endDate=None,
	540	startTime=None,
	541	endTime=None,
	542	walk=True,
	543	timezone='utc',
	544	code = None,
	545	online=False,
	546	ReadMode=None, **kwargs):
	547
	548	self.isConfig = True
	549
	550	self.path=path
	551	self.startDate=startDate
	552	self.endDate=endDate
	553	self.startTime=startTime
	554	self.endTime=endTime
	555	self.walk=walk
	556	#self.ReadMode=int(ReadMode)
	557
	558	pass
	559
	560
	561	def getData(self):
	562	'''
	563	Before starting this function, you should check that there is still an unread file,
	564	If there are still blocks to read or if the data block is empty.
	565
	566	You should call the file "read".
	567
	568	'''
	569
	570	if self.flagNoMoreFiles:
	571	self.dataOut.flagNoData = True
	572	print 'NoData se vuelve true'
	573	return 0
	574
	575	self.fp=self.path
	576	self.Files2Read(self.fp)
	577	self.readFile(self.fp)
	578
	579	self.dataOut.data_spc = self.dataOut_spc#self.data_spc.copy()
	580	self.dataOut.RadarConst = self.RadarConst
	581	self.dataOut.data_output=self.data_output
	582	self.dataOut.noise = self.dataOut.getNoise()
	583	#print 'ACAAAAAA', self.dataOut.noise
	584	self.dataOut.data_spc = self.dataOut.data_spc+self.dataOut.noise
	585	#print 'self.dataOut.noise',self.dataOut.noise
	586
	587
	588	return self.dataOut.data_spc
	589
	590
	591	def readFile(self,fp):
	592	'''
	593	You must indicate if you are reading in Online or Offline mode and load the
	594	The parameters for this file reading mode.
	595
	596	Then you must do 2 actions:
	597
	598	1. Get the BLTR FileHeader.
	599	2. Start reading the first block.
	600	'''
	601
	602	#The address of the folder is generated the name of the .fdt file that will be read
	603	print "File: ",self.fileSelector+1
	604
	605	if self.fileSelector < len(self.filenameList):
	606
	607	self.fpFile=str(fp)+'/'+str(self.filenameList[self.fileSelector])
	608
	609	if self.nextfileflag==True:
	610	self.fp = open(self.fpFile,"rb")
	611	self.nextfileflag==False
	612
	613	'''HERE STARTING THE FILE READING'''
	614
	615
	616	self.fheader = FileHeaderMIRA35c()
	617	self.fheader.FHread(self.fp) #Bltr FileHeader Reading
	618
	619
	620	self.SPARrawGate1 = self.fheader.SPARrawGate1
	621	self.SPARrawGate2 = self.fheader.SPARrawGate2
	622	self.Num_Hei = self.SPARrawGate2 - self.SPARrawGate1
	623	self.Num_Bins = self.fheader.PPARsft
	624	self.dataOut.nFFTPoints = self.fheader.PPARsft
	625
	626
	627	self.Num_inCoh = self.fheader.PPARavc
	628	self.dataOut.PRF = self.fheader.PPARprf
	629	self.dataOut.frequency = 34.8510*9
	630	self.Lambda = SPEED_OF_LIGHT/self.dataOut.frequency
	631	self.dataOut.ippSeconds= 1./float(self.dataOut.PRF)
	632
	633	pulse_width = self.fheader.PPARpdr * 10**-9
	634	self.__deltaHeigth = 0.5 * SPEED_OF_LIGHT * pulse_width
	635
	636	self.data_spc = numpy.zeros((self.Num_Hei, self.Num_Bins,2))#
	637	self.dataOut.HSDV = numpy.zeros((self.Num_Hei, 2))
	638
	639	self.Ze = numpy.zeros(self.Num_Hei)
	640	self.ETA = numpy.zeros(([2,self.Num_Hei]))
	641
	642
	643
	644	self.readBlock() #Block reading
	645
	646	else:
	647	print 'readFile FlagNoData becomes true'
	648	self.flagNoMoreFiles=True
	649	self.dataOut.flagNoData = True
	650	self.FileHeaderFlag == True
	651	return 0
	652
	653
	654
	655	def readBlock(self):
	656	'''
	657	It should be checked if the block has data, if it is not passed to the next file.
	658
	659	Then the following is done:
	660
	661	1. Read the RecordHeader
	662	2. Fill the buffer with the current block number.
	663
	664	'''
	665
	666	if self.PointerReader > 1180:
	667	self.fp.seek(self.PointerReader , os.SEEK_SET)
	668	self.FirstPoint = self.PointerReader
	669
	670	else :
	671	self.FirstPoint = 1180
	672
	673
	674
	675	self.srviHeader = SRVIHeader()
	676
	677	self.srviHeader.SRVIread(self.fp) #Se obtiene la cabecera del SRVI
	678
	679	self.blocksize = self.srviHeader.SizeOfDataBlock1 # Se obtiene el tamao del bloque
	680
	681	if self.blocksize == 148:
	682	print 'blocksize == 148 bug'
	683	jump = numpy.fromfile(self.fp,[('jump',numpy.str_,140)] ,1)
	684
	685	self.srviHeader.SRVIread(self.fp) #Se obtiene la cabecera del SRVI
	686
	687	if not self.srviHeader.SizeOfSRVI1:
	688	self.fileSelector+=1
	689	self.nextfileflag==True
	690	self.FileHeaderFlag == True
	691
	692	self.recordheader = RecordHeader()
	693	self.recordheader.RHread(self.fp)
	694	self.RadarConst = self.recordheader.RadarConst
	695	dwell = self.recordheader.time_t
	696	npw1 = self.recordheader.npw1
	697	npw2 = self.recordheader.npw2
	698
	699
	700	self.dataOut.channelList = range(1)
	701	self.dataOut.nIncohInt = self.Num_inCoh
	702	self.dataOut.nProfiles = self.Num_Bins
	703	self.dataOut.nCohInt = 1
	704	self.dataOut.windowOfFilter = 1
	705	self.dataOut.utctime = dwell
	706	self.dataOut.timeZone=0
	707
	708	self.dataOut.outputInterval = self.dataOut.getTimeInterval()
	709	self.dataOut.heightList = self.SPARrawGate1self.__deltaHeigth + numpy.array(range(self.Num_Hei))self.__deltaHeigth
	710
	711
	712
	713	self.HSDVsign = numpy.fromfile( self.fp, [('HSDV',numpy.str_,4)],1)
	714	self.SizeHSDV = numpy.fromfile( self.fp, [('SizeHSDV','<i4')],1)
	715	self.HSDV_Co = numpy.fromfile( self.fp, [('HSDV_Co','<f4')],self.Num_Hei)
	716	self.HSDV_Cx = numpy.fromfile( self.fp, [('HSDV_Cx','<f4')],self.Num_Hei)
	717
	718	self.COFAsign = numpy.fromfile( self.fp, [('COFA',numpy.str_,4)],1)
	719	self.SizeCOFA = numpy.fromfile( self.fp, [('SizeCOFA','<i4')],1)
	720	self.COFA_Co = numpy.fromfile( self.fp, [('COFA_Co','<f4')],self.Num_Hei)
	721	self.COFA_Cx = numpy.fromfile( self.fp, [('COFA_Cx','<f4')],self.Num_Hei)
	722
	723	self.ZSPCsign = numpy.fromfile(self.fp, [('ZSPCsign',numpy.str_,4)],1)
	724	self.SizeZSPC = numpy.fromfile(self.fp, [('SizeZSPC','<i4')],1)
	725
	726	self.dataOut.HSDV[0]=self.HSDV_Co[:][0]
	727	self.dataOut.HSDV[1]=self.HSDV_Cx[:][0]
	728
	729	for irg in range(self.Num_Hei):
	730	nspc = numpy.fromfile(self.fp, [('nspc','int16')],1)[0][0] # Number of spectral sub pieces containing significant power
	731
	732	for k in range(nspc):
	733	binIndex = numpy.fromfile(self.fp, [('binIndex','int16')],1)[0][0] # Index of the spectral bin where the piece is beginning
	734	nbins = numpy.fromfile(self.fp, [('nbins','int16')],1)[0][0] # Number of bins of the piece
	735
	736	#Co_Channel
	737	jbin = numpy.fromfile(self.fp, [('jbin','uint16')],nbins)[0][0] # Spectrum piece to be normaliced
	738	jmax = numpy.fromfile(self.fp, [('jmax','float32')],1)[0][0] # Maximun piece to be normaliced
	739
	740
	741	self.data_spc[irg,binIndex:binIndex+nbins,0] = self.data_spc[irg,binIndex:binIndex+nbins,0]+jbin/65530.*jmax
	742
	743	#Cx_Channel
	744	jbin = numpy.fromfile(self.fp, [('jbin','uint16')],nbins)[0][0]
	745	jmax = numpy.fromfile(self.fp, [('jmax','float32')],1)[0][0]
	746
	747
	748	self.data_spc[irg,binIndex:binIndex+nbins,1] = self.data_spc[irg,binIndex:binIndex+nbins,1]+jbin/65530.*jmax
	749
	750	for bin in range(self.Num_Bins):
	751
	752	self.data_spc[:,bin,0] = self.data_spc[:,bin,0] - self.dataOut.HSDV[:,0]
	753
	754	self.data_spc[:,bin,1] = self.data_spc[:,bin,1] - self.dataOut.HSDV[:,1]
	755
	756
	757	numpy.set_printoptions(threshold='nan')
	758
	759	self.data_spc = numpy.where(self.data_spc > 0. , self.data_spc, 0)
	760
	761	self.dataOut.COFA = numpy.array([self.COFA_Co , self.COFA_Cx])
	762
	763	print ' '
	764	print 'SPC',numpy.shape(self.dataOut.data_spc)
	765	#print 'SPC',self.dataOut.data_spc
	766
	767	noinor1 = 713031680
	768	noinor2 = 30
	769
	770	npw1 = 1#0*(npw1/10) noinor1 * noinor2
	771	npw2 = 1#0*(npw2/10) noinor1 * noinor2
	772	self.dataOut.NPW = numpy.array([npw1, npw2])
	773
	774	print ' '
	775
	776	self.data_spc = numpy.transpose(self.data_spc, (2,1,0))
	777	self.data_spc = numpy.fft.fftshift(self.data_spc, axes = 1)
	778
	779	self.data_spc = numpy.fliplr(self.data_spc)
	780
	781	self.data_spc = numpy.where(self.data_spc > 0. , self.data_spc, 0)
	782	self.dataOut_spc= numpy.ones([1, self.Num_Bins , self.Num_Hei])
	783	self.dataOut_spc[0,:,:] = self.data_spc[0,:,:]
	784	#print 'SHAPE', self.dataOut_spc.shape
	785	#For nyquist correction:
	786	#fix = 20 # ~3m/s
	787	#shift = self.Num_Bins/2 + fix
	788	#self.data_spc = numpy.array([ self.data_spc[: , self.Num_Bins-shift+1: , :] , self.data_spc[: , 0:self.Num_Bins-shift , :]])
	789
	790
	791
	792	'''Block Reading, the Block Data is received and Reshape is used to give it
	793	shape.
	794	'''
	795
	796	self.PointerReader = self.fp.tell()
	797
	798
	799
	800
	801
	802
	803	No newline at end of file

schainpy/model/proc/bltrproc_parameters.py created 644 +403 0

			@@ -0,0 +1,403
		1	'''
		2	Created on Oct 24, 2016
		3
		4	@author: roj- LouVD
		5	'''
		6
		7	import numpy
		8	import copy
		9	import datetime
		10	import time
		11	from time import gmtime
		12
		13	from numpy import transpose
		14
		15	from jroproc_base import ProcessingUnit, Operation
		16	from schainpy.model.data.jrodata import Parameters
		17
		18
		19	class BLTRParametersProc(ProcessingUnit):
		20	'''
		21	Processing unit for BLTR parameters data (winds)
		22
		23	Inputs:
		24	self.dataOut.nmodes - Number of operation modes
		25	self.dataOut.nchannels - Number of channels
		26	self.dataOut.nranges - Number of ranges
		27
		28	self.dataOut.data_SNR - SNR array
		29	self.dataOut.data_output - Zonal, Vertical and Meridional velocity array
		30	self.dataOut.height - Height array (km)
		31	self.dataOut.time - Time array (seconds)
		32
		33	self.dataOut.fileIndex -Index of the file currently read
		34	self.dataOut.lat - Latitude coordinate of BLTR location
		35
		36	self.dataOut.doy - Experiment doy (number of the day in the current year)
		37	self.dataOut.month - Experiment month
		38	self.dataOut.day - Experiment day
		39	self.dataOut.year - Experiment year
		40	'''
		41
		42	def __init__(self, **kwargs):
		43	'''
		44	Inputs: None
		45	'''
		46	ProcessingUnit.__init__(self, **kwargs)
		47	self.dataOut = Parameters()
		48	self.isConfig = False
		49
		50	def setup(self, mode):
		51	'''
		52	'''
		53	self.dataOut.mode = mode
		54
		55	def run(self, mode, snr_threshold=None):
		56	'''
		57	Inputs:
		58	mode = High resolution (0) or Low resolution (1) data
		59	snr_threshold = snr filter value
		60	'''
		61
		62	if not self.isConfig:
		63	self.setup(mode)
		64	self.isConfig = True
		65
		66	if self.dataIn.type == 'Parameters':
		67	self.dataOut.copy(self.dataIn)
		68
		69	self.dataOut.data_output = self.dataOut.data_output[mode]
		70	self.dataOut.heightList = self.dataOut.height[0]
		71	self.dataOut.data_SNR = self.dataOut.data_SNR[mode]
		72
		73	if snr_threshold is not None:
		74	SNRavg = numpy.average(self.dataOut.data_SNR, axis=0)
		75	SNRavgdB = 10*numpy.log10(SNRavg)
		76	for i in range(3):
		77	self.dataOut.data_output[i][SNRavgdB <= snr_threshold] = numpy.nan
		78
		79	# TODO
		80	class OutliersFilter(Operation):
		81
		82	def __init__(self, **kwargs):
		83	'''
		84	'''
		85	Operation.__init__(self, **kwargs)
		86
		87	def run(self, svalue2, method, factor, filter, npoints=9):
		88	'''
		89	Inputs:
		90	svalue - string to select array velocity
		91	svalue2 - string to choose axis filtering
		92	method - 0 for SMOOTH or 1 for MEDIAN
		93	factor - number used to set threshold
		94	filter - 1 for data filtering using the standard deviation criteria else 0
		95	npoints - number of points for mask filter
		96	'''
		97
		98	print ' Outliers Filter {} {} / threshold = {}'.format(svalue, svalue, factor)
		99
		100
		101	yaxis = self.dataOut.heightList
		102	xaxis = numpy.array([[self.dataOut.utctime]])
		103
		104	# Zonal
		105	value_temp = self.dataOut.data_output[0]
		106
		107	# Zonal
		108	value_temp = self.dataOut.data_output[1]
		109
		110	# Vertical
		111	value_temp = numpy.transpose(self.dataOut.data_output[2])
		112
		113	htemp = yaxis
		114	std = value_temp
		115	for h in range(len(htemp)):
		116	nvalues_valid = len(numpy.where(numpy.isfinite(value_temp[h]))[0])
		117	minvalid = npoints
		118
		119	#only if valid values greater than the minimum required (10%)
		120	if nvalues_valid > minvalid:
		121
		122	if method == 0:
		123	#SMOOTH
		124	w = value_temp[h] - self.Smooth(input=value_temp[h], width=npoints, edge_truncate=1)
		125
		126
		127	if method == 1:
		128	#MEDIAN
		129	w = value_temp[h] - self.Median(input=value_temp[h], width = npoints)
		130
		131	dw = numpy.std(w[numpy.where(numpy.isfinite(w))],ddof = 1)
		132
		133	threshold = dw*factor
		134	value_temp[numpy.where(w > threshold),h] = numpy.nan
		135	value_temp[numpy.where(w < -1*threshold),h] = numpy.nan
		136
		137
		138	#At the end
		139	if svalue2 == 'inHeight':
		140	value_temp = numpy.transpose(value_temp)
		141	output_array[:,m] = value_temp
		142
		143	if svalue == 'zonal':
		144	self.dataOut.data_output[0] = output_array
		145
		146	elif svalue == 'meridional':
		147	self.dataOut.data_output[1] = output_array
		148
		149	elif svalue == 'vertical':
		150	self.dataOut.data_output[2] = output_array
		151
		152	return self.dataOut.data_output
		153
		154
		155	def Median(self,input,width):
		156	'''
		157	Inputs:
		158	input - Velocity array
		159	width - Number of points for mask filter
		160
		161	'''
		162
		163	if numpy.mod(width,2) == 1:
		164	pc = int((width - 1) / 2)
		165	cont = 0
		166	output = []
		167
		168	for i in range(len(input)):
		169	if i >= pc and i < len(input) - pc:
		170	new2 = input[i-pc:i+pc+1]
		171	temp = numpy.where(numpy.isfinite(new2))
		172	new = new2[temp]
		173	value = numpy.median(new)
		174	output.append(value)
		175
		176	output = numpy.array(output)
		177	output = numpy.hstack((input[0:pc],output))
		178	output = numpy.hstack((output,input[-pc:len(input)]))
		179
		180	return output
		181
		182	def Smooth(self,input,width,edge_truncate = None):
		183	'''
		184	Inputs:
		185	input - Velocity array
		186	width - Number of points for mask filter
		187	edge_truncate - 1 for truncate the convolution product else
		188
		189	'''
		190
		191	if numpy.mod(width,2) == 0:
		192	real_width = width + 1
		193	nzeros = width / 2
		194	else:
		195	real_width = width
		196	nzeros = (width - 1) / 2
		197
		198	half_width = int(real_width)/2
		199	length = len(input)
		200
		201	gate = numpy.ones(real_width,dtype='float')
		202	norm_of_gate = numpy.sum(gate)
		203
		204	nan_process = 0
		205	nan_id = numpy.where(numpy.isnan(input))
		206	if len(nan_id[0]) > 0:
		207	nan_process = 1
		208	pb = numpy.zeros(len(input))
		209	pb[nan_id] = 1.
		210	input[nan_id] = 0.
		211
		212	if edge_truncate == True:
		213	output = numpy.convolve(input/norm_of_gate,gate,mode='same')
		214	elif edge_truncate == False or edge_truncate == None:
		215	output = numpy.convolve(input/norm_of_gate,gate,mode='valid')
		216	output = numpy.hstack((input[0:half_width],output))
		217	output = numpy.hstack((output,input[len(input)-half_width:len(input)]))
		218
		219	if nan_process:
		220	pb = numpy.convolve(pb/norm_of_gate,gate,mode='valid')
		221	pb = numpy.hstack((numpy.zeros(half_width),pb))
		222	pb = numpy.hstack((pb,numpy.zeros(half_width)))
		223	output[numpy.where(pb > 0.9999)] = numpy.nan
		224	input[nan_id] = numpy.nan
		225	return output
		226
		227	def Average(self,aver=0,nhaver=1):
		228	'''
		229	Inputs:
		230	aver - Indicates the time period over which is averaged or consensus data
		231	nhaver - Indicates the decimation factor in heights
		232
		233	'''
		234	nhpoints = 48
		235
		236	lat_piura = -5.17
		237	lat_huancayo = -12.04
		238	lat_porcuya = -5.8
		239
		240	if '%2.2f'%self.dataOut.lat == '%2.2f'%lat_piura:
		241	hcm = 3.
		242	if self.dataOut.year == 2003 :
		243	if self.dataOut.doy >= 25 and self.dataOut.doy < 64:
		244	nhpoints = 12
		245
		246	elif '%2.2f'%self.dataOut.lat == '%2.2f'%lat_huancayo:
		247	hcm = 3.
		248	if self.dataOut.year == 2003 :
		249	if self.dataOut.doy >= 25 and self.dataOut.doy < 64:
		250	nhpoints = 12
		251
		252
		253	elif '%2.2f'%self.dataOut.lat == '%2.2f'%lat_porcuya:
		254	hcm = 5.#2
		255
		256	pdata = 0.2
		257	taver = [1,2,3,4,6,8,12,24]
		258	t0 = 0
		259	tf = 24
		260	ntime =(tf-t0)/taver[aver]
		261	ti = numpy.arange(ntime)
		262	tf = numpy.arange(ntime) + taver[aver]
		263
		264
		265	old_height = self.dataOut.heightList
		266
		267	if nhaver > 1:
		268	num_hei = len(self.dataOut.heightList)/nhaver/self.dataOut.nmodes
		269	deltha = 0.05*nhaver
		270	minhvalid = pdata*nhaver
		271	for im in range(self.dataOut.nmodes):
		272	new_height = numpy.arange(num_hei)*deltha + self.dataOut.height[im,0] + deltha/2.
		273
		274
		275	data_fHeigths_List = []
		276	data_fZonal_List = []
		277	data_fMeridional_List = []
		278	data_fVertical_List = []
		279	startDTList = []
		280
		281
		282	for i in range(ntime):
		283	height = old_height
		284
		285	start = datetime.datetime(self.dataOut.year,self.dataOut.month,self.dataOut.day) + datetime.timedelta(hours = int(ti[i])) - datetime.timedelta(hours = 5)
		286	stop = datetime.datetime(self.dataOut.year,self.dataOut.month,self.dataOut.day) + datetime.timedelta(hours = int(tf[i])) - datetime.timedelta(hours = 5)
		287
		288
		289	limit_sec1 = time.mktime(start.timetuple())
		290	limit_sec2 = time.mktime(stop.timetuple())
		291
		292	t1 = numpy.where(self.f_timesec >= limit_sec1)
		293	t2 = numpy.where(self.f_timesec < limit_sec2)
		294	time_select = []
		295	for val_sec in t1[0]:
		296	if val_sec in t2[0]:
		297	time_select.append(val_sec)
		298
		299
		300	time_select = numpy.array(time_select,dtype = 'int')
		301	minvalid = numpy.ceil(pdata*nhpoints)
		302
		303	zon_aver = numpy.zeros([self.dataOut.nranges,self.dataOut.nmodes],dtype='f4') + numpy.nan
		304	mer_aver = numpy.zeros([self.dataOut.nranges,self.dataOut.nmodes],dtype='f4') + numpy.nan
		305	ver_aver = numpy.zeros([self.dataOut.nranges,self.dataOut.nmodes],dtype='f4') + numpy.nan
		306
		307	if nhaver > 1:
		308	new_zon_aver = numpy.zeros([num_hei,self.dataOut.nmodes],dtype='f4') + numpy.nan
		309	new_mer_aver = numpy.zeros([num_hei,self.dataOut.nmodes],dtype='f4') + numpy.nan
		310	new_ver_aver = numpy.zeros([num_hei,self.dataOut.nmodes],dtype='f4') + numpy.nan
		311
		312	if len(time_select) > minvalid:
		313	time_average = self.f_timesec[time_select]
		314
		315	for im in range(self.dataOut.nmodes):
		316
		317	for ih in range(self.dataOut.nranges):
		318	if numpy.sum(numpy.isfinite(self.f_zon[time_select,ih,im])) >= minvalid:
		319	zon_aver[ih,im] = numpy.nansum(self.f_zon[time_select,ih,im]) / numpy.sum(numpy.isfinite(self.f_zon[time_select,ih,im]))
		320
		321	if numpy.sum(numpy.isfinite(self.f_mer[time_select,ih,im])) >= minvalid:
		322	mer_aver[ih,im] = numpy.nansum(self.f_mer[time_select,ih,im]) / numpy.sum(numpy.isfinite(self.f_mer[time_select,ih,im]))
		323
		324	if numpy.sum(numpy.isfinite(self.f_ver[time_select,ih,im])) >= minvalid:
		325	ver_aver[ih,im] = numpy.nansum(self.f_ver[time_select,ih,im]) / numpy.sum(numpy.isfinite(self.f_ver[time_select,ih,im]))
		326
		327	if nhaver > 1:
		328	for ih in range(num_hei):
		329	hvalid = numpy.arange(nhaver) + nhaver*ih
		330
		331	if numpy.sum(numpy.isfinite(zon_aver[hvalid,im])) >= minvalid:
		332	new_zon_aver[ih,im] = numpy.nansum(zon_aver[hvalid,im]) / numpy.sum(numpy.isfinite(zon_aver[hvalid,im]))
		333
		334	if numpy.sum(numpy.isfinite(mer_aver[hvalid,im])) >= minvalid:
		335	new_mer_aver[ih,im] = numpy.nansum(mer_aver[hvalid,im]) / numpy.sum(numpy.isfinite(mer_aver[hvalid,im]))
		336
		337	if numpy.sum(numpy.isfinite(ver_aver[hvalid,im])) >= minvalid:
		338	new_ver_aver[ih,im] = numpy.nansum(ver_aver[hvalid,im]) / numpy.sum(numpy.isfinite(ver_aver[hvalid,im]))
		339	if nhaver > 1:
		340	zon_aver = new_zon_aver
		341	mer_aver = new_mer_aver
		342	ver_aver = new_ver_aver
		343	height = new_height
		344
		345
		346	tstart = time_average[0]
		347	tend = time_average[-1]
		348	startTime = time.gmtime(tstart)
		349
		350	year = startTime.tm_year
		351	month = startTime.tm_mon
		352	day = startTime.tm_mday
		353	hour = startTime.tm_hour
		354	minute = startTime.tm_min
		355	second = startTime.tm_sec
		356
		357	startDTList.append(datetime.datetime(year,month,day,hour,minute,second))
		358
		359
		360	o_height = numpy.array([])
		361	o_zon_aver = numpy.array([])
		362	o_mer_aver = numpy.array([])
		363	o_ver_aver = numpy.array([])
		364	if self.dataOut.nmodes > 1:
		365	for im in range(self.dataOut.nmodes):
		366
		367	if im == 0:
		368	h_select = numpy.where(numpy.bitwise_and(height[0,:] >=0,height[0,:] <= hcm,numpy.isfinite(height[0,:])))
		369	else:
		370	h_select = numpy.where(numpy.bitwise_and(height[1,:] > hcm,height[1,:] < 20,numpy.isfinite(height[1,:])))
		371
		372
		373	ht = h_select[0]
		374
		375	o_height = numpy.hstack((o_height,height[im,ht]))
		376	o_zon_aver = numpy.hstack((o_zon_aver,zon_aver[ht,im]))
		377	o_mer_aver = numpy.hstack((o_mer_aver,mer_aver[ht,im]))
		378	o_ver_aver = numpy.hstack((o_ver_aver,ver_aver[ht,im]))
		379
		380	data_fHeigths_List.append(o_height)
		381	data_fZonal_List.append(o_zon_aver)
		382	data_fMeridional_List.append(o_mer_aver)
		383	data_fVertical_List.append(o_ver_aver)
		384
		385
		386	else:
		387	h_select = numpy.where(numpy.bitwise_and(height[0,:] <= hcm,numpy.isfinite(height[0,:])))
		388	ht = h_select[0]
		389	o_height = numpy.hstack((o_height,height[im,ht]))
		390	o_zon_aver = numpy.hstack((o_zon_aver,zon_aver[ht,im]))
		391	o_mer_aver = numpy.hstack((o_mer_aver,mer_aver[ht,im]))
		392	o_ver_aver = numpy.hstack((o_ver_aver,ver_aver[ht,im]))
		393
		394	data_fHeigths_List.append(o_height)
		395	data_fZonal_List.append(o_zon_aver)
		396	data_fMeridional_List.append(o_mer_aver)
		397	data_fVertical_List.append(o_ver_aver)
		398
		399
		400	return startDTList, data_fHeigths_List, data_fZonal_List, data_fMeridional_List, data_fVertical_List
		401
		402
		403	No newline at end of file

schainpy/__init__.py +1 -1

              '''
              Created on Feb 7, 2012
              @author $Author$
              @version $Id$
              '''
-             __version__ = "2.3"
+             __version__ = '2.3'

schainpy/model/data/jrodata.py +9 0

              '''
              $Author: murco $
              $Id: JROData.py 173 2012-11-20 15:06:21Z murco $
              '''
              import copy
              import numpy
              import datetime
              from jroheaderIO import SystemHeader, RadarControllerHeader
              from schainpy import cSchain
              def getNumpyDtype(dataTypeCode):
                  if dataTypeCode == 0:
                      numpyDtype = numpy.dtype([('real','<i1'),('imag','<i1')])
                  elif dataTypeCode == 1:
                      numpyDtype = numpy.dtype([('real','<i2'),('imag','<i2')])
                  elif dataTypeCode == 2:
                      numpyDtype = numpy.dtype([('real','<i4'),('imag','<i4')])
                  elif dataTypeCode == 3:
                      numpyDtype = numpy.dtype([('real','<i8'),('imag','<i8')])
                  elif dataTypeCode == 4:
                      numpyDtype = numpy.dtype([('real','<f4'),('imag','<f4')])
                  elif dataTypeCode == 5:
                      numpyDtype = numpy.dtype([('real','<f8'),('imag','<f8')])
                  else:
                      raise ValueError, 'dataTypeCode was not defined'
                  return numpyDtype
              def getDataTypeCode(numpyDtype):
                  if numpyDtype == numpy.dtype([('real','<i1'),('imag','<i1')]):
                      datatype = 0
                  elif numpyDtype == numpy.dtype([('real','<i2'),('imag','<i2')]):
                      datatype = 1
                  elif numpyDtype == numpy.dtype([('real','<i4'),('imag','<i4')]):
                      datatype = 2
                  elif numpyDtype == numpy.dtype([('real','<i8'),('imag','<i8')]):
                      datatype = 3
                  elif numpyDtype == numpy.dtype([('real','<f4'),('imag','<f4')]):
                      datatype = 4
                  elif numpyDtype == numpy.dtype([('real','<f8'),('imag','<f8')]):
                      datatype = 5
                  else:
                      datatype = None
                  return datatype
              def hildebrand_sekhon(data, navg):
                  """
                  This method is for the objective determination of the noise level in Doppler spectra. This
                  implementation technique is based on the fact that the standard deviation of the spectral
                  densities is equal to the mean spectral density for white Gaussian noise
                  Inputs:
                      Data    :    heights
                      navg    :    numbers of averages
                  Return:
                      -1        :    any error
                      anoise    :    noise's level
                  """
                  sortdata = numpy.sort(data, axis=None)
              #     lenOfData = len(sortdata)
              #     nums_min = lenOfData*0.2
              #
              #     if nums_min <= 5:
              #         nums_min = 5
              #
              #     sump = 0.
              #
              #     sumq = 0.
              #
              #     j = 0
              #
              #     cont = 1
              #
              #     while((cont==1)and(j<lenOfData)):
              #
              #         sump += sortdata[j]
              #
              #         sumq += sortdata[j]**2
              #
              #         if j > nums_min:
              #             rtest = float(j)/(j-1) + 1.0/navg
              #             if ((sumq*j) > (rtest*sump**2)):
              #                 j = j - 1
              #                 sump  = sump - sortdata[j]
              #                 sumq =  sumq - sortdata[j]**2
              #                 cont = 0
              #
              #         j += 1
              #
              #     lnoise = sump /j
              #
              #     return lnoise
                  return cSchain.hildebrand_sekhon(sortdata, navg)
              class Beam:
                  def __init__(self):
                      self.codeList = []
                      self.azimuthList = []
                      self.zenithList = []
              class GenericData(object):
                  flagNoData = True
                  def copy(self, inputObj=None):
                      if inputObj == None:
                          return copy.deepcopy(self)
                      for key in inputObj.__dict__.keys():
                          attribute = inputObj.__dict__[key]
                          #If this attribute is a tuple or list
                          if type(inputObj.__dict__[key]) in (tuple, list):
                              self.__dict__[key] = attribute[:]
                              continue
                          #If this attribute is another object or instance
                          if hasattr(attribute, '__dict__'):
                              self.__dict__[key] = attribute.copy()
                              continue
                          self.__dict__[key] = inputObj.__dict__[key]
                  def deepcopy(self):
                      return copy.deepcopy(self)
                  def isEmpty(self):
                      return self.flagNoData
              class JROData(GenericData):
              #    m_BasicHeader = BasicHeader()
              #    m_ProcessingHeader = ProcessingHeader()
                  systemHeaderObj = SystemHeader()
                  radarControllerHeaderObj = RadarControllerHeader()
              #    data = None
                  type = None
                  datatype = None #dtype but in string
              #     dtype = None
              #    nChannels = None
              #    nHeights = None
                  nProfiles = None
                  heightList = None
                  channelList = None
                  flagDiscontinuousBlock = False
                  useLocalTime = False
                  utctime = None
                  timeZone = None
                  dstFlag = None
                  errorCount = None
                  blocksize = None
              #     nCode = None
              #
              #     nBaud = None
              #
              #     code = None
                  flagDecodeData = False #asumo q la data no esta decodificada
                  flagDeflipData = False #asumo q la data no esta sin flip
                  flagShiftFFT = False
              #     ippSeconds = None
              #     timeInterval = None
                  nCohInt = None
              #     noise = None
                  windowOfFilter = 1
                  #Speed of ligth
                  C = 3e8
                  frequency = 49.92e6
                  realtime = False
                  beacon_heiIndexList = None
                  last_block = None
                  blocknow = None
                  azimuth = None
                  zenith = None
                  beam = Beam()
                  profileIndex = None
                  def getNoise(self):
                      raise NotImplementedError
                  def getNChannels(self):
                      return len(self.channelList)
                  def getChannelIndexList(self):
                      return range(self.nChannels)
                  def getNHeights(self):
                      return len(self.heightList)
                  def getHeiRange(self, extrapoints=0):
                      heis = self.heightList
              #        deltah = self.heightList[1] - self.heightList[0]
              #
              #        heis.append(self.heightList[-1])
                      return heis
                  def getDeltaH(self):
                      delta = self.heightList[1] - self.heightList[0]
                      return delta
                  def getltctime(self):
                      if self.useLocalTime:
                          return self.utctime - self.timeZone*60
                      return self.utctime
                  def getDatatime(self):
                      datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
                      return datatimeValue
                  def getTimeRange(self):
                      datatime = []
                      datatime.append(self.ltctime)
                      datatime.append(self.ltctime + self.timeInterval+1)
                      datatime = numpy.array(datatime)
                      return datatime
                  def getFmaxTimeResponse(self):
                      period = (10**-6)*self.getDeltaH()/(0.15)
                      PRF = 1./(period * self.nCohInt)
                      fmax = PRF
                      return fmax
                  def getFmax(self):
                      PRF = 1./(self.ippSeconds * self.nCohInt)
                      fmax = PRF
                      return fmax
                  def getVmax(self):
                      _lambda = self.C/self.frequency
                      vmax = self.getFmax() * _lambda/2
                      return vmax
                  def get_ippSeconds(self):
                      '''
                      '''
                      return self.radarControllerHeaderObj.ippSeconds
                  def set_ippSeconds(self, ippSeconds):
                      '''
                      '''
                      self.radarControllerHeaderObj.ippSeconds = ippSeconds
                      return
                  def get_dtype(self):
                      '''
                      '''
                      return getNumpyDtype(self.datatype)
                  def set_dtype(self, numpyDtype):
                      '''
                      '''
                      self.datatype = getDataTypeCode(numpyDtype)
                  def get_code(self):
                      '''
                      '''
                      return self.radarControllerHeaderObj.code
                  def set_code(self, code):
                      '''
                      '''
                      self.radarControllerHeaderObj.code = code
                      return
                  def get_ncode(self):
                      '''
                      '''
                      return self.radarControllerHeaderObj.nCode
                  def set_ncode(self, nCode):
                      '''
                      '''
                      self.radarControllerHeaderObj.nCode = nCode
                      return
                  def get_nbaud(self):
                      '''
                      '''
                      return self.radarControllerHeaderObj.nBaud
                  def set_nbaud(self, nBaud):
                      '''
                      '''
                      self.radarControllerHeaderObj.nBaud = nBaud
                      return
                  nChannels = property(getNChannels, "I'm the 'nChannel' property.")
                  channelIndexList = property(getChannelIndexList, "I'm the 'channelIndexList' property.")
                  nHeights = property(getNHeights, "I'm the 'nHeights' property.")
                  #noise = property(getNoise, "I'm the 'nHeights' property.")
                  datatime = property(getDatatime, "I'm the 'datatime' property")
                  ltctime = property(getltctime, "I'm the 'ltctime' property")
                  ippSeconds = property(get_ippSeconds, set_ippSeconds)
                  dtype = property(get_dtype, set_dtype)
              #     timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
                  code = property(get_code, set_code)
                  nCode = property(get_ncode, set_ncode)
                  nBaud = property(get_nbaud, set_nbaud)
              class Voltage(JROData):
                  #data es un numpy array de 2 dmensiones (canales, alturas)
                  data = None
                  def __init__(self):
                      '''
                      Constructor
                      '''
                      self.useLocalTime = True
                      self.radarControllerHeaderObj = RadarControllerHeader()
                      self.systemHeaderObj = SystemHeader()
                      self.type = "Voltage"
                      self.data = None
              #         self.dtype = None
              #        self.nChannels = 0
              #        self.nHeights = 0
                      self.nProfiles = None
                      self.heightList = None
                      self.channelList = None
              #        self.channelIndexList = None
                      self.flagNoData = True
                      self.flagDiscontinuousBlock = False
                      self.utctime = None
                      self.timeZone = None
                      self.dstFlag = None
                      self.errorCount = None
                      self.nCohInt = None
                      self.blocksize = None
                      self.flagDecodeData = False #asumo q la data no esta decodificada
                      self.flagDeflipData = False #asumo q la data no esta sin flip
                      self.flagShiftFFT = False
                      self.flagDataAsBlock = False    #Asumo que la data es leida perfil a perfil
                      self.profileIndex = 0
                  def getNoisebyHildebrand(self, channel = None):
                      """
                      Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
                      Return:
                          noiselevel
                      """
                      if channel != None:
                          data = self.data[channel]
                          nChannels = 1
                      else:
                          data = self.data
                          nChannels = self.nChannels
                      noise = numpy.zeros(nChannels)
                      power = data * numpy.conjugate(data)
                      for thisChannel in range(nChannels):
                          if nChannels == 1:
                              daux = power[:].real
                          else:
                              daux = power[thisChannel,:].real
                          noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
                      return noise
                  def getNoise(self, type = 1, channel = None):
                      if type == 1:
                          noise = self.getNoisebyHildebrand(channel)
                      return noise
                  def getPower(self, channel = None):
                      if channel != None:
                          data = self.data[channel]
                      else:
                          data = self.data
                      power = data * numpy.conjugate(data)
                      powerdB = 10*numpy.log10(power.real)
                      powerdB = numpy.squeeze(powerdB)
                      return powerdB
                  def getTimeInterval(self):
                      timeInterval = self.ippSeconds * self.nCohInt
                      return timeInterval
                  noise = property(getNoise, "I'm the 'nHeights' property.")
                  timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
              class Spectra(JROData):
                  #data spc es un numpy array de 2 dmensiones (canales, perfiles, alturas)
                  data_spc = None
                  #data cspc es un numpy array de 2 dmensiones (canales, pares, alturas)
                  data_cspc = None
                  #data dc es un numpy array de 2 dmensiones (canales, alturas)
                  data_dc = None
                  #data power
                  data_pwr = None
                  nFFTPoints = None
              #     nPairs = None
                  pairsList = None
                  nIncohInt = None
                  wavelength = None #Necesario para cacular el rango de velocidad desde la frecuencia
                  nCohInt = None #se requiere para determinar el valor de timeInterval
                  ippFactor = None
                  profileIndex = 0
                  plotting = "spectra"
                  def __init__(self):
                      '''
                      Constructor
                      '''
                      self.useLocalTime = True
                      self.radarControllerHeaderObj = RadarControllerHeader()
                      self.systemHeaderObj = SystemHeader()
                      self.type = "Spectra"
              #        self.data = None
              #         self.dtype = None
              #        self.nChannels = 0
              #        self.nHeights = 0
                      self.nProfiles = None
                      self.heightList = None
                      self.channelList = None
              #        self.channelIndexList = None
                      self.pairsList = None
                      self.flagNoData = True
                      self.flagDiscontinuousBlock = False
                      self.utctime = None
                      self.nCohInt = None
                      self.nIncohInt = None
                      self.blocksize = None
                      self.nFFTPoints = None
                      self.wavelength = None
                      self.flagDecodeData = False #asumo q la data no esta decodificada
                      self.flagDeflipData = False #asumo q la data no esta sin flip
                      self.flagShiftFFT = False
                      self.ippFactor = 1
                      #self.noise = None
                      self.beacon_heiIndexList = []
                      self.noise_estimation = None
                  def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
                      """
                      Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
                      Return:
                          noiselevel
                      """
                      noise = numpy.zeros(self.nChannels)
                      for channel in range(self.nChannels):
                          daux = self.data_spc[channel,xmin_index:xmax_index,ymin_index:ymax_index]
                          noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
                      return noise
                  def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
                      if self.noise_estimation is not None:
                          return self.noise_estimation #this was estimated by getNoise Operation defined in jroproc_spectra.py
                      else:
                          noise = self.getNoisebyHildebrand(xmin_index, xmax_index, ymin_index, ymax_index)
                          return noise
                  def getFreqRangeTimeResponse(self, extrapoints=0):
                      deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints*self.ippFactor)
                      freqrange = deltafreq*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) - deltafreq/2
                      return freqrange
                  def getAcfRange(self, extrapoints=0):
                      deltafreq = 10./(self.getFmax() / (self.nFFTPoints*self.ippFactor))
                      freqrange = deltafreq*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) - deltafreq/2
                      return freqrange
                  def getFreqRange(self, extrapoints=0):
                      deltafreq = self.getFmax() / (self.nFFTPoints*self.ippFactor)
                      freqrange = deltafreq*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) - deltafreq/2
                      return freqrange
                  def getVelRange(self, extrapoints=0):
                      deltav = self.getVmax() / (self.nFFTPoints*self.ippFactor)
                      velrange = deltav*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) #- deltav/2
                      return velrange
                  def getNPairs(self):
                      return len(self.pairsList)
                  def getPairsIndexList(self):
                      return range(self.nPairs)
                  def getNormFactor(self):
                      pwcode = 1
                      if self.flagDecodeData:
                          pwcode = numpy.sum(self.code[0]**2)
                      #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
                      normFactor = self.nProfiles*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
                      return normFactor
                  def getFlagCspc(self):
                      if self.data_cspc is None:
                          return True
                      return False
                  def getFlagDc(self):
                      if self.data_dc is None:
                          return True
                      return False
                  def getTimeInterval(self):
                      timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles
                      return timeInterval
                  def getPower(self):
                      factor = self.normFactor
                      z = self.data_spc/factor
                      z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
                      avg = numpy.average(z, axis=1)
                      return 10*numpy.log10(avg)
                  def getCoherence(self, pairsList=None, phase=False):
                      z = []
                      if pairsList is None:
                          pairsIndexList = self.pairsIndexList
                      else:
                          pairsIndexList = []
                          for pair in pairsList:
                              if pair not in self.pairsList:
                                  raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair)
                              pairsIndexList.append(self.pairsList.index(pair))
                      for i in range(len(pairsIndexList)):
                          pair = self.pairsList[pairsIndexList[i]]
                          ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
                          powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
                          powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
                          avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
                          if phase:
                              data = numpy.arctan2(avgcoherenceComplex.imag,
                                                   avgcoherenceComplex.real)*180/numpy.pi
                          else:
                              data = numpy.abs(avgcoherenceComplex)
                          z.append(data)
                      return numpy.array(z)
                  def setValue(self, value):
                      print "This property should not be initialized"
                      return
                  nPairs = property(getNPairs, setValue, "I'm the 'nPairs' property.")
                  pairsIndexList = property(getPairsIndexList, setValue, "I'm the 'pairsIndexList' property.")
                  normFactor = property(getNormFactor, setValue, "I'm the 'getNormFactor' property.")
                  flag_cspc = property(getFlagCspc, setValue)
                  flag_dc = property(getFlagDc, setValue)
                  noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
                  timeInterval = property(getTimeInterval, setValue, "I'm the 'timeInterval' property")
              class SpectraHeis(Spectra):
                  data_spc = None
                  data_cspc = None
                  data_dc = None
                  nFFTPoints = None
              #     nPairs = None
                  pairsList = None
                  nCohInt = None
                  nIncohInt = None
                  def __init__(self):
                      self.radarControllerHeaderObj = RadarControllerHeader()
                      self.systemHeaderObj = SystemHeader()
                      self.type = "SpectraHeis"
              #         self.dtype = None
              #        self.nChannels = 0
              #        self.nHeights = 0
                      self.nProfiles = None
                      self.heightList = None
                      self.channelList = None
              #        self.channelIndexList = None
                      self.flagNoData = True
                      self.flagDiscontinuousBlock = False
              #         self.nPairs = 0
                      self.utctime = None
                      self.blocksize = None
                      self.profileIndex = 0
                      self.nCohInt = 1
                      self.nIncohInt = 1
                  def getNormFactor(self):
                      pwcode = 1
                      if self.flagDecodeData:
                          pwcode = numpy.sum(self.code[0]**2)
                      normFactor = self.nIncohInt*self.nCohInt*pwcode
                      return normFactor
                  def getTimeInterval(self):
                      timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                      return timeInterval
                  normFactor = property(getNormFactor, "I'm the 'getNormFactor' property.")
                  timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
              class Fits(JROData):
                  heightList = None
                  channelList = None
                  flagNoData = True
                  flagDiscontinuousBlock = False
                  useLocalTime = False
                  utctime = None
                  timeZone = None
              #     ippSeconds = None
              #     timeInterval = None
                  nCohInt = None
                  nIncohInt = None
                  noise = None
                  windowOfFilter = 1
                  #Speed of ligth
                  C = 3e8
                  frequency = 49.92e6
                  realtime = False
                  def __init__(self):
                      self.type = "Fits"
                      self.nProfiles = None
                      self.heightList = None
                      self.channelList = None
              #         self.channelIndexList = None
                      self.flagNoData = True
                      self.utctime = None
                      self.nCohInt = 1
                      self.nIncohInt = 1
                      self.useLocalTime = True
                      self.profileIndex = 0
              #         self.utctime = None
              #         self.timeZone = None
              #         self.ltctime = None
              #         self.timeInterval = None
              #         self.header = None
              #         self.data_header = None
              #         self.data = None
              #         self.datatime = None
              #         self.flagNoData = False
              #         self.expName = ''
              #         self.nChannels = None
              #         self.nSamples = None
              #         self.dataBlocksPerFile = None
              #         self.comments = ''
              #
                  def getltctime(self):
                      if self.useLocalTime:
                          return self.utctime - self.timeZone*60
                      return self.utctime
                  def getDatatime(self):
                      datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
                      return datatime
                  def getTimeRange(self):
                      datatime = []
                      datatime.append(self.ltctime)
                      datatime.append(self.ltctime + self.timeInterval)
                      datatime = numpy.array(datatime)
                      return datatime
                  def getHeiRange(self):
                      heis = self.heightList
                      return heis
                  def getNHeights(self):
                      return len(self.heightList)
                  def getNChannels(self):
                      return len(self.channelList)
                  def getChannelIndexList(self):
                      return range(self.nChannels)
                  def getNoise(self, type = 1):
                      #noise = numpy.zeros(self.nChannels)
                      if type == 1:
                          noise = self.getNoisebyHildebrand()
                      if type == 2:
                          noise = self.getNoisebySort()
                      if type == 3:
                          noise = self.getNoisebyWindow()
                      return noise
                  def getTimeInterval(self):
                      timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                      return timeInterval
                  datatime = property(getDatatime, "I'm the 'datatime' property")
                  nHeights = property(getNHeights, "I'm the 'nHeights' property.")
                  nChannels = property(getNChannels, "I'm the 'nChannel' property.")
                  channelIndexList = property(getChannelIndexList, "I'm the 'channelIndexList' property.")
                  noise = property(getNoise, "I'm the 'nHeights' property.")
                  ltctime = property(getltctime, "I'm the 'ltctime' property")
                  timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
              class Correlation(JROData):
                  noise = None
                  SNR = None
                  #--------------------------------------------------
                  mode = None
                  split = False
                  data_cf = None
                  lags = None
                  lagRange = None
                  pairsList = None
                  normFactor = None
                  #--------------------------------------------------
              #     calculateVelocity = None
                  nLags = None
                  nPairs = None
                  nAvg = None
                  def __init__(self):
                      '''
                      Constructor
                      '''
                      self.radarControllerHeaderObj = RadarControllerHeader()
                      self.systemHeaderObj = SystemHeader()
                      self.type = "Correlation"
                      self.data = None
                      self.dtype = None
                      self.nProfiles = None
                      self.heightList = None
                      self.channelList = None
                      self.flagNoData = True
                      self.flagDiscontinuousBlock = False
                      self.utctime = None
                      self.timeZone = None
                      self.dstFlag = None
                      self.errorCount = None
                      self.blocksize = None
                      self.flagDecodeData = False #asumo q la data no esta decodificada
                      self.flagDeflipData = False #asumo q la data no esta sin flip
                      self.pairsList = None
                      self.nPoints = None
                  def getPairsList(self):
                      return self.pairsList
                  def getNoise(self, mode = 2):
                      indR = numpy.where(self.lagR == 0)[0][0]
                      indT = numpy.where(self.lagT == 0)[0][0]
                      jspectra0 = self.data_corr[:,:,indR,:]
                      jspectra = copy.copy(jspectra0)
                      num_chan = jspectra.shape[0]
                      num_hei = jspectra.shape[2]
                      freq_dc = jspectra.shape[1]/2
                      ind_vel = numpy.array([-2,-1,1,2]) + freq_dc
                      if ind_vel[0]<0:
                          ind_vel[range(0,1)] = ind_vel[range(0,1)] + self.num_prof
                      if mode == 1:
                          jspectra[:,freq_dc,:] = (jspectra[:,ind_vel[1],:] + jspectra[:,ind_vel[2],:])/2 #CORRECCION
                      if mode == 2:
                          vel = numpy.array([-2,-1,1,2])
                          xx = numpy.zeros([4,4])
                          for fil in range(4):
                              xx[fil,:] = vel[fil]**numpy.asarray(range(4))
                          xx_inv = numpy.linalg.inv(xx)
                          xx_aux = xx_inv[0,:]
                          for ich in range(num_chan):
                              yy = jspectra[ich,ind_vel,:]
                              jspectra[ich,freq_dc,:] = numpy.dot(xx_aux,yy)
                              junkid = jspectra[ich,freq_dc,:]<=0
                              cjunkid = sum(junkid)
                              if cjunkid.any():
                                  jspectra[ich,freq_dc,junkid.nonzero()] = (jspectra[ich,ind_vel[1],junkid] + jspectra[ich,ind_vel[2],junkid])/2
                      noise = jspectra0[:,freq_dc,:] - jspectra[:,freq_dc,:]
                      return noise
                  def getTimeInterval(self):
                      timeInterval = self.ippSeconds * self.nCohInt * self.nProfiles
                      return timeInterval
                  def splitFunctions(self):
                      pairsList = self.pairsList
                      ccf_pairs = []
                      acf_pairs = []
                      ccf_ind = []
                      acf_ind = []
                      for l in range(len(pairsList)):
                          chan0 = pairsList[l][0]
                          chan1 = pairsList[l][1]
                          #Obteniendo pares de Autocorrelacion
                          if chan0 == chan1:
                              acf_pairs.append(chan0)
                              acf_ind.append(l)
                          else:
                              ccf_pairs.append(pairsList[l])
                              ccf_ind.append(l)
                      data_acf = self.data_cf[acf_ind]
                      data_ccf = self.data_cf[ccf_ind]
                      return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
                  def getNormFactor(self):
                      acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
                      acf_pairs = numpy.array(acf_pairs)
                      normFactor = numpy.zeros((self.nPairs,self.nHeights))
                      for p in range(self.nPairs):
                          pair = self.pairsList[p]
                          ch0 = pair[0]
                          ch1 = pair[1]
                          ch0_max = numpy.max(data_acf[acf_pairs==ch0,:,:], axis=1)
                          ch1_max = numpy.max(data_acf[acf_pairs==ch1,:,:], axis=1)
                          normFactor[p,:] = numpy.sqrt(ch0_max*ch1_max)
                      return normFactor
                  timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
                  normFactor = property(getNormFactor, "I'm the 'normFactor property'")
              class Parameters(Spectra):
                  experimentInfo = None   #Information about the experiment
                  #Information from previous data
                  inputUnit = None        #Type of data to be processed
                  operation = None        #Type of operation to parametrize
                  #normFactor = None       #Normalization Factor
                  groupList = None        #List of Pairs, Groups, etc
                  #Parameters
                  data_param = None       #Parameters obtained
                  data_pre = None         #Data Pre Parametrization
                  data_SNR = None         #Signal to Noise Ratio
              #     heightRange = None      #Heights
                  abscissaList = None    #Abscissa, can be velocities, lags or time
              #    noise = None            #Noise Potency
                  utctimeInit = None      #Initial UTC time
                  paramInterval = None    #Time interval to calculate Parameters in seconds
                  useLocalTime = True
                  #Fitting
                  data_error = None       #Error of the estimation
                  constants = None
                  library = None
                  #Output signal
                  outputInterval = None     #Time interval to calculate output signal in seconds
                  data_output = None       #Out signal
                  nAvg = None
                  noise_estimation = None
+                 GauSPC = None #Fit gaussian SPC
                  def __init__(self):
                      '''
                      Constructor
                      '''
                      self.radarControllerHeaderObj = RadarControllerHeader()
                      self.systemHeaderObj = SystemHeader()
                      self.type = "Parameters"
                  def getTimeRange1(self, interval):
                      datatime = []
                      if self.useLocalTime:
                          time1 = self.utctimeInit - self.timeZone*60
                      else:
                          time1 = self.utctimeInit
                      datatime.append(time1)
                      datatime.append(time1 + interval)
                      datatime = numpy.array(datatime)
                      return datatime
                  def getTimeInterval(self):
                      if hasattr(self, 'timeInterval1'):
                          return self.timeInterval1
                      else:
                          return self.paramInterval
+                 def setValue(self, value):
+                     print "This property should not be initialized"
+                     return
                  def getNoise(self):
                      return self.spc_noise
                  timeInterval = property(getTimeInterval)
+                 noise = property(getNoise, setValue, "I'm the 'Noise' property.")

schainpy/model/graphics/jroplot_data.py +57 -20

              import os
              import time
              import glob
              import datetime
              from multiprocessing import Process
              import zmq
              import numpy
              import matplotlib
              import matplotlib.pyplot as plt
              from mpl_toolkits.axes_grid1 import make_axes_locatable
              from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
              from schainpy.model.proc.jroproc_base import Operation
              from schainpy.utils import log
-             func = lambda x, pos: ('%s') %(datetime.datetime.fromtimestamp(x).strftime('%H:%M'))
+             jet_values = matplotlib.pyplot.get_cmap("jet", 100)(numpy.arange(100))[10:90]
+             blu_values = matplotlib.pyplot.get_cmap("seismic_r", 20)(numpy.arange(20))[10:15]
+             ncmap = matplotlib.colors.LinearSegmentedColormap.from_list("jro", numpy.vstack((blu_values, jet_values)))
+             matplotlib.pyplot.register_cmap(cmap=ncmap)
-             d1970 = datetime.datetime(1970, 1, 1)
+             func = lambda x, pos: '{}'.format(datetime.datetime.fromtimestamp(x).strftime('%H:%M'))
+             UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
+             CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'RdBu_r', 'seismic')]
              class PlotData(Operation, Process):
                  '''
                  Base class for Schain plotting operations
                  '''
                  CODE = 'Figure'
                  colormap = 'jro'
                  bgcolor = 'white'
                  CONFLATE = False
                  __MAXNUMX = 80
                  __missing = 1E30
                  def __init__(self, **kwargs):
                      Operation.__init__(self, plot=True, **kwargs)
                      Process.__init__(self)
                      self.kwargs['code'] = self.CODE
                      self.mp = False
                      self.data = None
                      self.isConfig = False
                      self.figures = []
                      self.axes = []
                      self.cb_axes = []
                      self.localtime = kwargs.pop('localtime', True)
                      self.show = kwargs.get('show', True)
                      self.save = kwargs.get('save', False)
                      self.colormap = kwargs.get('colormap', self.colormap)
                      self.colormap_coh = kwargs.get('colormap_coh', 'jet')
                      self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
                      self.colormaps = kwargs.get('colormaps', None)
                      self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
                      self.showprofile = kwargs.get('showprofile', False)
                      self.title = kwargs.get('wintitle', self.CODE.upper())
                      self.cb_label = kwargs.get('cb_label', None)
                      self.cb_labels = kwargs.get('cb_labels', None)
                      self.xaxis = kwargs.get('xaxis', 'frequency')
                      self.zmin = kwargs.get('zmin', None)
                      self.zmax = kwargs.get('zmax', None)
                      self.zlimits = kwargs.get('zlimits', None)
                      self.xmin = kwargs.get('xmin', None)
-                     if self.xmin is not None:
-                         self.xmin += 5
                      self.xmax = kwargs.get('xmax', None)
                      self.xrange = kwargs.get('xrange', 24)
                      self.ymin = kwargs.get('ymin', None)
                      self.ymax = kwargs.get('ymax', None)
                      self.xlabel = kwargs.get('xlabel', None)
                      self.__MAXNUMY = kwargs.get('decimation', 100)
                      self.showSNR = kwargs.get('showSNR', False)
                      self.oneFigure = kwargs.get('oneFigure', True)
                      self.width = kwargs.get('width', None)
                      self.height = kwargs.get('height', None)
                      self.colorbar = kwargs.get('colorbar', True)
                      self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
                      self.titles = ['' for __ in range(16)]
                  def __setup(self):
                      '''
                      Common setup for all figures, here figures and axes are created
                      '''
                      self.setup()
+                     self.time_label = 'LT' if self.localtime else 'UTC'
                      if self.width is None:
                          self.width = 8
                      self.figures = []
                      self.axes = []
                      self.cb_axes = []
                      self.pf_axes = []
                      self.cmaps = []
                      size = '15%' if self.ncols==1 else '30%'
                      pad = '4%' if self.ncols==1 else '8%'
                      if self.oneFigure:
                          if self.height is None:
                              self.height = 1.4*self.nrows + 1
                          fig = plt.figure(figsize=(self.width, self.height),
                                           edgecolor='k',
                                           facecolor='w')
                          self.figures.append(fig)
                          for n in range(self.nplots):
                              ax = fig.add_subplot(self.nrows, self.ncols, n+1)
                              ax.tick_params(labelsize=8)
                              ax.firsttime = True
+                             ax.index = 0
                              self.axes.append(ax)
                              if self.showprofile:
                                  cax = self.__add_axes(ax, size=size, pad=pad)
                                  cax.tick_params(labelsize=8)
                                  self.pf_axes.append(cax)
                      else:
                          if self.height is None:
                              self.height = 3
                          for n in range(self.nplots):
                              fig = plt.figure(figsize=(self.width, self.height),
                                                  edgecolor='k',
                                                  facecolor='w')
                              ax = fig.add_subplot(1, 1, 1)
                              ax.tick_params(labelsize=8)
                              ax.firsttime = True
+                             ax.index = 0
                              self.figures.append(fig)
                              self.axes.append(ax)
                              if self.showprofile:
                                  cax = self.__add_axes(ax, size=size, pad=pad)
                                  cax.tick_params(labelsize=8)
                                  self.pf_axes.append(cax)
                      for n in range(self.nrows):
                          if self.colormaps is not None:
                              cmap = plt.get_cmap(self.colormaps[n])
                          else:
                              cmap = plt.get_cmap(self.colormap)
                          cmap.set_bad(self.bgcolor, 1.)
                          self.cmaps.append(cmap)
+                     for fig in self.figures:
+                         fig.canvas.mpl_connect('key_press_event', self.event_key_press)
+                 def event_key_press(self, event):
+                     '''
+                     '''
+                     for ax in self.axes:
+                         if ax == event.inaxes:
+                             if event.key == 'down':
+                                 ax.index += 1
+                             elif event.key == 'up':
+                                 ax.index -= 1
+                             if ax.index < 0:
+                                 ax.index = len(CMAPS) - 1
+                             elif ax.index == len(CMAPS):
+                                 ax.index = 0
+                             cmap = CMAPS[ax.index]
+                             ax.cbar.set_cmap(cmap)
+                             ax.cbar.draw_all()
+                             ax.plt.set_cmap(cmap)
+                             ax.cbar.patch.figure.canvas.draw()
                  def __add_axes(self, ax, size='30%', pad='8%'):
                      '''
                      Add new axes to the given figure
                      '''
                      divider = make_axes_locatable(ax)
                      nax = divider.new_horizontal(size=size, pad=pad)
                      ax.figure.add_axes(nax)
                      return nax
+                     self.setup()
                  def setup(self):
                      '''
                      This method should be implemented in the child class, the following
                      attributes should be set:
                      self.nrows: number of rows
                      self.ncols: number of cols
                      self.nplots: number of plots (channels or pairs)
                      self.ylabel: label for Y axes
                      self.titles: list of axes title
                      '''
                      raise(NotImplementedError, 'Implement this method in child class')
                  def fill_gaps(self, x_buffer, y_buffer, z_buffer):
                      '''
                      Create a masked array for missing data
                      '''
                      if x_buffer.shape[0] < 2:
                          return x_buffer, y_buffer, z_buffer
                      deltas = x_buffer[1:] - x_buffer[0:-1]
                      x_median = numpy.median(deltas)
                      index = numpy.where(deltas > 5*x_median)
                      if len(index[0]) != 0:
                          z_buffer[::, index[0], ::] = self.__missing
                          z_buffer = numpy.ma.masked_inside(z_buffer,
 .99*self.__missing,
 .01*self.__missing)
                      return x_buffer, y_buffer, z_buffer
                  def decimate(self):
                      # dx = int(len(self.x)/self.__MAXNUMX) + 1
                      dy = int(len(self.y)/self.__MAXNUMY) + 1
                      # x = self.x[::dx]
                      x = self.x
                      y = self.y[::dy]
                      z = self.z[::, ::, ::dy]
                      return x, y, z
                  def format(self):
                      '''
                      Set min and max values, labels, ticks and titles
                      '''
                      if self.xmin is None:
                          xmin = self.min_time
                      else:
                          if self.xaxis is 'time':
                              dt = datetime.datetime.fromtimestamp(self.min_time)
                              xmin = (datetime.datetime.combine(dt.date(),
-                                                             datetime.time(int(self.xmin), 0, 0))-d1970).total_seconds()
+                                                             datetime.time(int(self.xmin), 0, 0))-UT1970).total_seconds()
                          else:
                              xmin = self.xmin
                      if self.xmax is None:
                          xmax = xmin+self.xrange*60*60
                      else:
                          if self.xaxis is 'time':
                              dt = datetime.datetime.fromtimestamp(self.min_time)
                              xmax = (datetime.datetime.combine(dt.date(),
-                                                             datetime.time(int(self.xmax), 0, 0))-d1970).total_seconds()
+                                                             datetime.time(int(self.xmax), 0, 0))-UT1970).total_seconds()
                          else:
                              xmax = self.xmax
                      ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
                      ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
                      ystep = 200 if ymax>= 800 else 100 if ymax>=400 else 50 if ymax>=200 else 20
                      for n, ax in enumerate(self.axes):
                          if ax.firsttime:
                              ax.set_facecolor(self.bgcolor)
                              ax.yaxis.set_major_locator(MultipleLocator(ystep))
                              if self.xaxis is 'time':
                                  ax.xaxis.set_major_formatter(FuncFormatter(func))
                                  ax.xaxis.set_major_locator(LinearLocator(9))
                              if self.xlabel is not None:
                                  ax.set_xlabel(self.xlabel)
                              ax.set_ylabel(self.ylabel)
                              ax.firsttime = False
                              if self.showprofile:
                                  self.pf_axes[n].set_ylim(ymin, ymax)
                                  self.pf_axes[n].set_xlim(self.zmin, self.zmax)
                                  self.pf_axes[n].set_xlabel('dB')
                                  self.pf_axes[n].grid(b=True, axis='x')
                                  [tick.set_visible(False) for tick in self.pf_axes[n].get_yticklabels()]
                              if self.colorbar:
-                                 cb = plt.colorbar(ax.plt, ax=ax, pad=0.02)
-                                 cb.ax.tick_params(labelsize=8)
+                                 ax.cbar = plt.colorbar(ax.plt, ax=ax, pad=0.02, aspect=10)
+                                 ax.cbar.ax.tick_params(labelsize=8)
                                  if self.cb_label:
-                                     cb.set_label(self.cb_label, size=8)
+                                     ax.cbar.set_label(self.cb_label, size=8)
                                  elif self.cb_labels:
-                                     cb.set_label(self.cb_labels[n], size=8)
+                                     ax.cbar.set_label(self.cb_labels[n], size=8)
-                         ax.set_title('{} - {} UTC'.format(
+                         ax.set_title('{} - {} {}'.format(
                                  self.titles[n],
-                                 datetime.datetime.fromtimestamp(self.max_time).strftime('%H:%M:%S')),
+                                 datetime.datetime.fromtimestamp(self.max_time).strftime('%H:%M:%S'),
+                                 self.time_label),
                              size=8)
                          ax.set_xlim(xmin, xmax)
                          ax.set_ylim(ymin, ymax)
                  def __plot(self):
                      '''
                      '''
                      log.success('Plotting', self.name)
                      self.plot()
                      self.format()
                      for n, fig in enumerate(self.figures):
                          if self.nrows == 0 or self.nplots == 0:
                              log.warning('No data', self.name)
                              continue
                          if self.show:
                              fig.show()
                          fig.tight_layout()
                          fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
                                                                               datetime.datetime.fromtimestamp(self.max_time).strftime('%Y/%m/%d')))
                          # fig.canvas.draw()
                          if self.save and self.data.ended:
                              channels = range(self.nrows)
                              if self.oneFigure:
                                  label = ''
                              else:
                                  label = '_{}'.format(channels[n])
                              figname = os.path.join(
                                  self.save,
                                  '{}{}_{}.png'.format(
                                      self.CODE,
                                      label,
                                      datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')
                                  )
                              )
                              print 'Saving figure: {}'.format(figname)
                              fig.savefig(figname)
                  def plot(self):
                      '''
                      '''
                      raise(NotImplementedError, 'Implement this method in child class')
                  def run(self):
                      log.success('Starting', self.name)
                      context = zmq.Context()
                      receiver = context.socket(zmq.SUB)
                      receiver.setsockopt(zmq.SUBSCRIBE, '')
                      receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
                      if 'server' in self.kwargs['parent']:
                          receiver.connect('ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
                      else:
                          receiver.connect("ipc:///tmp/zmq.plots")
                      while True:
                          try:
-                             self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
-                             self.min_time = self.data.times[0]
-                             self.max_time = self.data.times[-1]
+                             if self.localtime:
+                                 self.times = self.data.times - time.timezone
+                             else:
+                                 self.times = self.data.times
+                             self.min_time = self.times[0]
+                             self.max_time = self.times[-1]
                              if self.isConfig is False:
                                  self.__setup()
                                  self.isConfig = True
                              self.__plot()
                          except zmq.Again as e:
                              log.log('Waiting for data...')
                              if self.data:
                                  plt.pause(self.data.throttle)
                              else:
                                  time.sleep(2)
                  def close(self):
                      if self.data:
                          self.__plot()
              class PlotSpectraData(PlotData):
                  '''
                  Plot for Spectra data
                  '''
                  CODE = 'spc'
                  colormap = 'jro'
                  def setup(self):
                      self.nplots = len(self.data.channels)
                      self.ncols = int(numpy.sqrt(self.nplots)+ 0.9)
                      self.nrows = int((1.0*self.nplots/self.ncols) + 0.9)
                      self.width = 3.4*self.ncols
                      self.height = 3*self.nrows
                      self.cb_label = 'dB'
                      if self.showprofile:
                          self.width += 0.8*self.ncols
                      self.ylabel = 'Range [Km]'
                  def plot(self):
                      if self.xaxis == "frequency":
                          x = self.data.xrange[0]
                          self.xlabel = "Frequency (kHz)"
                      elif self.xaxis == "time":
                          x = self.data.xrange[1]
                          self.xlabel = "Time (ms)"
                      else:
                          x = self.data.xrange[2]
                          self.xlabel = "Velocity (m/s)"
                      if self.CODE == 'spc_mean':
                          x = self.data.xrange[2]
                          self.xlabel = "Velocity (m/s)"
                      self.titles = []
                      y = self.data.heights
                      self.y = y
                      z = self.data['spc']
                      for n, ax in enumerate(self.axes):
                          noise = self.data['noise'][n][-1]
                          if self.CODE == 'spc_mean':
                              mean = self.data['mean'][n][-1]
                          if ax.firsttime:
                              self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
                              self.xmin = self.xmin if self.xmin else -self.xmax
                              self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
                              self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
                              ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                   vmin=self.zmin,
                                                   vmax=self.zmax,
                                                   cmap=plt.get_cmap(self.colormap)
                                                  )
                              if self.showprofile:
                                  ax.plt_profile= self.pf_axes[n].plot(self.data['rti'][n][-1], y)[0]
                                  ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
                                                                       color="k", linestyle="dashed", lw=1)[0]
                              if self.CODE == 'spc_mean':
                                  ax.plt_mean = ax.plot(mean, y, color='k')[0]
                          else:
                              ax.plt.set_array(z[n].T.ravel())
                              if self.showprofile:
                                  ax.plt_profile.set_data(self.data['rti'][n][-1], y)
                                  ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
                              if self.CODE == 'spc_mean':
                                  ax.plt_mean.set_data(mean, y)
                          self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
                          self.saveTime = self.max_time
              class PlotCrossSpectraData(PlotData):
                  CODE = 'cspc'
                  zmin_coh = None
                  zmax_coh = None
                  zmin_phase = None
                  zmax_phase = None
                  def setup(self):
                      self.ncols = 4
                      self.nrows = len(self.data.pairs)
                      self.nplots = self.nrows*4
                      self.width = 3.4*self.ncols
                      self.height = 3*self.nrows
                      self.ylabel = 'Range [Km]'
                      self.showprofile = False
                  def plot(self):
                      if self.xaxis == "frequency":
                          x = self.data.xrange[0]
                          self.xlabel = "Frequency (kHz)"
                      elif self.xaxis == "time":
                          x = self.data.xrange[1]
                          self.xlabel = "Time (ms)"
                      else:
                          x = self.data.xrange[2]
                          self.xlabel = "Velocity (m/s)"
                      self.titles = []
                      y = self.data.heights
                      self.y = y
                      spc = self.data['spc']
                      cspc = self.data['cspc']
                      for n in range(self.nrows):
                          noise = self.data['noise'][n][-1]
                          pair = self.data.pairs[n]
                          ax = self.axes[4*n]
                          ax3 = self.axes[4*n+3]
                          if ax.firsttime:
                              self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
                              self.xmin = self.xmin if self.xmin else -self.xmax
                              self.zmin = self.zmin if self.zmin else numpy.nanmin(spc)
                              self.zmax = self.zmax if self.zmax else numpy.nanmax(spc)
                              ax.plt = ax.pcolormesh(x, y, spc[pair[0]].T,
                                                     vmin=self.zmin,
                                                     vmax=self.zmax,
                                                     cmap=plt.get_cmap(self.colormap)
                                                     )
                          else:
                              ax.plt.set_array(spc[pair[0]].T.ravel())
                          self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
                          ax = self.axes[4*n+1]
                          if ax.firsttime:
                              ax.plt = ax.pcolormesh(x, y, spc[pair[1]].T,
                                                     vmin=self.zmin,
                                                     vmax=self.zmax,
                                                     cmap=plt.get_cmap(self.colormap)
                                                     )
                          else:
                              ax.plt.set_array(spc[pair[1]].T.ravel())
                          self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
                          out = cspc[n]/numpy.sqrt(spc[pair[0]]*spc[pair[1]])
                          coh = numpy.abs(out)
                          phase = numpy.arctan2(out.imag, out.real)*180/numpy.pi
                          ax = self.axes[4*n+2]
                          if ax.firsttime:
                              ax.plt = ax.pcolormesh(x, y, coh.T,
                                                     vmin=0,
                                                     vmax=1,
                                                     cmap=plt.get_cmap(self.colormap_coh)
                                                     )
                          else:
                              ax.plt.set_array(coh.T.ravel())
                          self.titles.append('Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
                          ax = self.axes[4*n+3]
                          if ax.firsttime:
                              ax.plt = ax.pcolormesh(x, y, phase.T,
                                                     vmin=-180,
                                                     vmax=180,
                                                     cmap=plt.get_cmap(self.colormap_phase)
                                                     )
                          else:
                              ax.plt.set_array(phase.T.ravel())
                          self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
                          self.saveTime = self.max_time
              class PlotSpectraMeanData(PlotSpectraData):
                  '''
                  Plot for Spectra and Mean
                  '''
                  CODE = 'spc_mean'
                  colormap = 'jro'
              class PlotRTIData(PlotData):
                  '''
                  Plot for RTI data
                  '''
                  CODE = 'rti'
                  colormap = 'jro'
                  def setup(self):
                      self.xaxis = 'time'
                      self.ncols = 1
                      self.nrows = len(self.data.channels)
                      self.nplots = len(self.data.channels)
                      self.ylabel = 'Range [Km]'
                      self.cb_label = 'dB'
                      self.titles = ['{} Channel {}'.format(self.CODE.upper(), x) for x in range(self.nrows)]
                  def plot(self):
-                     self.x = self.data.times
+                     self.x = self.times
                      self.y = self.data.heights
                      self.z = self.data[self.CODE]
                      self.z = numpy.ma.masked_invalid(self.z)
                      for n, ax in enumerate(self.axes):
                          x, y, z = self.fill_gaps(*self.decimate())
                          self.zmin = self.zmin if self.zmin else numpy.min(self.z)
                          self.zmax = self.zmax if self.zmax else numpy.max(self.z)
                          if ax.firsttime:
                              ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                   vmin=self.zmin,
                                                   vmax=self.zmax,
                                                   cmap=plt.get_cmap(self.colormap)
                                                  )
                              if self.showprofile:
                                  ax.plot_profile= self.pf_axes[n].plot(self.data['rti'][n][-1], self.y)[0]
                                  ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y,
                                                                       color="k", linestyle="dashed", lw=1)[0]
                          else:
                              ax.collections.remove(ax.collections[0])
                              ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                     vmin=self.zmin,
                                                     vmax=self.zmax,
                                                     cmap=plt.get_cmap(self.colormap)
                                                    )
                              if self.showprofile:
                                  ax.plot_profile.set_data(self.data['rti'][n][-1], self.y)
                                  ax.plot_noise.set_data(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y)
                      self.saveTime = self.min_time
              class PlotCOHData(PlotRTIData):
                  '''
                  Plot for Coherence data
                  '''
                  CODE = 'coh'
                  def setup(self):
                      self.xaxis = 'time'
                      self.ncols = 1
                      self.nrows = len(self.data.pairs)
                      self.nplots = len(self.data.pairs)
                      self.ylabel = 'Range [Km]'
                      if self.CODE == 'coh':
                          self.cb_label = ''
                          self.titles = ['Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
                      else:
                          self.cb_label = 'Degrees'
                          self.titles = ['Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
              class PlotPHASEData(PlotCOHData):
                  '''
                  Plot for Phase map data
                  '''
                  CODE = 'phase'
                  colormap = 'seismic'
              class PlotNoiseData(PlotData):
                  '''
                  Plot for noise
                  '''
                  CODE = 'noise'
                  def setup(self):
                      self.xaxis = 'time'
                      self.ncols = 1
                      self.nrows = 1
                      self.nplots = 1
                      self.ylabel = 'Intensity [dB]'
                      self.titles = ['Noise']
                      self.colorbar = False
                  def plot(self):
-                     x = self.data.times
+                     x = self.times
                      xmin = self.min_time
                      xmax = xmin+self.xrange*60*60
                      Y = self.data[self.CODE]
                      if self.axes[0].firsttime:
                          for ch in self.data.channels:
                              y = Y[ch]
                              self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
                          plt.legend()
                      else:
                          for ch in self.data.channels:
                              y = Y[ch]
                              self.axes[0].lines[ch].set_data(x, y)
                      self.ymin = numpy.nanmin(Y) - 5
                      self.ymax = numpy.nanmax(Y) + 5
                      self.saveTime = self.min_time
              class PlotSNRData(PlotRTIData):
                  '''
                  Plot for SNR Data
                  '''
                  CODE = 'snr'
                  colormap = 'jet'
              class PlotDOPData(PlotRTIData):
                  '''
                  Plot for DOPPLER Data
                  '''
                  CODE = 'dop'
                  colormap = 'jet'
              class PlotSkyMapData(PlotData):
                  '''
                  Plot for meteors detection data
                  '''
                  CODE = 'met'
                  def setup(self):
                      self.ncols = 1
                      self.nrows = 1
                      self.width = 7.2
                      self.height = 7.2
                      self.xlabel = 'Zonal Zenith Angle (deg)'
                      self.ylabel = 'Meridional Zenith Angle (deg)'
                      if self.figure is None:
                          self.figure = plt.figure(figsize=(self.width, self.height),
                                                   edgecolor='k',
                                                   facecolor='w')
                      else:
                          self.figure.clf()
                      self.ax = plt.subplot2grid((self.nrows, self.ncols), (0, 0), 1, 1, polar=True)
                      self.ax.firsttime = True
                  def plot(self):
-                     arrayParameters = numpy.concatenate([self.data['param'][t] for t in self.data.times])
+                     arrayParameters = numpy.concatenate([self.data['param'][t] for t in self.times])
                      error = arrayParameters[:,-1]
                      indValid = numpy.where(error == 0)[0]
                      finalMeteor = arrayParameters[indValid,:]
                      finalAzimuth = finalMeteor[:,3]
                      finalZenith = finalMeteor[:,4]
                      x = finalAzimuth*numpy.pi/180
                      y = finalZenith
                      if self.ax.firsttime:
                          self.ax.plot = self.ax.plot(x, y, 'bo', markersize=5)[0]
                          self.ax.set_ylim(0,90)
                          self.ax.set_yticks(numpy.arange(0,90,20))
                          self.ax.set_xlabel(self.xlabel)
                          self.ax.set_ylabel(self.ylabel)
                          self.ax.yaxis.labelpad = 40
                          self.ax.firsttime = False
                      else:
                          self.ax.plot.set_data(x, y)
                      dt1 = datetime.datetime.fromtimestamp(self.min_time).strftime('%y/%m/%d %H:%M:%S')
                      dt2 = datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')
                      title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
                                                                                                   dt2,
                                                                                                   len(x))
                      self.ax.set_title(title, size=8)
                      self.saveTime = self.max_time
              class PlotParamData(PlotRTIData):
                  '''
                  Plot for data_param object
                  '''
                  CODE = 'param'
                  colormap = 'seismic'
                  def setup(self):
                      self.xaxis = 'time'
                      self.ncols = 1
                      self.nrows = self.data.shape(self.CODE)[0]
                      self.nplots = self.nrows
                      if self.showSNR:
                          self.nrows += 1
+                         self.nplots += 1
                      self.ylabel = 'Height [Km]'
                      self.titles = self.data.parameters \
                          if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
                      if self.showSNR:
                          self.titles.append('SNR')
                  def plot(self):
                      self.data.normalize_heights()
-                     self.x = self.data.times
+                     self.x = self.times
                      self.y = self.data.heights
                      if self.showSNR:
                          self.z = numpy.concatenate(
                              (self.data[self.CODE], self.data['snr'])
                          )
                      else:
                          self.z = self.data[self.CODE]
                      self.z = numpy.ma.masked_invalid(self.z)
                      for n, ax in enumerate(self.axes):
                          x, y, z = self.fill_gaps(*self.decimate())
                          if ax.firsttime:
                              if self.zlimits is not None:
                                  self.zmin, self.zmax = self.zlimits[n]
                              self.zmax = self.zmax if self.zmax is not None else numpy.nanmax(abs(self.z[:-1, :]))
                              self.zmin = self.zmin if self.zmin is not None else -self.zmax
                              ax.plt = ax.pcolormesh(x, y, z[n, :, :].T*self.factors[n],
                                                     vmin=self.zmin,
                                                     vmax=self.zmax,
                                                     cmap=self.cmaps[n]
                                                    )
                          else:
                              if self.zlimits is not None:
                                  self.zmin, self.zmax = self.zlimits[n]
                              ax.collections.remove(ax.collections[0])
                              ax.plt = ax.pcolormesh(x, y, z[n, :, :].T*self.factors[n],
                                                     vmin=self.zmin,
                                                     vmax=self.zmax,
                                                     cmap=self.cmaps[n]
                                                    )
                      self.saveTime = self.min_time
              class PlotOuputData(PlotParamData):
                  '''
                  Plot data_output object
                  '''
                  CODE = 'output'
                  colormap = 'seismic'

schainpy/model/graphics/jroplot_parameters.py +230 -24

              import os
              import datetime
              import numpy
              import inspect
              from figure import Figure, isRealtime, isTimeInHourRange
              from plotting_codes import *
+             class FitGauPlot(Figure):
+                 isConfig = None
+                 __nsubplots = None
+                 WIDTHPROF = None
+                 HEIGHTPROF = None
+                 PREFIX = 'fitgau'
+                 def __init__(self, **kwargs):
+                     Figure.__init__(self, **kwargs)
+                     self.isConfig = False
+                     self.__nsubplots = 1
+                     self.WIDTH = 250
+                     self.HEIGHT = 250
+                     self.WIDTHPROF = 120
+                     self.HEIGHTPROF = 0
+                     self.counter_imagwr = 0
+                     self.PLOT_CODE = SPEC_CODE
+                     self.FTP_WEI = None
+                     self.EXP_CODE = None
+                     self.SUB_EXP_CODE = None
+                     self.PLOT_POS = None
+                     self.__xfilter_ena = False
+                     self.__yfilter_ena = False
+                 def getSubplots(self):
+                     ncol = int(numpy.sqrt(self.nplots)+0.9)
+                     nrow = int(self.nplots*1./ncol + 0.9)
+                     return nrow, ncol
+                 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
+                     self.__showprofile = showprofile
+                     self.nplots = nplots
+                     ncolspan = 1
+                     colspan = 1
+                     if showprofile:
+                         ncolspan = 3
+                         colspan = 2
+                         self.__nsubplots = 2
+                     self.createFigure(id = id,
+                                       wintitle = wintitle,
+                                       widthplot = self.WIDTH + self.WIDTHPROF,
+                                       heightplot = self.HEIGHT + self.HEIGHTPROF,
+                                       show=show)
+                     nrow, ncol = self.getSubplots()
+                     counter = 0
+                     for y in range(nrow):
+                         for x in range(ncol):
+                             if counter >= self.nplots:
+                                 break
+                             self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
+                             if showprofile:
+                                 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
+                             counter += 1
+                 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
+                         xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
+                         save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
+                         server=None, folder=None, username=None, password=None,
+                         ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
+                         xaxis="frequency", colormap='jet', normFactor=None , GauSelector = 1):
+                     """
+                     Input:
+                         dataOut         :
+                         id        :
+                         wintitle        :
+                         channelList     :
+                         showProfile     :
+                         xmin            :    None,
+                         xmax            :    None,
+                         ymin            :    None,
+                         ymax            :    None,
+                         zmin            :    None,
+                         zmax            :    None
+                     """
+                     if realtime:
+                         if not(isRealtime(utcdatatime = dataOut.utctime)):
+                             print 'Skipping this plot function'
+                             return
+                     if channelList == None:
+                         channelIndexList = dataOut.channelIndexList
+                     else:
+                         channelIndexList = []
+                         for channel in channelList:
+                             if channel not in dataOut.channelList:
+                                 raise ValueError, "Channel %d is not in dataOut.channelList" %channel
+                             channelIndexList.append(dataOut.channelList.index(channel))
+             #         if normFactor is None:
+             #             factor = dataOut.normFactor
+             #         else:
+             #             factor = normFactor
+                     if xaxis == "frequency":
+                         x = dataOut.spc_range[0]
+                         xlabel = "Frequency (kHz)"
+                     elif xaxis == "time":
+                         x = dataOut.spc_range[1]
+                         xlabel = "Time (ms)"
+                     else:
+                         x = dataOut.spc_range[2]
+                         xlabel = "Velocity (m/s)"
+                     ylabel = "Range (Km)"
+                     y = dataOut.getHeiRange()
+                     z = dataOut.GauSPC[:,GauSelector,:,:] #GauSelector]    #dataOut.data_spc/factor
+                     print 'GausSPC', z[0,32,10:40]
+                     z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
+                     zdB = 10*numpy.log10(z)
+                     avg = numpy.average(z, axis=1)
+                     avgdB = 10*numpy.log10(avg)
+                     noise = dataOut.spc_noise
+                     noisedB = 10*numpy.log10(noise)
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
+                     title = wintitle + " Spectra"
+                     if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
+                         title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
+                     if not self.isConfig:
+                         nplots = len(channelIndexList)
+                         self.setup(id=id,
+                                    nplots=nplots,
+                                    wintitle=wintitle,
+                                    showprofile=showprofile,
+                                    show=show)
+                         if xmin == None: xmin = numpy.nanmin(x)
+                         if xmax == None: xmax = numpy.nanmax(x)
+                         if ymin == None: ymin = numpy.nanmin(y)
+                         if ymax == None: ymax = numpy.nanmax(y)
+                         if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
+                         if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
+                         self.FTP_WEI = ftp_wei
+                         self.EXP_CODE = exp_code
+                         self.SUB_EXP_CODE = sub_exp_code
+                         self.PLOT_POS = plot_pos
+                         self.isConfig = True
+                     self.setWinTitle(title)
+                     for i in range(self.nplots):
+                         index = channelIndexList[i]
+                         str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
+                         title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[index], noisedB[index], str_datetime)
+                         if len(dataOut.beam.codeList) != 0:
+                             title = "Ch%d:%4.2fdB,%2.2f,%2.2f:%s" %(dataOut.channelList[index], noisedB[index], dataOut.beam.azimuthList[index], dataOut.beam.zenithList[index], str_datetime)
+                         axes = self.axesList[i*self.__nsubplots]
+                         axes.pcolor(x, y, zdB[index,:,:],
+                                     xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
+                                     xlabel=xlabel, ylabel=ylabel, title=title, colormap=colormap,
+                                     ticksize=9, cblabel='')
+                         if self.__showprofile:
+                             axes = self.axesList[i*self.__nsubplots +1]
+                             axes.pline(avgdB[index,:], y,
+                                     xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
+                                     xlabel='dB', ylabel='', title='',
+                                     ytick_visible=False,
+                                     grid='x')
+                             noiseline = numpy.repeat(noisedB[index], len(y))
+                             axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
+                     self.draw()
+                     if figfile == None:
+                         str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
+                         name = str_datetime
+                         if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
+                             name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
+                         figfile = self.getFilename(name)
+                     self.save(figpath=figpath,
+                               figfile=figfile,
+                               save=save,
+                               ftp=ftp,
+                               wr_period=wr_period,
+                               thisDatetime=thisDatetime)
              class MomentsPlot(Figure):
                  isConfig = None
                  __nsubplots = None
                  WIDTHPROF = None
                  HEIGHTPROF = None
                  PREFIX = 'prm'
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.isConfig = False
                      self.__nsubplots = 1
                      self.WIDTH = 280
                      self.HEIGHT = 250
                      self.WIDTHPROF = 120
                      self.HEIGHTPROF = 0
                      self.counter_imagwr = 0
                      self.PLOT_CODE = MOMENTS_CODE
                      self.FTP_WEI = None
                      self.EXP_CODE = None
                      self.SUB_EXP_CODE = None
                      self.PLOT_POS = None
                  def getSubplots(self):
                      ncol = int(numpy.sqrt(self.nplots)+0.9)
                      nrow = int(self.nplots*1./ncol + 0.9)
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, showprofile=True, show=True):
                      self.__showprofile = showprofile
                      self.nplots = nplots
                      ncolspan = 1
                      colspan = 1
                      if showprofile:
                          ncolspan = 3
                          colspan = 2
                          self.__nsubplots = 2
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH + self.WIDTHPROF,
                                        heightplot = self.HEIGHT + self.HEIGHTPROF,
                                        show=show)
                      nrow, ncol = self.getSubplots()
                      counter = 0
                      for y in range(nrow):
                          for x in range(ncol):
                              if counter >= self.nplots:
                                  break
                              self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
                              if showprofile:
                                  self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
                              counter += 1
                  def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
                          save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
                      """
                      Input:
                          dataOut         :
                          id        :
                          wintitle        :
                          channelList     :
                          showProfile     :
                          xmin            :    None,
                          xmax            :    None,
                          ymin            :    None,
                          ymax            :    None,
                          zmin            :    None,
                          zmax            :    None
                      """
                      if dataOut.flagNoData:
                          return None
                      if realtime:
                          if not(isRealtime(utcdatatime = dataOut.utctime)):
                              print 'Skipping this plot function'
                              return
                      if channelList == None:
                          channelIndexList = dataOut.channelIndexList
                      else:
                          channelIndexList = []
                          for channel in channelList:
                              if channel not in dataOut.channelList:
                                  raise ValueError, "Channel %d is not in dataOut.channelList"
                              channelIndexList.append(dataOut.channelList.index(channel))
                      factor = dataOut.normFactor
                      x = dataOut.abscissaList
                      y = dataOut.heightList
                      z = dataOut.data_pre[channelIndexList,:,:]/factor
                      z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
                      avg = numpy.average(z, axis=1)
                      noise = dataOut.noise/factor
                      zdB = 10*numpy.log10(z)
                      avgdB = 10*numpy.log10(avg)
                      noisedB = 10*numpy.log10(noise)
                      #thisDatetime = dataOut.datatime
                      thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Parameters"
                      xlabel = "Velocity (m/s)"
                      ylabel = "Range (Km)"
                      update_figfile = False
                      if not self.isConfig:
                          nplots = len(channelIndexList)
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     showprofile=showprofile,
                                     show=show)
                          if xmin == None: xmin = numpy.nanmin(x)
                          if xmax == None: xmax = numpy.nanmax(x)
                          if ymin == None: ymin = numpy.nanmin(y)
                          if ymax == None: ymax = numpy.nanmax(y)
                          if zmin == None: zmin = numpy.nanmin(avgdB)*0.9
                          if zmax == None: zmax = numpy.nanmax(avgdB)*0.9
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                          self.SUB_EXP_CODE = sub_exp_code
                          self.PLOT_POS = plot_pos
                          self.isConfig = True
                          update_figfile = True
                      self.setWinTitle(title)
                      for i in range(self.nplots):
                          str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
                          title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[i], noisedB[i], str_datetime)
                          axes = self.axesList[i*self.__nsubplots]
                          axes.pcolor(x, y, zdB[i,:,:],
                                      xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
                                      xlabel=xlabel, ylabel=ylabel, title=title,
                                      ticksize=9, cblabel='')
                                      #Mean Line
                          mean = dataOut.data_param[i, 1, :]
                          axes.addpline(mean, y, idline=0, color="black", linestyle="solid", lw=1)
                          if self.__showprofile:
                              axes = self.axesList[i*self.__nsubplots +1]
                              axes.pline(avgdB[i], y,
                                      xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
                                      xlabel='dB', ylabel='', title='',
                                      ytick_visible=False,
                                      grid='x')
                              noiseline = numpy.repeat(noisedB[i], len(y))
                              axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
                      self.draw()
                      self.save(figpath=figpath,
                                figfile=figfile,
                                save=save,
                                ftp=ftp,
                                wr_period=wr_period,
                                thisDatetime=thisDatetime)
              class SkyMapPlot(Figure):
                  __isConfig = None
                  __nsubplots = None
                  WIDTHPROF = None
                  HEIGHTPROF = None
                  PREFIX = 'mmap'
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.isConfig = False
                      self.__nsubplots = 1
              #         self.WIDTH = 280
              #         self.HEIGHT = 250
                      self.WIDTH = 600
                      self.HEIGHT = 600
                      self.WIDTHPROF = 120
                      self.HEIGHTPROF = 0
                      self.counter_imagwr = 0
                      self.PLOT_CODE = MSKYMAP_CODE
                      self.FTP_WEI = None
                      self.EXP_CODE = None
                      self.SUB_EXP_CODE = None
                      self.PLOT_POS = None
                  def getSubplots(self):
                      ncol = int(numpy.sqrt(self.nplots)+0.9)
                      nrow = int(self.nplots*1./ncol + 0.9)
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, showprofile=False, show=True):
                      self.__showprofile = showprofile
                      self.nplots = nplots
                      ncolspan = 1
                      colspan = 1
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH, #+ self.WIDTHPROF,
                                        heightplot = self.HEIGHT,# + self.HEIGHTPROF,
                                        show=show)
                      nrow, ncol = 1,1
                      counter = 0
                      x = 0
                      y = 0
                      self.addAxes(1, 1, 0, 0, 1, 1, True)
                  def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
                          tmin=0, tmax=24, timerange=None,
                          save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
                      """
                      Input:
                          dataOut         :
                          id        :
                          wintitle        :
                          channelList     :
                          showProfile     :
                          xmin            :    None,
                          xmax            :    None,
                          ymin            :    None,
                          ymax            :    None,
                          zmin            :    None,
                          zmax            :    None
                      """
                      arrayParameters = dataOut.data_param
                      error = arrayParameters[:,-1]
                      indValid = numpy.where(error == 0)[0]
                      finalMeteor = arrayParameters[indValid,:]
                      finalAzimuth = finalMeteor[:,3]
                      finalZenith = finalMeteor[:,4]
                      x = finalAzimuth*numpy.pi/180
                      y = finalZenith
                      x1 = [dataOut.ltctime, dataOut.ltctime]
                      #thisDatetime = dataOut.datatime
                      thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
                      title = wintitle + " Parameters"
                      xlabel = "Zonal Zenith Angle (deg) "
                      ylabel = "Meridional Zenith Angle (deg)"
                      update_figfile = False
                      if not self.isConfig:
                          nplots = 1
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     showprofile=showprofile,
                                     show=show)
                          if self.xmin is None and self.xmax is None:
                              self.xmin, self.xmax = self.getTimeLim(x1, tmin, tmax, timerange)
                          if timerange != None:
                              self.timerange = timerange
                          else:
                              self.timerange = self.xmax - self.xmin
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                          self.SUB_EXP_CODE = sub_exp_code
                          self.PLOT_POS = plot_pos
                          self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
                          self.firstdate = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
                          self.isConfig = True
                          update_figfile = True
                      self.setWinTitle(title)
                      i = 0
                      str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
                      axes = self.axesList[i*self.__nsubplots]
                      nevents = axes.x_buffer.shape[0] + x.shape[0]
                      title = "Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n" %(self.firstdate,str_datetime,nevents)
                      axes.polar(x, y,
                                  title=title, xlabel=xlabel, ylabel=ylabel,
                                  ticksize=9, cblabel='')
                      self.draw()
                      self.save(figpath=figpath,
                                figfile=figfile,
                                save=save,
                                ftp=ftp,
                                wr_period=wr_period,
                                thisDatetime=thisDatetime,
                                update_figfile=update_figfile)
                      if dataOut.ltctime >= self.xmax:
                          self.isConfigmagwr = wr_period
                          self.isConfig = False
                          update_figfile = True
                          axes.__firsttime = True
                          self.xmin += self.timerange
                          self.xmax += self.timerange
              class WindProfilerPlot(Figure):
                  __isConfig = None
                  __nsubplots = None
                  WIDTHPROF = None
                  HEIGHTPROF = None
                  PREFIX = 'wind'
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.timerange = None
                      self.isConfig = False
                      self.__nsubplots = 1
                      self.WIDTH = 800
                      self.HEIGHT = 300
                      self.WIDTHPROF = 120
                      self.HEIGHTPROF = 0
                      self.counter_imagwr = 0
                      self.PLOT_CODE = WIND_CODE
                      self.FTP_WEI = None
                      self.EXP_CODE = None
                      self.SUB_EXP_CODE = None
                      self.PLOT_POS = None
                      self.tmin = None
                      self.tmax = None
                      self.xmin = None
                      self.xmax = None
                      self.figfile = None
                  def getSubplots(self):
                      ncol = 1
                      nrow = self.nplots
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, showprofile=True, show=True):
                      self.__showprofile = showprofile
                      self.nplots = nplots
                      ncolspan = 1
                      colspan = 1
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH + self.WIDTHPROF,
                                        heightplot = self.HEIGHT + self.HEIGHTPROF,
                                        show=show)
                      nrow, ncol = self.getSubplots()
                      counter = 0
                      for y in range(nrow):
                          if counter >= self.nplots:
                              break
                          self.addAxes(nrow, ncol*ncolspan, y, 0, colspan, 1)
                          counter += 1
                  def run(self, dataOut, id, wintitle="", channelList=None, showprofile='False',
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
                          zmax_ver = None, zmin_ver = None, SNRmin = None, SNRmax = None,
                          timerange=None, SNRthresh = None,
                          save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
                      """
                      Input:
                          dataOut         :
                          id        :
                          wintitle        :
                          channelList     :
                          showProfile     :
                          xmin            :    None,
                          xmax            :    None,
                          ymin            :    None,
                          ymax            :    None,
                          zmin            :    None,
                          zmax            :    None
                      """
              #         if timerange is not None:
              #             self.timerange = timerange
              #
              #         tmin = None
              #         tmax = None
-                     x = dataOut.getTimeRange1(dataOut.outputInterval)
+                     x = dataOut.getTimeRange1(dataOut.paramInterval)
                      y = dataOut.heightList
                      z = dataOut.data_output.copy()
                      nplots = z.shape[0]    #Number of wind dimensions estimated
                      nplotsw = nplots
                      #If there is a SNR function defined
                      if dataOut.data_SNR is not None:
                          nplots += 1
                          SNR = dataOut.data_SNR
                          SNRavg = numpy.average(SNR, axis=0)
                          SNRdB = 10*numpy.log10(SNR)
                          SNRavgdB = 10*numpy.log10(SNRavg)
                          if SNRthresh == None: SNRthresh = -5.0
                          ind = numpy.where(SNRavg < 10**(SNRthresh/10))[0]
                          for i in range(nplotsw):
                              z[i,ind] = numpy.nan
                      thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
                      #thisDatetime = datetime.datetime.now()
                      title = wintitle + "Wind"
                      xlabel = ""
                      ylabel = "Height (km)"
                      update_figfile = False
                      if not self.isConfig:
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     showprofile=showprofile,
                                     show=show)
                          if timerange is not None:
                              self.timerange = timerange
                          self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
                          if ymin == None: ymin = numpy.nanmin(y)
                          if ymax == None: ymax = numpy.nanmax(y)
                          if zmax == None: zmax = numpy.nanmax(abs(z[range(2),:]))
                          #if numpy.isnan(zmax): zmax = 50
                          if zmin == None: zmin = -zmax
                          if nplotsw == 3:
                              if zmax_ver == None: zmax_ver = numpy.nanmax(abs(z[2,:]))
                              if zmin_ver == None: zmin_ver = -zmax_ver
                          if dataOut.data_SNR is not None:
                              if SNRmin == None:  SNRmin = numpy.nanmin(SNRavgdB)
                              if SNRmax == None:  SNRmax = numpy.nanmax(SNRavgdB)
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                          self.SUB_EXP_CODE = sub_exp_code
                          self.PLOT_POS = plot_pos
                          self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
                          self.isConfig = True
                          self.figfile = figfile
                          update_figfile = True
                      self.setWinTitle(title)
                      if ((self.xmax - x[1]) < (x[1]-x[0])):
                          x[1] = self.xmax
                      strWind = ['Zonal', 'Meridional', 'Vertical']
                      strCb = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)']
                      zmaxVector = [zmax, zmax, zmax_ver]
                      zminVector = [zmin, zmin, zmin_ver]
                      windFactor = [1,1,100]
                      for i in range(nplotsw):
                          title = "%s Wind: %s" %(strWind[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
                          axes = self.axesList[i*self.__nsubplots]
                          z1 = z[i,:].reshape((1,-1))*windFactor[i]
                          #z1=numpy.ma.masked_where(z1==0.,z1)
                          axes.pcolorbuffer(x, y, z1,
                                      xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zminVector[i], zmax=zmaxVector[i],
                                      xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
                                      ticksize=9, cblabel=strCb[i], cbsize="1%", colormap="seismic" )
                      if dataOut.data_SNR is not None:
                          i += 1
                          title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
                          axes = self.axesList[i*self.__nsubplots]
                          SNRavgdB = SNRavgdB.reshape((1,-1))
                          axes.pcolorbuffer(x, y, SNRavgdB,
                                      xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
                                      xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
                                      ticksize=9, cblabel='', cbsize="1%", colormap="jet")
                      self.draw()
                      self.save(figpath=figpath,
                                figfile=figfile,
                                save=save,
                                ftp=ftp,
                                wr_period=wr_period,
                                thisDatetime=thisDatetime,
                                update_figfile=update_figfile)
-                     if dataOut.ltctime + dataOut.outputInterval >= self.xmax:
+                     if dataOut.ltctime + dataOut.paramInterval >= self.xmax:
                          self.counter_imagwr = wr_period
                          self.isConfig = False
                          update_figfile = True
              class ParametersPlot(Figure):
                  __isConfig = None
                  __nsubplots = None
                  WIDTHPROF = None
                  HEIGHTPROF = None
                  PREFIX = 'param'
                  nplots = None
                  nchan = None
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.timerange = None
                      self.isConfig = False
                      self.__nsubplots = 1
                      self.WIDTH = 800
                      self.HEIGHT = 180
                      self.WIDTHPROF = 120
                      self.HEIGHTPROF = 0
                      self.counter_imagwr = 0
                      self.PLOT_CODE = RTI_CODE
                      self.FTP_WEI = None
                      self.EXP_CODE = None
                      self.SUB_EXP_CODE = None
                      self.PLOT_POS = None
                      self.tmin = None
                      self.tmax = None
                      self.xmin = None
                      self.xmax = None
                      self.figfile = None
                  def getSubplots(self):
                      ncol = 1
                      nrow = self.nplots
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, show=True):
                      self.nplots = nplots
                      ncolspan = 1
                      colspan = 1
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH + self.WIDTHPROF,
                                        heightplot = self.HEIGHT + self.HEIGHTPROF,
                                        show=show)
                      nrow, ncol = self.getSubplots()
                      counter = 0
                      for y in range(nrow):
                          for x in range(ncol):
                              if counter >= self.nplots:
                                  break
                              self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
                              counter += 1
-                 def run(self, dataOut, id, wintitle="", channelList=None, paramIndex = 0, colormap=True,
+                 def run(self, dataOut, id, wintitle="", channelList=None, paramIndex = 0, colormap="jet",
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None, timerange=None,
                          showSNR=False, SNRthresh = -numpy.inf, SNRmin=None, SNRmax=None,
                          save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
                          server=None, folder=None, username=None, password=None,
-                         ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
+                         ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, HEIGHT=None):
                      """
                      Input:
                          dataOut         :
                          id        :
                          wintitle        :
                          channelList     :
                          showProfile     :
                          xmin            :    None,
                          xmax            :    None,
                          ymin            :    None,
                          ymax            :    None,
                          zmin            :    None,
                          zmax            :    None
                      """
-                     if colormap:
-                         colormap="jet"
-                     else:
-                         colormap="RdBu_r"
+                     if HEIGHT is not None:
+                         self.HEIGHT = HEIGHT
                      if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
                          return
                      if channelList == None:
                          channelIndexList = range(dataOut.data_param.shape[0])
                      else:
                          channelIndexList = []
                          for channel in channelList:
                              if channel not in dataOut.channelList:
                                  raise ValueError, "Channel %d is not in dataOut.channelList"
                              channelIndexList.append(dataOut.channelList.index(channel))
                      x = dataOut.getTimeRange1(dataOut.paramInterval)
                      y = dataOut.getHeiRange()
                      if dataOut.data_param.ndim == 3:
                          z = dataOut.data_param[channelIndexList,paramIndex,:]
                      else:
                          z = dataOut.data_param[channelIndexList,:]
                      if showSNR:
                          #SNR data
                          SNRarray = dataOut.data_SNR[channelIndexList,:]
                          SNRdB = 10*numpy.log10(SNRarray)
                          ind = numpy.where(SNRdB < SNRthresh)
                          z[ind] = numpy.nan
                      thisDatetime = dataOut.datatime
              #         thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
                      xlabel = ""
                      ylabel = "Range (Km)"
                      update_figfile = False
                      if not self.isConfig:
                          nchan = len(channelIndexList)
                          self.nchan = nchan
                          self.plotFact = 1
                          nplots = nchan
                          if showSNR:
                              nplots = nchan*2
                              self.plotFact = 2
                              if SNRmin == None:  SNRmin = numpy.nanmin(SNRdB)
                              if SNRmax == None:  SNRmax = numpy.nanmax(SNRdB)
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     show=show)
                          if timerange != None:
                              self.timerange = timerange
                          self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
                          if ymin == None: ymin = numpy.nanmin(y)
                          if ymax == None: ymax = numpy.nanmax(y)
                          if zmin == None: zmin = numpy.nanmin(z)
                          if zmax == None: zmax = numpy.nanmax(z)
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                          self.SUB_EXP_CODE = sub_exp_code
                          self.PLOT_POS = plot_pos
                          self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
                          self.isConfig = True
                          self.figfile = figfile
                          update_figfile = True
                      self.setWinTitle(title)
                      for i in range(self.nchan):
                          index = channelIndexList[i]
                          title = "Channel %d: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
                          axes = self.axesList[i*self.plotFact]
                          z1 = z[i,:].reshape((1,-1))
                          axes.pcolorbuffer(x, y, z1,
                                      xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
                                      xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
                                      ticksize=9, cblabel='', cbsize="1%",colormap=colormap)
                          if showSNR:
                              title = "Channel %d SNR: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
                              axes = self.axesList[i*self.plotFact + 1]
                              SNRdB1 = SNRdB[i,:].reshape((1,-1))
                              axes.pcolorbuffer(x, y, SNRdB1,
                                      xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
                                      xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
                                      ticksize=9, cblabel='', cbsize="1%",colormap='jet')
                      self.draw()
                      if dataOut.ltctime >= self.xmax:
                          self.counter_imagwr = wr_period
                          self.isConfig = False
                          update_figfile = True
                      self.save(figpath=figpath,
                                figfile=figfile,
                                save=save,
                                ftp=ftp,
                                wr_period=wr_period,
                                thisDatetime=thisDatetime,
                                update_figfile=update_figfile)
              class Parameters1Plot(Figure):
                  __isConfig = None
                  __nsubplots = None
                  WIDTHPROF = None
                  HEIGHTPROF = None
                  PREFIX = 'prm'
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.timerange = 2*60*60
                      self.isConfig = False
                      self.__nsubplots = 1
                      self.WIDTH = 800
                      self.HEIGHT = 180
                      self.WIDTHPROF = 120
                      self.HEIGHTPROF = 0
                      self.counter_imagwr = 0
                      self.PLOT_CODE = PARMS_CODE
                      self.FTP_WEI = None
                      self.EXP_CODE = None
                      self.SUB_EXP_CODE = None
                      self.PLOT_POS = None
                      self.tmin = None
                      self.tmax = None
                      self.xmin = None
                      self.xmax = None
                      self.figfile = None
                  def getSubplots(self):
                      ncol = 1
                      nrow = self.nplots
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, showprofile=True, show=True):
                      self.__showprofile = showprofile
                      self.nplots = nplots
                      ncolspan = 1
                      colspan = 1
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH + self.WIDTHPROF,
                                        heightplot = self.HEIGHT + self.HEIGHTPROF,
                                        show=show)
                      nrow, ncol = self.getSubplots()
                      counter = 0
                      for y in range(nrow):
                          for x in range(ncol):
                              if counter >= self.nplots:
                                  break
                              self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
                              if showprofile:
                                  self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
                              counter += 1
                  def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,timerange=None,
                          parameterIndex = None, onlyPositive = False,
                          SNRthresh = -numpy.inf, SNR = True, SNRmin = None, SNRmax = None, onlySNR = False,
                          DOP = True,
                          zlabel = "", parameterName = "", parameterObject = "data_param",
                          save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
                      #print inspect.getargspec(self.run).args
                      """
                      Input:
                          dataOut         :
                          id        :
                          wintitle        :
                          channelList     :
                          showProfile     :
                          xmin            :    None,
                          xmax            :    None,
                          ymin            :    None,
                          ymax            :    None,
                          zmin            :    None,
                          zmax            :    None
                      """
                      data_param = getattr(dataOut, parameterObject)
                      if channelList == None:
                          channelIndexList = numpy.arange(data_param.shape[0])
                      else:
                          channelIndexList = numpy.array(channelList)
                      nchan = len(channelIndexList) #Number of channels being plotted
                      if nchan < 1:
                          return
                      nGraphsByChannel = 0
                      if SNR:
                          nGraphsByChannel += 1
                      if DOP:
                          nGraphsByChannel += 1
                      if nGraphsByChannel < 1:
                          return
                      nplots = nGraphsByChannel*nchan
                      if timerange is not None:
                          self.timerange = timerange
                      #tmin = None
                      #tmax = None
                      if parameterIndex == None:
                          parameterIndex = 1
                      x = dataOut.getTimeRange1(dataOut.paramInterval)
                      y = dataOut.heightList
-                     z = data_param[channelIndexList,parameterIndex,:].copy()
-                     zRange = dataOut.abscissaList
-             #         nChannels = z.shape[0]    #Number of wind dimensions estimated
-             #        thisDatetime = dataOut.datatime
+                     if dataOut.data_param.ndim == 3:
+                         z = dataOut.data_param[channelIndexList,parameterIndex,:]
+                     else:
+                         z = dataOut.data_param[channelIndexList,:]
                      if dataOut.data_SNR is not None:
-                         SNRarray = dataOut.data_SNR[channelIndexList,:]
-                         SNRdB = 10*numpy.log10(SNRarray)
-             #             SNRavgdB = 10*numpy.log10(SNRavg)
-                         ind = numpy.where(SNRdB < 10**(SNRthresh/10))
-                         z[ind] = numpy.nan
+                         if dataOut.data_SNR.ndim == 2:
+                             SNRavg = numpy.average(dataOut.data_SNR, axis=0)
+                         else:
+                             SNRavg = dataOut.data_SNR
+                         SNRdB = 10*numpy.log10(SNRavg)
                      thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
                      xlabel = ""
                      ylabel = "Range (Km)"
-                     if (SNR and not onlySNR): nplots = 2*nplots
                      if onlyPositive:
                          colormap = "jet"
                          zmin = 0
                      else: colormap = "RdBu_r"
                      if not self.isConfig:
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     showprofile=showprofile,
                                     show=show)
                          self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
                          if ymin == None: ymin = numpy.nanmin(y)
                          if ymax == None: ymax = numpy.nanmax(y)
-                         if zmin == None: zmin = numpy.nanmin(zRange)
-                         if zmax == None: zmax = numpy.nanmax(zRange)
+                         if zmin == None: zmin = numpy.nanmin(z)
+                         if zmax == None: zmax = numpy.nanmax(z)
                          if SNR:
                              if SNRmin == None:  SNRmin = numpy.nanmin(SNRdB)
                              if SNRmax == None:  SNRmax = numpy.nanmax(SNRdB)
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                          self.SUB_EXP_CODE = sub_exp_code
                          self.PLOT_POS = plot_pos
                          self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
                          self.isConfig = True
                          self.figfile = figfile
                      self.setWinTitle(title)
                      if ((self.xmax - x[1]) < (x[1]-x[0])):
                          x[1] = self.xmax
                      for i in range(nchan):
                          if (SNR and not onlySNR): j = 2*i
                          else: j = i
                          j = nGraphsByChannel*i
                          if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
                              title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
                          if not onlySNR:
                              axes = self.axesList[j*self.__nsubplots]
                              z1 = z[i,:].reshape((1,-1))
                              axes.pcolorbuffer(x, y, z1,
                                      xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
                                      xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
                                      ticksize=9, cblabel=zlabel, cbsize="1%")
                          if DOP:
                              title = "%s Channel %d: %s" %(parameterName, channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
                              if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
                                  title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
                              axes = self.axesList[j]
                              z1 = z[i,:].reshape((1,-1))
                              axes.pcolorbuffer(x, y, z1,
                                          xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
                                          xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
                                          ticksize=9, cblabel=zlabel, cbsize="1%")
-                         if SNR:
-                             title = "Channel %d Signal Noise Ratio (SNR): %s" %(channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
-                             axes = self.axesList[(j)*self.__nsubplots]
-                             if not onlySNR:
-                                 axes = self.axesList[(j + 1)*self.__nsubplots]
-                             axes = self.axesList[(j + nGraphsByChannel-1)]
-                             z1 = SNRdB[i,:].reshape((1,-1))
+                         z1 = SNRdB.reshape((1,-1))
-                             axes.pcolorbuffer(x, y, z1,
-                                     xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
-                                     xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap="jet",
-                                     ticksize=9, cblabel=zlabel, cbsize="1%")
                      self.draw()
                      if x[1] >= self.axesList[0].xmax:
                          self.counter_imagwr = wr_period
                          self.isConfig = False
                          self.figfile = None
                      self.save(figpath=figpath,
                                figfile=figfile,
                                save=save,
                                ftp=ftp,
                                wr_period=wr_period,
                                thisDatetime=thisDatetime,
                                update_figfile=False)
              class SpectralFittingPlot(Figure):
                  __isConfig = None
                  __nsubplots = None
                  WIDTHPROF = None
                  HEIGHTPROF = None
                  PREFIX = 'prm'
                  N = None
                  ippSeconds = None
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.isConfig = False
                      self.__nsubplots = 1
                      self.PLOT_CODE = SPECFIT_CODE
                      self.WIDTH = 450
                      self.HEIGHT = 250
                      self.WIDTHPROF = 0
                      self.HEIGHTPROF = 0
                  def getSubplots(self):
                      ncol = int(numpy.sqrt(self.nplots)+0.9)
                      nrow = int(self.nplots*1./ncol + 0.9)
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, showprofile=False, show=True):
                      showprofile = False
                      self.__showprofile = showprofile
                      self.nplots = nplots
                      ncolspan = 5
                      colspan = 4
                      if showprofile:
                          ncolspan = 5
                          colspan = 4
                          self.__nsubplots = 2
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH + self.WIDTHPROF,
                                        heightplot = self.HEIGHT + self.HEIGHTPROF,
                                        show=show)
                      nrow, ncol = self.getSubplots()
                      counter = 0
                      for y in range(nrow):
                          for x in range(ncol):
                              if counter >= self.nplots:
                                  break
                              self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
                              if showprofile:
                                  self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
                              counter += 1
                  def run(self, dataOut, id, cutHeight=None, fit=False, wintitle="", channelList=None, showprofile=True,
                          xmin=None, xmax=None, ymin=None, ymax=None,
                          save=False, figpath='./', figfile=None, show=True):
                      """
                      Input:
                          dataOut         :
                          id        :
                          wintitle        :
                          channelList     :
                          showProfile     :
                          xmin            :    None,
                          xmax            :    None,
                          zmin            :    None,
                          zmax            :    None
                      """
                      if cutHeight==None:
                          h=270
                      heightindex = numpy.abs(cutHeight - dataOut.heightList).argmin()
                      cutHeight = dataOut.heightList[heightindex]
                      factor = dataOut.normFactor
                      x = dataOut.abscissaList[:-1]
                      #y = dataOut.getHeiRange()
                      z = dataOut.data_pre[:,:,heightindex]/factor
                      z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
                      avg = numpy.average(z, axis=1)
                      listChannels = z.shape[0]
                      #Reconstruct Function
                      if fit==True:
                          groupArray = dataOut.groupList
                          listChannels = groupArray.reshape((groupArray.size))
                          listChannels.sort()
                          spcFitLine = numpy.zeros(z.shape)
                          constants = dataOut.constants
                          nGroups = groupArray.shape[0]
                          nChannels = groupArray.shape[1]
                          nProfiles = z.shape[1]
                          for f in range(nGroups):
                              groupChann = groupArray[f,:]
                              p = dataOut.data_param[f,:,heightindex]
              #                 p = numpy.array([ 89.343967,0.14036615,0.17086219,18.89835291,1.58388365,1.55099167])
                              fitLineAux = dataOut.library.modelFunction(p, constants)*nProfiles
                              fitLineAux = fitLineAux.reshape((nChannels,nProfiles))
                              spcFitLine[groupChann,:] = fitLineAux
              #             spcFitLine = spcFitLine/factor
                          z = z[listChannels,:]
                          spcFitLine = spcFitLine[listChannels,:]
                          spcFitLinedB = 10*numpy.log10(spcFitLine)
                      zdB = 10*numpy.log10(z)
                      #thisDatetime = dataOut.datatime
                      thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Doppler Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
                      xlabel = "Velocity (m/s)"
                      ylabel = "Spectrum"
                      if not self.isConfig:
                          nplots = listChannels.size
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     showprofile=showprofile,
                                     show=show)
                          if xmin == None: xmin = numpy.nanmin(x)
                          if xmax == None: xmax = numpy.nanmax(x)
                          if ymin == None: ymin = numpy.nanmin(zdB)
                          if ymax == None: ymax = numpy.nanmax(zdB)+2
                          self.isConfig = True
                      self.setWinTitle(title)
                      for i in range(self.nplots):
              #             title = "Channel %d: %4.2fdB" %(dataOut.channelList[i]+1, noisedB[i])
                          title = "Height %4.1f km\nChannel %d:" %(cutHeight, listChannels[i])
                          axes = self.axesList[i*self.__nsubplots]
                          if fit == False:
                              axes.pline(x, zdB[i,:],
                                          xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
                                          xlabel=xlabel, ylabel=ylabel, title=title
                                          )
                          if fit == True:
                              fitline=spcFitLinedB[i,:]
                              y=numpy.vstack([zdB[i,:],fitline] )
                              legendlabels=['Data','Fitting']
                              axes.pmultilineyaxis(x, y,
                                          xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
                                          xlabel=xlabel, ylabel=ylabel, title=title,
                                          legendlabels=legendlabels, marker=None,
                                          linestyle='solid', grid='both')
                      self.draw()
                      self.save(figpath=figpath,
                                figfile=figfile,
                                save=save,
                                ftp=ftp,
                                wr_period=wr_period,
                                thisDatetime=thisDatetime)
              class EWDriftsPlot(Figure):
                  __isConfig = None
                  __nsubplots = None
                  WIDTHPROF = None
                  HEIGHTPROF = None
                  PREFIX = 'drift'
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.timerange = 2*60*60
                      self.isConfig = False
                      self.__nsubplots = 1
                      self.WIDTH = 800
                      self.HEIGHT = 150
                      self.WIDTHPROF = 120
                      self.HEIGHTPROF = 0
                      self.counter_imagwr = 0
                      self.PLOT_CODE = EWDRIFT_CODE
                      self.FTP_WEI = None
                      self.EXP_CODE = None
                      self.SUB_EXP_CODE = None
                      self.PLOT_POS = None
                      self.tmin = None
                      self.tmax = None
                      self.xmin = None
                      self.xmax = None
                      self.figfile = None
                  def getSubplots(self):
                      ncol = 1
                      nrow = self.nplots
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, showprofile=True, show=True):
                      self.__showprofile = showprofile
                      self.nplots = nplots
                      ncolspan = 1
                      colspan = 1
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH + self.WIDTHPROF,
                                        heightplot = self.HEIGHT + self.HEIGHTPROF,
                                        show=show)
                      nrow, ncol = self.getSubplots()
                      counter = 0
                      for y in range(nrow):
                          if counter >= self.nplots:
                              break
                          self.addAxes(nrow, ncol*ncolspan, y, 0, colspan, 1)
                          counter += 1
                  def run(self, dataOut, id, wintitle="", channelList=None,
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
                          zmaxVertical = None, zminVertical = None, zmaxZonal = None, zminZonal = None,
                          timerange=None, SNRthresh = -numpy.inf, SNRmin = None, SNRmax = None, SNR_1 = False,
                          save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
                      """
                      Input:
                          dataOut         :
                          id        :
                          wintitle        :
                          channelList     :
                          showProfile     :
                          xmin            :    None,
                          xmax            :    None,
                          ymin            :    None,
                          ymax            :    None,
                          zmin            :    None,
                          zmax            :    None
                      """
                      if timerange is not None:
                          self.timerange = timerange
                      tmin = None
                      tmax = None
                      x = dataOut.getTimeRange1(dataOut.outputInterval)
              #         y = dataOut.heightList
                      y = dataOut.heightList
                      z = dataOut.data_output
                      nplots = z.shape[0]    #Number of wind dimensions estimated
                      nplotsw = nplots
                      #If there is a SNR function defined
                      if dataOut.data_SNR is not None:
                          nplots += 1
                          SNR = dataOut.data_SNR
                          if SNR_1:
                              SNR += 1
                          SNRavg = numpy.average(SNR, axis=0)
                          SNRdB = 10*numpy.log10(SNR)
                          SNRavgdB = 10*numpy.log10(SNRavg)
                          ind = numpy.where(SNRavg < 10**(SNRthresh/10))[0]
                          for i in range(nplotsw):
                              z[i,ind] = numpy.nan
                      showprofile = False
              #        thisDatetime = dataOut.datatime
                      thisDatetime = datetime.datetime.utcfromtimestamp(x[1])
                      title = wintitle + " EW Drifts"
                      xlabel = ""
                      ylabel = "Height (Km)"
                      if not self.isConfig:
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     showprofile=showprofile,
                                     show=show)
                          self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
                          if ymin == None: ymin = numpy.nanmin(y)
                          if ymax == None: ymax = numpy.nanmax(y)
                          if zmaxZonal == None: zmaxZonal = numpy.nanmax(abs(z[0,:]))
                          if zminZonal == None: zminZonal = -zmaxZonal
                          if zmaxVertical == None: zmaxVertical = numpy.nanmax(abs(z[1,:]))
                          if zminVertical == None: zminVertical = -zmaxVertical
                          if dataOut.data_SNR is not None:
                              if SNRmin == None:  SNRmin = numpy.nanmin(SNRavgdB)
                              if SNRmax == None:  SNRmax = numpy.nanmax(SNRavgdB)
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                          self.SUB_EXP_CODE = sub_exp_code
                          self.PLOT_POS = plot_pos
                          self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
                          self.isConfig = True
                      self.setWinTitle(title)
                      if ((self.xmax - x[1]) < (x[1]-x[0])):
                          x[1] = self.xmax
                      strWind = ['Zonal','Vertical']
                      strCb = 'Velocity (m/s)'
                      zmaxVector = [zmaxZonal, zmaxVertical]
                      zminVector = [zminZonal, zminVertical]
                      for i in range(nplotsw):
                          title = "%s Drifts: %s" %(strWind[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
                          axes = self.axesList[i*self.__nsubplots]
                          z1 = z[i,:].reshape((1,-1))
                          axes.pcolorbuffer(x, y, z1,
                                      xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zminVector[i], zmax=zmaxVector[i],
                                      xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
                                      ticksize=9, cblabel=strCb, cbsize="1%", colormap="RdBu_r")
                      if dataOut.data_SNR is not None:
                          i += 1
                          if SNR_1:
                              title = "Signal Noise Ratio + 1 (SNR+1): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
                          else:
                              title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
                          axes = self.axesList[i*self.__nsubplots]
                          SNRavgdB = SNRavgdB.reshape((1,-1))
                          axes.pcolorbuffer(x, y, SNRavgdB,
                                      xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
                                      xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
                                      ticksize=9, cblabel='', cbsize="1%", colormap="jet")
                      self.draw()
                      if x[1] >= self.axesList[0].xmax:
                          self.counter_imagwr = wr_period
                          self.isConfig = False
                          self.figfile = None
              class PhasePlot(Figure):
                  __isConfig = None
                  __nsubplots = None
                  PREFIX = 'mphase'
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.timerange = 24*60*60
                      self.isConfig = False
                      self.__nsubplots = 1
                      self.counter_imagwr = 0
                      self.WIDTH = 600
                      self.HEIGHT = 300
                      self.WIDTHPROF = 120
                      self.HEIGHTPROF = 0
                      self.xdata = None
                      self.ydata = None
                      self.PLOT_CODE = MPHASE_CODE
                      self.FTP_WEI = None
                      self.EXP_CODE = None
                      self.SUB_EXP_CODE = None
                      self.PLOT_POS = None
                      self.filename_phase = None
                      self.figfile = None
                  def getSubplots(self):
                      ncol = 1
                      nrow = 1
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, showprofile=True, show=True):
                      self.__showprofile = showprofile
                      self.nplots = nplots
                      ncolspan = 7
                      colspan = 6
                      self.__nsubplots = 2
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH+self.WIDTHPROF,
                                        heightplot = self.HEIGHT+self.HEIGHTPROF,
                                        show=show)
                      nrow, ncol = self.getSubplots()
                      self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
                  def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
                          xmin=None, xmax=None, ymin=None, ymax=None,
                          timerange=None,
                          save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
                      tmin = None
                      tmax = None
                      x = dataOut.getTimeRange1(dataOut.outputInterval)
                      y = dataOut.getHeiRange()
                      #thisDatetime = dataOut.datatime
                      thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
                      title = wintitle + " Phase of Beacon Signal" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
                      xlabel = "Local Time"
                      ylabel = "Phase"
                      #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
                      phase_beacon = dataOut.data_output
                      update_figfile = False
                      if not self.isConfig:
                          self.nplots = phase_beacon.size
                          self.setup(id=id,
                                     nplots=self.nplots,
                                     wintitle=wintitle,
                                     showprofile=showprofile,
                                     show=show)
                          if timerange is not None:
                              self.timerange = timerange
                          self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
                          if ymin == None: ymin = numpy.nanmin(phase_beacon) - 10.0
                          if ymax == None: ymax = numpy.nanmax(phase_beacon) + 10.0
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                          self.SUB_EXP_CODE = sub_exp_code
                          self.PLOT_POS = plot_pos
                          self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
                          self.isConfig = True
                          self.figfile = figfile
                          self.xdata = numpy.array([])
                          self.ydata = numpy.array([])
                          #open file beacon phase
                          path = '%s%03d' %(self.PREFIX, self.id)
                          beacon_file = os.path.join(path,'%s.txt'%self.name)
                          self.filename_phase = os.path.join(figpath,beacon_file)
                          update_figfile = True
                      #store data beacon phase
                      #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
                      self.setWinTitle(title)
                      title = "Phase Offset %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
                      legendlabels = ["phase %d"%(chan) for chan in numpy.arange(self.nplots)]
                      axes = self.axesList[0]
                      self.xdata = numpy.hstack((self.xdata, x[0:1]))
                      if len(self.ydata)==0:
                          self.ydata = phase_beacon.reshape(-1,1)
                      else:
                          self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
                      axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
                                  xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
                                  xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
                                  XAxisAsTime=True, grid='both'
                                  )
                      self.draw()
                      self.save(figpath=figpath,
                        figfile=figfile,
                        save=save,
                        ftp=ftp,
                        wr_period=wr_period,
                        thisDatetime=thisDatetime,
                        update_figfile=update_figfile)
                      if dataOut.ltctime + dataOut.outputInterval >= self.xmax:
                          self.counter_imagwr = wr_period
                          self.isConfig = False
                          update_figfile = True
              class NSMeteorDetection1Plot(Figure):
                  isConfig = None
                  __nsubplots = None
                  WIDTHPROF = None
                  HEIGHTPROF = None
                  PREFIX = 'nsm'
                  zminList = None
                  zmaxList = None
                  cmapList = None
                  titleList = None
                  nPairs = None
                  nChannels = None
                  nParam = None
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.isConfig = False
                      self.__nsubplots = 1
                      self.WIDTH = 750
                      self.HEIGHT = 250
                      self.WIDTHPROF = 120
                      self.HEIGHTPROF = 0
                      self.counter_imagwr = 0
                      self.PLOT_CODE = SPEC_CODE
                      self.FTP_WEI = None
                      self.EXP_CODE = None
                      self.SUB_EXP_CODE = None
                      self.PLOT_POS = None
                      self.__xfilter_ena = False
                      self.__yfilter_ena = False
                  def getSubplots(self):
                      ncol = 3
                      nrow = int(numpy.ceil(self.nplots/3.0))
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, show=True):
                      self.nplots = nplots
                      ncolspan = 1
                      colspan = 1
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH + self.WIDTHPROF,
                                        heightplot = self.HEIGHT + self.HEIGHTPROF,
                                        show=show)
                      nrow, ncol = self.getSubplots()
                      counter = 0
                      for y in range(nrow):
                          for x in range(ncol):
                              if counter >= self.nplots:
                                  break
                              self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
                              counter += 1
                  def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
                          xmin=None, xmax=None, ymin=None, ymax=None, SNRmin=None, SNRmax=None,
                          vmin=None, vmax=None, wmin=None, wmax=None, mode = 'SA',
                          save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
                          xaxis="frequency"):
                      """
                      Input:
                          dataOut         :
                          id        :
                          wintitle        :
                          channelList     :
                          showProfile     :
                          xmin            :    None,
                          xmax            :    None,
                          ymin            :    None,
                          ymax            :    None,
                          zmin            :    None,
                          zmax            :    None
                      """
                      #SEPARAR EN DOS PLOTS
                      nParam = dataOut.data_param.shape[1] - 3
                      utctime = dataOut.data_param[0,0]
                      tmet = dataOut.data_param[:,1].astype(int)
                      hmet = dataOut.data_param[:,2].astype(int)
                      x = dataOut.abscissaList
                      y = dataOut.heightList
                      z = numpy.zeros((nParam, y.size, x.size - 1))
                      z[:,:] = numpy.nan
                      z[:,hmet,tmet] = dataOut.data_param[:,3:].T
                      z[0,:,:] = 10*numpy.log10(z[0,:,:])
                      xlabel = "Time (s)"
                      ylabel = "Range (km)"
                      thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
                      if not self.isConfig:
                          nplots = nParam
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     show=show)
                          if xmin is None: xmin = numpy.nanmin(x)
                          if xmax is None: xmax = numpy.nanmax(x)
                          if ymin is None: ymin = numpy.nanmin(y)
                          if ymax is None: ymax = numpy.nanmax(y)
                          if SNRmin is None: SNRmin = numpy.nanmin(z[0,:])
                          if SNRmax is None: SNRmax = numpy.nanmax(z[0,:])
                          if vmax is None: vmax = numpy.nanmax(numpy.abs(z[1,:]))
                          if vmin is None: vmin = -vmax
                          if wmin is None: wmin = 0
                          if wmax is None: wmax = 50
                          pairsList = dataOut.groupList
                          self.nPairs = len(dataOut.groupList)
                          zminList = [SNRmin, vmin, cmin] + [pmin]*self.nPairs
                          zmaxList = [SNRmax, vmax, cmax] + [pmax]*self.nPairs
                          titleList = ["SNR","Radial Velocity","Coherence"]
                          cmapList = ["jet","RdBu_r","jet"]
                          for i in range(self.nPairs):
                              strAux1 = "Phase Difference "+ str(pairsList[i][0]) + str(pairsList[i][1])
                              titleList = titleList + [strAux1]
                              cmapList = cmapList + ["RdBu_r"]
                          self.zminList = zminList
                          self.zmaxList = zmaxList
                          self.cmapList = cmapList
                          self.titleList = titleList
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                          self.SUB_EXP_CODE = sub_exp_code
                          self.PLOT_POS = plot_pos
                          self.isConfig = True
                      str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
                      for i in range(nParam):
                          title = self.titleList[i] + ": " +str_datetime
                          axes = self.axesList[i]
                          axes.pcolor(x, y, z[i,:].T,
                                      xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=self.zminList[i], zmax=self.zmaxList[i],
                                      xlabel=xlabel, ylabel=ylabel, title=title, colormap=self.cmapList[i],ticksize=9, cblabel='')
                      self.draw()
                      if figfile == None:
                          str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
                          name = str_datetime
                          if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
                              name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
                          figfile = self.getFilename(name)
                      self.save(figpath=figpath,
                                figfile=figfile,
                                save=save,
                                ftp=ftp,
                                wr_period=wr_period,
                                thisDatetime=thisDatetime)
              class NSMeteorDetection2Plot(Figure):
                  isConfig = None
                  __nsubplots = None
                  WIDTHPROF = None
                  HEIGHTPROF = None
                  PREFIX = 'nsm'
                  zminList = None
                  zmaxList = None
                  cmapList = None
                  titleList = None
                  nPairs = None
                  nChannels = None
                  nParam = None
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.isConfig = False
                      self.__nsubplots = 1
                      self.WIDTH = 750
                      self.HEIGHT = 250
                      self.WIDTHPROF = 120
                      self.HEIGHTPROF = 0
                      self.counter_imagwr = 0
                      self.PLOT_CODE = SPEC_CODE
                      self.FTP_WEI = None
                      self.EXP_CODE = None
                      self.SUB_EXP_CODE = None
                      self.PLOT_POS = None
                      self.__xfilter_ena = False
                      self.__yfilter_ena = False
                  def getSubplots(self):
                      ncol = 3
                      nrow = int(numpy.ceil(self.nplots/3.0))
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, show=True):
                      self.nplots = nplots
                      ncolspan = 1
                      colspan = 1
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH + self.WIDTHPROF,
                                        heightplot = self.HEIGHT + self.HEIGHTPROF,
                                        show=show)
                      nrow, ncol = self.getSubplots()
                      counter = 0
                      for y in range(nrow):
                          for x in range(ncol):
                              if counter >= self.nplots:
                                  break
                              self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
                              counter += 1
                  def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
                          xmin=None, xmax=None, ymin=None, ymax=None, SNRmin=None, SNRmax=None,
                          vmin=None, vmax=None, wmin=None, wmax=None, mode = 'SA',
                          save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
                          xaxis="frequency"):
                      """
                      Input:
                          dataOut         :
                          id        :
                          wintitle        :
                          channelList     :
                          showProfile     :
                          xmin            :    None,
                          xmax            :    None,
                          ymin            :    None,
                          ymax            :    None,
                          zmin            :    None,
                          zmax            :    None
                      """
                      #Rebuild matrix
                      utctime = dataOut.data_param[0,0]
                      cmet = dataOut.data_param[:,1].astype(int)
                      tmet = dataOut.data_param[:,2].astype(int)
                      hmet = dataOut.data_param[:,3].astype(int)
                      nParam = 3
                      nChan = len(dataOut.groupList)
                      x = dataOut.abscissaList
                      y = dataOut.heightList
                      z = numpy.full((nChan, nParam, y.size, x.size - 1),numpy.nan)
                      z[cmet,:,hmet,tmet] = dataOut.data_param[:,4:]
                      z[:,0,:,:] = 10*numpy.log10(z[:,0,:,:]) #logarithmic scale
                      z = numpy.reshape(z, (nChan*nParam, y.size, x.size-1))
                      xlabel = "Time (s)"
                      ylabel = "Range (km)"
                      thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
                      if not self.isConfig:
                          nplots = nParam*nChan
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     show=show)
                          if xmin is None: xmin = numpy.nanmin(x)
                          if xmax is None: xmax = numpy.nanmax(x)
                          if ymin is None: ymin = numpy.nanmin(y)
                          if ymax is None: ymax = numpy.nanmax(y)
                          if SNRmin is None: SNRmin = numpy.nanmin(z[0,:])
                          if SNRmax is None: SNRmax = numpy.nanmax(z[0,:])
                          if vmax is None: vmax = numpy.nanmax(numpy.abs(z[1,:]))
                          if vmin is None: vmin = -vmax
                          if wmin is None: wmin = 0
                          if wmax is None: wmax = 50
                          self.nChannels = nChan
                          zminList = []
                          zmaxList = []
                          titleList = []
                          cmapList = []
                          for i in range(self.nChannels):
                              strAux1 = "SNR Channel "+ str(i)
                              strAux2 = "Radial Velocity Channel "+ str(i)
                              strAux3 = "Spectral Width Channel "+ str(i)
                              titleList = titleList + [strAux1,strAux2,strAux3]
                              cmapList = cmapList + ["jet","RdBu_r","jet"]
                              zminList = zminList + [SNRmin,vmin,wmin]
                              zmaxList = zmaxList + [SNRmax,vmax,wmax]
                          self.zminList = zminList
                          self.zmaxList = zmaxList
                          self.cmapList = cmapList
                          self.titleList = titleList
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                          self.SUB_EXP_CODE = sub_exp_code
                          self.PLOT_POS = plot_pos
                          self.isConfig = True
                      str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
                      for i in range(self.nplots):
                          title = self.titleList[i] + ": " +str_datetime
                          axes = self.axesList[i]
                          axes.pcolor(x, y, z[i,:].T,
                                      xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=self.zminList[i], zmax=self.zmaxList[i],
                                      xlabel=xlabel, ylabel=ylabel, title=title, colormap=self.cmapList[i],ticksize=9, cblabel='')
                      self.draw()
                      if figfile == None:
                          str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
                          name = str_datetime
                          if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
                              name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
                          figfile = self.getFilename(name)
                      self.save(figpath=figpath,
                                figfile=figfile,
                                save=save,
                                ftp=ftp,
                                wr_period=wr_period,
                                thisDatetime=thisDatetime)

schainpy/model/graphics/jroplot_spectra.py +23 -16

              '''
              Created on Jul 9, 2014
              @author: roj-idl71
              '''
              import os
              import datetime
              import numpy
              from figure import Figure, isRealtime, isTimeInHourRange
              from plotting_codes import *
              class SpectraPlot(Figure):
                  isConfig = None
                  __nsubplots = None
                  WIDTHPROF = None
                  HEIGHTPROF = None
                  PREFIX = 'spc'
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.isConfig = False
                      self.__nsubplots = 1
                      self.WIDTH = 1000
                      self.HEIGHT = 1000
                      self.WIDTHPROF = 120
                      self.HEIGHTPROF = 0
                      self.counter_imagwr = 0
                      self.PLOT_CODE = SPEC_CODE
                      self.FTP_WEI = None
                      self.EXP_CODE = None
                      self.SUB_EXP_CODE = None
                      self.PLOT_POS = None
                      self.__xfilter_ena = False
                      self.__yfilter_ena = False
                  def getSubplots(self):
                      ncol = int(numpy.sqrt(self.nplots)+0.9)
                      nrow = int(self.nplots*1./ncol + 0.9)
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, showprofile=True, show=True):
                      self.__showprofile = showprofile
                      self.nplots = nplots
                      ncolspan = 1
                      colspan = 1
                      if showprofile:
                          ncolspan = 3
                          colspan = 2
                          self.__nsubplots = 2
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH + self.WIDTHPROF,
                                        heightplot = self.HEIGHT + self.HEIGHTPROF,
                                        show=show)
                      nrow, ncol = self.getSubplots()
                      counter = 0
                      for y in range(nrow):
                          for x in range(ncol):
                              if counter >= self.nplots:
                                  break
                              self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
                              if showprofile:
                                  self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
                              counter += 1
                  def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
                          save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
-                         xaxis="velocity", **kwargs):
+                         xaxis="frequency", colormap='jet', normFactor=None):
                      """
                      Input:
                          dataOut         :
                          id        :
                          wintitle        :
                          channelList     :
                          showProfile     :
                          xmin            :    None,
                          xmax            :    None,
                          ymin            :    None,
                          ymax            :    None,
                          zmin            :    None,
                          zmax            :    None
                      """
-                     colormap = kwargs.get('colormap','jet')
                      if realtime:
                          if not(isRealtime(utcdatatime = dataOut.utctime)):
                              print 'Skipping this plot function'
                              return
                      if channelList == None:
                          channelIndexList = dataOut.channelIndexList
                      else:
                          channelIndexList = []
                          for channel in channelList:
                              if channel not in dataOut.channelList:
                                  raise ValueError, "Channel %d is not in dataOut.channelList" %channel
                              channelIndexList.append(dataOut.channelList.index(channel))
+                     if normFactor is None:
                      factor = dataOut.normFactor
+                     else:
+                         factor = normFactor
                      if xaxis == "frequency":
                          x = dataOut.getFreqRange(1)/1000.
                          xlabel = "Frequency (kHz)"
                      elif xaxis == "time":
                          x = dataOut.getAcfRange(1)
                          xlabel = "Time (ms)"
                      else:
                          x = dataOut.getVelRange(1)
                          xlabel = "Velocity (m/s)"
                      ylabel = "Range (Km)"
                      y = dataOut.getHeiRange()
                      z = dataOut.data_spc/factor
                      z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
                      zdB = 10*numpy.log10(z)
                      avg = numpy.average(z, axis=1)
                      avgdB = 10*numpy.log10(avg)
                      noise = dataOut.getNoise()/factor
                      noisedB = 10*numpy.log10(noise)
                      thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Spectra"
                      if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
                          title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
                      if not self.isConfig:
                          nplots = len(channelIndexList)
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     showprofile=showprofile,
                                     show=show)
                          if xmin == None: xmin = numpy.nanmin(x)
                          if xmax == None: xmax = numpy.nanmax(x)
                          if ymin == None: ymin = numpy.nanmin(y)
                          if ymax == None: ymax = numpy.nanmax(y)
                          if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
                          if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                          self.SUB_EXP_CODE = sub_exp_code
                          self.PLOT_POS = plot_pos
                          self.isConfig = True
                      self.setWinTitle(title)
                      for i in range(self.nplots):
                          index = channelIndexList[i]
                          str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
                          title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[index], noisedB[index], str_datetime)
                          if len(dataOut.beam.codeList) != 0:
                              title = "Ch%d:%4.2fdB,%2.2f,%2.2f:%s" %(dataOut.channelList[index], noisedB[index], dataOut.beam.azimuthList[index], dataOut.beam.zenithList[index], str_datetime)
                          axes = self.axesList[i*self.__nsubplots]
                          axes.pcolor(x, y, zdB[index,:,:],
                                      xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
                                      xlabel=xlabel, ylabel=ylabel, title=title, colormap=colormap,
                                      ticksize=9, cblabel='')
                          if self.__showprofile:
                              axes = self.axesList[i*self.__nsubplots +1]
                              axes.pline(avgdB[index,:], y,
                                      xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
                                      xlabel='dB', ylabel='', title='',
                                      ytick_visible=False,
                                      grid='x')
                              noiseline = numpy.repeat(noisedB[index], len(y))
                              axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
                      self.draw()
                      if figfile == None:
                          str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
                          name = str_datetime
                          if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
                              name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
                          figfile = self.getFilename(name)
                      self.save(figpath=figpath,
                                figfile=figfile,
                                save=save,
                                ftp=ftp,
                                wr_period=wr_period,
                                thisDatetime=thisDatetime)
              class CrossSpectraPlot(Figure):
                  isConfig = None
                  __nsubplots = None
                  WIDTH = None
                  HEIGHT = None
                  WIDTHPROF = None
                  HEIGHTPROF = None
                  PREFIX = 'cspc'
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.isConfig = False
                      self.__nsubplots = 4
                      self.counter_imagwr = 0
                      self.WIDTH = 250
                      self.HEIGHT = 250
                      self.WIDTHPROF = 0
                      self.HEIGHTPROF = 0
                      self.PLOT_CODE = CROSS_CODE
                      self.FTP_WEI = None
                      self.EXP_CODE = None
                      self.SUB_EXP_CODE = None
                      self.PLOT_POS = None
                  def getSubplots(self):
                      ncol = 4
                      nrow = self.nplots
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, showprofile=True, show=True):
                      self.__showprofile = showprofile
                      self.nplots = nplots
                      ncolspan = 1
                      colspan = 1
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH + self.WIDTHPROF,
                                        heightplot = self.HEIGHT + self.HEIGHTPROF,
                                        show=True)
                      nrow, ncol = self.getSubplots()
                      counter = 0
                      for y in range(nrow):
                          for x in range(ncol):
                              self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
                              counter += 1
                  def run(self, dataOut, id, wintitle="", pairsList=None,
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
                          coh_min=None, coh_max=None, phase_min=None, phase_max=None,
                          save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
                          power_cmap='jet', coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
                          server=None, folder=None, username=None, password=None,
-                         ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0,
+                         ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None,
                          xaxis='frequency'):
                      """
                      Input:
                          dataOut         :
                          id        :
                          wintitle        :
                          channelList     :
                          showProfile     :
                          xmin            :    None,
                          xmax            :    None,
                          ymin            :    None,
                          ymax            :    None,
                          zmin            :    None,
                          zmax            :    None
                      """
                      if pairsList == None:
                          pairsIndexList = dataOut.pairsIndexList
                      else:
                          pairsIndexList = []
                          for pair in pairsList:
                              if pair not in dataOut.pairsList:
                                  raise ValueError, "Pair %s is not in dataOut.pairsList" %str(pair)
                              pairsIndexList.append(dataOut.pairsList.index(pair))
                      if not pairsIndexList:
                          return
                      if len(pairsIndexList) > 4:
                          pairsIndexList = pairsIndexList[0:4]
+                     if normFactor is None:
                      factor = dataOut.normFactor
+                     else:
+                         factor = normFactor
                      x = dataOut.getVelRange(1)
                      y = dataOut.getHeiRange()
                      z = dataOut.data_spc[:,:,:]/factor
                      z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
                      noise = dataOut.noise/factor
                      zdB = 10*numpy.log10(z)
                      noisedB = 10*numpy.log10(noise)
                      if coh_min == None:
                          coh_min = 0.0
                      if coh_max == None:
                          coh_max = 1.0
                      if phase_min == None:
                          phase_min = -180
                      if phase_max == None:
                          phase_max = 180
                      #thisDatetime = dataOut.datatime
                      thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Cross-Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
              #         xlabel = "Velocity (m/s)"
                      ylabel = "Range (Km)"
                      if xaxis == "frequency":
                          x = dataOut.getFreqRange(1)/1000.
                          xlabel = "Frequency (kHz)"
                      elif xaxis == "time":
                          x = dataOut.getAcfRange(1)
                          xlabel = "Time (ms)"
                      else:
                          x = dataOut.getVelRange(1)
                          xlabel = "Velocity (m/s)"
                      if not self.isConfig:
                          nplots = len(pairsIndexList)
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     showprofile=False,
                                     show=show)
                          avg = numpy.abs(numpy.average(z, axis=1))
                          avgdB = 10*numpy.log10(avg)
                          if xmin == None: xmin = numpy.nanmin(x)
                          if xmax == None: xmax = numpy.nanmax(x)
                          if ymin == None: ymin = numpy.nanmin(y)
                          if ymax == None: ymax = numpy.nanmax(y)
                          if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
                          if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                          self.SUB_EXP_CODE = sub_exp_code
                          self.PLOT_POS = plot_pos
                          self.isConfig = True
                      self.setWinTitle(title)
                      for i in range(self.nplots):
                          pair = dataOut.pairsList[pairsIndexList[i]]
                          chan_index0 = dataOut.channelList.index(pair[0])
                          chan_index1 = dataOut.channelList.index(pair[1])
                          str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
                          title = "Ch%d: %4.2fdB: %s" %(pair[0], noisedB[chan_index0], str_datetime)
                          zdB = 10.*numpy.log10(dataOut.data_spc[chan_index0,:,:]/factor)
                          axes0 = self.axesList[i*self.__nsubplots]
                          axes0.pcolor(x, y, zdB,
                                      xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
                                      xlabel=xlabel, ylabel=ylabel, title=title,
                                      ticksize=9, colormap=power_cmap, cblabel='')
                          title = "Ch%d: %4.2fdB: %s" %(pair[1], noisedB[chan_index1], str_datetime)
                          zdB = 10.*numpy.log10(dataOut.data_spc[chan_index1,:,:]/factor)
                          axes0 = self.axesList[i*self.__nsubplots+1]
                          axes0.pcolor(x, y, zdB,
                                      xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
                                      xlabel=xlabel, ylabel=ylabel, title=title,
                                      ticksize=9, colormap=power_cmap, cblabel='')
                          coherenceComplex = dataOut.data_cspc[pairsIndexList[i],:,:]/numpy.sqrt(dataOut.data_spc[chan_index0,:,:]*dataOut.data_spc[chan_index1,:,:])
                          coherence = numpy.abs(coherenceComplex)
              #            phase = numpy.arctan(-1*coherenceComplex.imag/coherenceComplex.real)*180/numpy.pi
                          phase = numpy.arctan2(coherenceComplex.imag, coherenceComplex.real)*180/numpy.pi
                          title = "Coherence Ch%d * Ch%d" %(pair[0], pair[1])
                          axes0 = self.axesList[i*self.__nsubplots+2]
                          axes0.pcolor(x, y, coherence,
                                      xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=coh_min, zmax=coh_max,
                                      xlabel=xlabel, ylabel=ylabel, title=title,
                                      ticksize=9, colormap=coherence_cmap, cblabel='')
                          title = "Phase Ch%d * Ch%d" %(pair[0], pair[1])
                          axes0 = self.axesList[i*self.__nsubplots+3]
                          axes0.pcolor(x, y, phase,
                                      xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=phase_min, zmax=phase_max,
                                      xlabel=xlabel, ylabel=ylabel, title=title,
                                      ticksize=9, colormap=phase_cmap, cblabel='')
                      self.draw()
                      self.save(figpath=figpath,
                                figfile=figfile,
                                save=save,
                                ftp=ftp,
                                wr_period=wr_period,
                                thisDatetime=thisDatetime)
              class RTIPlot(Figure):
                  __isConfig = None
                  __nsubplots = None
                  WIDTHPROF = None
                  HEIGHTPROF = None
                  PREFIX = 'rti'
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.timerange = None
                      self.isConfig = False
                      self.__nsubplots = 1
                      self.WIDTH = 800
                      self.HEIGHT = 180
                      self.WIDTHPROF = 120
                      self.HEIGHTPROF = 0
                      self.counter_imagwr = 0
                      self.PLOT_CODE = RTI_CODE
                      self.FTP_WEI = None
                      self.EXP_CODE = None
                      self.SUB_EXP_CODE = None
                      self.PLOT_POS = None
                      self.tmin = None
                      self.tmax = None
                      self.xmin = None
                      self.xmax = None
                      self.figfile = None
                  def getSubplots(self):
                      ncol = 1
                      nrow = self.nplots
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, showprofile=True, show=True):
                      self.__showprofile = showprofile
                      self.nplots = nplots
                      ncolspan = 1
                      colspan = 1
                      if showprofile:
                          ncolspan = 7
                          colspan = 6
                          self.__nsubplots = 2
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH + self.WIDTHPROF,
                                        heightplot = self.HEIGHT + self.HEIGHTPROF,
                                        show=show)
                      nrow, ncol = self.getSubplots()
                      counter = 0
                      for y in range(nrow):
                          for x in range(ncol):
                              if counter >= self.nplots:
                                  break
                              self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
                              if showprofile:
                                  self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
                              counter += 1
                  def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
-                         timerange=None,
+                         timerange=None, colormap='jet',
                          save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
                          server=None, folder=None, username=None, password=None,
-                         ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, **kwargs):
+                         ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None, HEIGHT=None):
                      """
                      Input:
                          dataOut         :
                          id        :
                          wintitle        :
                          channelList     :
                          showProfile     :
                          xmin            :    None,
                          xmax            :    None,
                          ymin            :    None,
                          ymax            :    None,
                          zmin            :    None,
                          zmax            :    None
                      """
-                     colormap = kwargs.get('colormap', 'jet')
+                     #colormap = kwargs.get('colormap', 'jet')
+                     if HEIGHT is not None:
+                         self.HEIGHT  = HEIGHT
                      if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
                          return
                      if channelList == None:
                          channelIndexList = dataOut.channelIndexList
                      else:
                          channelIndexList = []
                          for channel in channelList:
                              if channel not in dataOut.channelList:
                                  raise ValueError, "Channel %d is not in dataOut.channelList"
                              channelIndexList.append(dataOut.channelList.index(channel))
-                     if hasattr(dataOut, 'normFactor'):
+                     if normFactor is None:
                          factor = dataOut.normFactor
                      else:
-                         factor = 1
+                         factor = normFactor
              #         factor = dataOut.normFactor
                      x = dataOut.getTimeRange()
                      y = dataOut.getHeiRange()
-             #         z = dataOut.data_spc/factor
-             #         z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
-             #         avg = numpy.average(z, axis=1)
-             #         avgdB = 10.*numpy.log10(avg)
-                     avgdB = dataOut.getPower()
+                     z = dataOut.data_spc/factor
+                     z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
+                     avg = numpy.average(z, axis=1)
+                     avgdB = 10.*numpy.log10(avg)
+                     # avgdB = dataOut.getPower()
                      thisDatetime = dataOut.datatime
              #         thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " RTI" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
                      xlabel = ""
                      ylabel = "Range (Km)"
                      update_figfile = False
                      if dataOut.ltctime >= self.xmax:
                          self.counter_imagwr = wr_period
                          self.isConfig = False
                          update_figfile = True
                      if not self.isConfig:
                          nplots = len(channelIndexList)
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     showprofile=showprofile,
                                     show=show)
                          if timerange != None:
                              self.timerange = timerange
                          self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
                          noise = dataOut.noise/factor
                          noisedB = 10*numpy.log10(noise)
                          if ymin == None: ymin = numpy.nanmin(y)
                          if ymax == None: ymax = numpy.nanmax(y)
                          if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
                          if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                          self.SUB_EXP_CODE = sub_exp_code
                          self.PLOT_POS = plot_pos
                          self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
                          self.isConfig = True
                          self.figfile = figfile
                          update_figfile = True
                      self.setWinTitle(title)
                      for i in range(self.nplots):
                          index = channelIndexList[i]
                          title = "Channel %d: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
                          if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
                              title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
                          axes = self.axesList[i*self.__nsubplots]
                          zdB = avgdB[index].reshape((1,-1))
                          axes.pcolorbuffer(x, y, zdB,
                                      xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
                                      xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
                                      ticksize=9, cblabel='', cbsize="1%", colormap=colormap)
                          if self.__showprofile:
                              axes = self.axesList[i*self.__nsubplots +1]
                              axes.pline(avgdB[index], y,
                                      xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
                                      xlabel='dB', ylabel='', title='',
                                      ytick_visible=False,
                                      grid='x')
                      self.draw()
                      self.save(figpath=figpath,
                                figfile=figfile,
                                save=save,
                                ftp=ftp,
                                wr_period=wr_period,
                                thisDatetime=thisDatetime,
                                update_figfile=update_figfile)
              class CoherenceMap(Figure):
                  isConfig = None
                  __nsubplots = None
                  WIDTHPROF = None
                  HEIGHTPROF = None
                  PREFIX = 'cmap'
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.timerange = 2*60*60
                      self.isConfig = False
                      self.__nsubplots = 1
                      self.WIDTH = 800
                      self.HEIGHT = 180
                      self.WIDTHPROF = 120
                      self.HEIGHTPROF = 0
                      self.counter_imagwr = 0
                      self.PLOT_CODE = COH_CODE
                      self.FTP_WEI = None
                      self.EXP_CODE = None
                      self.SUB_EXP_CODE = None
                      self.PLOT_POS = None
                      self.counter_imagwr = 0
                      self.xmin = None
                      self.xmax = None
                  def getSubplots(self):
                      ncol = 1
                      nrow = self.nplots*2
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, showprofile=True, show=True):
                      self.__showprofile = showprofile
                      self.nplots = nplots
                      ncolspan = 1
                      colspan = 1
                      if showprofile:
                          ncolspan = 7
                          colspan = 6
                          self.__nsubplots = 2
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH + self.WIDTHPROF,
                                        heightplot = self.HEIGHT + self.HEIGHTPROF,
                                        show=True)
                      nrow, ncol = self.getSubplots()
                      for y in range(nrow):
                          for x in range(ncol):
                              self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
                              if showprofile:
                                  self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
                  def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
                          timerange=None, phase_min=None, phase_max=None,
                          save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
                          coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
                      if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
                          return
                      if pairsList == None:
                          pairsIndexList = dataOut.pairsIndexList
                      else:
                          pairsIndexList = []
                          for pair in pairsList:
                              if pair not in dataOut.pairsList:
                                  raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair)
                              pairsIndexList.append(dataOut.pairsList.index(pair))
                      if pairsIndexList == []:
                          return
                      if len(pairsIndexList) > 4:
                          pairsIndexList = pairsIndexList[0:4]
                      if phase_min == None:
                          phase_min = -180
                      if phase_max == None:
                          phase_max = 180
                      x = dataOut.getTimeRange()
                      y = dataOut.getHeiRange()
                      thisDatetime = dataOut.datatime
                      title = wintitle + " CoherenceMap" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
                      xlabel = ""
                      ylabel = "Range (Km)"
                      update_figfile = False
                      if not self.isConfig:
                          nplots = len(pairsIndexList)
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     showprofile=showprofile,
                                     show=show)
                          if timerange != None:
                              self.timerange = timerange
                          self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
                          if ymin == None: ymin = numpy.nanmin(y)
                          if ymax == None: ymax = numpy.nanmax(y)
                          if zmin == None: zmin = 0.
                          if zmax == None: zmax = 1.
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                          self.SUB_EXP_CODE = sub_exp_code
                          self.PLOT_POS = plot_pos
                          self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
                          self.isConfig = True
                          update_figfile = True
                      self.setWinTitle(title)
                      for i in range(self.nplots):
                          pair = dataOut.pairsList[pairsIndexList[i]]
                          ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i],:,:],axis=0)
                          powa = numpy.average(dataOut.data_spc[pair[0],:,:],axis=0)
                          powb = numpy.average(dataOut.data_spc[pair[1],:,:],axis=0)
                          avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
                          coherence = numpy.abs(avgcoherenceComplex)
                          z = coherence.reshape((1,-1))
                          counter = 0
                          title = "Coherence Ch%d * Ch%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
                          axes = self.axesList[i*self.__nsubplots*2]
                          axes.pcolorbuffer(x, y, z,
                                      xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
                                      xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
                                      ticksize=9, cblabel='', colormap=coherence_cmap, cbsize="1%")
                          if self.__showprofile:
                              counter += 1
                              axes = self.axesList[i*self.__nsubplots*2 + counter]
                              axes.pline(coherence, y,
                                      xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
                                      xlabel='', ylabel='', title='', ticksize=7,
                                      ytick_visible=False, nxticks=5,
                                      grid='x')
                          counter += 1
                          phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
                          z = phase.reshape((1,-1))
                          title = "Phase Ch%d * Ch%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
                          axes = self.axesList[i*self.__nsubplots*2 + counter]
                          axes.pcolorbuffer(x, y, z,
                                      xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=phase_min, zmax=phase_max,
                                      xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
                                      ticksize=9, cblabel='', colormap=phase_cmap, cbsize="1%")
                          if self.__showprofile:
                              counter += 1
                              axes = self.axesList[i*self.__nsubplots*2 + counter]
                              axes.pline(phase, y,
                                      xmin=phase_min, xmax=phase_max, ymin=ymin, ymax=ymax,
                                      xlabel='', ylabel='', title='', ticksize=7,
                                      ytick_visible=False, nxticks=4,
                                      grid='x')
                      self.draw()
                      if dataOut.ltctime >= self.xmax:
                          self.counter_imagwr = wr_period
                          self.isConfig = False
                          update_figfile = True
                      self.save(figpath=figpath,
                                figfile=figfile,
                                save=save,
                                ftp=ftp,
                                wr_period=wr_period,
                                thisDatetime=thisDatetime,
                                update_figfile=update_figfile)
              class PowerProfilePlot(Figure):
                  isConfig = None
                  __nsubplots = None
                  WIDTHPROF = None
                  HEIGHTPROF = None
                  PREFIX = 'spcprofile'
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.isConfig = False
                      self.__nsubplots = 1
                      self.PLOT_CODE = POWER_CODE
                      self.WIDTH = 300
                      self.HEIGHT = 500
                      self.counter_imagwr = 0
                  def getSubplots(self):
                      ncol = 1
                      nrow = 1
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, show):
                      self.nplots = nplots
                      ncolspan = 1
                      colspan = 1
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH,
                                        heightplot = self.HEIGHT,
                                        show=show)
                      nrow, ncol = self.getSubplots()
                      counter = 0
                      for y in range(nrow):
                          for x in range(ncol):
                              self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
                  def run(self, dataOut, id, wintitle="", channelList=None,
                          xmin=None, xmax=None, ymin=None, ymax=None,
                          save=False, figpath='./', figfile=None, show=True,
                          ftp=False, wr_period=1, server=None,
                          folder=None, username=None, password=None):
                      if channelList == None:
                          channelIndexList = dataOut.channelIndexList
                          channelList = dataOut.channelList
                      else:
                          channelIndexList = []
                          for channel in channelList:
                              if channel not in dataOut.channelList:
                                  raise ValueError, "Channel %d is not in dataOut.channelList"
                              channelIndexList.append(dataOut.channelList.index(channel))
                      factor = dataOut.normFactor
                      y = dataOut.getHeiRange()
                      #for voltage
                      if dataOut.type == 'Voltage':
                          x = dataOut.data[channelIndexList,:] * numpy.conjugate(dataOut.data[channelIndexList,:])
                          x = x.real
                          x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
                      #for spectra
                      if dataOut.type == 'Spectra':
                          x = dataOut.data_spc[channelIndexList,:,:]/factor
                          x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
                          x = numpy.average(x, axis=1)
                      xdB = 10*numpy.log10(x)
                      thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Power Profile %s" %(thisDatetime.strftime("%d-%b-%Y"))
                      xlabel = "dB"
                      ylabel = "Range (Km)"
                      if not self.isConfig:
                          nplots = 1
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     show=show)
                          if ymin == None: ymin = numpy.nanmin(y)
                          if ymax == None: ymax = numpy.nanmax(y)
                          if xmin == None: xmin = numpy.nanmin(xdB)*0.9
                          if xmax == None: xmax = numpy.nanmax(xdB)*1.1
                          self.isConfig = True
                      self.setWinTitle(title)
                      title = "Power Profile: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
                      axes = self.axesList[0]
                      legendlabels = ["channel %d"%x for x in channelList]
                      axes.pmultiline(xdB, y,
                              xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
                              xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
                              ytick_visible=True, nxticks=5,
                              grid='x')
                      self.draw()
                      self.save(figpath=figpath,
                                figfile=figfile,
                                save=save,
                                ftp=ftp,
                                wr_period=wr_period,
                                thisDatetime=thisDatetime)
              class SpectraCutPlot(Figure):
                  isConfig = None
                  __nsubplots = None
                  WIDTHPROF = None
                  HEIGHTPROF = None
                  PREFIX = 'spc_cut'
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.isConfig = False
                      self.__nsubplots = 1
                      self.PLOT_CODE = POWER_CODE
                      self.WIDTH = 700
                      self.HEIGHT = 500
                      self.counter_imagwr = 0
                  def getSubplots(self):
                      ncol = 1
                      nrow = 1
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, show):
                      self.nplots = nplots
                      ncolspan = 1
                      colspan = 1
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH,
                                        heightplot = self.HEIGHT,
                                        show=show)
                      nrow, ncol = self.getSubplots()
                      counter = 0
                      for y in range(nrow):
                          for x in range(ncol):
                              self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
                  def run(self, dataOut, id, wintitle="", channelList=None,
                          xmin=None, xmax=None, ymin=None, ymax=None,
                          save=False, figpath='./', figfile=None, show=True,
                          ftp=False, wr_period=1, server=None,
                          folder=None, username=None, password=None,
                          xaxis="frequency"):
                      if channelList == None:
                          channelIndexList = dataOut.channelIndexList
                          channelList = dataOut.channelList
                      else:
                          channelIndexList = []
                          for channel in channelList:
                              if channel not in dataOut.channelList:
                                  raise ValueError, "Channel %d is not in dataOut.channelList"
                              channelIndexList.append(dataOut.channelList.index(channel))
                      factor = dataOut.normFactor
                      y = dataOut.getHeiRange()
                      z = dataOut.data_spc/factor
                      z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
                      hei_index = numpy.arange(25)*3 + 20
                      if xaxis == "frequency":
                          x = dataOut.getFreqRange()/1000.
                          zdB = 10*numpy.log10(z[0,:,hei_index])
                          xlabel = "Frequency (kHz)"
                          ylabel = "Power (dB)"
                      elif xaxis == "time":
                          x = dataOut.getAcfRange()
                          zdB = z[0,:,hei_index]
                          xlabel = "Time (ms)"
                          ylabel = "ACF"
                      else:
                          x = dataOut.getVelRange()
                          zdB = 10*numpy.log10(z[0,:,hei_index])
                          xlabel = "Velocity (m/s)"
                          ylabel = "Power (dB)"
                      thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Range Cuts %s" %(thisDatetime.strftime("%d-%b-%Y"))
                      if not self.isConfig:
                          nplots = 1
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     show=show)
                          if xmin == None: xmin = numpy.nanmin(x)*0.9
                          if xmax == None: xmax = numpy.nanmax(x)*1.1
                          if ymin == None: ymin = numpy.nanmin(zdB)
                          if ymax == None: ymax = numpy.nanmax(zdB)
                          self.isConfig = True
                      self.setWinTitle(title)
                      title = "Spectra Cuts: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
                      axes = self.axesList[0]
                      legendlabels = ["Range = %dKm" %y[i] for i in hei_index]
                      axes.pmultilineyaxis( x, zdB,
                              xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
                              xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
                              ytick_visible=True, nxticks=5,
                              grid='x')
                      self.draw()
                      self.save(figpath=figpath,
                                figfile=figfile,
                                save=save,
                                ftp=ftp,
                                wr_period=wr_period,
                                thisDatetime=thisDatetime)
              class Noise(Figure):
                  isConfig = None
                  __nsubplots = None
                  PREFIX = 'noise'
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.timerange = 24*60*60
                      self.isConfig = False
                      self.__nsubplots = 1
                      self.counter_imagwr = 0
                      self.WIDTH = 800
                      self.HEIGHT = 400
                      self.WIDTHPROF = 120
                      self.HEIGHTPROF = 0
                      self.xdata = None
                      self.ydata = None
                      self.PLOT_CODE = NOISE_CODE
                      self.FTP_WEI = None
                      self.EXP_CODE = None
                      self.SUB_EXP_CODE = None
                      self.PLOT_POS = None
                      self.figfile = None
                      self.xmin = None
                      self.xmax = None
                  def getSubplots(self):
                      ncol = 1
                      nrow = 1
                      return nrow, ncol
                  def openfile(self, filename):
                      dirname = os.path.dirname(filename)
                      if not os.path.exists(dirname):
                          os.mkdir(dirname)
                      f = open(filename,'w+')
                      f.write('\n\n')
                      f.write('JICAMARCA RADIO OBSERVATORY - Noise \n')
                      f.write('DD MM YYYY  HH MM SS   Channel0    Channel1    Channel2    Channel3\n\n' )
                      f.close()
                  def save_data(self, filename_phase, data, data_datetime):
                      f=open(filename_phase,'a')
                      timetuple_data = data_datetime.timetuple()
                      day = str(timetuple_data.tm_mday)
                      month = str(timetuple_data.tm_mon)
                      year = str(timetuple_data.tm_year)
                      hour = str(timetuple_data.tm_hour)
                      minute = str(timetuple_data.tm_min)
                      second = str(timetuple_data.tm_sec)
                      data_msg = ''
                      for i in range(len(data)):
                          data_msg += str(data[i]) + '  '
                      f.write(day+' '+month+' '+year+'  '+hour+' '+minute+' '+second+'   ' + data_msg + '\n')
                      f.close()
                  def setup(self, id, nplots, wintitle, showprofile=True, show=True):
                      self.__showprofile = showprofile
                      self.nplots = nplots
                      ncolspan = 7
                      colspan = 6
                      self.__nsubplots = 2
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH+self.WIDTHPROF,
                                        heightplot = self.HEIGHT+self.HEIGHTPROF,
                                        show=show)
                      nrow, ncol = self.getSubplots()
                      self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
                  def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
                          xmin=None, xmax=None, ymin=None, ymax=None,
                          timerange=None,
                          save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
                      if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
                          return
                      if channelList == None:
                          channelIndexList = dataOut.channelIndexList
                          channelList = dataOut.channelList
                      else:
                          channelIndexList = []
                          for channel in channelList:
                              if channel not in dataOut.channelList:
                                  raise ValueError, "Channel %d is not in dataOut.channelList"
                              channelIndexList.append(dataOut.channelList.index(channel))
                      x = dataOut.getTimeRange()
                      #y = dataOut.getHeiRange()
                      factor = dataOut.normFactor
                      noise = dataOut.noise[channelIndexList]/factor
                      noisedB = 10*numpy.log10(noise)
                      thisDatetime = dataOut.datatime
                      title = wintitle + " Noise" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
                      xlabel = ""
                      ylabel = "Intensity (dB)"
                      update_figfile = False
                      if not self.isConfig:
                          nplots = 1
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     showprofile=showprofile,
                                     show=show)
                          if timerange != None:
                              self.timerange = timerange
                          self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
                          if ymin == None: ymin = numpy.floor(numpy.nanmin(noisedB)) - 10.0
                          if ymax == None: ymax = numpy.nanmax(noisedB) + 10.0
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                          self.SUB_EXP_CODE = sub_exp_code
                          self.PLOT_POS = plot_pos
                          self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
                          self.isConfig = True
                          self.figfile = figfile
                          self.xdata = numpy.array([])
                          self.ydata = numpy.array([])
                          update_figfile = True
                          #open file beacon phase
                          path = '%s%03d' %(self.PREFIX, self.id)
                          noise_file = os.path.join(path,'%s.txt'%self.name)
                          self.filename_noise = os.path.join(figpath,noise_file)
                      self.setWinTitle(title)
                      title = "Noise %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
                      legendlabels = ["channel %d"%(idchannel) for idchannel in channelList]
                      axes = self.axesList[0]
                      self.xdata = numpy.hstack((self.xdata, x[0:1]))
                      if len(self.ydata)==0:
                          self.ydata = noisedB.reshape(-1,1)
                      else:
                          self.ydata = numpy.hstack((self.ydata, noisedB.reshape(-1,1)))
                      axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
                                  xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
                                  xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
                                  XAxisAsTime=True, grid='both'
                                  )
                      self.draw()
                      if dataOut.ltctime >= self.xmax:
                          self.counter_imagwr = wr_period
                          self.isConfig = False
                          update_figfile = True
                      self.save(figpath=figpath,
                                figfile=figfile,
                                save=save,
                                ftp=ftp,
                                wr_period=wr_period,
                                thisDatetime=thisDatetime,
                                update_figfile=update_figfile)
                      #store data beacon phase
                      if save:
                          self.save_data(self.filename_noise, noisedB, thisDatetime)
              class BeaconPhase(Figure):
                  __isConfig = None
                  __nsubplots = None
                  PREFIX = 'beacon_phase'
                  def __init__(self, **kwargs):
                      Figure.__init__(self, **kwargs)
                      self.timerange = 24*60*60
                      self.isConfig = False
                      self.__nsubplots = 1
                      self.counter_imagwr = 0
                      self.WIDTH = 800
                      self.HEIGHT = 400
                      self.WIDTHPROF = 120
                      self.HEIGHTPROF = 0
                      self.xdata = None
                      self.ydata = None
                      self.PLOT_CODE = BEACON_CODE
                      self.FTP_WEI = None
                      self.EXP_CODE = None
                      self.SUB_EXP_CODE = None
                      self.PLOT_POS = None
                      self.filename_phase = None
                      self.figfile = None
                      self.xmin = None
                      self.xmax = None
                  def getSubplots(self):
                      ncol = 1
                      nrow = 1
                      return nrow, ncol
                  def setup(self, id, nplots, wintitle, showprofile=True, show=True):
                      self.__showprofile = showprofile
                      self.nplots = nplots
                      ncolspan = 7
                      colspan = 6
                      self.__nsubplots = 2
                      self.createFigure(id = id,
                                        wintitle = wintitle,
                                        widthplot = self.WIDTH+self.WIDTHPROF,
                                        heightplot = self.HEIGHT+self.HEIGHTPROF,
                                        show=show)
                      nrow, ncol = self.getSubplots()
                      self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
                  def save_phase(self, filename_phase):
                      f = open(filename_phase,'w+')
                      f.write('\n\n')
                      f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
                      f.write('DD MM YYYY  HH MM SS   pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
                      f.close()
                  def save_data(self, filename_phase, data, data_datetime):
                      f=open(filename_phase,'a')
                      timetuple_data = data_datetime.timetuple()
                      day = str(timetuple_data.tm_mday)
                      month = str(timetuple_data.tm_mon)
                      year = str(timetuple_data.tm_year)
                      hour = str(timetuple_data.tm_hour)
                      minute = str(timetuple_data.tm_min)
                      second = str(timetuple_data.tm_sec)
                      f.write(day+' '+month+' '+year+'  '+hour+' '+minute+' '+second+'   '+str(data[0])+'   '+str(data[1])+'   '+str(data[2])+'   '+str(data[3])+'\n')
                      f.close()
                  def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
                          xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
                          timerange=None,
                          save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
                      if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
                          return
                      if pairsList == None:
                          pairsIndexList = dataOut.pairsIndexList[:10]
                      else:
                          pairsIndexList = []
                          for pair in pairsList:
                              if pair not in dataOut.pairsList:
                                  raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair)
                              pairsIndexList.append(dataOut.pairsList.index(pair))
                      if pairsIndexList == []:
                          return
              #         if len(pairsIndexList) > 4:
              #             pairsIndexList = pairsIndexList[0:4]
                      hmin_index = None
                      hmax_index = None
                      if hmin != None and hmax != None:
                          indexes = numpy.arange(dataOut.nHeights)
                          hmin_list = indexes[dataOut.heightList >= hmin]
                          hmax_list = indexes[dataOut.heightList <= hmax]
                          if hmin_list.any():
                              hmin_index = hmin_list[0]
                          if hmax_list.any():
                              hmax_index = hmax_list[-1]+1
                      x = dataOut.getTimeRange()
                      #y = dataOut.getHeiRange()
                      thisDatetime = dataOut.datatime
                      title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
                      xlabel = "Local Time"
                      ylabel = "Phase (degrees)"
                      update_figfile = False
                      nplots = len(pairsIndexList)
                      #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
                      phase_beacon = numpy.zeros(len(pairsIndexList))
                      for i in range(nplots):
                          pair = dataOut.pairsList[pairsIndexList[i]]
                          ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
                          powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
                          powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
                          avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
                          phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
                          #print "Phase %d%d" %(pair[0], pair[1])
                          #print phase[dataOut.beacon_heiIndexList]
                          if dataOut.beacon_heiIndexList:
                              phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
                          else:
                              phase_beacon[i] = numpy.average(phase)
                      if not self.isConfig:
                          nplots = len(pairsIndexList)
                          self.setup(id=id,
                                     nplots=nplots,
                                     wintitle=wintitle,
                                     showprofile=showprofile,
                                     show=show)
                          if timerange != None:
                              self.timerange = timerange
                          self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
                          if ymin == None: ymin = 0
                          if ymax == None: ymax = 360
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                          self.SUB_EXP_CODE = sub_exp_code
                          self.PLOT_POS = plot_pos
                          self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
                          self.isConfig = True
                          self.figfile = figfile
                          self.xdata = numpy.array([])
                          self.ydata = numpy.array([])
                          update_figfile = True
                          #open file beacon phase
                          path = '%s%03d' %(self.PREFIX, self.id)
                          beacon_file = os.path.join(path,'%s.txt'%self.name)
                          self.filename_phase = os.path.join(figpath,beacon_file)
                          #self.save_phase(self.filename_phase)
                      #store data beacon phase
                      #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
                      self.setWinTitle(title)
                      title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
                      legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
                      axes = self.axesList[0]
                      self.xdata = numpy.hstack((self.xdata, x[0:1]))
                      if len(self.ydata)==0:
                          self.ydata = phase_beacon.reshape(-1,1)
                      else:
                          self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
                      axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
                                  xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
                                  xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
                                  XAxisAsTime=True, grid='both'
                                  )
                      self.draw()
                      if dataOut.ltctime >= self.xmax:
                          self.counter_imagwr = wr_period
                          self.isConfig = False
                          update_figfile = True
                      self.save(figpath=figpath,
                                figfile=figfile,
                                save=save,
                                ftp=ftp,
                                wr_period=wr_period,
                                thisDatetime=thisDatetime,
                                update_figfile=update_figfile)

schainpy/model/io/__init__.py +6 0

              '''
              $Author: murco $
              $Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $
              '''
              from jroIO_voltage import *
              from jroIO_spectra import *
              from jroIO_heispectra import *
              from jroIO_usrp import *
              from jroIO_digitalRF import *
              from jroIO_kamisr import *
              from jroIO_param import *
              from jroIO_hf import *
+             from jroIO_madrigal import *
+             from bltrIO_param import *
+             from jroIO_bltr import *
+             from jroIO_mira35c import *

schainpy/model/io/jroIO_base.py +86 -47

              '''
              Created on Jul 2, 2014
              @author: roj-idl71
              '''
              import os
              import sys
              import glob
              import time
              import numpy
              import fnmatch
              import inspect
-             import time, datetime
+             import time
+             import datetime
              import traceback
              import zmq
              try:
                  from gevent import sleep
              except:
                  from time import sleep
              from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
              from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
              LOCALTIME = True
              def isNumber(cad):
                  """
                  Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                  Excepciones:
                      Si un determinado string no puede ser convertido a numero
                  Input:
                      str, string al cual se le analiza para determinar si convertible a un numero o no
                  Return:
                      True    :    si el string es uno numerico
                      False   :    no es un string numerico
                  """
                  try:
-                     float( cad )
                      return True
                  except:
                      return False
              def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
                  """
                  Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
                  Inputs:
                      filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                      startUTSeconds      :    fecha inicial del rango seleccionado. La fecha esta dada en
                                               segundos contados desde 01/01/1970.
                      endUTSeconds        :    fecha final del rango seleccionado. La fecha esta dada en
                                               segundos contados desde 01/01/1970.
                  Return:
                      Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                      fecha especificado, de lo contrario retorna False.
                  Excepciones:
                      Si el archivo no existe o no puede ser abierto
                      Si la cabecera no puede ser leida.
                  """
                  basicHeaderObj = BasicHeader(LOCALTIME)
                  try:
                      fp = open(filename,'rb')
                  except IOError:
                      print "The file %s can't be opened" %(filename)
                      return 0
                  sts = basicHeaderObj.read(fp)
                  fp.close()
                  if not(sts):
                      print "Skipping the file %s because it has not a valid header" %(filename)
                      return 0
                  if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
                      return 0
                  return 1
              def isTimeInRange(thisTime, startTime, endTime):
                  if endTime >= startTime:
                      if (thisTime < startTime) or (thisTime > endTime):
                          return 0
                      return 1
                  else:
                      if (thisTime < startTime) and (thisTime > endTime):
                          return 0
                      return 1
              def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
                  """
                  Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                  Inputs:
                      filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                      startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                      endDate            :    fecha final del rango seleccionado en formato datetime.date
                      startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                      endTime            :    tiempo final del rango seleccionado en formato datetime.time
                  Return:
                      Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                      fecha especificado, de lo contrario retorna False.
                  Excepciones:
                      Si el archivo no existe o no puede ser abierto
                      Si la cabecera no puede ser leida.
                  """
                  try:
                      fp = open(filename,'rb')
                  except IOError:
                      print "The file %s can't be opened" %(filename)
                      return None
                  firstBasicHeaderObj = BasicHeader(LOCALTIME)
                  systemHeaderObj = SystemHeader()
                  radarControllerHeaderObj = RadarControllerHeader()
                  processingHeaderObj = ProcessingHeader()
                  lastBasicHeaderObj = BasicHeader(LOCALTIME)
                  sts = firstBasicHeaderObj.read(fp)
                  if not(sts):
                      print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
                      return None
                  if not systemHeaderObj.read(fp):
                      return None
                  if not radarControllerHeaderObj.read(fp):
                      return None
                  if not processingHeaderObj.read(fp):
                      return None
                  filesize = os.path.getsize(filename)
                  offset = processingHeaderObj.blockSize + 24 #header size
                  if filesize <= offset:
                      print "[Reading] %s: This file has not enough data" %filename
                      return None
                  fp.seek(-offset, 2)
                  sts = lastBasicHeaderObj.read(fp)
                  fp.close()
                  thisDatetime = lastBasicHeaderObj.datatime
                  thisTime_last_block = thisDatetime.time()
                  thisDatetime = firstBasicHeaderObj.datatime
                  thisDate = thisDatetime.date()
                  thisTime_first_block = thisDatetime.time()
                  #General case
                  #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
                  #-----------o----------------------------o-----------
                  #       startTime                     endTime
                  if endTime >= startTime:
                      if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
                          return None
                      return thisDatetime
                  #If endTime < startTime then endTime belongs to the next day
                  #<<<<<<<<<<<o                            o>>>>>>>>>>>
                  #-----------o----------------------------o-----------
                  #        endTime                    startTime
                  if (thisDate == startDate) and (thisTime_last_block < startTime):
                      return None
                  if (thisDate == endDate) and (thisTime_first_block > endTime):
                      return None
                  if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
                      return None
                  return thisDatetime
              def isFolderInDateRange(folder, startDate=None, endDate=None):
                  """
                  Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                  Inputs:
                      folder            :    nombre completo del directorio.
                                             Su formato deberia ser "/path_root/?YYYYDDD"
                                              siendo:
                                                  YYYY    :    Anio         (ejemplo 2015)
                                                  DDD     :    Dia del anio (ejemplo 305)
                      startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                      endDate            :    fecha final del rango seleccionado en formato datetime.date
                  Return:
                      Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                      fecha especificado, de lo contrario retorna False.
                  Excepciones:
                      Si el directorio no tiene el formato adecuado
                  """
                  basename = os.path.basename(folder)
                  if not isRadarFolder(basename):
                      print "The folder %s has not the rigth format" %folder
                      return 0
                  if startDate and endDate:
                      thisDate = getDateFromRadarFolder(basename)
                      if thisDate < startDate:
                          return 0
                      if thisDate > endDate:
                          return 0
                  return 1
              def isFileInDateRange(filename, startDate=None, endDate=None):
                  """
                  Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                  Inputs:
                      filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                                             Su formato deberia ser "?YYYYDDDsss"
                                              siendo:
                                                  YYYY    :    Anio         (ejemplo 2015)
                                                  DDD     :    Dia del anio (ejemplo 305)
                                                  sss     :    set
                      startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                      endDate            :    fecha final del rango seleccionado en formato datetime.date
                  Return:
                      Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                      fecha especificado, de lo contrario retorna False.
                  Excepciones:
                      Si el archivo no tiene el formato adecuado
                  """
                  basename = os.path.basename(filename)
                  if not isRadarFile(basename):
                      print "The filename %s has not the rigth format" %filename
                      return 0
                  if startDate and endDate:
                      thisDate = getDateFromRadarFile(basename)
                      if thisDate < startDate:
                          return 0
                      if thisDate > endDate:
                          return 0
                  return 1
              def getFileFromSet(path, ext, set):
                  validFilelist = []
                  fileList = os.listdir(path)
                  # 0 1234 567 89A BCDE
                  # H YYYY DDD SSS .ext
                  for thisFile in fileList:
                      try:
                          year = int(thisFile[1:5])
-                         doy  = int(thisFile[5:8])
                      except:
                          continue
                      if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                          continue
                      validFilelist.append(thisFile)
-                 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
+                 myfile = fnmatch.filter(
+                     validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
                  if len(myfile)!= 0:
                      return myfile[0]
                  else:
                      filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
                      print 'the filename %s does not exist'%filename
                      print '...going to the last file: '
                  if validFilelist:
-                     validFilelist = sorted( validFilelist, key=str.lower )
                      return validFilelist[-1]
                  return None
              def getlastFileFromPath(path, ext):
                  """
                  Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
                  al final de la depuracion devuelve el ultimo file de la lista que quedo.
                  Input:
                      fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                      ext         :    extension de los files contenidos en una carpeta
                  Return:
                      El ultimo file de una determinada carpeta, no se considera el path.
                  """
                  validFilelist = []
                  fileList = os.listdir(path)
                  # 0 1234 567 89A BCDE
                  # H YYYY DDD SSS .ext
                  for thisFile in fileList:
                      year = thisFile[1:5]
                      if not isNumber(year):
                          continue
                      doy = thisFile[5:8]
                      if not isNumber(doy):
                          continue
                      year = int(year)
                      doy = int(doy)
                      if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                          continue
                      validFilelist.append(thisFile)
                  if validFilelist:
-                     validFilelist = sorted( validFilelist, key=str.lower )
                      return validFilelist[-1]
                  return None
              def checkForRealPath(path, foldercounter, year, doy, set, ext):
                  """
                  Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
                  Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
                  el path exacto de un determinado file.
                  Example    :
                      nombre correcto del file es  .../.../D2009307/P2009307367.ext
                      Entonces la funcion prueba con las siguientes combinaciones
                          .../.../y2009307367.ext
                          .../.../Y2009307367.ext
                          .../.../x2009307/y2009307367.ext
                          .../.../x2009307/Y2009307367.ext
                          .../.../X2009307/y2009307367.ext
                          .../.../X2009307/Y2009307367.ext
                      siendo para este caso, la ultima combinacion de letras, identica al file buscado
                  Return:
                      Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
                      caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
                      para el filename
                  """
                  fullfilename = None
                  find_flag = False
                  filename = None
                  prefixDirList = [None,'d','D']
                  if ext.lower() == ".r": #voltage
                      prefixFileList = ['d','D']
                  elif ext.lower() == ".pdata": #spectra
                      prefixFileList = ['p','P']
                  else:
                      return None, filename
                  #barrido por las combinaciones posibles
                  for prefixDir in prefixDirList:
                      thispath = path
                      if prefixDir != None:
                          #formo el nombre del directorio xYYYYDDD (x=d o x=D)
                          if foldercounter == 0:
-                             thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
+                             thispath = os.path.join(path, "%s%04d%03d" %
+                                                     (prefixDir, year, doy))
                          else:
-                             thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
+                             thispath = os.path.join(path, "%s%04d%03d_%02d" % (
+                                 prefixDir, year, doy, foldercounter))
                      for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
-                         filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
-                         fullfilename = os.path.join( thispath, filename ) #formo el path completo
+                         # formo el nombre del file xYYYYDDDSSS.ext
+                         filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
+                         fullfilename = os.path.join(
+                             thispath, filename)  # formo el path completo
-                         if os.path.exists( fullfilename ): #verifico que exista
                              find_flag = True
                              break
                      if find_flag:
                          break
                  if not(find_flag):
                      return None, filename
                  return fullfilename, filename
              def isRadarFolder(folder):
                  try:
                      year = int(folder[1:5])
                      doy = int(folder[5:8])
                  except:
                      return 0
                  return 1
              def isRadarFile(file):
-                     try:
-                         year = int(file[1:5])
-                         doy = int(file[5:8])
-                         set = int(file[8:11])
-                     except:
-                         return 0
-                     return 1
              def getDateFromRadarFile(file):
                  try:
                      year = int(file[1:5])
                      doy = int(file[5:8])
                      set = int(file[8:11])
                  except:
                      return None
                  thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
                  return thisDate
              def getDateFromRadarFolder(folder):
                  try:
                      year = int(folder[1:5])
                      doy = int(folder[5:8])
                  except:
                      return None
                  thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
                  return thisDate
              class JRODataIO:
                  c = 3E8
                  isConfig = False
                  basicHeaderObj = None
                  systemHeaderObj = None
                  radarControllerHeaderObj = None
                  processingHeaderObj = None
                  dtype = None
                  pathList = []
                  filenameList = []
                  filename = None
                  ext = None
                  flagIsNewFile = 1
                  flagDiscontinuousBlock = 0
                  flagIsNewBlock = 0
                  fp = None
                  firstHeaderSize = 0
                  basicHeaderSize = 24
                  versionFile = 1103
                  fileSize = None
              #     ippSeconds = None
                  fileSizeByHeader = None
                  fileIndex = None
                  profileIndex = None
                  blockIndex = None
                  nTotalBlocks = None
                  maxTimeStep = 30
                  lastUTTime = None
                  datablock = None
                  dataOut = None
                  blocksize = None
                  getByBlock = False
                  def __init__(self):
                      raise NotImplementedError
                  def run(self):
                      raise NotImplementedError
                  def getDtypeWidth(self):
                      dtype_index = get_dtype_index(self.dtype)
                      dtype_width = get_dtype_width(dtype_index)
                      return dtype_width
                  def getAllowedArgs(self):
                      return inspect.getargspec(self.run).args
              class JRODataReader(JRODataIO):
                  online = 0
                  realtime = 0
                  nReadBlocks = 0
-                 delay  = 10   #number of seconds waiting a new file
-                 nTries  = 3  #quantity tries
-                 nFiles = 3   #number of files for searching
                  path = None
                  foldercounter = 0
                  flagNoMoreFiles = 0
                  datetimeList = []
                  __isFirstTimeOnline = 1
                  __printInfo = True
                  profileIndex = None
                  nTxs = 1
                  txIndex = None
                  #Added--------------------
                  selBlocksize = None
                  selBlocktime = None
                  def __init__(self):
                      """
                      This class is used to find data files
                      Example:
                          reader = JRODataReader()
                          fileList = reader.findDataFiles()
                      """
                      pass
                  def createObjByDefault(self):
                      """
                      """
                      raise NotImplementedError
                  def getBlockDimension(self):
                      raise NotImplementedError
                  def searchFilesOffLine(self,
                                         path,
                                         startDate=None,
                                         endDate=None,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.r',
                                         cursor=None,
                                         skip=None,
                                         walk=True):
                      self.filenameList = []
                      self.datetimeList = []
                      pathList = []
-                     dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
+                     dateList, pathList = self.findDatafiles(
+                         path, startDate, endDate, expLabel, ext, walk, include_path=True)
                      if dateList == []:
                          return [], []
                      if len(dateList) > 1:
                          print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
                      else:
                          print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
                      filenameList = []
                      datetimeList = []
                      for thisPath in pathList:
                          fileList = glob.glob1(thisPath, "*%s" %ext)
                          fileList.sort()
                          skippedFileList = []
                          if cursor is not None and skip is not None:
                              if skip == 0:
                                  skippedFileList = []
                              else:
-                                 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
+                                 skippedFileList = fileList[cursor *
+                                                            skip: cursor * skip + skip]
                          else:
                              skippedFileList = fileList
                          for file in skippedFileList:
                              filename = os.path.join(thisPath,file)
                              if not isFileInDateRange(filename, startDate, endDate):
                                  continue
-                             thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
+                             thisDatetime = isFileInTimeRange(
+                                 filename, startDate, endDate, startTime, endTime)
                              if not(thisDatetime):
                                  continue
                              filenameList.append(filename)
                              datetimeList.append(thisDatetime)
                      if not(filenameList):
                          print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
                          return [], []
                      print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
                      print
                      # for i in range(len(filenameList)):
                      #     print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                      self.filenameList = filenameList
                      self.datetimeList = datetimeList
                      return pathList, filenameList
-                 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
                      """
                      Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
                      devuelve el archivo encontrado ademas de otros datos.
                      Input:
                          path           :    carpeta donde estan contenidos los files que contiene data
                          expLabel        :     Nombre del subexperimento (subfolder)
                          ext              :    extension de los files
                          walk        :    Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
                      Return:
                          directory   :    eL directorio donde esta el file encontrado
                          filename    :    el ultimo file de una determinada carpeta
                          year        :    el anho
                          doy         :    el numero de dia del anho
                          set         :    el set del archivo
                      """
                      if not os.path.isdir(path):
                          return None, None, None, None, None, None
                      dirList = []
                      if not walk:
                          fullpath = path
                          foldercounter = 0
                      else:
                          #Filtra solo los directorios
                          for thisPath in os.listdir(path):
                              if not os.path.isdir(os.path.join(path,thisPath)):
                                  continue
                              if not isRadarFolder(thisPath):
                                  continue
                              dirList.append(thisPath)
                          if not(dirList):
                              return None, None, None, None, None, None
-                         dirList = sorted( dirList, key=str.lower )
                          doypath = dirList[-1]
-                         foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
+                         foldercounter = int(doypath.split('_')[1]) if len(
+                             doypath.split('_')) > 1 else 0
                          fullpath = os.path.join(path, doypath, expLabel)
-                     print "[Reading] %s folder was found: " %(fullpath )
                      if set == None:
                          filename = getlastFileFromPath(fullpath, ext)
                      else:
                          filename = getFileFromSet(fullpath, ext, set)
                      if not(filename):
                          return None, None, None, None, None, None
                      print "[Reading] %s file was found" %(filename)
                      if not(self.__verifyFile(os.path.join(fullpath, filename))):
                          return None, None, None, None, None, None
-                     year = int( filename[1:5] )
-                     doy  = int( filename[5:8] )
-                     set  = int( filename[8:11] )
                      return fullpath, foldercounter, filename, year, doy, set
                  def __setNextFileOffline(self):
                      idFile = self.fileIndex
                      while (True):
                          idFile += 1
                          if not(idFile < len(self.filenameList)):
                              self.flagNoMoreFiles = 1
              #                 print "[Reading] No more Files"
                              return 0
                          filename = self.filenameList[idFile]
                          if not(self.__verifyFile(filename)):
                              continue
                          fileSize = os.path.getsize(filename)
                          fp = open(filename,'rb')
                          break
                      self.flagIsNewFile = 1
                      self.fileIndex = idFile
                      self.filename = filename
                      self.fileSize = fileSize
                      self.fp = fp
              #         print "[Reading] Setting the file: %s"%self.filename
                      return 1
                  def __setNextFileOnline(self):
                      """
                      Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
                      no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
                      siguientes.
                      Affected:
                          self.flagIsNewFile
                          self.filename
                          self.fileSize
                          self.fp
                          self.set
                          self.flagNoMoreFiles
                      Return:
 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
 : si el file fue abierto con exito y esta listo a ser leido
                      Excepciones:
                          Si un determinado file no puede ser abierto
                      """
                      nFiles = 0
                      fileOk_flag = False
                      firstTime_flag = True
                      self.set += 1
                      if self.set > 999:
                          self.set = 0
                          self.foldercounter += 1
                      #busca el 1er file disponible
-                     fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
+                     fullfilename, filename = checkForRealPath(
+                         self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
                      if fullfilename:
                          if self.__verifyFile(fullfilename, False):
                              fileOk_flag = True
                      #si no encuentra un file entonces espera y vuelve a buscar
                      if not(fileOk_flag):
-                         for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
+                         # busco en los siguientes self.nFiles+1 files posibles
+                         for nFiles in range(self.nFiles + 1):
                              if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
                                  tries = self.nTries
                              else:
                                  tries = 1 #si no es la 1era vez entonces solo lo hace una vez
-                             for nTries in range( tries ):
                                  if firstTime_flag:
-                                     print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
-                                     sleep( self.delay )
                                  else:
                                      print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
-                                 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
+                                 fullfilename, filename = checkForRealPath(
+                                     self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
                                  if fullfilename:
                                      if self.__verifyFile(fullfilename):
                                          fileOk_flag = True
                                          break
                              if fileOk_flag:
                                  break
                              firstTime_flag = False
                              print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
                              self.set += 1
-                             if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
+                             # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
+                             if nFiles == (self.nFiles - 1):
                                  self.set = 0
                                  self.doy += 1
                                  self.foldercounter = 0
                      if fileOk_flag:
-                         self.fileSize = os.path.getsize( fullfilename )
                          self.filename = fullfilename
                          self.flagIsNewFile = 1
-                         if self.fp != None: self.fp.close()
+                         if self.fp != None:
+                             self.fp.close()
                          self.fp = open(fullfilename, 'rb')
                          self.flagNoMoreFiles = 0
              #             print '[Reading] Setting the file: %s' % fullfilename
                      else:
                          self.fileSize = 0
                          self.filename = None
                          self.flagIsNewFile = 0
                          self.fp = None
                          self.flagNoMoreFiles = 1
              #             print '[Reading] No more files to read'
                      return fileOk_flag
                  def setNextFile(self):
                      if self.fp != None:
                          self.fp.close()
                      if self.online:
                          newFile = self.__setNextFileOnline()
                      else:
                          newFile = self.__setNextFileOffline()
                      if not(newFile):
                          print '[Reading] No more files to read'
                          return 0
                      if self.verbose:
                          print '[Reading] Setting the file: %s' % self.filename
                      self.__readFirstHeader()
                      self.nReadBlocks = 0
                      return 1
                  def __waitNewBlock(self):
                      """
                      Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                      Si el modo de lectura es OffLine siempre retorn 0
                      """
                      if not self.online:
                          return 0
                      if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                          return 0
                      currentPointer = self.fp.tell()
                      neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
-                     for nTries in range( self.nTries ):
                          self.fp.close()
-                         self.fp = open( self.filename, 'rb' )
-                         self.fp.seek( currentPointer )
-                         self.fileSize = os.path.getsize( self.filename )
                          currentSize = self.fileSize - currentPointer
-                         if ( currentSize >= neededSize ):
                              self.basicHeaderObj.read(self.fp)
                              return 1
                          if self.fileSize == self.fileSizeByHeader:
              #                self.flagEoF = True
                              return 0
                          print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
-                         sleep( self.delay )
                      return 0
                  def waitDataBlock(self,pointer_location):
                      currentPointer = pointer_location
                      neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
-                     for nTries in range( self.nTries ):
                          self.fp.close()
-                         self.fp = open( self.filename, 'rb' )
-                         self.fp.seek( currentPointer )
-                         self.fileSize = os.path.getsize( self.filename )
                          currentSize = self.fileSize - currentPointer
-                         if ( currentSize >= neededSize ):
                              return 1
                          print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
-                         sleep( self.delay )
                      return 0
                  def __jumpToLastBlock(self):
                      if not(self.__isFirstTimeOnline):
                          return
                      csize = self.fileSize - self.fp.tell()
                      blocksize = self.processingHeaderObj.blockSize
                      #salta el primer bloque de datos
                      if csize > self.processingHeaderObj.blockSize:
                          self.fp.seek(self.fp.tell() + blocksize)
                      else:
                          return
                      csize = self.fileSize - self.fp.tell()
                      neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                      while True:
                          if self.fp.tell()<self.fileSize:
                              self.fp.seek(self.fp.tell() + neededsize)
                          else:
                              self.fp.seek(self.fp.tell() - neededsize)
                              break
              #        csize = self.fileSize - self.fp.tell()
              #        neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
              #        factor = int(csize/neededsize)
              #        if factor > 0:
              #            self.fp.seek(self.fp.tell() + factor*neededsize)
                      self.flagIsNewFile = 0
                      self.__isFirstTimeOnline = 0
                  def __setNewBlock(self):
                      #if self.server is None:
                      if self.fp == None:
                          return 0
              #         if self.online:
-             #             self.__jumpToLastBlock()
                      if self.flagIsNewFile:
                          self.lastUTTime = self.basicHeaderObj.utc
                          return 1
                      if self.realtime:
                          self.flagDiscontinuousBlock = 1
                          if not(self.setNextFile()):
                              return 0
                          else:
-                             return 1
                      #if self.server is None:
                      currentSize = self.fileSize - self.fp.tell()
                      neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                      if (currentSize >= neededSize):
                          self.basicHeaderObj.read(self.fp)
                          self.lastUTTime = self.basicHeaderObj.utc
                          return 1
                      # else:
                      #     self.basicHeaderObj.read(self.zHeader)
                      #     self.lastUTTime = self.basicHeaderObj.utc
                      #     return 1
                      if self.__waitNewBlock():
                          self.lastUTTime = self.basicHeaderObj.utc
                          return 1
                      #if self.server is None:
                      if not(self.setNextFile()):
                          return 0
-                     deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
+                     deltaTime = self.basicHeaderObj.utc - self.lastUTTime
                      self.lastUTTime = self.basicHeaderObj.utc
                      self.flagDiscontinuousBlock = 0
                      if deltaTime > self.maxTimeStep:
                          self.flagDiscontinuousBlock = 1
                      return 1
                  def readNextBlock(self):
                      #Skip block out of startTime and endTime
-                     while True:
-                         if not(self.__setNewBlock()):
                              return 0
                          if not(self.readBlock()):
                              return 0
                          self.getBasicHeader()
                          if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
                              print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
                                                                            self.processingHeaderObj.dataBlocksPerFile,
                                                                            self.dataOut.datatime.ctime())
                              continue
                          break
                      if self.verbose:
                          print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
                                                                    self.processingHeaderObj.dataBlocksPerFile,
                                                                    self.dataOut.datatime.ctime())
                      return 1
                  def __readFirstHeader(self):
                      self.basicHeaderObj.read(self.fp)
                      self.systemHeaderObj.read(self.fp)
                      self.radarControllerHeaderObj.read(self.fp)
                      self.processingHeaderObj.read(self.fp)
                      self.firstHeaderSize = self.basicHeaderObj.size
-                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
+                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
+                                                PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
                      if datatype == 0:
                          datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
                      elif datatype == 1:
                          datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
                      elif datatype == 2:
                          datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
                      elif datatype == 3:
                          datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
                      elif datatype == 4:
                          datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
                      elif datatype == 5:
                          datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
                      else:
                          raise ValueError, 'Data type was not defined'
                      self.dtype = datatype_str
                      #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
-                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
+                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
+                         self.firstHeaderSize + self.basicHeaderSize * \
+                         (self.processingHeaderObj.dataBlocksPerFile - 1)
              #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
              #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                      self.getBlockDimension()
                  def __verifyFile(self, filename, msgFlag=True):
                      msg = None
                      try:
                          fp = open(filename, 'rb')
                      except IOError:
                          if msgFlag:
                              print "[Reading] File %s can't be opened" % (filename)
                          return False
                      currentPosition = fp.tell()
                      neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
                      if neededSize == 0:
                          basicHeaderObj = BasicHeader(LOCALTIME)
                          systemHeaderObj = SystemHeader()
                          radarControllerHeaderObj = RadarControllerHeader()
                          processingHeaderObj = ProcessingHeader()
-                         if not( basicHeaderObj.read(fp) ):
                              fp.close()
                              return False
-                         if not( systemHeaderObj.read(fp) ):
                              fp.close()
                              return False
-                         if not( radarControllerHeaderObj.read(fp) ):
                              fp.close()
                              return False
-                         if not( processingHeaderObj.read(fp) ):
                              fp.close()
                              return False
                          neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
                      else:
                          msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
                      fp.close()
                      fileSize = os.path.getsize(filename)
                      currentSize = fileSize - currentPosition
                      if currentSize < neededSize:
                          if msgFlag and (msg != None):
                              print msg
                          return False
                      return True
                  def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
                      path_empty = True
                      dateList = []
                      pathList = []
                      multi_path = path.split(',')
                      if not walk:
                          for single_path in multi_path:
                              if not os.path.isdir(single_path):
                                  continue
                              fileList = glob.glob1(single_path, "*"+ext)
                              if not fileList:
                                  continue
                              path_empty = False
                              fileList.sort()
                              for thisFile in fileList:
                                  if not os.path.isfile(os.path.join(single_path, thisFile)):
                                      continue
                                  if not isRadarFile(thisFile):
                                      continue
                                  if not isFileInDateRange(thisFile, startDate, endDate):
                                      continue
                                  thisDate = getDateFromRadarFile(thisFile)
                                  if thisDate in dateList:
                                      continue
                                  dateList.append(thisDate)
                                  pathList.append(single_path)
                      else:
                          for single_path in multi_path:
                              if not os.path.isdir(single_path):
                                  continue
                              dirList = []
                              for thisPath in os.listdir(single_path):
                                  if not os.path.isdir(os.path.join(single_path,thisPath)):
                                      continue
                                  if not isRadarFolder(thisPath):
                                      continue
                                  if not isFolderInDateRange(thisPath, startDate, endDate):
                                      continue
                                  dirList.append(thisPath)
                              if not dirList:
                                  continue
                              dirList.sort()
                              for thisDir in dirList:
                                  datapath = os.path.join(single_path, thisDir, expLabel)
                                  fileList = glob.glob1(datapath, "*"+ext)
                                  if not fileList:
                                      continue
                                  path_empty = False
                                  thisDate = getDateFromRadarFolder(thisDir)
                                  pathList.append(datapath)
                                  dateList.append(thisDate)
                      dateList.sort()
                      if walk:
                          pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
                      else:
                          pattern_path = multi_path[0]
                      if path_empty:
                          print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
                      else:
                          if not dateList:
                              print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
                      if include_path:
                          return dateList, pathList
                      return dateList
                  def setup(self,
-                             path=None,
-                             startDate=None,
-                             endDate=None,
-                             startTime=datetime.time(0,0,0),
-                             endTime=datetime.time(23,59,59),
-                             set=None,
-                             expLabel = "",
-                             ext = None,
-                             online = False,
-                             delay = 60,
-                             walk = True,
-                             getblock = False,
-                             nTxs = 1,
-                             realtime=False,
-                             blocksize=None,
-                             blocktime=None,
-                             skip=None,
-                             cursor=None,
-                             warnings=True,
-                             verbose=True,
-                             server=None,
-                             **kwargs):
+                           format=None,
+                           oneDDict=None,
+                           twoDDict=None,
+                           ind2DList=None):
                      if server is not None:
                          if 'tcp://' in server:
                              address = server
                          else:
                              address = 'ipc:///tmp/%s' % server
                          self.server = address
                          self.context = zmq.Context()
                          self.receiver = self.context.socket(zmq.PULL)
                          self.receiver.connect(self.server)
                          time.sleep(0.5)
                          print '[Starting] ReceiverData from {}'.format(self.server)
-                     else:
                          self.server = None
                          if path == None:
                              raise ValueError, "[Reading] The path is not valid"
                          if ext == None:
                              ext = self.ext
                          if online:
                              print "[Reading] Searching files in online mode..."
-                             for nTries in range( self.nTries ):
-                                 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
+                                 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
+                                     path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
                                  if fullpath:
                                      break
                                  print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
-                                 sleep( self.delay )
                              if not(fullpath):
                                  print "[Reading] There 'isn't any valid file in %s" % path
                                  return
                              self.year = year
-                             self.doy  = doy
-                             self.set  = set - 1
                              self.path = path
                              self.foldercounter = foldercounter
                              last_set = None
                          else:
                              print "[Reading] Searching files in offline mode ..."
                              pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                              startTime=startTime, endTime=endTime,
                                                                              set=set, expLabel=expLabel, ext=ext,
                                                                              walk=walk, cursor=cursor,
                                                                              skip=skip)
                              if not(pathList):
                                  self.fileIndex = -1
                                  self.pathList = []
                                  self.filenameList = []
                                  return
                              self.fileIndex = -1
                              self.pathList = pathList
                              self.filenameList = filenameList
                              file_name = os.path.basename(filenameList[-1])
                              basename, ext = os.path.splitext(file_name)
                              last_set = int(basename[-3:])
                          self.online = online
                          self.realtime = realtime
                          self.delay = delay
                          ext = ext.lower()
                          self.ext = ext
                          self.getByBlock = getblock
                          self.nTxs = nTxs
                          self.startTime = startTime
                          self.endTime = endTime
                          #Added-----------------
                          self.selBlocksize = blocksize
                          self.selBlocktime = blocktime
                          # Verbose-----------
                          self.verbose = verbose
                          self.warnings = warnings
                          if not(self.setNextFile()):
                              if (startDate!=None) and (endDate!=None):
                                  print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                              elif startDate != None:
                                  print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
                              else:
                                  print "[Reading] No files"
                                  self.fileIndex = -1
                                  self.pathList = []
                                  self.filenameList = []
                                  return
                  #         self.getBasicHeader()
                          if last_set != None:
-                             self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
+                             self.dataOut.last_block = last_set * \
+                                 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
                      return
                  def getBasicHeader(self):
-                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
+                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
+. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
                      self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                      self.dataOut.timeZone = self.basicHeaderObj.timeZone
                      self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
                      self.dataOut.errorCount = self.basicHeaderObj.errorCount
                      self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
                      self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
              #         self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
                  def getFirstHeader(self):
                      raise NotImplementedError
                  def getData(self):
                      raise NotImplementedError
                  def hasNotDataInBuffer(self):
                      raise NotImplementedError
                  def readBlock(self):
                      raise NotImplementedError
                  def isEndProcess(self):
                      return self.flagNoMoreFiles
                  def printReadBlocks(self):
                      print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
                  def printTotalBlocks(self):
                      print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
                  def printNumberOfBlock(self):
+                     'SPAM!'
-                     if self.flagIsNewBlock:
-                         print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
-                                                                   self.processingHeaderObj.dataBlocksPerFile,
-                                                                   self.dataOut.datatime.ctime())
+             #         if self.flagIsNewBlock:
+             #             print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
+             #                                                       self.processingHeaderObj.dataBlocksPerFile,
+             #                                                       self.dataOut.datatime.ctime())
                  def printInfo(self):
                      if self.__printInfo == False:
                          return
                      self.basicHeaderObj.printInfo()
                      self.systemHeaderObj.printInfo()
                      self.radarControllerHeaderObj.printInfo()
                      self.processingHeaderObj.printInfo()
                      self.__printInfo = False
                  def run(self,
                          path=None,
                          startDate=None,
                          endDate=None,
                          startTime=datetime.time(0,0,0),
                          endTime=datetime.time(23,59,59),
                          set=None,
-                         expLabel = "",
-                         ext = None,
-                         online = False,
-                         delay = 60,
-                         walk = True,
-                         getblock = False,
-                         nTxs = 1,
                          realtime=False,
                          blocksize=None,
                          blocktime=None,
-                         queue=None,
                          skip=None,
                          cursor=None,
                          warnings=True,
                          server=None,
-                         verbose=True, **kwargs):
+                         verbose=True,
+                         format=None,
+                         oneDDict=None,
+                         twoDDict=None,
+                         ind2DList=None, **kwargs):
                      if not(self.isConfig):
                          self.setup(path=path,
                                     startDate=startDate,
                                     endDate=endDate,
                                     startTime=startTime,
                                     endTime=endTime,
                                     set=set,
                                     expLabel=expLabel,
                                     ext=ext,
                                     online=online,
                                     delay=delay,
                                     walk=walk,
                                     getblock=getblock,
                                     nTxs=nTxs,
                                     realtime=realtime,
                                     blocksize=blocksize,
                                     blocktime=blocktime,
                                     skip=skip,
                                     cursor=cursor,
                                     warnings=warnings,
                                     server=server,
-                                    verbose=verbose)
+                                    verbose=verbose,
+                                    format=format,
+                                    oneDDict=oneDDict,
+                                    twoDDict=twoDDict,
+                                    ind2DList=ind2DList)
                          self.isConfig = True
                      if server is None:
                          self.getData()
-                     else:
                          self.getFromServer()
              class JRODataWriter(JRODataIO):
                  """
                  Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
                  de los datos siempre se realiza por bloques.
                  """
                  blockIndex = 0
                  path = None
                  setFile = None
                  profilesPerBlock = None
                  blocksPerFile = None
                  nWriteBlocks = 0
                  fileDate = None
                  def __init__(self, dataOut=None):
                      raise NotImplementedError
                  def hasAllDataInBuffer(self):
                      raise NotImplementedError
                  def setBlockDimension(self):
                      raise NotImplementedError
                  def writeBlock(self):
                      raise NotImplementedError
                  def putData(self):
                      raise NotImplementedError
                  def getProcessFlags(self):
                      processFlags = 0
                      dtype_index = get_dtype_index(self.dtype)
                      procflag_dtype = get_procflag_dtype(dtype_index)
                      processFlags += procflag_dtype
                      if self.dataOut.flagDecodeData:
                          processFlags += PROCFLAG.DECODE_DATA
                      if self.dataOut.flagDeflipData:
                          processFlags += PROCFLAG.DEFLIP_DATA
                      if self.dataOut.code is not None:
                          processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                      if self.dataOut.nCohInt > 1:
                          processFlags += PROCFLAG.COHERENT_INTEGRATION
                      if self.dataOut.type == "Spectra":
                          if self.dataOut.nIncohInt > 1:
                              processFlags += PROCFLAG.INCOHERENT_INTEGRATION
                          if self.dataOut.data_dc is not None:
                              processFlags += PROCFLAG.SAVE_CHANNELS_DC
                          if self.dataOut.flagShiftFFT:
                              processFlags += PROCFLAG.SHIFT_FFT_DATA
                      return processFlags
                  def setBasicHeader(self):
                      self.basicHeaderObj.size = self.basicHeaderSize #bytes
                      self.basicHeaderObj.version = self.versionFile
                      self.basicHeaderObj.dataBlock = self.nTotalBlocks
                      utc = numpy.floor(self.dataOut.utctime)
-                     milisecond  = (self.dataOut.utctime - utc)* 1000.0
                      self.basicHeaderObj.utc = utc
                      self.basicHeaderObj.miliSecond = milisecond
                      self.basicHeaderObj.timeZone = self.dataOut.timeZone
                      self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
                      self.basicHeaderObj.errorCount = self.dataOut.errorCount
                  def setFirstHeader(self):
                      """
                      Obtiene una copia del First Header
                      Affected:
                          self.basicHeaderObj
                          self.systemHeaderObj
                          self.radarControllerHeaderObj
                          self.processingHeaderObj self.
                      Return:
                          None
                      """
                      raise NotImplementedError
                  def __writeFirstHeader(self):
                      """
                      Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
                      Affected:
                          __dataType
                      Return:
                          None
                      """
              #        CALCULAR PARAMETROS
-                     sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
+                     sizeLongHeader = self.systemHeaderObj.size + \
+                         self.radarControllerHeaderObj.size + self.processingHeaderObj.size
                      self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
                      self.basicHeaderObj.write(self.fp)
                      self.systemHeaderObj.write(self.fp)
                      self.radarControllerHeaderObj.write(self.fp)
                      self.processingHeaderObj.write(self.fp)
                  def __setNewBlock(self):
                      """
                      Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
                      Return:
 :    si no pudo escribir nada
 :    Si escribio el Basic el First Header
                      """
                      if self.fp == None:
                          self.setNextFile()
                      if self.flagIsNewFile:
                          return 1
                      if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
                          self.basicHeaderObj.write(self.fp)
                          return 1
-                     if not( self.setNextFile() ):
                          return 0
                      return 1
                  def writeNextBlock(self):
                      """
                      Selecciona el bloque siguiente de datos y los escribe en un file
                      Return:
 :    Si no hizo pudo escribir el bloque de datos
 :    Si no pudo escribir el bloque de datos
                      """
-                     if not( self.__setNewBlock() ):
                          return 0
                      self.writeBlock()
                      print "[Writing] Block No. %d/%d" %(self.blockIndex,
                                                          self.processingHeaderObj.dataBlocksPerFile)
                      return 1
                  def setNextFile(self):
                      """
                      Determina el siguiente file que sera escrito
                      Affected:
                          self.filename
                          self.subfolder
                          self.fp
                          self.setFile
                          self.flagIsNewFile
                      Return:
 :    Si el archivo no puede ser escrito
 :    Si el archivo esta listo para ser escrito
                      """
                      ext = self.ext
                      path = self.path
                      if self.fp != None:
                          self.fp.close()
-                     timeTuple = time.localtime( self.dataOut.utctime)
                      subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
-                     fullpath = os.path.join( path, subfolder )
                      setFile = self.setFile
-                     if not( os.path.exists(fullpath) ):
                          os.mkdir(fullpath)
                          setFile = -1 #inicializo mi contador de seteo
                      else:
-                         filesList = os.listdir( fullpath )
-                         if len( filesList ) > 0:
-                             filesList = sorted( filesList, key=str.lower )
                              filen = filesList[-1]
                              # el filename debera tener el siguiente formato
                              # 0 1234 567 89A BCDE (hex)
                              # x YYYY DDD SSS .ext
-                             if isNumber( filen[8:11] ):
-                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
+                                 # inicializo mi contador de seteo al seteo del ultimo file
+                                 setFile = int(filen[8:11])
                              else:
                                  setFile = -1
                          else:
                              setFile = -1 #inicializo mi contador de seteo
                      setFile += 1
                      #If this is a new day it resets some values
                      if self.dataOut.datatime.date() > self.fileDate:
                          setFile = 0
                          self.nTotalBlocks = 0
-                     filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
+                     filen = '%s%4.4d%3.3d%3.3d%s' % (
+                         self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
-                     filename = os.path.join( path, subfolder, filen )
-                     fp = open( filename,'wb' )
                      self.blockIndex = 0
                      #guardando atributos
                      self.filename = filename
                      self.subfolder = subfolder
                      self.fp = fp
                      self.setFile = setFile
                      self.flagIsNewFile = 1
                      self.fileDate = self.dataOut.datatime.date()
                      self.setFirstHeader()
                      print '[Writing] Opening file: %s'%self.filename
                      self.__writeFirstHeader()
                      return 1
                  def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
                      """
                      Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
                      Inputs:
                          path                :    directory where data will be saved
                          profilesPerBlock    :    number of profiles per block
                          set                 :    initial file set
                          datatype            :    An integer number that defines data type:
 : int8  (1 byte)
 : int16 (2 bytes)
 : int32 (4 bytes)
 : int64 (8 bytes)
 : float32 (4 bytes)
 : double64 (8 bytes)
                      Return:
 :    Si no realizo un buen seteo
 :    Si realizo un buen seteo
                      """
                      if ext == None:
                          ext = self.ext
                      self.ext = ext.lower()
                      self.path = path
                      if set is None:
                          self.setFile = -1
                      else:
                          self.setFile = set - 1
                      self.blocksPerFile = blocksPerFile
                      self.profilesPerBlock = profilesPerBlock
                      self.dataOut = dataOut
                      self.fileDate = self.dataOut.datatime.date()
                      #By default
                      self.dtype = self.dataOut.dtype
                      if datatype is not None:
                          self.dtype = get_numpy_dtype(datatype)
                      if not(self.setNextFile()):
                          print "[Writing] There isn't a next file"
                          return 0
                      self.setBlockDimension()
                      return 1
                  def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
                      if not(self.isConfig):
-                         self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
+                         self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
+                                    set=set, ext=ext, datatype=datatype, **kwargs)
                          self.isConfig = True
                      self.putData()

schainpy/model/io/jroIO_param.py +2 -2

              import numpy
              import time
              import os
              import h5py
              import re
              import datetime
              from schainpy.model.data.jrodata import *
              from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
              # from jroIO_base import *
              from schainpy.model.io.jroIO_base import *
              import schainpy
              class ParamReader(ProcessingUnit):
                  '''
                  Reads HDF5 format files
                  path
                  startDate
                  endDate
                  startTime
                  endTime
                  '''
                  ext = ".hdf5"
                  optchar = "D"
                  timezone = None
                  startTime = None
                  endTime = None
                  fileIndex = None
                  utcList = None      #To select data in the utctime list
                  blockList = None    #List to blocks to be read from the file
                  blocksPerFile = None    #Number of blocks to be read
                  blockIndex = None
                  path = None
                  #List of Files
                  filenameList = None
                  datetimeList = None
                  #Hdf5 File
                  listMetaname = None
                  listMeta = None
                  listDataname = None
                  listData = None
                  listShapes = None
                  fp = None
                  #dataOut reconstruction
                  dataOut = None
                  def __init__(self, **kwargs):
                      ProcessingUnit.__init__(self, **kwargs)
                      self.dataOut = Parameters()
                      return
                  def setup(self, **kwargs):
                      path = kwargs['path']
                      startDate = kwargs['startDate']
                      endDate = kwargs['endDate']
                      startTime = kwargs['startTime']
                      endTime = kwargs['endTime']
                      walk = kwargs['walk']
                      if kwargs.has_key('ext'):
                          ext = kwargs['ext']
                      else:
                          ext = '.hdf5'
                      if kwargs.has_key('timezone'):
                          self.timezone = kwargs['timezone']
                      else:
                          self.timezone = 'lt'
                      print "[Reading] Searching files in offline mode ..."
                      pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                             startTime=startTime, endTime=endTime,
                                                                             ext=ext, walk=walk)
                      if not(filenameList):
                          print "There is no files into the folder: %s"%(path)
                          sys.exit(-1)
                      self.fileIndex = -1
                      self.startTime = startTime
                      self.endTime = endTime
                      self.__readMetadata()
                      self.__setNextFileOffline()
                      return
                  def searchFilesOffLine(self,
                                          path,
                                          startDate=None,
                                          endDate=None,
                                          startTime=datetime.time(0,0,0),
                                          endTime=datetime.time(23,59,59),
                                          ext='.hdf5',
                                          walk=True):
                      expLabel = ''
                      self.filenameList = []
                      self.datetimeList = []
                      pathList = []
                      JRODataObj = JRODataReader()
                      dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
                      if dateList == []:
                          print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
                                                                      datetime.datetime.combine(startDate,startTime).ctime(),
                                                                      datetime.datetime.combine(endDate,endTime).ctime())
                          return None, None
                      if len(dateList) > 1:
                          print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
                      else:
                          print "[Reading] data was found for the date %s" %(dateList[0])
                      filenameList = []
                      datetimeList = []
                      #----------------------------------------------------------------------------------
                      for thisPath in pathList:
              #             thisPath = pathList[pathDict[file]]
                          fileList = glob.glob1(thisPath, "*%s" %ext)
                          fileList.sort()
                          for file in fileList:
                              filename = os.path.join(thisPath,file)
                              if not isFileInDateRange(filename, startDate, endDate):
                                  continue
                              thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
                              if not(thisDatetime):
                                  continue
                              filenameList.append(filename)
                              datetimeList.append(thisDatetime)
                      if not(filenameList):
                          print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                          return None, None
                      print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
                      print
-                     for i in range(len(filenameList)):
-                         print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
+             #         for i in range(len(filenameList)):
+             #             print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                      self.filenameList = filenameList
                      self.datetimeList = datetimeList
                      return pathList, filenameList
                  def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
                      """
                      Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                      Inputs:
                          filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                          startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                          endDate            :    fecha final del rango seleccionado en formato datetime.date
                          startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                          endTime            :    tiempo final del rango seleccionado en formato datetime.time
                      Return:
                          Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                          fecha especificado, de lo contrario retorna False.
                      Excepciones:
                          Si el archivo no existe o no puede ser abierto
                          Si la cabecera no puede ser leida.
                      """
                      try:
                          fp = h5py.File(filename,'r')
                          grp1 = fp['Data']
                      except IOError:
                          traceback.print_exc()
                          raise IOError, "The file %s can't be opened" %(filename)
                      #chino rata
                      #In case has utctime attribute
                      grp2 = grp1['utctime']
              #         thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
                      thisUtcTime = grp2.value[0]
                      fp.close()
                      if self.timezone == 'lt':
                          thisUtcTime -= 5*3600
                      thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
              #         thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
                      thisDate = thisDatetime.date()
                      thisTime = thisDatetime.time()
                      startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
                      endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
                      #General case
                      #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
                      #-----------o----------------------------o-----------
                      #       startTime                     endTime
                      if endTime >= startTime:
                          thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
                          if numpy.any(thisUtcLog):   #If there is one block between the hours mentioned
                              return thisDatetime
                          return None
                      #If endTime < startTime then endTime belongs to the next day
                      #<<<<<<<<<<<o                            o>>>>>>>>>>>
                      #-----------o----------------------------o-----------
                      #        endTime                    startTime
                      if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
                          return None
                      if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
                          return None
                      if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
                          return None
                      return thisDatetime
                  def __setNextFileOffline(self):
                      self.fileIndex += 1
                      idFile = self.fileIndex
                      if not(idFile < len(self.filenameList)):
                          print "No more Files"
                          return 0
                      filename = self.filenameList[idFile]
                      filePointer = h5py.File(filename,'r')
                      self.filename = filename
                      self.fp = filePointer
                      print "Setting the file: %s"%self.filename
              #         self.__readMetadata()
                      self.__setBlockList()
                      self.__readData()
              #         self.nRecords = self.fp['Data'].attrs['blocksPerFile']
              #         self.nRecords = self.fp['Data'].attrs['nRecords']
                      self.blockIndex = 0
                      return 1
                  def __setBlockList(self):
                      '''
                      Selects the data within the times defined
                      self.fp
                      self.startTime
                      self.endTime
                      self.blockList
                      self.blocksPerFile
                      '''
                      fp = self.fp
                      startTime = self.startTime
                      endTime = self.endTime
                      grp = fp['Data']
                      thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
                      #ERROOOOR
                      if self.timezone == 'lt':
                          thisUtcTime -= 5*3600
                      thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
                      thisDate = thisDatetime.date()
                      thisTime = thisDatetime.time()
                      startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                      endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                      ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
                      self.blockList = ind
                      self.blocksPerFile = len(ind)
                      return
                  def __readMetadata(self):
                      '''
                      Reads Metadata
                      self.pathMeta
                      self.listShapes
                      self.listMetaname
                      self.listMeta
                      '''
              #         grp = self.fp['Data']
              #         pathMeta = os.path.join(self.path, grp.attrs['metadata'])
              #
              #         if pathMeta == self.pathMeta:
              #             return
              #         else:
              #             self.pathMeta = pathMeta
              #
              #         filePointer = h5py.File(self.pathMeta,'r')
              #         groupPointer = filePointer['Metadata']
                      filename = self.filenameList[0]
                      fp = h5py.File(filename,'r')
                      gp = fp['Metadata']
                      listMetaname = []
                      listMetadata = []
                      for item in gp.items():
                          name = item[0]
                          if name=='array dimensions':
                              table = gp[name][:]
                              listShapes = {}
                              for shapes in table:
                                  listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
                          else:
                              data = gp[name].value
                              listMetaname.append(name)
                              listMetadata.append(data)
              #                 if name=='type':
              #                     self.__initDataOut(data)
                      self.listShapes = listShapes
                      self.listMetaname = listMetaname
                      self.listMeta = listMetadata
                      fp.close()
                      return
                  def __readData(self):
                      grp = self.fp['Data']
                      listdataname = []
                      listdata = []
                      for item in grp.items():
                          name = item[0]
                          listdataname.append(name)
                          array = self.__setDataArray(grp[name],self.listShapes[name])
                          listdata.append(array)
                      self.listDataname = listdataname
                      self.listData = listdata
                      return
                  def __setDataArray(self, dataset, shapes):
                      nDims = shapes[0]
                      nDim2 = shapes[1]      #Dimension 0
                      nDim1 = shapes[2]      #Dimension 1, number of Points or Parameters
                      nDim0 = shapes[3]      #Dimension 2, number of samples or ranges
                      mode = shapes[4]        #Mode of storing
                      blockList = self.blockList
                      blocksPerFile = self.blocksPerFile
                      #Depending on what mode the data was stored
                      if mode == 0:       #Divided in channels
                          arrayData = dataset.value.astype(numpy.float)[0][blockList]
                      if mode == 1:     #Divided in parameter
                          strds = 'table'
                          nDatas = nDim1
                          newShapes = (blocksPerFile,nDim2,nDim0)
                      elif mode==2:       #Concatenated in a table
                          strds = 'table0'
                          arrayData = dataset[strds].value
                          #Selecting part of the dataset
                          utctime = arrayData[:,0]
                          u, indices = numpy.unique(utctime, return_index=True)
                          if blockList.size != indices.size:
                              indMin = indices[blockList[0]]
                              if blockList[1] + 1 >= indices.size:
                                  arrayData = arrayData[indMin:,:]
                              else:
                                  indMax = indices[blockList[1] + 1]
                                  arrayData = arrayData[indMin:indMax,:]
                          return arrayData
                      #    One dimension
                      if nDims == 0:
                          arrayData = dataset.value.astype(numpy.float)[0][blockList]
                      #    Two dimensions
                      elif nDims == 2:
                          arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
                          newShapes = (blocksPerFile,nDim0)
                          nDatas = nDim1
                          for i in range(nDatas):
                              data = dataset[strds + str(i)].value
                              arrayData[:,i,:] = data[blockList,:]
                      #    Three dimensions
                      else:
                          arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
                          for i in range(nDatas):
                              data = dataset[strds + str(i)].value
                              for b in range(blockList.size):
                                  arrayData[b,:,i,:] = data[:,:,blockList[b]]
                      return arrayData
                  def __setDataOut(self):
                      listMeta = self.listMeta
                      listMetaname = self.listMetaname
                      listDataname = self.listDataname
                      listData = self.listData
                      listShapes = self.listShapes
                      blockIndex = self.blockIndex
              #         blockList = self.blockList
                      for i in range(len(listMeta)):
                          setattr(self.dataOut,listMetaname[i],listMeta[i])
                      for j in range(len(listData)):
                          nShapes = listShapes[listDataname[j]][0]
                          mode = listShapes[listDataname[j]][4]
                          if nShapes == 1:
                              setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
                          elif nShapes > 1:
                              setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
                          elif mode==0:
                              setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
                          #Mode Meteors
                          elif mode ==2:
                              selectedData = self.__selectDataMode2(listData[j], blockIndex)
                              setattr(self.dataOut, listDataname[j], selectedData)
                      return
                  def __selectDataMode2(self, data, blockIndex):
                      utctime = data[:,0]
                      aux, indices = numpy.unique(utctime, return_inverse=True)
                      selInd = numpy.where(indices == blockIndex)[0]
                      selData = data[selInd,:]
                      return selData
                  def getData(self):
              #         if self.flagNoMoreFiles:
              #             self.dataOut.flagNoData = True
              #             print 'Process finished'
              #             return 0
              #
                      if self.blockIndex==self.blocksPerFile:
                           if not( self.__setNextFileOffline() ):
                              self.dataOut.flagNoData = True
                              return 0
              #         if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
              #             self.dataOut.flagNoData = True
              #             return 0
              #         self.__readData()
                      self.__setDataOut()
                      self.dataOut.flagNoData = False
                      self.blockIndex += 1
                      return
                  def run(self, **kwargs):
                      if not(self.isConfig):
                          self.setup(**kwargs)
              #             self.setObjProperties()
                          self.isConfig = True
                      self.getData()
                      return
              class ParamWriter(Operation):
                  '''
                  HDF5 Writer, stores parameters data in HDF5 format files
                  path:             path where the files will be stored
                  blocksPerFile:    number of blocks that will be saved in per HDF5 format file
                  mode:             selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
                  metadataList:     list of attributes that will be stored as metadata
                  dataList:         list of attributes that will be stores as data
                  '''
                  ext = ".hdf5"
                  optchar = "D"
                  metaoptchar = "M"
                  metaFile = None
                  filename = None
                  path = None
                  setFile = None
                  fp = None
                  grp = None
                  ds = None
                  firsttime = True
                  #Configurations
                  blocksPerFile = None
                  blockIndex = None
                  dataOut = None
                  #Data Arrays
                  dataList = None
                  metadataList = None
              #     arrayDim = None
                  dsList = None   #List of dictionaries with dataset properties
                  tableDim = None
              #     dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
                  dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
                  currentDay = None
                  lastTime = None
                  def __init__(self, **kwargs):
                      Operation.__init__(self, **kwargs)
                      self.isConfig = False
                      return
                  def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
                      self.path = path
                      self.blocksPerFile = blocksPerFile
                      self.metadataList = metadataList
                      self.dataList = dataList
                      self.dataOut = dataOut
                      self.mode = mode
                      if self.mode is not None:
                          self.mode = numpy.zeros(len(self.dataList)) + mode
                      else:
                          self.mode = numpy.ones(len(self.dataList))
                      arrayDim = numpy.zeros((len(self.dataList),5))
                      #Table dimensions
                      dtype0 = self.dtype
                      tableList = []
                      #Dictionary and list of tables
                      dsList = []
                      for i in range(len(self.dataList)):
                          dsDict = {}
                          dataAux = getattr(self.dataOut, self.dataList[i])
                          dsDict['variable'] = self.dataList[i]
                          #---------------------    Conditionals    ------------------------
                          #There is no data
                          if dataAux is None:
                              return 0
                          #Not array, just a number
                          #Mode 0
                          if type(dataAux)==float or type(dataAux)==int:
                              dsDict['mode'] = 0
                              dsDict['nDim'] = 0
                              arrayDim[i,0] = 0
                              dsList.append(dsDict)
                          #Mode 2: meteors
                          elif mode[i] == 2:
              #                 dsDict['nDim'] = 0
                              dsDict['dsName'] = 'table0'
                              dsDict['mode'] = 2      # Mode meteors
                              dsDict['shape'] = dataAux.shape[-1]
                              dsDict['nDim'] = 0
                              dsDict['dsNumber'] = 1
                              arrayDim[i,3] = dataAux.shape[-1]
                              arrayDim[i,4] = mode[i]         #Mode the data was stored
                              dsList.append(dsDict)
                          #Mode 1
                          else:
                              arrayDim0 = dataAux.shape       #Data dimensions
                              arrayDim[i,0] = len(arrayDim0)  #Number of array dimensions
                              arrayDim[i,4] = mode[i]         #Mode the data was stored
                              strtable = 'table'
                              dsDict['mode'] = 1      # Mode parameters
                              # Three-dimension arrays
                              if len(arrayDim0) == 3:
                                  arrayDim[i,1:-1] = numpy.array(arrayDim0)
                                  nTables = int(arrayDim[i,2])
                                  dsDict['dsNumber'] = nTables
                                  dsDict['shape'] = arrayDim[i,2:4]
                                  dsDict['nDim'] = 3
                                  for j in range(nTables):
                                      dsDict = dsDict.copy()
                                      dsDict['dsName'] = strtable + str(j)
                                      dsList.append(dsDict)
                              # Two-dimension arrays
                              elif len(arrayDim0) == 2:
                                  arrayDim[i,2:-1] = numpy.array(arrayDim0)
                                  nTables = int(arrayDim[i,2])
                                  dsDict['dsNumber'] = nTables
                                  dsDict['shape'] = arrayDim[i,3]
                                  dsDict['nDim'] = 2
                                  for j in range(nTables):
                                      dsDict = dsDict.copy()
                                      dsDict['dsName'] = strtable + str(j)
                                      dsList.append(dsDict)
                              # One-dimension arrays
                              elif len(arrayDim0) == 1:
                                  arrayDim[i,3] = arrayDim0[0]
                                  dsDict['shape'] = arrayDim0[0]
                                  dsDict['dsNumber'] = 1
                                  dsDict['dsName'] = strtable + str(0)
                                  dsDict['nDim'] = 1
                                  dsList.append(dsDict)
                          table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
                          tableList.append(table)
              #         self.arrayDim = arrayDim
                      self.dsList = dsList
                      self.tableDim = numpy.array(tableList, dtype = dtype0)
                      self.blockIndex = 0
                      timeTuple = time.localtime(dataOut.utctime)
                      self.currentDay = timeTuple.tm_yday
                      return 1
                  def putMetadata(self):
                      fp = self.createMetadataFile()
                      self.writeMetadata(fp)
                      fp.close()
                      return
                  def createMetadataFile(self):
                      ext = self.ext
                      path = self.path
                      setFile = self.setFile
                      timeTuple = time.localtime(self.dataOut.utctime)
                      subfolder = ''
                      fullpath = os.path.join( path, subfolder )
                      if not( os.path.exists(fullpath) ):
                          os.mkdir(fullpath)
                          setFile = -1 #inicializo mi contador de seteo
                      subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                      fullpath = os.path.join( path, subfolder )
                      if not( os.path.exists(fullpath) ):
                          os.mkdir(fullpath)
                          setFile = -1 #inicializo mi contador de seteo
                      else:
                          filesList = os.listdir( fullpath )
                          filesList = sorted( filesList, key=str.lower )
                          if len( filesList ) > 0:
                              filesList = [k for k in filesList if 'M' in k]
                              filen = filesList[-1]
                              # el filename debera tener el siguiente formato
                              # 0 1234 567 89A BCDE (hex)
                              # x YYYY DDD SSS .ext
                              if isNumber( filen[8:11] ):
                                  setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                              else:
                                  setFile = -1
                          else:
                              setFile = -1 #inicializo mi contador de seteo
                      if self.setType is None:
                          setFile += 1
                          file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
                                                         timeTuple.tm_year,
                                                         timeTuple.tm_yday,
                                                         setFile,
                                                         ext )
                      else:
                          setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                          file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
                                                         timeTuple.tm_year,
                                                         timeTuple.tm_yday,
                                                         setFile,
                                                         ext )
                      filename = os.path.join( path, subfolder, file )
                      self.metaFile = file
                      #Setting HDF5 File
                      fp = h5py.File(filename,'w')
                      return fp
                  def writeMetadata(self, fp):
                      grp = fp.create_group("Metadata")
                      grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
                      for i in range(len(self.metadataList)):
                          grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
                      return
                  def timeFlag(self):
                      currentTime = self.dataOut.utctime
                      if self.lastTime is None:
                          self.lastTime = currentTime
                      #Day
                      timeTuple = time.localtime(currentTime)
                      dataDay = timeTuple.tm_yday
                      #Time
                      timeDiff = currentTime - self.lastTime
                      #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
                      if dataDay != self.currentDay:
                          self.currentDay = dataDay
                          return True
                      elif timeDiff > 3*60*60:
                          self.lastTime = currentTime
                          return True
                      else:
                          self.lastTime = currentTime
                          return False
                  def setNextFile(self):
                      ext = self.ext
                      path = self.path
                      setFile = self.setFile
                      mode = self.mode
                      timeTuple = time.localtime(self.dataOut.utctime)
                      subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                      fullpath = os.path.join( path, subfolder )
                      if os.path.exists(fullpath):
                          filesList = os.listdir( fullpath )
                          filesList = [k for k in filesList if 'D' in k]
                          if len( filesList ) > 0:
                              filesList = sorted( filesList, key=str.lower )
                              filen = filesList[-1]
                              # el filename debera tener el siguiente formato
                              # 0 1234 567 89A BCDE (hex)
                              # x YYYY DDD SSS .ext
                              if isNumber( filen[8:11] ):
                                  setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                              else:
                                  setFile = -1
                          else:
                              setFile = -1 #inicializo mi contador de seteo
                      else:
                          os.makedirs(fullpath)
                          setFile = -1 #inicializo mi contador de seteo
                      if self.setType is None:
                          setFile += 1
                          file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
                                                         timeTuple.tm_year,
                                                         timeTuple.tm_yday,
                                                         setFile,
                                                         ext )
                      else:
                          setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                          file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
                                                         timeTuple.tm_year,
                                                         timeTuple.tm_yday,
                                                         setFile,
                                                         ext )
                      filename = os.path.join( path, subfolder, file )
                      #Setting HDF5 File
                      fp = h5py.File(filename,'w')
                      #write metadata
                      self.writeMetadata(fp)
                      #Write data
                      grp = fp.create_group("Data")
              #         grp.attrs['metadata'] = self.metaFile
              #         grp.attrs['blocksPerFile'] = 0
                      ds = []
                      data = []
                      dsList = self.dsList
                      i = 0
                      while i < len(dsList):
                          dsInfo = dsList[i]
                          #One-dimension data
                          if dsInfo['mode'] == 0:
              #                 ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
                              ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
                              ds.append(ds0)
                              data.append([])
                              i += 1
                              continue
              #                 nDimsForDs.append(nDims[i])
                          elif dsInfo['mode'] == 2:
                              grp0 = grp.create_group(dsInfo['variable'])
                              ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
                              ds.append(ds0)
                              data.append([])
                              i += 1
                              continue
                          elif dsInfo['mode'] == 1:
                              grp0 = grp.create_group(dsInfo['variable'])
                              for j in range(dsInfo['dsNumber']):
                                  dsInfo = dsList[i]
                                  tableName = dsInfo['dsName']
                                  shape = int(dsInfo['shape'])
                                  if dsInfo['nDim'] == 3:
                                      ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
                                  else:
                                      ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
                                  ds.append(ds0)
                                  data.append([])
                                  i += 1
              #                     nDimsForDs.append(nDims[i])
                      fp.flush()
                      fp.close()
              #         self.nDatas = nDatas
              #         self.nDims = nDims
              #         self.nDimsForDs = nDimsForDs
                      #Saving variables
                      print 'Writing the file: %s'%filename
                      self.filename = filename
              #         self.fp = fp
              #         self.grp = grp
              #         self.grp.attrs.modify('nRecords', 1)
                      self.ds = ds
                      self.data = data
              #         self.setFile = setFile
                      self.firsttime = True
                      self.blockIndex = 0
                      return
                  def putData(self):
                      if self.blockIndex == self.blocksPerFile or self.timeFlag():
                          self.setNextFile()
              #         if not self.firsttime:
                      self.readBlock()
                      self.setBlock()     #Prepare data to be written
                      self.writeBlock()   #Write data
                      return
                  def readBlock(self):
                      '''
                      data Array configured
                      self.data
                      '''
                      dsList = self.dsList
                      ds = self.ds
                              #Setting HDF5 File
                      fp = h5py.File(self.filename,'r+')
                      grp = fp["Data"]
                      ind = 0
              #         grp.attrs['blocksPerFile'] = 0
                      while ind < len(dsList):
                          dsInfo = dsList[ind]
                          if dsInfo['mode'] == 0:
                              ds0 = grp[dsInfo['variable']]
                              ds[ind] = ds0
                              ind += 1
                          else:
                              grp0 = grp[dsInfo['variable']]
                              for j in range(dsInfo['dsNumber']):
                                  dsInfo = dsList[ind]
                                  ds0 = grp0[dsInfo['dsName']]
                                  ds[ind] = ds0
                                  ind += 1
                      self.fp = fp
                      self.grp = grp
                      self.ds = ds
                      return
                  def setBlock(self):
                      '''
                      data Array configured
                      self.data
                      '''
                      #Creating Arrays
                      dsList = self.dsList
                      data = self.data
                      ind = 0
                      while ind < len(dsList):
                          dsInfo = dsList[ind]
                          dataAux = getattr(self.dataOut, dsInfo['variable'])
                          mode = dsInfo['mode']
                          nDim = dsInfo['nDim']
                          if mode == 0 or mode == 2 or nDim == 1:
                              data[ind] = dataAux
                              ind += 1
              #             elif nDim == 1:
              #                 data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
              #                 ind += 1
                          elif nDim == 2:
                              for j in range(dsInfo['dsNumber']):
                                  data[ind] = dataAux[j,:]
                                  ind += 1
                          elif nDim == 3:
                              for j in range(dsInfo['dsNumber']):
                                  data[ind] = dataAux[:,j,:]
                                  ind += 1
                      self.data = data
                      return
                  def writeBlock(self):
                      '''
                      Saves the block in the HDF5 file
                      '''
                      dsList = self.dsList
                      for i in range(len(self.ds)):
                          dsInfo = dsList[i]
                          nDim = dsInfo['nDim']
                          mode = dsInfo['mode']
                          #    First time
                          if self.firsttime:
              #                 self.ds[i].resize(self.data[i].shape)
              #                 self.ds[i][self.blockIndex,:] = self.data[i]
                              if type(self.data[i]) == numpy.ndarray:
                                  if nDim == 3:
                                      self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
                                      self.ds[i].resize(self.data[i].shape)
                                  if mode == 2:
                                      self.ds[i].resize(self.data[i].shape)
                              self.ds[i][:] = self.data[i]
                          else:
                          #    From second time
                              #    Meteors!
                              if mode == 2:
                                  dataShape = self.data[i].shape
                                  dsShape = self.ds[i].shape
                                  self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
                                  self.ds[i][dsShape[0]:,:] = self.data[i]
                              #    No dimension
                              elif mode == 0:
                                  self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
                                  self.ds[i][0,-1] = self.data[i]
                              #    One dimension
                              elif nDim == 1:
                                  self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
                                  self.ds[i][-1,:] = self.data[i]
                              #    Two dimension
                              elif nDim == 2:
                                  self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
                                  self.ds[i][self.blockIndex,:] = self.data[i]
                              #    Three dimensions
                              elif nDim == 3:
                                  self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
                                  self.ds[i][:,:,-1] = self.data[i]
                      self.firsttime = False
                      self.blockIndex += 1
                      #Close to save changes
                      self.fp.flush()
                      self.fp.close()
                      return
                  def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
                      if not(self.isConfig):
                          flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
                                                metadataList=metadataList, dataList=dataList, mode=mode, **kwargs)
                          if not(flagdata):
                              return
                          self.isConfig = True
              #             self.putMetadata()
                          self.setNextFile()
                      self.putData()
                      return

schainpy/model/proc/__init__.py +1 0

              '''
              $Author: murco $
              $Id: Processor.py 1 2012-11-12 18:56:07Z murco $
              '''
              from jroproc_voltage import *
              from jroproc_spectra import *
              from jroproc_heispectra import *
              from jroproc_amisr import *
              from jroproc_correlation import *
              from jroproc_parameters import *
              from jroproc_spectra_lags import *
              from jroproc_spectra_acf import *
+             from bltrproc_parameters import *

schainpy/model/utils/jroutils_publish.py +4 -1

              '''
              @author: Juan C. Espinoza
              '''
              import time
              import json
              import numpy
              import paho.mqtt.client as mqtt
              import zmq
              import datetime
              from zmq.utils.monitor import recv_monitor_message
              from functools import wraps
              from threading import Thread
              from multiprocessing import Process
              from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
              from schainpy.model.data.jrodata import JROData
              from schainpy.utils import log
              MAXNUMX = 100
              MAXNUMY = 100
              class PrettyFloat(float):
                  def __repr__(self):
                      return '%.2f' % self
              def roundFloats(obj):
                  if isinstance(obj, list):
                      return map(roundFloats, obj)
                  elif isinstance(obj, float):
                      return round(obj, 2)
              def decimate(z, MAXNUMY):
                  dy = int(len(z[0])/MAXNUMY) + 1
                  return z[::, ::dy]
              class throttle(object):
                  '''
                  Decorator that prevents a function from being called more than once every
                  time period.
                  To create a function that cannot be called more than once a minute, but
                  will sleep until it can be called:
                  @throttle(minutes=1)
                  def foo():
                    pass
                  for i in range(10):
                    foo()
                    print "This function has run %s times." % i
                  '''
                  def __init__(self, seconds=0, minutes=0, hours=0):
                      self.throttle_period = datetime.timedelta(
                          seconds=seconds, minutes=minutes, hours=hours
                      )
                      self.time_of_last_call = datetime.datetime.min
                  def __call__(self, fn):
                      @wraps(fn)
                      def wrapper(*args, **kwargs):
                          now = datetime.datetime.now()
                          time_since_last_call = now - self.time_of_last_call
                          time_left = self.throttle_period - time_since_last_call
                          if time_left > datetime.timedelta(seconds=0):
                              return
                          self.time_of_last_call = datetime.datetime.now()
                          return fn(*args, **kwargs)
                      return wrapper
              class Data(object):
                  '''
                  Object to hold data to be plotted
                  '''
                  def __init__(self, plottypes, throttle_value):
                      self.plottypes = plottypes
                      self.throttle = throttle_value
                      self.ended = False
                      self.__times = []
                  def __str__(self):
                      dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
                      return 'Data[{}][{}]'.format(';'.join(dum), len(self.__times))
                  def __len__(self):
                      return len(self.__times)
                  def __getitem__(self, key):
                      if key not in self.data:
                          raise KeyError(log.error('Missing key: {}'.format(key)))
                      if 'spc' in key:
                          ret = self.data[key]
                      else:
                          ret = numpy.array([self.data[key][x] for x in self.times])
                          if ret.ndim > 1:
                              ret = numpy.swapaxes(ret, 0, 1)
                      return ret
                  def setup(self):
                      '''
                      Configure object
                      '''
                      self.ended = False
                      self.data = {}
                      self.__times = []
                      self.__heights = []
                      self.__all_heights = set()
                      for plot in self.plottypes:
+                         if 'snr' in plot:
+                             plot = 'snr'
                          self.data[plot] = {}
                  def shape(self, key):
                      '''
                      Get the shape of the one-element data for the given key
                      '''
                      if len(self.data[key]):
                          if 'spc' in key:
                              return self.data[key].shape
                          return self.data[key][self.__times[0]].shape
                      return (0,)
                  def update(self, dataOut):
                      '''
                      Update data object with new dataOut
                      '''
                      tm = dataOut.utctime
                      if tm in self.__times:
                          return
                      self.parameters = getattr(dataOut, 'parameters', [])
                      self.pairs = dataOut.pairsList
                      self.channels = dataOut.channelList
-                     self.xrange = (dataOut.getFreqRange(1)/1000. , dataOut.getAcfRange(1) , dataOut.getVelRange(1))
                      self.interval = dataOut.getTimeInterval()
+                     if 'spc' in self.plottypes or 'cspc' in self.plottypes:
+                         self.xrange = (dataOut.getFreqRange(1)/1000. , dataOut.getAcfRange(1) , dataOut.getVelRange(1))
                      self.__heights.append(dataOut.heightList)
                      self.__all_heights.update(dataOut.heightList)
                      self.__times.append(tm)
                      for plot in self.plottypes:
                          if plot == 'spc':
                              z = dataOut.data_spc/dataOut.normFactor
                              self.data[plot] = 10*numpy.log10(z)
                          if plot == 'cspc':
                              self.data[plot] = dataOut.data_cspc
                          if plot == 'noise':
                              self.data[plot][tm] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
                          if plot == 'rti':
                              self.data[plot][tm] = dataOut.getPower()
                          if plot == 'snr_db':
                              self.data['snr'][tm] = dataOut.data_SNR
                          if plot == 'snr':
                              self.data[plot][tm] = 10*numpy.log10(dataOut.data_SNR)
                          if plot == 'dop':
                              self.data[plot][tm] = 10*numpy.log10(dataOut.data_DOP)
                          if plot == 'mean':
                              self.data[plot][tm] = dataOut.data_MEAN
                          if plot == 'std':
                              self.data[plot][tm] = dataOut.data_STD
                          if plot == 'coh':
                              self.data[plot][tm] = dataOut.getCoherence()
                          if plot == 'phase':
                              self.data[plot][tm] = dataOut.getCoherence(phase=True)
                          if plot == 'output':
                              self.data[plot][tm] = dataOut.data_output
                          if plot == 'param':
                              self.data[plot][tm] = dataOut.data_param
                  def normalize_heights(self):
                      '''
                      Ensure same-dimension of the data for different heighList
                      '''
                      H = numpy.array(list(self.__all_heights))
                      H.sort()
                      for key in self.data:
                          shape = self.shape(key)[:-1] + H.shape
                          for tm, obj in self.data[key].items():
                              h = self.__heights[self.__times.index(tm)]
                              if H.size == h.size:
                                  continue
                              index = numpy.where(numpy.in1d(H, h))[0]
                              dummy = numpy.zeros(shape) + numpy.nan
                              if len(shape) == 2:
                                  dummy[:, index] = obj
                              else:
                                  dummy[index] = obj
                              self.data[key][tm] = dummy
                      self.__heights = [H for tm in self.__times]
                  def jsonify(self, decimate=False):
                      '''
                      Convert data to json
                      '''
                      ret = {}
                      tm = self.times[-1]
                      for key, value in self.data:
                          if key in ('spc', 'cspc'):
                              ret[key] = roundFloats(self.data[key].to_list())
                          else:
                              ret[key] = roundFloats(self.data[key][tm].to_list())
                      ret['timestamp'] = tm
                      ret['interval'] = self.interval
                  @property
                  def times(self):
                      '''
                      Return the list of times of the current data
                      '''
                      ret = numpy.array(self.__times)
                      ret.sort()
                      return ret
                  @property
                  def heights(self):
                      '''
                      Return the list of heights of the current data
                      '''
                      return numpy.array(self.__heights[-1])
              class PublishData(Operation):
                  '''
                  Operation to send data over zmq.
                  '''
                  def __init__(self, **kwargs):
                      """Inicio."""
                      Operation.__init__(self, **kwargs)
                      self.isConfig = False
                      self.client = None
                      self.zeromq = None
                      self.mqtt = None
                  def on_disconnect(self, client, userdata, rc):
                      if rc != 0:
                          log.warning('Unexpected disconnection.')
                      self.connect()
                  def connect(self):
                      log.warning('trying to connect')
                      try:
                          self.client.connect(
                              host=self.host,
                              port=self.port,
                              keepalive=60*10,
                              bind_address='')
                          self.client.loop_start()
                          # self.client.publish(
                          #     self.topic + 'SETUP',
                          #     json.dumps(setup),
                          #     retain=True
                          #     )
                      except:
                          log.error('MQTT Conection error.')
                          self.client = False
                  def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, verbose=True, **kwargs):
                      self.counter = 0
                      self.topic = kwargs.get('topic', 'schain')
                      self.delay = kwargs.get('delay', 0)
                      self.plottype = kwargs.get('plottype', 'spectra')
                      self.host = kwargs.get('host', "10.10.10.82")
                      self.port = kwargs.get('port', 3000)
                      self.clientId = clientId
                      self.cnt = 0
                      self.zeromq = zeromq
                      self.mqtt = kwargs.get('plottype', 0)
                      self.client = None
                      self.verbose = verbose
                      setup = []
                      if mqtt is 1:
                          self.client = mqtt.Client(
                              client_id=self.clientId + self.topic + 'SCHAIN',
                              clean_session=True)
                          self.client.on_disconnect = self.on_disconnect
                          self.connect()
                          for plot in self.plottype:
                              setup.append({
                                  'plot': plot,
                                  'topic': self.topic + plot,
                                  'title': getattr(self, plot + '_' + 'title', False),
                                  'xlabel': getattr(self, plot + '_' + 'xlabel', False),
                                  'ylabel': getattr(self, plot + '_' + 'ylabel', False),
                                  'xrange': getattr(self, plot + '_' + 'xrange', False),
                                  'yrange': getattr(self, plot + '_' + 'yrange', False),
                                  'zrange': getattr(self, plot + '_' + 'zrange', False),
                              })
                      if zeromq is 1:
                          context = zmq.Context()
                          self.zmq_socket = context.socket(zmq.PUSH)
                          server = kwargs.get('server', 'zmq.pipe')
                          if 'tcp://' in server:
                              address = server
                          else:
                              address = 'ipc:///tmp/%s' % server
                          self.zmq_socket.connect(address)
                          time.sleep(1)
                  def publish_data(self):
                      self.dataOut.finished = False
                      if self.mqtt is 1:
                          yData = self.dataOut.heightList[:2].tolist()
                          if self.plottype == 'spectra':
                              data = getattr(self.dataOut, 'data_spc')
                              z = data/self.dataOut.normFactor
                              zdB = 10*numpy.log10(z)
                              xlen, ylen = zdB[0].shape
                              dx = int(xlen/MAXNUMX) + 1
                              dy = int(ylen/MAXNUMY) + 1
                              Z = [0 for i in self.dataOut.channelList]
                              for i in self.dataOut.channelList:
                                  Z[i] = zdB[i][::dx, ::dy].tolist()
                              payload = {
                                  'timestamp': self.dataOut.utctime,
                                  'data': roundFloats(Z),
                                  'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
                                  'interval': self.dataOut.getTimeInterval(),
                                  'type': self.plottype,
                                  'yData': yData
                              }
                          elif self.plottype in ('rti', 'power'):
                              data = getattr(self.dataOut, 'data_spc')
                              z = data/self.dataOut.normFactor
                              avg = numpy.average(z, axis=1)
                              avgdB = 10*numpy.log10(avg)
                              xlen, ylen = z[0].shape
                              dy = numpy.floor(ylen/self.__MAXNUMY) + 1
                              AVG = [0 for i in self.dataOut.channelList]
                              for i in self.dataOut.channelList:
                                  AVG[i] = avgdB[i][::dy].tolist()
                              payload = {
                                  'timestamp': self.dataOut.utctime,
                                  'data': roundFloats(AVG),
                                  'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
                                  'interval': self.dataOut.getTimeInterval(),
                                  'type': self.plottype,
                                  'yData': yData
                              }
                          elif self.plottype == 'noise':
                              noise = self.dataOut.getNoise()/self.dataOut.normFactor
                              noisedB = 10*numpy.log10(noise)
                              payload = {
                                  'timestamp': self.dataOut.utctime,
                                  'data': roundFloats(noisedB.reshape(-1, 1).tolist()),
                                  'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
                                  'interval': self.dataOut.getTimeInterval(),
                                  'type': self.plottype,
                                  'yData': yData
                              }
                          elif self.plottype == 'snr':
                              data = getattr(self.dataOut, 'data_SNR')
                              avgdB = 10*numpy.log10(data)
                              ylen = data[0].size
                              dy = numpy.floor(ylen/self.__MAXNUMY) + 1
                              AVG = [0 for i in self.dataOut.channelList]
                              for i in self.dataOut.channelList:
                                  AVG[i] = avgdB[i][::dy].tolist()
                              payload = {
                                  'timestamp': self.dataOut.utctime,
                                  'data': roundFloats(AVG),
                                  'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
                                  'type': self.plottype,
                                  'yData': yData
                              }
                          else:
                              print "Tipo de grafico invalido"
                              payload = {
                                  'data': 'None',
                                  'timestamp': 'None',
                                  'type': None
                              }
                          self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0)
                      if self.zeromq is 1:
                          if self.verbose:
                              log.log(
                                  '{} - {}'.format(self.dataOut.type, self.dataOut.datatime),
                                  'Sending'
                              )
                          self.zmq_socket.send_pyobj(self.dataOut)
                  def run(self, dataOut, **kwargs):
                      self.dataOut = dataOut
                      if not self.isConfig:
                          self.setup(**kwargs)
                          self.isConfig = True
                      self.publish_data()
                      time.sleep(self.delay)
                  def close(self):
                      if self.zeromq is 1:
                          self.dataOut.finished = True
                          self.zmq_socket.send_pyobj(self.dataOut)
                          time.sleep(0.1)
                          self.zmq_socket.close()
                      if self.client:
                          self.client.loop_stop()
                          self.client.disconnect()
              class ReceiverData(ProcessingUnit):
                  def __init__(self, **kwargs):
                      ProcessingUnit.__init__(self, **kwargs)
                      self.isConfig = False
                      server = kwargs.get('server', 'zmq.pipe')
                      if 'tcp://' in server:
                          address = server
                      else:
                          address = 'ipc:///tmp/%s' % server
                      self.address = address
                      self.dataOut = JROData()
                  def setup(self):
                      self.context = zmq.Context()
                      self.receiver = self.context.socket(zmq.PULL)
                      self.receiver.bind(self.address)
                      time.sleep(0.5)
                      log.success('ReceiverData from {}'.format(self.address))
                  def run(self):
                      if not self.isConfig:
                          self.setup()
                          self.isConfig = True
                      self.dataOut = self.receiver.recv_pyobj()
                      log.log('{} - {}'.format(self.dataOut.type,
                                               self.dataOut.datatime.ctime(),),
                              'Receiving')
              class PlotterReceiver(ProcessingUnit, Process):
                  throttle_value = 5
                  def __init__(self, **kwargs):
                      ProcessingUnit.__init__(self, **kwargs)
                      Process.__init__(self)
                      self.mp = False
                      self.isConfig = False
                      self.isWebConfig = False
                      self.connections = 0
                      server = kwargs.get('server', 'zmq.pipe')
                      plot_server = kwargs.get('plot_server', 'zmq.web')
                      if 'tcp://' in server:
                          address = server
                      else:
                          address = 'ipc:///tmp/%s' % server
                      if 'tcp://' in plot_server:
                          plot_address = plot_server
                      else:
                          plot_address = 'ipc:///tmp/%s' % plot_server
                      self.address = address
                      self.plot_address = plot_address
                      self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')]
                      self.realtime = kwargs.get('realtime', False)
                      self.throttle_value = kwargs.get('throttle', 5)
                      self.sendData = self.initThrottle(self.throttle_value)
                      self.dates = []
                      self.setup()
                  def setup(self):
                      self.data = Data(self.plottypes, self.throttle_value)
                      self.isConfig = True
                  def event_monitor(self, monitor):
                      events = {}
                      for name in dir(zmq):
                          if name.startswith('EVENT_'):
                              value = getattr(zmq, name)
                              events[value] = name
                      while monitor.poll():
                          evt = recv_monitor_message(monitor)
                          if evt['event'] == 32:
                              self.connections += 1
                          if evt['event'] == 512:
                              pass
                          evt.update({'description': events[evt['event']]})
                          if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
                              break
                      monitor.close()
                      print('event monitor thread done!')
                  def initThrottle(self, throttle_value):
                      @throttle(seconds=throttle_value)
                      def sendDataThrottled(fn_sender, data):
                          fn_sender(data)
                      return sendDataThrottled
                  def send(self, data):
                      log.success('Sending {}'.format(data), self.name)
                      self.sender.send_pyobj(data)
                  def run(self):
                      log.success(
                          'Starting from {}'.format(self.address),
                          self.name
                      )
                      self.context = zmq.Context()
                      self.receiver = self.context.socket(zmq.PULL)
                      self.receiver.bind(self.address)
                      monitor = self.receiver.get_monitor_socket()
                      self.sender = self.context.socket(zmq.PUB)
                      if self.realtime:
                          self.sender_web = self.context.socket(zmq.PUB)
                          self.sender_web.connect(self.plot_address)
                          time.sleep(1)
                      if 'server' in self.kwargs:
                          self.sender.bind("ipc:///tmp/{}.plots".format(self.kwargs['server']))
                      else:
                          self.sender.bind("ipc:///tmp/zmq.plots")
                      time.sleep(2)
                      t = Thread(target=self.event_monitor, args=(monitor,))
                      t.start()
                      while True:
                          dataOut = self.receiver.recv_pyobj()
                          dt = datetime.datetime.fromtimestamp(dataOut.utctime).date()
                          sended = False
                          if dt not in self.dates:
                              if self.data:
                                  self.data.ended = True
                                  self.send(self.data)
                                  sended = True
                              self.data.setup()
                              self.dates.append(dt)
                          self.data.update(dataOut)
                          if dataOut.finished is True:
                              self.connections -= 1
                              if self.connections == 0 and dt in self.dates:
                                  self.data.ended = True
                                  self.send(self.data)
                                  self.data.setup()
                          else:
                              if self.realtime:
                                  self.send(self.data)
                                  # self.sender_web.send_string(self.data.jsonify())
                              else:
                                  if not sended:
                                      self.sendData(self.send, self.data)
                      return
                  def sendToWeb(self):
                      if not self.isWebConfig:
                          context = zmq.Context()
                          sender_web_config = context.socket(zmq.PUB)
                          if 'tcp://' in self.plot_address:
                              dum, address, port = self.plot_address.split(':')
                              conf_address = '{}:{}:{}'.format(dum, address, int(port)+1)
                          else:
                              conf_address = self.plot_address + '.config'
                          sender_web_config.bind(conf_address)
                          time.sleep(1)
                          for kwargs in self.operationKwargs.values():
                              if 'plot' in kwargs:
                                  log.success('[Sending] Config data to web for {}'.format(kwargs['code'].upper()))
                                  sender_web_config.send_string(json.dumps(kwargs))
                          self.isWebConfig = True
  No newline at end of file

schainpy/project.py removed 0 -34

NO CONTENT: file was removed

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages