schain-jc Commit - r568:486c66596e4c

Merge with branch schain_julia_drifts from rev. 803 to 995....

Miguel Valdez -

r568:486c66596e4c

parent child

Context file:

r568:486c66596e4c

Expand all files

schainpy/model/io/jroIO_usrp.py created 644 +517 0

This diff has been collapsed as it changes many lines, (517 lines changed) Show them Hide them
		@@ -0,0 +1,517
	1	'''
	2	Created on Jul 3, 2014
	3
	4	@author: roj-idl71
	5	'''
	6	import datetime
	7	import numpy
	8
	9	try:
	10	from gevent import sleep
	11	except:
	12	from time import sleep
	13
	14	from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
	15	from schainpy.model.data.jrodata import Voltage
	16	from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
	17
	18	try:
	19	import digital_rf_hdf5
	20	except:
	21	print 'You should install "digital_rf_hdf5" module if you want to read USRP data'
	22
	23	class USRPReader(ProcessingUnit):
	24	'''
	25	classdocs
	26	'''
	27
	28	def __init__(self):
	29	'''
	30	Constructor
	31	'''
	32
	33	ProcessingUnit.__init__(self)
	34
	35	self.dataOut = Voltage()
	36	self.__printInfo = True
	37	self.__flagDiscontinuousBlock = False
	38	self.__bufferIndex = 9999999
	39
	40	self.__ippKm = None
	41	self.__codeType = 0
	42	self.__nCode = None
	43	self.__nBaud = None
	44	self.__code = None
	45
	46	def __getCurrentSecond(self):
	47
	48	return self.__thisUnixSample/self.__sample_rate
	49
	50	thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
	51
	52	def __setFileHeader(self):
	53	'''
	54	In this method will be initialized every parameter of dataOut object (header, no data)
	55	'''
	56
	57	self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
	58	txA=0,
	59	txB=0,
	60	nWindows=1,
	61	nHeights=self.__nSamples,
	62	firstHeight=self.__firstHeigth,
	63	deltaHeight=self.__deltaHeigth,
	64	codeType=self.__codeType,
	65	nCode=self.__nCode, nBaud=self.__nBaud,
	66	code = self.__code)
	67
	68	self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
	69	nProfiles=1024,
	70	nChannels=len(self.__channelList),
	71	adcResolution=14)
	72
	73	self.dataOut.type = "Voltage"
	74
	75	self.dataOut.data = None
	76
	77	self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
	78
	79	# self.dataOut.nChannels = 0
	80
	81	# self.dataOut.nHeights = 0
	82
	83	self.dataOut.nProfiles = 1
	84
	85	self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
	86
	87	self.dataOut.channelList = self.__channelList
	88
	89	self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
	90
	91	# self.dataOut.channelIndexList = None
	92
	93	self.dataOut.flagNoData = True
	94
	95	#Set to TRUE if the data is discontinuous
	96	self.dataOut.flagDiscontinuousBlock = False
	97
	98	self.dataOut.utctime = None
	99
	100	self.dataOut.timeZone = self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
	101
	102	self.dataOut.dstFlag = 0
	103
	104	self.dataOut.errorCount = 0
	105
	106	self.dataOut.nCohInt = 1
	107
	108	self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
	109
	110	self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
	111
	112	self.dataOut.flagShiftFFT = False
	113
	114	self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
	115
	116	#Time interval between profiles
	117	#self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
	118
	119	self.dataOut.frequency = self.__frequency
	120
	121	self.dataOut.realtime = self.__online
	122
	123	def setup(self, path = None,
	124	startDate = None,
	125	endDate = None,
	126	startTime = datetime.time(0,0,0),
	127	endTime = datetime.time(23,59,59),
	128	channelList = None,
	129	nSamples = None,
	130	ippKm = None,
	131	online = False,
	132	wait = 60,
	133	nbuffer = 1024*4):
	134	'''
	135	In this method we should set all initial parameters.
	136
	137	Inputs:
	138	path
	139	startDate
	140	endDate
	141	startTime
	142	endTime
	143	set
	144	expLabel
	145	ext
	146	online
	147	wait
	148	'''
	149	try:
	150	self.digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
	151	except:
	152	self.digitalReadObj = digital_rf_hdf5.read_hdf5(path)
	153
	154	channelNameList = self.digitalReadObj.get_channels()
	155
	156	if not channelNameList:
	157	raise IOError, "[Reading] The path doesn,t have any files .. "
	158
	159	if not channelList:
	160	channelList = range(len(channelNameList))
	161
	162	########## Reading metadata ######################
	163
	164	metadata_dict = self.digitalReadObj.get_rf_file_metadata(channelNameList[channelList[0]])
	165
	166	self.__sample_rate = metadata_dict['sample_rate'][0]
	167	self.__samples_per_file = metadata_dict['samples_per_file'][0]
	168	self.__deltaHeigth = 1e6*0.15/self.__sample_rate
	169
	170	this_metadata_file = self.digitalReadObj.get_metadata(channelNameList[channelList[0]])
	171
	172	self.__frequency = this_metadata_file['center_frequencies'].value
	173	try:
	174	self.__timezone = this_metadata_file['timezone'].value
	175	except:
	176	self.__timezone = 0
	177
	178	self.__firstHeigth = 0
	179
	180	try:
	181	codeType = this_metadata_file['codeType'].value
	182	except:
	183	codeType = 0
	184
	185	nCode = 0
	186	nBaud = 0
	187	code = None
	188
	189	if codeType:
	190	nCode = this_metadata_file['nCode'].value
	191	nBaud = this_metadata_file['nBaud'].value
	192	code = this_metadata_file['code'].value
	193
	194	if not ippKm:
	195	try:
	196	#seconds to km
	197	ippKm = 1e60.15this_metadata_file['ipp'].value
	198	except:
	199	ippKm = None
	200
	201	####################################################
	202	startUTCSecond = None
	203	endUTCSecond = None
	204
	205	if startDate:
	206	startDatetime = datetime.datetime.combine(startDate, startTime)
	207	startUTCSecond = (startDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
	208
	209	if endDate:
	210	endDatetime = datetime.datetime.combine(endDate, endTime)
	211	endUTCSecond = (endDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
	212
	213	start_index, end_index = self.digitalReadObj.get_bounds(channelNameList[channelList[0]])
	214
	215	if not startUTCSecond:
	216	startUTCSecond = start_index/self.__sample_rate
	217
	218	if start_index > startUTCSecond*self.__sample_rate:
	219	startUTCSecond = start_index/self.__sample_rate
	220
	221	if not endUTCSecond:
	222	endUTCSecond = end_index/self.__sample_rate
	223
	224	if end_index < endUTCSecond*self.__sample_rate:
	225	endUTCSecond = end_index/self.__sample_rate
	226
	227	if not nSamples:
	228	if not ippKm:
	229	raise ValueError, "[Reading] nSamples or ippKm should be defined"
	230
	231	nSamples = ippKm / (1e6*0.15/self.__sample_rate)
	232
	233	channelBoundList = []
	234	channelNameListFiltered = []
	235
	236	for thisIndexChannel in channelList:
	237	thisChannelName = channelNameList[thisIndexChannel]
	238	start_index, end_index = self.digitalReadObj.get_bounds(thisChannelName)
	239	channelBoundList.append((start_index, end_index))
	240	channelNameListFiltered.append(thisChannelName)
	241
	242	self.profileIndex = 0
	243
	244	self.__ippKm = ippKm
	245	self.__codeType = codeType
	246	self.__nCode = nCode
	247	self.__nBaud = nBaud
	248	self.__code = code
	249
	250	self.__datapath = path
	251	self.__online = online
	252	self.__channelList = channelList
	253	self.__channelNameList = channelNameListFiltered
	254	self.__channelBoundList = channelBoundList
	255	self.__nSamples = nSamples
	256	self.__samples_to_read = nbuffer*nSamples
	257	self.__nChannels = len(self.__channelList)
	258
	259	self.__startUTCSecond = startUTCSecond
	260	self.__endUTCSecond = endUTCSecond
	261
	262	self.__timeInterval = 1.0 * self.__samples_to_read/self.__sample_rate #Time interval
	263
	264	if online:
	265	# self.__thisUnixSample = int(endUTCSecondself.__sample_rate - 4self.__samples_to_read)
	266	startUTCSecond = numpy.floor(endUTCSecond)
	267
	268	self.__thisUnixSample = int(startUTCSecond*self.__sample_rate) - self.__samples_to_read
	269
	270	self.__data_buffer = numpy.zeros((self.__nChannels, self.__samples_to_read), dtype = numpy.complex)
	271
	272	self.__setFileHeader()
	273	self.isConfig = True
	274
	275	print "[Reading] USRP Data was found from %s to %s " %(
	276	datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
	277	datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
	278	)
	279
	280	print "[Reading] Starting process from ", datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone), " to ", datetime.datetime.utcfromtimestamp(endUTCSecond - self.__timezone)
	281
	282	def __reload(self):
	283
	284	if not self.__online:
	285	return
	286
	287	# print
	288	# print "%s not in range [%s, %s]" %(
	289	# datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
	290	# datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
	291	# datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
	292	# )
	293	print "[Reading] reloading metadata ..."
	294
	295	self.digitalReadObj.reload(complete_update=True)
	296
	297	start_index, end_index = self.digitalReadObj.get_bounds(self.__channelNameList[self.__channelList[0]])
	298
	299	if start_index > self.__startUTCSecond*self.__sample_rate:
	300	self.__startUTCSecond = 1.0*start_index/self.__sample_rate
	301
	302	if end_index > self.__endUTCSecond*self.__sample_rate:
	303	self.__endUTCSecond = 1.0*end_index/self.__sample_rate
	304	print
	305	print "[Reading] New timerange found [%s, %s] " %(
	306	datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
	307	datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
	308	)
	309
	310	return True
	311
	312	return False
	313
	314	def __readNextBlock(self, seconds=30, volt_scale = 218776):
	315	'''
	316	'''
	317
	318	#Set the next data
	319	self.__flagDiscontinuousBlock = False
	320	self.__thisUnixSample += self.__samples_to_read
	321
	322	if self.__thisUnixSample + 2self.__samples_to_read > self.__endUTCSecondself.__sample_rate:
	323	print "[Reading] There are no more data into selected timerange"
	324
	325	self.__reload()
	326
	327	if self.__thisUnixSample + 2self.__samples_to_read > self.__endUTCSecondself.__sample_rate:
	328	self.__thisUnixSample -= self.__samples_to_read
	329	return False
	330
	331	indexChannel = 0
	332
	333	dataOk = False
	334
	335	for thisChannelName in self.__channelNameList:
	336
	337	try:
	338	result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
	339	self.__samples_to_read,
	340	thisChannelName)
	341
	342	except IOError, e:
	343	#read next profile
	344	self.__flagDiscontinuousBlock = True
	345	print e
	346	break
	347
	348	if result.shape[0] != self.__samples_to_read:
	349	self.__flagDiscontinuousBlock = True
	350	print "[Reading] %s: Too few samples were found, just %d samples" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
	351	result.shape[0])
	352	break
	353
	354	self.__data_buffer[indexChannel,:] = result*volt_scale
	355
	356	indexChannel += 1
	357
	358	dataOk = True
	359
	360	self.__utctime = self.__thisUnixSample/self.__sample_rate
	361
	362	if not dataOk:
	363	return False
	364
	365	print "[Reading] %s: %d samples <> %f sec" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
	366	self.__samples_to_read,
	367	self.__timeInterval)
	368
	369	self.__bufferIndex = 0
	370
	371	return True
	372
	373	def __isBufferEmpty(self):
	374
	375	if self.__bufferIndex <= self.__samples_to_read - self.__nSamples:
	376	return False
	377
	378	return True
	379
	380	def getData(self, seconds=30, nTries=5):
	381
	382	'''
	383	This method gets the data from files and put the data into the dataOut object
	384
	385	In addition, increase el the buffer counter in one.
	386
	387	Return:
	388	data : retorna un perfil de voltages (alturas * canales) copiados desde el
	389	buffer. Si no hay mas archivos a leer retorna None.
	390
	391	Affected:
	392	self.dataOut
	393	self.profileIndex
	394	self.flagDiscontinuousBlock
	395	self.flagIsNewBlock
	396	'''
	397
	398	err_counter = 0
	399	self.dataOut.flagNoData = True
	400
	401	if self.__isBufferEmpty():
	402
	403	self.__flagDiscontinuousBlock = False
	404
	405	while True:
	406	if self.__readNextBlock():
	407	break
	408
	409	if self.__thisUnixSample > self.__endUTCSecond*self.__sample_rate:
	410	return False
	411
	412	if self.__flagDiscontinuousBlock:
	413	print '[Reading] discontinuous block found ... continue with the next block'
	414	continue
	415
	416	if not self.__online:
	417	return False
	418
	419	err_counter += 1
	420	if err_counter > nTries:
	421	return False
	422
	423	print '[Reading] waiting %d seconds to read a new block' %seconds
	424	sleep(seconds)
	425
	426	self.dataOut.data = self.__data_buffer[:,self.__bufferIndex:self.__bufferIndex+self.__nSamples]
	427	self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex)/self.__sample_rate
	428	self.dataOut.flagNoData = False
	429	self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
	430
	431	self.__bufferIndex += self.__nSamples
	432	self.profileIndex += 1
	433
	434	return True
	435
	436	def printInfo(self):
	437	'''
	438	'''
	439	if self.__printInfo == False:
	440	return
	441
	442	# self.systemHeaderObj.printInfo()
	443	# self.radarControllerHeaderObj.printInfo()
	444
	445	self.__printInfo = False
	446
	447	def printNumberOfBlock(self):
	448	'''
	449	'''
	450
	451	print self.profileIndex
	452
	453	def run(self, **kwargs):
	454	'''
	455	This method will be called many times so here you should put all your code
	456	'''
	457
	458	if not self.isConfig:
	459	self.setup(**kwargs)
	460
	461	self.getData()
	462
	463	return
	464
	465	class USRPWriter(Operation):
	466	'''
	467	classdocs
	468	'''
	469
	470	def __init__(self):
	471	'''
	472	Constructor
	473	'''
	474	self.dataOut = None
	475
	476	def setup(self, dataIn, path, blocksPerFile, set=0, ext=None):
	477	'''
	478	In this method we should set all initial parameters.
	479
	480	Input:
	481	dataIn : Input data will also be outputa data
	482
	483	'''
	484	self.dataOut = dataIn
	485
	486
	487
	488
	489
	490	self.isConfig = True
	491
	492	return
	493
	494	def run(self, dataIn, **kwargs):
	495	'''
	496	This method will be called many times so here you should put all your code
	497
	498	Inputs:
	499
	500	dataIn : object with the data
	501
	502	'''
	503
	504	if not self.isConfig:
	505	self.setup(dataIn, **kwargs)
	506
	507
	508	if __name__ == '__main__':
	509
	510	readObj = USRPReader()
	511
	512	while True:
	513	readObj.run(path='/Volumes/DATA/haystack/passive_radar/')
	514	# readObj.printInfo()
	515	readObj.printNumberOfBlock()
	516
	517	No newline at end of file

schainpy/model/io/jroIO_usrp_api.py created 644 +135 0

			@@ -0,0 +1,135
		1	'''
		2	Created on Jul 15, 2014
		3
		4	@author: roj-idl71
		5	'''
		6	import time
		7	import threading
		8	import cPickle
		9
		10	try:
		11	from gevent import sleep
		12	except:
		13	from time import sleep
		14
		15	SERIALIZER = cPickle
		16
		17	# from schainpy.serializer import DynamicSerializer
		18	from schainpy.model.io.jroIO_usrp import USRPReader
		19	from schainpy.serializer.DataTranslate import obj2Serial
		20
		21	class USRPReaderAPI(USRPReader, threading.Thread):
		22
		23	# __isBufferEmpty = True
		24
		25	__DATAKEYLIST = ['data','utctime','flagNoData']
		26
		27	def __init__(self, serializer='msgpack'):
		28
		29	threading.Thread.__init__(self)
		30	USRPReader.__init__(self)
		31
		32	# self.__serializerObj = DynamicSerializer.DynamicSerializer('msgpack')
		33	self.__mySerial = None
		34	self.__isBufferEmpty = True
		35
		36	self.setSerializer(serializer)
		37
		38	def setSerializer(self, serializer):
		39
		40	self.__serializer = serializer
		41
		42	def getSerializer(self):
		43
		44	return self.__serializer
		45
		46	def getProfileIndex(self):
		47
		48	return self.profileIndex
		49
		50	def getSerialMetaData(self):
		51
		52	if self.__isBufferEmpty:
		53	ini = time.time()
		54
		55	while True:
		56
		57	if not self.__isBufferEmpty:
		58	break
		59
		60	if time.time() - ini > 20:
		61	break
		62
		63	sleep(1e-12)
		64
		65
		66	# if not self.getData():
		67	# self.__isBufferEmpty = False
		68	# return None
		69
		70	if self.dataOut.flagNoData:
		71	return None
		72
		73	myMetadataSerial = obj2Serial(self.dataOut,
		74	serializer = self.__serializer)
		75
		76	return myMetadataSerial
		77
		78	def getSerialData(self):
		79
		80	if self.__isBufferEmpty:
		81	ini = time.time()
		82
		83	while True:
		84
		85	if not self.__isBufferEmpty:
		86	break
		87
		88	if time.time() - ini > 20:
		89	break
		90
		91	sleep(1e-12)
		92
		93
		94	# if not self.getData():
		95	# self.__isBufferEmpty = False
		96	# return None
		97
		98	if self.dataOut.flagNoData:
		99	return None
		100
		101	self.__isBufferEmpty = True
		102
		103	return self.__mySerial
		104
		105	def run(self):
		106
		107	'''
		108	This method will be called many times so here you should put all your code
		109	'''
		110
		111	if not self.isConfig:
		112	raise IOError, 'setup() method has to be called before start()'
		113
		114	while True:
		115
		116	if not self.__isBufferEmpty:
		117	sleep(1e-12)
		118	continue
		119
		120	if not self.getData():
		121	break
		122
		123	print ".",
		124
		125	self.__mySerial = obj2Serial(self.dataOut,
		126	keyList = self.__DATAKEYLIST,
		127	serializer = self.__serializer)
		128	self.__isBufferEmpty = False
		129
		130	# print self.profileIndex
		131	# print 'wait 1 second'
		132
		133	# sleep(0.1)
		134
		135	return No newline at end of file

schainpy/serializer/DataTranslate.py created 644 +115 0

			@@ -0,0 +1,115
		1	'''
		2	Created on Jul 15, 2014
		3
		4	@author: Miguel Urco
		5	'''
		6	from JROSerializer import DynamicSerializer
		7
		8	PICKLE_SERIALIZER = DynamicSerializer('cPickle')
		9	MSGPACK_SERIALIZER = DynamicSerializer('msgpack')
		10
		11	from schainpy.model.data.jrodata import *
		12
		13	CLASSNAME_KEY = 'classname__'
		14
		15	def isNotClassVar(myObj):
		16
		17	return not hasattr(myObj,'__dict__')
		18
		19	def isDictFormat(thisValue):
		20
		21	if type(thisValue) != type({}):
		22	return False
		23
		24	if CLASSNAME_KEY not in thisValue.keys():
		25	return False
		26
		27	return True
		28
		29	def obj2Dict(myObj, keyList=[]):
		30
		31	if not keyList:
		32	keyList = myObj.__dict__.keys()
		33
		34	myDict = {}
		35
		36	myDict[CLASSNAME_KEY] = myObj.__class__.__name__
		37
		38	for thisKey, thisValue in myObj.__dict__.items():
		39
		40	if thisKey not in keyList:
		41	continue
		42
		43	if isNotClassVar(thisValue):
		44	myDict[thisKey] = thisValue
		45	continue
		46
		47	## If this value is another class instance
		48	myNewDict = obj2Dict(thisValue)
		49	myDict[thisKey] = myNewDict
		50
		51	return myDict
		52
		53	def dict2Obj(myDict):
		54	'''
		55	'''
		56
		57	if CLASSNAME_KEY not in myDict.keys():
		58	return None
		59
		60	className = eval(myDict[CLASSNAME_KEY])
		61
		62	myObj = className()
		63
		64	for thisKey, thisValue in myDict.items():
		65
		66	if thisKey == CLASSNAME_KEY:
		67	continue
		68
		69	if not isDictFormat(thisValue):
		70	setattr(myObj, thisKey, thisValue)
		71	continue
		72
		73	myNewObj = dict2Obj(thisValue)
		74	setattr(myObj, thisKey, myNewObj)
		75
		76	return myObj
		77
		78	def obj2Serial(myObj, serializer='msgpack', **kwargs):
		79
		80	if serializer == 'cPickle':
		81	SERIALIZER = PICKLE_SERIALIZER
		82	else:
		83	SERIALIZER = MSGPACK_SERIALIZER
		84
		85	myDict = obj2Dict(myObj, **kwargs)
		86	mySerial = SERIALIZER.dumps(myDict)
		87
		88	return mySerial
		89
		90	def serial2Dict(mySerial, serializer='msgpack'):
		91
		92	if serializer == 'cPickle':
		93	SERIALIZER = PICKLE_SERIALIZER
		94	else:
		95	SERIALIZER = MSGPACK_SERIALIZER
		96
		97	return SERIALIZER.loads(mySerial)
		98
		99	def serial2Obj(mySerial, metadataDict = {}, serializer='msgpack'):
		100
		101	if serializer == 'cPickle':
		102	SERIALIZER = PICKLE_SERIALIZER
		103	else:
		104	SERIALIZER = MSGPACK_SERIALIZER
		105
		106	myDataDict = SERIALIZER.loads(mySerial)
		107
		108	if not metadataDict:
		109	myObj = dict2Obj(myDataDict)
		110	return myObj
		111
		112	metadataDict.update(myDataDict)
		113	myObj = dict2Obj(metadataDict)
		114
		115	return myObj No newline at end of file

schainpy/serializer/JROSerializer.py created 644 +93 0

			@@ -0,0 +1,93
		1	'''
		2	Created on Jul 17, 2014
		3
		4	@author: roj-idl71
		5	'''
		6
		7	import cPickle
		8	import msgpack_numpy
		9	import jsonpickle
		10	import yaml
		11
		12	# import JROMsgpack
		13	# import JROJsonpickle
		14
		15	class Serializer(object):
		16
		17	def __init__(self):
		18
		19	self.serializer = None
		20
		21	def dumps(self, obj, **kwargs):
		22
		23	return self.serializer.dumps(obj, **kwargs)
		24
		25	def loads(self, obj, **kwargs):
		26	return self.serializer.loads(obj, **kwargs)
		27
		28	class cPickleSerializer(Serializer):
		29
		30	def __init__(self):
		31	self.serializer = cPickle
		32
		33	def dumps(self, obj, **kwargs):
		34	return self.serializer.dumps(obj, 2)
		35
		36	def loads(self, obj, **kwargs):
		37	return self.serializer.loads(obj)
		38
		39	class msgpackSerializer(Serializer):
		40
		41	def __init__(self):
		42
		43	self.serializer = msgpack_numpy
		44
		45	def dumps(self, obj, **kwargs):
		46	return self.serializer.packb(obj)
		47
		48	def loads(self, obj, **kwargs):
		49	return self.serializer.unpackb(obj)
		50
		51	class jsonpickleSerializer(Serializer):
		52
		53	def __init__(self):
		54
		55	self.serializer = jsonpickle
		56
		57	def dumps(self, obj, **kwargs):
		58	return self.serializer.encode(obj, **kwargs)
		59
		60	def loads(self, obj, **kwargs):
		61	return self.serializer.decode(obj, **kwargs)
		62
		63	class yamlSerializer(Serializer):
		64
		65	def __init__(self):
		66
		67	self.serializer = yaml
		68
		69	def dumps(self, obj, **kwargs):
		70	return self.serializer.dump(obj, **kwargs)
		71
		72	def loads(self, obj, **kwargs):
		73	return self.serializer.load(obj, **kwargs)
		74
		75	class DynamicSerializer(Serializer):
		76
		77	def __init__(self, mode = 'cPickle'):
		78
		79	if mode == 'cPickle':
		80	self.serializer = cPickleSerializer()
		81
		82	if mode == 'jsonpickle':
		83	self.serializer = jsonpickleSerializer()
		84
		85	if mode == 'yaml':
		86	self.serializer = yamlSerializer()
		87
		88	if mode == 'msgpack':
		89	self.serializer = msgpackSerializer()
		90
		91
		92	if __name__ == '__main__':
		93	pass No newline at end of file

schainpy/serializer/__init__.py created 644 0 0

NO CONTENT: new file 100644

schainpy/serializer/test/DynamicObject.py created 644 +816 0

This diff has been collapsed as it changes many lines, (816 lines changed) Show them Hide them
		@@ -0,0 +1,816
	1	'''
	2	The DynamicObject module supports dynamic loading of YAML
	3	defined objects into Python class objects. Object can
	4	be sub-classed to allow direct binding of methods having
	5	matching signatures.
	6
	7	$Id$
	8	'''
	9
	10	import urllib
	11	import os
	12	import re
	13	import yaml # YAML Ain't Markup Language
	14	import numpy as np
	15	import copy
	16	import inspect
	17	import PrecisionTime
	18	import time
	19	import sys
	20	import datetime
	21	import collections
	22
	23	# Replacement Loader for PyYAML to keep dictionaries in-order:
	24	import OrderedYAML
	25	#OrderedYAML.collections
	26
	27	class Object(object):
	28	""" Loads a YAML defined python class dynamically using the supplied URI,
	29	which may be a file, directory, web hyper-link, or hyper-linked directory. """
	30
	31	# Dictionary containing all known Object class names and corresponding class objects
	32	dynamicClasses = collections.OrderedDict()
	33
	34	def __init__(self, object_uri=None, revision=None, recursive=False):
	35	if isinstance(object_uri, file):
	36	# URI is a yaml file - read it.
	37	self.yaml = file.read()
	38	elif object_uri == None:
	39	self.yaml = None
	40	elif isinstance(object_uri, str):
	41	if object_uri.endswith('.yml'):
	42	# URI is a web hyper-linked yaml file - read it.
	43	self.yaml = urllib.urlopen(object_uri).read()
	44	else:
	45	# URI is a (hyper-linked?) directory - try reading it.
	46	#print "URI is a directory."
	47	try:
	48	self.files = self.__parseLink(object_uri, recursive)
	49	except IOError:
	50	# URI is a local directory - get a list of YAML files in it
	51	self.files = self.__getYamlFiles(object_uri, recursive)
	52
	53	# For each YAML file found, create a new DynamicObject of it:
	54	self.yaml = []
	55	for fn in self.files:
	56	self.yaml.append(Object(fn))
	57	else:
	58	print "Invalid URI supplied: %s"%(object_uri,)
	59
	60	def __parseLink(self, object_uri, recursive):
	61	""" Returns a listing of all YAML files located in the
	62	hyper-link directory given by page. """
	63	page = urllib.urlopen(object_uri).read()
	64	#print "URI is a URL directory: %s"%(object_uri,)
	65	pattern = re.compile(r'<a href="[^"]*">')
	66
	67	# List of files contained in the directory at the given URL, ignoring
	68	# any "?" / GET query-string locations given:
	69	files = [x[9:-2] for x in pattern.findall(page) if not x[9:-2].startswith('?')]
	70	#print files
	71
	72	yamlFiles = []
	73	dirs = []
	74	for fn in files:
	75	if not fn.startswith('/'): # Ignore absolute paths...
	76	path = os.path.join(object_uri, fn)
	77	#print path
	78
	79	# Keep list of YAML files found...
	80	if fn.endswith('.yml'):
	81	yamlFiles.append(path)
	82
	83	# Keep list of directories found...
	84	elif recursive and fn.endswith('/'):
	85	dirs.append(path)
	86
	87	if recursive:
	88	#print dirs
	89	for path in dirs:
	90	yamlFiles += self.__parseLink(path,recursive)
	91
	92	return yamlFiles
	93
	94	def __getYamlFiles(self, local_dir, recursive):
	95	""" Returns a listing of all YAML files located in the given
	96	directory, recursing if requested. """
	97	yamlFiles = []
	98	dirs = []
	99	for fn in os.listdir(local_dir):
	100	path = os.path.join(local_dir, fn)
	101
	102	# List of YAML files found...
	103	if fn.endswith('.yml'):
	104	yamlFiles.append(path)
	105
	106	# List of directories found...
	107	elif recursive and os.path.isdir(path):
	108	dirs.append(path)
	109
	110	# Recurse if desired:
	111	if recursive:
	112	for path in dirs:
	113	yamlFiles += self.__getYamlFiles(path,recursive)
	114
	115	return yamlFiles
	116
	117	def equals(self, obj, compare_time_created=True):
	118	""" Returns True iff self has identical attributes
	119	(numerically) to obj (no extras) """
	120
	121	if not isinstance(obj, Object): return False
	122
	123	self_keys = self.__dict__.keys()
	124	obj_keys = obj.__dict__.keys()
	125	if not self_keys == obj_keys:
	126	return False
	127	for key in self_keys:
	128	obj_keys.remove(key)
	129
	130	self_value, obj_value = self.__dict__[key], obj.__dict__[key]
	131	if isinstance(self_value, Object):
	132	if not self_value.equals(obj_value, compare_time_created):
	133	return False
	134	elif isinstance(self_value, np.ndarray):
	135	m1 = map(repr,self_value.flat)
	136	m2 = map(repr,obj_value.flat)
	137	ret = m1 == m2
	138	if not ret:
	139	return False
	140	else:
	141	if not self_value == obj_value:
	142	# Return False iff the different times are important
	143	return key == '__time_created' and not compare_time_created
	144
	145	return obj_keys == [] # no more keys --> the objects are identical
	146
	147	def sizeof(self):
	148	""" Recursively computes the size in bytes of the given Dynamic Object """
	149	sz = 0
	150	values = self.__dict__.values()
	151	for val in values:
	152	if isinstance(val, Object): sz += val.sizeof()
	153	elif isinstance(val, np.ndarray): sz += val.nbytes
	154	elif hasattr(val, 'dtype') and hasattr(val.dtype, 'itemsize'): sz += val.dtype.itemsize
	155	else: sz += sys.getsizeof(val)
	156	return sz
	157
	158	# Automatic methods for accessing meta-data
	159	getters = ['__object_name', '__revision_number', '__revision_id', '__revision_source', '__revision_tag', '__time_created']
	160	def getObjectName(self): return self.__class__.meta_attributes['__object_name']
	161	def getRevisionNumber(self): return self.__class__.meta_attributes['__revision_number']
	162	def getRevisionId(self): return self.__class__.meta_attributes['__revision_id']
	163	def getRevisionSource(self): return self.__class__.meta_attributes['__revision_source']
	164	def getRevisionTag(self): return self.__class__.meta_attributes['__revision_tag']
	165	def getTimeCreated(self): return getattr(self, "__time_created")
	166
	167	"""
	168	__getters = [('ObjectName', getObjectName), ('RevisionNumber', getRevisionNumber),
	169	('RevisionId', getRevisionId), ('RevisionSource', getRevisionSource),
	170	('RevisionTag', getRevisionTag)]
	171	def __repr__(self):
	172	meta_atts = repr([(x[0], x[1](self)) for x in Object.__getters])
	173	atts = repr(self.__dict__)
	174	return "Object(%s, %s)"%(atts, meta_atts)
	175	"""
	176
	177
	178	class SignatureException(Exception):
	179	""" Exception thrown when a data or method signature is unknown or invalid
	180	for a particular Object. """
	181	def __init__(self, value): self.value = value
	182	def __str__(self): return repr(self.value)
	183
	184	class _IDLTag(object):
	185	""" IDLTag (aka Interface Definition Language Tag) is an abstract helper class
	186	used by the Factory to define built-in tags used
	187	specifically for our IDL """
	188	def __init__(self, yamlString):
	189	self.yamlString = yamlString
	190	def __repr__(self):
	191	return self.yamlString
	192
	193	class _Reference(_IDLTag):
	194	""" Helper class for Factory: Objects can be composed
	195	of other objects, requiring a Reference to the other object. """
	196	def __repr__(self):
	197	return "Ref(%s)"%(self.yamlString,)
	198
	199	class _Method(_IDLTag):
	200	""" Helper class for Factory: Objects have methods
	201	associated with them - this tag tells the Factory that a method
	202	signature follows (in dict format) """
	203	def __repr__(self):
	204	return "Method(%r)"%(self.yamlString,)
	205
	206	class Binary(Object):
	207	def __init__(self, binary_type, value=None):
	208	self.binary_type = binary_type
	209	self.value = value
	210
	211	import Lookup
	212
	213	class BuiltinDtype(_IDLTag):
	214	""" Helper class for Factory: Object parameters each
	215	have a certain data type (either dtype.xxxx for numpy compatible data
	216	types, or one of the generic python data types (i.e. int, bool, str...)
	217
	218	__addYamlConstructor in Factory registers all of the tags
	219	listed as keys in the dtypes dictionary."""
	220
	221	def __init__(self, yamlString, tag=None):
	222	self.tag = tag[1:]
	223	super(BuiltinDtype, self).__init__(yamlString)
	224	#print self.tag
	225	try: self.dtype = Lookup.numpy_dtypes[self.tag]
	226	except KeyError: self.dtype = Lookup.builtin_objects[self.tag]
	227
	228	def __repr__(self):
	229	return "_BuiltinType(%s,%s)"%(self.yamlString, self.tag)
	230
	231	# Register hexadecimal representation of numpy dtypes in YAML
	232
	233	class _Parameter:
	234	""" Helper class for Factory: Contains the name, default
	235	value, and length (if an array) of an object initialization parameter. """
	236
	237	def __init__(self, name, hasDefault=False, default=None, length=None, classType=None):
	238	self.name = name
	239	self.hasDefault = hasDefault
	240	self.default = default
	241	self.length = length
	242	if isinstance(classType, None.__class__) and not isinstance(default, None.__class__):
	243	self.classType = default.__class__
	244	else:
	245	self.classType = classType
	246
	247	class _UnresolvedType:
	248	""" Used to indicate a data type which has not yet been parsed (i.e. for
	249	recursive data-types. """
	250
	251	def __init__(self, yamlObject):
	252	# Either the name of the class we couldn't resolve, or a dictionary
	253	# containing the name and a default value
	254	self.yamlObject = yamlObject
	255
	256	class UnresolvedTypeException(Exception):
	257	""" Raised when a !ref tag is used, but the reference cannot be resolved """
	258	pass
	259
	260	def get_class(kls):
	261	""" Returns a pointer to the class instance with the name kls
	262	Function acquired from http://stackoverflow.com/questions/452969/ """
	263	parts = kls.split('.')
	264	module = ".".join(parts[:-1])
	265	m = __import__( module )
	266	for comp in parts[1:]:
	267	m = getattr(m, comp)
	268	return m
	269
	270	# Aliased constructor & representer adders for easily swapping between Ordered and non-Ordered:
	271	def add_constructor(tag, constructor):
	272	#yaml.add_constructor(tag, constructor)
	273	OrderedYAML.Loader.add_constructor(tag, constructor)
	274	def add_representer(cls, representer):
	275	#yaml.add_representer(cls, representer)
	276	OrderedYAML.Dumper.add_representer(cls, representer)
	277
	278	# Implicit constructor for _Reference objects using the !ref tag:
	279	def __ref_constructor(loader, node):
	280	if isinstance(node, yaml.nodes.MappingNode):
	281	return _Reference(loader.construct_mapping(node))
	282	else:
	283	return _Reference(loader.construct_scalar(node))
	284	add_constructor(u'!ref', __ref_constructor)
	285
	286	# Method constructor using !method tag:
	287	def __method_constructor(loader, node):
	288	if isinstance(node, yaml.nodes.MappingNode):
	289	return _Method(loader.construct_mapping(node))
	290	else:
	291	return _Method(loader.construct_scalar(node))
	292	add_constructor(u'!method', __method_constructor)
	293
	294	# Generic constructor for any _BuiltinDtype
	295	def __dtype_constructor(loader, node):
	296	if isinstance(node, yaml.nodes.SequenceNode):
	297	ret = BuiltinDtype(loader.construct_sequence(node), tag=node.tag)
	298	elif isinstance(node, yaml.nodes.MappingNode):
	299	ret = BuiltinDtype(loader.construct_mapping(node), tag=node.tag)
	300	else:
	301	ret = BuiltinDtype(loader.construct_scalar(node), tag=node.tag)
	302	return ret
	303
	304	# Register YAML constructors for each builtin type:
	305	for dtype in Lookup.numpy_dtypes.keys() + Lookup.builtin_objects.keys():
	306	add_constructor(u'!%s'%(dtype,), __dtype_constructor)
	307
	308	class FactoryLoader(OrderedYAML.Loader):
	309	""" A YAML Loader specifically designed to load YAML object definitions
	310	(as opposed to actual instances of the objects) """
	311
	312	def construct_yaml_timestamp(self, node):
	313	""" Make empty timestamps (None/null) acceptable, otherwise parse the timestamp """
	314	if node.value == u'':
	315	name = 'YAML_DEFN_LOADED_INCORRECTLY' # in case we forget to fix the name...
	316	return _Parameter(name, hasDefault=False, classType=datetime.datetime)
	317	else:
	318	return yaml.constructor.SafeConstructor.construct_yaml_timestamp(self, node)
	319
	320	# Override default timestamp constructor:
	321	FactoryLoader.add_constructor(
	322	u'tag:yaml.org,2002:timestamp',
	323	FactoryLoader.construct_yaml_timestamp
	324	)
	325
	326	import DynamicYAML
	327	class Factory:
	328	""" Load a YAML defined python class and create a class with initialization
	329	provided by this factory. This is intended as an abstract class to be sub-classed
	330	to enable complex initialization on object instantiation.
	331
	332	Factory subclasses should override __buildClass()."""
	333
	334	def __init__(self, dynamic_object=None, yaml=None, typeCheck='strong', parse=True, revision_dict=None):
	335	if revision_dict != None: self.revision_dict = revision_dict # Remember for when we build each individual class
	336	else:
	337	self.revision_dict = {\
	338	"__revision_number": 0,
	339	"__revision_id": 'unknown',
	340	"__revision_source": 'unknown',
	341	"__revision_tag": 'unknown'}
	342	if parse:
	343	if dynamic_object:
	344	self.parse(dynamic_object, typeCheck=typeCheck)
	345	else:
	346	dyno = Object()
	347	dyno.yaml = yaml
	348	self.parse(dyno, typeCheck=typeCheck)
	349
	350	def parse(self, dynamic_object, typeCheck='strong'):
	351	"""
	352	Initializer for a Factory, converting the given dynamic_object
	353	containing a (text) YAML object definition into the corresponding class-type
	354	with initializer.
	355
	356	typeCheck parameter can be one of 'strong' or 'cast':
	357	'strong': Class initializer should raise a TypeError when given
	358	anything but the correct type
	359	'cast': Class initializer should attempt to cast any input to the correct type
	360	"""
	361
	362	# Remember what kind of type-checking to do:
	363	if typeCheck not in ['strong', 'cast']:
	364	raise Exception('Incorrect input for typeCheck: %s\nExpected "strong" or "cast"'%(typeCheck))
	365	self.typeCheck = typeCheck
	366
	367	# Get a list of the objects to build:
	368	if isinstance(dynamic_object.yaml, list):
	369	objects = dynamic_object.yaml
	370	else:
	371	objects = [dynamic_object]
	372
	373	# Generate a dictionary of classes from the DynamicObjects given:
	374	self.classes = dict()
	375	for obj in objects:
	376
	377	# This loader breaks nothing anymore #everything currently
	378	loader = FactoryLoader(obj.yaml)
	379	#loader = yaml.Loader(obj.yaml)
	380
	381	# Dictionary with method and data signatures for the current object:
	382	objDefn = []
	383	while loader.check_data():
	384	objDefn.append(loader.get_data())
	385	loader.dispose()
	386
	387	# Parse the dictionary into a class definition:
	388	objClass = self.__buildClass(objDefn)
	389	self.classes.update(objClass)
	390
	391	def parseMethodSignature(self, sigName, methDict):
	392	""" Returns the python method corresponding to the given signature
	393	(given signature should be in the loaded YAML dict format.
	394
	395	Override this method for recognizing complex method signatures. """
	396
	397	raise SignatureException("Object abstract base class doesn't support any method signatures.")
	398
	399	def parseDataSignature(self, sigName, sig):
	400	""" Returns the Parameter object corresponding to the given signature.
	401
	402	This method should be overridden for recognizing complex data signatures
	403	(don't forget to call super(sig) for built-in data types though!) """
	404
	405	# Is the object an array with explicit default elements?:
	406	if isinstance(sig.yamlString, list):
	407	#length = len(sig.yamlString)
	408	if 'dtype' in sig.tag:
	409	default = np.array(sig.yamlString, dtype=sig.dtype)
	410	elif 'binary' == sig.tag:
	411	default = Binary(sig.yamlString["type"])
	412	else:
	413	default = sig.yamlString
	414	return _Parameter(sigName, True, default, length=None)
	415
	416	# Is the object an array with length and default value given?:
	417	if isinstance(sig.yamlString, dict) and "len" in sig.yamlString.keys():
	418	length = sig.yamlString["len"]
	419
	420	# Shape is given as something like [[],[]], not [2,2] - convert
	421	if isinstance(length, list):
	422
	423	def get_shape(lst):
	424	""" Gets the shape of a list recursively filled with empty lists """
	425	if lst == []: return [0]
	426	return [len(lst)] + get_shape(lst[0])
	427
	428	if len(length) > 0:
	429	if isinstance(length[0], list):
	430	length = get_shape(length)
	431	else:
	432	pass
	433	else:
	434	length = [0] # convert [] to [0] (numpy interprets [] as [1] for shapes)
	435
	436
	437	if 'complex' in sig.tag:
	438	imag = sig.yamlString["default"]["imag"]
	439	real = sig.yamlString["default"]["real"]
	440	default = sig.dtype(real) + sig.dtype(imag*1j)
	441	elif 'binary' == sig.tag:
	442	default = Binary(sig.yamlString["type"])
	443	else:
	444	default = sig.dtype(sig.yamlString["default"])
	445
	446	return _Parameter(sigName, True, default, length)
	447
	448	# The object is singular, with a given value:
	449	if 'complex' in sig.tag:
	450	imag = sig.yamlString["imag"]
	451	real = sig.yamlString["real"]
	452	default = sig.dtype(real) + sig.dtype(imag*1j)
	453	return _Parameter(sigName, True, default)
	454	elif 'binary' == sig.tag:
	455	default = Binary(sig.yamlString["type"])
	456	return _Parameter(sigName, False, default, classType=Binary)
	457	elif 'timestamp' in sig.tag:
	458	if isinstance(sig.yamlString, dict):
	459	if sig.tag in ['timestamp_picosecond', 'timestamp_ps']:
	460	try: s = sig.yamlString['second']
	461	except KeyError: s = sig.yamlString['s']
	462	try: ps = sig.yamlString['picosecond']
	463	except KeyError: ps = sig.yamlString['ps']
	464	return _Parameter(sigName, True, PrecisionTime.psTime(s, ps))
	465	elif sig.tag in ['timestamp_nanosecond', 'timestamp_ns']:
	466	try: s = sig.yamlString['second']
	467	except KeyError: s = sig.yamlString['s']
	468	try: ns = sig.yamlString['nanosecond']
	469	except KeyError: ns = sig.yamlString['ns']
	470	return _Parameter(sigName, True, PrecisionTime.nsTime(s, ns))
	471	else:
	472	if sig.tag in ['timestamp_picosecond', 'timestamp_ps']:
	473	return _Parameter(sigName, False, classType=PrecisionTime.psTime)
	474	elif sig.tag in ['timestamp_nanosecond', 'timestamp_ns']:
	475	return _Parameter(sigName, False, classType=PrecisionTime.nsTime)
	476	else:
	477	default = sig.dtype(sig.yamlString)
	478	return _Parameter(sigName, True, default) # not binary
	479
	480
	481
	482	def __parsePythonType(self, sigName, sig):
	483	""" Returns a _Parameter object, similar to parseDataSignature, but
	484	for a basic python type. """
	485
	486	if isinstance(sig, collections.OrderedDict):
	487	default = dict(sig) # Type-check user-defined !!maps as dicts, not OrderedDicts.
	488	else:
	489	default = sig # The signature sig is the default value itself
	490	return _Parameter(sigName, True, default)
	491
	492	def __parseReferenceSignature(self, sigName, ref_object, objClasses):
	493	""" Takes a reference object ref_object to be named sigName, and
	494	produces a _Parameter object with default value of None. """
	495
	496	# List of names of classes we've created so far:
	497	#print [x for x in objClasses]
	498	names = objClasses.keys()
	499
	500	if ref_object.yamlString in names:
	501	defaultType = objClasses[ref_object.yamlString]
	502	return _Parameter(sigName, classType=defaultType)
	503	else:
	504	try:
	505	# Try to find the class type in globals:
	506	className = objClasses[str(ref_object.yamlString)]
	507	defaultType = get_class(className)
	508	except (ValueError, KeyError):
	509	defaultType = _UnresolvedType(ref_object.yamlString)
	510	#raise NameError("Invalid reference to module %s"%(className,))
	511
	512	return _Parameter(sigName, classType=defaultType)
	513
	514	def __buildInitializer(self, className, classData):
	515	""" Constructs the initializer for an object which expects parameters
	516	listed in classData as input upon initialization. """
	517
	518	# Type of type-checking to use:
	519	strong = (self.typeCheck == 'strong')
	520	#cast = (self.typeCheck == 'cast')
	521
	522	def typeCheck(param, arg):
	523	"""
	524	Checks to see if the type of arg matches that of the corresponding param,
	525	casting arg to the correct type if desired.
	526	"""
	527	if isinstance(arg, param.classType): return arg
	528	if isinstance(arg, np.ndarray) and arg.dtype.type == param.classType:
	529	if not param.hasDefault: return arg
	530	if param.default.shape == (): return arg
	531	if param.default.shape[-1] == 0: return arg
	532	if arg.shape == param.default.shape: return arg
	533	if isinstance(arg, None.__class__): return arg
	534	if strong:
	535	raise TypeError("Incorrect input type on strong type-checking."+\
	536	" Expected %s - got %s"%(param.classType,arg.__class__))
	537	else:
	538	# If the parameter corresponding to the given argument has a non-NoneType default
	539	# value, then attempt to cast the argument into the correct parameter type
	540	if param.hasDefault and param.default != None:
	541	if isinstance(param.default, np.ndarray):
	542	return np.array(arg, dtype=param.default.dtype)
	543	else:
	544	return param.default.__class__(arg)
	545	else:
	546	return param.classType(arg)
	547
	548	"""
	549	attributes = {"__object_name": className,
	550	"__revision_number": self.svn_revision_number,
	551	"__revision_id": 'unknown',
	552	"__revision_source": 'unknown',
	553	"__revision_tag": 'unknown'}
	554	"""
	555	attributes = {} # Create new attributes dict for this particular class object
	556	attributes.update(self.revision_dict) # Revision info now passed into the factory
	557	attributes['__object_name'] = className
	558
	559	def init(_self, args, *kwargs):
	560	""" Dynamically generated initializer. """
	561
	562	# meta-data goes in the class, not the objects (commented the following out):
	563	"""
	564	# Initialize automatic class data
	565	for attr,value in attributes.items():
	566	try:
	567	value = kwargs[attr] # Are we given a value to over-ride with?
	568	del kwargs[attr] # Ignore the meta attribute later
	569	except KeyError:
	570	pass
	571	setattr(_self, attr, value)
	572	"""
	573
	574	# Set default values first (assume no parameters):
	575	for param in classData:
	576	if param.length:
	577	if isinstance(param.length, int): param.length = [param.length]
	578	default = np.empty(param.length, dtype=param.classType)
	579	if param.hasDefault:
	580	# Initialize array with default array value given:
	581	flatIter = default.flat
	582	for i in range(len(flatIter)):
	583	flatIter[i] = copy.deepcopy(param.default)
	584	else:
	585	# Initialize to None if no default given:
	586	default.fill(None)
	587	else:
	588	default = param.default
	589	setattr(_self, param.name, copy.deepcopy(default))
	590
	591	# Set attributes given by standard args:
	592	for i in range(len(args)):
	593	arg = typeCheck(classData[i], args[i])
	594	setattr(_self, classData[i].name, arg)
	595
	596	# Set named attributes (given by dictionary kwargs):
	597	for key,value in kwargs.items():
	598
	599	try: keyIndex = [param.name for param in classData].index(key)
	600	except ValueError:
	601	raise TypeError("'%s' is an invalid keyword argument"%(key,))
	602	arg = typeCheck(classData[keyIndex],value)
	603	#setattr(_self, key, value)
	604	setattr(_self, key, arg)
	605
	606
	607	# Object instantiation / creation time (if not already present):
	608	if not kwargs.has_key('__time_created'):
	609	setattr(_self, "__time_created", np.float64(time.time()))
	610
	611	return init, attributes
	612
	613	def __findClass(self, className, localClasses):
	614	""" Looks for the given className first in the given dictionary of localClasses
	615	then in the global definitions, returning the corresponding class object. Raises
	616	a KeyError if the class cannot be found. """
	617
	618	# If class definition was in the YAML file, extend that one:
	619	if className in localClasses.keys():
	620	return localClasses[className]
	621
	622	# Else try finding the class definition in our global scope:
	623	try: classObj = get_class(className)
	624	except KeyError:
	625	raise KeyError("Class '%s' not found in given YAML scope or global scope."%(className,))
	626	return classObj
	627
	628	def __buildClass(self, objDefn):
	629	""" Takes an object definition list / dictionary objDefn (loaded from a YAML
	630	object definition file) and creates a class, dynamically binding
	631	method and data signatures to the new class.
	632
	633	This method only performs a basic binding of method and data signatures to
	634	the new class. Object(s) having more complex initialization requirements
	635	should be given their own Factory subclass, overriding this
	636	and other methods."""
	637
	638	# objDefn is a list of dictionaries found in the YAML file - build each one...
	639	objClasses = dict()
	640	objClassesRev = dict()
	641
	642	# A list of all _Parameter objects created, used to resolve recursive
	643	# or "tangled" data structures
	644	allClassData = []
	645
	646	for document in objDefn:
	647	# Each document can contain multiple objects - build each one.
	648	# (NOTE: objects can cross reference each other in the same document
	649	# need to resolve Reference objects as last step)
	650	for objClassName in document.keys():
	651
	652	# The dictionary containing method & data signatures:
	653	objDict = document[objClassName]
	654
	655	# Extract data / attribute definitions (signatures) from the YAML dictionary
	656	# as well as method signatures and which classes this class extends:
	657	classData = []
	658	classMethods = dict()
	659	classBases = [Object]
	660
	661	# List structured documents result in a list of dicts each with one key:
	662	if isinstance(objDict, list): keys = [param.keys()[0] for param in objDict]
	663	# Otherwise the parameter names are just the keys of the dict
	664	else: keys = objDict.keys() # if key not found, raises AttributeError
	665
	666	for sigName in keys:
	667	#print sigName
	668	sig = objDict[sigName]
	669	#for f in _BuiltinDtype.python_dtypes: print f.__class__
	670	if sigName == '__extends':
	671	if isinstance(sig, str):
	672	sig = [sig]
	673	if isinstance(sig, list):
	674	for className in sig:
	675	newBase = self.__findClass(className, objClasses)
	676
	677	# Remove Object extension if newBase extends it already:
	678	if Object in classBases and Object in inspect.getmro(newBase):
	679	classBases.remove(Object)
	680	classBases += [newBase]
	681	else:
	682	raise TypeError("Incorrect format for extending classes - %s"%(sig,))
	683	elif isinstance(sig, BuiltinDtype):
	684	classData.append(self.parseDataSignature(sigName, sig))
	685	elif isinstance(sig, Lookup.python_dtypes):
	686	classData.append(self.__parsePythonType(sigName, sig))
	687	elif isinstance(sig, _Reference):
	688	classData.append(self.__parseReferenceSignature(sigName, sig, objClasses))
	689	elif isinstance(sig, _Method):
	690	classMethods[sigName] = self.parseMethodSignature(sigName, sig.yamlString)
	691	elif isinstance(sig, (PrecisionTime.nsTime, PrecisionTime.psTime)):
	692	classData.append(_Parameter(sigName, True, sig))
	693	elif isinstance(sig, _Parameter): # sig is already a parameter (we skipped a step)
	694	sig.name = sigName # we didn't know the name during load time - fill that in now
	695	classData.append(sig)
	696	else:
	697	msg = "Factory abstract base class doesn't " +\
	698	"support the following signature: %r \"%s\""%(sig.__class__,str(sig))
	699	print sig.__class__
	700	raise SignatureException(msg)
	701
	702	# Built-in attribute for all Dynamic Objects:
	703	classData.append(_Parameter('__time_created', classType=np.float64))
	704
	705	# Turn the object data / attributes into a usable __init__ method:
	706	classMethods["__init__"], meta_attributes = self.__buildInitializer(objClassName, classData)
	707
	708	# Keep a record of the _Parameters created for later type resolution
	709	allClassData.extend(classData)
	710
	711	"""
	712	__automaticMethods = {
	713	"getObjectName": lambda _self: getattr(_self, '__object_name'),
	714	"getRevisionNumber": lambda _self: getattr(_self, '__revision_number'),
	715	"getRevisionId": lambda _self: getattr(_self, '__revision_id'),
	716	"getRevisionSource": lambda _self: getattr(_self, '__revision_source'),
	717	"getRevisionTag": lambda _self: getattr(_self, '__revision_tag')
	718	}
	719	classMethods.update(__automaticMethods)
	720	"""
	721
	722	# Put the method signatures into a namespace for the new class,
	723	# then dynamically build the class from this namespace.
	724	classNamespace = classMethods
	725	classNamespace["meta_attributes"] = meta_attributes
	726	cls = type(str(objClassName), tuple(classBases), classNamespace)
	727	objClasses[objClassName] = cls
	728	objClassesRev['%s.%s'%(objClassName,cls.meta_attributes["__revision_number"])] = cls
	729
	730	# Create and register a constructor (loading) and representer (dumping) for the new class cls
	731	def construct_dynamic_object(loader, node):
	732	kwargs = loader.construct_mapping(node)
	733	# Remove revision control from loaded objects (info is in the class object!)
	734	for arg in kwargs.keys():
	735	if arg in getattr(Object, 'getters') and arg != '__time_created':
	736	del kwargs[arg]
	737	return cls(**kwargs)
	738	revision = cls.meta_attributes["__revision_number"]
	739	DynamicYAML.Loader.add_constructor(u'!%s.%s'%(str(objClassName),revision), construct_dynamic_object)
	740
	741	represent_dynamic_object = DynamicYAML.Dumper.represent_dynamic_object
	742	DynamicYAML.Dumper.add_representer(cls, represent_dynamic_object)
	743
	744	def findClass(className):
	745	""" Search for the most recently added class object with given className """
	746	try:
	747	return objClasses[className] # Look for reference to object in same YAML defn file:
	748	except KeyError:
	749	# Now look for reference to class object loaded from any YAML defn file, loading the
	750	# most recent version / revision (number) of the definition
	751	for dynClass in Object.dynamicClasses.keys()[::-1]:
	752	if dynClass.startswith(className):
	753	return Object.dynamicClasses[dynClass]
	754
	755	# Still unresolved - raise exception:
	756	allDynamicClasses = repr(objClasses.keys() + Object.dynamicClasses.keys())
	757	raise UnresolvedTypeException("Cannot resolve type '%s': Name not found in %s"%(className,allDynamicClasses))
	758
	759
	760	def resolve(param):
	761
	762	# Reference is just a string - that's the class name:
	763	if isinstance(param.classType.yamlObject, (str, unicode)):
	764	className = str(param.classType.yamlObject)
	765	param.classType = findClass(className)
	766	return
	767
	768	# Reference is a dict containing class name and / or default values:
	769	if not isinstance(param.classType.yamlObject, dict):
	770	raise UnresolvedTypeException("Cannot resolve reference of type '%s'"%(param.classType.yamlObject.__class__,))
	771
	772	# Definitely a dict:
	773	refDict = param.classType.yamlObject
	774
	775	# Determine the name of the class being referenced
	776	try:
	777	className = refDict["type"]
	778	except KeyError:
	779	raise KeyError("No 'type' key in reference dictionary for parameter '%s'"%(param.name,))
	780
	781	# Determine the class object corresponding to the class name
	782	param.classType = findClass(className)
	783
	784	try:
	785	defaultParams = refDict["default"]
	786	except KeyError:
	787	defaultParams = None
	788
	789	if defaultParams != None:
	790	for sub_param in defaultParams:
	791	if isinstance(sub_param.classType, _UnresolvedType):
	792	resolve(sub_param)
	793	param.default = param.classType( **defaultParams ) # Create the default object
	794	param.hasDefault = True
	795	else:
	796	param.hasDefault = False # for good measure
	797
	798	# Is it an object array?:
	799	if "len" in refDict.keys():
	800	param.length = refDict["len"]
	801
	802	# Resolve any unresolved data-types:
	803	for param in allClassData:
	804	if isinstance(param.classType, _UnresolvedType):
	805	resolve(param)
	806
	807	Object.dynamicClasses.update(objClassesRev)
	808	return objClasses
	809
	810	def load_defn(yaml):
	811	""" Shortcut for producing a single DynamicObject class object from
	812	the provided yaml definition in string format """
	813	return Factory(yaml=yaml).classes.values()[0]
	814
	815
	816

schainpy/serializer/test/DynamicSerializer.py created 644 +69 0

			@@ -0,0 +1,69
		1	#
		2	# rps 6/9/2014
		3	# mit haystack obs
		4	#
		5	# wrapper for Karl's code
		6
		7	import DynamicObject # used for serial/deserial of complex python objects
		8	import Serializer # used for serial/deserial of complex python
		9
		10	#
		11	class DynamicSerializer:
		12	#
		13	#------------------------------------------------------
		14	#
		15	def __init__(self,which='yaml'):
		16	#
		17	# choices are: yaml, msgpack, hdf5, json
		18	#
		19	self.err_f = False
		20	self.whichList = ['yaml', 'msgpack', 'hdf5', 'json'] # from Serialzer.py
		21	self.err_f,self.serializer = self.initSerializer(which)
		22	#
		23	#------------------------------------------------------
		24	#
		25	def initSerializer(self,which):
		26	#
		27	# calls REU student code that works but hasn't been walked-through
		28	# it's a dynamic serializer not strictly a yaml serializer
		29	#
		30	err_f = False
		31	match_f = False
		32	serializer = None
		33	ii = 0
		34	while ii < len(self.whichList):
		35	if (self.whichList[ii] == which):
		36	match_f = True
		37	break
		38	ii = ii + 1
		39	# end while
		40	if not match_f:
		41	err_f = True
		42	else:
		43	serializer = which
		44	serializer = Serializer.serializers[serializer]()
		45
		46	return err_f,serializer
		47	# end initSerializer
		48	#
		49	# --------------------------------------------------
		50	#
		51	def loads(self,element): # borrows name from json module (json - to - python)
		52	retval = self.serializer.fromSerial(element) # de-serialize
		53	return retval
		54	# end loads
		55	#
		56	# --------------------------------------------------
		57	#
		58	def dumps(self,element): # borrows name from json module (python - to - json)
		59	retval = self.serializer.toSerial(element) # serialize
		60	return retval
		61	# end dumps
		62	#
		63	# --------------------------------------------------
		64	#
		65	# end class DynamicSerializer
		66
		67	if __name__ == "__main__":
		68	DynamicSerializer()
		69	print "DynamicSerializer ran" No newline at end of file

schainpy/serializer/test/DynamicYAML.py created 644 +221 0

			@@ -0,0 +1,221
		1	'''
		2	Module containing YAML Loader and Dumper for DynamicObjects
		3	as well as built-in data types (numpy, PrecisionTime, datetime, Binary, ...)
		4
		5	$Id$
		6	'''
		7
		8	import yaml
		9	import OrderedYAML
		10	import DynamicObject
		11	import binascii
		12	import numpy as np
		13	import PrecisionTime
		14	import Lookup
		15	import pysvn
		16
		17	def load_defn(source, rev='head', repo=""):
		18	""" Import YAML definition(s) from given 'source' SVN location
		19	with specific revision number 'rev'. Returns a dict of the object
		20	names -> class object instances.
		21
		22	NOTE: Object defns with same name & revision number will conflict /
		23	cause issues (regardless of svn location). """
		24	client = pysvn.Client()
		25
		26	if rev == 'head':
		27	#yaml = client.cat(source)
		28	rev = client.info(repo).revision.number
		29
		30	if source.startswith('http'):
		31	yaml = client.cat("%s?p=%d"%(source, rev))
		32	else:
		33	pysvn_rev = pysvn.Revision(pysvn.opt_revision_kind.number, rev)
		34	yaml = client.cat(source, pysvn_rev)
		35
		36	revision_dict = {\
		37	"__revision_number": rev,
		38	"__revision_id": 'unknown',
		39	"__revision_source": source,
		40	"__revision_tag": 'unknown'}
		41
		42	return DynamicObject.Factory(yaml=yaml, revision_dict=revision_dict).classes
		43
		44	class Loader(OrderedYAML.Loader):
		45
		46	def __init__(self, stream):
		47	OrderedYAML.Loader.__init__(self, stream)
		48
		49	def construct_object(self, node, deep=False):
		50	""" Unresolved tags on mapping nodes come from un-imported YAML definitions - import it """
		51	resolved = node.tag in self.yaml_constructors
		52	resolved = resolved or any([node.tag.startswith(x) for x in self.yaml_multi_constructors])
		53	if isinstance(node, yaml.nodes.MappingNode) and not resolved:
		54	data = self.construct_mapping(self, node)
		55	self.constructed_objects[node] = data
		56	del self.recursive_objects[node]
		57	if data.has_key('__revision_source'):
		58	# TODO: Handle password authentication
		59	client = pysvn.Client()
		60	source = data['__revision_source']
		61	if source.startswith('http'):
		62	rev = data['__revision_number']
		63	defn = client.cat("%s?p=%d"%(source, rev))
		64	else:
		65	rev = pysvn.Revision(pysvn.opt_revision_kind.number, data['__revision_number'])
		66	defn = client.cat(source, revision=rev)
		67	DynamicObject.Factory(yaml=defn) # Register the object
		68
		69	constructor = self.yaml_constructors["%s.%s"%(data['__revision_name'], data['__revision_number'])]
		70	return constructor(node)
		71	else:
		72	raise Exception("Cannot load object with tag '%s' - cannot find YAML object definition (no __revision_source included)")
		73	else:
		74	return yaml.Loader.construct_object(self, node, deep=deep)
		75
		76	class Dumper(OrderedYAML.Dumper):
		77
		78	def __init__(self, stream, args, *kwargs):
		79	OrderedYAML.Dumper.__init__(self, stream, args, *kwargs)
		80
		81	def represent_dynamic_object(self, obj):
		82	"""
		83	Override the !!python/object:__main__.xxx syntax with
		84	!ObjectName.zzz where zzz is the revision number of the Object obj
		85	"""
		86
		87	state = {}
		88	state.update(obj.__dict__.items())
		89	state.update(obj.__class__.meta_attributes.items())
		90	name = obj.getObjectName() # obj.__class__.__name__
		91	revision = obj.getRevisionNumber()
		92	return self.represent_mapping(u'!%s.%s' % (name, revision), state)
		93
		94	# Dtypes to be stored as hex in YAML streams / strings
		95	hex_dtypes = ['float', 'complex', 'half', 'single', 'double']
		96
		97	# Register hex constructors for the numpy / built-in dtypes:
		98	dtypes = Lookup.numpy_dtypes
		99
		100	# Inverse lookup for accessing tags given a class instance:
		101	cls_dtypes = dict([(v,k) for (k,v) in dtypes.items()])
		102
		103	# Representer for numpy arrays:
		104	def ndarray_representer(dumper, obj):
		105	#if isinstance(obj, np.ndarray):
		106	tag = 'dtype.'+obj.dtype.type.__name__
		107	hexlify = any([x in tag for x in hex_dtypes])
		108	np_ary = obj
		109	#hex_ary = np.empty(np_ary.shape, dtype=yaml.nodes.ScalarNode)
		110	np_flat, hex_flat = np_ary.flat, [] #hex_ary.flat
		111	hex_flat.append(dumper.represent_sequence(u'tag:yaml.org,2002:seq', list(np_ary.shape), flow_style=True))
		112	if hexlify:
		113	lst = []
		114	for i in range(len(np_flat)):
		115	value = u'%s'%(np_flat[i],)
		116	node = dumper.represent_scalar(u'tag:yaml.org,2002:str', value, style='')
		117	lst.append(node)
		118	hex_flat.append(yaml.nodes.SequenceNode(u'tag:yaml.org,2002:seq', lst, flow_style=True))
		119	lst = []
		120	for i in range(len(np_flat)):
		121	if hexlify: value = u'%s'%(binascii.hexlify(np_flat[i]),)
		122	else: value = u'%s'%(np_flat[i],)
		123	node = dumper.represent_scalar(u'tag:yaml.org,2002:str', value, style='')
		124	if hexlify: lst.append(node)
		125	else: hex_flat.append(node)
		126	if hexlify: hex_flat.append(yaml.nodes.SequenceNode(u'tag:yaml.org,2002:seq', lst, flow_style=True))
		127	return yaml.nodes.SequenceNode(u'!%s'%(tag,), hex_flat, flow_style=True)
		128	Dumper.add_representer(np.ndarray, ndarray_representer)
		129
		130	# Constructor for ndarrays with arbitrary (specified) dtype:
		131	def ndarray_constructor(loader, node, dtype, hexlify=False):
		132	shape = loader.construct_sequence(node.value.pop(0))
		133	np_ary = np.empty(shape, dtype=dtype)
		134	np_flat = np_ary.flat # Flat iterator
		135	if hexlify:
		136	node.value[1].tag = node.tag
		137	node = node.value[1] # only look at hexlified values
		138	for i in range(len(node.value)):
		139	# Over-ride the 'tag:yaml.org,2002:str' tag with correct data type
		140	node.value[i].tag = node.tag
		141	value = loader.construct_object(node.value[i])
		142	#if hexlify:
		143	# value = binascii.unhexlify(value)
		144	# value = np.frombuffer(value, dtype=dtype)
		145	np_flat[i] = value
		146	return np_ary
		147
		148	class __dtype_con:
		149
		150	def __init__(self, tag):
		151	# Whether or not to convert to hex:
		152	hexlify = any([x in tag for x in hex_dtypes])
		153	dtype = dtypes[tag]
		154
		155	# Mutable list containing constructor & representer info
		156	self.fncn_attributes = [tag, hexlify, dtype]
		157
		158	def dtype_constructor(loader, node):
		159	tag, hexlify, dtype = self.fncn_attributes
		160	if isinstance(node, yaml.nodes.SequenceNode):
		161	return ndarray_constructor(loader, node, dtype, hexlify=hexlify)
		162	else: # isinstance(node, yaml.nodes.ScalarNode):
		163	value = loader.construct_scalar(node)
		164	dtype = dtypes[node.tag[1:]]
		165	if hexlify:
		166	value = binascii.unhexlify(value)
		167	value = np.frombuffer(value, dtype=dtype)[0]
		168	else:
		169	value = dtype(value)
		170	return value
		171
		172	def dtype_representer(dumper, obj):
		173	tag, hexlify, dtype = self.fncn_attributes
		174	if isinstance(obj, float): obj = np.float64(obj)
		175	if hexlify: value = u'%s'%(binascii.hexlify(obj),)
		176	else: value = u'%s'%(obj,)
		177	try: tag = u'!%s'%(cls_dtypes[obj.__class__]) # 'dtype.'+obj.__class__.__name__ # bullshit...
		178	except KeyError: tag = ''
		179	node = dumper.represent_scalar(tag, value, style='')
		180	return node
		181
		182	self.dtype_constructor = dtype_constructor
		183	self.dtype_representer = dtype_representer
		184
		185	keys = [x for x in dtypes.keys() if x != 'dtype.int' and x != 'dtype.bool']
		186	print keys
		187
		188	n = len(keys)
		189	print n
		190	i=0
		191
		192	for tag in keys:
		193	dtype = __dtype_con(tag)
		194	dtype_constructor = dtype.dtype_constructor
		195	dtype_representer = dtype.dtype_representer
		196	Loader.add_constructor(u'!%s'%(tag,), dtype_constructor)
		197	Dumper.add_representer(dtypes[tag], dtype_representer)
		198
		199	# Precision time constructors & representers:
		200	def ns_rep(dumper, obj):
		201	state = {'second': obj.__dict__['second'], 'nanosecond': obj.__dict__['nanosecond']}
		202	return dumper.represent_mapping(u'!timestamp_ns', state)
		203	def ps_rep(dumper, obj):
		204	state = {'second': obj.__dict__['second'], 'picosecond': obj.__dict__['picosecond']}
		205	return dumper.represent_mapping(u'!timestamp_ps', state)
		206	def ns_con(loader, node): return PrecisionTime.nsTime(**loader.construct_mapping(node))
		207	def ps_con(loader, node): return PrecisionTime.psTime(**loader.construct_mapping(node))
		208
		209	Dumper.add_representer(PrecisionTime.nsTime, ns_rep)
		210	Dumper.add_representer(PrecisionTime.psTime, ps_rep)
		211	Loader.add_constructor(u'!timestamp_ns', ns_con)
		212	Loader.add_constructor(u'!timestamp_nanosecond', ns_con)
		213	Loader.add_constructor(u'!timestamp_ps', ps_con)
		214	Loader.add_constructor(u'!timestamp_picosecond', ps_con)
		215
		216	# Binary object constructor & representer:
		217	def bin_rep(dumper, obj): return dumper.represent_mapping(u'!binary', obj.__dict__)
		218	def bin_con(loader, node): return DynamicObject.Binary(**loader.construct_mapping(node))
		219	Dumper.add_representer(DynamicObject.Binary, bin_rep)
		220	Loader.add_constructor(u'!binary', bin_con)
		221

schainpy/serializer/test/Lookup.py created 644 +62 0

			@@ -0,0 +1,62
		1	'''
		2	Helper module for DynamicObject module - contains dictionaries
		3	of data types built-in to our YAML IDL, converting backing and forth between
		4	strings / YAML tags and python class instances.
		5
		6	$Id$
		7	'''
		8
		9	import datetime
		10	import numpy as np
		11	import PrecisionTime
		12	import DynamicObject
		13	Binary = DynamicObject.Binary
		14	import platform
		15	import collections
		16
		17	# Implicit Types:
		18	python_dtypes = tuple([bool,int,long,float,str,datetime.datetime,list,
		19	set,dict,tuple,unicode])
		20
		21	# Numpy Data-types:
		22	numpy_dtypes = {'dtype.bool': bool, 'dtype.int': np.int, 'dtype.int8': np.int8,
		23	'dtype.int16': np.int16, 'dtype.int32': np.int32, 'dtype.int64': np.int64,
		24	'dtype.uint8': np.uint8, 'dtype.uint16': np.uint16, 'dtype.uint32': np.uint32,
		25	'dtype.uint64': np.uint64, 'dtype.float': np.float, 'dtype.float16': np.float16,
		26	'dtype.float32': np.float32, 'dtype.float64': np.float64, 'dtype.complex': np.complex,
		27	'dtype.complex64': np.complex64, 'dtype.complex128': np.complex128,
		28	'dtype.byte': np.byte, 'dtype.short': np.short, 'dtype.intc': np.intc,
		29	'dtype.longlong': np.longlong, 'dtype.intp': np.intp, 'dtype.ubyte': np.ubyte,
		30	'dtype.ushort': np.ushort, 'dtype.uintc': np.uintc, 'dtype.uint': np.uint,
		31	'dtype.uintc': np.uintc, 'dtype.uint': np.uint, 'dtype.ulonglong': np.ulonglong,
		32	'dtype.uintp': np.uintp, 'dtype.half': np.half, 'dtype.single': np.single,
		33	'dtype.double': np.double, 'dtype.longfloat': np.longfloat,
		34	'dtype.csingle': np.csingle, 'dtype.clongfloat': np.clongfloat, 'dtype.long': np.long}
		35
		36	if platform.architecture()[0] != '32bit': # 64bit - certain numpy types exist
		37	numpy_dtypes.update({'dtype.float128': np.float128, 'dtype.complex256': np.complex256})
		38	bit32 = False
		39	else:
		40	bit32 = True
		41	#else: # 32 bit - fix 32 bit integer issue.
		42	# np.int32 = np.int
		43	# bit32 = True
		44
		45	# Built-in objects:
		46	builtin_objects = {'binary': Binary, 'nsTime': PrecisionTime.nsTime, 'psTime': PrecisionTime.psTime,
		47	'timestamp_ns': PrecisionTime.nsTime, 'timestamp_ps': PrecisionTime.psTime,
		48	'timestamp_nanosecond': PrecisionTime.nsTime, 'timestamp_picosecond': PrecisionTime.psTime,
		49	'datetime': datetime.datetime, 'Binary': Binary}
		50
		51	builtin_objects_simple = {'nsTime': PrecisionTime.nsTime, 'psTime': PrecisionTime.psTime,
		52	'binary': Binary, 'datetime': datetime.datetime,
		53	'Binary': Binary}
		54
		55	# Inverse lookup for accessing tags given a class instance:
		56	cls_dtypes = dict([(v,k) for (k,v) in numpy_dtypes.items()])
		57	obj_dtypes = dict([(v,k) for (k,v) in builtin_objects_simple.items()])
		58
		59	# Pointer to the list of all Object classes created, as located in the Object module / class:
		60	dynamicClasses = DynamicObject.Object.dynamicClasses
		61
		62

schainpy/serializer/test/OrderedYAML.py created 644 +89 0

			@@ -0,0 +1,89
		1	'''
		2	A YAML Loader and Dumper which provide ordered dictionaries in place
		3	of dictionaries (to keep the order of attributes as
		4	found in the original YAML object file).
		5
		6	This module is modified from a submission on pyyaml.org:
		7	http://pyyaml.org/attachment/ticket/161/use_ordered_dict.py
		8
		9	$Id$
		10	'''
		11
		12	import yaml
		13	import collections
		14
		15	class Loader(yaml.loader.Loader):
		16	""" YAML Loader producing OrderedDicts in place of dicts """
		17
		18	def __init__(self, stream):
		19	yaml.loader.Loader.__init__(self, stream)
		20
		21	def construct_ordered_mapping(self, node, deep=False):
		22	""" Replacement mapping constructor producing an OrderedDict """
		23	if not isinstance(node, yaml.MappingNode):
		24	raise yaml.constructor.ConstructorError(None, None,
		25	"expected a mapping node, but found %s" % node.id,
		26	node.start_mark)
		27	mapping = collections.OrderedDict()
		28	for key_node, value_node in node.value:
		29	key = self.construct_object(key_node, deep=deep)
		30	if not isinstance(key, collections.Hashable):
		31	raise yaml.constructor.ConstructorError("while constructing a mapping", node.start_mark,
		32	"found unhashable key", key_node.start_mark)
		33	value = self.construct_object(value_node, deep=deep)
		34	mapping[key] = value
		35	return mapping
		36	# yaml.constructor.BaseConstructor.construct_mapping = construct_ordered_mapping
		37
		38	def construct_mapping(self, node, deep=False):
		39	return self.construct_ordered_mapping(node, deep=deep)
		40
		41	def construct_yaml_map_with_ordered_dict(self, node):
		42	data = collections.OrderedDict()
		43	yield data
		44	value = self.construct_mapping(node)
		45	data.update(value)
		46
		47	class Dumper(yaml.dumper.Dumper):
		48	""" YAML Dumper producing documents from OrderedDicts """
		49
		50	def __init__(self, stream, args, *kwargs):
		51	yaml.dumper.Dumper.__init__(self, stream, args, *kwargs)
		52
		53	def represent_ordered_mapping(self, tag, mapping, flow_style=None):
		54	""" Replacement mapping representer for OrderedDicts """
		55	value = []
		56	node = yaml.MappingNode(tag, value, flow_style=flow_style)
		57	if self.alias_key is not None:
		58	self.represented_objects[self.alias_key] = node
		59	best_style = True
		60	if hasattr(mapping, 'items'):
		61	mapping = list(mapping.items())
		62	for item_key, item_value in mapping:
		63	node_key = self.represent_data(item_key)
		64	node_value = self.represent_data(item_value)
		65	if not (isinstance(node_key, yaml.ScalarNode) and not node_key.style):
		66	best_style = False
		67	if not (isinstance(node_value, yaml.ScalarNode) and not node_value.style):
		68	best_style = False
		69	value.append((node_key, node_value))
		70	if flow_style is None:
		71	if self.default_flow_style is not None:
		72	node.flow_style = self.default_flow_style
		73	else:
		74	node.flow_style = best_style
		75	return node
		76	# yaml.representer.BaseRepresenter.represent_mapping = represent_ordered_mapping
		77
		78	def represent_mapping(self, tag, mapping, flow_style=None):
		79	return self.represent_ordered_mapping(tag, mapping, flow_style=flow_style)
		80
		81	# Loader.add_constructor(
		82	# u'tag:yaml.org,2002:map',
		83	# Loader.construct_yaml_map_with_ordered_dict
		84	# )
		85	#
		86	# Dumper.add_representer(
		87	# collections.OrderedDict,
		88	# yaml.representer.SafeRepresenter.represent_dict
		89	# )

schainpy/serializer/test/PrecisionTime.py created 644 +211 0

			@@ -0,0 +1,211
		1	#!/usr/local/midas/bin/python
		2
		3	"""PrecisionTime.py is a collection of python classes to manipulate times with high
		4	precision using integer logic.
		5
		6	Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu May 24, 2007
		7
		8	$Id$
		9	"""
		10	import types
		11
		12	class nsTime:
		13	"""nsTime is a class to handle times given as UT second (integer) and nanosecond (integer)
		14
		15	If nanosecond > 1E9, seconds will be added to second
		16	"""
		17
		18	def __init__(self, second, nanosecond):
		19	self.second = int(second)
		20	if self.second < 0:
		21	raise ValueError, 'seconds must be greater than 0, not %i' % (self.second)
		22	nanosecond = long(nanosecond)
		23	if nanosecond < 0:
		24	raise ValueError, 'nanoseconds must be greater 0, not %i' % (nanosecond)
		25	addSec = nanosecond / 1000000000
		26	if addSec > 0:
		27	self.second += addSec
		28	self.nanosecond = nanosecond % 1000000000
		29	self.totalNS = long(self.nanosecond) + long(self.second) * 1000000000
		30
		31
		32	def __add__(self, other):
		33	"""__add__ another nsTime to this one and return a new one as result
		34	"""
		35	nsResult = self.nanosecond + other.nanosecond
		36	addSec = int(nsResult / 1000000000)
		37	newSec = self.second + other.second + addSec
		38	newNS = nsResult % 1000000000
		39	return(nsTime(newSec, newNS))
		40
		41
		42	def increase(self, other):
		43	"""increase adds other to self, changing self (rather than creating a new object)
		44	"""
		45	nsResult = self.nanosecond + other.nanosecond
		46	addSec = int(nsResult / 1000000000)
		47	self.second = self.second + other.second + addSec
		48	self.nanosecond = nsResult % 1000000000
		49	self.totalNS = long(self.nanosecond) + long(self.second) * 1000000000
		50
		51
		52	def __sub__(self, other):
		53	"""__sub__ another nsTime from this one and return a new one as result
		54	"""
		55	nsResult = self.nanosecond - other.nanosecond
		56	if nsResult < 0:
		57	addSec = 1
		58	nsResult += 1000000000
		59	else:
		60	addSec = 0
		61	newSec = (self.second - other.second) - addSec
		62	return(nsTime(newSec, nsResult))
		63
		64
		65	def multiply(self, factor):
		66	"""multiply this nsTime times an integer
		67	"""
		68	if type(factor) not in (types.IntType, types.LongType):
		69	raise ValueError, 'Illegal type %s passed into nsTime.multiply' % (str(type(factor)))
		70	newTotalNS = self.totalNS * factor
		71	newSeconds = int(newTotalNS / 1000000000)
		72	newNanoseconds = int(newTotalNS - (newSeconds * 1000000000))
		73	return(nsTime(newSeconds, newNanoseconds))
		74
		75
		76	def integerDivision(self, other):
		77	"""integerDivision returns the total number of other nsTimes that fit in self
		78	"""
		79	return(int(self.totalNS / other.totalNS))
		80
		81	def getUnixTime(self):
		82	""" getUnixTime() returns a Unix style time as a float. """
		83	return(float(self.second) + float(self.nanosecond)/1.0e9)
		84
		85	def __mod__(self, other):
		86	"""__mod__ implements self % other.
		87	"""
		88	if type(other) in (types.IntType, types.LongType):
		89	return self.totalNS % other
		90	else:
		91	return self.totalNS % other.totalNS
		92
		93	def __eq__(self, other):
		94	""" equality of two nsTime objects """
		95	if not (hasattr(other, 'second') and hasattr(other, 'nanosecond')): return False
		96	return self.__cmp__(other) == 0
		97
		98	def __cmp__(self, other):
		99	"""compare two nsTime objects
		100	"""
		101	result = cmp(self.second, other.second)
		102	if result != 0:
		103	return(result)
		104
		105	return(cmp(self.nanosecond, other.nanosecond))
		106
		107
		108	def __str__(self):
		109	return '%d.%09d' % (self.second, self.nanosecond)
		110
		111
		112	class psTime:
		113	"""psTime is a class to handle times given as UT second (integer) and picosecond (integer)
		114
		115	If picosecond > 1E12, seconds will be added to second
		116	"""
		117
		118	def __init__(self, second, picosecond):
		119	self.second = int(second)
		120	if self.second < 0:
		121	raise ValueError, 'seconds must be greater than 0, not %i' % (self.second)
		122	picosecond = long(picosecond)
		123	if picosecond < 0:
		124	raise ValueError, 'picoseconds must be greater 0, not %i' % (picosecond)
		125	addSec = picosecond / 1000000000000
		126	if addSec > 0:
		127	self.second += addSec
		128	self.picosecond = picosecond % 1000000000000
		129	self.totalPS = long(self.picosecond) + long(self.second) * 1000000000000
		130
		131
		132	def __add__(self, other):
		133	"""__add__ another psTime to this one and return a new one as result
		134	"""
		135	psResult = self.picosecond + other.picosecond
		136	addSec = int(psResult / 1000000000000)
		137	newSec = self.second + other.second + addSec
		138	newPS = psResult % 1000000000000
		139	return(psTime(newSec, newPS))
		140
		141
		142	def increase(self, other):
		143	"""increase adds other to self, changing self (rather than creating a new object)
		144	"""
		145	psResult = self.picosecond + other.picosecond
		146	addSec = int(psResult / 1000000000000)
		147	self.second = self.second + other.second + addSec
		148	self.picosecond = psResult % 1000000000000
		149	self.totalPS = long(self.picosecond) + long(self.second) * 1000000000000
		150
		151
		152	def __sub__(self, other):
		153	"""__sub__ another psTime from this one and return a new one as result
		154	"""
		155	psResult = self.picosecond - other.picosecond
		156	if psResult < 0:
		157	addSec = 1
		158	psResult += 1000000000000
		159	else:
		160	addSec = 0
		161	newSec = (self.second - other.second) - addSec
		162	return(psTime(newSec, psResult))
		163
		164
		165	def multiply(self, factor):
		166	"""multiply this psTime times an integer
		167	"""
		168	if type(factor) not in (types.IntType, types.LongType):
		169	raise ValueError, 'Illegal type %s passed into psTime.multiply' % (str(type(factor)))
		170	newTotalPS = self.totalPS * factor
		171	newSeconds = int(newTotalPS / 1000000000000)
		172	newPicoseconds = int(newTotalPS - (newSeconds * 1000000000000))
		173	return(psTime(newSeconds, newPicoseconds))
		174
		175
		176	def integerDivision(self, other):
		177	"""integerDivision returns the total number of other psTimes that fit in self
		178	"""
		179	return(int(self.totalPS / other.totalPS))
		180
		181	def getUnixTime(self):
		182	""" getUnixTime() returns a Unix style time as a float. """
		183	return(float(self.second) + float(self.picosecond)/1.0e12)
		184
		185	def __mod__(self, other):
		186	"""__mod__ implements self % other.
		187	"""
		188	if type(other) in (types.IntType, types.LongType):
		189	return self.totalPS % other
		190	else:
		191	return self.totalPS % other.totalPS
		192
		193	def __eq__(self, other):
		194	""" equality of two psTime objects """
		195	if not (hasattr(other, 'second') and hasattr(other, 'picosecond')): return False
		196	return self.__cmp__(other) == 0
		197
		198	def __cmp__(self, other):
		199	"""compare two psTime objects
		200	"""
		201	result = cmp(self.second, other.second)
		202	if result != 0:
		203	return(result)
		204
		205	return(cmp(self.picosecond, other.picosecond))
		206
		207
		208	def __str__(self):
		209	return '%d.%12d' % (self.second, self.picosecond)
		210
		211

schainpy/serializer/test/Serializer.py created 644 +373 0

			@@ -0,0 +1,373
		1	'''
		2	Module containing classes with serialization and de-serialization services.
		3
		4	$Id$
		5	'''
		6
		7	import Lookup
		8	import numpy as np
		9	import zlib
		10	import binascii
		11	import yaml
		12	import DynamicObject
		13	import DynamicYAML
		14	import PrecisionTime
		15	import datetime
		16	import re
		17	import os
		18	#import json
		19	import jsonpickle
		20	import jpickle
		21	import h5py
		22	import msgpack
		23
		24	class CompressionException(Exception): pass
		25
		26	class Serializer:
		27	""" Base class for pickle-like serialization
		28	of DynamicObjects (with compression available) """
		29
		30	def __init__(self):
		31	pass
		32
		33	def dump(self, obj, file_name, compression=None):
		34	""" Dumps obj to file_name, serializing the obj with toSerial() """
		35	string = self.dumps(obj, compression)
		36	open(file_name, 'w').write(string)
		37
		38	def dumps(self, obj, compression=None):
		39	""" Returns serialized string representing obj, using toSerial() to serialize """
		40	if compression == 'gzip':
		41	return zlib.compress(self.toSerial(obj))
		42	elif compression in [None, '']:
		43	return self.toSerial(obj)
		44	else:
		45	raise CompressionException("Invalid decompression type '%r'"%(compression,))
		46
		47	def load(self, file_name, compression=None):
		48	""" Returns the Object located in file_name, using fromSerial() to deserialize """
		49	string = open(file_name, 'r').read()
		50	return self.loads(string, compression)
		51
		52	def loads(self, string, compression=None):
		53	""" Returns the Object serialized as the given string """
		54	if compression == 'gzip':
		55	return self.fromSerial(zlib.decompress(string))
		56	elif compression in [None, '']:
		57	return self.fromSerial(string)
		58	else:
		59	raise CompressionException("Invalid compression type '%r'"%(compression,))
		60
		61	def fromSerial(self, string):
		62	""" Deserializes the given string """
		63	return string
		64
		65	def toSerial(self, obj):
		66	""" Serializes the given object """
		67	return repr(obj)
		68
		69	class YAMLSerializer(Serializer):
		70	""" Serializes a Object to/from YAML format """
		71
		72	def __init__(self):
		73	Serializer.__init__(self)
		74
		75	def fromSerial(self, string):
		76	loader = DynamicYAML.Loader
		77	return yaml.load(string, Loader=loader)
		78
		79	def toSerial(self, obj):
		80	dumper = DynamicYAML.Dumper
		81	return yaml.dump(obj, Dumper=dumper)
		82
		83	# Regular expression taken from yaml.constructor.py
		84	timestamp_regexp_str = str(\
		85	ur'^(?P<year>[0-9][0-9][0-9][0-9])'
		86	ur'-(?P<month>[0-9][0-9]?)'
		87	ur'-(?P<day>[0-9][0-9]?)'
		88	ur'(?:(?:[Tt]\|[ \t]+)'
		89	ur'(?P<hour>[0-9][0-9]?)'
		90	ur':(?P<minute>[0-9][0-9])'
		91	ur':(?P<second>[0-9][0-9])'
		92	ur'(?:\.(?P<fraction>[0-9]*))?'
		93	ur'(?:[ \t]*(?P<tz>Z\|(?P<tz_sign>[-+])(?P<tz_hour>[0-9][0-9]?)'
		94	ur'(?::(?P<tz_minute>[0-9][0-9]))?))?)?$')
		95	timestamp_regexp = re.compile(timestamp_regexp_str, re.X)
		96
		97	def construct_timestamp(value):
		98	""" Taken & modified from yaml.constructor.py """
		99
		100	match = timestamp_regexp.match(value)
		101	#print "&%s&"%(value,)
		102	#print timestamp_regexp_str
		103	values = match.groupdict()
		104	year = int(values['year'])
		105	month = int(values['month'])
		106	day = int(values['day'])
		107	if not values['hour']:
		108	return datetime.date(year, month, day)
		109	hour = int(values['hour'])
		110	minute = int(values['minute'])
		111	second = int(values['second'])
		112	fraction = 0
		113	if values['fraction']:
		114	fraction = values['fraction'][:6]
		115	while len(fraction) < 6:
		116	fraction += '0'
		117	fraction = int(fraction)
		118	delta = None
		119	if values['tz_sign']:
		120	tz_hour = int(values['tz_hour'])
		121	tz_minute = int(values['tz_minute'] or 0)
		122	delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute)
		123	if values['tz_sign'] == '-':
		124	delta = -delta
		125	data = datetime.datetime(year, month, day, hour, minute, second, fraction)
		126	if delta:
		127	data -= delta
		128	return data
		129
		130	class MessagePackSerializer(Serializer):
		131	""" Serializes a Object to/from MessagePack format """
		132
		133	def __fromSerial(self, msg_dict):
		134	if not isinstance(msg_dict, (dict, list, tuple)):
		135	return msg_dict # msg_dict is a value - return it
		136	if isinstance(msg_dict, dict) and msg_dict.has_key('__meta_attributes'):
		137	meta_attr = msg_dict['__meta_attributes']
		138	msg_dict.pop('__meta_attributes')
		139	if meta_attr.has_key('type'):
		140	if meta_attr['type'] == 'datetime':
		141	return construct_timestamp(str(msg_dict['ts']))
		142	elif meta_attr['type'] == 'nsTime':
		143	msg_dict.pop('totalNS')
		144	elif meta_attr['type'] == 'psTime':
		145	msg_dict.pop('totalPS')
		146	try: dtype = Lookup.cls_dtypes[meta_attr['type']]
		147	except KeyError: dtype = Lookup.builtin_objects[meta_attr['type']]
		148	return dtype(**msg_dict)
		149	else:
		150	for key in msg_dict.keys():
		151	msg_dict[key] = self.__fromSerial(msg_dict[key])
		152	cls = Lookup.dynamicClasses['%s.%s'%(meta_attr['__object_name'],meta_attr['__revision_number'])]
		153	return cls(**msg_dict)
		154	elif msg_dict == ():
		155	return []
		156	elif isinstance(msg_dict[0], str) and msg_dict[1] in Lookup.numpy_dtypes and\
		157	isinstance(msg_dict, tuple) and len(msg_dict) == 2:
		158	value = binascii.unhexlify(msg_dict[0])
		159	return np.frombuffer(value, dtype=Lookup.numpy_dtypes[msg_dict[1]])[0]
		160
		161	tup = isinstance(msg_dict, tuple)
		162	if tup and len(msg_dict) > 1 and msg_dict[0] in Lookup.numpy_dtypes.keys():
		163	msg_flat = list(msg_dict)
		164	dtypeName = msg_flat.pop(0)
		165	dtype = Lookup.numpy_dtypes[dtypeName]
		166	shape = msg_flat.pop(0)
		167	obj = np.empty(shape, dtype=dtype)
		168	np_flat = obj.flat
		169	for i in range(len(np_flat)):
		170	if isinstance(msg_flat[i], float):
		171	value = msg_flat[i]
		172	else:
		173	value = self.__fromSerial((msg_flat[i], dtypeName))
		174	np_flat[i] = value
		175	return obj
		176	else:
		177	return msg_dict
		178
		179	def fromSerial(self, string):
		180	msg_dict = msgpack.unpackb(string)
		181	return self.__fromSerial(msg_dict)
		182
		183	def __toSerial(self, obj):
		184
		185	if isinstance(obj, (PrecisionTime.nsTime, PrecisionTime.psTime, DynamicObject.Binary, datetime.datetime)):
		186	msg_dict = {}
		187	if isinstance(obj, datetime.datetime):
		188	msg_dict['ts'] = obj.isoformat(' ')
		189	else:
		190	msg_dict.update(obj.__dict__)
		191	msg_dict['__meta_attributes'] = {'type': obj.__class__.__name__}
		192	return msg_dict
		193	elif isinstance(obj, DynamicObject.Object):
		194	msg_dict = {}
		195	for key, value in obj.__dict__.items():
		196	msg_dict[key] = self.__toSerial(value)
		197
		198	msg_dict['__meta_attributes'] = obj.__class__.meta_attributes
		199	return msg_dict
		200	elif isinstance(obj, np.ndarray):
		201	np_flat = obj.flat
		202	msg_flat = []
		203	msg_flat.append(Lookup.cls_dtypes[obj.dtype.type]) # dtype is first element
		204	msg_flat.append(obj.shape) # shape of array is second element
		205	for i in range(len(np_flat)):
		206	toSer = self.__toSerial(np_flat[i])
		207	if isinstance(toSer, tuple):
		208	msg_flat.append(toSer[0])
		209	else:
		210	msg_flat.append(toSer)
		211	return list(msg_flat)
		212
		213	is_builtin = obj.__class__ in Lookup.numpy_dtypes.values()
		214	#is_python = isinstance(obj, Lookup.python_dtypes)
		215	if is_builtin: # and not is_python:
		216	try:
		217	#print obj.__class__
		218	msg_dict = (binascii.hexlify(obj), Lookup.cls_dtypes[obj.__class__])
		219	return msg_dict
		220	except TypeError: # numpy dtype is a built-in python type... force the hexlify:
		221	if not Lookup.bit32:
		222	if obj.__class__ == int: return (binascii.hexlify(np.int64(obj)), 'dtype.int64')
		223	elif obj.__class__ == float: return (binascii.hexlify(np.float64(obj)), 'dtype.float64')
		224	else:
		225	#print np.int32(obj).__class__, obj.__class__
		226	if obj.__class__ == int: return (binascii.hexlify(np.int32(obj)), 'dtype.int32')
		227	elif obj.__class__ == float: return (binascii.hexlify(np.float32(obj)), 'dtype.float32')
		228	raise
		229	else:
		230	return obj
		231
		232	def toSerial(self, obj):
		233	#if Lookup.bit32 and np.int32 != np.int: np.int32 = np.int
		234	toSer = self.__toSerial(obj)
		235	#print toSer
		236	value = msgpack.packb(toSer)
		237	return value
		238
		239	class HDF5Serializer(Serializer):
		240	""" Serializes a Object to/from HDF5 format """
		241
		242	tmp_num = 0
		243
		244	def __fromSerial(self, grp):
		245
		246	if isinstance(grp, h5py.Dataset):
		247	return grp.value
		248
		249	elif isinstance(grp, h5py.Group) and '__type' in grp.keys():
		250	typ = grp['__type'].value
		251	if typ == 'datetime':
		252	return construct_timestamp(str(grp['ts'].value))
		253	elif typ == '_null':
		254	return None
		255	elif typ == 'tuple':
		256	return tuple(grp['tuple'])
		257	elif typ == 'empty_list':
		258	return []
		259	try: cls = Lookup.builtin_objects_simple[typ]
		260	except KeyError: cls = Lookup.dynamicClasses[typ]
		261	args = []
		262	for key in grp.keys():
		263	fromSer = self.__fromSerial(grp[key])
		264	args.append((key, fromSer))
		265	kwargs = dict(args)
		266	kwargs.pop('__type')
		267	return cls(**kwargs)
		268	#else:
		269	# return grp.value
		270
		271
		272	def fromSerial(self, string):
		273	HDF5Serializer.tmp_num += 1
		274	fn = 'tmp%d.hdf5'%(HDF5Serializer.tmp_num-1,)
		275	fp = open(fn, 'wb')
		276	fp.write(string)
		277	fp.flush(), fp.close()
		278
		279	root = h5py.File(fn, driver='core')
		280	try:
		281	fromSer = self.__fromSerial(root['dataset'])
		282	except:
		283	root.flush(), root.close()
		284	os.remove(fn)
		285	raise
		286
		287	root.flush(), root.close()
		288	os.remove(fn)
		289
		290	return fromSer
		291
		292	def __toSerial(self, obj, grp, name):
		293
		294	if isinstance(obj, datetime.datetime):
		295	sub_grp = grp.create_group(name)
		296	sub_grp['__type'] = 'datetime'
		297	sub_grp['ts'] = obj.isoformat(' ')
		298
		299	elif isinstance(obj, tuple(Lookup.builtin_objects_simple.values())):
		300	sub_grp = grp.create_group(name)
		301	sub_grp['__type'] = Lookup.obj_dtypes[obj.__class__]
		302	for key, value in obj.__dict__.items():
		303	if value != None and key not in ['totalNS', 'totalPS']:
		304	sub_grp[key] = value
		305
		306	elif obj == None:
		307	sub_grp = grp.create_group(name)
		308	sub_grp['__type'] = '_null'
		309
		310	elif isinstance(obj, DynamicObject.Object):
		311	# Create the new group and assign unique identifier for this type of DynamicObject
		312	sub_grp = grp.create_group(name)
		313	tag = '%s.%s'%(obj.getObjectName(), obj.getRevisionNumber())
		314	sub_grp['__type'] = tag
		315	# Put all of the DynamicObject's attributes into the new h5py group
		316	for key, value in obj.__dict__.items():
		317	self.__toSerial(value, sub_grp, key)
		318
		319	elif isinstance(obj, tuple):
		320	sub_grp = grp.create_group(name)
		321	sub_grp['__type'] = 'tuple'
		322	sub_grp['tuple'] = obj
		323
		324	elif isinstance(obj, list) and len(obj) == 0:
		325	sub_grp = grp.create_group(name)
		326	sub_grp['__type'] = 'empty_list'
		327
		328	else:
		329	grp[name] = obj
		330
		331	def toSerial(self, obj):
		332	HDF5Serializer.tmp_num += 1
		333	fn = 'tmp%d.hdf5'%(HDF5Serializer.tmp_num,)
		334	root = h5py.File(fn, driver='core')
		335	try:
		336	self.__toSerial(obj, root, 'dataset')
		337	except:
		338	root.flush(), root.close()
		339	os.remove(fn)
		340	raise
		341	root.flush(), root.close()
		342
		343	fp = open(fn, 'rb')
		344	msg = fp.read()
		345	fp.close()
		346	os.remove(fn)
		347
		348	return msg
		349
		350	# Alias for the standard json serializer:
		351	class jsonSerializer(Serializer):
		352	def fromSerial(self, string):
		353	#return json.loads(string)
		354	return jsonpickle.decode(string)
		355	def toSerial(self, string):
		356	#return json.dumps(string)
		357	return jsonpickle.encode(string, max_depth=500)
		358
		359	# Dict mapping from serializer type to corresponding class object:
		360	serializers = {'yaml': YAMLSerializer,
		361	'msgpack': MessagePackSerializer,
		362	'hdf5': HDF5Serializer,
		363	'json': jsonSerializer}
		364
		365	instances = {'yaml': YAMLSerializer(),
		366	'msgpack': MessagePackSerializer(),
		367	'hdf5': HDF5Serializer(),
		368	'json': jsonSerializer()}
		369
		370	serial_types = dict([(v,u) for u,v in serializers.items()])
		371
		372	compression_types = ['gzip', '']
		373

schainpy/serializer/test/__init__.py created 644 0 0

NO CONTENT: new file 100644

schainpy/serializer/test/serialtest.py created 644 +187 0

			@@ -0,0 +1,187
		1
		2	#
		3	# when Serializer is imported alone, fault indicates this package is
		4	# dependent on lookup, but don't import Lookup, instead:
		5	#
		6	import DynamicObject # dependent on pysvn
		7	import Serializer # dependent on Dynamic Object
		8
		9	import msgpack
		10	import redis
		11
		12	import numpy as np
		13
		14	class NamedObject():
		15
		16	#---------------------------------------------
		17	@staticmethod
		18	def indexListMatch(list_of_lists, name, value, name2=None, value2=None, unique_f=False):
		19	#
		20	# for each list's <element> attribute compare with value
		21	# if match, return True plus list
		22	# else return False plus empty list
		23	#
		24	# search needs to be named part of class for object else .<value> is unrecognized
		25	#
		26	# unique_f finds non-uniqueness
		27
		28
		29	index = [] # return empty indices
		30	list_data = [] # return empty list
		31	ii = 0
		32	for theList in list_of_lists:
		33
		34	cmd0 = "theList.%s == value" % (name)
		35	cmd1 = "isInlist(theList.%s,value)" % name
		36	# if name is valid then
		37	# match name against value
		38	# match name (as list) against value
		39	if (eval(cmd0) or eval(cmd1)):
		40	if (name2 != None):
		41	cmd2 = "theList.%s == value2" % name2
		42	cmd3 = "isInlist(theList.%s,value2)" % name2
		43	if (eval(cmd2) or eval(cmd3)):
		44	if (unique_f):
		45	index = index + [ii]
		46	list_data = list_data + [theList] # save list of lists if non-unique
		47	# don't exit on match, may be non-unique
		48	else:
		49	list_data = theList # save the list
		50	index = [ii]
		51	break
		52	else:
		53	if (unique_f):
		54	index = index + [ii]
		55	list_data = list_data + [theList] # list of lists if non-unique
		56	else:
		57	list_data = theList
		58	index = [ii]
		59	break # exit on match
		60	#endif
		61	ii = ii + 1
		62	#end for
		63
		64	return index, list_data # return indices of matches and list (or list of lists)
		65
		66	#end indexListMatch
		67
		68	#---------------------------------------------
		69	@staticmethod
		70	def namedListMatch(list_of_lists, name, value, name2=None, value2=None, unique_f=None):
		71	#
		72	# for each list's <element> attribute compare with value
		73	# if match, return True plus list
		74	# else return False plus empty list
		75	#
		76	# search needs to be named part of class for object else .<value> is unrecognized
		77	#
		78	# unique_f finds non-uniqueness ('None' is same as False)
		79
		80	match_f = False
		81	list_data = [] # initialize
		82
		83	for theList in list_of_lists:
		84
		85	cmd0 = "theList.%s == value" % (name)
		86	cmd1 = "isInlist(theList.%s,value)" % name
		87	# if name is valid then
		88	# match name against value
		89	# match name (as list) against value
		90	if (eval(cmd0) or eval(cmd1)):
		91	if (name2 != None):
		92	cmd2 = "theList.%s == value2" % name2
		93	cmd3 = "isInlist(theList.%s,value2)" % name2
		94	if (eval(cmd2) or eval(cmd3)):
		95	match_f = True
		96	if (unique_f):
		97	list_data = list_data + [theList] # save list of lists if non-unique
		98	# don't exit on match, may be non-unique
		99	else:
		100	list_data = theList # save the list
		101	break
		102	else:
		103	match_f = True
		104	if (unique_f):
		105	list_data = list_data + [theList] # list of lists if non-unique
		106	else:
		107	list_data = theList
		108	break # exit on match
		109	#endif
		110	#end for
		111
		112	return match_f, list_data # return match, and list (or list of lists)
		113
		114	#end namedListMatch
		115
		116	#---------------------------------------------
		117	@staticmethod
		118	def combineLists(object):
		119	#
		120	# used for dumping elements in list of lists for debugging
		121	#
		122	ret_list =[]
		123	ii = 0
		124	while ii < len(object):
		125	ret_list = ret_list + [object[ii].list] # not a real list, so can't use built-in list iterator
		126	ii = ii + 1
		127	return ret_list
		128
		129	# end combineLists
		130
		131
		132
		133	class StateListObject(NamedObject):
		134	def __init__(self, concurrent=None, hierarchical=None, history=None, state=None):
		135	self.concurrent = concurrent
		136	self.hierarchical = hierarchical
		137	self.history = history
		138	self.state = state
		139	self.list = [self.concurrent, self.hierarchical, self.history, self.state ]
		140	#end class StateListObject
		141
		142	source_object = "my test string"
		143
		144	serializer = "yaml"
		145	#
		146	# python versioning issue (ver 2.7 -> else path)
		147	#
		148	if isinstance(serializer,Serializer.Serializer):
		149	serial_type = Serializer.serial_types[serializer]
		150	else:
		151	serial_type = serializer
		152	serializer = Serializer.serializers[serializer]()
		153
		154
		155
		156	datastr = serializer.toSerial(source_object)
		157
		158	dest_object = serializer.fromSerial(datastr)
		159
		160	print "dest_object=",dest_object
		161
		162	myObject = StateListObject(hierarchical="yes",state=np.array([1,2,3.0]))
		163
		164	datastr = serializer.toSerial(myObject)
		165
		166	packed = msgpack.packb(datastr)
		167
		168	try:
		169	r= redis.StrictRedis(host='localhost',port=6379,db=0)
		170	except Exception as eobj:
		171	print "is the redis server running?",eobj
		172	else:
		173
		174	r.set('baz',packed) # converts to string
		175	x = r.get('baz')
		176
		177	unpacked = msgpack.unpackb(x)
		178
		179	dest_object = serializer.fromSerial(unpacked)
		180
		181	print "val1=",dest_object.hierarchical
		182	val2 = dest_object.state
		183	print "val2=",val2
		184	# can numpy array be used as array?
		185	print val2.shape
		186
		187

schainpy/serializer/test/test _serializer_speed.py created 644 +175 0

			@@ -0,0 +1,175
		1	'''
		2	Created on Jul 16, 2014
		3
		4	@author: roj-idl71
		5	'''
		6	"""
		7	Dependencies:
		8
		9	pip install tabulate simplejson python-cjson ujson yajl msgpack-python
		10
		11	"""
		12
		13	from timeit import timeit
		14	from tabulate import tabulate
		15
		16	setup = '''d = {
		17	'words': """
		18	Lorem ipsum dolor sit amet, consectetur adipiscing
		19	elit. Mauris adipiscing adipiscing placerat.
		20	Vestibulum augue augue,
		21	pellentesque quis sollicitudin id, adipiscing.
		22	""",
		23	'boolean' : False,
		24	'list': range(10),
		25	'dict': dict((str(i),'a') for i in xrange(10)),
		26	'int': 100,
		27	'float': 100.123456,
		28	}'''
		29
		30	setup = '''import numpy;
		31	import datetime;
		32	d = {
		33	'words': """
		34	Lorem ipsum dolor sit amet, consectetur adipiscing
		35	elit. Mauris adipiscing adipiscing placerat.
		36	Vestibulum augue augue,
		37	pellentesque quis sollicitudin id, adipiscing.
		38	""",
		39	'boolean' : False,
		40	'list': range(10),
		41	'dict': dict((str(i),'a') for i in xrange(10)),
		42	'int': 100,
		43	'float': 100.123456,
		44	'datetime' : datetime.datetime(2001,1,1,10,10,10)
		45	}'''
		46
		47
		48	setup_pickle = '%s ; import pickle ; src = pickle.dumps(d)' % setup
		49	setup_pickle2 = '%s ; import pickle ; src = pickle.dumps(d, 2)' % setup
		50	setup_cpickle = '%s ; import cPickle ; src = cPickle.dumps(d)' % setup
		51	setup_cpickle2 = '%s ; import cPickle ; src = cPickle.dumps(d, 2)' % setup
		52	setup_json = '%s ; import json; src = json.dumps(d)' % setup
		53	setup_ujson = '%s ; import ujson; src = ujson.encode(d)' % setup
		54	setup_cjson = '%s ; import cjson; src = cjson.encode(d)' % setup
		55	setup_simplejson = '%s ; import simplejson; src = simplejson.dump(d)' % setup
		56	setup_jsonpickle = '%s ; import jsonpickle; src = jsonpickle.encode(d)' % setup
		57	setup_yaml = '%s ; import yaml; src = yaml.dump(d)' % setup
		58	setup_msgpack = '%s ; import msgpack; src = msgpack.dumps(d)' % setup
		59	setup_msgpack_np = '%s; import msgpack_numpy as msgpack; src = msgpack.dumps(d)' % setup
		60
		61	tests = [
		62	# (title, setup, enc_test, dec_test, result)
		63	('cPickle (binary)', 'import cPickle; %s' % setup_cpickle2, 'cPickle.dumps(d, 2)', 'r = cPickle.loads(src)', 'print r'),
		64	('cPickle (ascii)', 'import cPickle; %s' % setup_cpickle, 'cPickle.dumps(d, 0)', 'r = cPickle.loads(src)', 'print r'),
		65	('pickle (binary)', 'import pickle; %s' % setup_pickle2, 'pickle.dumps(d, 2)', 'r = pickle.loads(src)', 'print r'),
		66	('pickle (ascii)', 'import pickle; %s' % setup_pickle, 'pickle.dumps(d, 0)', 'r = pickle.loads(src)', 'print r'),
		67	('jsonpickle', 'import jsonpickle; %s' % setup_jsonpickle, 'jsonpickle.encode(d)', 'r = jsonpickle.decode(src)', 'print r'),
		68	# ('msgpack-numpy-python', '%s' % setup_msgpack_np, 'msgpack.dumps(d)', 'r = msgpack.loads(src)', 'print r'),
		69	('ujson', 'import ujson; %s' % setup_ujson, 'ujson.encode(d)', 'r = ujson.decode(src)', 'print r'),
		70	# ('msgpack-python', 'import msgpack; %s' % setup_msgpack, 'msgpack.dumps(d)', 'r = msgpack.loads(src)', 'print r'),
		71	# ('json', 'import json; %s' % setup_json, 'json.dumps(d)', 'r = json.loads(src)', 'print r'),
		72	# ('python-cjson-1.0.5', 'import cjson; %s' % setup_cjson, 'cjson.encode(d)', 'r = cjson.decode(src)', 'print r'),
		73	# ('simplejson-3.3.1', 'import simplejson; %s' % setup_json, 'simplejson.dumps(d)', 'r = simplejson.loads(src)', 'print r'),
		74	('yaml', 'import yaml; %s' % setup_yaml, 'yaml.dump(d)', 'r = yaml.load(src)', 'print r'),
		75	]
		76
		77	loops = 1
		78	enc_table = []
		79	dec_table = []
		80
		81	print "Running tests (%d loops each)" % loops
		82
		83	for title, mod, enc, dec, msg in tests:
		84	print title
		85
		86	### Getting the package size
		87	exec mod
		88	size = len("".join(src))
		89
		90	print " [Encode]", enc
		91	result = timeit(enc, mod, number=loops)
		92	enc_table.append([title, result, size])
		93
		94	print " [Decode]", dec
		95	result = timeit(dec, mod, number=loops)
		96	dec_table.append([title, result])
		97
		98	print " Result"
		99	result = timeit(msg, mod+';'+dec, number=1)
		100
		101	enc_table.sort(key=lambda x: x[1])
		102	enc_table.insert(0, ['Package', 'Seconds', 'Size'])
		103
		104	dec_table.sort(key=lambda x: x[1])
		105	dec_table.insert(0, ['Package', 'Seconds'])
		106
		107	print "\nEncoding Test (%d loops)" % loops
		108	print tabulate(enc_table, headers="firstrow")
		109
		110	print "\nDecoding Test (%d loops)" % loops
		111	print tabulate(dec_table, headers="firstrow")
		112
		113	"""
		114	OUTPUT:
		115
		116	Running tests (15000 loops each)
		117	pickle (ascii)
		118	[Encode] pickle.dumps(d, 0)
		119	[Decode] pickle.loads(src)
		120	pickle (binary)
		121	[Encode] pickle.dumps(d, 2)
		122	[Decode] pickle.loads(src)
		123	cPickle (ascii)
		124	[Encode] cPickle.dumps(d, 0)
		125	[Decode] cPickle.loads(src)
		126	cPickle (binary)
		127	[Encode] cPickle.dumps(d, 2)
		128	[Decode] cPickle.loads(src)
		129	json
		130	[Encode] json.dumps(d)
		131	[Decode] json.loads(src)
		132	simplejson-3.3.1
		133	[Encode] simplejson.dumps(d)
		134	[Decode] simplejson.loads(src)
		135	python-cjson-1.0.5
		136	[Encode] cjson.encode(d)
		137	[Decode] cjson.decode(src)
		138	ujson-1.33
		139	[Encode] ujson.dumps(d)
		140	[Decode] ujson.loads(src)
		141	yajl 0.3.5
		142	[Encode] yajl.dumps(d)
		143	[Decode] yajl.loads(src)
		144	msgpack-python-0.3.0
		145	[Encode] msgpack.dumps(d)
		146	[Decode] msgpack.loads(src)
		147
		148	Encoding Test (15000 loops)
		149	Package Seconds
		150	-------------------- ---------
		151	ujson-1.33 0.232215
		152	msgpack-python-0.3.0 0.241945
		153	cPickle (binary) 0.305273
		154	yajl 0.3.5 0.634148
		155	python-cjson-1.0.5 0.680604
		156	json 0.780438
		157	simplejson-3.3.1 1.04763
		158	cPickle (ascii) 1.62062
		159	pickle (ascii) 14.0497
		160	pickle (binary) 15.4712
		161
		162	Decoding Test (15000 loops)
		163	Package Seconds
		164	-------------------- ---------
		165	msgpack-python-0.3.0 0.240885
		166	cPickle (binary) 0.393152
		167	ujson-1.33 0.396875
		168	python-cjson-1.0.5 0.694321
		169	yajl 0.3.5 0.748369
		170	simplejson-3.3.1 0.780531
		171	cPickle (ascii) 1.38561
		172	json 1.65921
		173	pickle (binary) 5.20554
		174	pickle (ascii) 17.8767
		175	""" No newline at end of file

schainpy/test/__init__.py created 644 +5 0

			@@ -0,0 +1,5
		1	'''
		2	Created on Jul 2, 2014
		3
		4	@author: roj-idl71
		5	'''

schainpy/zerorpc/MyClient01.py created 644 +147 0

			@@ -0,0 +1,147
		1	'''
		2	Created on Jul 11, 2014
		3
		4	@author: roj-idl71
		5	'''
		6	import time
		7	from gevent import sleep
		8
		9	import zerorpc
		10	from schainpy.model import *
		11	from schainpy.serializer.DataTranslate import serial2Obj, serial2Dict
		12	# import schainpy.model.io.jroIO_usrp
		13
		14	def createObjVolt():
		15	'''
		16	This function creates a processing object "VoltProc" with some operations.
		17	such as: "CohInt", "Scope", etc
		18	These class are found inside schainpy.model.proc and schainpy.model.graphics
		19	'''
		20	procObj = VoltageProc()
		21
		22	opObj = CohInt()
		23	procObj.addOperation(opObj, 1)
		24
		25	opObj = Scope()
		26	procObj.addOperation(opObj, 2)
		27
		28	return procObj
		29
		30	def createObjSpec():
		31	'''
		32	This function creates a processing object "SpecProc" with some operation objects
		33	such as: "IncohInt", "SpectraPlot", "RTIPlot", etc
		34	These class are found inside schainpy.model.proc and schainpy.model.graphics
		35	'''
		36
		37	procObj = SpectraProc()
		38
		39	opObj = IncohInt()
		40	procObj.addOperation(opObj, objId = 1)
		41
		42	opObj = SpectraPlot()
		43	procObj.addOperation(opObj, objId = 2)
		44
		45	opObj = RTIPlot()
		46	procObj.addOperation(opObj, objId = 3)
		47
		48	opObj = SpectraPlot()
		49	procObj.addOperation(opObj, objId = 4)
		50
		51	opObj = RTIPlot()
		52	procObj.addOperation(opObj, objId = 5)
		53
		54	return procObj
		55
		56	def processingSpec(procObj, dataInObj):
		57
		58	procObj.setInput(dataInObj)
		59	procObj.run(nFFTPoints = 16)
		60
		61	procObj.call(opType = "external",
		62	opId = 1,
		63	n=1)
		64
		65	procObj.call(opType = "external",
		66	opId = 2,
		67	id=191,
		68	zmin=-100,
		69	zmax=-40)
		70
		71	procObj.call(opType = "external",
		72	opId = 3,
		73	id=192,
		74	zmin=-100,
		75	zmax=-40,
		76	timerange=10*60)
		77
		78	# procObj.call(opType = "self",
		79	# opName = "selectChannels",
		80	# channelList = [0,1])
		81	#
		82	# procObj.call(opType = "self",
		83	# opName = "selectHeights",
		84	# minHei = 300,
		85	# maxHei = 400)
		86	#
		87	# procObj.call(opType = "external",
		88	# opId = 4,
		89	# id=193,
		90	# zmin=-100,
		91	# zmax=-40)
		92	#
		93	# procObj.call(opType = "external",
		94	# opId = 5,
		95	# id=194,
		96	# zmin=-100,
		97	# zmax=-40,
		98	# timerange=10*60)
		99
		100	def printSpeed(deltaTime, mySerial):
		101
		102	####################
		103	size = len(mySerial)/1024.
		104	vel = 1.0*size / deltaTime
		105
		106	print "Index [", replayerObj.getProfileIndex(), "]: ",
		107	print "Total time %5.2f ms, Data size %5.2f KB, Speed %5.2f MB/s" %(deltaTime, size, vel)
		108	####################
		109
		110	if __name__ == '__main__':
		111
		112	procObj = createObjSpec()
		113
		114	replayerObj = zerorpc.Client()
		115	replayerObj.connect("tcp://127.0.0.1:4242")
		116
		117	serializer = replayerObj.getSerializer()
		118
		119	ini = time.time()
		120	mySerialMetadata = replayerObj.getSerialMetaData()
		121	deltaTime = (time.time() - ini)*1024
		122
		123	printSpeed(deltaTime, mySerialMetadata)
		124
		125	myMetaDict = serial2Dict(mySerialMetadata,
		126	serializer = serializer)
		127	# print myMetaDict
		128	while True:
		129	ini = time.time()
		130	mySerialData = replayerObj.getSerialData()
		131	deltaTime = (time.time() - ini)*1024
		132
		133	if not mySerialData:
		134	print "No more data"
		135	break
		136
		137	# myDataDict = SERIALIZER.loads(mySerialData)
		138	# print myDataDict
		139	# continue
		140
		141	printSpeed(deltaTime, mySerialData)
		142
		143	dataInObj = serial2Obj(mySerialData,
		144	metadataDict=myMetaDict,
		145	serializer = serializer)
		146	processingSpec(procObj, dataInObj)
		147	sleep(1e-1) No newline at end of file

schainpy/zerorpc/MyClient02.py created 644 +13 0

			@@ -0,0 +1,13
		1	'''
		2	Created on Jul 11, 2014
		3
		4	@author: roj-idl71
		5	'''
		6
		7	import zerorpc
		8
		9	if __name__ == '__main__':
		10	c = zerorpc.Client()
		11	c.connect("tcp://127.0.0.1:4242")
		12	c.load("file2") # AAAHH! The previously loaded model gets overwritten here!
		13	print c.getModelName() No newline at end of file

schainpy/zerorpc/MyServer.py created 644 +33 0

			@@ -0,0 +1,33
		1	'''
		2	Created on Jul 11, 2014
		3
		4	@author: roj-idl71
		5	'''
		6	# import sys
		7	import datetime
		8	import zerorpc
		9
		10	from schainpy.model.io.jrodataIO import USRPReaderAPI
		11	# from schainpy.serializer.DataTranslate import serial2Obj
		12
		13	if __name__ == '__main__':
		14
		15	replayerObj = USRPReaderAPI(serializer='msgpack')
		16
		17	replayerObj.setup(path='/Volumes/DATA/haystack/passive_radar/',
		18	startDate=datetime.date(2000,1,1),
		19	endDate=datetime.date(2015,1,1),
		20	startTime=datetime.time(0,0,0),
		21	endTime=datetime.time(23,59,59),
		22	online=1,
		23	nSamples=500,
		24	channelList = [0,1,2,3,4,5,6,7])
		25
		26	replayerObj.start()
		27
		28	print "Initializing 'zerorpc' server"
		29	s = zerorpc.Server(replayerObj)
		30	s.bind("tcp://0.0.0.0:4242")
		31	s.run()
		32
		33	print "End" No newline at end of file

schainpy/zerorpc/__init__.py created 644 0 0

NO CONTENT: new file 100644

schainpy/zerorpc/test/testServer.py created 644 +177 0

			@@ -0,0 +1,177
		1	'''
		2	Created on Jul 15, 2014
		3
		4	@author: roj-idl71
		5	'''
		6
		7	import sys
		8	import yaml
		9	import numpy
		10	import jsonpickle
		11
		12	# import schainpy.serializer.DynamicSerializer as DynamicSerializer
		13
		14
		15
		16	def isNotClassVar(myObj):
		17
		18	return not hasattr(myObj,'__dict__')
		19
		20	def isDictFormat(thisValue):
		21
		22	if type(thisValue) != type({}):
		23	return False
		24
		25	if '__name__' not in thisValue.keys():
		26	return False
		27
		28	return True
		29
		30	def obj2Dict(myObj):
		31
		32	myDict = {}
		33
		34	myDict['__name__'] = myObj.__class__.__name__
		35
		36	for thisKey, thisValue in myObj.__dict__.items():
		37
		38	if isNotClassVar(thisValue):
		39	myDict[thisKey] = thisValue
		40	continue
		41
		42	## If this value is another class instance
		43	myNewDict = obj2Dict(thisValue)
		44	myDict[thisKey] = myNewDict
		45
		46	return myDict
		47
		48	def dict2Obj(myDict):
		49	'''
		50	'''
		51
		52	if '__name__' not in myDict.keys():
		53	return None
		54
		55	className = eval(myDict['__name__'])
		56
		57	myObj = className()
		58
		59	for thisKey, thisValue in myDict.items():
		60
		61	if thisKey == '__name__':
		62	continue
		63
		64	if not isDictFormat(thisValue):
		65	setattr(myObj, thisKey, thisValue)
		66	continue
		67
		68	myNewObj = dict2Obj(thisValue)
		69	setattr(myObj, thisKey, myNewObj)
		70
		71	return myObj
		72
		73	class myTestClass3(object):
		74
		75	def __init__(self):
		76	'''
		77	'''
		78	self.y1 = 'y1'
		79	self.y2 = 'y2'
		80
		81	class myTestClass2(object):
		82
		83	def __init__(self):
		84	'''
		85	'''
		86	self.x1 = 'x1'
		87	self.x2 = 'x2'
		88	self.otherObj = myTestClass3()
		89
		90
		91	class myTestClass(object):
		92
		93	flagNoData = True
		94	value1 = 1
		95	value2 = 2
		96	myObj = None
		97
		98	def __init__(self):
		99
		100	'''
		101	'''
		102	self.flagNoData = True
		103	self.value1 = 1
		104	self.value2 = 2
		105	self.myObj = myTestClass2()
		106
		107	def get_dtype(self):
		108
		109	'''
		110	'''
		111	return self.value1
		112
		113	def set_dtype(self, value):
		114
		115	'''
		116	'''
		117
		118	self.value1 = value
		119
		120	dtype = property(get_dtype, set_dtype)
		121
		122	def myMsgPackTest():
		123
		124	import msgpack
		125	import msgpack_numpy as m
		126	import numpy as np
		127
		128	x = np.random.rand(5)
		129	x_enc = m.encode(x)
		130	x_rec = m.decode(x_enc)
		131
		132	print x_rec
		133	#
		134	# x_enc = msgpack.packb(x, default=m.encoder)
		135	# x_rec = msgpack.unpackb(x_enc, object_hook=m.decoder)
		136
		137	if __name__ == '__main__':
		138
		139	myMsgPackTest()
		140
		141	sys.exit()
		142
		143	serializerObj = DynamicSerializer.DynamicSerializer('json')
		144	serializerObj = jsonpickle
		145
		146	myTestObj = myTestClass()
		147
		148	myTestObj.flagNoData = False
		149	myTestObj.value1 = [1+3.4j,4,'5',]
		150	myTestObj.value2 = {'x2': numpy.complex(1,2),'x1': 'x1'}
		151	# myTestObj.myObj.x2 = numpy.arange(15, dtype=numpy.complex)
		152
		153	myDict = obj2Dict(myTestObj)
		154
		155	myNewObj = dict2Obj(myDict)
		156
		157	# print myDict
		158	# print myTestObj.__dict__
		159	# print myNewObj.__dict__
		160
		161	# sys.exit()
		162	print myDict
		163
		164	newSerial = serializerObj.encode(myDict)
		165	# print newSerial
		166
		167	newDict = serializerObj.decode(newSerial)
		168	print newDict
		169
		170	myNewObj = dict2Obj(newDict)
		171
		172	print
		173	print
		174	print 50*'###'
		175	print myTestObj.__dict__
		176	print myNewObj.__dict__
		177	No newline at end of file

schainpy/zerorpc/test/testServer_Voltage.py created 644 +46 0

			@@ -0,0 +1,46
		1	'''
		2	Created on Jul 15, 2014
		3
		4	@author: roj-idl71
		5	'''
		6
		7	import sys
		8	import cPickle
		9
		10	from schainpy.model.data.jrodata import Voltage
		11	# from schainpy.model.io.jrodataIO import USRPReaderMP
		12	from schainpy.serializer.DynamicSerializer import DynamicSerializer
		13	from schainpy.serializer.DataTranslate import obj2Dict, dict2Obj
		14
		15
		16	if __name__ == "__main__":
		17
		18	serializerObj = DynamicSerializer('yaml')
		19
		20	myTestObj = Voltage()
		21
		22	myDict = obj2Dict(myTestObj)
		23
		24	myNewObj = dict2Obj(myDict)
		25
		26	# print myDict
		27	# print myTestObj.__dict__
		28	# print myNewObj.__dict__
		29	#
		30	# print
		31	# print '#############################'
		32	# print
		33	# newValue = serializerObj.dumps(myDict)
		34	# print newValue
		35	#
		36	# newValue = serializerObj.loads(newValue)
		37	# print newValue
		38
		39
		40	print '###########CPICKLE##################'
		41	print myDict
		42	newSerialized = cPickle.dumps(myDict, 2)
		43	# print newValue
		44
		45	newDict = cPickle.loads(newSerialized)
		46	print newDict No newline at end of file

setup.py created 644 +22 0

			@@ -0,0 +1,22
		1	'''
		2	Created on Jul 16, 2014
		3
		4	@author: roj-idl71
		5	'''
		6
		7	from distutils.core import setup, Extension
		8
		9	setup(name="schainpy",
		10	version="1.0",
		11	description="Python tools to read, write and process Jicamarca data",
		12	author="Miguel Urco",
		13	author_email="miguel.urco@jro.igp.gob.pe",
		14	url="http://jro.igp.gob.pe",
		15	packages = {'schainpy', 'schainpy.model',
		16	'schainpy.model.data',
		17	'schainpy.model.graphics',
		18	'schainpy.model.io',
		19	'schainpy.model.proc',
		20	'schainpy.model.utils'},
		21	py_modules=['schainpy.serializer.DataTranslate',
		22	'schainpy.serializer.JROSerializer']) No newline at end of file

schainpy/controller.py +78 -18

              Created on September , 2012
              @author:
              '''
-             from xml.etree.ElementTree import Element, SubElement, ElementTree
+             from xml.etree.ElementTree import Element, SubElement
              from xml.etree import ElementTree as ET
              from xml.dom import minidom
-             import datetime
+             #import datetime
              from model import *
              import ast
                      self.id = parmElement.get('id')
                      self.name = parmElement.get('name')
                      self.value = parmElement.get('value')
-                     self.format = parmElement.get('format')
+                     self.format = str.lower(parmElement.get('format'))
+                     #Compatible with old signal chain version
+                     if self.format == 'int' and self.name == 'idfigure':
+                         self.name = 'id'
                  def printattr(self):
                      print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
                  def __init__(self):
-                     id = 0
-                     name = None
-                     priority = None
-                     type = 'self'
+                     self.id = 0
+                     self.name = None
+                     self.priority = None
+                     self.type = 'self'
                  def __getNewId(self):
                      self.type = opElement.get('type')
                      self.priority = opElement.get('priority')
+                     #Compatible with old signal chain version
+                     #Use of 'run' method instead 'init'
+                     if self.type == 'self' and self.name == 'init':
+                         self.name = 'run'
                      self.parmConfObjList = []
                      parmElementList = opElement.getiterator(ParameterConf().getElementName())
                      for parmElement in parmElementList:
                          parmConfObj = ParameterConf()
                          parmConfObj.readXml(parmElement)
-                         self.parmConfObjList.append(parmConfObj)
+                         #Compatible with old signal chain version
+                         #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
+                         if self.type != 'self' and self.name == 'Plot':
+                             if parmConfObj.format == 'str' and parmConfObj.name == 'type':
+                                 self.name = parmConfObj.value
+                                 continue
+                         self.parmConfObjList.append(parmConfObj)
                  def printattr(self):
                      print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
                      self.name = upElement.get('name')
                      self.datatype = upElement.get('datatype')
                      self.inputId = upElement.get('inputId')
+                     #Compatible with old signal chain version
+                     if self.ELEMENTNAME == ReadUnitConf().getElementName():
+                         if 'Reader' not in self.name:
+                             self.name += 'Reader'
+                     if self.ELEMENTNAME == ProcUnitConf().getElementName():
+                         if 'Proc' not in self.name:
+                             self.name += 'Proc'
                      self.opConfObjList = []
                      opElementList = upElement.getiterator(OperationConf().getElementName())
                      self.name = name
                      self.description = description
-                 def addReadUnit(self, datatype, **kwargs):
+                 def addReadUnit(self, datatype=None, name=None, **kwargs):
+                     #Compatible with old signal chain version
+                     if datatype==None and name==None:
+                         raise ValueError, "datatype or name should be defined"
+                     if name==None:
+                         if 'Reader' in datatype:
+                             name = datatype
+                         else:
+                             name = '%sReader' %(datatype)
+                     if datatype==None:
+                         datatype = name.replace('Reader','')
                      id = self.__getNewId()
-                     name = '%s' %(datatype)
                      readUnitConfObj = ReadUnitConf()
                      readUnitConfObj.setup(id, name, datatype, **kwargs)
                      return readUnitConfObj
-                 def addProcUnit(self, datatype, inputId):
+                 def addProcUnit(self, inputId, datatype=None, name=None):
+                     #Compatible with old signal chain version
+                     if datatype==None and name==None:
+                         raise ValueError, "datatype or name should be defined"
+                     if name==None:
+                         if 'Proc' in datatype:
+                             name = datatype
+                         else:
+                             name = '%sProc' %(datatype)
+                     if datatype==None:
+                         datatype = name.replace('Proc','')
                      id = self.__getNewId()
-                     name = '%s' %(datatype)
                      procUnitConfObj = ProcUnitConf()
                      procUnitConfObj.setup(id, name, datatype, inputId)
                      self.makeXml()
-                     print prettify(self.projectElement)
+                     #print prettify(self.projectElement)
                      ElementTree(self.projectElement).write(filename, method='xml')
              #        for readUnitConfObj in self.readUnitConfObjList:
              #            readUnitConfObj.run()
+                     print
+                     print "*"*40
+                     print "   Starting SIGNAL CHAIN PROCESSING  "
+                     print "*"*40
+                     print
+                     keyList = self.procUnitConfObjDict.keys()
+                     keyList.sort()
                      while(True):
                          finalSts = False
-                         for procUnitConfObj in self.procUnitConfObjDict.values():
-                             #print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
+                         for procKey in keyList:
+             #                 print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
+                             procUnitConfObj = self.procUnitConfObjDict[procKey]
                              sts = procUnitConfObj.run()
                              finalSts = finalSts or sts
                          #If every process unit finished so end process
                          if not(finalSts):
-                             print "Every process units have finished"
+                             print "Every process unit have finished"
                              break
              if __name__ == '__main__':

schainpy/model/__init__.py +12 -5

		@@ -1,5 +1,12
1		from model.data.jrodata import *
2		from model.io.jrodataIO import *
3		from model.proc.jroprocessing import *
4		from model.graphics.jroplot import *
5		from model.utils.jroutils import * No newline at end of file
	1	#from schainpy.model.data.jrodata import *
	2	# from schainpy.model.io.jrodataIO import *
	3	# from schainpy.model.proc.jroprocessing import *
	4	# from schainpy.model.graphics.jroplot import *
	5	# from schainpy.model.utils.jroutils import *
	6	# from schainpy.serializer import *
	7
	8	from data import *
	9	from io import *
	10	from proc import *
	11	from graphics import *
	12	from utils import *

schainpy/model/data/__init__.py +3 0

			@@ -0,0 +1,3
		1	from jrodata import *
		2	from jroheaderIO import *
		3	from jroamisr import * No newline at end of file

schainpy/model/data/jroamisr.py +2 -1

              import numpy
+             import copy
              class Beam:
                  def __init__(self):
                      self.channelList = [0]#self.dataIn.channelList esto solo aplica para el caso de AMISR
                      self.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
-                     self.flagTimeBlock = None#self.dataIn.flagTimeBlock
+                     self.flagDiscontinuousBlock = None#self.dataIn.flagDiscontinuousBlock
                      #self.utctime = #self.firstdatatime
                      self.flagDecodeData = None#self.dataIn.flagDecodeData #asumo q la data esta decodificada
                      self.flagDeflipData = None#self.dataIn.flagDeflipData #asumo q la data esta sin flip

schainpy/model/data/jrodata.py +104 -31

                  return datatype
              def hildebrand_sekhon(data, navg):
+                 """
+                 This method is for the objective determination of the noise level in Doppler spectra. This
+                 implementation technique is based on the fact that the standard deviation of the spectral
+                 densities is equal to the mean spectral density for white Gaussian noise
-                 data = data.copy()
+                 Inputs:
+                     Data    :    heights
+                     navg    :    numbers of averages
+                 Return:
+                     -1        :    any error
+                     anoise    :    noise's level
+                 """
                  sortdata = numpy.sort(data,axis=None)
                  lenOfData = len(sortdata)
                  channelList = None
-                 flagTimeBlock = False
+                 flagDiscontinuousBlock = False
                  useLocalTime = False
                  blocksize = None
-                 nCode = None
-                 nBaud = None
-                 code = None
+             #     nCode = None
+             #
+             #     nBaud = None
+             #
+             #     code = None
                  flagDecodeData = False #asumo q la data no esta decodificada
                  nCohInt = None
-                 noise = None
+             #     noise = None
                  windowOfFilter = 1
                      '''
                      self.datatype = getDataTypeCode(numpyDtype)
+                 def get_code(self):
+                     '''
+                     '''
+                     return self.radarControllerHeaderObj.code
+                 def set_code(self, code):
+                     '''
+                     '''
+                     self.radarControllerHeaderObj.code = code
+                     return
+                 def get_ncode(self):
+                     '''
+                     '''
+                     return self.radarControllerHeaderObj.nCode
+                 def set_ncode(self, nCode):
+                     '''
+                     '''
+                     self.radarControllerHeaderObj.nCode = nCode
+                     return
+                 def get_nbaud(self):
+                     '''
+                     '''
+                     return self.radarControllerHeaderObj.nBaud
+                 def set_nbaud(self, nBaud):
+                     '''
+                     '''
+                     self.radarControllerHeaderObj.nBaud = nBaud
+                     return
              #     def getTimeInterval(self):
              #
              #         raise IOError, "This method should be implemented inside each Class"
                  ippSeconds = property(get_ippSeconds, set_ippSeconds)
                  dtype = property(get_dtype, set_dtype)
              #     timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
+                 code = property(get_code, set_code)
+                 nCode = property(get_ncode, set_ncode)
+                 nBaud = property(get_nbaud, set_nbaud)
              class Voltage(JROData):
                      self.flagNoData = True
-                     self.flagTimeBlock = False
+                     self.flagDiscontinuousBlock = False
                      self.utctime = None
                      self.flagShiftFFT = False
                      self.flagDataAsBlock = False    #Asumo que la data es leida perfil a perfil
                      self.profileIndex = 0
-                 def getNoisebyHildebrand(self):
+                 def getNoisebyHildebrand(self, channel = None):
                      """
                      Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
                          noiselevel
                      """
-                     for channel in range(self.nChannels):
-                         daux = self.data_spc[channel,:,:]
-                         self.noise[channel] = hildebrand_sekhon(daux, self.nCohInt)
+                     if channel != None:
+                         data = self.data[channel]
+                         nChannels = 1
+                     else:
+                         data = self.data
+                         nChannels = self.nChannels
+                     noise = numpy.zeros(nChannels)
+                     power = data * numpy.conjugate(data)
-                     return self.noise
-                 def getNoise(self, type = 1):
+                     for thisChannel in range(nChannels):
+                         if nChannels == 1:
+                             daux = power[:].real
+                         else:
+                             daux = power[thisChannel,:].real
+                         noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
-                     self.noise = numpy.zeros(self.nChannels)
+                     return noise
+                 def getNoise(self, type = 1, channel = None):
                      if type == 1:
-                         noise = self.getNoisebyHildebrand()
+                         noise = self.getNoisebyHildebrand(channel)
                      return 10*numpy.log10(noise)
+                 def getPower(self, channel = None):
+                     if channel != None:
+                         data = self.data[channel]
+                     else:
+                         data = self.data
+                     power = data * numpy.conjugate(data)
+                     return 10*numpy.log10(power.real)
                  def getTimeInterval(self):
                      timeInterval = self.ippSeconds * self.nCohInt
                      self.flagNoData = True
-                     self.flagTimeBlock = False
+                     self.flagDiscontinuousBlock = False
                      self.utctime = None
                      self.noise_estimation = None
-                 def getNoisebyHildebrand(self):
+                 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
                      """
                      Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
                      """
                      noise = numpy.zeros(self.nChannels)
                      for channel in range(self.nChannels):
-                         daux = self.data_spc[channel,:,:]
+                         daux = self.data_spc[channel,xmin_index:xmax_index,ymin_index:ymax_index]
                          noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
                      return noise
-                 def getNoise(self):
+                 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
                      if self.noise_estimation != None:
                          return self.noise_estimation #this was estimated by getNoise Operation defined in jroproc_spectra.py
                      else:
-                         noise = self.getNoisebyHildebrand()
+                         noise = self.getNoisebyHildebrand(xmin_index, xmax_index, ymin_index, ymax_index)
                          return noise
                      self.flagNoData = True
-                     self.flagTimeBlock = False
+                     self.flagDiscontinuousBlock = False
              #         self.nPairs = 0
                  flagNoData = True
-                 flagTimeBlock = False
+                 flagDiscontinuousBlock = False
                  useLocalTime = False
                  def getNoise(self, type = 1):
-                     self.noise = numpy.zeros(self.nChannels)
+                     #noise = numpy.zeros(self.nChannels)
                      if type == 1:
                          noise = self.getNoisebyHildebrand()
                      self.flagNoData = True
-                     self.flagTimeBlock = False
+                     self.flagDiscontinuousBlock = False
                      self.utctime = None
                  noise = None            #Noise Potency
-                 utctimeInit = None      #Initial UTC time
+                 initUtcTime = None      #Initial UTC time
                  paramInterval = None    #Time interval to calculate Parameters in seconds
                      datatime = []
-                     datatime.append(self.utctimeInit)
-                     datatime.append(self.utctimeInit + self.outputInterval - 1)
+                     datatime.append(self.initUtcTime)
+                     datatime.append(self.initUtcTime + self.outputInterval - 1)
                      datatime = numpy.array(datatime)

schainpy/model/data/jroheaderIO.py +63 -52

                      print "#"*100
                      for key in self.__dict__.keys():
                          print "%s = %s" %(key, self.__dict__[key])
              class BasicHeader(Header):
                  size = None
                  adcResolution = None
                  pciDioBusWidth = None
-                 def __init__(self):
+                 def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWith=0):
                      self.size = 24
-                     self.nSamples = 0
-                     self.nProfiles = 0
-                     self.nChannels = 0
-                     self.adcResolution = 0
-                     self.pciDioBusWidth = 0
+                     self.nSamples = nSamples
+                     self.nProfiles = nProfiles
+                     self.nChannels = nChannels
+                     self.adcResolution = adcResolution
+                     self.pciDioBusWidth = pciDioBusWith
                  def read(self, fp):
                      try:
                          header = numpy.fromfile(fp,SYSTEM_STRUCTURE,1)
                          self.size = header['nSize'][0]
                  __C = 3e8
-                 def __init__(self):
+                 def __init__(self, expType=2, nTx=1,
+                              ippKm=None, txA=0, txB=0,
+                              nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None,
+                              numTaus=0, line6Function=0, line5Function=0, fClock=0,
+                              prePulseBefore=0, prePulseAfter=0,
+                              codeType=0, nCode=0, nBaud=0, code=None,
+                              flip1=0, flip2=0):
                      self.size = 116
-                     self.expType = 0
-                     self.nTx = 0
-                     self.ipp = 0
-                     self.txA = 0
-                     self.txB = 0
-                     self.nWindows = 0
-                     self.numTaus = 0
-                     self.codeType = 0
-                     self.line6Function = 0
-                     self.line5Function = 0
-                     self.fClock = 0
-                     self.prePulseBefore = 0
-                     self.prePulserAfter = 0
-                     self.rangeIpp = 0
-                     self.rangeTxA = 0
-                     self.rangeTxB = 0
-                     self.samplingWindow = None
-                     self.nHeights = None
-                     self.firstHeight = None
-                     self.deltaHeight = None
-                     self.samplesWin = None
-                     self.nCode = None
-                     self.nBaud = None
-                     self.code = None
-                     self.flip1 = None
-                     self.flip2 = None
+                     self.expType = expType
+                     self.nTx = nTx
+                     self.ipp = ippKm
+                     self.txA = txA
+                     self.txB = txB
+                     self.rangeIpp = ippKm
+                     self.rangeTxA = txA
+                     self.rangeTxB = txB
+                     self.nWindows = nWindows
+                     self.numTaus = numTaus
+                     self.codeType = codeType
+                     self.line6Function = line6Function
+                     self.line5Function = line5Function
+                     self.fClock = fClock
+                     self.prePulseBefore = prePulseBefore
+                     self.prePulserAfter = prePulseAfter
+                     self.nHeights = nHeights
+                     self.firstHeight = firstHeight
+                     self.deltaHeight = deltaHeight
+                     self.samplesWin = nHeights
+                     self.nCode = nCode
+                     self.nBaud = nBaud
+                     self.code = code
+                     self.flip1 = flip1
+                     self.flip2 = flip2
              #         self.dynamic = numpy.array([],numpy.dtype('byte'))
                  def read(self, fp):
                      try:
                          startFp = fp.tell()
                          header = numpy.fromfile(fp,RADAR_STRUCTURE,1)
              #             backFp = fp.tell() - jumpFp
              #             fp.seek(backFp)
-                         self.samplingWindow = numpy.fromfile(fp,SAMPLING_STRUCTURE,self.nWindows)
+                         samplingWindow = numpy.fromfile(fp,SAMPLING_STRUCTURE,self.nWindows)
-                         self.nHeights = int(numpy.sum(self.samplingWindow['nsa']))
-                         self.firstHeight = self.samplingWindow['h0']
-                         self.deltaHeight = self.samplingWindow['dh']
-                         self.samplesWin = self.samplingWindow['nsa']
+                         self.nHeights = int(numpy.sum(samplingWindow['nsa']))
+                         self.firstHeight = samplingWindow['h0']
+                         self.deltaHeight = samplingWindow['dh']
+                         self.samplesWin = samplingWindow['nsa']
                          self.Taus = numpy.fromfile(fp,'<f4',self.numTaus)
                          if self.codeType != 0:
                              self.nCode = int(numpy.fromfile(fp,'<u4',1))
                              self.nBaud = int(numpy.fromfile(fp,'<u4',1))
-                             self.code = numpy.empty([self.nCode,self.nBaud],dtype='u1')
+                             self.code = numpy.empty([self.nCode,self.nBaud],dtype='i1')
                              for ic in range(self.nCode):
                                  temp = numpy.fromfile(fp,'u4',int(numpy.ceil(self.nBaud/32.)))
                                  for ib in range(self.nBaud-1,-1,-1):
                      #dynamic = self.dynamic
                      #dynamic.tofile(fp)
-                     samplingWindow = self.samplingWindow
+                     sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin)
+                     samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE)
                      samplingWindow.tofile(fp)
                      if self.numTaus > 0:
                  flag_cspc = None
                  def __init__(self):
                      self.size = 0
                      self.dtype = 0
                      self.blockSize = 0
                      self.nIncohInt = 0
                      self.totalSpectra = 0
-                     self.samplingWindow = 0
                      self.nHeights = 0
                      self.firstHeight = 0
                      self.deltaHeight = 0
                      self.nIncohInt = int(header['nIncoherentIntegrations'][0])
                      self.totalSpectra = int(header['nTotalSpectra'][0])
-                     self.samplingWindow = numpy.fromfile(fp,SAMPLING_STRUCTURE,self.nWindows)
+                     samplingWindow = numpy.fromfile(fp,SAMPLING_STRUCTURE,self.nWindows)
-                     self.nHeights = int(numpy.sum(self.samplingWindow['nsa']))
-                     self.firstHeight = float(self.samplingWindow['h0'][0])
-                     self.deltaHeight = float(self.samplingWindow['dh'][0])
-                     self.samplesWin = self.samplingWindow['nsa'][0]
+                     self.nHeights = int(numpy.sum(samplingWindow['nsa']))
+                     self.firstHeight = float(samplingWindow['h0'][0])
+                     self.deltaHeight = float(samplingWindow['dh'][0])
+                     self.samplesWin = samplingWindow['nsa'][0]
                      self.spectraComb = numpy.fromfile(fp,'u1',2*self.totalSpectra)
              #        if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
                      return 1
                  def write(self, fp):
                      headerTuple = (self.size,
                                     self.dtype,
                                     self.blockSize,

schainpy/model/graphics/__init__.py +5 0

			@@ -0,0 +1,5
		1	from jroplot_voltage import *
		2	from jroplot_spectra import *
		3	from jroplot_heispectra import *
		4	from jroplot_correlation import *
		5	from jroplot_parameters import * No newline at end of file

schainpy/model/graphics/figure.py +9 -8

                  def __del__(self):
-                     self.__driver.closeFigure()
+                     self.__driver.closeFigure(True)
                  def getFilename(self, name, ext='.png'):
                      return widthscreen, heightscreen
-                 def getTimeLim(self, x, xmin=None, xmax=None, timerange=None, timezone=0):
+                 def getTimeLim(self, x, xmin=None, xmax=None, timerange=None):
                      if self.xmin != None and self.xmax != None:
                          if timerange == None:
                          return xmin, xmax
-                     if timerange != None and self.xmin == None and self.xmax == None:
-                         txmin = x[0] - x[0]%timerange
+                     if timerange == None and (xmin==None or xmax==None):
+                         raise ValueError, "timerange or xmin+xmax should be defined"
+                     if timerange != None:
+                         txmin = x[0] - x[0] % min(timerange/10, 10*60)
                      else:
-                         txmin = numpy.min(x)
-                         timerange = self.timerange
+                         txmin = x[0] - x[0] % 10*60
                      thisdatetime = datetime.datetime.utcfromtimestamp(txmin)
                      thisdate = datetime.datetime.combine(thisdatetime.date(), datetime.time(0,0,0))
-                     if xmin == None and xmax == None:
+                     if timerange != None:
                          xmin = (thisdatetime - thisdate).seconds/(60*60.)
                          xmax = xmin + timerange/(60*60.)

schainpy/model/graphics/jroplot_correlation.py +13 -11

                  def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
-                         save=False, figpath='', figfile=None, show=True, ftp=False, wr_period=1,
+                         save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
              #         noise = dataOut.noise/factor
                      #thisDatetime = dataOut.datatime
-                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Correlation"
                      xlabel = "Lag T (s)"
                      ylabel = "Range (Km)"
                      self.draw()
-                     if figfile == None:
-                         str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
-                         figfile = self.getFilename(name = str_datetime)
-                     if figpath != '':
+                     if save:
+                         if figfile == None:
+                             str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
+                             figfile = self.getFilename(name = str_datetime)
                          self.counter_imagwr += 1
                          if (self.counter_imagwr>=wr_period):
                              # store png plot to local folder
                              self.saveFigure(figpath, figfile)
-                             # store png plot to FTP server according to RT-Web format
-                             name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
-                             ftp_filename = os.path.join(figpath, name)
-                             self.saveFigure(figpath, ftp_filename)
+                             # store png plot to FTP server according to RT-Web format
+                             if ftp:
+                                 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
+                                 ftp_filename = os.path.join(figpath, name)
+                                 self.saveFigure(figpath, ftp_filename)
                              self.counter_imagwr = 0

schainpy/model/graphics/jroplot_heispectra.py +18 -16

              '''
+             Created on Jul 9, 2014
-             @author: Daniel Suarez
+             @author: roj-idl71
              '''
              import os
              import datetime
              import numpy
                  def run(self, dataOut, id, wintitle="", channelList=None,
                          xmin=None, xmax=None, ymin=None, ymax=None, save=False,
-                         figpath='', figfile=None, ftp=False, wr_period=1, show=True,
+                         figpath='./', figfile=None, ftp=False, wr_period=1, show=True,
                          server=None, folder=None, username=None, password=None):
                      """
                      y = datadB
                      #thisDatetime = dataOut.datatime
-                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
                      xlabel = ""
                      #para 1Mhz descomentar la siguiente linea
                      self.draw()
-                     if figfile == None:
-                         str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
-                         figfile = self.getFilename(name = str_datetime)
-                     if figpath != '':
+                     if save:
+                         if figfile == None:
+                             str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
+                             figfile = self.getFilename(name = str_datetime)
                          self.counter_imagwr += 1
                          if (self.counter_imagwr>=wr_period):
                              # store png plot to local folder
                  def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
                          xmin=None, xmax=None, ymin=None, ymax=None,
                          timerange=None,
-                         save=False, figpath='', figfile=None, ftp=False, wr_period=1, show=True,
+                         save=False, figpath='./', figfile=None, ftp=False, wr_period=1, show=True,
                          server=None, folder=None, username=None, password=None):
                      if channelList == None:
              #        noisedB = 10*numpy.log10(noise)
                      #thisDatetime = dataOut.datatime
-                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " RTI: %s" %(thisDatetime.strftime("%d-%b-%Y"))
                      xlabel = "Local Time"
                      ylabel = "Intensity (dB)"
                          del self.ydata
                          self.__isConfig = False
-                     if self.figfile == None:
-                         str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
-                         self.figfile = self.getFilename(name = str_datetime)
-                     if figpath != '':
+                     if save:
+                         if self.figfile == None:
+                             str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
+                             self.figfile = self.getFilename(name = str_datetime)
                          self.counter_imagwr += 1
                          if (self.counter_imagwr>=wr_period):
                              # store png plot to local folder

schainpy/model/graphics/jroplot_parameters.py +67 -54

                  def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
-                         save=False, figpath='', figfile=None, show=True, ftp=False, wr_period=1,
+                         save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
                      noisedB = 10*numpy.log10(noise)
                      #thisDatetime = dataOut.datatime
-                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Parameters"
                      xlabel = "Velocity (m/s)"
                      ylabel = "Range (Km)"
                      self.draw()
-                     if figfile == None:
-                         str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
-                         figfile = self.getFilename(name = str_datetime)
-                     if figpath != '':
+                     if save:
+                         if figfile == None:
+                             str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
+                             figfile = self.getFilename(name = str_datetime)
                          self.counter_imagwr += 1
                          if (self.counter_imagwr>=wr_period):
                              # store png plot to local folder
                              self.saveFigure(figpath, figfile)
-                             # store png plot to FTP server according to RT-Web format
-                             name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
-                             ftp_filename = os.path.join(figpath, name)
-                             self.saveFigure(figpath, ftp_filename)
                              self.counter_imagwr = 0
+                             # store png plot to FTP server according to RT-Web format
+                             if ftp:
+                                 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
+                                 ftp_filename = os.path.join(figpath, name)
+                                 self.saveFigure(figpath, ftp_filename)
              class SkyMapPlot(Figure):
                      #thisDatetime = dataOut.datatime
-                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Parameters"
                      xlabel = "Zonal Zenith Angle (deg) "
                      ylabel = "Meridional Zenith Angle (deg)"
                              if figfile == None:
                                  figfile = self.getFilename(name = self.name)
                              self.saveFigure(figpath, figfile)
+                             self.counter_imagwr = 0
                              if ftp:
                                  #provisionalmente envia archivos en el formato de la web en tiempo real
                                      self.counter_imagwr = 0
                                      raise ValueError, 'Error FTP'
-                             self.counter_imagwr = 0
              class WindProfilerPlot(Figure):
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
                          zmax_ver = None, zmin_ver = None, SNRmin = None, SNRmax = None,
                          timerange=None, SNRthresh = None,
-                         save=False, figpath='', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
+                         save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
                      """
                      showprofile = False
              #        thisDatetime = dataOut.datatime
-                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + "Wind"
                      xlabel = ""
                      ylabel = "Range (Km)"
                      self.draw()
-                     if self.figfile == None:
-                         str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
-                         self.figfile = self.getFilename(name = str_datetime)
-                     if figpath != '':
+                     if save:
+                         if self.figfile == None:
+                             str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
+                             self.figfile = self.getFilename(name = str_datetime)
                          self.counter_imagwr += 1
                          if (self.counter_imagwr>=wr_period):
                              # store png plot to local folder
                              self.saveFigure(figpath, self.figfile)
-                             # store png plot to FTP server according to RT-Web format
-                             name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
-                             ftp_filename = os.path.join(figpath, name)
-                             self.saveFigure(figpath, ftp_filename)
                              self.counter_imagwr = 0
+                             if ftp:
+                                 # store png plot to FTP server according to RT-Web format
+                                 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
+                                 ftp_filename = os.path.join(figpath, name)
+                                 self.saveFigure(figpath, ftp_filename)
                      if x[1] >= self.axesList[0].xmax:
                          self.counter_imagwr = wr_period
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,timerange=None,
                          parameterIndex = None, onlyPositive = False,
                          SNRthresh = -numpy.inf, SNR = True, SNRmin = None, SNRmax = None,
                          zlabel = "", parameterName = "", parameterObject = "data_param",
-                         save=False, figpath='', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
+                         save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
                          ind = numpy.where(SNRdB < 10**(SNRthresh/10))
                          z[ind] = numpy.nan
-                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
                      xlabel = ""
                      ylabel = "Range (Km)"
-                     self.draw()
-                     if self.figfile == None:
-                         str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
-                         self.figfile = self.getFilename(name = str_datetime)
+                     self.draw()
-                     if figpath != '':
+                     if save:
+                         if self.figfile == None:
+                             str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
+                             self.figfile = self.getFilename(name = str_datetime)
                          self.counter_imagwr += 1
                          if (self.counter_imagwr>=wr_period):
-                             # store png plot to local folder
+                             # store png plot to local folder
                              self.saveFigure(figpath, self.figfile)
-                             # store png plot to FTP server according to RT-Web format
-                             name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
-                             ftp_filename = os.path.join(figpath, name)
-                             self.saveFigure(figpath, ftp_filename)
                              self.counter_imagwr = 0
+                             if ftp:
+                                 # store png plot to FTP server according to RT-Web format
+                                 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
+                                 ftp_filename = os.path.join(figpath, name)
+                                 self.saveFigure(figpath, ftp_filename)
                      if x[1] >= self.axesList[0].xmax:
                          self.counter_imagwr = wr_period
                          self.__isConfig = False
                      zdB = 10*numpy.log10(z)
                      #thisDatetime = dataOut.datatime
-                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Doppler Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
                      xlabel = "Velocity (m/s)"
                      ylabel = "Spectrum"
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
                          zmaxVertical = None, zminVertical = None, zmaxZonal = None, zminZonal = None,
                          timerange=None, SNRthresh = -numpy.inf, SNRmin = None, SNRmax = None, SNR_1 = False,
-                         save=False, figpath='', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
+                         save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
                      """
                                      ticksize=9, cblabel='', cbsize="1%", colormap="jet")
                      self.draw()
-                     if self.figfile == None:
-                         str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
-                         self.figfile = self.getFilename(name = str_datetime)
-                     if figpath != '':
+                     if save:
+                         if self.figfile == None:
+                             str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
+                             self.figfile = self.getFilename(name = str_datetime)
                          self.counter_imagwr += 1
                          if (self.counter_imagwr>=wr_period):
-                             # store png plot to local folder
+                             # store png plot to local folder
                              self.saveFigure(figpath, self.figfile)
-                             # store png plot to FTP server according to RT-Web format
-                             name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
-                             ftp_filename = os.path.join(figpath, name)
-                             self.saveFigure(figpath, ftp_filename)
                              self.counter_imagwr = 0
+                             if ftp:
+                                 # store png plot to FTP server according to RT-Web format
+                                 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
+                                 ftp_filename = os.path.join(figpath, name)
+                                 self.saveFigure(figpath, ftp_filename)
                      if x[1] >= self.axesList[0].xmax:
                          self.counter_imagwr = wr_period

schainpy/model/graphics/jroplot_spectra.py +169 -150

              '''
-             @author: Daniel Suarez
+             Created on Jul 9, 2014
+             @author: roj-idl71
              '''
              import os
              import datetime
                      self.SUB_EXP_CODE = None
                      self.PLOT_POS = None
+                     self.__xfilter_ena = False
+                     self.__yfilter_ena = False
                  def getSubplots(self):
                      ncol = int(numpy.sqrt(self.nplots)+0.9)
                  def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
-                         save=False, figpath='', figfile=None, show=True, ftp=False, wr_period=1,
+                         save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
                          zmax            :    None
                      """
-                     if dataOut.flagNoData:
-                         return None
                      if realtime:
                          if not(isRealtime(utcdatatime = dataOut.utctime)):
                              print 'Skipping this plot function'
                      z = dataOut.data_spc[channelIndexList,:,:]/factor
                      z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
-                     avg = numpy.average(z, axis=1)
-                     #avg = numpy.nanmean(z, axis=1)
-                     noise = dataOut.noise/factor
                      zdB = 10*numpy.log10(z)
+                     avg = numpy.nanmean(z, axis=1)
                      avgdB = 10*numpy.log10(avg)
+                     noise = dataOut.getNoise()/factor
                      noisedB = 10*numpy.log10(noise)
-                     #thisDatetime = dataOut.datatime
-                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Spectra"
                      if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
                          title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
                          if xmax == None: xmax = numpy.nanmax(x)
                          if ymin == None: ymin = numpy.nanmin(y)
                          if ymax == None: ymax = numpy.nanmax(y)
-                         if zmin == None: zmin = numpy.nanmin(avgdB)*0.9
-                         if zmax == None: zmax = numpy.nanmax(avgdB)*0.9
+                         if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
+                         if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                          self.SUB_EXP_CODE = sub_exp_code
                          if self.__showprofile:
                              axes = self.axesList[i*self.__nsubplots +1]
-                             axes.pline(avgdB[i], y,
+                             axes.pline(avgdB[i,:], y,
                                      xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
                                      xlabel='dB', ylabel='', title='',
                                      ytick_visible=False,
                              axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
                      self.draw()
-                     if figfile == None:
-                         str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
-                         figfile = self.getFilename(name = str_datetime)
-                         name = str_datetime
-                         if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
-                             name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
-                         figfile = self.getFilename(name)
-                     if figpath != '':
+                     if save:
+                         if figfile == None:
+                             str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
+                             figfile = self.getFilename(name = str_datetime)
+                             name = str_datetime
+                             if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
+                                 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
+                             figfile = self.getFilename(name)
                          self.counter_imagwr += 1
                          if (self.counter_imagwr>=wr_period):
-                             # store png plot to local folder
+                             # store png plot to local folder
                              self.saveFigure(figpath, figfile)
-                             # store png plot to FTP server according to RT-Web format
-                             name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
-                             ftp_filename = os.path.join(figpath, name)
-                             self.saveFigure(figpath, ftp_filename)
                              self.counter_imagwr = 0
+                             if ftp:
+                                 # store png plot to FTP server according to RT-Web format
+                                 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
+                                 ftp_filename = os.path.join(figpath, name)
+                                 self.saveFigure(figpath, ftp_filename)
              class CrossSpectraPlot(Figure):
                  def run(self, dataOut, id, wintitle="", pairsList=None,
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
-                         save=False, figpath='', figfile=None, ftp=False, wr_period=1,
+                         save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
                          power_cmap='jet', coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
                          pairsIndexList = []
                          for pair in pairsList:
                              if pair not in dataOut.pairsList:
-                                 raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair)
+                                 raise ValueError, "Pair %s is not in dataOut.pairsList" %str(pair)
                              pairsIndexList.append(dataOut.pairsList.index(pair))
                      if pairsIndexList == []:
                      x = dataOut.getVelRange(1)
                      y = dataOut.getHeiRange()
                      z = dataOut.data_spc[:,:,:]/factor
-             #        z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
-                     avg = numpy.abs(numpy.average(z, axis=1))
+                     z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
                      noise = dataOut.noise/factor
                      zdB = 10*numpy.log10(z)
-                     avgdB = 10*numpy.log10(avg)
                      noisedB = 10*numpy.log10(noise)
                      #thisDatetime = dataOut.datatime
-                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Cross-Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
                      xlabel = "Velocity (m/s)"
                      ylabel = "Range (Km)"
                                     showprofile=False,
                                     show=show)
+                         avg = numpy.abs(numpy.average(z, axis=1))
+                         avgdB = 10*numpy.log10(avg)
                          if xmin == None: xmin = numpy.nanmin(x)
                          if xmax == None: xmax = numpy.nanmax(x)
                          if ymin == None: ymin = numpy.nanmin(y)
                          if ymax == None: ymax = numpy.nanmax(y)
-                         if zmin == None: zmin = numpy.nanmin(avgdB)*0.9
-                         if zmax == None: zmax = numpy.nanmax(avgdB)*0.9
+                         if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
+                         if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                      self.draw()
-                     if figfile == None:
-                         str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
-                         figfile = self.getFilename(name = str_datetime)
-                     if figpath != '':
+                     if save != '':
+                         if figfile == None:
+                             str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
+                             figfile = self.getFilename(name = str_datetime)
                          self.counter_imagwr += 1
                          if (self.counter_imagwr>=wr_period):
-                             # store png plot to local folder
+                             # store png plot to local folder
                              self.saveFigure(figpath, figfile)
-                             # store png plot to FTP server according to RT-Web format
-                             name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
-                             ftp_filename = os.path.join(figpath, name)
-                             self.saveFigure(figpath, ftp_filename)
                              self.counter_imagwr = 0
+                             if ftp:
+                                 # store png plot to FTP server according to RT-Web format
+                                 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
+                                 ftp_filename = os.path.join(figpath, name)
+                                 self.saveFigure(figpath, ftp_filename)
              class RTIPlot(Figure):
                  def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
                          timerange=None,
-                         save=False, figpath='', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
+                         save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
                                  raise ValueError, "Channel %d is not in dataOut.channelList"
                              channelIndexList.append(dataOut.channelList.index(channel))
-                     if timerange != None:
-                         self.timerange = timerange
+             #         if timerange != None:
+             #             self.timerange = timerange
                      #tmin = None
                      #tmax = None
              #        thisDatetime = dataOut.datatime
-                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " RTI" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
                      xlabel = ""
                      ylabel = "Range (Km)"
                                     showprofile=showprofile,
                                     show=show)
-                         self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
-             #             if timerange != None:
-             #                 self.timerange = timerange
-             #                 self.xmin, self.tmax = self.getTimeLim(x, xmin, xmax, timerange)
+                         if timerange != None:
+                             self.timerange = timerange
+                         self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
+                         noise = dataOut.noise/factor
+                         noisedB = 10*numpy.log10(noise)
                          if ymin == None: ymin = numpy.nanmin(y)
                          if ymax == None: ymax = numpy.nanmax(y)
-                         if zmin == None: zmin = numpy.nanmin(avgdB)*0.9
-                         if zmax == None: zmax = numpy.nanmax(avgdB)*0.9
+                         if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
+                         if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
                          self.FTP_WEI = ftp_wei
                          self.EXP_CODE = exp_code
                      self.draw()
-                     if self.figfile == None:
-                         str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
-                         self.figfile = self.getFilename(name = str_datetime)
-                     if figpath != '':
+                     if save:
+                         if self.figfile == None:
+                             str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
+                             self.figfile = self.getFilename(name = str_datetime)
                          self.counter_imagwr += 1
                          if (self.counter_imagwr>=wr_period):
-                             # store png plot to local folder
+                             # store png plot to local folder
                              self.saveFigure(figpath, self.figfile)
-                             # store png plot to FTP server according to RT-Web format
-                             name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
-                             ftp_filename = os.path.join(figpath, name)
-                             self.saveFigure(figpath, ftp_filename)
                              self.counter_imagwr = 0
+                             if ftp:
+                                 # store png plot to FTP server according to RT-Web format
+                                 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
+                                 ftp_filename = os.path.join(figpath, name)
+                                 self.saveFigure(figpath, ftp_filename)
                      if x[1] >= self.axesList[0].xmax:
                          self.counter_imagwr = wr_period
                  def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
                          xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
                          timerange=None,
-                         save=False, figpath='', figfile=None, ftp=False, wr_period=1,
+                         save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
                          coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
                                  raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair)
                              pairsIndexList.append(dataOut.pairsList.index(pair))
-                     if timerange != None:
-                         self.timerange = timerange
                      if pairsIndexList == []:
                          return
                      y = dataOut.getHeiRange()
                      #thisDatetime = dataOut.datatime
-                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " CoherenceMap" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
                      xlabel = ""
                      ylabel = "Range (Km)"
                                     showprofile=showprofile,
                                     show=show)
-                         #tmin, tmax = self.getTimeLim(x, xmin, xmax)
+                         if timerange != None:
+                             self.timerange = timerange
                          self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
                          self.counter_imagwr = wr_period
                          self.__isConfig = False
-                     if figfile == None:
-                         str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
-                         figfile = self.getFilename(name = str_datetime)
-                     if figpath != '':
+                     if save:
+                         if figfile == None:
+                             str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
+                             figfile = self.getFilename(name = str_datetime)
                          self.counter_imagwr += 1
                          if (self.counter_imagwr>=wr_period):
-                             # store png plot to local folder
+                             # store png plot to local folder
                              self.saveFigure(figpath, figfile)
-                             # store png plot to FTP server according to RT-Web format
-                             name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
-                             ftp_filename = os.path.join(figpath, name)
-                             self.saveFigure(figpath, ftp_filename)
                              self.counter_imagwr = 0
+                             if ftp:
+                                 # store png plot to FTP server according to RT-Web format
+                                 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
+                                 ftp_filename = os.path.join(figpath, name)
+                                 self.saveFigure(figpath, ftp_filename)
              class PowerProfile(Figure):
                  isConfig = None
                  def run(self, dataOut, id, wintitle="", channelList=None,
                          xmin=None, xmax=None, ymin=None, ymax=None,
-                         save=False, figpath='', figfile=None, show=True, wr_period=1,
-                         server=None, folder=None, username=None, password=None,):
+                         save=False, figpath='./', figfile=None, show=True,
+                         ftp=False, wr_period=1, server=None,
+                         folder=None, username=None, password=None):
-                     if dataOut.flagNoData:
-                         return None
                      if channelList == None:
                          channelIndexList = dataOut.channelIndexList
                                  raise ValueError, "Channel %d is not in dataOut.channelList"
                              channelIndexList.append(dataOut.channelList.index(channel))
-                     try:
-                         factor = dataOut.normFactor
-                     except:
-                         factor = 1
+                     factor = dataOut.normFactor
                      y = dataOut.getHeiRange()
                      xdB = 10*numpy.log10(x)
-                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Power Profile %s" %(thisDatetime.strftime("%d-%b-%Y"))
                      xlabel = "dB"
                      ylabel = "Range (Km)"
                          if ymin == None: ymin = numpy.nanmin(y)
                          if ymax == None: ymax = numpy.nanmax(y)
                          if xmin == None: xmin = numpy.nanmin(xdB)*0.9
-                         if xmax == None: xmax = numpy.nanmax(xdB)*0.9
+                         if xmax == None: xmax = numpy.nanmax(xdB)*1.1
                          self.__isConfig = True
                      self.draw()
-                     if figfile == None:
-                         str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
-                         figfile = self.getFilename(name = str_datetime)
-                     if figpath != '':
+                     if save:
+                         if figfile == None:
+                             str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
+                             figfile = self.getFilename(name = str_datetime)
                          self.counter_imagwr += 1
                          if (self.counter_imagwr>=wr_period):
-                             # store png plot to local folder
+                             # store png plot to local folder
                              self.saveFigure(figpath, figfile)
-                             # store png plot to FTP server according to RT-Web format
-                             #name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
-                             #ftp_filename = os.path.join(figpath, name)
-                             #self.saveFigure(figpath, ftp_filename)
                              self.counter_imagwr = 0
              class Noise(Figure):
                  isConfig = None
                      self.PLOT_POS = None
                      self.figfile = None
+                     self.xmin = None
+                     self.xmax = None
                  def getSubplots(self):
                      ncol = 1
                  def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
                          xmin=None, xmax=None, ymin=None, ymax=None,
                          timerange=None,
-                         save=False, figpath='', figfile=None, show=True, ftp=False, wr_period=1,
+                         save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
                                  raise ValueError, "Channel %d is not in dataOut.channelList"
                              channelIndexList.append(dataOut.channelList.index(channel))
-                     if timerange != None:
-                         self.timerange = timerange
-                     tmin = None
-                     tmax = None
                      x = dataOut.getTimeRange()
-                     y = dataOut.getHeiRange()
+                     #y = dataOut.getHeiRange()
                      factor = dataOut.normFactor
                      noise = dataOut.noise/factor
                      noisedB = 10*numpy.log10(noise)
                      #thisDatetime = dataOut.datatime
-                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Noise" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
                      xlabel = ""
                      ylabel = "Intensity (dB)"
                                     showprofile=showprofile,
                                     show=show)
-                         tmin, tmax = self.getTimeLim(x, xmin, xmax)
-                         if ymin == None: ymin = numpy.nanmin(noisedB) - 10.0
+                         if timerange != None:
+                             self.timerange = timerange
+                         self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
+                         if ymin == None: ymin = numpy.floor(numpy.nanmin(noisedB)) - 10.0
                          if ymax == None: ymax = numpy.nanmax(noisedB) + 10.0
                          self.FTP_WEI = ftp_wei
                      axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
-                                 xmin=tmin, xmax=tmax, ymin=ymin, ymax=ymax,
+                                 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
                                  xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
                                  XAxisAsTime=True, grid='both'
                                  )
                          del self.ydata
                          self.__isConfig = False
-                     if self.figfile == None:
-                         str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
-                         self.figfile = self.getFilename(name = str_datetime)
-                     if figpath != '':
+                     if save != '':
+                         if self.figfile == None:
+                             str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
+                             self.figfile = self.getFilename(name = str_datetime)
                          self.counter_imagwr += 1
                          if (self.counter_imagwr>=wr_period):
-                             # store png plot to local folder
+                             # store png plot to local folder
                              self.saveFigure(figpath, self.figfile)
-                             # store png plot to FTP server according to RT-Web format
-                             name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
-                             ftp_filename = os.path.join(figpath, name)
-                             self.saveFigure(figpath, ftp_filename)
                              self.counter_imagwr = 0
+                             if ftp:
+                                 # store png plot to FTP server according to RT-Web format
+                                 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
+                                 ftp_filename = os.path.join(figpath, name)
+                                 self.saveFigure(figpath, ftp_filename)
              class BeaconPhase(Figure):
                      self.figfile = None
+                     self.xmin = None
+                     self.xmax = None
                  def getSubplots(self):
                      ncol = 1
                  def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
                          xmin=None, xmax=None, ymin=None, ymax=None,
                          timerange=None,
-                         save=False, figpath='', figfile=None, show=True, ftp=False, wr_period=1,
+                         save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
                          server=None, folder=None, username=None, password=None,
                          ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
              #         if len(pairsIndexList) > 4:
              #             pairsIndexList = pairsIndexList[0:4]
-                     if timerange != None:
-                         self.timerange = timerange
-                     tmin = None
-                     tmax = None
                      x = dataOut.getTimeRange()
-                     y = dataOut.getHeiRange()
+                     #y = dataOut.getHeiRange()
                      #thisDatetime = dataOut.datatime
-                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      title = wintitle + " Phase of Beacon Signal" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
                      xlabel = "Local Time"
                      ylabel = "Phase"
                                     showprofile=showprofile,
                                     show=show)
-                         tmin, tmax = self.getTimeLim(x, xmin, xmax)
+                         if timerange != None:
+                             self.timerange = timerange
+                         self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
                          if ymin == None: ymin = numpy.nanmin(phase_beacon) - 10.0
                          if ymax == None: ymax = numpy.nanmax(phase_beacon) + 10.0
                      axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
-                                 xmin=tmin, xmax=tmax, ymin=ymin, ymax=ymax,
+                                 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
                                  xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
                                  XAxisAsTime=True, grid='both'
                                  )
                          del self.ydata
                          self.__isConfig = False
-                     if self.figfile == None:
-                         str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
-                         self.figfile = self.getFilename(name = str_datetime)
-                     if figpath != '':
+                     if save:
+                         if self.figfile == None:
+                             str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
+                             self.figfile = self.getFilename(name = str_datetime)
                          self.counter_imagwr += 1
                          if (self.counter_imagwr>=wr_period):
-                             # store png plot to local folder
+                             # store png plot to local folder
                              self.saveFigure(figpath, self.figfile)
-                             # store png plot to FTP server according to RT-Web format
-                             name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
-                             ftp_filename = os.path.join(figpath, name)
-                             self.saveFigure(figpath, ftp_filename)
                              self.counter_imagwr = 0
+                             if ftp:
+                                 # store png plot to FTP server according to RT-Web format
+                                 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
+                                 ftp_filename = os.path.join(figpath, name)
+                                 self.saveFigure(figpath, ftp_filename)

schainpy/model/graphics/jroplot_voltage.py +6 -6

              '''
-             @author: Daniel Suarez
-             '''
+             Created on Jul 9, 2014
+             @author: roj-idl71
+             '''
+             import os
              import datetime
              import numpy
                  def run(self, dataOut, id, wintitle="", channelList=None,
                          xmin=None, xmax=None, ymin=None, ymax=None, save=False,
                          figpath='./', figfile=None, show=True, wr_period=1,
-                         server=None, folder=None, username=None, password=None, type='power'):
+                         ftp=False, server=None, folder=None, username=None, password=None, type='power'):
                      """
                          ymin            :    None,
                          ymax            :    None,
                      """
-                     if dataOut.flagNoData:
-                         return None
                      if channelList == None:
                          channelIndexList = dataOut.channelIndexList
                      y = dataOut.data[channelIndexList,:] * numpy.conjugate(dataOut.data[channelIndexList,:])
                      y = y.real
-                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
+                     thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
                      if type == "power":
                          self.plot_power(dataOut.heightList,

schainpy/model/graphics/mpldriver.py 0 -3

              def pcolor(imesh, z, xlabel='', ylabel='', title=''):
                  z = z.T
                  ax = imesh.get_axes()
                  printLabels(ax, xlabel, ylabel, title)
                  imesh.set_array(z.ravel())
              def addpcolor(ax, x, y, z, zmin, zmax, xlabel='', ylabel='', title='', colormap='jet'):

schainpy/model/io/__init__.py +30 0

			@@ -0,0 +1,30
		1	'''
		2
		3	$Author: murco $
		4	$Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $
		5	'''
		6
		7	from jroIO_voltage import *
		8	from jroIO_spectra import *
		9	from jroIO_heispectra import *
		10	from jroIO_usrp import *
		11
		12	try:
		13	from jroIO_usrp_api import *
		14	except:
		15	print "jroIO_usrp_api could not be imported"
		16
		17	try:
		18	from jroIO_amisr import *
		19	except:
		20	print "jroIO_amisr could not be imported"
		21
		22	try:
		23	from jroIO_HDF5 import *
		24	except:
		25	print "jroIO_HDF5 could not be imported"
		26
		27	try:
		28	from jroIO_hf import *
		29	except:
		30	print "jroIO_hf could not be imported" No newline at end of file

schainpy/model/io/jroIO_HDF5.py +3 -3

              import h5py
              import re
-             from model.data.jrodata import *
-             from model.proc.jroproc_base import ProcessingUnit, Operation
-             from model.io.jroIO_base import *
+             from schainpy.model.data.jrodata import *
+             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
+             from schainpy.model.io.jroIO_base import *
              class HDF5Reader(ProcessingUnit):

schainpy/model/io/jroIO_amisr.py +2 -2

              import h5py
              import numpy
-             from model.proc.jroproc_base import ProcessingUnit, Operation
-             from model.data.jroamisr import AMISR
+             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
+             from schainpy.model.data.jroamisr import AMISR
              class RadacHeader():
                  def __init__(self, fp):

schainpy/model/io/jroIO_base.py +40 -47

              '''
+             Created on Jul 2, 2014
+             @author: roj-idl71
              '''
              import os
              import sys
              #import h5py
              import traceback
-             #try:
-             #    import pyfits
-             #except:
-             #    print "pyfits module has not been imported, it should be installed to save files in fits format"
-             #from jrodata import *
-             #from jroheaderIO import *
-             #from jroprocessing import *
-             #import re
-             #from xml.etree.ElementTree import Element, SubElement, ElementTree
-             LOCALTIME = True #-18000
+             try:
+                 from gevent import sleep
+             except:
+                 from time import sleep
+             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
-             from model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
+             LOCALTIME = True
-             def isNumber(str):
+             def isNumber(cad):
                  """
                  Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                      False   :    no es un string numerico
                  """
                  try:
-                     float( str )
+                     float( cad )
                      return True
                  except:
                      return False
                  flagIsNewFile = 1
-                 flagTimeBlock = 0
+                 flagDiscontinuousBlock = 0
                  flagIsNewBlock = 0
                          fileList = glob.glob1(thisPath, "*%s" %ext)
                          if len(fileList) < 1:
                              continue
                          fileList.sort()
                          pathDict.setdefault(fileList[0])
                          pathDict[fileList[0]] = i
                              for nTries in range( tries ):
                                  if firstTime_flag:
                                      print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
-                                     time.sleep( self.delay )
+                                     sleep( self.delay )
                                  else:
                                      print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
              #                self.flagEoF = True
                              return 0
-                         print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
-                         time.sleep( self.delay )
+                         print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
+                         sleep( self.delay )
                      return 0
                          if ( currentSize >= neededSize ):
                              return 1
-                         print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
-                         time.sleep( self.delay )
+                         print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
+                         sleep( self.delay )
                      return 0
                      deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
-                     self.flagTimeBlock = 0
+                     self.flagDiscontinuousBlock = 0
                      if deltaTime > self.maxTimeStep:
-                         self.flagTimeBlock = 1
+                         self.flagDiscontinuousBlock = 1
                      return 1
                      except IOError:
                          traceback.print_exc()
                          if msgFlag:
-                             print "The file %s can't be opened" % (filename)
+                             print "[Reading] The file %s can't be opened" % (filename)
                          return False
                      neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
                          except IOError:
                              traceback.print_exc()
                              if msgFlag:
-                                 print "\tThe file %s is empty or it hasn't enough data" % filename
+                                 print "[Reading] The file %s is empty or it hasn't enough data" % filename
                              fp.close()
                              return False
                      else:
-                         msg = "\tSkipping the file %s due to it hasn't enough data" %filename
+                         msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
                      fp.close()
                      fileSize = os.path.getsize(filename)
                              nTxs = 1):
                      if path == None:
-                         raise ValueError, "The path is not valid"
+                         raise ValueError, "[Reading] The path is not valid"
                      if ext == None:
                          ext = self.ext
                      if online:
-                         print "Searching files in online mode..."
+                         print "[Reading] Searching files in online mode..."
                          for nTries in range( self.nTries ):
                              fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
                              if fullpath:
                                  break
-                             print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
-                             time.sleep( self.delay )
+                             print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
+                             sleep( self.delay )
                          if not(fullpath):
-                             print "There 'isn't valied files in %s" % path
+                             print "There 'isn't any valid file in %s" % path
                              return None
                          self.year = year
                          last_set = None
                      else:
-                         print "Searching files in offline mode ..."
+                         print "[Reading] Searching files in offline mode ..."
                          pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                             startTime=startTime, endTime=endTime,
                                                                             set=set, expLabel=expLabel, ext=ext,
                                                                             walk=walk)
                          if not(pathList):
-                             print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
+                             print "[Reading] No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
                                                                      datetime.datetime.combine(startDate,startTime).ctime(),
                                                                      datetime.datetime.combine(endDate,endTime).ctime())
                      if not(self.setNextFile()):
                          if (startDate!=None) and (endDate!=None):
-                             print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
+                             print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                          elif startDate != None:
-                             print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
+                             print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
                          else:
-                             print "No files"
+                             print "[Reading] No files"
                          sys.exit(-1)
                      self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
-                     self.dataOut.flagTimeBlock = self.flagTimeBlock
+                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                      self.dataOut.timeZone = self.basicHeaderObj.timeZone
                  def printReadBlocks(self):
-                     print "Number of read blocks per file %04d" %self.nReadBlocks
+                     print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
                  def printTotalBlocks(self):
-                     print "Number of read blocks %04d" %self.nTotalBlocks
+                     print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
                  def printNumberOfBlock(self):
                      if self.flagIsNewBlock:
-                         print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
+                         print "[Reading] Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
                          self.dataOut.blocknow = self.basicHeaderObj.dataBlock
                  def printInfo(self):
                      setFile = self.setFile
                      setFile += 1
-                     file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
+                     filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                      timeTuple.tm_year,
                                                      timeTuple.tm_yday,
                                                      setFile,
                                                      ext )
-                     filename = os.path.join( path, subfolder, file )
+                     filename = os.path.join( path, subfolder, filen )
                      fp = open( filename,'wb' )
                      self.setFirstHeader()
-                     print 'Writing the file: %s'%self.filename
+                     print '[Writing] file: %s'%self.filename
                      self.__writeFirstHeader()
                      self.dataOut = dataOut
                      if not(self.setNextFile()):
-                         print "There isn't a next file"
+                         print "[Writing] There isn't a next file"
                          return 0
                      self.setBlockDimension()

schainpy/model/io/jroIO_example.py +8 -4

                      ProcessingUnit.__init__(self)
-             #         self.dataIn = None
+             #
-             #         self.isConfig = False
                      #Is really necessary create the output object in the initializer
                      self.dataOut = Voltage()
                      '''
+                     '''
+                     Add code
+                     '''
                      self.isConfig = True
                  def run(self, **kwargs):
                      if not self.isConfig:
                          self.setup(**kwargs)
+                     '''
+                     Add code
+                     '''
              class Writer(Operation):
                  '''
                  classdocs

schainpy/model/io/jroIO_heispectra.py +85 -14

              '''
+             Created on Jul 3, 2014
+             @author: roj-idl71
              '''
              import os, sys
              import glob
              try:
+                 from gevent import sleep
+             except:
+                 from time import sleep
+             try:
                  import pyfits
              except:
                  """
                  """
              from xml.etree.ElementTree import ElementTree
              from jroIO_base import isDoyFolder, isNumber
-             from model.proc.jroproc_base import Operation, ProcessingUnit
+             from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
+             class Fits:
+                 name=None
+                 format=None
+                 array =None
+                 data =None
+                 thdulist=None
+                 prihdr=None
+                 hdu=None
+                 def __init__(self):
+                     pass
+                 def setColF(self,name,format,array):
+                     self.name=name
+                     self.format=format
+                     self.array=array
+                     a1=numpy.array([self.array],dtype=numpy.float32)
+                     self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)
+                     return self.col1
+             #    def setColP(self,name,format,data):
+             #        self.name=name
+             #        self.format=format
+             #        self.data=data
+             #        a2=numpy.array([self.data],dtype=numpy.float32)
+             #        self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
+             #        return self.col2
+                 def writeData(self,name,format,data):
+                     self.name=name
+                     self.format=format
+                     self.data=data
+                     a2=numpy.array([self.data],dtype=numpy.float32)
+                     self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
+                     return self.col2
+                 def cFImage(self,idblock,year,month,day,hour,minute,second):
+                     self.hdu= pyfits.PrimaryHDU(idblock)
+                     self.hdu.header.set("Year",year)
+                     self.hdu.header.set("Month",month)
+                     self.hdu.header.set("Day",day)
+                     self.hdu.header.set("Hour",hour)
+                     self.hdu.header.set("Minute",minute)
+                     self.hdu.header.set("Second",second)
+                     return self.hdu
+                 def Ctable(self,colList):
+                     self.cols=pyfits.ColDefs(colList)
+                     self.tbhdu = pyfits.new_table(self.cols)
+                     return self.tbhdu
+                 def CFile(self,hdu,tbhdu):
+                     self.thdulist=pyfits.HDUList([hdu,tbhdu])
+                 def wFile(self,filename):
+                     if os.path.isfile(filename):
+                         os.remove(filename)
+                     self.thdulist.writeto(filename)
              class ParameterConf:
                  ELEMENTNAME = 'Parameter'
                      setFile = self.setFile
                      setFile += 1
-                     file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
+                     thisFile = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                      timeTuple.tm_year,
                                                      timeTuple.tm_yday,
                                                      setFile,
                                                      ext )
-                     filename = os.path.join( path, subfolder, file )
+                     filename = os.path.join( path, subfolder, thisFile )
                      self.blockIndex = 0
                      self.filename = filename
                      self.setFile = 0
                      self.flagNoMoreFiles = 0
                      self.flagIsNewFile = 1
-                     self.flagTimeBlock = None
+                     self.flagDiscontinuousBlock = None
                      self.fileIndex = None
                      self.filename = None
                      self.fileSize = None
                          fileList = glob.glob1(thisPath, "*%s" %ext)
                          fileList.sort()
-                         for file in fileList:
+                         for thisFile in fileList:
-                             filename = os.path.join(thisPath,file)
+                             filename = os.path.join(thisPath,thisFile)
                              thisDatetime = self.isFileinThisTime(filename, startTime, endTime)
                              if not(thisDatetime):
                              return 1
                          print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
-                         time.sleep( self.delay )
+                         sleep( self.delay )
                      return 0
                      deltaTime = self.utc - self.lastUTTime
-                     self.flagTimeBlock = 0
+                     self.flagDiscontinuousBlock = 0
                      if deltaTime > self.maxTimeStep:
-                         self.flagTimeBlock = 1
+                         self.flagDiscontinuousBlock = 1
                      return 1
                          print 'Process finished'
                          return 0
-                     self.flagTimeBlock = 0
+                     self.flagDiscontinuousBlock = 0
                      self.flagIsNewBlock = 0
                      if not(self.readNextBlock()):
                  subfolder = None
                  def __init__(self):
-                     self.wrObj = FITS()
+                     self.wrObj = Fits()
              #        self.dataOut = dataOut
                      self.nTotalBlocks=0
              #        self.set = None
              #            self.setFile = 0
                      #make the filename
-                     file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
+                     thisFile = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
-                     filename = os.path.join(self.wrpath,self.subfolder, file)
+                     filename = os.path.join(self.wrpath,self.subfolder, thisFile)
                      idblock = numpy.array([self.idblock],dtype="int64")
                      header=self.wrObj.cFImage(idblock=idblock,

schainpy/model/io/jroIO_spectra.py +17 -14

              '''
-             '''
+             Created on Jul 2, 2014
+             @author: roj-idl71
+             '''
              import numpy
              from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
-             from model.proc.jroproc_base import ProcessingUnit, Operation
-             from model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
-             from model.data.jrodata import Spectra
+             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
+             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
+             from schainpy.model.data.jrodata import Spectra
              class SpectraReader(JRODataReader, ProcessingUnit):
                  """
              #         self.ippSeconds = 0
-                     self.flagTimeBlock = 0
+                     self.flagDiscontinuousBlock = 0
                      self.flagIsNewBlock = 0
                      self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
-                     self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip
+                     self.dataOut.flagDeflipData = False #asumo q la data esta sin flip
                      if self.radarControllerHeaderObj.code != None:
-                         self.dataOut.nCode = self.radarControllerHeaderObj.nCode
-                         self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
-                         self.dataOut.code = self.radarControllerHeaderObj.code
+             #             self.dataOut.nCode = self.radarControllerHeaderObj.nCode
+             #
+             #             self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
+             #
+             #             self.dataOut.code = self.radarControllerHeaderObj.code
                          self.dataOut.flagDecodeData = True
                      Affected:
                          self.dataOut
-                         self.flagTimeBlock
+                         self.flagDiscontinuousBlock
                          self.flagIsNewBlock
                      """
                          print 'Process finished'
                          return 0
-                     self.flagTimeBlock = 0
+                     self.flagDiscontinuousBlock = 0
                      self.flagIsNewBlock = 0
                      if self.__hasNotDataInBuffer():
                      self.nWriteBlocks += 1
                      self.blockIndex += 1
+                     print "[Writing] Block = ", self.blockIndex
                  def putData(self):
                      """
                      self.flagIsNewBlock = 0
-                     if self.dataOut.flagTimeBlock:
+                     if self.dataOut.flagDiscontinuousBlock:
                          self.data_spc.fill(0)
                          self.data_cspc.fill(0)
                          self.data_dc.fill(0)

schainpy/model/io/jroIO_voltage.py +20 -17

              '''
+             Created on Jul 2, 2014
+             @author: roj-idl71
              '''
              import numpy
              from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
-             from model.proc.jroproc_base import ProcessingUnit, Operation
-             from model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
-             from model.data.jrodata import Voltage
+             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
+             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
+             from schainpy.model.data.jrodata import Voltage
              class VoltageReader(JRODataReader, ProcessingUnit):
                  """
              #         self.ippSeconds = 0
-                     self.flagTimeBlock = 0
+                     self.flagDiscontinuousBlock = 0
                      self.flagIsNewBlock = 0
                          self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
              #         self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
-                     if self.radarControllerHeaderObj.code != None:
-                         self.dataOut.nCode = self.radarControllerHeaderObj.nCode
-                         self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
-                         self.dataOut.code = self.radarControllerHeaderObj.code
+             #
+             #         if self.radarControllerHeaderObj.code != None:
+             #
+             #             self.dataOut.nCode = self.radarControllerHeaderObj.nCode
+             #
+             #             self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
+             #
+             #             self.dataOut.code = self.radarControllerHeaderObj.code
                      self.dataOut.dtype = self.dtype
                      Affected:
                          self.dataOut
                          self.profileIndex
-                         self.flagTimeBlock
+                         self.flagDiscontinuousBlock
                          self.flagIsNewBlock
                      """
                          print 'Process finished'
                          return 0
-                     self.flagTimeBlock = 0
+                     self.flagDiscontinuousBlock = 0
                      self.flagIsNewBlock = 0
                      if self.__hasNotDataInBuffer():
                                                   self.processingHeaderObj.profilesPerBlock,
                                                   self.processingHeaderObj.nHeights),
                                                   dtype=numpy.dtype('complex64'))
                  def writeBlock(self):
                      """
                      self.blockIndex += 1
                      self.nTotalBlocks += 1
+                     print "[Writing] Block = ", self.blockIndex
                  def putData(self):
                      """
                      Setea un bloque de datos y luego los escribe en un file
                      self.flagIsNewBlock = 0
-                     if self.dataOut.flagTimeBlock:
+                     if self.dataOut.flagDiscontinuousBlock:
                          self.datablock.fill(0)
                          self.profileIndex = 0
                          self.setNextFile()

schainpy/model/proc/__init__.py +12 0

			@@ -0,0 +1,12
		1	'''
		2
		3	$Author: murco $
		4	$Id: Processor.py 1 2012-11-12 18:56:07Z murco $
		5	'''
		6
		7	from jroproc_voltage import *
		8	from jroproc_spectra import *
		9	from jroproc_heispectra import *
		10	from jroproc_amisr import *
		11	from jroproc_correlation import *
		12	from jroproc_parameters import * No newline at end of file

schainpy/model/proc/cfunctions.pyx ~~schainpy/model/cfunctions.pyx~~ renamed +1 -1

-             import numpy
+             # import numpy
              cimport numpy
              def decoder(numpy.ndarray[numpy.complex_t, ndim=2] fft_code, numpy.ndarray[numpy.complex_t, ndim=2] data):

schainpy/model/proc/jroproc_amisr.py +1 -1

              '''
              import numpy
              from jroproc_base import ProcessingUnit, Operation
-             from model.data.jroamisr import AMISR
+             from schainpy.model.data.jroamisr import AMISR
              class AMISRProc(ProcessingUnit):
                  def __init__(self):

schainpy/model/proc/jroproc_base.py +33 -18

              '''
+             $Author: murco $
+             $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
              '''
-             class ProcessingUnit:
+             class ProcessingUnit(object):
                  """
                  Esta es la clase base para el procesamiento de datos.
                          **kwargs     :    diccionario con los nombres y valores de la funcion a ejecutar.
                      """
+                     #Checking the inputs
                      if name == 'run':
                          if not self.checkInputs():
                  def call(self, opType, opName=None, opId=None, **kwargs):
                      """
-                     Return True si ejecuta la operacion "operationConf.name" con los
-                     argumentos "**kwargs". False si la operacion no se ha ejecutado.
-                     La operacion puede ser de dos tipos:
+                     Return True si ejecuta la operacion interna nombrada "opName" o la operacion externa
+                     identificada con el id "opId"; con los argumentos "**kwargs".
-. Un metodo propio de esta clase:
-                             operation.type = "self"
+                     False si la operacion no se ha ejecutado.
+                     Input:
+                         opType    :    Puede ser "self" o "external"
+                             La operacion puede ser de dos tipos (callMethod or callObject):
-. El metodo "run" de un objeto del tipo Operation o de un derivado de ella:
-                             operation.type = "other".
+. Un metodo propio de esta clase:
+                                 opType = "self"
-                            Este objeto de tipo Operation debe de haber sido agregado antes con el metodo:
-                            "addOperation" e identificado con el operation.id
+. El metodo "run" de un objeto del tipo Operation o de un derivado de ella:
+                                 opType = "other" or "external".
-                     con el id de la operacion.
-                     Input:
+                         opName    : Si la operacion es interna (opType = 'self'), entonces el "opName" sera
+                                     usada para llamar a un metodo interno de la clase Processing
+                         opId    :    Si la operacion es externa (opType = 'other'), entonces el "opId" sera
+                                     usada para llamar al metodo "run" de la clase Operation registrada con ese Id
-                         Operation    :    Objeto del tipo operacion con los atributos: name, type y id.
+                     Exception:
+                            Este objeto de tipo Operation debe de haber sido agregado antes con el metodo:
+                            "addOperation" e identificado con el valor "opId"  = el id de la operacion.
+                            De lo contrario retornara un error del tipo IOError
                      """
                      raise ValueError, "Not implemented"
-             class Operation():
+             class Operation(object):
                  """
                  Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
                  def run(self, dataIn, **kwargs):
                      """
-                     Realiza las operaciones necesarias sobre la dataIn.data y actualiza los atributos del objeto dataIn.
+                     Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
+                     atributos del objeto dataIn.
                      Input:

schainpy/model/proc/jroproc_correlation.py +2 -2

              import numpy
              from jroproc_base import ProcessingUnit, Operation
-             from model.data.jrodata import Correlation
+             from schainpy.model.data.jrodata import Correlation
              class CorrelationProc(ProcessingUnit):
                      self.dataOut.nCode = self.dataIn.nCode
                      self.dataOut.code = self.dataIn.code
              #        self.dataOut.nProfiles = self.dataOut.nFFTPoints
-                     self.dataOut.flagTimeBlock = self.dataIn.flagTimeBlock
+                     self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
                      self.dataOut.utctime = self.firstdatatime
                      self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
                      self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip

schainpy/model/proc/jroproc_heispectra.py +2 -2

              import numpy
              from jroproc_base import ProcessingUnit, Operation
-             from model.data.jrodata import SpectraHeis
+             from schainpy.model.data.jrodata import SpectraHeis
              class SpectraHeisProc(ProcessingUnit):
                      self.dataOut.nFFTPoints = self.dataIn.nHeights
              #        self.dataOut.channelIndexList = self.dataIn.channelIndexList
              #        self.dataOut.flagNoData = self.dataIn.flagNoData
-                     self.dataOut.flagTimeBlock = self.dataIn.flagTimeBlock
+                     self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
                      self.dataOut.utctime = self.dataIn.utctime
              #        self.dataOut.utctime = self.firstdatatime
                      self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada

schainpy/model/proc/jroproc_parameters.py +2 -2

              import itertools
              from jroproc_base import ProcessingUnit, Operation
-             from model.data.jrodata import Parameters
+             from schainpy.model.data.jrodata import Parameters
              class ParametersProc(ProcessingUnit):
                      self.dataOut.nCode = self.dataIn.nCode
                      self.dataOut.code = self.dataIn.code
              #        self.dataOut.nProfiles = self.dataOut.nFFTPoints
-                     self.dataOut.flagTimeBlock = self.dataIn.flagTimeBlock
+                     self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
                      self.dataOut.utctime = self.firstdatatime
                      self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
                      self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip

schainpy/model/proc/jroproc_spectra.py +5 -4

              import math
              from jroproc_base import ProcessingUnit, Operation
-             from model.data.jrodata import Spectra
-             from model.data.jrodata import hildebrand_sekhon
+             from schainpy.model.data.jrodata import Spectra
+             from schainpy.model.data.jrodata import hildebrand_sekhon
              class SpectraProc(ProcessingUnit):
                      self.dataOut.code = self.dataIn.code
                      self.dataOut.nProfiles = self.dataOut.nFFTPoints
              #        self.dataOut.channelIndexList = self.dataIn.channelIndexList
-                     self.dataOut.flagTimeBlock = self.dataIn.flagTimeBlock
+                     self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
                      self.dataOut.utctime = self.firstdatatime
                      self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
                      self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip
                              raise ValueError, "This SpectraProc.run() need nFFTPoints input variable"
                          if nProfiles == None:
-                             raise ValueError, "This SpectraProc.run() need nProfiles input variable"
+                             nProfiles = nFFTPoints
+             #                 raise ValueError, "This SpectraProc.run() need nProfiles input variable"
                          if ippFactor == None:

schainpy/model/proc/jroproc_voltage.py +158 -14

		@@ -1,8 +1,7
1	1	import numpy
2	2
3	3	from jroproc_base import ProcessingUnit, Operation
4		from model.data.jrodata import Voltage
5
	4	from schainpy.model.data.jrodata import Voltage
6	5
7	6	class VoltageProc(ProcessingUnit):
8	7
		@@ -207,7 +206,7 class VoltageProc(ProcessingUnit):
207	206	return 1
208	207
209	208
210		def filterByHeights(self, window, ~~axis~~=2):
	209	def filterByHeights(self, window):
211	210
212	211	deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
213	212
		@@ -233,8 +232,17 class VoltageProc(ProcessingUnit):
233	232	self.dataOut.data = buffer.copy()
234	233	self.dataOut.heightList = numpy.arange(self.dataOut.heightList[0],newdelta*(self.dataOut.nHeights-r)/window,newdelta)
235	234	self.dataOut.windowOfFilter = window
	235
	236	def setH0(self, h0, deltaHeight = None):
236	237
237		return 1
	238	if not deltaHeight:
	239	deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
	240
	241	nHeights = self.dataOut.nHeights
	242
	243	newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
	244
	245	self.dataOut.heightList = newHeiRange
238	246
239	247	def deFlip(self, channelList = []):
240	248
		@@ -267,8 +275,6 class VoltageProc(ProcessingUnit):
267	275
268	276	self.dataOut.data = data
269	277
270
271
272	278	def setRadarFrequency(self, frequency=None):
273	279
274	280	if frequency != None:
		@@ -571,13 +577,15 class Decoder(Operation):
571	577
572	578	def convolutionInFreqOpt(self, data):
573	579
574		fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
575
576		data = cfunctions.decoder(fft_code, data)
577
578		datadec = data#[:,:]
579
580		~~return~~ datadec
	580	raise NotImplementedError
	581
	582	# fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
	583	#
	584	# data = cfunctions.decoder(fft_code, data)
	585	#
	586	# datadec = data#[:,:]
	587	#
	588	# return datadec
581	589
582	590	def convolutionInTime(self, data):
583	591
		@@ -887,4 +895,140 class Reshaper(Operation):
887	895
888	896	dataOut.nProfiles = dataOut.data.shape[1]
889	897
890		dataOut.ippSeconds *= factor No newline at end of file
	898	dataOut.ippSeconds *= factor
	899
	900	import collections
	901	from scipy.stats import mode
	902
	903	class Synchronize(Operation):
	904
	905	isConfig = False
	906	__profIndex = 0
	907
	908	def __init__(self):
	909
	910	Operation.__init__(self)
	911	# self.isConfig = False
	912	self.__powBuffer = None
	913	self.__startIndex = 0
	914	self.__pulseFound = False
	915
	916	def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
	917
	918	#Read data
	919
	920	powerdB = dataOut.getPower(channel = channel)
	921	noisedB = dataOut.getNoise(channel = channel)[0]
	922
	923	self.__powBuffer.extend(powerdB.flatten())
	924
	925	dataArray = numpy.array(self.__powBuffer)
	926
	927	filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
	928
	929	maxValue = numpy.nanmax(filteredPower)
	930
	931	if maxValue < noisedB + 10:
	932	#No se encuentra ningun pulso de transmision
	933	return None
	934
	935	maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
	936
	937	if len(maxValuesIndex) < 2:
	938	#Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
	939	return None
	940
	941	phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
	942
	943	#Seleccionar solo valores con un espaciamiento de nSamples
	944	pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
	945
	946	if len(pulseIndex) < 2:
	947	#Solo se encontro un pulso de transmision con ancho mayor a 1
	948	return None
	949
	950	spacing = pulseIndex[1:] - pulseIndex[:-1]
	951
	952	#remover senales que se distancien menos de 10 unidades o muestras
	953	#(No deberian existir IPP menor a 10 unidades)
	954
	955	realIndex = numpy.where(spacing > 10 )[0]
	956
	957	if len(realIndex) < 2:
	958	#Solo se encontro un pulso de transmision con ancho mayor a 1
	959	return None
	960
	961	#Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
	962	realPulseIndex = pulseIndex[realIndex]
	963
	964	period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
	965
	966	print "IPP = %d samples" %period
	967
	968	self.__newNSamples = dataOut.nHeights #int(period)
	969	self.__startIndex = int(realPulseIndex[0])
	970
	971	return 1
	972
	973
	974	def setup(self, nSamples, nChannels, buffer_size = 4):
	975
	976	self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
	977	maxlen = buffer_size*nSamples)
	978
	979	bufferList = []
	980
	981	for i in range(nChannels):
	982	bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) + numpy.NAN,
	983	maxlen = buffer_size*nSamples)
	984
	985	bufferList.append(bufferByChannel)
	986
	987	self.__nSamples = nSamples
	988	self.__nChannels = nChannels
	989	self.__bufferList = bufferList
	990
	991	def run(self, dataOut, channel = 0):
	992
	993	if not self.isConfig:
	994	nSamples = dataOut.nHeights
	995	nChannels = dataOut.nChannels
	996	self.setup(nSamples, nChannels)
	997	self.isConfig = True
	998
	999	#Append new data to internal buffer
	1000	for thisChannel in range(self.__nChannels):
	1001	bufferByChannel = self.__bufferList[thisChannel]
	1002	bufferByChannel.extend(dataOut.data[thisChannel])
	1003
	1004	if self.__pulseFound:
	1005	self.__startIndex -= self.__nSamples
	1006
	1007	#Finding Tx Pulse
	1008	if not self.__pulseFound:
	1009	indexFound = self.__findTxPulse(dataOut, channel)
	1010
	1011	if indexFound == None:
	1012	dataOut.flagNoData = True
	1013	return
	1014
	1015	self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
	1016	self.__pulseFound = True
	1017	self.__startIndex = indexFound
	1018
	1019	#If pulse was found ...
	1020	for thisChannel in range(self.__nChannels):
	1021	bufferByChannel = self.__bufferList[thisChannel]
	1022	#print self.__startIndex
	1023	x = numpy.array(bufferByChannel)
	1024	self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
	1025
	1026	deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
	1027	dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
	1028	# dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
	1029
	1030	dataOut.data = self.__arrayBuffer
	1031
	1032	self.__startIndex += self.__newNSamples
	1033
	1034	return No newline at end of file

schainpy/model/utils/__init__.py +7 0

			@@ -0,0 +1,7
		1	'''
		2
		3	$Author: murco $
		4	$Id: Processor.py 1 2012-11-12 18:56:07Z murco $
		5	'''
		6
		7	from jroutils_ftp import * No newline at end of file

schainpy/model/utils/jroutils_ftp.py +1 -1

              import glob
              import ftplib
              import multiprocessing
-             from model.proc.jroproc_base import ProcessingUnit, Operation
+             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
              class FTP(object):
                  """

schainpy/test/testRawData.py +71 -128

		@@ -1,3 +1,8
	1	import os, sys
	2
	3	path = os.path.split(os.getcwd())[0]
	4	sys.path.append(path)
	5
1	6	from controller import *
2	7
3	8	if __name__ == '__main__':
		@@ -9,148 +14,86 if __name__ == '__main__':
9	14
10	15	controllerObj.setup(id = '191', name='test01', description=desc)
11	16
12		readUnitConfObj = controllerObj.addReadUnit(datatype='Voltage',
13		path='/~~remote/puma/2012_06/Meteors~~',
14		startDate='201~~2/06/21~~',
15		endDate='201~~2/06/21~~',
16		startTime='~~04:0~~0:00',
17		endTime='~~05:59:59~~',
	17	readUnitConfObj = controllerObj.addReadUnit(datatype='VoltageReader',
	18	path='/Volumes/SOUSY/',
	19	startDate='2014/10/28',
	20	endDate='2014/10/28',
	21	startTime='15:40:00',
	22	endTime='16:20:00',
18	23	online=0,
19	24	walk=1)
20	25
21		# opObj00 = readUnitConfObj.addOperation(name='print~~Info~~')
22
23		procUnitConfObj0 = controllerObj.addProcUnit(datatype='Voltage', inputId=readUnitConfObj.getId())
	26	opObj00 = readUnitConfObj.addOperation(name='printNumberOfBlock')
24	27
25		opObj10 = procUnitConfObj0.addOperation(name='selectChannels')
26		opObj10.addParameter(name='channelList', value='0,1,2', format='intlist')
27		#
28		opObj10 = procUnitConfObj0.addOperation(name='selectHeights')
29		opObj10.addParameter(name='minHei', value='140', format='float')
30		opObj10.addParameter(name='maxHei', value='180', format='float')
	28	procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc',
	29	inputId=readUnitConfObj.getId())
31	30
32		opObj12 = procUnitConfObj0.addOperation(name='~~Decoder~~', optype='~~other~~')
33
34		opObj12 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
35		opObj12.addParameter(name='n', value='4', format='int')
	31	# opObj10 = procUnitConfObj0.addOperation(name='Synchronize', optype='external')
36	32
37		# opObj11 = procUnitConfObj0.addOperation(name='~~Scope', optype='other~~')
38		# opObj11.addParameter(name='~~idfigure~~', value='10', format='int')
39		# opObj11.addParameter(name='~~wintitle', value='Voltage~~', format='~~str~~')
40		# opObj11.addParameter(name='zmin', value='40', format='int')
41		# opObj11.addParameter(name='zmax', value='90', format='int')
42
43		procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj0.getId())
44		procUnitConfObj1.addParameter(name='nFFTPoints', value='16', format='int')
45		procUnitConfObj1.addParameter(name='pairsList', value='(0,1),(0,2),(1,2)', format='pairslist')
46
	33	opObj10 = procUnitConfObj0.addOperation(name='selectHeights')
	34	opObj10.addParameter(name='minHei', value='0', format='float')
	35	opObj10.addParameter(name='maxHei', value='8', format='float')
47	36
48		opObj12 = procUnitConfObj1.addOperation(name='~~IncohInt'~~, ~~optype~~=~~'other~~')
49		opObj12.addParameter(name='n', value='10', format='int')
50
51		opObj11 = procUnitConfObj1.addOperation(name='~~SpectraPlot~~', optype='~~other~~')
52		opObj11.addParameter(name='~~idfigur~~e', value='1', format='int')
53		opObj11.addParameter(name='~~wintitl~~e', value='~~SpectraPlot0~~', format='~~str~~')
54		opObj11.addParameter(name='~~zmi~~n', value='35', format='int')
55		opObj11.addParameter(name='zmax', value='90', format='int')
56		opObj11.addParameter(name='showprofile', value='1', format='int')
57		opObj11.addParameter(name='figpath', value='/home/roj-idl71/Data/RAWDATA/BIESTATIC/RAWDATA_8CH/graphs')
58		opObj11.addParameter(name='save', value='1', format='int')
	37	opObj10 = procUnitConfObj0.addOperation(name='filterByHeights')
	38	opObj10.addParameter(name='window', value='2', format='float')
	39
	40	opObj10 = procUnitConfObj0.addOperation(name='Decoder', optype='external')
	41	opObj10.addParameter(name='code', value='1,-1', format='intlist')
	42	opObj10.addParameter(name='nCode', value='2', format='float')
	43	opObj10.addParameter(name='nBaud', value='1', format='float')
	44
59	45
60		opObj11 = procUnitConfObj1.addOperation(name='C~~rossSpectraPlo~~t', optype='~~other~~')
61		opObj11.addParameter(name='~~idfigure~~', value='2', format='int')
62		opObj11.addParameter(name='wintitle', value='CrossSpectraPlot', format='str')
63		opObj11.addParameter(name='zmin', value='35', format='int')
64		opObj11.addParameter(name='zmax', value='90', format='int')
65		opObj11.addParameter(name='figpath', value='/home/roj-idl71/Data/RAWDATA/BIESTATIC/RAWDATA_8CH/graphs')
66		opObj11.addParameter(name='save', value='1', format='int')
67
	46	opObj10 = procUnitConfObj0.addOperation(name='CohInt', optype='external')
	47	opObj10.addParameter(name='n', value='1296', format='float')
68	48
69		opObj11 = procUnitConfObj1.addOperation(name='CoherenceMap', optype='other')
70		opObj11.addParameter(name='idfigure', value='3', format='int')
71		opObj11.addParameter(name='wintitle', value='CoherenciaMap', format='str')
72		# opObj11.addParameter(name='timerange', value=str(60), format='int')
73		opObj11.addParameter(name='figpath', value='/home/roj-idl71/Data/RAWDATA/BIESTATIC/RAWDATA_8CH/graphs')
	49	procUnitConfObj1 = controllerObj.addProcUnit(datatype='SpectraProc',
	50	inputId=procUnitConfObj0.getId())
	51
	52	#Creating a processing object with its parameters
	53	#schainpy.model.proc.jroproc_spectra.SpectraProc.run()
	54	#If you need to add more parameters can use the "addParameter method"
	55	procUnitConfObj1.addParameter(name='nFFTPoints', value='128', format='int')
	56
	57	opObj10 = procUnitConfObj1.addOperation(name='IncohInt', optype='external')
	58	opObj10.addParameter(name='n', value='2', format='float')
	59
	60	#Using internal methods
	61	#schainpy.model.proc.jroproc_spectra.SpectraProc.selectChannels()
	62	# opObj10 = procUnitConfObj1.addOperation(name='selectChannels')
	63	# opObj10.addParameter(name='channelList', value='0,1', format='intlist')
	64
	65	#Using internal methods
	66	#schainpy.model.proc.jroproc_spectra.SpectraProc.selectHeights()
	67	# opObj10 = procUnitConfObj1.addOperation(name='selectHeights')
	68	# opObj10.addParameter(name='minHei', value='90', format='float')
	69	# opObj10.addParameter(name='maxHei', value='180', format='float')
	70
	71	#Using external methods (new modules)
	72	# #schainpy.model.proc.jroproc_spectra.IncohInt.setup()
	73	# opObj12 = procUnitConfObj1.addOperation(name='IncohInt', optype='other')
	74	# opObj12.addParameter(name='n', value='1', format='int')
	75
	76	#Using external methods (new modules)
	77	#schainpy.model.graphics.jroplot_spectra.SpectraPlot.setup()
	78	opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
	79	opObj11.addParameter(name='id', value='11', format='int')
	80	opObj11.addParameter(name='wintitle', value='SpectraPlot', format='str')
	81	opObj11.addParameter(name='zmin', value='-60', format='int')
	82	opObj11.addParameter(name='zmax', value='10', format='int')
74	83	opObj11.addParameter(name='save', value='1', format='int')
75	84
	85	#Using external methods (new modules)
	86	#schainpy.model.graphics.jroplot_spectra.RTIPlot.setup()
76	87	opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
77		opObj11.addParameter(name='id~~figure~~', value='4', format='int')
	88	opObj11.addParameter(name='id', value='30', format='int')
78	89	opObj11.addParameter(name='wintitle', value='RTI', format='str')
79		# opObj11.addParameter(name='~~timerange', value=str(60*60)~~, format='int')
80		opObj11.addParameter(name='zmin', value='35', format='int')
81		opObj11.addParameter(name='~~zmax~~', value='90', format='int')
82		opObj11.addParameter(name='figpath', value='/home/roj-idl71/Data/RAWDATA/BIESTATIC/RAWDATA_8CH/graphs')
	90	opObj11.addParameter(name='zmin', value='-60', format='int')
	91	opObj11.addParameter(name='zmax', value='-10', format='int')
	92	opObj11.addParameter(name='showprofile', value='1', format='int')
	93	# opObj11.addParameter(name='timerange', value=str(56060*60), format='int')
	94	opObj11.addParameter(name='xmin', value='14', format='float')
	95	opObj11.addParameter(name='xmax', value='23.9', format='float')
83	96	opObj11.addParameter(name='save', value='1', format='int')
84
85		# procUnitConfObj2 = controllerObj.addProcUnit(datatype='Voltage', inputId=procUnitConfObj0.getId())
86		#
87		# opObj12 = procUnitConfObj2.addOperation(name='CohInt', optype='other')
88		# opObj12.addParameter(name='n', value='2', format='int')
89		# opObj12.addParameter(name='overlapping', value='1', format='int')
90		#
91		# procUnitConfObj3 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj2.getId())
92		# procUnitConfObj3.addParameter(name='nFFTPoints', value='32', format='int')
93		#
94		# opObj11 = procUnitConfObj3.addOperation(name='SpectraPlot', optype='other')
95		# opObj11.addParameter(name='idfigure', value='2', format='int')
96		# opObj11.addParameter(name='wintitle', value='SpectraPlot1', format='str')
97		# opObj11.addParameter(name='zmin', value='40', format='int')
98		# opObj11.addParameter(name='zmax', value='90', format='int')
99		# opObj11.addParameter(name='showprofile', value='1', format='int')
100
101		# opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
102		# opObj11.addParameter(name='idfigure', value='10', format='int')
103		# opObj11.addParameter(name='wintitle', value='RTI', format='str')
104		## opObj11.addParameter(name='xmin', value='21', format='float')
105		## opObj11.addParameter(name='xmax', value='22', format='float')
106		# opObj11.addParameter(name='zmin', value='40', format='int')
107		# opObj11.addParameter(name='zmax', value='90', format='int')
108		# opObj11.addParameter(name='showprofile', value='1', format='int')
109		# opObj11.addParameter(name='timerange', value=str(60), format='int')
110
111		# opObj10 = procUnitConfObj1.addOperation(name='selectChannels')
112		# opObj10.addParameter(name='channelList', value='0,2,4,6', format='intlist')
113		#
114		# opObj12 = procUnitConfObj1.addOperation(name='IncohInt', optype='other')
115		# opObj12.addParameter(name='n', value='2', format='int')
116		#
117		# opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
118		# opObj11.addParameter(name='idfigure', value='2', format='int')
119		# opObj11.addParameter(name='wintitle', value='SpectraPlot10', format='str')
120		# opObj11.addParameter(name='zmin', value='70', format='int')
121		# opObj11.addParameter(name='zmax', value='90', format='int')
122		#
123		# opObj10 = procUnitConfObj1.addOperation(name='selectChannels')
124		# opObj10.addParameter(name='channelList', value='2,6', format='intlist')
125		#
126		# opObj12 = procUnitConfObj1.addOperation(name='IncohInt', optype='other')
127		# opObj12.addParameter(name='n', value='2', format='int')
128		#
129		# opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
130		# opObj11.addParameter(name='idfigure', value='3', format='int')
131		# opObj11.addParameter(name='wintitle', value='SpectraPlot10', format='str')
132		# opObj11.addParameter(name='zmin', value='70', format='int')
133		# opObj11.addParameter(name='zmax', value='90', format='int')
134
135
136		# opObj12 = procUnitConfObj1.addOperation(name='decoder')
137		# opObj12.addParameter(name='ncode', value='2', format='int')
138		# opObj12.addParameter(name='nbauds', value='8', format='int')
139		# opObj12.addParameter(name='code0', value='001110011', format='int')
140		# opObj12.addParameter(name='code1', value='001110011', format='int')
141
142
143
144		# procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj1.getId())
145		#
146		# opObj21 = procUnitConfObj2.addOperation(name='IncohInt', optype='other')
147		# opObj21.addParameter(name='n', value='2', format='int')
148		#
149		# opObj11 = procUnitConfObj2.addOperation(name='SpectraPlot', optype='other')
150		# opObj11.addParameter(name='idfigure', value='4', format='int')
151		# opObj11.addParameter(name='wintitle', value='SpectraPlot OBJ 2', format='str')
152		# opObj11.addParameter(name='zmin', value='70', format='int')
153		# opObj11.addParameter(name='zmax', value='90', format='int')
154	97
155	98	print "Escribiendo el archivo XML"
156	99

schainpy/model/io/jrodataIO.py removed 0 -9

NO CONTENT: file was removed

schainpy/model/proc/jroprocessing.py removed 0 -6

NO CONTENT: file was removed

schainpy/model/utils/jroutils.py removed 0 -1

NO CONTENT: file was removed

schainpy/test/EWDriftsApp.py removed 0 -117

NO CONTENT: file was removed

schainpy/test/FaradayApp.py removed 0 -62

NO CONTENT: file was removed

schainpy/test/FaradayPw.py removed 0 -151

NO CONTENT: file was removed

schainpy/test/FitsExp.py removed 0 -45

NO CONTENT: file was removed

schainpy/test/ImagingApp.py removed 0 -83

NO CONTENT: file was removed

schainpy/test/Imaging_EwDrifts.py removed 0 -187

NO CONTENT: file was removed

schainpy/test/Imaging_Plots.py removed 0 -79

NO CONTENT: file was removed

schainpy/test/Imaging_Proc.py removed 0 -71

NO CONTENT: file was removed

schainpy/test/Imaging_Proc_Driver4.py removed 0 -170

NO CONTENT: file was removed

schainpy/test/MeteorApp.py removed 0 -121

NO CONTENT: file was removed

schainpy/test/Meteor_JARS.py removed 0 -93

NO CONTENT: file was removed

schainpy/test/Meteor_Jasmet.py removed 0 -111

NO CONTENT: file was removed

schainpy/test/Meteor_Jasmet_Plot.py removed 0 -72

NO CONTENT: file was removed

schainpy/test/Meteor_Jasmet_Proc.py removed 0 -64

NO CONTENT: file was removed

schainpy/test/SunExperiment.py removed 0 -87

NO CONTENT: file was removed

schainpy/test/hydra.py removed 0 -127

NO CONTENT: file was removed

schainpy/test/isrproc.py removed 0 -114

NO CONTENT: file was removed

schainpy/test/mst_eej_isr.py removed 0 -258

NO CONTENT: file was removed

schainpy/test/mstisr.py removed 0 -186

NO CONTENT: file was removed

schainpy/test/testMeteors.py removed 0 -166

NO CONTENT: file was removed

schainpy/test/testProcData.py removed 0 -151

NO CONTENT: file was removed

schainpy/test/testcode.py removed 0 -146

NO CONTENT: file was removed

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages