schain Commit - r1251:6d7191c1eaa6 · Jicamarca Repository

Cleaning IO modules (base, voltaje and spectra)

jespinoza -

r1251:6d7191c1eaa6

parent child

Context file:

r1251:6d7191c1eaa6

Collapse all files

schainpy/model/__init__.py 0 -6

@@ -1,9 +1,3
1	#from schainpy.model.data.jrodata import *
2	# from schainpy.model.io.jrodataIO import *
3	# from schainpy.model.proc.jroprocessing import *
4	# from schainpy.model.graphics.jroplot import *
5	# from schainpy.model.utils.jroutils import *
6	# from schainpy.serializer import *
7		1
8	from .graphics import *	2	from .graphics import *
9	from .data import *	3	from .data import *

schainpy/model/graphics/jroplot_base.py +1 -1

                     self.localtime = kwargs.pop('localtime', True)
                     self.show = kwargs.get('show', True)
                     self.save = kwargs.get('save', False)
-                    self.save_period = kwargs.get('save_period', 2)
+                    self.save_period = kwargs.get('save_period', 1)
                     self.ftp = kwargs.get('ftp', False)
                     self.colormap = kwargs.get('colormap', self.colormap)
                     self.colormap_coh = kwargs.get('colormap_coh', 'jet')

schainpy/model/io/jroIO_base.py +386 -652

This diff has been collapsed as it changes many lines, (1038 lines changed) Show them Hide them
	@@ -1,8 +1,8
	1	'''	1	"""
	2	Created on Jul 2, 2014	2	Created on Jul 2, 2014
	3		3
	4	@author: roj-idl71	4	@author: roj-idl71
	5	'''	5	"""
	6	import os	6	import os
	7	import sys	7	import sys
	8	import glob	8	import glob
	@@ -12,7 +12,6 import fnmatch
	12	import inspect	12	import inspect
	13	import time	13	import time
	14	import datetime	14	import datetime
	15	import traceback
	16	import zmq	15	import zmq
	17		16
	18	from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader	17	from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
	@@ -21,6 +20,17 from schainpy.utils import log
	21	import schainpy.admin	20	import schainpy.admin
	22		21
	23	LOCALTIME = True	22	LOCALTIME = True
			23	DT_DIRECTIVES = {
			24	'%Y': 4,
			25	'%y': 2,
			26	'%m': 2,
			27	'%d': 2,
			28	'%j': 3,
			29	'%H': 2,
			30	'%M': 2,
			31	'%S': 2,
			32	'%f': 6
			33	}
	24		34
	25		35
	26	def isNumber(cad):	36	def isNumber(cad):
	@@ -362,70 +372,6 def getlastFileFromPath(path, ext):
	362	return None	372	return None
	363		373
	364		374
	365	def checkForRealPath(path, foldercounter, year, doy, set, ext):
	366	"""
	367	Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
	368	Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
	369	el path exacto de un determinado file.
	370
	371	Example :
	372	nombre correcto del file es .../.../D2009307/P2009307367.ext
	373
	374	Entonces la funcion prueba con las siguientes combinaciones
	375	.../.../y2009307367.ext
	376	.../.../Y2009307367.ext
	377	.../.../x2009307/y2009307367.ext
	378	.../.../x2009307/Y2009307367.ext
	379	.../.../X2009307/y2009307367.ext
	380	.../.../X2009307/Y2009307367.ext
	381	siendo para este caso, la ultima combinacion de letras, identica al file buscado
	382
	383	Return:
	384	Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
	385	caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
	386	para el filename
	387	"""
	388	fullfilename = None
	389	find_flag = False
	390	filename = None
	391
	392	prefixDirList = [None, 'd', 'D']
	393	if ext.lower() == ".r": # voltage
	394	prefixFileList = ['d', 'D']
	395	elif ext.lower() == ".pdata": # spectra
	396	prefixFileList = ['p', 'P']
	397	else:
	398	return None, filename
	399
	400	# barrido por las combinaciones posibles
	401	for prefixDir in prefixDirList:
	402	thispath = path
	403	if prefixDir != None:
	404	# formo el nombre del directorio xYYYYDDD (x=d o x=D)
	405	if foldercounter == 0:
	406	thispath = os.path.join(path, "%s%04d%03d" %
	407	(prefixDir, year, doy))
	408	else:
	409	thispath = os.path.join(path, "%s%04d%03d_%02d" % (
	410	prefixDir, year, doy, foldercounter))
	411	for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
	412	# formo el nombre del file xYYYYDDDSSS.ext
	413	filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
	414	fullfilename = os.path.join(
	415	thispath, filename) # formo el path completo
	416
	417	if os.path.exists(fullfilename): # verifico que exista
	418	find_flag = True
	419	break
	420	if find_flag:
	421	break
	422
	423	if not(find_flag):
	424	return None, filename
	425
	426	return fullfilename, filename
	427
	428
	429	def isRadarFolder(folder):	375	def isRadarFolder(folder):
	430	try:	376	try:
	431	year = int(folder[1:5])	377	year = int(folder[1:5])
	@@ -469,85 +415,73 def getDateFromRadarFolder(folder):
	469	thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)	415	thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
	470	return thisDate	416	return thisDate
	471		417
			418	def parse_format(s, fmt):
			419
			420	for i in range(fmt.count('%')):
			421	x = fmt.index('%')
			422	d = DT_DIRECTIVES[fmt[x:x+2]]
			423	fmt = fmt.replace(fmt[x:x+2], s[x:x+d])
			424	return fmt
	472		425
	473	class JRODataIO:	426	class Reader(object):
	474		427
	475	c = 3E8	428	c = 3E8
	476
	477	isConfig = False	429	isConfig = False
	478
	479	basicHeaderObj = None
	480
	481	systemHeaderObj = None
	482
	483	radarControllerHeaderObj = None
	484
	485	processingHeaderObj = None
	486
	487	dtype = None	430	dtype = None
	488
	489	pathList = []	431	pathList = []
	490
	491	filenameList = []	432	filenameList = []
	492		433	datetimeList = []
	493	filename = None	434	filename = None
	494
	495	ext = None	435	ext = None
	496
	497	flagIsNewFile = 1	436	flagIsNewFile = 1
	498
	499	flagDiscontinuousBlock = 0	437	flagDiscontinuousBlock = 0
	500
	501	flagIsNewBlock = 0	438	flagIsNewBlock = 0
	502		439	flagNoMoreFiles = 0
	503	fp = None	440	fp = None
	504
	505	firstHeaderSize = 0	441	firstHeaderSize = 0
	506
	507	basicHeaderSize = 24	442	basicHeaderSize = 24
	508
	509	versionFile = 1103	443	versionFile = 1103
	510
	511	fileSize = None	444	fileSize = None
	512
	513	# ippSeconds = None
	514
	515	fileSizeByHeader = None	445	fileSizeByHeader = None
	516		446	fileIndex = -1
	517	fileIndex = None
	518
	519	profileIndex = None	447	profileIndex = None
	520		448	blockIndex = 0
	521	blockIndex = None	449	nTotalBlocks = 0
	522
	523	nTotalBlocks = None
	524
	525	maxTimeStep = 30	450	maxTimeStep = 30
	526
	527	lastUTTime = None	451	lastUTTime = None
	528
	529	datablock = None	452	datablock = None
	530
	531	dataOut = None	453	dataOut = None
	532
	533	blocksize = None
	534
	535	getByBlock = False	454	getByBlock = False
	536		455	path = None
	537	def __init__(self):	456	startDate = None
	538		457	endDate = None
	539	raise NotImplementedError	458	startTime = datetime.time(0, 0, 0)
			459	endTime = datetime.time(23, 59, 59)
			460	set = None
			461	expLabel = ""
			462	online = False
			463	delay = 60
			464	nTries = 3 # quantity tries
			465	nFiles = 3 # number of files for searching
			466	walk = True
			467	getblock = False
			468	nTxs = 1
			469	realtime = False
			470	blocksize = 0
			471	blocktime = None
			472	warnings = True
			473	verbose = True
			474	server = None
			475	format = None
			476	oneDDict = None
			477	twoDDict = None
			478	independentParam = None
			479	filefmt = None
			480	folderfmt = None
	540		481
	541	def run(self):	482	def run(self):
	542		483
	543	raise NotImplementedError	484	raise NotImplementedError
	544
	545	def getDtypeWidth(self):
	546
	547	dtype_index = get_dtype_index(self.dtype)
	548	dtype_width = get_dtype_width(dtype_index)
	549
	550	return dtype_width
	551		485
	552	def getAllowedArgs(self):	486	def getAllowedArgs(self):
	553	if hasattr(self, '__attrs__'):	487	if hasattr(self, '__attrs__'):
	@@ -555,321 +489,169 class JRODataIO:
	555	else:	489	else:
	556	return inspect.getargspec(self.run).args	490	return inspect.getargspec(self.run).args
	557		491
			492	def set_kwargs(self, **kwargs):
	558		493
	559	class JRODataReader(JRODataIO):	494	for key, value in kwargs.items():
	560		495	setattr(self, key, value)
	561	online = 0	496
	562		497	def find_folders(self, path, startDate, endDate, folderfmt, last=False):
	563	realtime = 0
	564
	565	nReadBlocks = 0
	566
	567	delay = 10 # number of seconds waiting a new file
	568
	569	nTries = 3 # quantity tries
	570
	571	nFiles = 3 # number of files for searching
	572
	573	path = None
	574
	575	foldercounter = 0
	576
	577	flagNoMoreFiles = 0
	578
	579	datetimeList = []
	580
	581	__isFirstTimeOnline = 1
	582
	583	__printInfo = True
	584		498
	585	profileIndex = None	499	folders = [x for f in path.split(',')
	586		500	for x in os.listdir(f) if os.path.isdir(os.path.join(f, x))]
	587	nTxs = 1	501	folders.sort()
	588		502
	589	txIndex = None	503	if last:
			504	folders = [folders[-1]]
	590		505
	591	# Added--------------------	506	for folder in folders:
	592		507	try:
	593	selBlocksize = None	508	dt = datetime.datetime.strptime(parse_format(folder, folderfmt), folderfmt).date()
	594		509	if dt >= startDate and dt <= endDate:
	595	selBlocktime = None	510	yield os.path.join(path, folder)
	596		511	else:
	597	def __init__(self):	512	log.log('Skiping folder {}'.format(folder), self.name)
	598	"""	513	except Exception as e:
	599	This class is used to find data files	514	log.log('Skiping folder {}'.format(folder), self.name)
	600		515	continue
	601	~~Example:~~	516	return
	602	reader = JRODataReader()	517
	603	fileList = reader.findDataFiles()	518	def find_files(self, folders, ext, filefmt, startDate=None, endDate=None,
			519	expLabel='', last=False):
			520
			521	for path in folders:
			522	files = glob.glob1(path, '*{}'.format(ext))
			523	files.sort()
			524	if last:
			525	if files:
			526	fo = files[-1]
			527	try:
			528	dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
			529	yield os.path.join(path, expLabel, fo)
			530	except Exception as e:
			531	pass
			532	return None
			533	else:
			534	return None
	604		535
	605	"""	536	for fo in files:
	606	pass	537	try:
			538	dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
			539	if dt >= startDate and dt <= endDate:
			540	yield os.path.join(path, expLabel, fo)
			541	else:
			542	log.log('Skiping file {}'.format(fo), self.name)
			543	except Exception as e:
			544	log.log('Skiping file {}'.format(fo), self.name)
			545	continue
	607		546
	608	def createObjByDefault(self):	547	def searchFilesOffLine(self, path, startDate, endDate,
	609	"""	548	expLabel, ext, walk,
			549	filefmt, folderfmt):
			550	"""Search files in offline mode for the given arguments
	610		551
			552	Return:
			553	Generator of files
	611	"""	554	"""
	612	raise NotImplementedError
	613
	614	def getBlockDimension(self):
	615
	616	raise NotImplementedError
	617		555
	618	def searchFilesOffLine(self,	556	if walk:
	619	path,	557	folders = self.find_folders(
	620	startDate=~~None~~,	558	path, startDate, endDate, folderfmt)
	621	endDate=None,
	622	startTime=datetime.time(0, 0, 0),
	623	endTime=datetime.time(23, 59, 59),
	624	set=None,
	625	expLabel='',
	626	ext='.r',
	627	cursor=None,
	628	skip=None,
	629	walk=True):
	630
	631	self.filenameList = []
	632	self.datetimeList = []
	633
	634	pathList = []
	635
	636	dateList, pathList = self.findDatafiles(
	637	path, startDate, endDate, expLabel, ext, walk, include_path=True)
	638
	639	if dateList == []:
	640	return [], []
	641
	642	if len(dateList) > 1:
	643	print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)))
	644	else:	559	else:
	645	print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]))	560	folders = path.split(',')
	646
	647	filenameList = []
	648	datetimeList = []
	649		561
	650	for thisPath in pathList:	562	return self.find_files(
	651		563	folders, ext, filefmt, startDate, endDate, expLabel)
	652	fileList = glob.glob1(thisPath, "*%s" % ext)	564
	653	fileList.sort()	565	def searchFilesOnLine(self, path, startDate, endDate,
	654		566	expLabel, ext, walk,
	655	for file in fileList:	567	filefmt, folderfmt):
	656		568	"""Search for the last file of the last folder
	657	filename = os.path.join(thisPath, file)	569
	658		570	Arguments:
	659	if not isFileInDateRange(filename, startDate, endDate):	571	path : carpeta donde estan contenidos los files que contiene data
	660	continue	572	expLabel : Nombre del subexperimento (subfolder)
	661		573	ext : extension de los files
	662	thisDatetime = isFileInTimeRange(
	663	filename, startDate, endDate, startTime, endTime)
	664
	665	if not(thisDatetime):
	666	continue
	667
	668	filenameList.append(filename)
	669	datetimeList.append(thisDatetime)
	670
	671	if cursor is not None and skip is not None:
	672	filenameList = filenameList[cursor * skip:cursor * skip + skip]
	673	datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
	674
	675	if not(filenameList):
	676	print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path))
	677	return [], []
	678
	679	print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime))
	680
	681	# for i in range(len(filenameList)):
	682	# print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
	683
	684	self.filenameList = filenameList
	685	self.datetimeList = datetimeList
	686
	687	return pathList, filenameList
	688
	689	def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
	690	"""
	691	Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
	692	devuelve el archivo encontrado ademas de otros datos.
	693
	694	Input:
	695	path : carpeta donde estan contenidos los files que contiene data
	696
	697	expLabel : Nombre del subexperimento (subfolder)
	698
	699	ext : extension de los files
	700
	701	walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)	574	walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
	702		575
	703	Return:	576	Return:
	704	directory : eL directorio donde esta el file encontrado	577	generator with the full path of last filename
	705	filename : el ultimo file de una determinada carpeta
	706	year : el anho
	707	doy : el numero de dia del anho
	708	set : el set del archivo
	709
	710
	711	"""	578	"""
	712	if not os.path.isdir(path):	579
	713	return None, None, None, None, None, None	580	if walk:
	714		581	folders = self.find_folders(
	715	dirList = []	582	path, startDate, endDate, folderfmt, last=True)
	716
	717	if not walk:
	718	fullpath = path
	719	foldercounter = 0
	720	else:	583	else:
	721	# Filtra solo los directorios	584	folders = path.split(',')
	722	for thisPath in os.listdir(path):	585
	723	if not os.path.isdir(os.path.join(path, thisPath)):	586	return self.find_files(
	724	continue	587	folders, ext, filefmt, startDate, endDate, expLabel, last=True)
	725	if not isRadarFolder(thisPath):
	726	continue
	727
	728	dirList.append(thisPath)
	729
	730	if not(dirList):
	731	return None, None, None, None, None, None
	732
	733	dirList = sorted(dirList, key=str.lower)
	734		588
	735	doypath = dirList[-1]	589	def setNextFile(self):
	736	foldercounter = int(doypath.split('_')[1]) if len(	590	"""Set the next file to be readed open it and parse de file header"""
	737	doypath.split('_')) > 1 else 0
	738	fullpath = os.path.join(path, doypath, expLabel)
	739		591
	740	print("[Reading] %s folder was found: " % (fullpath))	592	if self.fp != None:
			593	self.fp.close()
	741		594
	742	if set == None:	595	if self.online:
	743	filename = getlastFileFromPath(fullpath, ext)	596	newFile = self.setNextFileOnline()
	744	else:	597	else:
	745	filename = getFileFromSet(fullpath, ext, set)	598	newFile = self.setNextFileOffline()
	746		599
	747	if not(~~filenam~~e):	600	if not(newFile):
	748	return None, None, None, None, None, None	601	if self.online:
	749		602	raise schainpy.admin.SchainError('Time to wait for new files reach')
	750	print("[Reading] %s file was found" % (filename))	603	else:
	751		604	if self.fileIndex == -1:
	752	if not(self.__verifyFile(os.path.join(fullpath, filename))):	605	raise schainpy.admin.SchainWarning('No files found in the given path')
	753	return None, None, None, None, None, None	606	else:
	754		607	raise schainpy.admin.SchainWarning('No more files to read')
	755	year = int(filename[1:5])	608
	756	doy = int(filename[5:8])	609	if not(self.verifyFile(self.filename)):
	757	set = int(filename[8:11])	610	self.setNextFile()
	758		611
	759	return fullpath, foldercounter, filename, year, doy, set	612	log.log('Opening file: %s' % self.filename, self.name)
	760
	761	def __setNextFileOffline(self):
	762
	763	idFile = self.fileIndex
	764
	765	while (True):
	766	idFile += 1
	767	if not(idFile < len(self.filenameList)):
	768	self.flagNoMoreFiles = 1
	769	return 0
	770
	771	filename = self.filenameList[idFile]
	772
	773	if not(self.__verifyFile(filename)):
	774	continue
	775
	776	fileSize = os.path.getsize(filename)
	777	fp = open(filename, 'rb')
	778	break
	779
	780	self.flagIsNewFile = 1
	781	self.fileIndex = idFile
	782	self.filename = filename
	783	self.fileSize = fileSize
	784	self.fp = fp
	785
	786	# print "[Reading] Setting the file: %s"%self.filename
	787		613
	788	return 1	614	self.readFirstHeader()
			615	self.nReadBlocks = 0
	789		616
	790	def __setNextFileOnline(self):	617	def setNextFileOnline(self):
	791	"""	618	"""Check for the next file to be readed in online mode.
	792	Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
	793	no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
	794	siguientes.
	795		619
	796	~~Affected~~:	620	Set:
	797	self.flagIsNewFile
	798	self.filename	621	self.filename
	799	self.fileSize
	800	self.fp	622	self.fp
	801	self.set	623	self.filesize
	802	self.flagNoMoreFiles	624
	803
	804	Return:	625	Return:
	805	0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado	626	boolean
	806	1 : si el file fue abierto con exito y esta listo a ser leido
	807		627
	808	Excepciones:
	809	Si un determinado file no puede ser abierto
	810	"""	628	"""
	811	nFiles = 0	629	nextFile = True
	812	~~fileOk_flag~~ = False	630	nextDay = False
	813	firstTime_flag = True
	814
	815	self.set += 1
	816		631
	817	if self.set > 999:	632	for nFiles in range(self.nFiles+1):
	818	self.set = 0	633	for nTries in range(self.nTries):
	819	self.foldercounter += 1	634	fullfilename, filename = self.checkForRealPath(nextFile, nextDay)
	820		635	if fullfilename is not None:
	821	# busca el 1er file disponible
	822	fullfilename, filename = checkForRealPath(
	823	self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
	824	if fullfilename:
	825	if self.__verifyFile(fullfilename, False):
	826	fileOk_flag = True
	827
	828	# si no encuentra un file entonces espera y vuelve a buscar
	829	if not(fileOk_flag):
	830	# busco en los siguientes self.nFiles+1 files posibles
	831	for nFiles in range(self.nFiles + 1):
	832
	833	if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
	834	tries = self.nTries
	835	else:
	836	tries = 1 # si no es la 1era vez entonces solo lo hace una vez
	837
	838	for nTries in range(tries):
	839	if firstTime_flag:
	840	log.warning(
	841	"Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1),
	842	self.name)
	843	time.sleep(self.delay)
	844	else:
	845	log.warning(
	846	"Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext),
	847	self.name)
	848
	849	fullfilename, filename = checkForRealPath(
	850	self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
	851	if fullfilename:
	852	if self.__verifyFile(fullfilename):
	853	fileOk_flag = True
	854	break
	855
	856	if fileOk_flag:
	857	break	636	break
	858
	859	firstTime_flag = False
	860
	861	log.warning(	637	log.warning(
	862	'Skipping the file {} due to this file doesn\'t exist'.format(filename),	638	"Waiting %0.2f sec for the next file: \"%s\" , try %02d ..." % (self.delay, filename, nTries + 1),
	863	self.name)	639	self.name)
	864	self.~~set~~ += 1	640	time.sleep(self.delay)
			641	nextFile = False
			642	continue
			643
			644	if fullfilename:
			645	break
			646
			647	self.nTries = 1
			648	nextFile = True
	865		649
	866	# si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta	650	if nFiles == (self.nFiles - 1):
	867	if nFiles == (self.nFiles - 1):	651	log.log('Trying with next day...', self.name)
	868	self.set = 0	652	nextDay = True
	869	self.doy += 1
	870	self.foldercounter = 0
	871		653
	872	if f~~ileOk_flag~~:	654	if fullfilename:
	873	self.fileSize = os.path.getsize(fullfilename)	655	self.fileSize = os.path.getsize(fullfilename)
	874	self.filename = fullfilename	656	self.filename = fullfilename
	875	self.flagIsNewFile = 1	657	self.flagIsNewFile = 1
	@@ -877,36 +659,123 class JRODataReader(JRODataIO):
	877	self.fp.close()	659	self.fp.close()
	878	self.fp = open(fullfilename, 'rb')	660	self.fp = open(fullfilename, 'rb')
	879	self.flagNoMoreFiles = 0	661	self.flagNoMoreFiles = 0
	880	else:	662	self.fileIndex += 1
	881	raise schainpy.admin.SchainError('Time for waiting new files reach')	663	return 1
	882	self.fileSize = 0	664	else:
	883	self.filename = None	665	return 0
	884	self.flagIsNewFile = 0	666
	885	self.fp = None	667	def setNextFileOffline(self):
			668	"""Open the next file to be readed in offline mode"""
			669
			670	try:
			671	filename = next(self.filenameList)
			672	self.fileIndex +=1
			673	except StopIteration:
	886	self.flagNoMoreFiles = 1	674	self.flagNoMoreFiles = 1
			675	return 0
	887		676
	888	return fileOk_flag	677	self.filename = filename
			678	self.fileSize = os.path.getsize(filename)
			679	self.fp = open(filename, 'rb')
			680	self.flagIsNewFile = 1
	889		681
	890	def setNextFile(self):	682	return 1
	891	if self.fp != None:	683
	892	self.fp.close()	684	def verifyFile(self, filename):
			685	"""Check for a valid file
			686
			687	Arguments:
			688	filename -- full path filename
			689
			690	Return:
			691	boolean
			692	"""
	893		693
	894	if self.online:	694	return True
	895	newFile = self.__setNextFileOnline()
	896	else:
	897	newFile = self.__setNextFileOffline()
	898		695
	899	if not(newFile):	696	def checkForRealPath(self, nextFile, nextDay):
	900	raise schainpy.admin.SchainWarning('No more files to read')	697	"""Check if the next file to be readed exists"""
	901
	902		698
	903	if self.verbose:	699	raise NotImplementedError
	904	print('[Reading] Setting the file: %s' % self.filename)	700
			701	def readFirstHeader(self):
			702	"""Parse the file header"""
	905		703
	906	self.__readFirstHeader()	704	pass
	907	self.nReadBlocks = 0	705
	908	return 1	706	class JRODataReader(Reader):
			707
			708	utc = 0
			709	nReadBlocks = 0
			710	foldercounter = 0
			711	firstHeaderSize = 0
			712	basicHeaderSize = 24
			713	__isFirstTimeOnline = 1
			714	__printInfo = True
			715	filefmt = "%Y%j**"
			716	folderfmt = "*%Y%j"
	909		717
			718	def getDtypeWidth(self):
			719
			720	dtype_index = get_dtype_index(self.dtype)
			721	dtype_width = get_dtype_width(dtype_index)
			722
			723	return dtype_width
			724
			725	def checkForRealPath(self, nextFile, nextDay):
			726	"""Check if the next file to be readed exists.
			727
			728	Example :
			729	nombre correcto del file es .../.../D2009307/P2009307367.ext
			730
			731	Entonces la funcion prueba con las siguientes combinaciones
			732	.../.../y2009307367.ext
			733	.../.../Y2009307367.ext
			734	.../.../x2009307/y2009307367.ext
			735	.../.../x2009307/Y2009307367.ext
			736	.../.../X2009307/y2009307367.ext
			737	.../.../X2009307/Y2009307367.ext
			738	siendo para este caso, la ultima combinacion de letras, identica al file buscado
			739
			740	Return:
			741	str -- fullpath of the file
			742	"""
			743
			744
			745	if nextFile:
			746	self.set += 1
			747	if nextDay:
			748	self.set = 0
			749	self.doy += 1
			750	foldercounter = 0
			751	prefixDirList = [None, 'd', 'D']
			752	if self.ext.lower() == ".r": # voltage
			753	prefixFileList = ['d', 'D']
			754	elif self.ext.lower() == ".pdata": # spectra
			755	prefixFileList = ['p', 'P']
			756
			757	# barrido por las combinaciones posibles
			758	for prefixDir in prefixDirList:
			759	thispath = self.path
			760	if prefixDir != None:
			761	# formo el nombre del directorio xYYYYDDD (x=d o x=D)
			762	if foldercounter == 0:
			763	thispath = os.path.join(self.path, "%s%04d%03d" %
			764	(prefixDir, self.year, self.doy))
			765	else:
			766	thispath = os.path.join(self.path, "%s%04d%03d_%02d" % (
			767	prefixDir, self.year, self.doy, foldercounter))
			768	for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
			769	# formo el nombre del file xYYYYDDDSSS.ext
			770	filename = "%s%04d%03d%03d%s" % (prefixFile, self.year, self.doy, self.set, self.ext)
			771	fullfilename = os.path.join(
			772	thispath, filename)
			773
			774	if os.path.exists(fullfilename):
			775	return fullfilename, filename
			776
			777	return None, filename
			778
	910	def __waitNewBlock(self):	779	def __waitNewBlock(self):
	911	"""	780	"""
	912	Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.	781	Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
	@@ -972,48 +841,12 class JRODataReader(JRODataIO):
	972		841
	973	return 0	842	return 0
	974		843
	975	def __jumpToLastBlock(self):
	976
	977	if not(self.__isFirstTimeOnline):
	978	return
	979
	980	csize = self.fileSize - self.fp.tell()
	981	blocksize = self.processingHeaderObj.blockSize
	982
	983	# salta el primer bloque de datos
	984	if csize > self.processingHeaderObj.blockSize:
	985	self.fp.seek(self.fp.tell() + blocksize)
	986	else:
	987	return
	988
	989	csize = self.fileSize - self.fp.tell()
	990	neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
	991	while True:
	992
	993	if self.fp.tell() < self.fileSize:
	994	self.fp.seek(self.fp.tell() + neededsize)
	995	else:
	996	self.fp.seek(self.fp.tell() - neededsize)
	997	break
	998
	999	# csize = self.fileSize - self.fp.tell()
	1000	# neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
	1001	# factor = int(csize/neededsize)
	1002	# if factor > 0:
	1003	# self.fp.seek(self.fp.tell() + factor*neededsize)
	1004
	1005	self.flagIsNewFile = 0
	1006	self.__isFirstTimeOnline = 0
	1007
	1008	def __setNewBlock(self):	844	def __setNewBlock(self):
	1009	# if self.server is None:
	1010	if self.fp == None:
	1011	return 0
	1012		845
	1013	# if self.o~~nli~~ne:	846	if self.fp == None:
	1014	# self.__jumpToLastBlock()	847	return 0
	1015		848
	1016	if self.flagIsNewFile:	849	if self.flagIsNewFile:
	1017	self.lastUTTime = self.basicHeaderObj.utc	850	self.lastUTTime = self.basicHeaderObj.utc
	1018	return 1	851	return 1
	1019		852
	@@ -1023,21 +856,19 class JRODataReader(JRODataIO):
	1023	return 0	856	return 0
	1024	else:	857	else:
	1025	return 1	858	return 1
	1026	# if self.server is None:	859
	1027	currentSize = self.fileSize - self.fp.tell()	860	currentSize = self.fileSize - self.fp.tell()
	1028	neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize	861	neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
			862
	1029	if (currentSize >= neededSize):	863	if (currentSize >= neededSize):
	1030	self.basicHeaderObj.read(self.fp)	864	self.basicHeaderObj.read(self.fp)
	1031	self.lastUTTime = self.basicHeaderObj.utc	865	self.lastUTTime = self.basicHeaderObj.utc
	1032	return 1	866	return 1
	1033	~~# else:~~	867
	1034	# self.basicHeaderObj.read(self.zHeader)
	1035	# self.lastUTTime = self.basicHeaderObj.utc
	1036	# return 1
	1037	if self.__waitNewBlock():	868	if self.__waitNewBlock():
	1038	self.lastUTTime = self.basicHeaderObj.utc	869	self.lastUTTime = self.basicHeaderObj.utc
	1039	return 1	870	return 1
	1040	# if self.server is None:	871
	1041	if not(self.setNextFile()):	872	if not(self.setNextFile()):
	1042	return 0	873	return 0
	1043		874
	@@ -1053,10 +884,8 class JRODataReader(JRODataIO):
	1053		884
	1054	def readNextBlock(self):	885	def readNextBlock(self):
	1055		886
	1056	# Skip block out of startTime and endTime
	1057	while True:	887	while True:
	1058	if ~~not~~(self.__setNewBlock()):	888	self.__setNewBlock()
	1059	raise schainpy.admin.SchainWarning('No more files to read')
	1060		889
	1061	if not(self.readBlock()):	890	if not(self.readBlock()):
	1062	return 0	891	return 0
	@@ -1076,13 +905,12 class JRODataReader(JRODataIO):
	1076	self.dataOut.datatime.ctime()))	905	self.dataOut.datatime.ctime()))
	1077	return 1	906	return 1
	1078		907
	1079	def __readFirstHeader(self):	908	def readFirstHeader(self):
	1080		909
	1081	self.basicHeaderObj.read(self.fp)	910	self.basicHeaderObj.read(self.fp)
	1082	self.systemHeaderObj.read(self.fp)	911	self.systemHeaderObj.read(self.fp)
	1083	self.radarControllerHeaderObj.read(self.fp)	912	self.radarControllerHeaderObj.read(self.fp)
	1084	self.processingHeaderObj.read(self.fp)	913	self.processingHeaderObj.read(self.fp)
	1085
	1086	self.firstHeaderSize = self.basicHeaderObj.size	914	self.firstHeaderSize = self.basicHeaderObj.size
	1087		915
	1088	datatype = int(numpy.log2((self.processingHeaderObj.processFlags &	916	datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
	@@ -1111,7 +939,7 class JRODataReader(JRODataIO):
	1111	# self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)	939	# self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
	1112	self.getBlockDimension()	940	self.getBlockDimension()
	1113		941
	1114	def __verifyFile(self, filename, msgFlag=True):	942	def verifyFile(self, filename, msgFlag=True):
	1115		943
	1116	msg = None	944	msg = None
	1117		945
	@@ -1268,48 +1096,17 class JRODataReader(JRODataIO):
	1268		1096
	1269	return dateList	1097	return dateList
	1270		1098
	1271	def setup(self,	1099	def setup(self, **kwargs):
	1272	path=None,	1100
	1273	startDate=None,	1101	self.set_kwargs(**kwargs)
	1274	endDate=None,	1102	if not self.ext.startswith('.'):
	1275	startTime=datetime.time(0, 0, 0),	1103	self.ext = '.{}'.format(self.ext)
	1276	endTime=datetime.time(23, 59, 59),
	1277	set=None,
	1278	expLabel="",
	1279	ext=None,
	1280	online=False,
	1281	delay=60,
	1282	walk=True,
	1283	getblock=False,
	1284	nTxs=1,
	1285	realtime=False,
	1286	blocksize=None,
	1287	blocktime=None,
	1288	skip=None,
	1289	cursor=None,
	1290	warnings=True,
	1291	verbose=True,
	1292	server=None,
	1293	format=None,
	1294	oneDDict=None,
	1295	twoDDict=None,
	1296	independentParam=None):
	1297		1104
	1298	self.online = online	1105	if self.server is not None:
	1299	self.realtime = realtime	1106	if 'tcp://' in self.server:
	1300	self.delay = delay
	1301	self.getByBlock = getblock
	1302	self.nTxs = nTxs
	1303	self.startTime = startTime
	1304	self.endTime = endTime
	1305	self.endDate = endDate
	1306	self.startDate = startDate
	1307
	1308	if server is not None:
	1309	if 'tcp://' in server:
	1310	address = server	1107	address = server
	1311	else:	1108	else:
	1312	address = 'ipc:///tmp/%s' % server	1109	address = 'ipc:///tmp/%s' % self.server
	1313	self.server = address	1110	self.server = address
	1314	self.context = zmq.Context()	1111	self.context = zmq.Context()
	1315	self.receiver = self.context.socket(zmq.PULL)	1112	self.receiver = self.context.socket(zmq.PULL)
	@@ -1318,84 +1115,46 class JRODataReader(JRODataIO):
	1318	print('[Starting] ReceiverData from {}'.format(self.server))	1115	print('[Starting] ReceiverData from {}'.format(self.server))
	1319	else:	1116	else:
	1320	self.server = None	1117	self.server = None
	1321	if path == None:	1118	if self.path == None:
	1322	raise ValueError("[Reading] The path is not valid")	1119	raise ValueError("[Reading] The path is not valid")
	1323		1120
	1324	if ~~ext~~ == None:	1121	if self.online:
	1325	ext = self.ext	1122	log.log("[Reading] Searching files in online mode...", self.name)
	1326
	1327	if online:
	1328	print("[Reading] Searching files in online mode...")
	1329		1123
	1330	for nTries in range(self.nTries):	1124	for nTries in range(self.nTries):
	1331	fullpath, ~~foldercounter~~, ~~file~~, ~~year~~, ~~doy~~, ~~set~~ = self.__searchFilesOnLine(	1125	fullpath = self.searchFilesOnLine(self.path, self.startDate,
	1332	~~path~~=~~path~~, ~~expLabel~~=expLabel, ~~ext~~=ext, ~~walk~~=walk, ~~set~~=~~set~~)	1126	self.endDate, self.expLabel, self.ext, self.walk,
	1333		1127	self.filefmt, self.folderfmt)
			1128
			1129	try:
			1130	fullpath = next(fullpath)
			1131	except:
			1132	fullpath = None
			1133
	1334	if fullpath:	1134	if fullpath:
	1335	break	1135	break
	1336		1136
	1337	print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1))	1137	log.warning(
			1138	'Waiting {} sec for a valid file in {}: try {} ...'.format(
			1139	self.delay, self.path, nTries + 1),
			1140	self.name)
	1338	time.sleep(self.delay)	1141	time.sleep(self.delay)
	1339		1142
	1340	if not(fullpath):	1143	if not(fullpath):
	1341	raise schainpy.admin.SchainError(~~'There isn~~\'~~t any valid file in {}'~~.~~format~~(~~path~~))	1144	raise schainpy.admin.SchainError(
	1342	return	1145	'There isn\'t any valid file in {}'.format(self.path))
	1343
	1344	self.year = year
	1345	self.doy = doy
	1346	self.set = set - 1
	1347	self.path = path
	1348	self.foldercounter = foldercounter
	1349	last_set = None
	1350	else:
	1351	print("[Reading] Searching files in offline mode ...")
	1352	pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
	1353	startTime=startTime, endTime=endTime,
	1354	set=set, expLabel=expLabel, ext=ext,
	1355	walk=walk, cursor=cursor,
	1356	skip=skip)
	1357
	1358	if not(pathList):
	1359	self.fileIndex = -1
	1360	self.pathList = []
	1361	self.filenameList = []
	1362	return
	1363
	1364	self.fileIndex = -1
	1365	self.pathList = pathList
	1366	self.filenameList = filenameList
	1367	file_name = os.path.basename(filenameList[-1])
	1368	basename, ext = os.path.splitext(file_name)
	1369	last_set = int(basename[-3:])
	1370		1146
			1147	pathname, filename = os.path.split(fullpath)
			1148	self.year = int(filename[1:5])
			1149	self.doy = int(filename[5:8])
			1150	self.set = int(filename[8:11]) - 1
			1151	else:
			1152	log.log("Searching files in {}".format(self.path), self.name)
			1153	self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
			1154	self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
	1371		1155
	1372	~~ext~~ = ~~ext~~.lower()	1156	self.setNextFile()
	1373	self.ext = ext
	1374
	1375	# Added-----------------
	1376	self.selBlocksize = blocksize
	1377	self.selBlocktime = blocktime
	1378
	1379	# Verbose-----------
	1380	self.verbose = verbose
	1381	self.warnings = warnings
	1382
	1383	if not(self.setNextFile()):
	1384	if (startDate != None) and (endDate != None):
	1385	print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
	1386	elif startDate != None:
	1387	print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
	1388	else:
	1389	print("[Reading] No files")
	1390
	1391	self.fileIndex = -1
	1392	self.pathList = []
	1393	self.filenameList = []
	1394	return
	1395		1157
	1396	if last_set != None:
	1397	self.dataOut.last_block = last_set * \
	1398	self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
	1399	return	1158	return
	1400		1159
	1401	def getBasicHeader(self):	1160	def getBasicHeader(self):
	@@ -1415,7 +1174,7 class JRODataReader(JRODataIO):
	1415		1174
	1416	self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs	1175	self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
	1417		1176
	1418	# self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs	1177	# self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
	1419		1178
	1420	def getFirstHeader(self):	1179	def getFirstHeader(self):
	1421		1180
	@@ -1448,10 +1207,10 class JRODataReader(JRODataIO):
	1448	def printNumberOfBlock(self):	1207	def printNumberOfBlock(self):
	1449	'SPAM!'	1208	'SPAM!'
	1450		1209
	1451	# if self.flagIsNewBlock:	1210	# if self.flagIsNewBlock:
	1452	# print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,	1211	# print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
	1453	# self.processingHeaderObj.dataBlocksPerFile,	1212	# self.processingHeaderObj.dataBlocksPerFile,
	1454	# self.dataOut.datatime.ctime())	1213	# self.dataOut.datatime.ctime())
	1455		1214
	1456	def printInfo(self):	1215	def printInfo(self):
	1457		1216
	@@ -1465,85 +1224,57 class JRODataReader(JRODataIO):
	1465		1224
	1466	self.__printInfo = False	1225	self.__printInfo = False
	1467		1226
	1468	def run(self,	1227	def run(self, **kwargs):
	1469	path=None,	1228	"""
	1470	startDate=None,	1229
	1471	endDate=None,	1230	Arguments:
	1472	startTime=datetime.time(0, 0, 0),	1231	path :
	1473	endTime=datetime.time(23, 59, 59),	1232	startDate :
	1474	~~set~~=~~None~~,	1233	endDate :
	1475	expLabel="",	1234	startTime :
	1476	ext=~~None~~,	1235	endTime :
	1477	online=False,	1236	set :
	1478	~~delay~~=60,	1237	expLabel :
	1479	walk=True,	1238	ext :
	1480	getblock=False,	1239	online :
	1481	nTxs=1,	1240	delay :
	1482	realtime=False,	1241	walk :
	1483	~~blocksize~~=~~None~~,	1242	getblock :
	1484	blocktime=None,	1243	nTxs :
	1485	skip=None,	1244	realtime :
	1486	~~cursor~~=~~None~~,	1245	blocksize :
	1487	warnings=True,	1246	blocktime :
	1488	server=None,	1247	skip :
	1489	~~verbose~~=~~True~~,	1248	cursor :
	1490	format=None,	1249	warnings :
	1491	oneDDict=None,	1250	server :
	1492	twoDDict=None,	1251	verbose :
	1493	independentParam=None, **kwargs):	1252	format :
			1253	oneDDict :
			1254	twoDDict :
			1255	independentParam :
			1256	"""
	1494		1257
	1495	if not(self.isConfig):	1258	if not(self.isConfig):
	1496	self.setup(~~path~~=~~path~~,	1259	self.setup(**kwargs)
	1497	startDate=startDate,
	1498	endDate=endDate,
	1499	startTime=startTime,
	1500	endTime=endTime,
	1501	set=set,
	1502	expLabel=expLabel,
	1503	ext=ext,
	1504	online=online,
	1505	delay=delay,
	1506	walk=walk,
	1507	getblock=getblock,
	1508	nTxs=nTxs,
	1509	realtime=realtime,
	1510	blocksize=blocksize,
	1511	blocktime=blocktime,
	1512	skip=skip,
	1513	cursor=cursor,
	1514	warnings=warnings,
	1515	server=server,
	1516	verbose=verbose,
	1517	format=format,
	1518	oneDDict=oneDDict,
	1519	twoDDict=twoDDict,
	1520	independentParam=independentParam)
	1521	self.isConfig = True	1260	self.isConfig = True
	1522	if server is None:	1261	if self.server is None:
	1523	self.getData()	1262	self.getData()
	1524	else:	1263	else:
	1525	self.getFromServer()	1264	self.getFromServer()
	1526		1265
	1527		1266
	1528	class JRODataWriter(~~JRODataIO~~):	1267	class JRODataWriter(Reader):
	1529		1268
	1530	"""	1269	"""
	1531	Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura	1270	Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
	1532	de los datos siempre se realiza por bloques.	1271	de los datos siempre se realiza por bloques.
	1533	"""	1272	"""
	1534		1273
	1535	blockIndex = 0
	1536
	1537	path = None
	1538
	1539	setFile = None	1274	setFile = None
	1540
	1541	profilesPerBlock = None	1275	profilesPerBlock = None
	1542
	1543	blocksPerFile = None	1276	blocksPerFile = None
	1544
	1545	nWriteBlocks = 0	1277	nWriteBlocks = 0
	1546
	1547	fileDate = None	1278	fileDate = None
	1548		1279
	1549	def __init__(self, dataOut=None):	1280	def __init__(self, dataOut=None):
	@@ -1561,6 +1292,13 class JRODataWriter(JRODataIO):
	1561	def putData(self):	1292	def putData(self):
	1562	raise NotImplementedError	1293	raise NotImplementedError
	1563		1294
			1295	def getDtypeWidth(self):
			1296
			1297	dtype_index = get_dtype_index(self.dtype)
			1298	dtype_width = get_dtype_width(dtype_index)
			1299
			1300	return dtype_width
			1301
	1564	def getProcessFlags(self):	1302	def getProcessFlags(self):
	1565		1303
	1566	processFlags = 0	1304	processFlags = 0
	@@ -1688,8 +1426,7 class JRODataWriter(JRODataIO):
	1688	return 1	1426	return 1
	1689		1427
	1690	def setNextFile(self):	1428	def setNextFile(self):
	1691	"""	1429	"""Determina el siguiente file que sera escrito
	1692	Determina el siguiente file que sera escrito
	1693		1430
	1694	Affected:	1431	Affected:
	1695	self.filename	1432	self.filename
	@@ -1708,6 +1445,9 class JRODataWriter(JRODataIO):
	1708	if self.fp != None:	1445	if self.fp != None:
	1709	self.fp.close()	1446	self.fp.close()
	1710		1447
			1448	if not os.path.exists(path):
			1449	os.mkdir(path)
			1450
	1711	timeTuple = time.localtime(self.dataOut.utctime)	1451	timeTuple = time.localtime(self.dataOut.utctime)
	1712	subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)	1452	subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
	1713		1453
	@@ -1748,15 +1488,12 class JRODataWriter(JRODataIO):
	1748	fp = open(filename, 'wb')	1488	fp = open(filename, 'wb')
	1749		1489
	1750	self.blockIndex = 0	1490	self.blockIndex = 0
	1751
	1752	# guardando atributos
	1753	self.filename = filename	1491	self.filename = filename
	1754	self.subfolder = subfolder	1492	self.subfolder = subfolder
	1755	self.fp = fp	1493	self.fp = fp
	1756	self.setFile = setFile	1494	self.setFile = setFile
	1757	self.flagIsNewFile = 1	1495	self.flagIsNewFile = 1
	1758	self.fileDate = self.dataOut.datatime.date()	1496	self.fileDate = self.dataOut.datatime.date()
	1759
	1760	self.setFirstHeader()	1497	self.setFirstHeader()
	1761		1498
	1762	print('[Writing] Opening file: %s' % self.filename)	1499	print('[Writing] Opening file: %s' % self.filename)
	@@ -1799,12 +1536,9 class JRODataWriter(JRODataIO):
	1799	self.setFile = set - 1	1536	self.setFile = set - 1
	1800		1537
	1801	self.blocksPerFile = blocksPerFile	1538	self.blocksPerFile = blocksPerFile
	1802
	1803	self.profilesPerBlock = profilesPerBlock	1539	self.profilesPerBlock = profilesPerBlock
	1804
	1805	self.dataOut = dataOut	1540	self.dataOut = dataOut
	1806	self.fileDate = self.dataOut.datatime.date()	1541	self.fileDate = self.dataOut.datatime.date()
	1807	# By default
	1808	self.dtype = self.dataOut.dtype	1542	self.dtype = self.dataOut.dtype
	1809		1543
	1810	if datatype is not None:	1544	if datatype is not None:
	@@ -1828,4 +1562,4 class JRODataWriter(JRODataIO):
	1828		1562
	1829	self.dataOut = dataOut	1563	self.dataOut = dataOut
	1830	self.putData()	1564	self.putData()
	1831	return self.dataOut No newline at end of file	1565	return self.dataOut

schainpy/model/io/jroIO_spectra.py +15 -163

@@ -53,24 +53,6 class SpectraReader(JRODataReader, ProcessingUnit):
53		53
54	"""	54	"""
55		55
56	pts2read_SelfSpectra = 0
57
58	pts2read_CrossSpectra = 0
59
60	pts2read_DCchannels = 0
61
62	ext = ".pdata"
63
64	optchar = "P"
65
66	dataOut = None
67
68	nRdChannels = None
69
70	nRdPairs = None
71
72	rdPairList = []
73
74	def __init__(self):#, **kwargs):	56	def __init__(self):#, **kwargs):
75	"""	57	"""
76	Inicializador de la clase SpectraReader para la lectura de datos de espectros.	58	Inicializador de la clase SpectraReader para la lectura de datos de espectros.
@@ -89,89 +71,24 class SpectraReader(JRODataReader, ProcessingUnit):
89	Return : None	71	Return : None
90	"""	72	"""
91		73
92	#Eliminar de la base la herencia	74	ProcessingUnit.__init__(self)
93	ProcessingUnit.__init__(self)#, **kwargs)
94
95		75
96	self.pts2read_SelfSpectra = 0	76	self.pts2read_SelfSpectra = 0
97
98	self.pts2read_CrossSpectra = 0	77	self.pts2read_CrossSpectra = 0
99		78	self.pts2read_DCchannels = 0
100	self.pts2read_DCchannels = 0
101
102	self.datablock = None
103
104	self.utc = None
105
106	self.ext = ".pdata"	79	self.ext = ".pdata"
107
108	self.optchar = "P"	80	self.optchar = "P"
109
110	self.basicHeaderObj = BasicHeader(LOCALTIME)	81	self.basicHeaderObj = BasicHeader(LOCALTIME)
111
112	self.systemHeaderObj = SystemHeader()	82	self.systemHeaderObj = SystemHeader()
113
114	self.radarControllerHeaderObj = RadarControllerHeader()	83	self.radarControllerHeaderObj = RadarControllerHeader()
115
116	self.processingHeaderObj = ProcessingHeader()	84	self.processingHeaderObj = ProcessingHeader()
117
118	self.online = 0
119
120	self.fp = None
121
122	self.idFile = None
123
124	self.dtype = None
125
126	self.fileSizeByHeader = None
127
128	self.filenameList = []
129
130	self.filename = None
131
132	self.fileSize = None
133
134	self.firstHeaderSize = 0
135
136	self.basicHeaderSize = 24
137
138	self.pathList = []
139
140	self.lastUTTime = 0	85	self.lastUTTime = 0
141
142	self.maxTimeStep = 30	86	self.maxTimeStep = 30
143		87	self.dataOut = Spectra()
144	self.~~flagNoMoreFiles~~ = 0	88	self.profileIndex = 1
145		89	self.nRdChannels = None
146	self.set = 0	90	self.nRdPairs = None
147		91	self.rdPairList = []
148	self.path = None
149
150	self.delay = 60 #seconds
151
152	self.nTries = 3 #quantity tries
153
154	self.nFiles = 3 #number of files for searching
155
156	self.nReadBlocks = 0
157
158	self.flagIsNewFile = 1
159
160	self.__isFirstTimeOnline = 1
161
162
163	self.flagDiscontinuousBlock = 0
164
165	self.flagIsNewBlock = 0
166
167	self.nTotalBlocks = 0
168
169	self.blocksize = 0
170
171	self.dataOut = self.createObjByDefault()
172
173	self.profileIndex = 1 #Always
174
175		92
176	def createObjByDefault(self):	93	def createObjByDefault(self):
177		94
@@ -224,9 +141,6 class SpectraReader(JRODataReader, ProcessingUnit):
224	self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)	141	self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
225	self.blocksize += self.pts2read_DCchannels	142	self.blocksize += self.pts2read_DCchannels
226		143
227	# self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
228
229
230	def readBlock(self):	144	def readBlock(self):
231	"""	145	"""
232	Lee el bloque de datos desde la posicion actual del puntero del archivo	146	Lee el bloque de datos desde la posicion actual del puntero del archivo
@@ -248,7 +162,7 class SpectraReader(JRODataReader, ProcessingUnit):
248	Exceptions:	162	Exceptions:
249	Si un bloque leido no es un bloque valido	163	Si un bloque leido no es un bloque valido
250	"""	164	"""
251	blockOk_flag = False	165
252	fpointer = self.fp.tell()	166	fpointer = self.fp.tell()
253		167
254	spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )	168	spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
@@ -262,7 +176,6 class SpectraReader(JRODataReader, ProcessingUnit):
262	dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )	176	dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
263	dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D	177	dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
264		178
265
266	if not self.processingHeaderObj.shif_fft:	179	if not self.processingHeaderObj.shif_fft:
267	#desplaza a la derecha en el eje 2 determinadas posiciones	180	#desplaza a la derecha en el eje 2 determinadas posiciones
268	shift = int(self.processingHeaderObj.profilesPerBlock/2)	181	shift = int(self.processingHeaderObj.profilesPerBlock/2)
@@ -298,39 +211,19 class SpectraReader(JRODataReader, ProcessingUnit):
298	def getFirstHeader(self):	211	def getFirstHeader(self):
299		212
300	self.getBasicHeader()	213	self.getBasicHeader()
301
302	self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()	214	self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
303
304	self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()	215	self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
305
306	# self.dataOut.ippSeconds = self.ippSeconds
307
308	# self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.processingHeaderObj.profilesPerBlock
309
310	self.dataOut.dtype = self.dtype	216	self.dataOut.dtype = self.dtype
311
312	# self.dataOut.nPairs = self.nPairs
313
314	self.dataOut.pairsList = self.rdPairList	217	self.dataOut.pairsList = self.rdPairList
315
316	self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock	218	self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
317
318	self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock	219	self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
319
320	self.dataOut.nCohInt = self.processingHeaderObj.nCohInt	220	self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
321
322	self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt	221	self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
323
324	xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight	222	xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
325
326	self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)	223	self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
327
328	self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))	224	self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
329
330	self.dataOut.flagShiftFFT = True #Data is always shifted	225	self.dataOut.flagShiftFFT = True #Data is always shifted
331
332	self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada	226	self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
333
334	self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data esta sin flip	227	self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data esta sin flip
335		228
336	def getData(self):	229	def getData(self):
@@ -347,7 +240,6 class SpectraReader(JRODataReader, ProcessingUnit):
347		240
348	Affected:	241	Affected:
349	self.dataOut	242	self.dataOut
350
351	self.flagDiscontinuousBlock	243	self.flagDiscontinuousBlock
352	self.flagIsNewBlock	244	self.flagIsNewBlock
353	"""	245	"""
@@ -372,20 +264,16 class SpectraReader(JRODataReader, ProcessingUnit):
372	return 0	264	return 0
373		265
374	self.getBasicHeader()	266	self.getBasicHeader()
375
376	self.getFirstHeader()	267	self.getFirstHeader()
377
378	self.dataOut.data_spc = self.data_spc	268	self.dataOut.data_spc = self.data_spc
379
380	self.dataOut.data_cspc = self.data_cspc	269	self.dataOut.data_cspc = self.data_cspc
381
382	self.dataOut.data_dc = self.data_dc	270	self.dataOut.data_dc = self.data_dc
383
384	self.dataOut.flagNoData = False	271	self.dataOut.flagNoData = False
385
386	self.dataOut.realtime = self.online	272	self.dataOut.realtime = self.online
387		273
388	return self.dataOut.data_spc	274	return self.dataOut.data_spc
		275
		276
389	@MPDecorator	277	@MPDecorator
390	class SpectraWriter(JRODataWriter, Operation):	278	class SpectraWriter(JRODataWriter, Operation):
391		279
@@ -394,22 +282,6 class SpectraWriter(JRODataWriter, Operation):
394	de los datos siempre se realiza por bloques.	282	de los datos siempre se realiza por bloques.
395	"""	283	"""
396		284
397	ext = ".pdata"
398
399	optchar = "P"
400
401	shape_spc_Buffer = None
402
403	shape_cspc_Buffer = None
404
405	shape_dc_Buffer = None
406
407	data_spc = None
408
409	data_cspc = None
410
411	data_dc = None
412
413	def __init__(self):	285	def __init__(self):
414	"""	286	"""
415	Inicializador de la clase SpectraWriter para la escritura de datos de espectros.	287	Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
@@ -426,41 +298,21 class SpectraWriter(JRODataWriter, Operation):
426		298
427	Operation.__init__(self)	299	Operation.__init__(self)
428		300
429	self.~~nTotalBlocks~~ = 0	301	self.ext = ".pdata"
430		302	self.optchar = "P"
		303	self.shape_spc_Buffer = None
		304	self.shape_cspc_Buffer = None
		305	self.shape_dc_Buffer = None
431	self.data_spc = None	306	self.data_spc = None
432
433	self.data_cspc = None	307	self.data_cspc = None
434
435	self.data_dc = None	308	self.data_dc = None
436
437	self.fp = None
438
439	self.flagIsNewFile = 1
440
441	self.nTotalBlocks = 0
442
443	self.flagIsNewBlock = 0
444
445	self.setFile = None	309	self.setFile = None
446
447	self.dtype = None
448
449	self.path = None
450
451	self.noMoreFiles = 0	310	self.noMoreFiles = 0
452
453	self.filename = None
454
455	self.basicHeaderObj = BasicHeader(LOCALTIME)	311	self.basicHeaderObj = BasicHeader(LOCALTIME)
456
457	self.systemHeaderObj = SystemHeader()	312	self.systemHeaderObj = SystemHeader()
458
459	self.radarControllerHeaderObj = RadarControllerHeader()	313	self.radarControllerHeaderObj = RadarControllerHeader()
460
461	self.processingHeaderObj = ProcessingHeader()	314	self.processingHeaderObj = ProcessingHeader()
462		315
463
464	def hasAllDataInBuffer(self):	316	def hasAllDataInBuffer(self):
465	return 1	317	return 1
466		318

schainpy/model/io/jroIO_voltage.py +8 -82

@@ -57,12 +57,7 class VoltageReader(JRODataReader, ProcessingUnit):
57		57
58	"""	58	"""
59		59
60	ext = ".r"	60	def __init__(self):
61
62	optchar = "D"
63	dataOut = None
64
65	def __init__(self):#, **kwargs):
66	"""	61	"""
67	Inicializador de la clase VoltageReader para la lectura de datos de voltage.	62	Inicializador de la clase VoltageReader para la lectura de datos de voltage.
68		63
@@ -81,89 +76,19 class VoltageReader(JRODataReader, ProcessingUnit):
81	None	76	None
82	"""	77	"""
83		78
84	ProcessingUnit.__init__(self)~~#, **kwargs)~~	79	ProcessingUnit.__init__(self)
85		80
86	self.isConfig = False
87
88	self.datablock = None
89
90	self.utc = 0
91
92	self.ext = ".r"	81	self.ext = ".r"
93
94	self.optchar = "D"	82	self.optchar = "D"
95
96	self.basicHeaderObj = BasicHeader(LOCALTIME)	83	self.basicHeaderObj = BasicHeader(LOCALTIME)
97
98	self.systemHeaderObj = SystemHeader()	84	self.systemHeaderObj = SystemHeader()
99
100	self.radarControllerHeaderObj = RadarControllerHeader()	85	self.radarControllerHeaderObj = RadarControllerHeader()
101
102	self.processingHeaderObj = ProcessingHeader()	86	self.processingHeaderObj = ProcessingHeader()
103
104	self.online = 0
105
106	self.fp = None
107
108	self.idFile = None
109
110	self.dtype = None
111
112	self.fileSizeByHeader = None
113
114	self.filenameList = []
115
116	self.filename = None
117
118	self.fileSize = None
119
120	self.firstHeaderSize = 0
121
122	self.basicHeaderSize = 24
123
124	self.pathList = []
125
126	self.filenameList = []
127
128	self.lastUTTime = 0	87	self.lastUTTime = 0
129		88	self.profileIndex = 2**32 - 1
130	self.maxTimeStep = 30	89	self.dataOut = Voltage()
131		90	self.selBlocksize = None
132	self.flagNoMoreFiles = 0	91	self.selBlocktime = None
133
134	self.set = 0
135
136	self.path = None
137
138	self.profileIndex = 2**32 - 1
139
140	self.delay = 3 # seconds
141
142	self.nTries = 3 # quantity tries
143
144	self.nFiles = 3 # number of files for searching
145
146	self.nReadBlocks = 0
147
148	self.flagIsNewFile = 1
149
150	self.__isFirstTimeOnline = 1
151
152	# self.ippSeconds = 0
153
154	self.flagDiscontinuousBlock = 0
155
156	self.flagIsNewBlock = 0
157
158	self.nTotalBlocks = 0
159
160	self.blocksize = 0
161
162	self.dataOut = self.createObjByDefault()
163
164	self.nTxs = 1
165
166	self.txIndex = 0
167		92
168	def createObjByDefault(self):	93	def createObjByDefault(self):
169		94
@@ -545,6 +470,7 class VoltageReader(JRODataReader, ProcessingUnit):
545		470
546	return self.dataOut.data	471	return self.dataOut.data
547		472
		473
548	@MPDecorator	474	@MPDecorator
549	class VoltageWriter(JRODataWriter, Operation):	475	class VoltageWriter(JRODataWriter, Operation):
550	"""	476	"""

schainpy/model/io/jroIO_example.py removed 0 -175

NO CONTENT: file was removed

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages