schain Commit - r589:341989823444 · Jicamarca Repository

-USRP Reader was added to Signal Chain GUI...

Miguel Valdez -

r589:341989823444

parent child

Context file:

r589:341989823444

Collapse all files

The requested changes are too big and content was truncated. Show full diff

schainpy/gui/viewcontroller/parametersModel.py created 644 +87 0

@@ -0,0 +1,87
	1	"""
	2	Classes to save parameters from Windows.
	3
	4	-Project window
	5	-Voltage window
	6	-Spectra window
	7	-SpectraHeis window
	8	-Correlation window
	9
	10	"""
	11
	12	class ProjectParms():
	13
	14	parmsOk = False
	15	project_name = None
	16	datatype = None
	17	ext = None
	18	dpath = None
	19	startDate = None
	20	endDate = None
	21	startTime = None
	22	endTime = None
	23	online = None
	24	delay = None
	25	walk = None
	26	expLabel = None
	27	set = None
	28	ippKm = None
	29
	30	def __init__(self):
	31
	32	self.parmsOk = True
	33	self.expLabel = ''
	34	self.set = None
	35	self.ippKm = None
	36	self.walk = None
	37	self.delay = None
	38
	39	def getDatatypeIndex(self):
	40
	41	indexDatatype = None
	42
	43	if self.datatype.lower() == 'voltage':
	44	indexDatatype = 0
	45	if self.datatype.lower() == 'spectra':
	46	indexDatatype = 1
	47	if self.datatype.lower() == 'fits':
	48	indexDatatype = 2
	49	if self.datatype.lower() == 'usrp':
	50	indexDatatype = 3
	51
	52	return indexDatatype
	53
	54	def getExt(self):
	55
	56	ext = None
	57
	58	if self.datatype.lower() == 'voltage':
	59	ext = '.r'
	60	if self.datatype.lower() == 'spectra':
	61	ext = '.pdata'
	62	if self.datatype.lower() == 'fits':
	63	ext = '.fits'
	64	if self.datatype.lower() == 'usrp':
	65	ext = '.hdf5'
	66
	67	return ext
	68
	69	def set(self, project_name, datatype, ext, dpath, online,
	70	startDate=None, endDate=None, startTime=None, endTime=None,
	71	delay=None, walk=None, set=None, ippKm=None, parmsOk=True):
	72
	73	project_name = project_name
	74	datatype = datatype
	75	ext = ext
	76	dpath = dpath
	77	startDate = startDate
	78	endDate = endDate
	79	startTime = startTime
	80	endTime = endTime
	81	online = online
	82	delay = delay
	83	walk = walk
	84	set = set
	85	ippKm = ippKm
	86
	87	self.parmsOk = parmsOk No newline at end of file

schainpy/gui/viewcontroller/propertiesViewModel.py created 644 +211 0

@@ -0,0 +1,211
	1	# -- coding: utf-8 --
	2	"""
	3	This module contains every model class to create, modify and show a property tree on a GUI.
	4	"""
	5
	6	from PyQt4 import QtCore
	7	import itertools
	8
	9	HORIZONTAL_HEADERS = ("Property","Value " )
	10
	11	HORIZONTAL = ("RAMA :",)
	12
	13	class PropertyBuffer():
	14
	15	def __init__(self):
	16
	17	self.clear()
	18
	19	def clear(self):
	20
	21	self.headerList = []
	22	self.parmList = []
	23	self.valueList = []
	24
	25	def append(self, header, parm, value):
	26
	27	self.headerList.append(header)
	28	self.parmList.append(parm)
	29	self.valueList.append(value)
	30
	31	return
	32
	33	def get(self):
	34
	35	return self.headerList, self.parmList, self.valueList
	36
	37	def getPropertyModel(self):
	38
	39	propertiesModel = TreeModel()
	40	propertiesModel.showProperties(self.headerList, self.parmList, self.valueList)
	41
	42	return propertiesModel
	43
	44
	45	class TreeModel(QtCore.QAbstractItemModel):
	46	'''
	47	a model to display a few names, ordered by encabezado
	48
	49	'''
	50	def __init__(self ,parent=None):
	51	super(TreeModel, self).__init__(parent)
	52	self.people = []
	53
	54	def initProjectView(self):
	55	"""
	56	Reemplazo del método showtree
	57	"""
	58	HORIZONTAL_HEADERS = ("Property","Value " )
	59	HORIZONTAL = ("RAMA :",)
	60	self.rootItem = TreeItem(None, "ALL", None)
	61	self.parents = {0 : self.rootItem}
	62	self.__setupModelData()
	63
	64	def initPUVoltageView(self):
	65	HORIZONTAL_HEADERS = ("Operation"," Parameter Value " )
	66	HORIZONTAL = ("RAMA :",)
	67	self.rootItem = TreeItem(None, "ALL", None)
	68	self.parents = {0 : self.rootItem}
	69	self.__setupModelData()
	70
	71	def showProperties(self,headerList, parmList, valueList):
	72	"""
	73	set2Obje
	74	"""
	75	for header, parameter, value in itertools.izip(headerList, parmList, valueList):
	76	person = person_class(header, parameter, value)
	77	self.people.append(person)
	78
	79	self.rootItem = TreeItem(None, "ALL", None)
	80	self.parents = {0 : self.rootItem}
	81	self.__setupModelData()
	82
	83	def columnCount(self, parent=None):
	84	if parent and parent.isValid():
	85	return parent.internalPointer().columnCount()
	86	else:
	87	return len(HORIZONTAL_HEADERS)
	88
	89	def data(self, index, role):
	90	if not index.isValid():
	91	return QtCore.QVariant()
	92
	93	item = index.internalPointer()
	94	if role == QtCore.Qt.DisplayRole:
	95	return item.data(index.column())
	96	if role == QtCore.Qt.UserRole:
	97	if item:
	98	return item.person
	99
	100	return QtCore.QVariant()
	101
	102	def index(self, row, column, parent):
	103	if not self.hasIndex(row, column, parent):
	104	return QtCore.QModelIndex()
	105
	106	if not parent.isValid():
	107	parentItem = self.rootItem
	108	else:
	109	parentItem = parent.internalPointer()
	110
	111	childItem = parentItem.child(row)
	112	if childItem:
	113	return self.createIndex(row, column, childItem)
	114	else:
	115	return QtCore.QModelIndex()
	116
	117	def parent(self, index):
	118	if not index.isValid():
	119	return QtCore.QModelIndex()
	120
	121	childItem = index.internalPointer()
	122	if not childItem:
	123	return QtCore.QModelIndex()
	124
	125	parentItem = childItem.parent()
	126
	127	if parentItem == self.rootItem:
	128	return QtCore.QModelIndex()
	129
	130	return self.createIndex(parentItem.row(), 0, parentItem)
	131
	132	def rowCount(self, parent=QtCore.QModelIndex()):
	133	if parent.column() > 0:
	134	return 0
	135	if not parent.isValid():
	136	p_Item = self.rootItem
	137	else:
	138	p_Item = parent.internalPointer()
	139	return p_Item.childCount()
	140
	141	def __setupModelData(self):
	142	for person in self.people:
	143	if person.value:
	144	encabezado = person.header
	145
	146	if not self.parents.has_key(encabezado):
	147	newparent = TreeItem(None, encabezado, self.rootItem)
	148	self.rootItem.appendChild(newparent)
	149
	150	self.parents[encabezado] = newparent
	151
	152	parentItem = self.parents[encabezado]
	153	newItem = TreeItem(person, "", parentItem)
	154	parentItem.appendChild(newItem)
	155
	156	class person_class(object):
	157	'''
	158	a trivial custom data object
	159	'''
	160	def __init__(self, header, parameter, value):
	161	self.header = header
	162	self.parameter = parameter
	163	self.value = value
	164
	165	def __repr__(self):
	166	return "PERSON - %s %s"% (self.parameter, self.header)
	167
	168	class TreeItem(object):
	169	'''
	170	a python object used to return row/column data, and keep note of
	171	it's parents and/or children
	172	'''
	173	def __init__(self, person, header, parentItem):
	174	self.person = person
	175	self.parentItem = parentItem
	176	self.header = header
	177	self.childItems = []
	178
	179	def appendChild(self, item):
	180	self.childItems.append(item)
	181
	182	def child(self, row):
	183	return self.childItems[row]
	184
	185	def childCount(self):
	186	return len(self.childItems)
	187
	188	def columnCount(self):
	189	return 2
	190
	191	def data(self, column):
	192	if self.person == None:
	193	if column == 0:
	194	return QtCore.QVariant(self.header)
	195	if column == 1:
	196	return QtCore.QVariant("")
	197	else:
	198	if column == 0:
	199	return QtCore.QVariant(self.person.parameter)
	200	if column == 1:
	201	return QtCore.QVariant(self.person.value)
	202	return QtCore.QVariant()
	203
	204	def parent(self):
	205	return self.parentItem
	206
	207	def row(self):
	208	if self.parentItem:
	209	return self.parentItem.childItems.index(self)
	210	return 0
	211	No newline at end of file

schainpy/controller.py +38 -11

             '''
             Created on September , 2012
             @author:
             '''
             from xml.etree.ElementTree import Element, SubElement
             from xml.etree import ElementTree as ET
             from xml.dom import minidom
             #import datetime
             from model import *
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             import ast
             def prettify(elem):
                 """Return a pretty-printed XML string for the Element.
                 """
                 rough_string = ET.tostring(elem, 'utf-8')
                 reparsed = minidom.parseString(rough_string)
                 return reparsed.toprettyxml(indent="  ")
             class ParameterConf():
                 id = None
                 name = None
                 value = None
                 format = None
                 __formated_value = None
                 ELEMENTNAME = 'Parameter'
                 def __init__(self):
                     self.format = 'str'
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getValue(self):
                     if self.__formated_value != None:
                         return self.__formated_value
                     value = self.value
                     if self.format == 'bool':
                         value = int(value)
                     if self.format == 'list':
                         strList = value.split(',')
                         self.__formated_value = strList
                         return self.__formated_value
                     if self.format == 'intlist':
                         """
                         Example:
                             value = (0,1,2)
                         """
                         value = value.replace('(', '')
                         value = value.replace(')', '')
                         value = value.replace('[', '')
                         value = value.replace(']', '')
                         strList = value.split(',')
                         intList = [int(x) for x in strList]
                         self.__formated_value = intList
                         return self.__formated_value
                     if self.format == 'floatlist':
                         """
                         Example:
                             value = (0.5, 1.4, 2.7)
                         """
                         value = value.replace('(', '')
                         value = value.replace(')', '')
                         value = value.replace('[', '')
                         value = value.replace(']', '')
                         strList = value.split(',')
                         floatList = [float(x) for x in strList]
                         self.__formated_value = floatList
                         return self.__formated_value
                     if self.format == 'date':
                         strList = value.split('/')
                         intList = [int(x) for x in strList]
                         date = datetime.date(intList[0], intList[1], intList[2])
                         self.__formated_value = date
                         return self.__formated_value
                     if self.format == 'time':
                         strList = value.split(':')
                         intList = [int(x) for x in strList]
                         time = datetime.time(intList[0], intList[1], intList[2])
                         self.__formated_value = time
                         return self.__formated_value
                     if self.format == 'pairslist':
                         """
                         Example:
                             value = (0,1),(1,2)
                         """
                         value = value.replace('(', '')
                         value = value.replace(')', '')
                         value = value.replace('[', '')
                         value = value.replace(']', '')
                         strList = value.split(',')
                         intList = [int(item) for item in strList]
                         pairList = []
                         for i in range(len(intList)/2):
                             pairList.append((intList[i*2], intList[i*2 + 1]))
                         self.__formated_value = pairList
                         return self.__formated_value
                     if self.format == 'multilist':
                         """
                         Example:
                             value = (0,1,2),(3,4,5)
                         """
                         multiList = ast.literal_eval(value)
                         self.__formated_value = multiList
                         return self.__formated_value
                     format_func = eval(self.format)
                     self.__formated_value = format_func(value)
                     return self.__formated_value
                 def setup(self, id, name, value, format='str'):
                     self.id = id
                     self.name = name
                     self.value = str(value)
                     self.format = str.lower(format)
                 def update(self, name, value, format='str'):
                     self.name = name
                     self.value = str(value)
                     self.format = format
                 def makeXml(self, opElement):
                     parmElement = SubElement(opElement, self.ELEMENTNAME)
                     parmElement.set('id', str(self.id))
                     parmElement.set('name', self.name)
                     parmElement.set('value', self.value)
                     parmElement.set('format', self.format)
                 def readXml(self, parmElement):
                     self.id = parmElement.get('id')
                     self.name = parmElement.get('name')
                     self.value = parmElement.get('value')
                     self.format = str.lower(parmElement.get('format'))
                     #Compatible with old signal chain version
                     if self.format == 'int' and self.name == 'idfigure':
                         self.name = 'id'
                 def printattr(self):
                     print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
             class OperationConf():
                 id = None
                 name = None
                 priority = None
                 type = None
                 parmConfObjList = []
                 ELEMENTNAME = 'Operation'
                 def __init__(self):
-                    self.id = 0
+                    self.id = '0'
                     self.name = None
                     self.priority = None
                     self.type = 'self'
                 def __getNewId(self):
                     return int(self.id)*10 + len(self.parmConfObjList) + 1
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getParameterObjList(self):
                     return self.parmConfObjList
                 def getParameterObj(self, parameterName):
                     for parmConfObj in self.parmConfObjList:
                         if parmConfObj.name != parameterName:
                             continue
                         return parmConfObj
                     return None
                 def getParameterObjfromValue(self,parameterValue):
                     for parmConfObj in self.parmConfObjList:
                         if parmConfObj.getValue() != parameterValue:
                             continue
                         return parmConfObj.getValue()
                     return None
                 def getParameterValue(self, parameterName):
                     parameterObj = self.getParameterObj(parameterName)
                     value = parameterObj.getValue()
                     return value
                 def setup(self, id, name, priority, type):
                     self.id = id
                     self.name = name
                     self.type = type
                     self.priority = priority
                     self.parmConfObjList = []
                 def removeParameters(self):
                     for obj in self.parmConfObjList:
                         del obj
                     self.parmConfObjList = []
                 def addParameter(self, name, value, format='str'):
                     id = self.__getNewId()
                     parmConfObj = ParameterConf()
                     parmConfObj.setup(id, name, value, format)
                     self.parmConfObjList.append(parmConfObj)
                     return parmConfObj
                 def changeParameter(self, name, value, format='str'):
                     parmConfObj = self.getParameterObj(name)
                     parmConfObj.update(name, value, format)
                     return parmConfObj
                 def makeXml(self, upElement):
                     opElement = SubElement(upElement, self.ELEMENTNAME)
                     opElement.set('id', str(self.id))
                     opElement.set('name', self.name)
                     opElement.set('type', self.type)
                     opElement.set('priority', str(self.priority))
                     for parmConfObj in self.parmConfObjList:
                         parmConfObj.makeXml(opElement)
                 def readXml(self, opElement):
                     self.id = opElement.get('id')
                     self.name = opElement.get('name')
                     self.type = opElement.get('type')
                     self.priority = opElement.get('priority')
                     #Compatible with old signal chain version
                     #Use of 'run' method instead 'init'
                     if self.type == 'self' and self.name == 'init':
                         self.name = 'run'
                     self.parmConfObjList = []
                     parmElementList = opElement.getiterator(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                         parmConfObj.readXml(parmElement)
                         #Compatible with old signal chain version
                         #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
                         if self.type != 'self' and self.name == 'Plot':
                             if parmConfObj.format == 'str' and parmConfObj.name == 'type':
                                 self.name = parmConfObj.value
                                 continue
                         self.parmConfObjList.append(parmConfObj)
                 def printattr(self):
                     print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
                                                                           self.id,
                                                                           self.name,
                                                                           self.type,
                                                                           self.priority)
                     for parmConfObj in self.parmConfObjList:
                         parmConfObj.printattr()
                 def createObject(self):
                     if self.type == 'self':
                         raise ValueError, "This operation type cannot be created"
                     if self.type == 'external' or self.type == 'other':
                         className = eval(self.name)
                         opObj = className()
                     return opObj
             class ProcUnitConf():
                 id = None
                 name = None
                 datatype = None
                 inputId = None
                 parentId = None
                 opConfObjList = []
                 procUnitObj = None
                 opObjList = []
                 ELEMENTNAME = 'ProcUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.opConfObjList = []
                     self.procUnitObj = None
                     self.opObjDict = {}
                 def __getPriority(self):
                     return len(self.opConfObjList)+1
                 def __getNewId(self):
                     return int(self.id)*10 + len(self.opConfObjList) + 1
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getId(self):
-                    return str(self.id)
+                    return self.id
                 def getInputId(self):
-                    return str(self.inputId)
+                    return self.inputId
                 def getOperationObjList(self):
                     return self.opConfObjList
                 def getOperationObj(self, name=None):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.name != name:
                             continue
                         return opConfObj
                     return None
                 def getOpObjfromParamValue(self,value=None):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
                             continue
                         return opConfObj
                     return None
                 def getProcUnitObj(self):
                     return self.procUnitObj
                 def setup(self, id, name, datatype, inputId, parentId=None):
                     self.id = id
                     self.name = name
                     self.datatype = datatype
                     self.inputId = inputId
                     self.parentId = parentId
                     self.opConfObjList = []
                     self.addOperation(name='run', optype='self')
                 def removeOperations(self):
                     for obj in self.opConfObjList:
                         del obj
                     self.opConfObjList = []
                     self.addOperation(name='run')
                 def addParameter(self, **kwargs):
                     '''
                     Add parameters to "run" operation
                     '''
                     opObj = self.opConfObjList[0]
                     opObj.addParameter(**kwargs)
                     return opObj
                 def addOperation(self, name, optype='self'):
                     id = self.__getNewId()
                     priority = self.__getPriority()
                     opConfObj = OperationConf()
                     opConfObj.setup(id, name=name, priority=priority, type=optype)
                     self.opConfObjList.append(opConfObj)
                     return opConfObj
                 def makeXml(self, procUnitElement):
                     upElement = SubElement(procUnitElement, self.ELEMENTNAME)
                     upElement.set('id', str(self.id))
                     upElement.set('name', self.name)
                     upElement.set('datatype', self.datatype)
                     upElement.set('inputId', str(self.inputId))
                     for opConfObj in self.opConfObjList:
                         opConfObj.makeXml(upElement)
                 def readXml(self, upElement):
                     self.id = upElement.get('id')
                     self.name = upElement.get('name')
                     self.datatype = upElement.get('datatype')
                     self.inputId = upElement.get('inputId')
+                    if self.inputId == 'None':
+                        self.inputId = '0'
                     self.opConfObjList = []
                     opElementList = upElement.getiterator(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                         opConfObj.readXml(opElement)
                         self.opConfObjList.append(opConfObj)
                 def printattr(self):
                     print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
                                                                          self.id,
                                                                          self.name,
                                                                          self.datatype,
                                                                          self.inputId)
                     for opConfObj in self.opConfObjList:
                         opConfObj.printattr()
                 def createObjects(self):
                     className = eval(self.name)
                     procUnitObj = className()
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type == 'self':
                             continue
                         opObj = opConfObj.createObject()
                         self.opObjDict[opConfObj.id] = opObj
                         procUnitObj.addOperation(opObj, opConfObj.id)
                     self.procUnitObj = procUnitObj
                     return procUnitObj
                 def run(self):
                     finalSts = False
                     for opConfObj in self.opConfObjList:
                         kwargs = {}
                         for parmConfObj in opConfObj.getParameterObjList():
                             if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
                                 continue
                             kwargs[parmConfObj.name] = parmConfObj.getValue()
                         #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
                         sts = self.procUnitObj.call(opType = opConfObj.type,
                                                     opName = opConfObj.name,
                                                     opId = opConfObj.id,
                                                      **kwargs)
                         finalSts = finalSts or sts
                     return finalSts
                 def close(self):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type == 'self':
                             continue
                         opObj = self.procUnitObj.getOperationObj(opConfObj.id)
                         opObj.close()
                     self.procUnitObj.close()
                     return
             class ReadUnitConf(ProcUnitConf):
                 path = None
                 startDate = None
                 endDate = None
                 startTime = None
                 endTime = None
                 ELEMENTNAME = 'ReadUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
-                    self.inputId = 0
+                    self.inputId = None
+                    self.parentId = None
                     self.opConfObjList = []
                     self.opObjList = []
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, **kwargs):
                     self.id = id
                     self.name = name
                     self.datatype = datatype
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
+                    self.inputId = '0'
+                    self.parentId = parentId
                     self.addRunOperation(**kwargs)
-                def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, **kwargs):
+                def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
+                    if name==None:
+                        if 'Reader' in datatype:
+                            name = datatype
+                        else:
+                            name = '%sReader' %(datatype)
+                    if datatype==None:
+                        datatype = name.replace('Reader','')
                     self.datatype = datatype
+                    self.name = name
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
+                    self.inputId = None
+                    self.parentId = parentId
                     self.updateRunOperation(**kwargs)
                 def addRunOperation(self, **kwargs):
                     opObj = self.addOperation(name = 'run', optype = 'self')
                     opObj.addParameter(name='datatype' , value=self.datatype, format='str')
                     opObj.addParameter(name='path'     , value=self.path, format='str')
                     opObj.addParameter(name='startDate' , value=self.startDate, format='date')
                     opObj.addParameter(name='endDate'   , value=self.endDate, format='date')
                     opObj.addParameter(name='startTime' , value=self.startTime, format='time')
                     opObj.addParameter(name='endTime'   , value=self.endTime, format='time')
                     for key, value in kwargs.items():
                         opObj.addParameter(name=key, value=value, format=type(value).__name__)
                     return opObj
                 def updateRunOperation(self, **kwargs):
                     opObj = self.getOperationObj(name = 'run')
                     opObj.removeParameters()
                     opObj.addParameter(name='datatype' , value=self.datatype, format='str')
                     opObj.addParameter(name='path'     , value=self.path, format='str')
                     opObj.addParameter(name='startDate' , value=self.startDate, format='date')
                     opObj.addParameter(name='endDate'   , value=self.endDate, format='date')
                     opObj.addParameter(name='startTime' , value=self.startTime, format='time')
                     opObj.addParameter(name='endTime'   , value=self.endTime, format='time')
                     for key, value in kwargs.items():
                         opObj.addParameter(name=key, value=value, format=type(value).__name__)
                     return opObj
             class Project():
                 id = None
                 name = None
                 description = None
             #    readUnitConfObjList = None
                 procUnitConfObjDict = None
                 ELEMENTNAME = 'Project'
                 def __init__(self, control=None, dataq=None):
                     self.id = None
                     self.name = None
                     self.description = None
                     self.procUnitConfObjDict = {}
                     #global data_q
                     #data_q = dataq
                     if control==None:
                         control = {}
                         control['stop'] = False
                         control['pause'] = False
                     self.control = control
                 def __getNewId(self):
                     id = int(self.id)*10 + len(self.procUnitConfObjDict) + 1
                     return str(id)
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getId(self):
                     return self.id
                 def setup(self, id, name, description):
                     self.id = id
                     self.name = name
                     self.description = description
                 def update(self, name, description):
                     self.name = name
                     self.description = description
                 def addReadUnit(self, datatype=None, name=None, **kwargs):
                     #Compatible with old signal chain version
                     if datatype==None and name==None:
                         raise ValueError, "datatype or name should be defined"
                     if name==None:
                         if 'Reader' in datatype:
                             name = datatype
                         else:
                             name = '%sReader' %(datatype)
                     if datatype==None:
                         datatype = name.replace('Reader','')
-                    id = self.__getNewId()
+                    idReadUnit = self.__getNewId()
                     readUnitConfObj = ReadUnitConf()
-                    readUnitConfObj.setup(id, name, datatype, parentId=self.id, **kwargs)
+                    readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
                     self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     return readUnitConfObj
-                def addProcUnit(self, inputId=0, datatype=None, name=None):
+                def addProcUnit(self, inputId='0', datatype=None, name=None):
                     #Compatible with old signal chain version
                     if datatype==None and name==None:
                         raise ValueError, "datatype or name should be defined"
                     if name==None:
                         if 'Proc' in datatype:
                             name = datatype
                         else:
                             name = '%sProc' %(datatype)
                     if datatype==None:
                         datatype = name.replace('Proc','')
-                    id = self.__getNewId()
+                    idProcUnit = self.__getNewId()
                     procUnitConfObj = ProcUnitConf()
-                    procUnitConfObj.setup(id, name, datatype, inputId, parentId=self.id)
+                    procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
                     self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                     return procUnitConfObj
                 def removeProcUnit(self, id):
                     if id in self.procUnitConfObjDict.keys():
                         self.procUnitConfObjDict.pop(id)
                 def getReadUnitId(self):
                     readUnitConfObj = self.getReadUnitObj()
                     return readUnitConfObj.id
                 def getReadUnitObj(self):
                     for obj in self.procUnitConfObjDict.values():
                         if obj.getElementName() == "ReadUnit":
                             return obj
                     return None
                 def getProcUnitObj(self, id):
                     return self.procUnitConfObjDict[id]
                 def getProcUnitObjByName(self, name):
                     for obj in self.procUnitConfObjDict.values():
                         if obj.name == name:
                             return obj
                     return None
                 def makeXml(self):
                     projectElement = Element('Project')
                     projectElement.set('id', str(self.id))
                     projectElement.set('name', self.name)
                     projectElement.set('description', self.description)
             #        for readUnitConfObj in self.readUnitConfObjList:
             #            readUnitConfObj.makeXml(projectElement)
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         procUnitConfObj.makeXml(projectElement)
                     self.projectElement = projectElement
                 def writeXml(self, filename):
                     self.makeXml()
                     #print prettify(self.projectElement)
                     ElementTree(self.projectElement).write(filename, method='xml')
                 def readXml(self, filename):
                     #tree = ET.parse(filename)
                     self.projectElement = None
             #        self.readUnitConfObjList = []
                     self.procUnitConfObjDict = {}
                     self.projectElement = ElementTree().parse(filename)
                     self.project = self.projectElement.tag
                     self.id = self.projectElement.get('id')
                     self.name = self.projectElement.get('name')
                     self.description = self.projectElement.get('description')
                     readUnitElementList = self.projectElement.getiterator(ReadUnitConf().getElementName())
                     for readUnitElement in readUnitElementList:
                         readUnitConfObj = ReadUnitConf()
                         readUnitConfObj.readXml(readUnitElement)
+                        if readUnitConfObj.parentId == None:
+                            readUnitConfObj.parentId = self.id
                         self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     procUnitElementList = self.projectElement.getiterator(ProcUnitConf().getElementName())
                     for procUnitElement in procUnitElementList:
                         procUnitConfObj = ProcUnitConf()
                         procUnitConfObj.readXml(procUnitElement)
+                        if procUnitConfObj.parentId == None:
+                            procUnitConfObj.parentId = self.id
                         self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                 def printattr(self):
                     print "Project[%s]: name = %s, description = %s" %(self.id,
                                                                           self.name,
                                                                           self.description)
             #        for readUnitConfObj in self.readUnitConfObjList:
             #            readUnitConfObj.printattr()
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         procUnitConfObj.printattr()
                 def createObjects(self):
             #        for readUnitConfObj in self.readUnitConfObjList:
             #            readUnitConfObj.createObjects()
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         procUnitConfObj.createObjects()
                 def __connect(self, objIN, thisObj):
                     thisObj.setInput(objIN.getOutputObj())
                 def connectObjects(self):
                     for thisPUConfObj in self.procUnitConfObjDict.values():
                         inputId = thisPUConfObj.getInputId()
                         if int(inputId) == 0:
                             continue
                         #Get input object
                         puConfINObj = self.procUnitConfObjDict[inputId]
                         puObjIN = puConfINObj.getProcUnitObj()
                         #Get current object
                         thisPUObj = thisPUConfObj.getProcUnitObj()
                         self.__connect(puObjIN, thisPUObj)
                 def run(self):
             #        for readUnitConfObj in self.readUnitConfObjList:
             #            readUnitConfObj.run()
                     print
                     print "*"*40
                     print "   Starting SIGNAL CHAIN PROCESSING  "
                     print "*"*40
                     print
                     keyList = self.procUnitConfObjDict.keys()
                     keyList.sort()
                     while(True):
                         finalSts = False
                         for procKey in keyList:
             #                 print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
                             procUnitConfObj = self.procUnitConfObjDict[procKey]
                             sts = procUnitConfObj.run()
                             finalSts = finalSts or sts
                         #If every process unit finished so end process
                         if not(finalSts):
                             print "Every process unit have finished"
                             break
                         if self.control['pause']:
                             print "Process suspended"
                             while True:
                                 sleep(0.1)
                                 if not self.control['pause']:
                                     break
                                 if self.control['stop']:
                                     break
                             print "Process reinitialized"
                         if self.control['stop']:
                             print "Process stopped"
                             break
                     #Closing every process
                     for procKey in keyList:
                         procUnitConfObj = self.procUnitConfObjDict[procKey]
                         procUnitConfObj.close()
                     print "Process finished"
                 def start(self, filename):
                     self.writeXml(filename)
                     self.readXml(filename)
                     self.createObjects()
                     self.connectObjects()
                     self.run()
             if __name__ == '__main__':
                 desc = "Segundo Test"
                 filename = "schain.xml"
                 controllerObj = Project()
                 controllerObj.setup(id = '191', name='test01', description=desc)
                 readUnitConfObj = controllerObj.addReadUnit(datatype='Voltage',
                                                             path='data/rawdata/',
                                                             startDate='2011/01/01',
                                                             endDate='2012/12/31',
                                                             startTime='00:00:00',
                                                             endTime='23:59:59',
                                                             online=1,
                                                             walk=1)
             #    opObj00 = readUnitConfObj.addOperation(name='printInfo')
                 procUnitConfObj0 = controllerObj.addProcUnit(datatype='Voltage', inputId=readUnitConfObj.getId())
                 opObj10 = procUnitConfObj0.addOperation(name='selectChannels')
                 opObj10.addParameter(name='channelList', value='3,4,5', format='intlist')
                 opObj10 = procUnitConfObj0.addOperation(name='selectHeights')
                 opObj10.addParameter(name='minHei', value='90', format='float')
                 opObj10.addParameter(name='maxHei', value='180', format='float')
                 opObj12 = procUnitConfObj0.addOperation(name='CohInt', optype='external')
                 opObj12.addParameter(name='n', value='10', format='int')
                 procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj0.getId())
                 procUnitConfObj1.addParameter(name='nFFTPoints', value='32', format='int')
             #    procUnitConfObj1.addParameter(name='pairList', value='(0,1),(0,2),(1,2)', format='')
                 opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
                 opObj11.addParameter(name='idfigure', value='1', format='int')
                 opObj11.addParameter(name='wintitle', value='SpectraPlot0', format='str')
                 opObj11.addParameter(name='zmin', value='40', format='int')
                 opObj11.addParameter(name='zmax', value='90', format='int')
                 opObj11.addParameter(name='showprofile', value='1', format='int')
             #    opObj11 = procUnitConfObj1.addOperation(name='CrossSpectraPlot', optype='external')
             #    opObj11.addParameter(name='idfigure', value='2', format='int')
             #    opObj11.addParameter(name='wintitle', value='CrossSpectraPlot', format='str')
             #    opObj11.addParameter(name='zmin', value='40', format='int')
             #    opObj11.addParameter(name='zmax', value='90', format='int')
             #    procUnitConfObj2 = controllerObj.addProcUnit(datatype='Voltage', inputId=procUnitConfObj0.getId())
             #
             #    opObj12 = procUnitConfObj2.addOperation(name='CohInt', optype='external')
             #    opObj12.addParameter(name='n', value='2', format='int')
             #    opObj12.addParameter(name='overlapping', value='1', format='int')
             #
             #    procUnitConfObj3 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj2.getId())
             #    procUnitConfObj3.addParameter(name='nFFTPoints', value='32', format='int')
             #
             #    opObj11 = procUnitConfObj3.addOperation(name='SpectraPlot', optype='external')
             #    opObj11.addParameter(name='idfigure', value='2', format='int')
             #    opObj11.addParameter(name='wintitle', value='SpectraPlot1', format='str')
             #    opObj11.addParameter(name='zmin', value='40', format='int')
             #    opObj11.addParameter(name='zmax', value='90', format='int')
             #    opObj11.addParameter(name='showprofile', value='1', format='int')
             #    opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='external')
             #    opObj11.addParameter(name='idfigure', value='10', format='int')
             #    opObj11.addParameter(name='wintitle', value='RTI', format='str')
             ##    opObj11.addParameter(name='xmin', value='21', format='float')
             ##    opObj11.addParameter(name='xmax', value='22', format='float')
             #    opObj11.addParameter(name='zmin', value='40', format='int')
             #    opObj11.addParameter(name='zmax', value='90', format='int')
             #    opObj11.addParameter(name='showprofile', value='1', format='int')
             #    opObj11.addParameter(name='timerange', value=str(60), format='int')
             #    opObj10 = procUnitConfObj1.addOperation(name='selectChannels')
             #    opObj10.addParameter(name='channelList', value='0,2,4,6', format='intlist')
             #
             #    opObj12 = procUnitConfObj1.addOperation(name='IncohInt', optype='external')
             #    opObj12.addParameter(name='n', value='2', format='int')
             #
             #    opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
             #    opObj11.addParameter(name='idfigure', value='2', format='int')
             #    opObj11.addParameter(name='wintitle', value='SpectraPlot10', format='str')
             #    opObj11.addParameter(name='zmin', value='70', format='int')
             #    opObj11.addParameter(name='zmax', value='90', format='int')
             #
             #    opObj10 = procUnitConfObj1.addOperation(name='selectChannels')
             #    opObj10.addParameter(name='channelList', value='2,6', format='intlist')
             #
             #    opObj12 = procUnitConfObj1.addOperation(name='IncohInt', optype='external')
             #    opObj12.addParameter(name='n', value='2', format='int')
             #
             #    opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
             #    opObj11.addParameter(name='idfigure', value='3', format='int')
             #    opObj11.addParameter(name='wintitle', value='SpectraPlot10', format='str')
             #    opObj11.addParameter(name='zmin', value='70', format='int')
             #    opObj11.addParameter(name='zmax', value='90', format='int')
             #    opObj12 = procUnitConfObj1.addOperation(name='decoder')
             #    opObj12.addParameter(name='ncode', value='2', format='int')
             #    opObj12.addParameter(name='nbauds', value='8', format='int')
             #    opObj12.addParameter(name='code0', value='001110011', format='int')
             #    opObj12.addParameter(name='code1', value='001110011', format='int')
             #    procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj1.getId())
             #
             #    opObj21 = procUnitConfObj2.addOperation(name='IncohInt', optype='external')
             #    opObj21.addParameter(name='n', value='2', format='int')
             #
             #    opObj11 = procUnitConfObj2.addOperation(name='SpectraPlot', optype='external')
             #    opObj11.addParameter(name='idfigure', value='4', format='int')
             #    opObj11.addParameter(name='wintitle', value='SpectraPlot OBJ 2', format='str')
             #    opObj11.addParameter(name='zmin', value='70', format='int')
             #    opObj11.addParameter(name='zmax', value='90', format='int')
                 print "Escribiendo el archivo XML"
                 controllerObj.writeXml(filename)
                 print "Leyendo el archivo XML"
                 controllerObj.readXml(filename)
                 #controllerObj.printattr()
                 controllerObj.createObjects()
                 controllerObj.connectObjects()
                 controllerObj.run()
   No newline at end of file

schainpy/gui/viewcontroller/basicwindow.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/gui/viewcontroller/comm.py +11 -1

             import threading
             import Queue
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             from schainpy.controller  import Project
             from command import *
             class ControllerThread(threading.Thread):
                 def __init__(self, filename, data_q=None):
                     super(ControllerThread, self).__init__()
+                    self.setDaemon(True)
                     self.filename = filename
                     self.data_q = data_q
                     self.control = {'stop':False,'pause':False}
                 def stop(self):
                     self.control['stop'] = True
                 def pause(self):
                     self.control['pause'] = not(self.control['pause'])
                 def run(self):
                     self.control['stop'] = False
                     self.control['pause'] = False
                     self.controllerObj = Project(self.control, self.data_q)
                     self.controllerObj.readXml(self.filename)
                     self.controllerObj.createObjects()
                     self.controllerObj.connectObjects()
                     self.controllerObj.run()
             class CommCtrlProcessThread(threading.Thread):
                 """ Implements the threading.Thread interface (start, join, etc.) and
                     can be controlled via the cmd_q Queue attribute. Replies are placed in
                     the reply_q Queue attribute.
                 """
                 def __init__(self, cmd_q=Queue.Queue(), reply_q=Queue.Queue()):
                     super(CommCtrlProcessThread, self).__init__()
                     self.cmd_q = cmd_q
             #         self.reply_q = reply_q
             #         self.print_q = Queue.Queue()
             #         self.data_q = Queue.Queue()
                     self.alive = threading.Event()
                     self.setDaemon(True)
                     self.alive.set()
                     self.socket = None
                     self.socketIO = None
                     self.mySocket = None
                     self.controllerObj = None
                     self.handlers = {
                         ProcessCommand.PROCESS: self._handle_ioPROCESSTHREAD,
                         ProcessCommand.MESSAGE: self._handle_ioMESSAGE,
                         ProcessCommand.DATA: self._handle_ioDATA,
                         ProcessCommand.STOP: self._handle_ioSTOP,
                         ProcessCommand.PAUSE: self._handle_ioPAUSE
                     }
                 def run(self):
                     while self.alive.isSet():
                         try:
                             cmd = self.cmd_q.get(True, 0.1)
                             self.handlers[cmd.type](cmd)
                         except Queue.Empty as e:
-                            sleep(0.1)
                             continue
                 def isRunning(self):
                     if self.controllerObj == None:
                         return False
                     if self.controllerObj.isAlive():
                         return True
                     return False
                 def _handle_ioPROCESSTHREAD(self, cmd):
                     filename = cmd.data
                     self.controllerObj = ControllerThread(filename=filename)
                     self.controllerObj.start()
                 def _handle_ioPAUSE(self, cmd):
                     self.controllerObj.pause()
                 def _handle_ioSTOP(self, cmd):
                     self.controllerObj.stop()
+                    while self.controllerObj.isAlive():
+                        self.console.clear()
+                        self.console.append("Close graphics before continue...")
+                        sleep(0.1)
                     self.controllerObj.join()
             #         print "Process thread finished"
                 def _handle_ioDATA(self, cmd):
                     self.reply_q.put(self._success_reply_data(data=cmd.data))
                 def _handle_ioMESSAGE(self, cmd):
                     self.reply_q.put(self._success_reply_message(data=cmd.data))
                 def _success_reply_data(self, data=None):
                     return ClientReply(ClientReply.DATA, data)
                 def _success_reply_message(self, data=None):
                     return ClientReply(ClientReply.MESSAGE, data)
                 def join(self, timeout=None):
                     self.alive.clear()
                     threading.Thread.join(self, timeout)
   No newline at end of file

schainpy/gui/viewer/windows/ui_project.py +2 0

             from PyQt4 import QtCore, QtGui
             try:
                 _fromUtf8 = QtCore.QString.fromUtf8
             except AttributeError:
                 def _fromUtf8(s):
                     return s
             try:
                 _encoding = QtGui.QApplication.UnicodeUTF8
                 def _translate(context, text, disambig):
                     return QtGui.QApplication.translate(context, text, disambig, _encoding)
             except AttributeError:
                 def _translate(context, text, disambig):
                     return QtGui.QApplication.translate(context, text, disambig)
             class Ui_ProjectTab(object):
                 def setupUi(self):
                     self.tabProject = QtGui.QWidget()
                     self.tabProject.setObjectName(_fromUtf8("tabProject"))
                     self.gridLayout_15 = QtGui.QGridLayout(self.tabProject)
                     self.gridLayout_15.setObjectName(_fromUtf8("gridLayout_15"))
                     self.frame = QtGui.QFrame(self.tabProject)
                     self.frame.setFrameShape(QtGui.QFrame.StyledPanel)
                     self.frame.setFrameShadow(QtGui.QFrame.Raised)
                     self.frame.setObjectName(_fromUtf8("frame"))
                     self.gridLayout_2 = QtGui.QGridLayout(self.frame)
                     self.gridLayout_2.setObjectName(_fromUtf8("gridLayout_2"))
                     self.label = QtGui.QLabel(self.frame)
                     self.label.setObjectName(_fromUtf8("label"))
                     self.gridLayout_2.addWidget(self.label, 0, 0, 1, 1)
                     self.proName = QtGui.QLineEdit(self.frame)
                     self.proName.setObjectName(_fromUtf8("proName"))
                     self.gridLayout_2.addWidget(self.proName, 0, 1, 1, 8)
                     self.label_11 = QtGui.QLabel(self.frame)
                     self.label_11.setObjectName(_fromUtf8("label_11"))
                     self.gridLayout_2.addWidget(self.label_11, 1, 0, 1, 1)
                     self.proComDataType = QtGui.QComboBox(self.frame)
                     self.proComDataType.setObjectName(_fromUtf8("proComDataType"))
                     self.proComDataType.addItem(_fromUtf8(""))
                     self.proComDataType.addItem(_fromUtf8(""))
                     self.proComDataType.addItem(_fromUtf8(""))
+                    self.proComDataType.addItem(_fromUtf8(""))
                     self.gridLayout_2.addWidget(self.proComDataType, 1, 1, 1, 5)
                     self.proDataType = QtGui.QLineEdit(self.frame)
                     self.proDataType.setObjectName(_fromUtf8("proDataType"))
                     self.gridLayout_2.addWidget(self.proDataType, 1, 6, 1, 3)
                     self.label_15 = QtGui.QLabel(self.frame)
                     self.label_15.setObjectName(_fromUtf8("label_15"))
                     self.gridLayout_2.addWidget(self.label_15, 2, 0, 1, 1)
                     self.proToolPath = QtGui.QToolButton(self.frame)
                     self.proToolPath.setObjectName(_fromUtf8("proToolPath"))
                     self.gridLayout_2.addWidget(self.proToolPath, 2, 1, 1, 1)
                     self.proDataPath = QtGui.QLineEdit(self.frame)
                     self.proDataPath.setObjectName(_fromUtf8("proDataPath"))
                     self.gridLayout_2.addWidget(self.proDataPath, 2, 2, 1, 7)
                     self.label_23 = QtGui.QLabel(self.frame)
                     self.label_23.setObjectName(_fromUtf8("label_23"))
                     self.gridLayout_2.addWidget(self.label_23, 3, 0, 1, 1)
                     self.proComReadMode = QtGui.QComboBox(self.frame)
                     self.proComReadMode.setObjectName(_fromUtf8("proComReadMode"))
                     self.proComReadMode.addItem(_fromUtf8(""))
                     self.proComReadMode.addItem(_fromUtf8(""))
                     self.gridLayout_2.addWidget(self.proComReadMode, 3, 1, 1, 2)
                     self.label_33 = QtGui.QLabel(self.frame)
                     self.label_33.setObjectName(_fromUtf8("label_33"))
                     self.gridLayout_2.addWidget(self.label_33, 3, 5, 1, 2)
                     self.proDelay = QtGui.QLineEdit(self.frame)
                     self.proDelay.setObjectName(_fromUtf8("proDelay"))
                     self.gridLayout_2.addWidget(self.proDelay, 3, 8, 1, 1)
                     self.label_32 = QtGui.QLabel(self.frame)
                     self.label_32.setObjectName(_fromUtf8("label_32"))
                     self.gridLayout_2.addWidget(self.label_32, 4, 0, 1, 1)
                     self.proComWalk = QtGui.QComboBox(self.frame)
                     self.proComWalk.setObjectName(_fromUtf8("proComWalk"))
                     self.proComWalk.addItem(_fromUtf8(""))
                     self.proComWalk.addItem(_fromUtf8(""))
                     self.gridLayout_2.addWidget(self.proComWalk, 4, 1, 1, 8)
                     self.proLoadButton = QtGui.QPushButton(self.frame)
                     self.proLoadButton.setObjectName(_fromUtf8("proLoadButton"))
                     self.gridLayout_2.addWidget(self.proLoadButton, 5, 0, 1, 9)
                     self.label_10 = QtGui.QLabel(self.frame)
                     self.label_10.setObjectName(_fromUtf8("label_10"))
                     self.gridLayout_2.addWidget(self.label_10, 3, 3, 1, 1)
                     self.proSet = QtGui.QLineEdit(self.frame)
                     self.proSet.setObjectName(_fromUtf8("proSet"))
                     self.gridLayout_2.addWidget(self.proSet, 3, 4, 1, 1)
                     self.gridLayout_15.addWidget(self.frame, 0, 0, 1, 1)
                     self.frame_2 = QtGui.QFrame(self.tabProject)
                     self.frame_2.setFrameShape(QtGui.QFrame.StyledPanel)
                     self.frame_2.setFrameShadow(QtGui.QFrame.Raised)
                     self.frame_2.setObjectName(_fromUtf8("frame_2"))
                     self.gridLayout_10 = QtGui.QGridLayout(self.frame_2)
                     self.gridLayout_10.setObjectName(_fromUtf8("gridLayout_10"))
                     self.label_27 = QtGui.QLabel(self.frame_2)
                     self.label_27.setObjectName(_fromUtf8("label_27"))
                     self.gridLayout_10.addWidget(self.label_27, 0, 0, 1, 1)
                     self.proComStartDate = QtGui.QComboBox(self.frame_2)
                     self.proComStartDate.setObjectName(_fromUtf8("proComStartDate"))
                     self.gridLayout_10.addWidget(self.proComStartDate, 0, 1, 1, 1)
                     self.label_28 = QtGui.QLabel(self.frame_2)
                     self.label_28.setObjectName(_fromUtf8("label_28"))
                     self.gridLayout_10.addWidget(self.label_28, 1, 0, 1, 1)
                     self.proComEndDate = QtGui.QComboBox(self.frame_2)
                     self.proComEndDate.setObjectName(_fromUtf8("proComEndDate"))
                     self.gridLayout_10.addWidget(self.proComEndDate, 1, 1, 1, 1)
                     self.label_2 = QtGui.QLabel(self.frame_2)
                     self.label_2.setObjectName(_fromUtf8("label_2"))
                     self.gridLayout_10.addWidget(self.label_2, 2, 0, 1, 1)
                     self.proStartTime = QtGui.QTimeEdit(self.frame_2)
                     self.proStartTime.setObjectName(_fromUtf8("proStartTime"))
                     self.gridLayout_10.addWidget(self.proStartTime, 2, 1, 1, 1)
                     self.label_3 = QtGui.QLabel(self.frame_2)
                     self.label_3.setObjectName(_fromUtf8("label_3"))
                     self.gridLayout_10.addWidget(self.label_3, 3, 0, 1, 1)
                     self.proEndTime = QtGui.QTimeEdit(self.frame_2)
                     self.proEndTime.setObjectName(_fromUtf8("proEndTime"))
                     self.gridLayout_10.addWidget(self.proEndTime, 3, 1, 1, 1)
                     self.label_30 = QtGui.QLabel(self.frame_2)
                     self.label_30.setObjectName(_fromUtf8("label_30"))
                     self.gridLayout_10.addWidget(self.label_30, 4, 0, 1, 1)
                     self.proDescription = QtGui.QTextEdit(self.frame_2)
                     self.proDescription.setObjectName(_fromUtf8("proDescription"))
                     self.gridLayout_10.addWidget(self.proDescription, 4, 1, 1, 1)
                     self.gridLayout_15.addWidget(self.frame_2, 1, 0, 1, 1)
                     self.frame_3 = QtGui.QFrame(self.tabProject)
                     self.frame_3.setFrameShape(QtGui.QFrame.StyledPanel)
                     self.frame_3.setFrameShadow(QtGui.QFrame.Raised)
                     self.frame_3.setObjectName(_fromUtf8("frame_3"))
                     self.gridLayout_14 = QtGui.QGridLayout(self.frame_3)
                     self.gridLayout_14.setObjectName(_fromUtf8("gridLayout_14"))
                     self.proOk = QtGui.QPushButton(self.frame_3)
                     self.proOk.setObjectName(_fromUtf8("proOk"))
                     self.gridLayout_14.addWidget(self.proOk, 0, 0, 1, 1)
                     self.proClear = QtGui.QPushButton(self.frame_3)
                     self.proClear.setObjectName(_fromUtf8("proClear"))
                     self.gridLayout_14.addWidget(self.proClear, 0, 1, 1, 1)
                     self.gridLayout_15.addWidget(self.frame_3, 2, 0, 1, 1)
                     self.tabWidgetProject.addTab(self.tabProject, _fromUtf8(""))
                 def retranslateUi(self):
                     self.label.setText(_translate("MainWindow", "Project Name :", None))
                     self.label_11.setText(_translate("MainWindow", "DataType :", None))
                     self.proComDataType.setItemText(0, _translate("MainWindow", "Voltage", None))
                     self.proComDataType.setItemText(1, _translate("MainWindow", "Spectra", None))
                     self.proComDataType.setItemText(2, _translate("MainWindow", "Fits", None))
+                    self.proComDataType.setItemText(3, _translate("MainWindow", "USRP", None))
                     self.label_15.setText(_translate("MainWindow", "DataPath :", None))
                     self.proToolPath.setText(_translate("MainWindow", "...", None))
                     self.label_23.setText(_translate("MainWindow", "Read Mode:", None))
                     self.proComReadMode.setItemText(0, _translate("MainWindow", "Offline", None))
                     self.proComReadMode.setItemText(1, _translate("MainWindow", "Online", None))
                     self.label_33.setText(_translate("MainWindow", "Delay:", None))
                     self.label_32.setText(_translate("MainWindow", "Walk :", None))
                     self.proComWalk.setItemText(0, _translate("MainWindow", "On Files", None))
                     self.proComWalk.setItemText(1, _translate("MainWindow", "On Folder", None))
                     self.proLoadButton.setText(_translate("MainWindow", "Load", None))
                     self.label_10.setText(_translate("MainWindow", "Set:", None))
                     self.label_27.setText(_translate("MainWindow", "Star Date:", None))
                     self.label_28.setText(_translate("MainWindow", "End Date:", None))
                     self.label_2.setText(_translate("MainWindow", "Start Time:", None))
                     self.label_3.setText(_translate("MainWindow", "End Time:", None))
                     self.label_30.setText(_translate("MainWindow", "Description:", None))
                     self.proOk.setText(_translate("MainWindow", "Ok", None))
                     self.proClear.setText(_translate("MainWindow", "Clear", None))
                     self.tabWidgetProject.setTabText(self.tabWidgetProject.indexOf(self.tabProject), _translate("MainWindow", "Project", None))
   No newline at end of file

schainpy/model/io/jroIO_HDF5.py +1 -1

             import numpy
             import time
             import os
             import h5py
             import re
             from schainpy.model.data.jrodata import *
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
             from schainpy.model.io.jroIO_base import *
             class HDF5Reader(ProcessingUnit):
                 ext = ".hdf5"
                 optchar = "D"
                 timezone = None
                 secStart = None
                 secEnd = None
                 fileIndex = None
                 blockIndex = None
                 blocksPerFile = None
                 path = None
                 #List of Files
                 filenameList = None
                 datetimeList = None
                 #Hdf5 File
                 fpMetadata = None
                 pathMeta = None
                 listMetaname = None
                 listMeta = None
                 listDataname = None
                 listData = None
                 listShapes = None
                 fp = None
                 #dataOut reconstruction
                 dataOut = None
                 nRecords = None
                 def __init__(self):
                     self.dataOut = self.__createObjByDefault()
                     return
                 def __createObjByDefault(self):
                     dataObj = Parameters()
                     return dataObj
                 def setup(self,path=None,
                                 startDate=None,
                                 endDate=None,
                                 startTime=datetime.time(0,0,0),
                                 endTime=datetime.time(23,59,59),
                                 walk=True,
                                 timezone='ut',
                                 all=0,
                                 online=False,
                                 ext=None):
                     if ext==None:
                         ext = self.ext
                     self.timezone = timezone
             #         self.all = all
             #         self.online = online
                     self.path = path
                     startDateTime = datetime.datetime.combine(startDate,startTime)
                     endDateTime = datetime.datetime.combine(endDate,endTime)
                     secStart = (startDateTime-datetime.datetime(1970,1,1)).total_seconds()
                     secEnd = (endDateTime-datetime.datetime(1970,1,1)).total_seconds()
                     self.secStart = secStart
                     self.secEnd = secEnd
                     if not(online):
                         #Busqueda de archivos offline
                         self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, secStart, secEnd, walk)
                     else:
                         self.__searchFilesOnline(path, walk)
                     if not(self.filenameList):
                         print "There is no files into the folder: %s"%(path)
                         sys.exit(-1)
             #         self.__getExpParameters()
                     self.fileIndex = -1
                     self.__setNextFileOffline()
                     self.__readMetadata()
                     self.blockIndex = 0
                     return
                 def __searchFilesOffline(self,
                                         path,
                                         startDate,
                                         endDate,
                                         ext,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         secStart = 0,
                                         secEnd = numpy.inf,
                                         walk=True):
             #         self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
             #
             #         self.__checkPath()
             #
             #         self.__findDataForDates()
             #
             #         self.__selectDataForTimes()
             #
             #         for i in range(len(self.filenameList)):
             #             print "%s" %(self.filenameList[i])
                     pathList = []
                     if not walk:
                         #pathList.append(path)
                         multi_path = path.split(',')
                         for single_path in multi_path:
                             pathList.append(single_path)
                     else:
                         #dirList = []
                         multi_path = path.split(',')
                         for single_path in multi_path:
                             dirList = []
                             for thisPath in os.listdir(single_path):
                                 if not os.path.isdir(os.path.join(single_path,thisPath)):
                                     continue
-                                if not isDoyFolder(thisPath):
+                                if not isRadarFolder(thisPath):
                                     continue
                                 dirList.append(thisPath)
                             if not(dirList):
                                 return None, None
                             thisDate = startDate
                             while(thisDate <= endDate):
                                 year = thisDate.timetuple().tm_year
                                 doy = thisDate.timetuple().tm_yday
                                 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
                                 if len(matchlist) == 0:
                                     thisDate += datetime.timedelta(1)
                                     continue
                                 for match in matchlist:
                                     pathList.append(os.path.join(single_path,match))
                                 thisDate += datetime.timedelta(1)
                     if pathList == []:
                         print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
                         return None, None
                     print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
                     filenameList = []
                     datetimeList = []
                     pathDict = {}
                     filenameList_to_sort = []
                     for i in range(len(pathList)):
                         thisPath = pathList[i]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         pathDict.setdefault(fileList[0])
                         pathDict[fileList[0]] = i
                         filenameList_to_sort.append(fileList[0])
                     filenameList_to_sort.sort()
                     for file in filenameList_to_sort:
                         thisPath = pathList[pathDict[file]]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             thisDatetime = self.__isFileinThisTime(filename, secStart, secEnd)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print "Any file was found for the time range %s - %s" %(startTime, endTime)
                         return None, None
                     print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
                     print
                     for i in range(len(filenameList)):
                         print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __isFileinThisTime(self, filename, startSeconds, endSeconds):
                     """
                     Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                     Inputs:
                         filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                         startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                         endTime            :    tiempo final del rango seleccionado en formato datetime.time
                     Return:
                         Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                         fecha especificado, de lo contrario retorna False.
                     Excepciones:
                         Si el archivo no existe o no puede ser abierto
                         Si la cabecera no puede ser leida.
                     """
                     try:
                         fp = fp = h5py.File(filename,'r')
                     except IOError:
                         traceback.print_exc()
                         raise IOError, "The file %s can't be opened" %(filename)
                     grp = fp['Data']
                     timeAux = grp['time']
                     time0 = timeAux[:][0].astype(numpy.float)   #Time Vector
                     fp.close()
                     if self.timezone == 'lt':
                         time0 -= 5*3600
                     boolTimer = numpy.logical_and(time0 >= startSeconds,time0 < endSeconds)
                     if not (numpy.any(boolTimer)):
                         return None
                     thisDatetime = datetime.datetime.utcfromtimestamp(time0[0])
                     return thisDatetime
                 def __checkPath(self):
                     if os.path.exists(self.path):
                         self.status = 1
                     else:
                         self.status = 0
                         print 'Path:%s does not exists'%self.path
                     return
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     idFile += 1
                     if not(idFile < len(self.filenameList)):
                         print "No more Files"
                         return 0
                     filename = self.filenameList[idFile]
                     filePointer = h5py.File(filename,'r')
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fp = filePointer
                     print "Setting the file: %s"%self.filename
                     self.__readMetadata()
                     self.__setBlockList()
             #         self.nRecords = self.fp['Data'].attrs['blocksPerFile']
                     self.nRecords = self.fp['Data'].attrs['nRecords']
                     self.blockIndex = 0
                     return 1
                 def __setBlockList(self):
                     '''
                     self.fp
                     self.startDateTime
                     self.endDateTime
                     self.blockList
                     self.blocksPerFile
                     '''
                     filePointer = self.fp
                     secStart = self.secStart
                     secEnd = self.secEnd
                     grp = filePointer['Data']
                     timeVector = grp['time'].value.astype(numpy.float)[0]
                     if self.timezone == 'lt':
                         timeVector -= 5*3600
                     ind = numpy.where(numpy.logical_and(timeVector >= secStart , timeVector < secEnd))[0]
                     self.blockList = ind
                     self.blocksPerFile = len(ind)
                     return
                 def __readMetadata(self):
                     '''
                     self.pathMeta
                     self.listShapes
                     self.listMetaname
                     self.listMeta
                     '''
                     grp = self.fp['Data']
                     pathMeta = os.path.join(self.path, grp.attrs['metadata'])
                     if pathMeta == self.pathMeta:
                         return
                     else:
                         self.pathMeta = pathMeta
                     filePointer = h5py.File(self.pathMeta,'r')
                     groupPointer = filePointer['Metadata']
                     listMetaname = []
                     listMetadata = []
                     for item in groupPointer.items():
                         name = item[0]
                         if name=='array dimensions':
                             table = groupPointer[name][:]
                             listShapes = {}
                             for shapes in table:
                                 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4]])
                         else:
                             data = groupPointer[name].value
                             listMetaname.append(name)
                             listMetadata.append(data)
                             if name=='type':
                                 self.__initDataOut(data)
                     filePointer.close()
                     self.listShapes = listShapes
                     self.listMetaname = listMetaname
                     self.listMeta = listMetadata
                     return
                 def __readData(self):
                     grp = self.fp['Data']
                     listdataname = []
                     listdata = []
                     for item in grp.items():
                         name = item[0]
                         if name == 'time':
                             listdataname.append('utctime')
                             timeAux = grp[name].value.astype(numpy.float)[0]
                             listdata.append(timeAux)
                             continue
                         listdataname.append(name)
                         array = self.__setDataArray(self.nRecords, grp[name],self.listShapes[name])
                         listdata.append(array)
                     self.listDataname = listdataname
                     self.listData = listdata
                     return
                 def __setDataArray(self, nRecords, dataset, shapes):
                     nChannels = shapes[0]    #Dimension 0
                     nPoints = shapes[1]      #Dimension 1, number of Points or Parameters
                     nSamples = shapes[2]     #Dimension 2, number of samples or ranges
                     mode = shapes[3]
             #         if nPoints>1:
             #             arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
             #         else:
             #             arrayData = numpy.zeros((nRecords,nChannels,nSamples))
             #
             #         chn = 'channel'
             #
             #         for i in range(nChannels):
             #
             #             data = dataset[chn + str(i)].value
             #
             #             if nPoints>1:
             #                 data = numpy.rollaxis(data,2)
             #
             #             arrayData[:,i,:] = data
                     arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
                     doSqueeze = False
                     if mode == 0:
                         strds = 'channel'
                         nDatas = nChannels
                         newShapes = (nRecords,nPoints,nSamples)
                         if nPoints == 1:
                             doSqueeze = True
                             axisSqueeze = 2
                     else:
                         strds = 'param'
                         nDatas = nPoints
                         newShapes = (nRecords,nChannels,nSamples)
                         if nChannels == 1:
                             doSqueeze = True
                             axisSqueeze = 1
                     for i in range(nDatas):
                         data = dataset[strds + str(i)].value
                         data = data.reshape(newShapes)
                         if mode == 0:
                             arrayData[:,i,:,:] = data
                         else:
                             arrayData[:,:,i,:] = data
                     if doSqueeze:
                             arrayData = numpy.squeeze(arrayData, axis=axisSqueeze)
                     return arrayData
                 def __initDataOut(self, type):
             #         if type =='Parameters':
             #             self.dataOut = Parameters()
             #         elif type =='Spectra':
             #             self.dataOut = Spectra()
             #         elif type =='Voltage':
             #             self.dataOut = Voltage()
             #         elif type =='Correlation':
             #             self.dataOut = Correlation()
                     return
                 def __setDataOut(self):
                     listMeta = self.listMeta
                     listMetaname = self.listMetaname
                     listDataname = self.listDataname
                     listData = self.listData
                     blockIndex = self.blockIndex
                     blockList = self.blockList
                     for i in range(len(listMeta)):
                         setattr(self.dataOut,listMetaname[i],listMeta[i])
                     for j in range(len(listData)):
                         if listDataname[j]=='utctime':
             #                 setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex]])
                             setattr(self.dataOut,'utctimeInit',listData[j][blockList[blockIndex]])
                             continue
                         setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex],:])
                     return self.dataOut.data_param
                 def getData(self):
             #         if self.flagNoMoreFiles:
             #             self.dataOut.flagNoData = True
             #             print 'Process finished'
             #             return 0
             #
                     if self.blockIndex==self.blocksPerFile:
                          if not( self.__setNextFileOffline() ):
                             self.dataOut.flagNoData = True
                             return 0
             #
             #         if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
             #             self.dataOut.flagNoData = True
             #             return 0
                     self.__readData()
                     self.__setDataOut()
                     self.dataOut.flagNoData = False
                     self.blockIndex += 1
                     return
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
             #             self.setObjProperties()
                         self.isConfig = True
                     self.getData()
                     return
             class HDF5Writer(Operation):
                 ext = ".hdf5"
                 optchar = "D"
                 metaoptchar = "M"
                 metaFile = None
                 path = None
                 setFile = None
                 fp = None
                 grp = None
                 ds = None
                 firsttime = True
                 #Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None
                 #Data Arrays
                 dataList = None
                 metadataList = None
                 arrayDim = None
                 tableDim = None
             #     dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
                 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
                 mode = None
                 nDatas = None    #Number of datasets to be stored per array
                 nDims = None  #Number Dimensions in each dataset
                 def __init__(self):
                     Operation.__init__(self)
                     self.isConfig = False
                     return
                 def setup(self, dataOut, **kwargs):
                     self.path = kwargs['path']
                     if kwargs.has_key('ext'):
                         self.ext = kwargs['ext']
                     if kwargs.has_key('blocksPerFile'):
                         self.blocksPerFile = kwargs['blocksPerFile']
                     else:
                         self.blocksPerFile = 10
                     self.metadataList = kwargs['metadataList']
                     self.dataList = kwargs['dataList']
                     self.dataOut = dataOut
                     if kwargs.has_key('mode'):
                         mode = kwargs['mode']
                         if type(mode) == int:
                             mode = numpy.zeros(len(self.dataList)) + mode
                     else:
                         mode = numpy.zeros(len(self.dataList))
                     self.mode = mode
                     arrayDim = numpy.zeros((len(self.dataList),5))
                     #Table dimensions
                     dtype0 = self.dtype
                     tableList = []
                     for i in range(len(self.dataList)):
                         dataAux = getattr(self.dataOut, self.dataList[i])
                         if type(dataAux)==float or type(dataAux)==int:
                             arrayDim[i,0] = 1
                         else:
                             arrayDim0 = dataAux.shape
                             arrayDim[i,0] = len(arrayDim0)
                             arrayDim[i,4] = mode[i]
                             if len(arrayDim0) == 3:
                                 arrayDim[i,1:-1] = numpy.array(arrayDim0)
                             elif len(arrayDim0) == 2:
                                 arrayDim[i,2:-1] = numpy.array(arrayDim0) #nHeights
                             elif len(arrayDim0) == 1:
                                 arrayDim[i,3] = arrayDim0
                             elif len(arrayDim0) == 0:
                                 arrayDim[i,0] = 1
                                 arrayDim[i,3] = 1
                         table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
                         tableList.append(table)
                     self.arrayDim = arrayDim
                     self.tableDim = numpy.array(tableList, dtype = dtype0)
                     self.blockIndex = 0
                     return
                 def putMetadata(self):
                     fp = self.createMetadataFile()
                     self.writeMetadata(fp)
                     fp.close()
                     return
                 def createMetadataFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = ''
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     setFile += 1
                     file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, file )
                     self.metaFile = file
                     #Setting HDF5 File
                     fp = h5py.File(filename,'w')
                     return fp
                 def writeMetadata(self, fp):
                     grp = fp.create_group("Metadata")
                     grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
                     for i in range(len(self.metadataList)):
                         grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
                     return
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     mode = self.mode
                     if self.fp != None:
                         self.fp.close()
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     setFile += 1
                     file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, file )
                     #Setting HDF5 File
                     fp = h5py.File(filename,'w')
                     grp = fp.create_group("Data")
                     grp.attrs['metadata'] = self.metaFile
             #         grp.attrs['blocksPerFile'] = 0
                     ds = []
                     data = []
                     nDatas = numpy.zeros(len(self.dataList))
                     nDims = self.arrayDim[:,0]
                     for i in range(len(self.dataList)):
                         if nDims[i]==1:
                             ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,None) , chunks = True, dtype='S20')
                             ds.append(ds0)
                             data.append([])
                         else:
                             if mode[i]==0:
                                 strMode = "channel"
                                 nDatas[i] = self.arrayDim[i,1]
                             else:
                                 strMode = "param"
                                 nDatas[i] = self.arrayDim[i,2]
                             if nDims[i]==2:
                                 nDatas[i] = self.arrayDim[i,2]
                             grp0 = grp.create_group(self.dataList[i])
                             for j in range(int(nDatas[i])):
                                 tableName = strMode + str(j)
                                 if nDims[i] == 3:
                                     ds0 = grp0.create_dataset(tableName, (1,1,1) , maxshape=(None,None,None), chunks=True)
                                 else:
                                     ds0 = grp0.create_dataset(tableName, (1,1) , maxshape=(None,None), chunks=True)
                                 ds.append(ds0)
                                 data.append([])
                     self.nDatas = nDatas
                     self.nDims = nDims
                     #Saving variables
                     print 'Writing the file: %s'%filename
                     self.fp = fp
                     self.grp = grp
                     self.ds = ds
                     self.data = data
                     self.setFile = setFile
                     self.firsttime = True
                     self.blockIndex = 0
                     return
                 def putData(self):
                     self.setBlock()
                     self.writeBlock()
                     if self.blockIndex == self.blocksPerFile:
                         self.setNextFile()
                     return
                 def setBlock(self):
                     '''
                     data Array configured
                     self.data
                     '''
                     #Creating Arrays
                     data = self.data
                     nDatas = self.nDatas
                     nDims = self.nDims
                     mode = self.mode
                     ind = 0
                     for i in range(len(self.dataList)):
                         dataAux = getattr(self.dataOut,self.dataList[i])
                         if nDims[i] == 1:
                             data[ind] = numpy.array([str(dataAux)]).reshape((1,1))
                             if not self.firsttime:
                                 data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind]))
                             ind += 1
                         else:
                             for j in range(int(nDatas[i])):
                                 if (mode[i] == 0) or (nDims[i] == 2):       #In case division per channel or Dimensions is only 1
                                     data[ind] = dataAux[j,:]
                                 else:
                                     data[ind] = dataAux[:,j,:]
                                 if nDims[i] == 3:
                                     data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1))
                                     if not self.firsttime:
                                         data[ind] = numpy.dstack((self.ds[ind][:], data[ind]))
                                 else:
                                     data[ind] = data[ind].reshape((1,data[ind].shape[0]))
                                     if not self.firsttime:
                                         data[ind] = numpy.vstack((self.ds[ind][:], data[ind]))
                                 ind += 1
                     self.data = data
                     return
                 def writeBlock(self):
                     '''
                     Saves the block in the HDF5 file
                     '''
                     for i in range(len(self.ds)):
                         self.ds[i].resize(self.data[i].shape)
                         self.ds[i][:] = self.data[i]
                     self.blockIndex += 1
                     self.grp.attrs.modify('nRecords', self.blockIndex)
                     self.firsttime = False
                     return
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                         self.putMetadata()
                         self.setNextFile()
                     self.putData()
                     return

schainpy/model/io/jroIO_base.py +129 -26

             '''
             Created on Jul 2, 2014
             @author: roj-idl71
             '''
             import os
             import sys
             import glob
             import time
             import numpy
             import fnmatch
             import time, datetime
             #import h5py
             import traceback
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             LOCALTIME = True
             def isNumber(cad):
                 """
                 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                 Excepciones:
                     Si un determinado string no puede ser convertido a numero
                 Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                 Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                 """
                 try:
                     float( cad )
                     return True
                 except:
                     return False
             def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
                 """
                 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startUTSeconds      :    fecha inicial del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                     endUTSeconds        :    fecha final del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 try:
                     fp = open(filename,'rb')
                 except IOError:
                     traceback.print_exc()
                     raise IOError, "The file %s can't be opened" %(filename)
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 if not(sts):
                     print "Skipping the file %s because it has not a valid header" %(filename)
                     return 0
                 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
                     return 0
                 return 1
             def isFileinThisTime(filename, startTime, endTime):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                     endTime            :    tiempo final del rango seleccionado en formato datetime.time
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 try:
                     fp = open(filename,'rb')
                 except IOError:
                     traceback.print_exc()
                     raise IOError, "The file %s can't be opened" %(filename)
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 thisDatetime = basicHeaderObj.datatime
                 thisTime = thisDatetime.time()
                 if not(sts):
                     print "Skipping the file %s because it has not a valid header" %(filename)
                     return None
                 if not ((startTime <= thisTime) and (endTime > thisTime)):
                     return None
                 return thisDatetime
             def getFileFromSet(path, ext, set):
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     try:
                         year = int(thisFile[1:5])
                         doy  = int(thisFile[5:8])
                     except:
                         continue
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
                 if len(myfile)!= 0:
                     return myfile[0]
                 else:
                     filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
                     print 'the filename %s does not exist'%filename
                     print '...going to the last file: '
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def getlastFileFromPath(path, ext):
                 """
                 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
                 al final de la depuracion devuelve el ultimo file de la lista que quedo.
                 Input:
                     fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                     ext         :    extension de los files contenidos en una carpeta
                 Return:
                     El ultimo file de una determinada carpeta, no se considera el path.
                 """
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     year = thisFile[1:5]
                     if not isNumber(year):
                         continue
                     doy = thisFile[5:8]
                     if not isNumber(doy):
                         continue
                     year = int(year)
                     doy = int(doy)
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def checkForRealPath(path, foldercounter, year, doy, set, ext):
                 """
                 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
                 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
                 el path exacto de un determinado file.
                 Example    :
                     nombre correcto del file es  .../.../D2009307/P2009307367.ext
                     Entonces la funcion prueba con las siguientes combinaciones
                         .../.../y2009307367.ext
                         .../.../Y2009307367.ext
                         .../.../x2009307/y2009307367.ext
                         .../.../x2009307/Y2009307367.ext
                         .../.../X2009307/y2009307367.ext
                         .../.../X2009307/Y2009307367.ext
                     siendo para este caso, la ultima combinacion de letras, identica al file buscado
                 Return:
                     Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
                     caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
                     para el filename
                 """
                 fullfilename = None
                 find_flag = False
                 filename = None
                 prefixDirList = [None,'d','D']
                 if ext.lower() == ".r": #voltage
                     prefixFileList = ['d','D']
                 elif ext.lower() == ".pdata": #spectra
                     prefixFileList = ['p','P']
                 else:
                     return None, filename
                 #barrido por las combinaciones posibles
                 for prefixDir in prefixDirList:
                     thispath = path
                     if prefixDir != None:
                         #formo el nombre del directorio xYYYYDDD (x=d o x=D)
                         if foldercounter == 0:
                             thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
                         else:
                             thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
                     for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
                         filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
                         fullfilename = os.path.join( thispath, filename ) #formo el path completo
                         if os.path.exists( fullfilename ): #verifico que exista
                             find_flag = True
                             break
                     if find_flag:
                         break
                 if not(find_flag):
                     return None, filename
                 return fullfilename, filename
-            def isDoyFolder(folder):
+            def isRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
-                except:
-                    return 0
-                try:
                     doy = int(folder[5:8])
                 except:
                     return 0
                 return 1
+            def isRadarFile(file):
+                    try:
+                        year = int(file[1:5])
+                        doy = int(file[5:8])
+                        set = int(file[8:11])
+                    except:
+                        return 0
+                    return 1
+            def getDateFromRadarFile(file):
+                    try:
+                        year = int(file[1:5])
+                        doy = int(file[5:8])
+                        set = int(file[8:11])
+                    except:
+                        return None
+                    thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
+                    return thisDate
             class JRODataIO:
                 c = 3E8
                 isConfig = False
                 basicHeaderObj = None
                 systemHeaderObj = None
                 radarControllerHeaderObj = None
                 processingHeaderObj = None
                 online = 0
                 dtype = None
                 pathList = []
                 filenameList = []
                 filename = None
                 ext = None
                 flagIsNewFile = 1
                 flagDiscontinuousBlock = 0
                 flagIsNewBlock = 0
                 fp = None
                 firstHeaderSize = 0
                 basicHeaderSize = 24
                 versionFile = 1103
                 fileSize = None
             #     ippSeconds = None
                 fileSizeByHeader = None
                 fileIndex = None
                 profileIndex = None
                 blockIndex = None
                 nTotalBlocks = None
                 maxTimeStep = 30
                 lastUTTime = None
                 datablock = None
                 dataOut = None
                 blocksize = None
                 getByBlock = False
                 def __init__(self):
                     raise ValueError, "Not implemented"
                 def run(self):
                     raise ValueError, "Not implemented"
             class JRODataReader(JRODataIO):
                 nReadBlocks = 0
                 delay  = 10   #number of seconds waiting a new file
                 nTries  = 3  #quantity tries
                 nFiles = 3   #number of files for searching
                 path = None
                 foldercounter = 0
                 flagNoMoreFiles = 0
                 datetimeList = []
                 __isFirstTimeOnline = 1
                 __printInfo = True
                 profileIndex = None
                 nTxs = 1
                 txIndex = None
                 def __init__(self):
                     """
                     """
-                    raise ValueError, "This method has not been implemented"
+            #         raise NotImplementedError, "This method has not been implemented"
                 def createObjByDefault(self):
                     """
                     """
-                    raise ValueError, "This method has not been implemented"
+                    raise NotImplementedError, "This method has not been implemented"
                 def getBlockDimension(self):
-                    raise ValueError, "No implemented"
+                    raise NotImplementedError, "No implemented"
                 def __searchFilesOffLine(self,
                                         path,
-                                        startDate,
+                                        startDate=None,
-                                        endDate,
+                                        endDate=None,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.r',
                                         walk=True):
+                    self.filenameList = []
+                    self.datetimeList = []
                     pathList = []
                     if not walk:
                         #pathList.append(path)
                         multi_path = path.split(',')
                         for single_path in multi_path:
                             pathList.append(single_path)
                     else:
                         #dirList = []
                         multi_path = path.split(',')
                         for single_path in multi_path:
                             dirList = []
                             for thisPath in os.listdir(single_path):
                                 if not os.path.isdir(os.path.join(single_path,thisPath)):
                                     continue
-                                if not isDoyFolder(thisPath):
+                                if not isRadarFolder(thisPath):
                                     continue
                                 dirList.append(thisPath)
                             if not(dirList):
                                 return None, None
-                            thisDate = startDate
+                            if startDate and endDate:
+                                thisDate = startDate
-                            while(thisDate <= endDate):
-                                year = thisDate.timetuple().tm_year
+                                while(thisDate <= endDate):
-                                doy = thisDate.timetuple().tm_yday
+                                    year = thisDate.timetuple().tm_year
+                                    doy = thisDate.timetuple().tm_yday
-                                matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
-                                if len(matchlist) == 0:
+                                    matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
+                                    if len(matchlist) == 0:
+                                        thisDate += datetime.timedelta(1)
+                                        continue
+                                    for match in matchlist:
+                                        pathList.append(os.path.join(single_path,match,expLabel))
                                     thisDate += datetime.timedelta(1)
-                                    continue
+                            else:
-                                for match in matchlist:
+                                for thiDir in dirList:
-                                    pathList.append(os.path.join(single_path,match,expLabel))
+                                    pathList.append(os.path.join(single_path,thiDir,expLabel))
-                                thisDate += datetime.timedelta(1)
                     if pathList == []:
                         print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
                         return None, None
                     print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
                     filenameList = []
                     datetimeList = []
                     pathDict = {}
                     filenameList_to_sort = []
                     for i in range(len(pathList)):
                         thisPath = pathList[i]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         if len(fileList) < 1:
                             continue
                         fileList.sort()
                         pathDict.setdefault(fileList[0])
                         pathDict[fileList[0]] = i
                         filenameList_to_sort.append(fileList[0])
                     filenameList_to_sort.sort()
                     for file in filenameList_to_sort:
                         thisPath = pathList[pathDict[file]]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             thisDatetime = isFileinThisTime(filename, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print "Any file was found for the time range %s - %s" %(startTime, endTime)
                         return None, None
                     print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
                     print
                     for i in range(len(filenameList)):
                         print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
                     """
                     Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
                     devuelve el archivo encontrado ademas de otros datos.
                     Input:
                         path           :    carpeta donde estan contenidos los files que contiene data
                         expLabel        :     Nombre del subexperimento (subfolder)
                         ext              :    extension de los files
                         walk        :    Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
                     Return:
                         directory   :    eL directorio donde esta el file encontrado
                         filename    :    el ultimo file de una determinada carpeta
                         year        :    el anho
                         doy         :    el numero de dia del anho
                         set         :    el set del archivo
                     """
                     dirList = []
                     if not walk:
                         fullpath = path
                         foldercounter = 0
                     else:
                         #Filtra solo los directorios
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
-                            if not isDoyFolder(thisPath):
+                            if not isRadarFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None, None, None, None, None
                         dirList = sorted( dirList, key=str.lower )
                         doypath = dirList[-1]
                         foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
                         fullpath = os.path.join(path, doypath, expLabel)
                     print "%s folder was found: " %(fullpath )
                     if set == None:
                         filename = getlastFileFromPath(fullpath, ext)
                     else:
                         filename = getFileFromSet(fullpath, ext, set)
                     if not(filename):
                         return None, None, None, None, None, None
                     print "%s file was found" %(filename)
                     if not(self.__verifyFile(os.path.join(fullpath, filename))):
                         return None, None, None, None, None, None
                     year = int( filename[1:5] )
                     doy  = int( filename[5:8] )
                     set  = int( filename[8:11] )
                     return fullpath, foldercounter, filename, year, doy, set
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
             #                 print "[Reading] No more Files"
                             return 0
                         filename = self.filenameList[idFile]
                         if not(self.__verifyFile(filename)):
                             continue
                         fileSize = os.path.getsize(filename)
                         fp = open(filename,'rb')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fp = fp
             #         print "[Reading] Setting the file: %s"%self.filename
                     return 1
                 def __setNextFileOnline(self):
                     """
                     Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
                     no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
                     siguientes.
                     Affected:
                         self.flagIsNewFile
                         self.filename
                         self.fileSize
                         self.fp
                         self.set
                         self.flagNoMoreFiles
                     Return:
 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
 : si el file fue abierto con exito y esta listo a ser leido
                     Excepciones:
                         Si un determinado file no puede ser abierto
                     """
                     nFiles = 0
                     fileOk_flag = False
                     firstTime_flag = True
                     self.set += 1
                     if self.set > 999:
                         self.set = 0
                         self.foldercounter += 1
                     #busca el 1er file disponible
                     fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                     if fullfilename:
                         if self.__verifyFile(fullfilename, False):
                             fileOk_flag = True
                     #si no encuentra un file entonces espera y vuelve a buscar
                     if not(fileOk_flag):
                         for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
                             if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
                                 tries = self.nTries
                             else:
                                 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
                             for nTries in range( tries ):
                                 if firstTime_flag:
                                     print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
                                     sleep( self.delay )
                                 else:
                                     print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
                                 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                                 if fullfilename:
                                     if self.__verifyFile(fullfilename):
                                         fileOk_flag = True
                                         break
                             if fileOk_flag:
                                 break
                             firstTime_flag = False
                             print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
                             self.set += 1
                             if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
                                 self.set = 0
                                 self.doy += 1
                                 self.foldercounter = 0
                     if fileOk_flag:
                         self.fileSize = os.path.getsize( fullfilename )
                         self.filename = fullfilename
                         self.flagIsNewFile = 1
                         if self.fp != None: self.fp.close()
                         self.fp = open(fullfilename, 'rb')
                         self.flagNoMoreFiles = 0
             #             print '[Reading] Setting the file: %s' % fullfilename
                     else:
                         self.fileSize = 0
                         self.filename = None
                         self.flagIsNewFile = 0
                         self.fp = None
                         self.flagNoMoreFiles = 1
             #             print '[Reading] No more files to read'
                     return fileOk_flag
                 def setNextFile(self):
                     if self.fp != None:
                         self.fp.close()
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         print '[Reading] No more files to read'
                         return 0
                     print '[Reading] Setting the file: %s' % self.filename
                     self.__readFirstHeader()
                     self.nReadBlocks = 0
                     return 1
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             self.basicHeaderObj.read(self.fp)
                             return 1
                         if self.fileSize == self.fileSizeByHeader:
             #                self.flagEoF = True
                             return 0
                         print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         sleep( self.delay )
                     return 0
                 def waitDataBlock(self,pointer_location):
                     currentPointer = pointer_location
                     neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             return 1
                         print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         sleep( self.delay )
                     return 0
                 def __jumpToLastBlock(self):
                     if not(self.__isFirstTimeOnline):
                         return
                     csize = self.fileSize - self.fp.tell()
                     blocksize = self.processingHeaderObj.blockSize
                     #salta el primer bloque de datos
                     if csize > self.processingHeaderObj.blockSize:
                         self.fp.seek(self.fp.tell() + blocksize)
                     else:
                         return
                     csize = self.fileSize - self.fp.tell()
                     neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     while True:
                         if self.fp.tell()<self.fileSize:
                             self.fp.seek(self.fp.tell() + neededsize)
                         else:
                             self.fp.seek(self.fp.tell() - neededsize)
                             break
             #        csize = self.fileSize - self.fp.tell()
             #        neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
             #        factor = int(csize/neededsize)
             #        if factor > 0:
             #            self.fp.seek(self.fp.tell() + factor*neededsize)
                     self.flagIsNewFile = 0
                     self.__isFirstTimeOnline = 0
                 def __setNewBlock(self):
                     if self.fp == None:
                         return 0
                     if self.online:
                         self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         return 1
                     self.lastUTTime = self.basicHeaderObj.utc
                     currentSize = self.fileSize - self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     if (currentSize >= neededSize):
                         self.basicHeaderObj.read(self.fp)
                         return 1
                     if self.__waitNewBlock():
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
                     self.flagDiscontinuousBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagDiscontinuousBlock = 1
                     return 1
                 def readNextBlock(self):
                     if not(self.__setNewBlock()):
                         return 0
                     if not(self.readBlock()):
                         return 0
                     return 1
                 def __readFirstHeader(self):
                     self.basicHeaderObj.read(self.fp)
                     self.systemHeaderObj.read(self.fp)
                     self.radarControllerHeaderObj.read(self.fp)
                     self.processingHeaderObj.read(self.fp)
                     self.firstHeaderSize = self.basicHeaderObj.size
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
                     else:
                         raise ValueError, 'Data type was not defined'
                     self.dtype = datatype_str
                     #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
             #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
             #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                     self.getBlockDimension()
                 def __verifyFile(self, filename, msgFlag=True):
                     msg = None
                     try:
                         fp = open(filename, 'rb')
                         currentPosition = fp.tell()
                     except IOError:
                         traceback.print_exc()
                         if msgFlag:
                             print "[Reading] The file %s can't be opened" % (filename)
                         return False
                     neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
                     if neededSize == 0:
                         basicHeaderObj = BasicHeader(LOCALTIME)
                         systemHeaderObj = SystemHeader()
                         radarControllerHeaderObj = RadarControllerHeader()
                         processingHeaderObj = ProcessingHeader()
                         try:
                             if not( basicHeaderObj.read(fp) ): raise IOError
                             if not( systemHeaderObj.read(fp) ): raise IOError
                             if not( radarControllerHeaderObj.read(fp) ): raise IOError
                             if not( processingHeaderObj.read(fp) ): raise IOError
             #                 data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
                             neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
                         except IOError:
                             traceback.print_exc()
                             if msgFlag:
                                 print "[Reading] The file %s is empty or it hasn't enough data" % filename
                             fp.close()
                             return False
                     else:
                         msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
                     fp.close()
                     fileSize = os.path.getsize(filename)
                     currentSize = fileSize - currentPosition
                     if currentSize < neededSize:
                         if msgFlag and (msg != None):
                             print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
                         return False
                     return True
+                def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True):
+                    dateList = []
+                    pathList = []
+                    if not walk:
+                        #pathList.append(path)
+                        multi_path = path.split(',')
+                        for single_path in multi_path:
+                            ok = False
+                            fileList = glob.glob1(single_path, "*"+ext)
+                            for thisFile in fileList:
+                                if not os.path.isfile(os.path.join(single_path, thisFile)):
+                                    continue
+                                if not isRadarFile(thisFile):
+                                    continue
+                                ok = True
+                                thisDate = getDateFromRadarFile(thisFile)
+                                if thisDate not in dateList:
+                                    dateList.append(thisDate)
+                            if ok:
+                                pathList.append(single_path)
+                        return dateList
+                    multi_path = path.split(',')
+                    for single_path in multi_path:
+                        dirList = []
+                        for thisPath in os.listdir(single_path):
+                            if not os.path.isdir(os.path.join(single_path,thisPath)):
+                                continue
+                            if not isRadarFolder(thisPath):
+                                continue
+                            dirList.append(thisPath)
+                        if not dirList:
+                            return dateList
+                        if startDate and endDate:
+                            thisDate = startDate
+                            while(thisDate <= endDate):
+                                year = thisDate.timetuple().tm_year
+                                doy = thisDate.timetuple().tm_yday
+                                matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
+                                if len(matchlist) == 0:
+                                    thisDate += datetime.timedelta(1)
+                                    continue
+                                for match in matchlist:
+                                    pathList.append(os.path.join(single_path,match,expLabel))
+                                    dateList.append(thisDate)
+                                thisDate += datetime.timedelta(1)
+                        else:
+                            for thiDir in dirList:
+                                year = int(folder[1:5])
+                                doy = int(folder[5:8])
+                                thisDate = datetime.date(year,1,1) + datetime.timedelta(doy-1)
+                                pathList.append(os.path.join(single_path,thiDir,expLabel))
+                                dateList.append(thisDate)
+                    return dateList
                 def setup(self,
                             path=None,
                             startDate=None,
                             endDate=None,
                             startTime=datetime.time(0,0,0),
                             endTime=datetime.time(23,59,59),
                             set=None,
                             expLabel = "",
                             ext = None,
                             online = False,
                             delay = 60,
                             walk = True,
                             getblock = False,
                             nTxs = 1):
                     if path == None:
                         raise ValueError, "[Reading] The path is not valid"
                     if ext == None:
                         ext = self.ext
                     if online:
                         print "[Reading] Searching files in online mode..."
                         for nTries in range( self.nTries ):
                             fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
                             if fullpath:
                                 break
                             print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
                             sleep( self.delay )
                         if not(fullpath):
                             print "[Reading] There 'isn't any valid file in %s" % path
                             return None
                         self.year = year
                         self.doy  = doy
                         self.set  = set - 1
                         self.path = path
                         self.foldercounter = foldercounter
                         last_set = None
                     else:
                         print "[Reading] Searching files in offline mode ..."
                         pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            set=set, expLabel=expLabel, ext=ext,
                                                                            walk=walk)
                         if not(pathList):
                             print "[Reading] No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime())
                             sys.exit(-1)
                         self.fileIndex = -1
                         self.pathList = pathList
                         self.filenameList = filenameList
                         file_name = os.path.basename(filenameList[-1])
                         basename, ext = os.path.splitext(file_name)
                         last_set = int(basename[-3:])
                     self.online = online
                     self.delay = delay
                     ext = ext.lower()
                     self.ext = ext
                     self.getByBlock = getblock
                     self.nTxs = int(nTxs)
                     if not(self.setNextFile()):
                         if (startDate!=None) and (endDate!=None):
                             print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                         elif startDate != None:
                             print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
                         else:
                             print "[Reading] No files"
                         sys.exit(-1)
             #        self.updateDataHeader()
                     if last_set != None:
                         self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
                     return
                 def getBasicHeader(self):
                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                     self.dataOut.timeZone = self.basicHeaderObj.timeZone
                     self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
                     self.dataOut.errorCount = self.basicHeaderObj.errorCount
                     self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
                     self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
                 def getFirstHeader(self):
                     raise ValueError, "This method has not been implemented"
                 def getData(self):
                     raise ValueError, "This method has not been implemented"
                 def hasNotDataInBuffer(self):
                     raise ValueError, "This method has not been implemented"
                 def readBlock(self):
                     raise ValueError, "This method has not been implemented"
                 def isEndProcess(self):
                     return self.flagNoMoreFiles
                 def printReadBlocks(self):
                     print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
                 def printTotalBlocks(self):
                     print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
                 def printNumberOfBlock(self):
                     if self.flagIsNewBlock:
                         print "[Reading] Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
                         self.dataOut.blocknow = self.basicHeaderObj.dataBlock
                 def printInfo(self):
                     if self.__printInfo == False:
                         return
                     self.basicHeaderObj.printInfo()
                     self.systemHeaderObj.printInfo()
                     self.radarControllerHeaderObj.printInfo()
                     self.processingHeaderObj.printInfo()
                     self.__printInfo = False
                 def run(self, **kwargs):
                     if not(self.isConfig):
             #            self.dataOut = dataOut
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
             class JRODataWriter(JRODataIO):
                 """
                 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 blockIndex = 0
                 path = None
                 setFile = None
                 profilesPerBlock = None
                 blocksPerFile = None
                 nWriteBlocks = 0
                 def __init__(self, dataOut=None):
                     raise ValueError, "Not implemented"
                 def hasAllDataInBuffer(self):
                     raise ValueError, "Not implemented"
                 def setBlockDimension(self):
                     raise ValueError, "Not implemented"
                 def writeBlock(self):
                     raise ValueError, "No implemented"
                 def putData(self):
                     raise ValueError, "No implemented"
                 def setBasicHeader(self):
                     self.basicHeaderObj.size = self.basicHeaderSize #bytes
                     self.basicHeaderObj.version = self.versionFile
                     self.basicHeaderObj.dataBlock = self.nTotalBlocks
                     utc = numpy.floor(self.dataOut.utctime)
                     milisecond  = (self.dataOut.utctime - utc)* 1000.0
                     self.basicHeaderObj.utc = utc
                     self.basicHeaderObj.miliSecond = milisecond
                     self.basicHeaderObj.timeZone = self.dataOut.timeZone
                     self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
                     self.basicHeaderObj.errorCount = self.dataOut.errorCount
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj self.
                     Return:
                         None
                     """
                     raise ValueError, "No implemented"
                 def __writeFirstHeader(self):
                     """
                     Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
                     Affected:
                         __dataType
                     Return:
                         None
                     """
             #        CALCULAR PARAMETROS
                     sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
                     self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
                     self.basicHeaderObj.write(self.fp)
                     self.systemHeaderObj.write(self.fp)
                     self.radarControllerHeaderObj.write(self.fp)
                     self.processingHeaderObj.write(self.fp)
                     self.dtype = self.dataOut.dtype
                 def __setNewBlock(self):
                     """
                     Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
                     Return:
 :    si no pudo escribir nada
 :    Si escribio el Basic el First Header
                     """
                     if self.fp == None:
                         self.setNextFile()
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
                         self.basicHeaderObj.write(self.fp)
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     """
                     Selecciona el bloque siguiente de datos y los escribe en un file
                     Return:
 :    Si no hizo pudo escribir el bloque de datos
 :    Si no pudo escribir el bloque de datos
                     """
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     return 1
                 def setNextFile(self):
                     """
                     Determina el siguiente file que sera escrito
                     Affected:
                         self.filename
                         self.subfolder
                         self.fp
                         self.setFile
                         self.flagIsNewFile
                     Return:
 :    Si el archivo no puede ser escrito
 :    Si el archivo esta listo para ser escrito
                     """
                     ext = self.ext
                     path = self.path
                     if self.fp != None:
                         self.fp.close()
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         self.setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 self.setFile = -1
                         else:
                             self.setFile = -1 #inicializo mi contador de seteo
                     setFile = self.setFile
                     setFile += 1
                     filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, filen )
                     fp = open( filename,'wb' )
                     self.blockIndex = 0
                     #guardando atributos
                     self.filename = filename
                     self.subfolder = subfolder
                     self.fp = fp
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     self.setFirstHeader()
                     print '[Writing] file: %s'%self.filename
                     self.__writeFirstHeader()
                     return 1
                 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):
                     """
                     Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
                     Inputs:
                         path      :    el path destino en el cual se escribiran los files a crear
                         format    :    formato en el cual sera salvado un file
                         set       :    el setebo del file
                     Return:
 :    Si no realizo un buen seteo
 :    Si realizo un buen seteo
                     """
                     if ext == None:
                         ext = self.ext
                     ext = ext.lower()
                     self.ext = ext
                     self.path = path
                     self.setFile = set - 1
                     self.blocksPerFile = blocksPerFile
                     self.profilesPerBlock = profilesPerBlock
                     self.dataOut = dataOut
                     if not(self.setNextFile()):
                         print "[Writing] There isn't a next file"
                         return 0
                     self.setBlockDimension()
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()

schainpy/model/io/jroIO_heispectra.py +2 -2

             '''
             Created on Jul 3, 2014
             @author: roj-idl71
             '''
             import os, sys
             import time, datetime
             import numpy
             import fnmatch
             import glob
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             try:
                 import pyfits
             except:
                 """
                 """
             from xml.etree.ElementTree import ElementTree
-            from jroIO_base import isDoyFolder, isNumber
+            from jroIO_base import isRadarFolder, isNumber
             from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
             class Fits:
                 name=None
                 format=None
                 array =None
                 data =None
                 thdulist=None
                 prihdr=None
                 hdu=None
                 def __init__(self):
                     pass
                 def setColF(self,name,format,array):
                     self.name=name
                     self.format=format
                     self.array=array
                     a1=numpy.array([self.array],dtype=numpy.float32)
                     self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)
                     return self.col1
             #    def setColP(self,name,format,data):
             #        self.name=name
             #        self.format=format
             #        self.data=data
             #        a2=numpy.array([self.data],dtype=numpy.float32)
             #        self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
             #        return self.col2
                 def writeData(self,name,format,data):
                     self.name=name
                     self.format=format
                     self.data=data
                     a2=numpy.array([self.data],dtype=numpy.float32)
                     self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
                     return self.col2
                 def cFImage(self,idblock,year,month,day,hour,minute,second):
                     self.hdu= pyfits.PrimaryHDU(idblock)
                     self.hdu.header.set("Year",year)
                     self.hdu.header.set("Month",month)
                     self.hdu.header.set("Day",day)
                     self.hdu.header.set("Hour",hour)
                     self.hdu.header.set("Minute",minute)
                     self.hdu.header.set("Second",second)
                     return self.hdu
                 def Ctable(self,colList):
                     self.cols=pyfits.ColDefs(colList)
                     self.tbhdu = pyfits.new_table(self.cols)
                     return self.tbhdu
                 def CFile(self,hdu,tbhdu):
                     self.thdulist=pyfits.HDUList([hdu,tbhdu])
                 def wFile(self,filename):
                     if os.path.isfile(filename):
                         os.remove(filename)
                     self.thdulist.writeto(filename)
             class ParameterConf:
                 ELEMENTNAME = 'Parameter'
                 def __init__(self):
                     self.name = ''
                     self.value = ''
                 def readXml(self, parmElement):
                     self.name = parmElement.get('name')
                     self.value = parmElement.get('value')
                 def getElementName(self):
                     return self.ELEMENTNAME
             class Metadata:
                 def __init__(self, filename):
                     self.parmConfObjList = []
                     self.readXml(filename)
                 def readXml(self, filename):
                     self.projectElement = None
                     self.procUnitConfObjDict = {}
                     self.projectElement = ElementTree().parse(filename)
                     self.project = self.projectElement.tag
                     parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                         parmConfObj.readXml(parmElement)
                         self.parmConfObjList.append(parmConfObj)
             class FitsWriter(Operation):
                 def __init__(self):
                     self.isConfig = False
                     self.dataBlocksPerFile = None
                     self.blockIndex = 0
                     self.flagIsNewFile = 1
                     self.fitsObj = None
                     self.optchar = 'P'
                     self.ext = '.fits'
                     self.setFile = 0
                 def setFitsHeader(self, dataOut, metadatafile):
                     header_data = pyfits.PrimaryHDU()
                     metadata4fits = Metadata(metadatafile)
                     for parameter in metadata4fits.parmConfObjList:
                         parm_name = parameter.name
                         parm_value = parameter.value
             #             if parm_value == 'fromdatadatetime':
             #                 value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
             #             elif parm_value == 'fromdataheights':
             #                 value = dataOut.nHeights
             #             elif parm_value == 'fromdatachannel':
             #                 value = dataOut.nChannels
             #             elif parm_value == 'fromdatasamples':
             #                 value = dataOut.nFFTPoints
             #             else:
             #                 value = parm_value
                         header_data.header[parm_name] = parm_value
                     header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
                     header_data.header['CHANNELLIST'] = str(dataOut.channelList)
                     header_data.header['NCHANNELS'] = dataOut.nChannels
                     #header_data.header['HEIGHTS'] = dataOut.heightList
                     header_data.header['NHEIGHTS'] = dataOut.nHeights
                     header_data.header['IPPSECONDS'] = dataOut.ippSeconds
                     header_data.header['NCOHINT'] = dataOut.nCohInt
                     header_data.header['NINCOHINT'] = dataOut.nIncohInt
                     header_data.header['TIMEZONE'] = dataOut.timeZone
                     header_data.header['NBLOCK'] = self.blockIndex
                     header_data.writeto(self.filename)
                     self.addExtension(dataOut.heightList,'HEIGHTLIST')
                 def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):
                     self.path = path
                     self.dataOut = dataOut
                     self.metadatafile = metadatafile
                     self.dataBlocksPerFile = dataBlocksPerFile
                 def open(self):
                     self.fitsObj = pyfits.open(self.filename, mode='update')
                 def addExtension(self, data, tagname):
                     self.open()
                     extension = pyfits.ImageHDU(data=data, name=tagname)
                     #extension.header['TAG'] = tagname
                     self.fitsObj.append(extension)
                     self.write()
                 def addData(self, data):
                     self.open()
                     extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])
                     extension.header['UTCTIME'] = self.dataOut.utctime
                     self.fitsObj.append(extension)
                     self.blockIndex += 1
                     self.fitsObj[0].header['NBLOCK'] = self.blockIndex
                     self.write()
                 def write(self):
                     self.fitsObj.flush(verbose=True)
                     self.fitsObj.close()
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         self.setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             if isNumber( filen[8:11] ):
                                 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 self.setFile = -1
                         else:
                             self.setFile = -1 #inicializo mi contador de seteo
                     setFile = self.setFile
                     setFile += 1
                     thisFile = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, thisFile )
                     self.blockIndex = 0
                     self.filename = filename
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     print 'Writing the file: %s'%self.filename
                     self.setFitsHeader(self.dataOut, self.metadatafile)
                     return 1
                 def writeBlock(self):
                     self.addData(self.dataOut.data_spc)
                     self.flagIsNewFile = 0
                 def __setNewBlock(self):
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.dataBlocksPerFile:
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     return 1
                 def putData(self):
                     if self.flagIsNewFile:
                         self.setNextFile()
                     self.writeNextBlock()
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
             class FitsReader(ProcessingUnit):
             #     __TIMEZONE = time.timezone
                 expName = None
                 datetimestr = None
                 utc = None
                 nChannels = None
                 nSamples = None
                 dataBlocksPerFile = None
                 comments = None
                 lastUTTime = None
                 header_dict = None
                 data = None
                 data_header_dict = None
                 def __init__(self):
                     self.isConfig = False
                     self.ext = '.fits'
                     self.setFile = 0
                     self.flagNoMoreFiles = 0
                     self.flagIsNewFile = 1
                     self.flagDiscontinuousBlock = None
                     self.fileIndex = None
                     self.filename = None
                     self.fileSize = None
                     self.fitsObj = None
                     self.timeZone = None
                     self.nReadBlocks = 0
                     self.nTotalBlocks = 0
                     self.dataOut = self.createObjByDefault()
                     self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()
                     self.blockIndex = 1
                 def createObjByDefault(self):
                     dataObj = Fits()
                     return dataObj
                 def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):
                     try:
                         fitsObj = pyfits.open(filename,'readonly')
                     except:
                         raise IOError, "The file %s can't be opened" %(filename)
                     header = fitsObj[0].header
                     struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")
                     utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS
                     ltc = utc
                     if useLocalTime:
                         ltc -= time.timezone
                     thisDatetime = datetime.datetime.utcfromtimestamp(ltc)
                     thisTime = thisDatetime.time()
                     if not ((startTime <= thisTime) and (endTime > thisTime)):
                         return None
                     return thisDatetime
                 def __setNextFileOnline(self):
                     raise ValueError, "No implemented"
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print "No more Files"
                             return 0
                         filename = self.filenameList[idFile]
             #            if not(self.__verifyFile(filename)):
             #                continue
                         fileSize = os.path.getsize(filename)
                         fitsObj = pyfits.open(filename,'readonly')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fitsObj = fitsObj
                     self.blockIndex = 0
                     print "Setting the file: %s"%self.filename
                     return 1
                 def __setValuesFromHeader(self):
                     self.dataOut.header = self.header_dict
                     self.dataOut.expName = self.expName
                     self.dataOut.nChannels = self.nChannels
                     self.dataOut.timeZone = self.timeZone
                     self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
                     self.dataOut.comments = self.comments
             #         self.dataOut.timeInterval = self.timeInterval
                     self.dataOut.channelList = self.channelList
                     self.dataOut.heightList = self.heightList
                     self.dataOut.nCohInt = self.nCohInt
                     self.dataOut.nIncohInt = self.nIncohInt
                 def readHeader(self):
                     headerObj = self.fitsObj[0]
                     self.header_dict = headerObj.header
                     if 'EXPNAME' in headerObj.header.keys():
                         self.expName = headerObj.header['EXPNAME']
                     if 'DATATYPE' in headerObj.header.keys():
                         self.dataType = headerObj.header['DATATYPE']
                     self.datetimestr = headerObj.header['DATETIME']
                     channelList = headerObj.header['CHANNELLIST']
                     channelList = channelList.split('[')
                     channelList = channelList[1].split(']')
                     channelList = channelList[0].split(',')
                     channelList = [int(ch) for ch in channelList]
                     self.channelList = channelList
                     self.nChannels = headerObj.header['NCHANNELS']
                     self.nHeights = headerObj.header['NHEIGHTS']
                     self.ippSeconds = headerObj.header['IPPSECONDS']
                     self.nCohInt = headerObj.header['NCOHINT']
                     self.nIncohInt = headerObj.header['NINCOHINT']
                     self.dataBlocksPerFile = headerObj.header['NBLOCK']
                     self.timeZone = headerObj.header['TIMEZONE']
             #         self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                     if 'COMMENT' in headerObj.header.keys():
                         self.comments = headerObj.header['COMMENT']
                     self.readHeightList()
                 def readHeightList(self):
                     self.blockIndex = self.blockIndex + 1
                     obj = self.fitsObj[self.blockIndex]
                     self.heightList = obj.data
                     self.blockIndex = self.blockIndex + 1
                 def readExtension(self):
                     obj = self.fitsObj[self.blockIndex]
                     self.heightList = obj.data
                     self.blockIndex = self.blockIndex + 1
                 def setNextFile(self):
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         return 0
                     self.readHeader()
                     self.__setValuesFromHeader()
                     self.nReadBlocks = 0
             #         self.blockIndex = 1
                     return 1
                 def __searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.fits',
                                         walk=True):
                     pathList = []
                     if not walk:
                         pathList.append(path)
                     else:
                         dirList = []
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
-                            if not isDoyFolder(thisPath):
+                            if not isRadarFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None
                         thisDate = startDate
                         while(thisDate <= endDate):
                             year = thisDate.timetuple().tm_year
                             doy = thisDate.timetuple().tm_yday
                             matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
                             if len(matchlist) == 0:
                                 thisDate += datetime.timedelta(1)
                                 continue
                             for match in matchlist:
                                 pathList.append(os.path.join(path,match,expLabel))
                             thisDate += datetime.timedelta(1)
                     if pathList == []:
                         print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
                         return None, None
                     print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
                     filenameList = []
                     datetimeList = []
                     for i in range(len(pathList)):
                         thisPath = pathList[i]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for thisFile in fileList:
                             filename = os.path.join(thisPath,thisFile)
                             thisDatetime = self.isFileinThisTime(filename, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print "Any file was found for the time range %s - %s" %(startTime, endTime)
                         return None, None
                     print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
                     print
                     for i in range(len(filenameList)):
                         print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def setup(self, path=None,
                             startDate=None,
                             endDate=None,
                             startTime=datetime.time(0,0,0),
                             endTime=datetime.time(23,59,59),
                             set=0,
                             expLabel = "",
                             ext = None,
                             online = False,
                             delay = 60,
                             walk = True):
                     if path == None:
                         raise ValueError, "The path is not valid"
                     if ext == None:
                         ext = self.ext
                     if not(online):
                         print "Searching files in offline mode ..."
                         pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            set=set, expLabel=expLabel, ext=ext,
                                                                            walk=walk)
                         if not(pathList):
                             print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime())
                             sys.exit(-1)
                         self.fileIndex = -1
                         self.pathList = pathList
                         self.filenameList = filenameList
                     self.online = online
                     self.delay = delay
                     ext = ext.lower()
                     self.ext = ext
                     if not(self.setNextFile()):
                         if (startDate!=None) and (endDate!=None):
                             print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                         elif startDate != None:
                             print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
                         else:
                             print "No files"
                         sys.exit(-1)
                 def readBlock(self):
                     dataObj = self.fitsObj[self.blockIndex]
                     self.data = dataObj.data
                     self.data_header_dict = dataObj.header
                     self.utc = self.data_header_dict['UTCTIME']
                     self.flagIsNewFile = 0
                     self.blockIndex += 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def __jumpToLastBlock(self):
                     raise ValueError, "No implemented"
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             self.__rdBasicHeader()
                             return 1
                         print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         sleep( self.delay )
                     return 0
                 def __setNewBlock(self):
                     if self.online:
                         self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         return 1
                     self.lastUTTime = self.utc
                     if self.online:
                         if self.__waitNewBlock():
                             return 1
                     if self.nReadBlocks < self.dataBlocksPerFile:
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.utc - self.lastUTTime
                     self.flagDiscontinuousBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagDiscontinuousBlock = 1
                     return 1
                 def readNextBlock(self):
                     if not(self.__setNewBlock()):
                         return 0
                     if not(self.readBlock()):
                         return 0
                     return 1
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print 'Process finished'
                         return 0
                     self.flagDiscontinuousBlock = 0
                     self.flagIsNewBlock = 0
                     if not(self.readNextBlock()):
                         return 0
                     if self.data == None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.dataOut.data = self.data
                     self.dataOut.data_header = self.data_header_dict
                     self.dataOut.utctime = self.utc
             #         self.dataOut.header = self.header_dict
             #         self.dataOut.expName = self.expName
             #         self.dataOut.nChannels = self.nChannels
             #         self.dataOut.timeZone = self.timeZone
             #         self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
             #         self.dataOut.comments = self.comments
             # #         self.dataOut.timeInterval = self.timeInterval
             #         self.dataOut.channelList = self.channelList
             #         self.dataOut.heightList = self.heightList
                     self.dataOut.flagNoData = False
                     return self.dataOut.data
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
             class SpectraHeisWriter(Operation):
             #    set = None
                 setFile = None
                 idblock = None
                 doypath = None
                 subfolder = None
                 def __init__(self):
                     self.wrObj = Fits()
             #        self.dataOut = dataOut
                     self.nTotalBlocks=0
             #        self.set = None
                     self.setFile = None
                     self.idblock = 0
                     self.wrpath = None
                     self.doypath = None
                     self.subfolder = None
                     self.isConfig = False
                 def isNumber(str):
                     """
                     Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                     Excepciones:
                     Si un determinado string no puede ser convertido a numero
                     Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                     Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                     """
                     try:
                         float( str )
                         return True
                     except:
                         return False
                 def setup(self, dataOut, wrpath):
                     if not(os.path.exists(wrpath)):
                         os.mkdir(wrpath)
                     self.wrpath = wrpath
             #        self.setFile = 0
                     self.dataOut = dataOut
                 def putData(self):
                     name= time.localtime( self.dataOut.utctime)
                     ext=".fits"
                     if self.doypath == None:
                        self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
                        self.doypath = os.path.join( self.wrpath, self.subfolder )
                        os.mkdir(self.doypath)
                     if self.setFile == None:
             #           self.set = self.dataOut.set
                        self.setFile = 0
             #        if self.set != self.dataOut.set:
             ##            self.set = self.dataOut.set
             #            self.setFile = 0
                     #make the filename
                     thisFile = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
                     filename = os.path.join(self.wrpath,self.subfolder, thisFile)
                     idblock = numpy.array([self.idblock],dtype="int64")
                     header=self.wrObj.cFImage(idblock=idblock,
                                             year=time.gmtime(self.dataOut.utctime).tm_year,
                                             month=time.gmtime(self.dataOut.utctime).tm_mon,
                                             day=time.gmtime(self.dataOut.utctime).tm_mday,
                                             hour=time.gmtime(self.dataOut.utctime).tm_hour,
                                             minute=time.gmtime(self.dataOut.utctime).tm_min,
                                             second=time.gmtime(self.dataOut.utctime).tm_sec)
                     c=3E8
                     deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
                     colList = []
                     colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
                     colList.append(colFreq)
                     nchannel=self.dataOut.nChannels
                     for i in range(nchannel):
                         col = self.wrObj.writeData(name="PCh"+str(i+1),
                                                      format=str(self.dataOut.nFFTPoints)+'E',
                                                       data=10*numpy.log10(self.dataOut.data_spc[i,:]))
                         colList.append(col)
                     data=self.wrObj.Ctable(colList=colList)
                     self.wrObj.CFile(header,data)
                     self.wrObj.wFile(filename)
                     #update the setFile
                     self.setFile += 1
                     self.idblock += 1
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()

schainpy/model/io/jroIO_usrp.py +67 -7

             '''
             Created on Jul 3, 2014
             @author: roj-idl71
             '''
             import datetime
             import numpy
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
             from schainpy.model.data.jrodata import Voltage
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
             try:
                 import digital_rf_hdf5
             except:
                 print 'You should install "digital_rf_hdf5" module if you want to read USRP data'
             class USRPReader(ProcessingUnit):
                 '''
                 classdocs
                 '''
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     ProcessingUnit.__init__(self)
                     self.dataOut = Voltage()
                     self.__printInfo = True
                     self.__flagDiscontinuousBlock = False
                     self.__bufferIndex = 9999999
                     self.__ippKm = None
                     self.__codeType = 0
                     self.__nCode = None
                     self.__nBaud = None
                     self.__code = None
                 def __getCurrentSecond(self):
                     return self.__thisUnixSample/self.__sample_rate
                 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
                 def __setFileHeader(self):
                     '''
                     In this method will be initialized every parameter of dataOut object (header, no data)
                     '''
                     self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
                                                                                   txA=0,
                                                                                   txB=0,
                                                                                   nWindows=1,
                                                                                   nHeights=self.__nSamples,
                                                                                   firstHeight=self.__firstHeigth,
                                                                                   deltaHeight=self.__deltaHeigth,
                                                                                   codeType=self.__codeType,
                                                                                   nCode=self.__nCode, nBaud=self.__nBaud,
                                                                                   code = self.__code)
                     self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
                                                                 nProfiles=1024,
                                                                 nChannels=len(self.__channelList),
                                                                 adcResolution=14)
                     self.dataOut.type = "Voltage"
                     self.dataOut.data = None
                     self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
             #        self.dataOut.nChannels = 0
             #        self.dataOut.nHeights = 0
                     self.dataOut.nProfiles = 1
                     self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
                     self.dataOut.channelList = self.__channelList
                     self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
             #        self.dataOut.channelIndexList = None
                     self.dataOut.flagNoData = True
                     #Set to TRUE if the data is discontinuous
                     self.dataOut.flagDiscontinuousBlock = False
                     self.dataOut.utctime = None
                     self.dataOut.timeZone = self.__timezone/60  #timezone like jroheader, difference in minutes between UTC and localtime
                     self.dataOut.dstFlag = 0
                     self.dataOut.errorCount = 0
                     self.dataOut.nCohInt = 1
                     self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
                     self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
                     self.dataOut.flagShiftFFT = False
                     self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
                     #Time interval between profiles
                     #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.__frequency
                     self.dataOut.realtime = self.__online
+                def findDatafiles(self, path, startDate=None, endDate=None):
+                    try:
+                        digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
+                    except:
+                        digitalReadObj = digital_rf_hdf5.read_hdf5(path)
+                    channelNameList = digitalReadObj.get_channels()
+                    metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
+                    sample_rate = metadata_dict['sample_rate'][0]
+                    this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
+                    try:
+                        timezone = this_metadata_file['timezone'].value
+                    except:
+                        timezone = 0
+                    startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(channelNameList[0])/sample_rate - timezone
+                    startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
+                    endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
+                    if not startDate:
+                        startDate = startDatetime.date()
+                    if not endDate:
+                        endDate = endDatatime.date()
+                    dateList = []
+                    thisDatetime = startDatetime
+                    while(thisDatetime<=endDatatime):
+                        thisDate = thisDatetime.date()
+                        if thisDate < startDate:
+                            continue
+                        if thisDate > endDate:
+                            break
+                        dateList.append(thisDate)
+                        thisDatetime += datetime.timedelta(1)
+                    return dateList
                 def setup(self, path = None,
                                 startDate = None,
                                 endDate = None,
                                 startTime = datetime.time(0,0,0),
                                 endTime = datetime.time(23,59,59),
                                 channelList = None,
                                 nSamples = None,
-                                ippKm = None,
+                                ippKm = 60,
                                 online = False,
-                                wait = 60,
+                                delay = 60,
-                                nbuffer = 1024*4):
+                                buffer_size = None,
+                                nbuffer = 1024,
+                                **kwargs):
                     '''
                     In this method we should set all initial parameters.
                     Inputs:
                         path
                         startDate
                         endDate
                         startTime
                         endTime
                         set
                         expLabel
                         ext
                         online
-                        wait
+                        delay
                     '''
+                    if not buffer_size:
+                        buffer_size = nbuffer
                     try:
                         self.digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
                     except:
                         self.digitalReadObj = digital_rf_hdf5.read_hdf5(path)
                     channelNameList = self.digitalReadObj.get_channels()
                     if not channelNameList:
                         raise IOError, "[Reading] The path doesn,t have any files .. "
                     if not channelList:
                         channelList = range(len(channelNameList))
                     ##########  Reading metadata ######################
                     metadata_dict = self.digitalReadObj.get_rf_file_metadata(channelNameList[channelList[0]])
                     self.__sample_rate = metadata_dict['sample_rate'][0]
                     self.__samples_per_file = metadata_dict['samples_per_file'][0]
                     self.__deltaHeigth = 1e6*0.15/self.__sample_rate
                     this_metadata_file = self.digitalReadObj.get_metadata(channelNameList[channelList[0]])
                     self.__frequency = this_metadata_file['center_frequencies'].value
                     try:
                         self.__timezone = this_metadata_file['timezone'].value
                     except:
                         self.__timezone = 0
                     self.__firstHeigth = 0
                     try:
                         codeType = this_metadata_file['codeType'].value
                     except:
                         codeType = 0
                     nCode = 0
                     nBaud = 0
                     code = None
                     if codeType:
                         nCode = this_metadata_file['nCode'].value
                         nBaud = this_metadata_file['nBaud'].value
                         code = this_metadata_file['code'].value
                     if not ippKm:
                         try:
                             #seconds to km
                             ippKm = 1e6*0.15*this_metadata_file['ipp'].value
                         except:
                             ippKm = None
                     ####################################################
                     startUTCSecond = None
                     endUTCSecond = None
                     if startDate:
                         startDatetime = datetime.datetime.combine(startDate, startTime)
                         startUTCSecond = (startDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
                     if endDate:
                         endDatetime = datetime.datetime.combine(endDate, endTime)
                         endUTCSecond = (endDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
                     start_index, end_index = self.digitalReadObj.get_bounds(channelNameList[channelList[0]])
                     if not startUTCSecond:
                         startUTCSecond = start_index/self.__sample_rate
                     if start_index > startUTCSecond*self.__sample_rate:
                         startUTCSecond = start_index/self.__sample_rate
                     if not endUTCSecond:
                         endUTCSecond = end_index/self.__sample_rate
                     if end_index < endUTCSecond*self.__sample_rate:
                         endUTCSecond = end_index/self.__sample_rate
                     if not nSamples:
                         if not ippKm:
                             raise ValueError, "[Reading] nSamples or ippKm should be defined"
                         nSamples = ippKm / (1e6*0.15/self.__sample_rate)
                     channelBoundList = []
                     channelNameListFiltered = []
                     for thisIndexChannel in channelList:
                         thisChannelName =  channelNameList[thisIndexChannel]
                         start_index, end_index = self.digitalReadObj.get_bounds(thisChannelName)
                         channelBoundList.append((start_index, end_index))
                         channelNameListFiltered.append(thisChannelName)
                     self.profileIndex = 0
+                    self.__delay = delay
                     self.__ippKm = ippKm
                     self.__codeType = codeType
                     self.__nCode = nCode
                     self.__nBaud = nBaud
                     self.__code = code
                     self.__datapath = path
                     self.__online = online
                     self.__channelList = channelList
                     self.__channelNameList = channelNameListFiltered
                     self.__channelBoundList = channelBoundList
                     self.__nSamples = nSamples
-                    self.__samples_to_read = nbuffer*nSamples
+                    self.__samples_to_read = buffer_size*nSamples
                     self.__nChannels = len(self.__channelList)
                     self.__startUTCSecond = startUTCSecond
                     self.__endUTCSecond = endUTCSecond
                     self.__timeInterval = 1.0 * self.__samples_to_read/self.__sample_rate #Time interval
                     if online:
             #             self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
                         startUTCSecond = numpy.floor(endUTCSecond)
                     self.__thisUnixSample = int(startUTCSecond*self.__sample_rate) - self.__samples_to_read
                     self.__data_buffer = numpy.zeros((self.__nChannels, self.__samples_to_read), dtype = numpy.complex)
                     self.__setFileHeader()
                     self.isConfig = True
                     print "[Reading] USRP Data was found from %s to %s " %(
                                                                   datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
                                                                   datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
                                                                   )
                     print "[Reading] Starting process from ", datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone), " to ", datetime.datetime.utcfromtimestamp(endUTCSecond - self.__timezone)
                 def __reload(self):
                     if not self.__online:
                         return
             #         print
             #         print "%s not in range [%s, %s]" %(
             #                                           datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
             #                                           datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
             #                                           datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
             #                                           )
                     print "[Reading] reloading metadata ..."
-                    self.digitalReadObj.reload(complete_update=True)
+                    try:
+                        self.digitalReadObj.reload(complete_update=True)
+                    except:
+                        self.digitalReadObj.reload()
                     start_index, end_index = self.digitalReadObj.get_bounds(self.__channelNameList[self.__channelList[0]])
                     if start_index > self.__startUTCSecond*self.__sample_rate:
                         self.__startUTCSecond = 1.0*start_index/self.__sample_rate
                     if end_index > self.__endUTCSecond*self.__sample_rate:
                         self.__endUTCSecond = 1.0*end_index/self.__sample_rate
                         print
                         print "[Reading] New timerange found [%s, %s] " %(
                                                                   datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
                                                                   datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
                                                                   )
                         return True
                     return False
                 def __readNextBlock(self, seconds=30, volt_scale = 218776):
                     '''
                     '''
                     #Set the next data
                     self.__flagDiscontinuousBlock = False
                     self.__thisUnixSample += self.__samples_to_read
                     if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
                         print "[Reading] There are no more data into selected timerange"
                         self.__reload()
                         if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
                             self.__thisUnixSample -=  self.__samples_to_read
                             return False
                     indexChannel = 0
                     dataOk = False
                     for thisChannelName in self.__channelNameList:
                         try:
                             result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
                                                                           self.__samples_to_read,
                                                                           thisChannelName)
                         except IOError, e:
                             #read next profile
                             self.__flagDiscontinuousBlock = True
                             print e
                             break
                         if result.shape[0] != self.__samples_to_read:
                             self.__flagDiscontinuousBlock = True
                             print "[Reading] %s: Too few samples were found, just %d samples" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                                             result.shape[0])
                             break
                         self.__data_buffer[indexChannel,:] = result*volt_scale
                         indexChannel += 1
                         dataOk = True
                     self.__utctime = self.__thisUnixSample/self.__sample_rate
                     if not dataOk:
                         return False
                     print "[Reading] %s: %d samples <> %f sec" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                      self.__samples_to_read,
                                                                      self.__timeInterval)
                     self.__bufferIndex = 0
                     return True
                 def __isBufferEmpty(self):
                     if self.__bufferIndex <= self.__samples_to_read - self.__nSamples:
                         return False
                     return True
                 def getData(self, seconds=30, nTries=5):
                     '''
                     This method gets the data from files and put the data into the dataOut object
                     In addition, increase el the buffer counter in one.
                     Return:
                         data    :    retorna un perfil de voltages (alturas * canales) copiados desde el
                                      buffer. Si no hay mas archivos a leer retorna None.
                     Affected:
                         self.dataOut
                         self.profileIndex
                         self.flagDiscontinuousBlock
                         self.flagIsNewBlock
                     '''
                     err_counter = 0
                     self.dataOut.flagNoData = True
                     if self.__isBufferEmpty():
                         self.__flagDiscontinuousBlock = False
                         while True:
                             if self.__readNextBlock():
                                 break
                             if self.__thisUnixSample > self.__endUTCSecond*self.__sample_rate:
                                 return False
                             if self.__flagDiscontinuousBlock:
                                 print '[Reading] discontinuous block found ... continue with the next block'
                                 continue
                             if not self.__online:
                                 return False
                             err_counter += 1
                             if err_counter > nTries:
                                 return False
                             print '[Reading] waiting %d seconds to read a new block' %seconds
                             sleep(seconds)
                     self.dataOut.data = self.__data_buffer[:,self.__bufferIndex:self.__bufferIndex+self.__nSamples]
                     self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex)/self.__sample_rate
                     self.dataOut.flagNoData = False
                     self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
                     self.__bufferIndex += self.__nSamples
                     self.profileIndex += 1
                     return True
                 def printInfo(self):
                     '''
                     '''
                     if self.__printInfo == False:
                         return
             #         self.systemHeaderObj.printInfo()
             #         self.radarControllerHeaderObj.printInfo()
                     self.__printInfo = False
                 def printNumberOfBlock(self):
                     '''
                     '''
                     print self.profileIndex
                 def run(self, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
-                    self.getData()
+                    self.getData(seconds=self.__delay)
                     return
             class USRPWriter(Operation):
                 '''
                 classdocs
                 '''
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.dataOut = None
                 def setup(self, dataIn, path, blocksPerFile, set=0, ext=None):
                     '''
                     In this method we should set all initial parameters.
                     Input:
                         dataIn        :        Input data will also be outputa data
                     '''
                     self.dataOut = dataIn
                     self.isConfig = True
                     return
                 def run(self, dataIn, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     Inputs:
                         dataIn        :        object with the data
                     '''
                     if not self.isConfig:
                         self.setup(dataIn, **kwargs)
             if __name__ == '__main__':
                 readObj = USRPReader()
                 while True:
                     readObj.run(path='/Volumes/DATA/haystack/passive_radar/')
             #         readObj.printInfo()
                     readObj.printNumberOfBlock()
   No newline at end of file

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages