##// END OF EJS Templates
Rewrite controller, remove MPDecorator to units (keep for plots an writers) use of queues for interproc comm instead of zmq, self operations are no longer supported
Rewrite controller, remove MPDecorator to units (keep for plots an writers) use of queues for interproc comm instead of zmq, self operations are no longer supported

File last commit:

r1167:1f521b07c958
r1287:af11e4aac00c
Show More
DynamicYAML.py
220 lines | 9.0 KiB | text/x-python | PythonLexer
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 '''
Module containing YAML Loader and Dumper for DynamicObjects
as well as built-in data types (numpy, PrecisionTime, datetime, Binary, ...)
$Id$
'''
import yaml
import OrderedYAML
import DynamicObject
import binascii
import numpy as np
import PrecisionTime
import Lookup
import pysvn
def load_defn(source, rev='head', repo=""):
""" Import YAML definition(s) from given 'source' SVN location
with specific revision number 'rev'. Returns a dict of the object
names -> class object instances.
NOTE: Object defns with same name & revision number will conflict /
cause issues (regardless of svn location). """
client = pysvn.Client()
if rev == 'head':
#yaml = client.cat(source)
rev = client.info(repo).revision.number
if source.startswith('http'):
yaml = client.cat("%s?p=%d"%(source, rev))
else:
pysvn_rev = pysvn.Revision(pysvn.opt_revision_kind.number, rev)
yaml = client.cat(source, pysvn_rev)
revision_dict = {\
"__revision_number": rev,
"__revision_id": 'unknown',
"__revision_source": source,
"__revision_tag": 'unknown'}
return DynamicObject.Factory(yaml=yaml, revision_dict=revision_dict).classes
class Loader(OrderedYAML.Loader):
def __init__(self, stream):
OrderedYAML.Loader.__init__(self, stream)
def construct_object(self, node, deep=False):
""" Unresolved tags on mapping nodes come from un-imported YAML definitions - import it """
resolved = node.tag in self.yaml_constructors
resolved = resolved or any([node.tag.startswith(x) for x in self.yaml_multi_constructors])
if isinstance(node, yaml.nodes.MappingNode) and not resolved:
data = self.construct_mapping(self, node)
self.constructed_objects[node] = data
del self.recursive_objects[node]
George Yong
Python 2to3, Spectra (all operations) working
r1167 if '__revision_source' in data:
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 # TODO: Handle password authentication
client = pysvn.Client()
source = data['__revision_source']
if source.startswith('http'):
rev = data['__revision_number']
defn = client.cat("%s?p=%d"%(source, rev))
else:
rev = pysvn.Revision(pysvn.opt_revision_kind.number, data['__revision_number'])
defn = client.cat(source, revision=rev)
DynamicObject.Factory(yaml=defn) # Register the object
constructor = self.yaml_constructors["%s.%s"%(data['__revision_name'], data['__revision_number'])]
return constructor(node)
else:
raise Exception("Cannot load object with tag '%s' - cannot find YAML object definition (no __revision_source included)")
else:
return yaml.Loader.construct_object(self, node, deep=deep)
class Dumper(OrderedYAML.Dumper):
def __init__(self, stream, *args, **kwargs):
OrderedYAML.Dumper.__init__(self, stream, *args, **kwargs)
def represent_dynamic_object(self, obj):
"""
Override the !!python/object:__main__.xxx syntax with
!ObjectName.zzz where zzz is the revision number of the Object obj
"""
state = {}
George Yong
Python 2to3, Spectra (all operations) working
r1167 state.update(list(obj.__dict__.items()))
state.update(list(obj.__class__.meta_attributes.items()))
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 name = obj.getObjectName() # obj.__class__.__name__
revision = obj.getRevisionNumber()
George Yong
Python 2to3, Spectra (all operations) working
r1167 return self.represent_mapping('!%s.%s' % (name, revision), state)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568
# Dtypes to be stored as hex in YAML streams / strings
hex_dtypes = ['float', 'complex', 'half', 'single', 'double']
# Register hex constructors for the numpy / built-in dtypes:
dtypes = Lookup.numpy_dtypes
# Inverse lookup for accessing tags given a class instance:
George Yong
Python 2to3, Spectra (all operations) working
r1167 cls_dtypes = dict([(v,k) for (k,v) in list(dtypes.items())])
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568
# Representer for numpy arrays:
def ndarray_representer(dumper, obj):
#if isinstance(obj, np.ndarray):
tag = 'dtype.'+obj.dtype.type.__name__
hexlify = any([x in tag for x in hex_dtypes])
np_ary = obj
#hex_ary = np.empty(np_ary.shape, dtype=yaml.nodes.ScalarNode)
np_flat, hex_flat = np_ary.flat, [] #hex_ary.flat
George Yong
Python 2to3, Spectra (all operations) working
r1167 hex_flat.append(dumper.represent_sequence('tag:yaml.org,2002:seq', list(np_ary.shape), flow_style=True))
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 if hexlify:
lst = []
for i in range(len(np_flat)):
George Yong
Python 2to3, Spectra (all operations) working
r1167 value = '%s'%(np_flat[i],)
node = dumper.represent_scalar('tag:yaml.org,2002:str', value, style='')
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 lst.append(node)
George Yong
Python 2to3, Spectra (all operations) working
r1167 hex_flat.append(yaml.nodes.SequenceNode('tag:yaml.org,2002:seq', lst, flow_style=True))
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 lst = []
for i in range(len(np_flat)):
George Yong
Python 2to3, Spectra (all operations) working
r1167 if hexlify: value = '%s'%(binascii.hexlify(np_flat[i]),)
else: value = '%s'%(np_flat[i],)
node = dumper.represent_scalar('tag:yaml.org,2002:str', value, style='')
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 if hexlify: lst.append(node)
else: hex_flat.append(node)
George Yong
Python 2to3, Spectra (all operations) working
r1167 if hexlify: hex_flat.append(yaml.nodes.SequenceNode('tag:yaml.org,2002:seq', lst, flow_style=True))
return yaml.nodes.SequenceNode('!%s'%(tag,), hex_flat, flow_style=True)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 Dumper.add_representer(np.ndarray, ndarray_representer)
# Constructor for ndarrays with arbitrary (specified) dtype:
def ndarray_constructor(loader, node, dtype, hexlify=False):
shape = loader.construct_sequence(node.value.pop(0))
np_ary = np.empty(shape, dtype=dtype)
np_flat = np_ary.flat # Flat iterator
if hexlify:
node.value[1].tag = node.tag
node = node.value[1] # only look at hexlified values
for i in range(len(node.value)):
# Over-ride the 'tag:yaml.org,2002:str' tag with correct data type
node.value[i].tag = node.tag
value = loader.construct_object(node.value[i])
#if hexlify:
# value = binascii.unhexlify(value)
# value = np.frombuffer(value, dtype=dtype)
np_flat[i] = value
return np_ary
class __dtype_con:
def __init__(self, tag):
# Whether or not to convert to hex:
hexlify = any([x in tag for x in hex_dtypes])
dtype = dtypes[tag]
# Mutable list containing constructor & representer info
self.fncn_attributes = [tag, hexlify, dtype]
def dtype_constructor(loader, node):
tag, hexlify, dtype = self.fncn_attributes
if isinstance(node, yaml.nodes.SequenceNode):
return ndarray_constructor(loader, node, dtype, hexlify=hexlify)
else: # isinstance(node, yaml.nodes.ScalarNode):
value = loader.construct_scalar(node)
dtype = dtypes[node.tag[1:]]
if hexlify:
value = binascii.unhexlify(value)
value = np.frombuffer(value, dtype=dtype)[0]
else:
value = dtype(value)
return value
def dtype_representer(dumper, obj):
tag, hexlify, dtype = self.fncn_attributes
if isinstance(obj, float): obj = np.float64(obj)
George Yong
Python 2to3, Spectra (all operations) working
r1167 if hexlify: value = '%s'%(binascii.hexlify(obj),)
else: value = '%s'%(obj,)
try: tag = '!%s'%(cls_dtypes[obj.__class__]) # 'dtype.'+obj.__class__.__name__ # bullshit...
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 except KeyError: tag = ''
node = dumper.represent_scalar(tag, value, style='')
return node
self.dtype_constructor = dtype_constructor
self.dtype_representer = dtype_representer
George Yong
Python 2to3, Spectra (all operations) working
r1167 keys = [x for x in list(dtypes.keys()) if x != 'dtype.int' and x != 'dtype.bool']
print(keys)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568
n = len(keys)
George Yong
Python 2to3, Spectra (all operations) working
r1167 print(n)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 i=0
for tag in keys:
dtype = __dtype_con(tag)
dtype_constructor = dtype.dtype_constructor
dtype_representer = dtype.dtype_representer
George Yong
Python 2to3, Spectra (all operations) working
r1167 Loader.add_constructor('!%s'%(tag,), dtype_constructor)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 Dumper.add_representer(dtypes[tag], dtype_representer)
# Precision time constructors & representers:
def ns_rep(dumper, obj):
state = {'second': obj.__dict__['second'], 'nanosecond': obj.__dict__['nanosecond']}
George Yong
Python 2to3, Spectra (all operations) working
r1167 return dumper.represent_mapping('!timestamp_ns', state)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 def ps_rep(dumper, obj):
state = {'second': obj.__dict__['second'], 'picosecond': obj.__dict__['picosecond']}
George Yong
Python 2to3, Spectra (all operations) working
r1167 return dumper.represent_mapping('!timestamp_ps', state)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 def ns_con(loader, node): return PrecisionTime.nsTime(**loader.construct_mapping(node))
def ps_con(loader, node): return PrecisionTime.psTime(**loader.construct_mapping(node))
Dumper.add_representer(PrecisionTime.nsTime, ns_rep)
Dumper.add_representer(PrecisionTime.psTime, ps_rep)
George Yong
Python 2to3, Spectra (all operations) working
r1167 Loader.add_constructor('!timestamp_ns', ns_con)
Loader.add_constructor('!timestamp_nanosecond', ns_con)
Loader.add_constructor('!timestamp_ps', ps_con)
Loader.add_constructor('!timestamp_picosecond', ps_con)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568
# Binary object constructor & representer:
George Yong
Python 2to3, Spectra (all operations) working
r1167 def bin_rep(dumper, obj): return dumper.represent_mapping('!binary', obj.__dict__)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 def bin_con(loader, node): return DynamicObject.Binary(**loader.construct_mapping(node))
Dumper.add_representer(DynamicObject.Binary, bin_rep)
George Yong
Python 2to3, Spectra (all operations) working
r1167 Loader.add_constructor('!binary', bin_con)