##// END OF EJS Templates
Python 2to3, Spectra (all operations) working
Python 2to3, Spectra (all operations) working

File last commit:

r1167:1f521b07c958
r1167:1f521b07c958
Show More
DynamicYAML.py
220 lines | 9.0 KiB | text/x-python | PythonLexer
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 '''
Module containing YAML Loader and Dumper for DynamicObjects
as well as built-in data types (numpy, PrecisionTime, datetime, Binary, ...)
$Id$
'''
import yaml
import OrderedYAML
import DynamicObject
import binascii
import numpy as np
import PrecisionTime
import Lookup
import pysvn
def load_defn(source, rev='head', repo=""):
""" Import YAML definition(s) from given 'source' SVN location
with specific revision number 'rev'. Returns a dict of the object
names -> class object instances.
NOTE: Object defns with same name & revision number will conflict /
cause issues (regardless of svn location). """
client = pysvn.Client()
if rev == 'head':
#yaml = client.cat(source)
rev = client.info(repo).revision.number
if source.startswith('http'):
yaml = client.cat("%s?p=%d"%(source, rev))
else:
pysvn_rev = pysvn.Revision(pysvn.opt_revision_kind.number, rev)
yaml = client.cat(source, pysvn_rev)
revision_dict = {\
"__revision_number": rev,
"__revision_id": 'unknown',
"__revision_source": source,
"__revision_tag": 'unknown'}
return DynamicObject.Factory(yaml=yaml, revision_dict=revision_dict).classes
class Loader(OrderedYAML.Loader):
def __init__(self, stream):
OrderedYAML.Loader.__init__(self, stream)
def construct_object(self, node, deep=False):
""" Unresolved tags on mapping nodes come from un-imported YAML definitions - import it """
resolved = node.tag in self.yaml_constructors
resolved = resolved or any([node.tag.startswith(x) for x in self.yaml_multi_constructors])
if isinstance(node, yaml.nodes.MappingNode) and not resolved:
data = self.construct_mapping(self, node)
self.constructed_objects[node] = data
del self.recursive_objects[node]
George Yong
Python 2to3, Spectra (all operations) working
r1167 if '__revision_source' in data:
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 # TODO: Handle password authentication
client = pysvn.Client()
source = data['__revision_source']
if source.startswith('http'):
rev = data['__revision_number']
defn = client.cat("%s?p=%d"%(source, rev))
else:
rev = pysvn.Revision(pysvn.opt_revision_kind.number, data['__revision_number'])
defn = client.cat(source, revision=rev)
DynamicObject.Factory(yaml=defn) # Register the object
constructor = self.yaml_constructors["%s.%s"%(data['__revision_name'], data['__revision_number'])]
return constructor(node)
else:
raise Exception("Cannot load object with tag '%s' - cannot find YAML object definition (no __revision_source included)")
else:
return yaml.Loader.construct_object(self, node, deep=deep)
class Dumper(OrderedYAML.Dumper):
def __init__(self, stream, *args, **kwargs):
OrderedYAML.Dumper.__init__(self, stream, *args, **kwargs)
def represent_dynamic_object(self, obj):
"""
Override the !!python/object:__main__.xxx syntax with
!ObjectName.zzz where zzz is the revision number of the Object obj
"""
state = {}
George Yong
Python 2to3, Spectra (all operations) working
r1167 state.update(list(obj.__dict__.items()))
state.update(list(obj.__class__.meta_attributes.items()))
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 name = obj.getObjectName() # obj.__class__.__name__
revision = obj.getRevisionNumber()
George Yong
Python 2to3, Spectra (all operations) working
r1167 return self.represent_mapping('!%s.%s' % (name, revision), state)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568
# Dtypes to be stored as hex in YAML streams / strings
hex_dtypes = ['float', 'complex', 'half', 'single', 'double']
# Register hex constructors for the numpy / built-in dtypes:
dtypes = Lookup.numpy_dtypes
# Inverse lookup for accessing tags given a class instance:
George Yong
Python 2to3, Spectra (all operations) working
r1167 cls_dtypes = dict([(v,k) for (k,v) in list(dtypes.items())])
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568
# Representer for numpy arrays:
def ndarray_representer(dumper, obj):
#if isinstance(obj, np.ndarray):
tag = 'dtype.'+obj.dtype.type.__name__
hexlify = any([x in tag for x in hex_dtypes])
np_ary = obj
#hex_ary = np.empty(np_ary.shape, dtype=yaml.nodes.ScalarNode)
np_flat, hex_flat = np_ary.flat, [] #hex_ary.flat
George Yong
Python 2to3, Spectra (all operations) working
r1167 hex_flat.append(dumper.represent_sequence('tag:yaml.org,2002:seq', list(np_ary.shape), flow_style=True))
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 if hexlify:
lst = []
for i in range(len(np_flat)):
George Yong
Python 2to3, Spectra (all operations) working
r1167 value = '%s'%(np_flat[i],)
node = dumper.represent_scalar('tag:yaml.org,2002:str', value, style='')
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 lst.append(node)
George Yong
Python 2to3, Spectra (all operations) working
r1167 hex_flat.append(yaml.nodes.SequenceNode('tag:yaml.org,2002:seq', lst, flow_style=True))
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 lst = []
for i in range(len(np_flat)):
George Yong
Python 2to3, Spectra (all operations) working
r1167 if hexlify: value = '%s'%(binascii.hexlify(np_flat[i]),)
else: value = '%s'%(np_flat[i],)
node = dumper.represent_scalar('tag:yaml.org,2002:str', value, style='')
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 if hexlify: lst.append(node)
else: hex_flat.append(node)
George Yong
Python 2to3, Spectra (all operations) working
r1167 if hexlify: hex_flat.append(yaml.nodes.SequenceNode('tag:yaml.org,2002:seq', lst, flow_style=True))
return yaml.nodes.SequenceNode('!%s'%(tag,), hex_flat, flow_style=True)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 Dumper.add_representer(np.ndarray, ndarray_representer)
# Constructor for ndarrays with arbitrary (specified) dtype:
def ndarray_constructor(loader, node, dtype, hexlify=False):
shape = loader.construct_sequence(node.value.pop(0))
np_ary = np.empty(shape, dtype=dtype)
np_flat = np_ary.flat # Flat iterator
if hexlify:
node.value[1].tag = node.tag
node = node.value[1] # only look at hexlified values
for i in range(len(node.value)):
# Over-ride the 'tag:yaml.org,2002:str' tag with correct data type
node.value[i].tag = node.tag
value = loader.construct_object(node.value[i])
#if hexlify:
# value = binascii.unhexlify(value)
# value = np.frombuffer(value, dtype=dtype)
np_flat[i] = value
return np_ary
class __dtype_con:
def __init__(self, tag):
# Whether or not to convert to hex:
hexlify = any([x in tag for x in hex_dtypes])
dtype = dtypes[tag]
# Mutable list containing constructor & representer info
self.fncn_attributes = [tag, hexlify, dtype]
def dtype_constructor(loader, node):
tag, hexlify, dtype = self.fncn_attributes
if isinstance(node, yaml.nodes.SequenceNode):
return ndarray_constructor(loader, node, dtype, hexlify=hexlify)
else: # isinstance(node, yaml.nodes.ScalarNode):
value = loader.construct_scalar(node)
dtype = dtypes[node.tag[1:]]
if hexlify:
value = binascii.unhexlify(value)
value = np.frombuffer(value, dtype=dtype)[0]
else:
value = dtype(value)
return value
def dtype_representer(dumper, obj):
tag, hexlify, dtype = self.fncn_attributes
if isinstance(obj, float): obj = np.float64(obj)
George Yong
Python 2to3, Spectra (all operations) working
r1167 if hexlify: value = '%s'%(binascii.hexlify(obj),)
else: value = '%s'%(obj,)
try: tag = '!%s'%(cls_dtypes[obj.__class__]) # 'dtype.'+obj.__class__.__name__ # bullshit...
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 except KeyError: tag = ''
node = dumper.represent_scalar(tag, value, style='')
return node
self.dtype_constructor = dtype_constructor
self.dtype_representer = dtype_representer
George Yong
Python 2to3, Spectra (all operations) working
r1167 keys = [x for x in list(dtypes.keys()) if x != 'dtype.int' and x != 'dtype.bool']
print(keys)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568
n = len(keys)
George Yong
Python 2to3, Spectra (all operations) working
r1167 print(n)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 i=0
for tag in keys:
dtype = __dtype_con(tag)
dtype_constructor = dtype.dtype_constructor
dtype_representer = dtype.dtype_representer
George Yong
Python 2to3, Spectra (all operations) working
r1167 Loader.add_constructor('!%s'%(tag,), dtype_constructor)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 Dumper.add_representer(dtypes[tag], dtype_representer)
# Precision time constructors & representers:
def ns_rep(dumper, obj):
state = {'second': obj.__dict__['second'], 'nanosecond': obj.__dict__['nanosecond']}
George Yong
Python 2to3, Spectra (all operations) working
r1167 return dumper.represent_mapping('!timestamp_ns', state)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 def ps_rep(dumper, obj):
state = {'second': obj.__dict__['second'], 'picosecond': obj.__dict__['picosecond']}
George Yong
Python 2to3, Spectra (all operations) working
r1167 return dumper.represent_mapping('!timestamp_ps', state)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 def ns_con(loader, node): return PrecisionTime.nsTime(**loader.construct_mapping(node))
def ps_con(loader, node): return PrecisionTime.psTime(**loader.construct_mapping(node))
Dumper.add_representer(PrecisionTime.nsTime, ns_rep)
Dumper.add_representer(PrecisionTime.psTime, ps_rep)
George Yong
Python 2to3, Spectra (all operations) working
r1167 Loader.add_constructor('!timestamp_ns', ns_con)
Loader.add_constructor('!timestamp_nanosecond', ns_con)
Loader.add_constructor('!timestamp_ps', ps_con)
Loader.add_constructor('!timestamp_picosecond', ps_con)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568
# Binary object constructor & representer:
George Yong
Python 2to3, Spectra (all operations) working
r1167 def bin_rep(dumper, obj): return dumper.represent_mapping('!binary', obj.__dict__)
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
r568 def bin_con(loader, node): return DynamicObject.Binary(**loader.construct_mapping(node))
Dumper.add_representer(DynamicObject.Binary, bin_rep)
George Yong
Python 2to3, Spectra (all operations) working
r1167 Loader.add_constructor('!binary', bin_con)