DynamicYAML.py
220 lines
| 9.0 KiB
| text/x-python
|
PythonLexer
|
r568 | ''' | ||
Module containing YAML Loader and Dumper for DynamicObjects | ||||
as well as built-in data types (numpy, PrecisionTime, datetime, Binary, ...) | ||||
$Id$ | ||||
''' | ||||
import yaml | ||||
import OrderedYAML | ||||
import DynamicObject | ||||
import binascii | ||||
import numpy as np | ||||
import PrecisionTime | ||||
import Lookup | ||||
import pysvn | ||||
def load_defn(source, rev='head', repo=""): | ||||
""" Import YAML definition(s) from given 'source' SVN location | ||||
with specific revision number 'rev'. Returns a dict of the object | ||||
names -> class object instances. | ||||
NOTE: Object defns with same name & revision number will conflict / | ||||
cause issues (regardless of svn location). """ | ||||
client = pysvn.Client() | ||||
if rev == 'head': | ||||
#yaml = client.cat(source) | ||||
rev = client.info(repo).revision.number | ||||
if source.startswith('http'): | ||||
yaml = client.cat("%s?p=%d"%(source, rev)) | ||||
else: | ||||
pysvn_rev = pysvn.Revision(pysvn.opt_revision_kind.number, rev) | ||||
yaml = client.cat(source, pysvn_rev) | ||||
revision_dict = {\ | ||||
"__revision_number": rev, | ||||
"__revision_id": 'unknown', | ||||
"__revision_source": source, | ||||
"__revision_tag": 'unknown'} | ||||
return DynamicObject.Factory(yaml=yaml, revision_dict=revision_dict).classes | ||||
class Loader(OrderedYAML.Loader): | ||||
def __init__(self, stream): | ||||
OrderedYAML.Loader.__init__(self, stream) | ||||
def construct_object(self, node, deep=False): | ||||
""" Unresolved tags on mapping nodes come from un-imported YAML definitions - import it """ | ||||
resolved = node.tag in self.yaml_constructors | ||||
resolved = resolved or any([node.tag.startswith(x) for x in self.yaml_multi_constructors]) | ||||
if isinstance(node, yaml.nodes.MappingNode) and not resolved: | ||||
data = self.construct_mapping(self, node) | ||||
self.constructed_objects[node] = data | ||||
del self.recursive_objects[node] | ||||
|
r1167 | if '__revision_source' in data: | ||
|
r568 | # TODO: Handle password authentication | ||
client = pysvn.Client() | ||||
source = data['__revision_source'] | ||||
if source.startswith('http'): | ||||
rev = data['__revision_number'] | ||||
defn = client.cat("%s?p=%d"%(source, rev)) | ||||
else: | ||||
rev = pysvn.Revision(pysvn.opt_revision_kind.number, data['__revision_number']) | ||||
defn = client.cat(source, revision=rev) | ||||
DynamicObject.Factory(yaml=defn) # Register the object | ||||
constructor = self.yaml_constructors["%s.%s"%(data['__revision_name'], data['__revision_number'])] | ||||
return constructor(node) | ||||
else: | ||||
raise Exception("Cannot load object with tag '%s' - cannot find YAML object definition (no __revision_source included)") | ||||
else: | ||||
return yaml.Loader.construct_object(self, node, deep=deep) | ||||
class Dumper(OrderedYAML.Dumper): | ||||
def __init__(self, stream, *args, **kwargs): | ||||
OrderedYAML.Dumper.__init__(self, stream, *args, **kwargs) | ||||
def represent_dynamic_object(self, obj): | ||||
""" | ||||
Override the !!python/object:__main__.xxx syntax with | ||||
!ObjectName.zzz where zzz is the revision number of the Object obj | ||||
""" | ||||
state = {} | ||||
|
r1167 | state.update(list(obj.__dict__.items())) | ||
state.update(list(obj.__class__.meta_attributes.items())) | ||||
|
r568 | name = obj.getObjectName() # obj.__class__.__name__ | ||
revision = obj.getRevisionNumber() | ||||
|
r1167 | return self.represent_mapping('!%s.%s' % (name, revision), state) | ||
|
r568 | |||
# Dtypes to be stored as hex in YAML streams / strings | ||||
hex_dtypes = ['float', 'complex', 'half', 'single', 'double'] | ||||
# Register hex constructors for the numpy / built-in dtypes: | ||||
dtypes = Lookup.numpy_dtypes | ||||
# Inverse lookup for accessing tags given a class instance: | ||||
|
r1167 | cls_dtypes = dict([(v,k) for (k,v) in list(dtypes.items())]) | ||
|
r568 | |||
# Representer for numpy arrays: | ||||
def ndarray_representer(dumper, obj): | ||||
#if isinstance(obj, np.ndarray): | ||||
tag = 'dtype.'+obj.dtype.type.__name__ | ||||
hexlify = any([x in tag for x in hex_dtypes]) | ||||
np_ary = obj | ||||
#hex_ary = np.empty(np_ary.shape, dtype=yaml.nodes.ScalarNode) | ||||
np_flat, hex_flat = np_ary.flat, [] #hex_ary.flat | ||||
|
r1167 | hex_flat.append(dumper.represent_sequence('tag:yaml.org,2002:seq', list(np_ary.shape), flow_style=True)) | ||
|
r568 | if hexlify: | ||
lst = [] | ||||
for i in range(len(np_flat)): | ||||
|
r1167 | value = '%s'%(np_flat[i],) | ||
node = dumper.represent_scalar('tag:yaml.org,2002:str', value, style='') | ||||
|
r568 | lst.append(node) | ||
|
r1167 | hex_flat.append(yaml.nodes.SequenceNode('tag:yaml.org,2002:seq', lst, flow_style=True)) | ||
|
r568 | lst = [] | ||
for i in range(len(np_flat)): | ||||
|
r1167 | if hexlify: value = '%s'%(binascii.hexlify(np_flat[i]),) | ||
else: value = '%s'%(np_flat[i],) | ||||
node = dumper.represent_scalar('tag:yaml.org,2002:str', value, style='') | ||||
|
r568 | if hexlify: lst.append(node) | ||
else: hex_flat.append(node) | ||||
|
r1167 | if hexlify: hex_flat.append(yaml.nodes.SequenceNode('tag:yaml.org,2002:seq', lst, flow_style=True)) | ||
return yaml.nodes.SequenceNode('!%s'%(tag,), hex_flat, flow_style=True) | ||||
|
r568 | Dumper.add_representer(np.ndarray, ndarray_representer) | ||
# Constructor for ndarrays with arbitrary (specified) dtype: | ||||
def ndarray_constructor(loader, node, dtype, hexlify=False): | ||||
shape = loader.construct_sequence(node.value.pop(0)) | ||||
np_ary = np.empty(shape, dtype=dtype) | ||||
np_flat = np_ary.flat # Flat iterator | ||||
if hexlify: | ||||
node.value[1].tag = node.tag | ||||
node = node.value[1] # only look at hexlified values | ||||
for i in range(len(node.value)): | ||||
# Over-ride the 'tag:yaml.org,2002:str' tag with correct data type | ||||
node.value[i].tag = node.tag | ||||
value = loader.construct_object(node.value[i]) | ||||
#if hexlify: | ||||
# value = binascii.unhexlify(value) | ||||
# value = np.frombuffer(value, dtype=dtype) | ||||
np_flat[i] = value | ||||
return np_ary | ||||
class __dtype_con: | ||||
def __init__(self, tag): | ||||
# Whether or not to convert to hex: | ||||
hexlify = any([x in tag for x in hex_dtypes]) | ||||
dtype = dtypes[tag] | ||||
# Mutable list containing constructor & representer info | ||||
self.fncn_attributes = [tag, hexlify, dtype] | ||||
def dtype_constructor(loader, node): | ||||
tag, hexlify, dtype = self.fncn_attributes | ||||
if isinstance(node, yaml.nodes.SequenceNode): | ||||
return ndarray_constructor(loader, node, dtype, hexlify=hexlify) | ||||
else: # isinstance(node, yaml.nodes.ScalarNode): | ||||
value = loader.construct_scalar(node) | ||||
dtype = dtypes[node.tag[1:]] | ||||
if hexlify: | ||||
value = binascii.unhexlify(value) | ||||
value = np.frombuffer(value, dtype=dtype)[0] | ||||
else: | ||||
value = dtype(value) | ||||
return value | ||||
def dtype_representer(dumper, obj): | ||||
tag, hexlify, dtype = self.fncn_attributes | ||||
if isinstance(obj, float): obj = np.float64(obj) | ||||
|
r1167 | if hexlify: value = '%s'%(binascii.hexlify(obj),) | ||
else: value = '%s'%(obj,) | ||||
try: tag = '!%s'%(cls_dtypes[obj.__class__]) # 'dtype.'+obj.__class__.__name__ # bullshit... | ||||
|
r568 | except KeyError: tag = '' | ||
node = dumper.represent_scalar(tag, value, style='') | ||||
return node | ||||
self.dtype_constructor = dtype_constructor | ||||
self.dtype_representer = dtype_representer | ||||
|
r1167 | keys = [x for x in list(dtypes.keys()) if x != 'dtype.int' and x != 'dtype.bool'] | ||
print(keys) | ||||
|
r568 | |||
n = len(keys) | ||||
|
r1167 | print(n) | ||
|
r568 | i=0 | ||
for tag in keys: | ||||
dtype = __dtype_con(tag) | ||||
dtype_constructor = dtype.dtype_constructor | ||||
dtype_representer = dtype.dtype_representer | ||||
|
r1167 | Loader.add_constructor('!%s'%(tag,), dtype_constructor) | ||
|
r568 | Dumper.add_representer(dtypes[tag], dtype_representer) | ||
# Precision time constructors & representers: | ||||
def ns_rep(dumper, obj): | ||||
state = {'second': obj.__dict__['second'], 'nanosecond': obj.__dict__['nanosecond']} | ||||
|
r1167 | return dumper.represent_mapping('!timestamp_ns', state) | ||
|
r568 | def ps_rep(dumper, obj): | ||
state = {'second': obj.__dict__['second'], 'picosecond': obj.__dict__['picosecond']} | ||||
|
r1167 | return dumper.represent_mapping('!timestamp_ps', state) | ||
|
r568 | def ns_con(loader, node): return PrecisionTime.nsTime(**loader.construct_mapping(node)) | ||
def ps_con(loader, node): return PrecisionTime.psTime(**loader.construct_mapping(node)) | ||||
Dumper.add_representer(PrecisionTime.nsTime, ns_rep) | ||||
Dumper.add_representer(PrecisionTime.psTime, ps_rep) | ||||
|
r1167 | Loader.add_constructor('!timestamp_ns', ns_con) | ||
Loader.add_constructor('!timestamp_nanosecond', ns_con) | ||||
Loader.add_constructor('!timestamp_ps', ps_con) | ||||
Loader.add_constructor('!timestamp_picosecond', ps_con) | ||||
|
r568 | |||
# Binary object constructor & representer: | ||||
|
r1167 | def bin_rep(dumper, obj): return dumper.represent_mapping('!binary', obj.__dict__) | ||
|
r568 | def bin_con(loader, node): return DynamicObject.Binary(**loader.construct_mapping(node)) | ||
Dumper.add_representer(DynamicObject.Binary, bin_rep) | ||||
|
r1167 | Loader.add_constructor('!binary', bin_con) | ||