DataManager.py @ b4d05bd

ESS_GUIESS_GUI_batch_fittingESS_GUI_bumps_abstractionESS_GUI_iss1116ESS_GUI_iss879ESS_GUI_openclESS_GUI_orderingESS_GUI_sync_sascalc

Last change on this file since b4d05bd was 0261bc1, checked in by Piotr Rozyczko <rozyczko@…>, 7 years ago
Added unit tests for recent functionality
Property mode set to `100755`
File size: 15.8 KB

Rev	Line
[dc5ef15]	1	################################################################################
	2	#This software was developed by the University of Tennessee as part of the
	3	#Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
	4	#project funded by the US National Science Foundation.
	5	#
	6	#See the license text in license.txt
	7	#
	8	#copyright 2010, University of Tennessee
	9	################################################################################
	10	"""
	11	This module manages all data loaded into the application. Data_manager makes
	12	available all data loaded for the current perspective.
	13
	14	All modules "creating Data" posts their data to data_manager .
	15	Data_manager make these new data available for all other perspectives.
	16	"""
	17	#
	18	# REDUNDANT CLASS!!!
	19	# ALL THE FUNCTIONALITY IS BEING MOVED TO GuiManager.py
	20	#
	21	import os
	22	import copy
	23	import logging
	24	import json
	25	import time
[b3e8629]	26	from io import StringIO
[dc5ef15]	27	import numpy as np
	28
	29	from sas.qtgui.Plotting.PlotterData import Data1D
	30	from sas.qtgui.Plotting.PlotterData import Data2D
	31	from sas.qtgui.Plotting.Plottables import Plottable
	32	from sas.qtgui.Plotting.Plottables import PlottableTheory1D
	33	from sas.qtgui.Plotting.Plottables import PlottableFit1D
	34	from sas.qtgui.Plotting.Plottables import Text
	35	from sas.qtgui.Plotting.Plottables import Chisq
	36	from sas.qtgui.Plotting.Plottables import View
	37
	38	from sas.qtgui.Utilities import GuiUtils
	39	from sas.qtgui.MainWindow.DataState import DataState
	40	import sas.sascalc.dataloader.data_info as DataInfo
	41
	42	# used for import/export
	43	from sas.sascalc.dataloader.data_info import Sample, Source, Vector
	44
	45	class DataManager(object):
	46	"""
	47	Manage a list of data
	48	"""
	49	def __init__(self):
	50	"""
	51	Store opened path and data object created at the loading time
	52	:param auto_plot: if True the datamanager sends data to plotting
	53	plugin.
	54	:param auto_set_data: if True the datamanager sends to the current
	55	perspective
	56	"""
	57	self.stored_data = {}
	58	self.message = ""
	59	self.data_name_dict = {}
	60	self.count = 0
	61	self.list_of_id = []
	62	self.time_stamp = time.time()
	63
	64	def __str__(self):
	65	_str = ""
	66	_str += "No of states is %s \n" % str(len(self.stored_data))
	67	n_count = 0
[b3e8629]	68	for value in list(self.stored_data.values()):
[dc5ef15]	69	n_count += 1
	70	_str += "State No %s \n" % str(n_count)
	71	_str += str(value) + "\n"
	72	return _str
	73
	74	def create_gui_data(self, data, path=None):
	75	"""
	76	Receive data from loader and create a data to use for guiframe
	77	"""
	78
	79	if issubclass(Data2D, data.__class__):
	80	new_plot = Data2D(image=None, err_image=None)
	81	else:
	82	new_plot = Data1D(x=[], y=[], dx=None, dy=None)
	83
	84	new_plot.copy_from_datainfo(data)
	85	data.clone_without_data(clone=new_plot)
	86	#creating a name for data
	87	title = ""
[f4a1433]	88	file_name = os.path.basename(path) if path is not None else os.path.basename(data.filename)
[dc5ef15]	89	if file_name:
	90	name = file_name
	91	elif data.run:
	92	name = data.run[0]
	93	else:
	94	name = "data"
	95	name = self.rename(name)
	96	#find title
	97	if data.title.strip():
	98	title = data.title
	99	if title.strip() == "":
	100	title = file_name
	101
[f4a1433]	102	stripped_filename = new_plot.filename.strip()
	103	if not stripped_filename:
	104	new_plot.filename = file_name
	105	if stripped_filename != os.path.basename(stripped_filename):
[dc5ef15]	106	new_plot.filename = file_name
	107
	108	new_plot.name = name
	109	new_plot.title = title
	110	## allow to highlight data when plotted
	111	new_plot.interactive = True
	112	## when 2 data have the same id override the 1 st plotted
	113	self.time_stamp += 1
	114	new_plot.id = str(name) + str(self.time_stamp)
	115	##group_id specify on which panel to plot this data
	116	new_plot.group_id = str(name) + str(self.time_stamp)
	117	new_plot.is_data = True
	118	new_plot.path = path
	119	new_plot.list_group_id = []
	120	##post data to plot
	121	# plot data
	122	return new_plot
	123
	124	def rename(self, name):
	125	"""
	126	rename data
	127	"""
[0261bc1]	128	# name of the data allow to differentiate data when plotted
	129	try:
	130	name = GuiUtils.parseName(name=name, expression="_")
	131	except TypeError:
	132	# bad name sent to rename
	133	return None
[dc5ef15]	134
	135	max_char = name.find("[")
	136	if max_char < 0:
	137	max_char = len(name)
	138	name = name[0:max_char]
	139
	140	if name not in self.data_name_dict:
	141	self.data_name_dict[name] = 0
	142	else:
	143	self.data_name_dict[name] += 1
	144	name = name + " [" + str(self.data_name_dict[name]) + "]"
	145	return name
	146
	147
	148	def add_data(self, data_list):
	149	"""
	150	receive a list of
	151	"""
[b3e8629]	152	for id, data in data_list.items():
[dc5ef15]	153	if id in self.stored_data:
	154	msg = "Data manager already stores %s" % str(data.name)
	155	msg += ""
	156	logging.info(msg)
	157	data_state = self.stored_data[id]
	158	data_state.data = data
	159	else:
	160	data_state = DataState(data)
	161	data_state.id = id
	162	data_state.path = data.path
	163	self.stored_data[id] = data_state
	164
	165	def update_data(self, prev_data, new_data):
	166	"""
	167	"""
[b3e8629]	168	if prev_data.id not in list(self.stored_data.keys()):
[dc5ef15]	169	return None, {}
	170	data_state = self.stored_data[prev_data.id]
	171	self.stored_data[new_data.id] = data_state.clone()
	172	self.stored_data[new_data.id].data = new_data
[b3e8629]	173	if prev_data.id in list(self.stored_data.keys()):
[dc5ef15]	174	del self.stored_data[prev_data.id]
	175	return prev_data.id, {new_data.id: self.stored_data[new_data.id]}
	176
	177	def update_theory(self, theory, data_id=None, state=None):
	178	"""
	179	"""
	180	uid = data_id
	181	if data_id is None and theory is not None:
	182	uid = theory.id
[b3e8629]	183	if uid in list(self.stored_data.keys()):
[dc5ef15]	184	data_state = self.stored_data[uid]
	185	else:
	186	data_state = DataState()
	187	data_state.uid = uid
	188	data_state.set_theory(theory_data=theory, theory_state=state)
	189	self.stored_data[uid] = data_state
	190	return {uid: self.stored_data[uid]}
	191
	192
	193	def get_message(self):
	194	"""
	195	return message
	196	"""
	197	return self.message
	198
	199	def get_by_id(self, id_list=None):
	200	"""
	201	"""
	202	_selected_data = {}
	203	_selected_theory_list = {}
	204	if id_list is None:
	205	return
	206	for d_id in self.stored_data:
	207	for search_id in id_list:
	208	data_state = self.stored_data[d_id]
	209	data = data_state.data
	210	theory_list = data_state.get_theory()
	211	if search_id == d_id:
	212	_selected_data[search_id] = data
[b3e8629]	213	if search_id in list(theory_list.keys()):
[dc5ef15]	214	_selected_theory_list[search_id] = theory_list[search_id]
	215
	216	return _selected_data, _selected_theory_list
	217
	218
	219	def freeze(self, theory_id):
	220	"""
	221	"""
[b3e8629]	222	return self.freeze_theory(list(self.stored_data.keys()), theory_id)
[dc5ef15]	223
	224	def freeze_theory(self, data_id, theory_id):
	225	"""
	226	"""
	227	selected_theory = {}
	228	for d_id in data_id:
	229	if d_id in self.stored_data:
	230	data_state = self.stored_data[d_id]
	231	theory_list = data_state.get_theory()
	232	for t_id in theory_id:
[b3e8629]	233	if t_id in list(theory_list.keys()):
[dc5ef15]	234	theory_data, theory_state = theory_list[t_id]
	235	new_theory = copy.deepcopy(theory_data)
	236	new_theory.id = time.time()
	237	new_theory.is_data = True
	238	new_theory.name += '_@' + \
	239	str(new_theory.id)[7:-1].replace('.', '')
	240	new_theory.title = new_theory.name
	241	new_theory.label = new_theory.name
	242	selected_theory[new_theory.id] = DataState(new_theory)
	243	self.stored_data[new_theory.id] = \
	244	selected_theory[new_theory.id]
	245
	246	return selected_theory
	247
	248
	249	def delete_data(self, data_id, theory_id=None, delete_all=False):
	250	"""
	251	"""
	252	for d_id in data_id:
[b3e8629]	253	if d_id in list(self.stored_data.keys()):
[dc5ef15]	254	data_state = self.stored_data[d_id]
	255	if data_state.data.name in self.data_name_dict:
	256	del self.data_name_dict[data_state.data.name]
	257	del self.stored_data[d_id]
	258
	259	self.delete_theory(data_id, theory_id)
	260	if delete_all:
	261	self.stored_data = {}
	262	self.data_name_dict = {}
	263
	264	def delete_theory(self, data_id, theory_id):
	265	"""
	266	"""
	267	for d_id in data_id:
	268	if d_id in self.stored_data:
	269	data_state = self.stored_data[d_id]
	270	theory_list = data_state.get_theory()
[b3e8629]	271	if theory_id in list(theory_list.keys()):
[dc5ef15]	272	del theory_list[theory_id]
	273	#del pure theory
	274	self.delete_by_id(theory_id)
	275
	276	def delete_by_id(self, id_list=None):
	277	"""
	278	save data and path
	279	"""
	280	for id in id_list:
	281	if id in self.stored_data:
	282	del self.stored_data[id]
	283
	284	def get_by_name(self, name_list=None):
	285	"""
	286	return a list of data given a list of data names
	287	"""
	288	_selected_data = {}
	289	for selected_name in name_list:
[b3e8629]	290	for id, data_state in self.stored_data.items():
[dc5ef15]	291	if data_state.data.name == selected_name:
	292	_selected_data[id] = data_state.data
	293	return _selected_data
	294
	295	def delete_by_name(self, name_list=None):
	296	"""
	297	save data and path
	298	"""
	299	for selected_name in name_list:
[b3e8629]	300	for id, data_state in self.stored_data.items():
[dc5ef15]	301	if data_state.data.name == selected_name:
	302	del self.stored_data[id]
	303
[1420066]	304	def update_stored_data(self, name_list=None):
	305	""" update stored data after deleting files in Data Explorer """
	306	for selected_name in name_list:
[7d9c83c]	307	# Take the copy of current, possibly shorter stored_data dict
	308	stored_data = copy.deepcopy(self.stored_data)
[b3e8629]	309	for idx in list(stored_data.keys()):
[1420066]	310	if str(selected_name) in str(idx):
	311	del self.stored_data[idx]
	312
[dc5ef15]	313	def get_data_state(self, data_id):
	314	"""
	315	Send list of selected data
	316	"""
	317	_selected_data_state = {}
	318	for id in data_id:
[b3e8629]	319	if id in list(self.stored_data.keys()):
[dc5ef15]	320	_selected_data_state[id] = self.stored_data[id]
	321	return _selected_data_state
	322
	323	def get_all_data(self):
	324	"""
	325	return list of all available data
	326	"""
	327	return self.stored_data
	328
	329	def assign(self, other):
	330	self.stored_data = other.stored_data
	331	self.message = other.message
	332	self.data_name_dict = other.data_name_dict
	333	self.count = other.count
	334	self.list_of_id = other.list_of_id
	335	self.time_stamp = other.time_stamp
	336
	337	def save_to_writable(self, fp):
	338	"""
	339	save content of stored_data to fp (a .write()-supporting file-like object)
	340	"""
	341
	342	def add_type(dict, type):
	343	dict['__type__'] = type.__name__
	344	return dict
	345
	346	def jdefault(o):
	347	"""
	348	objects that can't otherwise be serialized need to be converted
	349	"""
	350	# tuples and sets (TODO: default JSONEncoder converts tuples to lists, create custom Encoder that preserves tuples)
	351	if isinstance(o, (tuple, set)):
	352	content = { 'data': list(o) }
	353	return add_type(content, type(o))
	354
	355	# "simple" types
	356	if isinstance(o, (Sample, Source, Vector)):
	357	return add_type(o.__dict__, type(o))
	358	if isinstance(o, (Plottable, View)):
	359	return add_type(o.__dict__, type(o))
	360
	361	# DataState
	362	if isinstance(o, DataState):
	363	# don't store parent
	364	content = o.__dict__.copy()
	365	content.pop('parent')
	366	return add_type(content, type(o))
	367
	368	# ndarray
	369	if isinstance(o, np.ndarray):
	370	buffer = StringIO()
	371	np.save(buffer, o)
	372	buffer.seek(0)
	373	content = { 'data': buffer.read().decode('latin-1') }
	374	return add_type(content, type(o))
	375
	376	# not supported
	377	logging.info("data cannot be serialized to json: %s" % type(o))
	378	return None
	379
	380	json.dump(self.stored_data, fp, indent=2, sort_keys=True, default=jdefault)
	381
	382
	383	def load_from_readable(self, fp):
	384	"""
	385	load content from tp to stored_data (a .read()-supporting file-like object)
	386	"""
	387
	388	supported = [
	389	tuple, set,
	390	Sample, Source, Vector,
	391	Plottable, Data1D, Data2D, PlottableTheory1D, PlottableFit1D, Text, Chisq, View,
	392	DataState, np.ndarray]
	393
	394	lookup = dict((cls.__name__, cls) for cls in supported)
	395
	396	class TooComplexException(Exception):
	397	pass
	398
	399	def simple_type(cls, data, level):
	400	class Empty(object):
	401	def __init__(self):
[b3e8629]	402	for key, value in data.items():
[dc5ef15]	403	setattr(self, key, generate(value, level))
	404
	405	# create target object
	406	o = Empty()
	407	o.__class__ = cls
	408
	409	return o
	410
	411	def construct(type, data, level):
	412	try:
	413	cls = lookup[type]
	414	except KeyError:
	415	logging.info('unknown type: %s' % type)
	416	return None
	417
	418	# tuples and sets
	419	if cls in (tuple, set):
	420	# convert list to tuple/set
	421	return cls(generate(data['data'], level))
	422
	423	# "simple" types
	424	if cls in (Sample, Source, Vector):
	425	return simple_type(cls, data, level)
	426	if issubclass(cls, Plottable) or (cls == View):
	427	return simple_type(cls, data, level)
	428
	429	# DataState
	430	if cls == DataState:
	431	o = simple_type(cls, data, level)
	432	o.parent = None # TODO: set to ???
	433	return o
	434
	435	# ndarray
	436	if cls == np.ndarray:
	437	buffer = StringIO()
	438	buffer.write(data['data'].encode('latin-1'))
	439	buffer.seek(0)
	440	return np.load(buffer)
	441
	442	logging.info('not implemented: %s, %s' % (type, cls))
	443	return None
	444
	445	def generate(data, level):
	446	if level > 16: # recursion limit (arbitrary number)
	447	raise TooComplexException()
	448	else:
	449	level += 1
	450
	451	if isinstance(data, dict):
	452	try:
	453	type = data['__type__']
	454	except KeyError:
	455	# if dictionary doesn't have __type__ then it is assumed to be just an ordinary dictionary
	456	o = {}
[b3e8629]	457	for key, value in data.items():
[dc5ef15]	458	o[key] = generate(value, level)
	459	return o
	460
	461	return construct(type, data, level)
	462
	463	if isinstance(data, list):
	464	return [generate(item, level) for item in data]
	465
	466	return data
	467
	468	new_stored_data = {}
[b3e8629]	469	for id, data in json.load(fp).items():
[dc5ef15]	470	try:
	471	new_stored_data[id] = generate(data, 0)
	472	except TooComplexException:
	473	logging.info('unable to load %s' % id)
	474
	475	self.stored_data = new_stored_data
	476

Note: See TracBrowser for help on using the repository browser.

SasView

source: sasview/src/sas/qtgui/MainWindow/DataManager.py @ b4d05bd

Download in other formats: