file_reader_base_class.py @ f0b9bce

ticket-1094-headless

Last change on this file since f0b9bce was c222c27, checked in by Jeff Krzywon <jkrzywon@…>, 6 years ago
Merge branch 'master' into ticket-1111
Property mode set to `100644`
File size: 19.7 KB

Rev	Line
[beba407]	1	"""
[b09095a]	2	This is the base file reader class most file readers should inherit from.
[beba407]	3	All generic functionality required for a file loader/reader is built into this
	4	class
	5	"""
	6
	7	import os
[7b50f14]	8	import sys
[8475d16]	9	import math
[beba407]	10	import logging
	11	from abc import abstractmethod
[574adc7]	12
	13	import numpy as np
	14	from .loader_exceptions import NoKnownLoaderException, FileContentsException,\
[da8bb53]	15	DataReaderException, DefaultReaderException
[574adc7]	16	from .data_info import Data1D, Data2D, DataInfo, plottable_1D, plottable_2D,\
[beba407]	17	combine_data_info_with_plottable
[3bab401]	18	from sas.sascalc.data_util.nxsunit import Converter
[beba407]	19
	20	logger = logging.getLogger(__name__)
	21
[7b50f14]	22	if sys.version_info[0] < 3:
	23	def decode(s):
	24	return s
	25	else:
	26	def decode(s):
	27	return s.decode() if isinstance(s, bytes) else s
[beba407]	28
[6fd7b20]	29	# Data 1D fields for iterative purposes
	30	FIELDS_1D = ('x', 'y', 'dx', 'dy', 'dxl', 'dxw')
	31	# Data 2D fields for iterative purposes
	32	FIELDS_2D = ('data', 'qx_data', 'qy_data', 'q_data', 'err_data',
[340291a]	33	'dqx_data', 'dqy_data', 'mask')
[c36c09f]	34	DEPRECATION_MESSAGE = ("\rThe extension of this file suggests the data set migh"
	35	"t not be fully reduced. Support for the reader associat"
	36	"ed with this file type has been removed. An attempt to "
[4a8d55c]	37	"load the file was made, but, should it be successful, "
	38	"SasView cannot guarantee the accuracy of the data.")
[6fd7b20]	39
[8d5e11c]	40
[beba407]	41	class FileReader(object):
[b09095a]	42	# String to describe the type of data this reader can load
	43	type_name = "ASCII"
	44	# Wildcards to display
	45	type = ["Text files (.txt\|.TXT)"]
[beba407]	46	# List of allowed extensions
	47	ext = ['.txt']
[c36c09f]	48	# Deprecated extensions
[dfcdbf8]	49	deprecated_extensions = ['.asc']
[beba407]	50	# Bypass extension check and try to load anyway
	51	allow_all = False
[b09095a]	52	# Able to import the unit converter
	53	has_converter = True
	54	# Default value of zero
	55	_ZERO = 1e-16
[beba407]	56
[cb11a25]	57	def __init__(self):
	58	# List of Data1D and Data2D objects to be sent back to data_loader
	59	self.output = []
	60	# Current plottable_(1D/2D) object being loaded in
	61	self.current_dataset = None
	62	# Current DataInfo object being loaded in
	63	self.current_datainfo = None
[3053a4a]	64	# File path sent to reader
	65	self.filepath = None
[cb11a25]	66	# Open file handle
	67	self.f_open = None
	68
[beba407]	69	def read(self, filepath):
	70	"""
[bc570f4]	71	Basic file reader
	72
[beba407]	73	:param filepath: The full or relative path to a file to be loaded
	74	"""
[3053a4a]	75	self.filepath = filepath
[beba407]	76	if os.path.isfile(filepath):
	77	basename, extension = os.path.splitext(os.path.basename(filepath))
[da8bb53]	78	self.extension = extension.lower()
[beba407]	79	# If the file type is not allowed, return nothing
[da8bb53]	80	if self.extension in self.ext or self.allow_all:
[beba407]	81	# Try to load the file, but raise an error if unable to.
	82	try:
[b09095a]	83	self.f_open = open(filepath, 'rb')
	84	self.get_file_contents()
[0b79323]	85
[bc570f4]	86	except DataReaderException as e:
[da8bb53]	87	self.handle_error_message(e.message)
[beba407]	88	except OSError as e:
[b09095a]	89	# If the file cannot be opened
[beba407]	90	msg = "Unable to open file: {}\n".format(filepath)
	91	msg += e.message
	92	self.handle_error_message(msg)
[b09095a]	93	finally:
[da8bb53]	94	# Close the file handle if it is open
[b09095a]	95	if not self.f_open.closed:
	96	self.f_open.close()
[425feff]	97	if any(filepath.lower().endswith(ext) for ext in
	98	self.deprecated_extensions):
	99	self.handle_error_message(DEPRECATION_MESSAGE)
[248ff73]	100	if len(self.output) > 0:
	101	# Sort the data that's been loaded
[3bab401]	102	self.convert_data_units()
	103	self.sort_data()
[beba407]	104	else:
	105	msg = "Unable to find file at: {}\n".format(filepath)
	106	msg += "Please check your file path and try again."
	107	self.handle_error_message(msg)
[a78433dd]	108
[b09095a]	109	# Return a list of parsed entries that data_loader can manage
[1576693]	110	final_data = self.output
	111	self.reset_state()
	112	return final_data
[beba407]	113
[61f329f0]	114	def reset_state(self):
	115	"""
	116	Resets the class state to a base case when loading a new data file so previous
	117	data files do not appear a second time
	118	"""
	119	self.current_datainfo = None
	120	self.current_dataset = None
[3053a4a]	121	self.filepath = None
[8475d16]	122	self.ind = None
[61f329f0]	123	self.output = []
	124
[26183bf]	125	def nextline(self):
	126	"""
	127	Returns the next line in the file as a string.
	128	"""
	129	#return self.f_open.readline()
[7b50f14]	130	return decode(self.f_open.readline())
[26183bf]	131
	132	def nextlines(self):
	133	"""
	134	Returns the next line in the file as a string.
	135	"""
	136	for line in self.f_open:
	137	#yield line
[7b50f14]	138	yield decode(line)
[26183bf]	139
	140	def readall(self):
	141	"""
	142	Returns the entire file as a string.
	143	"""
[7b50f14]	144	return decode(self.f_open.read())
[26183bf]	145
[beba407]	146	def handle_error_message(self, msg):
	147	"""
	148	Generic error handler to add an error to the current datainfo to
[20fa5fe]	149	propagate the error up the error chain.
[beba407]	150	:param msg: Error message
	151	"""
[dcb91cf]	152	if len(self.output) > 0:
[282bc3f]	153	self.output[-1].errors.append(msg)
[dcb91cf]	154	elif isinstance(self.current_datainfo, DataInfo):
[9b08354]	155	self.current_datainfo.errors.append(msg)
[beba407]	156	else:
	157	logger.warning(msg)
[425feff]	158	raise NoKnownLoaderException(msg)
[beba407]	159
	160	def send_to_output(self):
	161	"""
	162	Helper that automatically combines the info and set and then appends it
	163	to output
	164	"""
	165	data_obj = combine_data_info_with_plottable(self.current_dataset,
	166	self.current_datainfo)
	167	self.output.append(data_obj)
	168
[3bab401]	169	def sort_data(self):
[b09095a]	170	"""
	171	Sort 1D data along the X axis for consistency
	172	"""
	173	for data in self.output:
	174	if isinstance(data, Data1D):
[a78a02f]	175	# Normalize the units for
	176	data.x_unit = self.format_unit(data.x_unit)
[8f882fe]	177	data._xunit = data.x_unit
[a78a02f]	178	data.y_unit = self.format_unit(data.y_unit)
[8f882fe]	179	data._yunit = data.y_unit
[7477fb9]	180	# Sort data by increasing x and remove 1st point
[e3133dc]	181	ind = np.lexsort((data.y, data.x))
	182	data.x = self._reorder_1d_array(data.x, ind)
	183	data.y = self._reorder_1d_array(data.y, ind)
[b09095a]	184	if data.dx is not None:
[4660990]	185	if len(data.dx) == 0:
	186	data.dx = None
	187	continue
[e3133dc]	188	data.dx = self._reorder_1d_array(data.dx, ind)
[b09095a]	189	if data.dxl is not None:
[e3133dc]	190	data.dxl = self._reorder_1d_array(data.dxl, ind)
[b09095a]	191	if data.dxw is not None:
[e3133dc]	192	data.dxw = self._reorder_1d_array(data.dxw, ind)
[b09095a]	193	if data.dy is not None:
[4660990]	194	if len(data.dy) == 0:
	195	data.dy = None
	196	continue
[e3133dc]	197	data.dy = self._reorder_1d_array(data.dy, ind)
[b09095a]	198	if data.lam is not None:
[e3133dc]	199	data.lam = self._reorder_1d_array(data.lam, ind)
[b09095a]	200	if data.dlam is not None:
[e3133dc]	201	data.dlam = self._reorder_1d_array(data.dlam, ind)
[f02a0c6]	202	data = self._remove_nans_in_data(data)
[dcb91cf]	203	if len(data.x) > 0:
[248ff73]	204	data.xmin = np.min(data.x)
	205	data.xmax = np.max(data.x)
	206	data.ymin = np.min(data.y)
	207	data.ymax = np.max(data.y)
[3bab401]	208	elif isinstance(data, Data2D):
	209	# Normalize the units for
[8f882fe]	210	data.Q_unit = self.format_unit(data.Q_unit)
	211	data.I_unit = self.format_unit(data.I_unit)
	212	data._xunit = data.Q_unit
	213	data._yunit = data.Q_unit
	214	data._zunit = data.I_unit
[3bab401]	215	data.data = data.data.astype(np.float64)
	216	data.qx_data = data.qx_data.astype(np.float64)
	217	data.xmin = np.min(data.qx_data)
	218	data.xmax = np.max(data.qx_data)
	219	data.qy_data = data.qy_data.astype(np.float64)
	220	data.ymin = np.min(data.qy_data)
	221	data.ymax = np.max(data.qy_data)
	222	data.q_data = np.sqrt(data.qx_data * data.qx_data
	223	+ data.qy_data * data.qy_data)
	224	if data.err_data is not None:
	225	data.err_data = data.err_data.astype(np.float64)
	226	if data.dqx_data is not None:
	227	data.dqx_data = data.dqx_data.astype(np.float64)
	228	if data.dqy_data is not None:
	229	data.dqy_data = data.dqy_data.astype(np.float64)
	230	if data.mask is not None:
	231	data.mask = data.mask.astype(dtype=bool)
	232
[9fb4572]	233	if len(data.data.shape) == 2:
	234	n_rows, n_cols = data.data.shape
	235	data.y_bins = data.qy_data[0::int(n_cols)]
	236	data.x_bins = data.qx_data[:int(n_cols)]
	237	data.data = data.data.flatten()
[b1ec23d]	238	data = self._remove_nans_in_data(data)
[3bab401]	239	if len(data.data) > 0:
	240	data.xmin = np.min(data.qx_data)
	241	data.xmax = np.max(data.qx_data)
	242	data.ymin = np.min(data.qy_data)
[c222c27]	243	data.ymax = np.max(data.qy_data)
[b09095a]	244
[e3133dc]	245	@staticmethod
	246	def _reorder_1d_array(array, ind):
	247	"""
	248	Reorders a 1D array based on the indices passed as ind
	249	:param array: Array to be reordered
	250	:param ind: Indices used to reorder array
	251	:return: reordered array
	252	"""
	253	array = np.asarray(array, dtype=np.float64)
	254	return array[ind]
	255
	256	@staticmethod
[f02a0c6]	257	def _remove_nans_in_data(data):
[e3133dc]	258	"""
	259	Remove data points where nan is loaded
[a58b5a0]	260	:param data: 1D or 2D data object
	261	:return: data with nan points removed
[e3133dc]	262	"""
[f02a0c6]	263	if isinstance(data, Data1D):
[6fd7b20]	264	fields = FIELDS_1D
[f02a0c6]	265	elif isinstance(data, Data2D):
[6fd7b20]	266	fields = FIELDS_2D
[f02a0c6]	267	else:
[6fd7b20]	268	return data
[a58b5a0]	269	# Make array of good points - all others will be removed
[6fd7b20]	270	good = np.isfinite(getattr(data, fields[0]))
	271	for name in fields[1:]:
	272	array = getattr(data, name)
[e3133dc]	273	if array is not None:
[a58b5a0]	274	# Update good points only if not already changed
[6fd7b20]	275	good &= np.isfinite(array)
	276	if not np.all(good):
	277	for name in fields:
	278	array = getattr(data, name)
	279	if array is not None:
	280	setattr(data, name, array[good])
[e3133dc]	281	return data
[8475d16]	282
[3bab401]	283	@staticmethod
	284	def set_default_1d_units(data):
	285	"""
	286	Set the x and y axes to the default 1D units
	287	:param data: 1D data set
	288	:return:
	289	"""
[b1ec23d]	290	data.xaxis(r"\rm{Q}", '1/A')
	291	data.yaxis(r"\rm{Intensity}", "1/cm")
[3bab401]	292	return data
[9d786e5]	293
[3bab401]	294	@staticmethod
	295	def set_default_2d_units(data):
	296	"""
	297	Set the x and y axes to the default 2D units
	298	:param data: 2D data set
	299	:return:
	300	"""
	301	data.xaxis("\\rm{Q_{x}}", '1/A')
	302	data.yaxis("\\rm{Q_{y}}", '1/A')
	303	data.zaxis("\\rm{Intensity}", "1/cm")
	304	return data
	305
[dfcdbf8]	306	def convert_data_units(self, default_q_unit="1/A"):
[3bab401]	307	"""
	308	Converts al; data to the sasview default of units of A^{-1} for Q and
	309	cm^{-1} for I.
[8d5e11c]	310	:param default_q_unit: The default Q unit used by Sasview
[3bab401]	311	"""
[dfcdbf8]	312	convert_q = True
[3bab401]	313	new_output = []
	314	for data in self.output:
[058f6c3]	315	if data.isSesans:
	316	new_output.append(data)
	317	continue
[dfcdbf8]	318	try:
	319	file_x_unit = data._xunit
	320	data_conv_x = Converter(file_x_unit)
	321	except KeyError:
	322	logger.info("Unrecognized Q units in data file. No data "
	323	"conversion attempted")
	324	convert_q = False
	325	try:
	326
	327	if isinstance(data, Data1D):
	328	if convert_q:
	329	data.x = data_conv_x(data.x, units=default_q_unit)
	330	data._xunit = default_q_unit
	331	data.x_unit = default_q_unit
	332	if data.dx is not None:
	333	data.dx = data_conv_x(data.dx,
	334	units=default_q_unit)
	335	if data.dxl is not None:
	336	data.dxl = data_conv_x(data.dxl,
	337	units=default_q_unit)
	338	if data.dxw is not None:
	339	data.dxw = data_conv_x(data.dxw,
	340	units=default_q_unit)
	341	elif isinstance(data, Data2D):
	342	if convert_q:
	343	data.qx_data = data_conv_x(data.qx_data,
	344	units=default_q_unit)
	345	if data.dqx_data is not None:
	346	data.dqx_data = data_conv_x(data.dqx_data,
	347	units=default_q_unit)
	348	try:
	349	file_y_unit = data._yunit
	350	data_conv_y = Converter(file_y_unit)
	351	data.qy_data = data_conv_y(data.qy_data,
	352	units=default_q_unit)
	353	if data.dqy_data is not None:
	354	data.dqy_data = data_conv_y(data.dqy_data,
	355	units=default_q_unit)
	356	except KeyError:
	357	logger.info("Unrecognized Qy units in data file. No"
	358	" data conversion attempted")
	359	except KeyError:
	360	message = "Unable to convert Q units from {0} to 1/A."
	361	message.format(default_q_unit)
[282bc3f]	362	data.errors.append(message)
[058f6c3]	363	new_output.append(data)
[3bab401]	364	self.output = new_output
[0b79323]	365
[a78a02f]	366	def format_unit(self, unit=None):
	367	"""
	368	Format units a common way
	369	:param unit:
	370	:return:
	371	"""
	372	if unit:
	373	split = unit.split("/")
	374	if len(split) == 1:
	375	return unit
	376	elif split[0] == '1':
	377	return "{0}^".format(split[1]) + "{-1}"
	378	else:
	379	return "{0}*{1}^".format(split[0], split[1]) + "{-1}"
	380
[da8bb53]	381	def set_all_to_none(self):
	382	"""
	383	Set all mutable values to None for error handling purposes
	384	"""
	385	self.current_dataset = None
	386	self.current_datainfo = None
	387	self.output = []
	388
[7b07fbe]	389	def data_cleanup(self):
	390	"""
	391	Clean up the data sets and refresh everything
	392	:return: None
	393	"""
	394	self.remove_empty_q_values()
	395	self.send_to_output() # Combine datasets with DataInfo
	396	self.current_datainfo = DataInfo() # Reset DataInfo
	397
	398	def remove_empty_q_values(self):
[ad92c5a]	399	"""
	400	Remove any point where Q == 0
	401	"""
[7b07fbe]	402	if isinstance(self.current_dataset, plottable_1D):
	403	# Booleans for resolutions
	404	has_error_dx = self.current_dataset.dx is not None
	405	has_error_dxl = self.current_dataset.dxl is not None
	406	has_error_dxw = self.current_dataset.dxw is not None
	407	has_error_dy = self.current_dataset.dy is not None
	408	# Create arrays of zeros for non-existent resolutions
	409	if has_error_dxw and not has_error_dxl:
	410	array_size = self.current_dataset.dxw.size - 1
	411	self.current_dataset.dxl = np.append(self.current_dataset.dxl,
	412	np.zeros([array_size]))
	413	has_error_dxl = True
	414	elif has_error_dxl and not has_error_dxw:
	415	array_size = self.current_dataset.dxl.size - 1
	416	self.current_dataset.dxw = np.append(self.current_dataset.dxw,
	417	np.zeros([array_size]))
	418	has_error_dxw = True
	419	elif not has_error_dxl and not has_error_dxw and not has_error_dx:
	420	array_size = self.current_dataset.x.size - 1
	421	self.current_dataset.dx = np.append(self.current_dataset.dx,
	422	np.zeros([array_size]))
	423	has_error_dx = True
	424	if not has_error_dy:
	425	array_size = self.current_dataset.y.size - 1
	426	self.current_dataset.dy = np.append(self.current_dataset.dy,
	427	np.zeros([array_size]))
	428	has_error_dy = True
	429
	430	# Remove points where q = 0
	431	x = self.current_dataset.x
	432	self.current_dataset.x = self.current_dataset.x[x != 0]
	433	self.current_dataset.y = self.current_dataset.y[x != 0]
	434	if has_error_dy:
	435	self.current_dataset.dy = self.current_dataset.dy[x != 0]
	436	if has_error_dx:
	437	self.current_dataset.dx = self.current_dataset.dx[x != 0]
	438	if has_error_dxl:
	439	self.current_dataset.dxl = self.current_dataset.dxl[x != 0]
	440	if has_error_dxw:
	441	self.current_dataset.dxw = self.current_dataset.dxw[x != 0]
	442	elif isinstance(self.current_dataset, plottable_2D):
	443	has_error_dqx = self.current_dataset.dqx_data is not None
	444	has_error_dqy = self.current_dataset.dqy_data is not None
	445	has_error_dy = self.current_dataset.err_data is not None
	446	has_mask = self.current_dataset.mask is not None
	447	x = self.current_dataset.qx_data
	448	self.current_dataset.data = self.current_dataset.data[x != 0]
	449	self.current_dataset.qx_data = self.current_dataset.qx_data[x != 0]
	450	self.current_dataset.qy_data = self.current_dataset.qy_data[x != 0]
[deaa0c6]	451	self.current_dataset.q_data = np.sqrt(
	452	np.square(self.current_dataset.qx_data) + np.square(
	453	self.current_dataset.qy_data))
[7b07fbe]	454	if has_error_dy:
[8d5e11c]	455	self.current_dataset.err_data = self.current_dataset.err_data[
	456	x != 0]
[7b07fbe]	457	if has_error_dqx:
[8d5e11c]	458	self.current_dataset.dqx_data = self.current_dataset.dqx_data[
	459	x != 0]
[7b07fbe]	460	if has_error_dqy:
[8d5e11c]	461	self.current_dataset.dqy_data = self.current_dataset.dqy_data[
	462	x != 0]
[7b07fbe]	463	if has_mask:
	464	self.current_dataset.mask = self.current_dataset.mask[x != 0]
[ad92c5a]	465
	466	def reset_data_list(self, no_lines=0):
	467	"""
	468	Reset the plottable_1D object
	469	"""
	470	# Initialize data sets with arrays the maximum possible size
	471	x = np.zeros(no_lines)
	472	y = np.zeros(no_lines)
[4660990]	473	dx = np.zeros(no_lines)
	474	dy = np.zeros(no_lines)
	475	self.current_dataset = plottable_1D(x, y, dx, dy)
[ad92c5a]	476
[b09095a]	477	@staticmethod
	478	def splitline(line):
	479	"""
[20fa5fe]	480	Splits a line into pieces based on common delimiters
[b09095a]	481	:param line: A single line of text
	482	:return: list of values
	483	"""
	484	# Initial try for CSV (split on ,)
	485	toks = line.split(',')
	486	# Now try SCSV (split on ;)
	487	if len(toks) < 2:
	488	toks = line.split(';')
	489	# Now go for whitespace
	490	if len(toks) < 2:
	491	toks = line.split()
	492	return toks
	493
[beba407]	494	@abstractmethod
[b09095a]	495	def get_file_contents(self):
[beba407]	496	"""
[ad92c5a]	497	Reader specific class to access the contents of the file
[b09095a]	498	All reader classes that inherit from FileReader must implement
[beba407]	499	"""
	500	pass

Note: See TracBrowser for help on using the repository browser.

SasView

source: sasview/src/sas/sascalc/dataloader/file_reader_base_class.py @ f0b9bce

Download in other formats: