cansas_reader_HDF5.py @ a14fa99

ESS_GUIESS_GUI_DocsESS_GUI_batch_fittingESS_GUI_bumps_abstractionESS_GUI_iss1116ESS_GUI_iss879ESS_GUI_iss959ESS_GUI_openclESS_GUI_orderingESS_GUI_sync_sascalccostrafo411magnetic_scattrelease-4.1.1release-4.1.2release-4.2.2ticket-1009ticket-1094-headlessticket-1242-2d-resolutionticket-1243ticket-1249ticket885unittest-saveload

Last change on this file since a14fa99 was 995f4eb, checked in by lewis, 8 years ago
Ensure CanSAS 2.0 files are closed after reading
Property mode set to `100644`
File size: 20.1 KB

Rev	Line
[68aa210]	1	"""
	2	CanSAS 2D data reader for reading HDF5 formatted CanSAS files.
	3	"""
	4
	5	import h5py
	6	import numpy as np
	7	import re
	8	import os
	9	import sys
	10
[d72567e]	11	from sas.sascalc.dataloader.data_info import plottable_1D, plottable_2D, Data1D, Data2D, DataInfo, Process, Aperture
	12	from sas.sascalc.dataloader.data_info import Collimation, TransmissionSpectrum, Detector
	13	from sas.sascalc.dataloader.data_info import combine_data_info_with_plottable
	14
[68aa210]	15
	16
	17	class Reader():
	18	"""
[ad52d31]	19	A class for reading in CanSAS v2.0 data files. The existing iteration opens Mantid generated HDF5 formatted files
	20	with file extension .h5/.H5. Any number of data sets may be present within the file and any dimensionality of data
	21	may be used. Currently 1D and 2D SAS data sets are supported, but future implementations will include 1D and 2D
[d72567e]	22	SESANS data.
	23
	24	Any number of SASdata sets may be present in a SASentry and the data within can be either 1D I(Q) or 2D I(Qx, Qy).
[68aa210]	25
	26	:Dependencies:
[d72567e]	27	The CanSAS HDF5 reader requires h5py => v2.5.0 or later.
[68aa210]	28	"""
	29
	30	## CanSAS version
	31	cansas_version = 2.0
	32	## Logged warnings or messages
	33	logging = None
	34	## List of errors for the current data set
	35	errors = None
	36	## Raw file contents to be processed
	37	raw_data = None
[d72567e]	38	## Data info currently being read in
	39	current_datainfo = None
	40	## SASdata set currently being read in
[68aa210]	41	current_dataset = None
[d72567e]	42	## List of plottable1D objects that should be linked to the current_datainfo
	43	data1d = None
	44	## List of plottable2D objects that should be linked to the current_datainfo
	45	data2d = None
[68aa210]	46	## Data type name
[ad52d31]	47	type_name = "CanSAS 2.0"
[68aa210]	48	## Wildcards
[ad52d31]	49	type = ["CanSAS 2.0 HDF5 Files (.h5)\|.h5"]
[68aa210]	50	## List of allowed extensions
	51	ext = ['.h5', '.H5']
	52	## Flag to bypass extension check
	53	allow_all = False
	54	## List of files to return
	55	output = None
	56
	57	def read(self, filename):
	58	"""
[ad52d31]	59	This is the general read method that all SasView data_loaders must have.
[68aa210]	60
	61	:param filename: A path for an HDF5 formatted CanSAS 2D data file.
[d72567e]	62	:return: List of Data1D/2D objects and/or a list of errors.
[68aa210]	63	"""
[ad52d31]	64	## Reinitialize the class when loading a new data file to reset all class variables
[d72567e]	65	self.reset_class_variables()
[68aa210]	66	## Check that the file exists
	67	if os.path.isfile(filename):
	68	basename = os.path.basename(filename)
	69	_, extension = os.path.splitext(basename)
	70	# If the file type is not allowed, return empty list
	71	if extension in self.ext or self.allow_all:
	72	## Load the data file
	73	self.raw_data = h5py.File(filename, 'r')
	74	## Read in all child elements of top level SASroot
[d72567e]	75	self.read_children(self.raw_data, [])
[ad52d31]	76	## Add the last data set to the list of outputs
[68aa210]	77	self.add_data_set()
[995f4eb]	78	## Close the data file
	79	self.raw_data.close()
[68aa210]	80	## Return data set(s)
	81	return self.output
	82
[d72567e]	83	def reset_class_variables(self):
	84	"""
	85	Create the reader object and define initial states for class variables
	86	"""
	87	self.current_datainfo = None
	88	self.current_dataset = None
	89	self.data1d = []
	90	self.data2d = []
	91	self.raw_data = None
	92	self.errors = set()
	93	self.logging = []
	94	self.output = []
	95	self.parent_class = u''
	96	self.detector = Detector()
	97	self.collimation = Collimation()
	98	self.aperture = Aperture()
	99	self.process = Process()
	100	self.trans_spectrum = TransmissionSpectrum()
	101
	102	def read_children(self, data, parent_list):
[68aa210]	103	"""
[ad52d31]	104	A recursive method for stepping through the hierarchical data file.
[68aa210]	105
	106	:param data: h5py Group object of any kind
	107	:param parent: h5py Group parent name
	108	"""
	109
	110	## Loop through each element of the parent and process accordingly
	111	for key in data.keys():
	112	## Get all information for the current key
	113	value = data.get(key)
[d398285]	114	if value.attrs.get(u'canSAS_class') is not None:
	115	class_name = value.attrs.get(u'canSAS_class')
	116	else:
	117	class_name = value.attrs.get(u'NX_class')
[68aa210]	118	if class_name is not None:
	119	class_prog = re.compile(class_name)
	120	else:
	121	class_prog = re.compile(value.name)
	122
	123	if isinstance(value, h5py.Group):
[d72567e]	124	self.parent_class = class_name
	125	parent_list.append(key)
	126	## If this is a new sasentry, store the current data sets and create a fresh Data1D/2D object
[68aa210]	127	if class_prog.match(u'SASentry'):
	128	self.add_data_set(key)
[d72567e]	129	elif class_prog.match(u'SASdata'):
	130	self._initialize_new_data_set(parent_list)
[ad52d31]	131	## Recursion step to access data within the group
[d72567e]	132	self.read_children(value, parent_list)
	133	self.add_intermediate()
	134	parent_list.remove(key)
[68aa210]	135
	136	elif isinstance(value, h5py.Dataset):
	137	## If this is a dataset, store the data appropriately
	138	data_set = data[key][:]
[7bd6860a]	139	unit = self._get_unit(value)
[ac370c5]	140
[7bd6860a]	141	## I and Q Data
	142	if key == u'I':
	143	if type(self.current_dataset) is plottable_2D:
[ac370c5]	144	self.current_dataset.data = data_set
[7bd6860a]	145	self.current_dataset.zaxis("Intensity", unit)
	146	else:
	147	self.current_dataset.y = data_set.flatten()
	148	self.current_dataset.yaxis("Intensity", unit)
	149	continue
	150	elif key == u'Idev':
	151	if type(self.current_dataset) is plottable_2D:
	152	self.current_dataset.err_data = data_set.flatten()
	153	else:
	154	self.current_dataset.dy = data_set.flatten()
	155	continue
	156	elif key == u'Q':
	157	self.current_dataset.xaxis("Q", unit)
	158	if type(self.current_dataset) is plottable_2D:
	159	self.current_dataset.q = data_set.flatten()
	160	else:
	161	self.current_dataset.x = data_set.flatten()
	162	continue
	163	elif key == u'Qy':
	164	self.current_dataset.yaxis("Q_y", unit)
	165	self.current_dataset.qy_data = data_set.flatten()
	166	continue
	167	elif key == u'Qydev':
	168	self.current_dataset.dqy_data = data_set.flatten()
	169	continue
	170	elif key == u'Qx':
	171	self.current_dataset.xaxis("Q_x", unit)
	172	self.current_dataset.qx_data = data_set.flatten()
	173	continue
	174	elif key == u'Qxdev':
	175	self.current_dataset.dqx_data = data_set.flatten()
	176	continue
	177	elif key == u'Mask':
	178	self.current_dataset.mask = data_set.flatten()
	179	continue
[68aa210]	180
	181	for data_point in data_set:
	182	## Top Level Meta Data
	183	if key == u'definition':
[d72567e]	184	self.current_datainfo.meta_data['reader'] = data_point
[68aa210]	185	elif key == u'run':
[d72567e]	186	self.current_datainfo.run.append(data_point)
[68aa210]	187	elif key == u'title':
[d72567e]	188	self.current_datainfo.title = data_point
[68aa210]	189	elif key == u'SASnote':
[d72567e]	190	self.current_datainfo.notes.append(data_point)
[68aa210]	191
	192	## Sample Information
[eb98f24]	193	elif key == u'Title' and self.parent_class == u'SASsample': # CanSAS 2.0 format
[d72567e]	194	self.current_datainfo.sample.name = data_point
[eb98f24]	195	elif key == u'name' and self.parent_class == u'SASsample': # NXcanSAS format
[88d85c6]	196	self.current_datainfo.sample.name = data_point
[d72567e]	197	elif key == u'thickness' and self.parent_class == u'SASsample':
	198	self.current_datainfo.sample.thickness = data_point
	199	elif key == u'temperature' and self.parent_class == u'SASsample':
	200	self.current_datainfo.sample.temperature = data_point
[68aa210]	201
[ad52d31]	202	## Instrumental Information
[d72567e]	203	elif key == u'name' and self.parent_class == u'SASinstrument':
	204	self.current_datainfo.instrument = data_point
	205	elif key == u'name' and self.parent_class == u'SASdetector':
[ad52d31]	206	self.detector.name = data_point
[d72567e]	207	elif key == u'SDD' and self.parent_class == u'SASdetector':
	208	self.detector.distance = float(data_point)
[d398285]	209	self.detector.distance_unit = unit
[d72567e]	210	elif key == u'SSD' and self.parent_class == u'SAScollimation':
[ad52d31]	211	self.collimation.length = data_point
[d398285]	212	self.collimation.length_unit = unit
[d72567e]	213	elif key == u'name' and self.parent_class == u'SAScollimation':
[ad52d31]	214	self.collimation.name = data_point
	215
[68aa210]	216	## Process Information
[d72567e]	217	elif key == u'name' and self.parent_class == u'SASprocess':
[68aa210]	218	self.process.name = data_point
[eb98f24]	219	elif key == u'Title' and self.parent_class == u'SASprocess': # CanSAS 2.0 format
[68aa210]	220	self.process.name = data_point
[eb98f24]	221	elif key == u'name' and self.parent_class == u'SASprocess': # NXcanSAS format
[88d85c6]	222	self.process.name = data_point
[d72567e]	223	elif key == u'description' and self.parent_class == u'SASprocess':
[68aa210]	224	self.process.description = data_point
[d72567e]	225	elif key == u'date' and self.parent_class == u'SASprocess':
[68aa210]	226	self.process.date = data_point
[d72567e]	227	elif self.parent_class == u'SASprocess':
[ad52d31]	228	self.process.notes.append(data_point)
	229
	230	## Transmission Spectrum
[d72567e]	231	elif key == u'T' and self.parent_class == u'SAStransmission_spectrum':
[ad52d31]	232	self.trans_spectrum.transmission.append(data_point)
[d72567e]	233	elif key == u'Tdev' and self.parent_class == u'SAStransmission_spectrum':
[ad52d31]	234	self.trans_spectrum.transmission_deviation.append(data_point)
[d72567e]	235	elif key == u'lambda' and self.parent_class == u'SAStransmission_spectrum':
[ad52d31]	236	self.trans_spectrum.wavelength.append(data_point)
	237
	238	## Other Information
[d72567e]	239	elif key == u'wavelength' and self.parent_class == u'SASdata':
	240	self.current_datainfo.source.wavelength = data_point
	241	self.current_datainfo.source.wavelength.unit = unit
	242	elif key == u'radiation' and self.parent_class == u'SASsource':
	243	self.current_datainfo.source.radiation = data_point
	244	elif key == u'transmission' and self.parent_class == u'SASdata':
	245	self.current_datainfo.sample.transmission = data_point
[68aa210]	246
	247	## Everything else goes in meta_data
	248	else:
[d72567e]	249	new_key = self._create_unique_key(self.current_datainfo.meta_data, key)
	250	self.current_datainfo.meta_data[new_key] = data_point
[68aa210]	251
	252	else:
	253	## I don't know if this reachable code
	254	self.errors.add("ShouldNeverHappenException")
	255
[d72567e]	256	def add_intermediate(self):
[ad52d31]	257	"""
	258	This method stores any intermediate objects within the final data set after fully reading the set.
	259
	260	:param parent: The NXclass name for the h5py Group object that just finished being processed
	261	"""
	262
[d72567e]	263	if self.parent_class == u'SASprocess':
	264	self.current_datainfo.process.append(self.process)
[ad52d31]	265	self.process = Process()
[d72567e]	266	elif self.parent_class == u'SASdetector':
	267	self.current_datainfo.detector.append(self.detector)
[ad52d31]	268	self.detector = Detector()
[d72567e]	269	elif self.parent_class == u'SAStransmission_spectrum':
	270	self.current_datainfo.trans_spectrum.append(self.trans_spectrum)
[ad52d31]	271	self.trans_spectrum = TransmissionSpectrum()
[d72567e]	272	elif self.parent_class == u'SAScollimation':
	273	self.current_datainfo.collimation.append(self.collimation)
[ad52d31]	274	self.collimation = Collimation()
[d72567e]	275	elif self.parent_class == u'SASaperture':
[ad52d31]	276	self.collimation.aperture.append(self.aperture)
	277	self.aperture = Aperture()
[d72567e]	278	elif self.parent_class == u'SASdata':
	279	if type(self.current_dataset) is plottable_2D:
	280	self.data2d.append(self.current_dataset)
	281	elif type(self.current_dataset) is plottable_1D:
	282	self.data1d.append(self.current_dataset)
[68aa210]	283
	284	def final_data_cleanup(self):
	285	"""
[d72567e]	286	Does some final cleanup and formatting on self.current_datainfo and all data1D and data2D objects and then
	287	combines the data and info into Data1D and Data2D objects
[68aa210]	288	"""
	289
[d72567e]	290	## Type cast data arrays to float64
	291	if len(self.current_datainfo.trans_spectrum) > 0:
[ad52d31]	292	spectrum_list = []
[d72567e]	293	for spectrum in self.current_datainfo.trans_spectrum:
[ad52d31]	294	spectrum.transmission = np.delete(spectrum.transmission, [0])
	295	spectrum.transmission = spectrum.transmission.astype(np.float64)
	296	spectrum.transmission_deviation = np.delete(spectrum.transmission_deviation, [0])
	297	spectrum.transmission_deviation = spectrum.transmission_deviation.astype(np.float64)
	298	spectrum.wavelength = np.delete(spectrum.wavelength, [0])
	299	spectrum.wavelength = spectrum.wavelength.astype(np.float64)
[d72567e]	300	if len(spectrum.transmission) > 0:
	301	spectrum_list.append(spectrum)
	302	self.current_datainfo.trans_spectrum = spectrum_list
[68aa210]	303
	304	## Append errors to dataset and reset class errors
[d72567e]	305	self.current_datainfo.errors = self.errors
[68aa210]	306	self.errors.clear()
	307
[d72567e]	308	## Combine all plottables with datainfo and append each to output
	309	## Type cast data arrays to float64 and find min/max as appropriate
	310	for dataset in self.data2d:
	311	dataset.data = dataset.data.astype(np.float64)
	312	dataset.err_data = dataset.err_data.astype(np.float64)
	313	if dataset.qx_data is not None:
	314	dataset.xmin = np.min(dataset.qx_data)
	315	dataset.xmax = np.max(dataset.qx_data)
	316	dataset.qx_data = dataset.qx_data.astype(np.float64)
	317	if dataset.dqx_data is not None:
	318	dataset.dqx_data = dataset.dqx_data.astype(np.float64)
	319	if dataset.qy_data is not None:
	320	dataset.ymin = np.min(dataset.qy_data)
	321	dataset.ymax = np.max(dataset.qy_data)
	322	dataset.qy_data = dataset.qy_data.astype(np.float64)
	323	if dataset.dqy_data is not None:
	324	dataset.dqy_data = dataset.dqy_data.astype(np.float64)
	325	if dataset.q_data is not None:
	326	dataset.q_data = dataset.q_data.astype(np.float64)
	327	zeros = np.ones(dataset.data.size, dtype=bool)
	328	try:
	329	for i in range (0, dataset.mask.size - 1):
	330	zeros[i] = dataset.mask[i]
	331	except:
	332	self.errors.add(sys.exc_value)
	333	dataset.mask = zeros
	334	## Calculate the actual Q matrix
	335	try:
	336	if dataset.q_data.size <= 1:
	337	dataset.q_data = np.sqrt(dataset.qx_data * dataset.qx_data + dataset.qy_data * dataset.qy_data)
	338	except:
	339	dataset.q_data = None
[ac370c5]	340
	341	if dataset.data.ndim == 2:
	342	(n_rows, n_cols) = dataset.data.shape
	343	dataset.y_bins = dataset.qy_data[0::n_rows]
	344	dataset.x_bins = dataset.qx_data[:n_cols]
	345	dataset.data = dataset.data.flatten()
	346
[d72567e]	347	final_dataset = combine_data_info_with_plottable(dataset, self.current_datainfo)
	348	self.output.append(final_dataset)
	349
	350	for dataset in self.data1d:
	351	if dataset.x is not None:
	352	dataset.x = dataset.x.astype(np.float64)
	353	dataset.xmin = np.min(dataset.x)
	354	dataset.xmax = np.max(dataset.x)
	355	if dataset.y is not None:
	356	dataset.y = dataset.y.astype(np.float64)
	357	dataset.ymin = np.min(dataset.y)
	358	dataset.ymax = np.max(dataset.y)
	359	if dataset.dx is not None:
	360	dataset.dx = dataset.dx.astype(np.float64)
	361	if dataset.dxl is not None:
	362	dataset.dxl = dataset.dxl.astype(np.float64)
	363	if dataset.dxw is not None:
	364	dataset.dxw = dataset.dxw.astype(np.float64)
	365	if dataset.dy is not None:
	366	dataset.dy = dataset.dy.astype(np.float64)
	367	final_dataset = combine_data_info_with_plottable(dataset, self.current_datainfo)
	368	self.output.append(final_dataset)
	369
[68aa210]	370	def add_data_set(self, key=""):
	371	"""
	372	Adds the current_dataset to the list of outputs after preforming final processing on the data and then calls a
	373	private method to generate a new data set.
	374
	375	:param key: NeXus group name for current tree level
	376	"""
[d72567e]	377
	378	if self.current_datainfo and self.current_dataset:
[68aa210]	379	self.final_data_cleanup()
[d72567e]	380	self.data1d = []
	381	self.data2d = []
	382	self.current_datainfo = DataInfo()
[68aa210]	383
[54ba66e]	384
[d72567e]	385	def _initialize_new_data_set(self, parent_list = None):
[68aa210]	386	"""
	387	A private class method to generate a new 1D or 2D data object based on the type of data within the set.
	388	Outside methods should call add_data_set() to be sure any existing data is stored properly.
	389
[d72567e]	390	:param parent_list: List of names of parent elements
[68aa210]	391	"""
[d72567e]	392
	393	if parent_list is None:
	394	parent_list = []
	395	if self._find_intermediate(parent_list, "Qx"):
	396	self.current_dataset = plottable_2D()
[68aa210]	397	else:
	398	x = np.array(0)
	399	y = np.array(0)
[d72567e]	400	self.current_dataset = plottable_1D(x, y)
	401	self.current_datainfo.filename = self.raw_data.filename
[68aa210]	402
[d72567e]	403	def _find_intermediate(self, parent_list, basename=""):
[ad52d31]	404	"""
	405	A private class used to find an entry by either using a direct key or knowing the approximate basename.
	406
[d72567e]	407	:param parent_list: List of parents to the current level in the HDF5 file
	408	:param basename: Approximate name of an entry to search for
[ad52d31]	409	:return:
	410	"""
[d72567e]	411
	412	entry = False
	413	key_prog = re.compile(basename)
	414	top = self.raw_data
	415	for parent in parent_list:
	416	top = top.get(parent)
	417	for key in top.keys():
	418	if (key_prog.match(key)):
	419	entry = True
	420	break
[ad52d31]	421	return entry
	422
[68aa210]	423	def _create_unique_key(self, dictionary, name, numb=0):
	424	"""
	425	Create a unique key value for any dictionary to prevent overwriting
	426	Recurses until a unique key value is found.
	427
	428	:param dictionary: A dictionary with any number of entries
	429	:param name: The index of the item to be added to dictionary
	430	:param numb: The number to be appended to the name, starts at 0
[d72567e]	431	:return: The new name for the dictionary entry
[68aa210]	432	"""
	433	if dictionary.get(name) is not None:
	434	numb += 1
	435	name = name.split("_")[0]
	436	name += "_{0}".format(numb)
	437	name = self._create_unique_key(dictionary, name, numb)
[d398285]	438	return name
	439
	440	def _get_unit(self, value):
	441	"""
	442	Find the unit for a particular value within the h5py dictionary
	443
	444	:param value: attribute dictionary for a particular value set
[d72567e]	445	:return: unit for the value passed to the method
[d398285]	446	"""
	447	unit = value.attrs.get(u'units')
	448	if unit == None:
	449	unit = value.attrs.get(u'unit')
	450	## Convert the unit formats
	451	if unit == "1/A":
	452	unit = "A^{-1}"
	453	elif unit == "1/cm":
	454	unit = "cm^{-1}"
[54ba66e]	455	return unit

Note: See TracBrowser for help on using the repository browser.

SasView

source: sasview/src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py @ a14fa99

Download in other formats: