cansas_reader_HDF5.py @ 8dec7e7

ESS_GUIESS_GUI_DocsESS_GUI_batch_fittingESS_GUI_bumps_abstractionESS_GUI_iss1116ESS_GUI_iss879ESS_GUI_iss959ESS_GUI_openclESS_GUI_orderingESS_GUI_sync_sascalccostrafo411magnetic_scattrelease-4.2.2ticket-1009ticket-1094-headlessticket-1242-2d-resolutionticket-1243ticket-1249ticket885unittest-saveload

Last change on this file since 8dec7e7 was 8dec7e7, checked in by lewis, 7 years ago
Raise DefaultReaderException? in NXcanSAS reader and handle correctly
Property mode set to `100644`
File size: 26.2 KB

Line
1	"""
2	CanSAS 2D data reader for reading HDF5 formatted CanSAS files.
3	"""
4
5	import h5py
6	import numpy as np
7	import re
8	import os
9	import sys
10
11	from sas.sascalc.dataloader.data_info import plottable_1D, plottable_2D,\
12	Data1D, Data2D, DataInfo, Process, Aperture, Collimation, \
13	TransmissionSpectrum, Detector
14	from sas.sascalc.dataloader.data_info import combine_data_info_with_plottable
15	from sas.sascalc.dataloader.loader_exceptions import FileContentsException, DefaultReaderException
16	from sas.sascalc.dataloader.file_reader_base_class import FileReader
17
18
19	class Reader(FileReader):
20	"""
21	A class for reading in CanSAS v2.0 data files. The existing iteration opens
22	Mantid generated HDF5 formatted files with file extension .h5/.H5. Any
23	number of data sets may be present within the file and any dimensionality
24	of data may be used. Currently 1D and 2D SAS data sets are supported, but
25	future implementations will include 1D and 2D SESANS data.
26
27	Any number of SASdata sets may be present in a SASentry and the data within
28	can be either 1D I(Q) or 2D I(Qx, Qy).
29
30	Also supports reading NXcanSAS formatted HDF5 files
31
32	:Dependencies:
33	The CanSAS HDF5 reader requires h5py => v2.5.0 or later.
34	"""
35
36	# CanSAS version
37	cansas_version = 2.0
38	# Logged warnings or messages
39	logging = None
40	# List of errors for the current data set
41	errors = None
42	# Raw file contents to be processed
43	raw_data = None
44	# List of plottable1D objects that should be linked to the current_datainfo
45	data1d = None
46	# List of plottable2D objects that should be linked to the current_datainfo
47	data2d = None
48	# Data type name
49	type_name = "CanSAS 2.0"
50	# Wildcards
51	type = ["CanSAS 2.0 HDF5 Files (.h5)\|.h5"]
52	# List of allowed extensions
53	ext = ['.h5', '.H5']
54	# Flag to bypass extension check
55	allow_all = True
56
57	def get_file_contents(self):
58	"""
59	This is the general read method that all SasView data_loaders must have.
60
61	:param filename: A path for an HDF5 formatted CanSAS 2D data file.
62	:return: List of Data1D/2D objects and/or a list of errors.
63	"""
64	# Reinitialize when loading a new data file to reset all class variables
65	self.reset_class_variables()
66
67	filename = self.f_open.name
68	self.f_open.close() # IO handled by h5py
69
70	# Check that the file exists
71	if os.path.isfile(filename):
72	basename = os.path.basename(filename)
73	_, extension = os.path.splitext(basename)
74	# If the file type is not allowed, return empty list
75	if extension in self.ext or self.allow_all:
76	# Load the data file
77	try:
78	self.raw_data = h5py.File(filename, 'r')
79	except Exception as e:
80	if extension not in self.ext:
81	msg = "CanSAS2.0 HDF5 Reader could not load file {}".format(basename + extension)
82	raise DefaultReaderException(msg)
83	raise FileContentsException(e.message)
84	# Read in all child elements of top level SASroot
85	self.read_children(self.raw_data, [])
86	# Add the last data set to the list of outputs
87	self.add_data_set()
88	# Close the data file
89	self.raw_data.close()
90	# Return data set(s)
91	return self.output
92
93	def reset_class_variables(self):
94	"""
95	Create the reader object and define initial states for class variables
96	"""
97	self.current_datainfo = None
98	self.current_dataset = None
99	self.data1d = []
100	self.data2d = []
101	self.raw_data = None
102	self.errors = set()
103	self.logging = []
104	self.output = []
105	self.parent_class = u''
106	self.detector = Detector()
107	self.collimation = Collimation()
108	self.aperture = Aperture()
109	self.process = Process()
110	self.trans_spectrum = TransmissionSpectrum()
111
112	def read_children(self, data, parent_list):
113	"""
114	A recursive method for stepping through the hierarchical data file.
115
116	:param data: h5py Group object of any kind
117	:param parent: h5py Group parent name
118	"""
119
120	# Loop through each element of the parent and process accordingly
121	for key in data.keys():
122	# Get all information for the current key
123	value = data.get(key)
124	if value.attrs.get(u'canSAS_class') is not None:
125	class_name = value.attrs.get(u'canSAS_class')
126	else:
127	class_name = value.attrs.get(u'NX_class')
128	if class_name is not None:
129	class_prog = re.compile(class_name)
130	else:
131	class_prog = re.compile(value.name)
132
133	if isinstance(value, h5py.Group):
134	self.parent_class = class_name
135	parent_list.append(key)
136	# If a new sasentry, store the current data sets and create
137	# a fresh Data1D/2D object
138	if class_prog.match(u'SASentry'):
139	self.add_data_set(key)
140	elif class_prog.match(u'SASdata'):
141	self._initialize_new_data_set(parent_list)
142	# Recursion step to access data within the group
143	self.read_children(value, parent_list)
144	self.add_intermediate()
145	parent_list.remove(key)
146
147	elif isinstance(value, h5py.Dataset):
148	# If this is a dataset, store the data appropriately
149	data_set = data[key][:]
150	unit = self._get_unit(value)
151
152	# I and Q Data
153	if key == u'I':
154	if isinstance(self.current_dataset, plottable_2D):
155	self.current_dataset.data = data_set
156	self.current_dataset.zaxis("Intensity", unit)
157	else:
158	self.current_dataset.y = data_set.flatten()
159	self.current_dataset.yaxis("Intensity", unit)
160	continue
161	elif key == u'Idev':
162	if isinstance(self.current_dataset, plottable_2D):
163	self.current_dataset.err_data = data_set.flatten()
164	else:
165	self.current_dataset.dy = data_set.flatten()
166	continue
167	elif key == u'Q':
168	self.current_dataset.xaxis("Q", unit)
169	if isinstance(self.current_dataset, plottable_2D):
170	self.current_dataset.q = data_set.flatten()
171	else:
172	self.current_dataset.x = data_set.flatten()
173	continue
174	elif key == u'Qdev':
175	self.current_dataset.dx = data_set.flatten()
176	continue
177	elif key == u'dQw':
178	self.current_dataset.dxw = data_set.flatten()
179	continue
180	elif key == u'dQl':
181	self.current_dataset.dxl = data_set.flatten()
182	continue
183	elif key == u'Qy':
184	self.current_dataset.yaxis("Q_y", unit)
185	self.current_dataset.qy_data = data_set.flatten()
186	continue
187	elif key == u'Qydev':
188	self.current_dataset.dqy_data = data_set.flatten()
189	continue
190	elif key == u'Qx':
191	self.current_dataset.xaxis("Q_x", unit)
192	self.current_dataset.qx_data = data_set.flatten()
193	continue
194	elif key == u'Qxdev':
195	self.current_dataset.dqx_data = data_set.flatten()
196	continue
197	elif key == u'Mask':
198	self.current_dataset.mask = data_set.flatten()
199	continue
200	# Transmission Spectrum
201	elif (key == u'T'
202	and self.parent_class == u'SAStransmission_spectrum'):
203	self.trans_spectrum.transmission = data_set.flatten()
204	continue
205	elif (key == u'Tdev'
206	and self.parent_class == u'SAStransmission_spectrum'):
207	self.trans_spectrum.transmission_deviation = \
208	data_set.flatten()
209	continue
210	elif (key == u'lambda'
211	and self.parent_class == u'SAStransmission_spectrum'):
212	self.trans_spectrum.wavelength = data_set.flatten()
213	continue
214
215	for data_point in data_set:
216	# Top Level Meta Data
217	if key == u'definition':
218	self.current_datainfo.meta_data['reader'] = data_point
219	elif key == u'run':
220	self.current_datainfo.run.append(data_point)
221	try:
222	run_name = value.attrs['name']
223	run_dict = {data_point: run_name}
224	self.current_datainfo.run_name = run_dict
225	except:
226	pass
227	elif key == u'title':
228	self.current_datainfo.title = data_point
229	elif key == u'SASnote':
230	self.current_datainfo.notes.append(data_point)
231
232	# Sample Information
233	# CanSAS 2.0 format
234	elif key == u'Title' and self.parent_class == u'SASsample':
235	self.current_datainfo.sample.name = data_point
236	# NXcanSAS format
237	elif key == u'name' and self.parent_class == u'SASsample':
238	self.current_datainfo.sample.name = data_point
239	# NXcanSAS format
240	elif key == u'ID' and self.parent_class == u'SASsample':
241	self.current_datainfo.sample.name = data_point
242	elif (key == u'thickness'
243	and self.parent_class == u'SASsample'):
244	self.current_datainfo.sample.thickness = data_point
245	elif (key == u'temperature'
246	and self.parent_class == u'SASsample'):
247	self.current_datainfo.sample.temperature = data_point
248	elif (key == u'transmission'
249	and self.parent_class == u'SASsample'):
250	self.current_datainfo.sample.transmission = data_point
251	elif (key == u'x_position'
252	and self.parent_class == u'SASsample'):
253	self.current_datainfo.sample.position.x = data_point
254	elif (key == u'y_position'
255	and self.parent_class == u'SASsample'):
256	self.current_datainfo.sample.position.y = data_point
257	elif key == u'pitch' and self.parent_class == u'SASsample':
258	self.current_datainfo.sample.orientation.x = data_point
259	elif key == u'yaw' and self.parent_class == u'SASsample':
260	self.current_datainfo.sample.orientation.y = data_point
261	elif key == u'roll' and self.parent_class == u'SASsample':
262	self.current_datainfo.sample.orientation.z = data_point
263	elif (key == u'details'
264	and self.parent_class == u'SASsample'):
265	self.current_datainfo.sample.details.append(data_point)
266
267	# Instrumental Information
268	elif (key == u'name'
269	and self.parent_class == u'SASinstrument'):
270	self.current_datainfo.instrument = data_point
271	elif key == u'name' and self.parent_class == u'SASdetector':
272	self.detector.name = data_point
273	elif key == u'SDD' and self.parent_class == u'SASdetector':
274	self.detector.distance = float(data_point)
275	self.detector.distance_unit = unit
276	elif (key == u'slit_length'
277	and self.parent_class == u'SASdetector'):
278	self.detector.slit_length = float(data_point)
279	self.detector.slit_length_unit = unit
280	elif (key == u'x_position'
281	and self.parent_class == u'SASdetector'):
282	self.detector.offset.x = float(data_point)
283	self.detector.offset_unit = unit
284	elif (key == u'y_position'
285	and self.parent_class == u'SASdetector'):
286	self.detector.offset.y = float(data_point)
287	self.detector.offset_unit = unit
288	elif (key == u'pitch'
289	and self.parent_class == u'SASdetector'):
290	self.detector.orientation.x = float(data_point)
291	self.detector.orientation_unit = unit
292	elif key == u'roll' and self.parent_class == u'SASdetector':
293	self.detector.orientation.z = float(data_point)
294	self.detector.orientation_unit = unit
295	elif key == u'yaw' and self.parent_class == u'SASdetector':
296	self.detector.orientation.y = float(data_point)
297	self.detector.orientation_unit = unit
298	elif (key == u'beam_center_x'
299	and self.parent_class == u'SASdetector'):
300	self.detector.beam_center.x = float(data_point)
301	self.detector.beam_center_unit = unit
302	elif (key == u'beam_center_y'
303	and self.parent_class == u'SASdetector'):
304	self.detector.beam_center.y = float(data_point)
305	self.detector.beam_center_unit = unit
306	elif (key == u'x_pixel_size'
307	and self.parent_class == u'SASdetector'):
308	self.detector.pixel_size.x = float(data_point)
309	self.detector.pixel_size_unit = unit
310	elif (key == u'y_pixel_size'
311	and self.parent_class == u'SASdetector'):
312	self.detector.pixel_size.y = float(data_point)
313	self.detector.pixel_size_unit = unit
314	elif (key == u'distance'
315	and self.parent_class == u'SAScollimation'):
316	self.collimation.length = data_point
317	self.collimation.length_unit = unit
318	elif (key == u'name'
319	and self.parent_class == u'SAScollimation'):
320	self.collimation.name = data_point
321	elif (key == u'shape'
322	and self.parent_class == u'SASaperture'):
323	self.aperture.shape = data_point
324	elif (key == u'x_gap'
325	and self.parent_class == u'SASaperture'):
326	self.aperture.size.x = data_point
327	elif (key == u'y_gap'
328	and self.parent_class == u'SASaperture'):
329	self.aperture.size.y = data_point
330
331	# Process Information
332	elif (key == u'Title'
333	and self.parent_class == u'SASprocess'): # CanSAS 2.0
334	self.process.name = data_point
335	elif (key == u'name'
336	and self.parent_class == u'SASprocess'): # NXcanSAS
337	self.process.name = data_point
338	elif (key == u'description'
339	and self.parent_class == u'SASprocess'):
340	self.process.description = data_point
341	elif key == u'date' and self.parent_class == u'SASprocess':
342	self.process.date = data_point
343	elif key == u'term' and self.parent_class == u'SASprocess':
344	self.process.term = data_point
345	elif self.parent_class == u'SASprocess':
346	self.process.notes.append(data_point)
347
348	# Source
349	elif (key == u'wavelength'
350	and self.parent_class == u'SASdata'):
351	self.current_datainfo.source.wavelength = data_point
352	self.current_datainfo.source.wavelength_unit = unit
353	elif (key == u'incident_wavelength'
354	and self.parent_class == 'SASsource'):
355	self.current_datainfo.source.wavelength = data_point
356	self.current_datainfo.source.wavelength_unit = unit
357	elif (key == u'wavelength_max'
358	and self.parent_class == u'SASsource'):
359	self.current_datainfo.source.wavelength_max = data_point
360	self.current_datainfo.source.wavelength_max_unit = unit
361	elif (key == u'wavelength_min'
362	and self.parent_class == u'SASsource'):
363	self.current_datainfo.source.wavelength_min = data_point
364	self.current_datainfo.source.wavelength_min_unit = unit
365	elif (key == u'incident_wavelength_spread'
366	and self.parent_class == u'SASsource'):
367	self.current_datainfo.source.wavelength_spread = \
368	data_point
369	self.current_datainfo.source.wavelength_spread_unit = \
370	unit
371	elif (key == u'beam_size_x'
372	and self.parent_class == u'SASsource'):
373	self.current_datainfo.source.beam_size.x = data_point
374	self.current_datainfo.source.beam_size_unit = unit
375	elif (key == u'beam_size_y'
376	and self.parent_class == u'SASsource'):
377	self.current_datainfo.source.beam_size.y = data_point
378	self.current_datainfo.source.beam_size_unit = unit
379	elif (key == u'beam_shape'
380	and self.parent_class == u'SASsource'):
381	self.current_datainfo.source.beam_shape = data_point
382	elif (key == u'radiation'
383	and self.parent_class == u'SASsource'):
384	self.current_datainfo.source.radiation = data_point
385	elif (key == u'transmission'
386	and self.parent_class == u'SASdata'):
387	self.current_datainfo.sample.transmission = data_point
388
389	# Everything else goes in meta_data
390	else:
391	new_key = self._create_unique_key(
392	self.current_datainfo.meta_data, key)
393	self.current_datainfo.meta_data[new_key] = data_point
394
395	else:
396	# I don't know if this reachable code
397	self.errors.add("ShouldNeverHappenException")
398
399	def add_intermediate(self):
400	"""
401	This method stores any intermediate objects within the final data set
402	after fully reading the set.
403
404	:param parent: The NXclass name for the h5py Group object that just
405	finished being processed
406	"""
407
408	if self.parent_class == u'SASprocess':
409	self.current_datainfo.process.append(self.process)
410	self.process = Process()
411	elif self.parent_class == u'SASdetector':
412	self.current_datainfo.detector.append(self.detector)
413	self.detector = Detector()
414	elif self.parent_class == u'SAStransmission_spectrum':
415	self.current_datainfo.trans_spectrum.append(self.trans_spectrum)
416	self.trans_spectrum = TransmissionSpectrum()
417	elif self.parent_class == u'SAScollimation':
418	self.current_datainfo.collimation.append(self.collimation)
419	self.collimation = Collimation()
420	elif self.parent_class == u'SASaperture':
421	self.collimation.aperture.append(self.aperture)
422	self.aperture = Aperture()
423	elif self.parent_class == u'SASdata':
424	if isinstance(self.current_dataset, plottable_2D):
425	self.data2d.append(self.current_dataset)
426	elif isinstance(self.current_dataset, plottable_1D):
427	self.data1d.append(self.current_dataset)
428
429	def final_data_cleanup(self):
430	"""
431	Does some final cleanup and formatting on self.current_datainfo and
432	all data1D and data2D objects and then combines the data and info into
433	Data1D and Data2D objects
434	"""
435	# Type cast data arrays to float64
436	if len(self.current_datainfo.trans_spectrum) > 0:
437	spectrum_list = []
438	for spectrum in self.current_datainfo.trans_spectrum:
439	spectrum.transmission = np.delete(spectrum.transmission, [0])
440	spectrum.transmission = spectrum.transmission.astype(np.float64)
441	spectrum.transmission_deviation = np.delete(
442	spectrum.transmission_deviation, [0])
443	spectrum.transmission_deviation = \
444	spectrum.transmission_deviation.astype(np.float64)
445	spectrum.wavelength = np.delete(spectrum.wavelength, [0])
446	spectrum.wavelength = spectrum.wavelength.astype(np.float64)
447	if len(spectrum.transmission) > 0:
448	spectrum_list.append(spectrum)
449	self.current_datainfo.trans_spectrum = spectrum_list
450
451	# Append errors to dataset and reset class errors
452	self.current_datainfo.errors = self.errors
453	self.errors.clear()
454
455	# Combine all plottables with datainfo and append each to output
456	# Type cast data arrays to float64 and find min/max as appropriate
457	for dataset in self.data2d:
458	zeros = np.ones(dataset.data.size, dtype=bool)
459	try:
460	for i in range(0, dataset.mask.size - 1):
461	zeros[i] = dataset.mask[i]
462	except:
463	self.errors.add(sys.exc_value)
464	dataset.mask = zeros
465	# Calculate the actual Q matrix
466	try:
467	if dataset.q_data.size <= 1:
468	dataset.q_data = np.sqrt(dataset.qx_data
469	* dataset.qx_data
470	+ dataset.qy_data
471	* dataset.qy_data)
472	except:
473	dataset.q_data = None
474
475	if dataset.data.ndim == 2:
476	(n_rows, n_cols) = dataset.data.shape
477	dataset.y_bins = dataset.qy_data[0::n_cols]
478	dataset.x_bins = dataset.qx_data[:n_cols]
479	dataset.data = dataset.data.flatten()
480	self.current_dataset = dataset
481	self.send_to_output()
482
483	for dataset in self.data1d:
484	self.current_dataset = dataset
485	self.send_to_output()
486
487	def add_data_set(self, key=""):
488	"""
489	Adds the current_dataset to the list of outputs after preforming final
490	processing on the data and then calls a private method to generate a
491	new data set.
492
493	:param key: NeXus group name for current tree level
494	"""
495
496	if self.current_datainfo and self.current_dataset:
497	self.final_data_cleanup()
498	self.data1d = []
499	self.data2d = []
500	self.current_datainfo = DataInfo()
501
502
503	def _initialize_new_data_set(self, parent_list=None):
504	"""
505	A private class method to generate a new 1D or 2D data object based on
506	the type of data within the set. Outside methods should call
507	add_data_set() to be sure any existing data is stored properly.
508
509	:param parent_list: List of names of parent elements
510	"""
511
512	if parent_list is None:
513	parent_list = []
514	if self._find_intermediate(parent_list, "Qx"):
515	self.current_dataset = plottable_2D()
516	else:
517	x = np.array(0)
518	y = np.array(0)
519	self.current_dataset = plottable_1D(x, y)
520	self.current_datainfo.filename = self.raw_data.filename
521
522	def _find_intermediate(self, parent_list, basename=""):
523	"""
524	A private class used to find an entry by either using a direct key or
525	knowing the approximate basename.
526
527	:param parent_list: List of parents nodes in the HDF5 file
528	:param basename: Approximate name of an entry to search for
529	:return:
530	"""
531
532	entry = False
533	key_prog = re.compile(basename)
534	top = self.raw_data
535	for parent in parent_list:
536	top = top.get(parent)
537	for key in top.keys():
538	if key_prog.match(key):
539	entry = True
540	break
541	return entry
542
543	def _create_unique_key(self, dictionary, name, numb=0):
544	"""
545	Create a unique key value for any dictionary to prevent overwriting
546	Recurses until a unique key value is found.
547
548	:param dictionary: A dictionary with any number of entries
549	:param name: The index of the item to be added to dictionary
550	:param numb: The number to be appended to the name, starts at 0
551	:return: The new name for the dictionary entry
552	"""
553	if dictionary.get(name) is not None:
554	numb += 1
555	name = name.split("_")[0]
556	name += "_{0}".format(numb)
557	name = self._create_unique_key(dictionary, name, numb)
558	return name
559
560	def _get_unit(self, value):
561	"""
562	Find the unit for a particular value within the h5py dictionary
563
564	:param value: attribute dictionary for a particular value set
565	:return: unit for the value passed to the method
566	"""
567	unit = value.attrs.get(u'units')
568	if unit is None:
569	unit = value.attrs.get(u'unit')
570	# Convert the unit formats
571	if unit == "1/A":
572	unit = "A^{-1}"
573	elif unit == "1/cm":
574	unit = "cm^{-1}"
575	return unit

Note: See TracBrowser for help on using the repository browser.

SasView

source: sasview/src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py @ 8dec7e7

Download in other formats: