cansas_reader_HDF5.py @ cd57c7d4

ESS_GUIESS_GUI_DocsESS_GUI_batch_fittingESS_GUI_bumps_abstractionESS_GUI_iss1116ESS_GUI_iss879ESS_GUI_iss959ESS_GUI_openclESS_GUI_orderingESS_GUI_sync_sascalcmagnetic_scattrelease-4.2.2ticket-1009ticket-1094-headlessticket-1242-2d-resolutionticket-1243ticket-1249ticket885unittest-saveload

Last change on this file since cd57c7d4 was cd57c7d4, checked in by krzywon, 7 years ago

Merge branch 'master' into 4_1_issues

# Conflicts:
# src/sas/sascalc/dataloader/readers/cansas_reader.py

Property mode set to 100644

File size: 26.8 KB

Line
1	"""
2	CanSAS 2D data reader for reading HDF5 formatted CanSAS files.
3	"""
4
5	import h5py
6	import numpy as np
7	import re
8	import os
9	import sys
10
11	from sas.sascalc.dataloader.data_info import plottable_1D, plottable_2D,\
12	Data1D, Data2D, DataInfo, Process, Aperture, Collimation, \
13	TransmissionSpectrum, Detector
14	from sas.sascalc.dataloader.data_info import combine_data_info_with_plottable
15	from sas.sascalc.dataloader.loader_exceptions import FileContentsException, DefaultReaderException
16	from sas.sascalc.dataloader.file_reader_base_class import FileReader
17
18
19	class Reader(FileReader):
20	"""
21	A class for reading in CanSAS v2.0 data files. The existing iteration opens
22	Mantid generated HDF5 formatted files with file extension .h5/.H5. Any
23	number of data sets may be present within the file and any dimensionality
24	of data may be used. Currently 1D and 2D SAS data sets are supported, but
25	future implementations will include 1D and 2D SESANS data.
26
27	Any number of SASdata sets may be present in a SASentry and the data within
28	can be either 1D I(Q) or 2D I(Qx, Qy).
29
30	Also supports reading NXcanSAS formatted HDF5 files
31
32	:Dependencies:
33	The CanSAS HDF5 reader requires h5py => v2.5.0 or later.
34	"""
35
36	# CanSAS version
37	cansas_version = 2.0
38	# Logged warnings or messages
39	logging = None
40	# List of errors for the current data set
41	errors = None
42	# Raw file contents to be processed
43	raw_data = None
44	# List of plottable1D objects that should be linked to the current_datainfo
45	data1d = None
46	# List of plottable2D objects that should be linked to the current_datainfo
47	data2d = None
48	# Data type name
49	type_name = "CanSAS 2.0"
50	# Wildcards
51	type = ["CanSAS 2.0 HDF5 Files (.h5)\|.h5"]
52	# List of allowed extensions
53	ext = ['.h5', '.H5']
54	# Flag to bypass extension check
55	allow_all = True
56
57	def get_file_contents(self):
58	"""
59	This is the general read method that all SasView data_loaders must have.
60
61	:param filename: A path for an HDF5 formatted CanSAS 2D data file.
62	:return: List of Data1D/2D objects and/or a list of errors.
63	"""
64	# Reinitialize when loading a new data file to reset all class variables
65	self.reset_class_variables()
66
67	filename = self.f_open.name
68	self.f_open.close() # IO handled by h5py
69
70	# Check that the file exists
71	if os.path.isfile(filename):
72	basename = os.path.basename(filename)
73	_, extension = os.path.splitext(basename)
74	# If the file type is not allowed, return empty list
75	if extension in self.ext or self.allow_all:
76	# Load the data file
77	try:
78	self.raw_data = h5py.File(filename, 'r')
79	except Exception as e:
80	if extension not in self.ext:
81	msg = "CanSAS2.0 HDF5 Reader could not load file {}".format(basename + extension)
82	raise DefaultReaderException(msg)
83	raise FileContentsException(e.message)
84	try:
85	# Read in all child elements of top level SASroot
86	self.read_children(self.raw_data, [])
87	# Add the last data set to the list of outputs
88	self.add_data_set()
89	except Exception as exc:
90	raise FileContentsException(exc.message)
91	finally:
92	# Close the data file
93	self.raw_data.close()
94
95	for dataset in self.output:
96	if isinstance(dataset, Data1D):
97	if dataset.x.size < 5:
98	self.output = []
99	raise FileContentsException("Fewer than 5 data points found.")
100
101	def reset_class_variables(self):
102	"""
103	Create the reader object and define initial states for class variables
104	"""
105	self.current_datainfo = None
106	self.current_dataset = None
107	self.data1d = []
108	self.data2d = []
109	self.raw_data = None
110	self.errors = set()
111	self.logging = []
112	self.output = []
113	self.parent_class = u''
114	self.detector = Detector()
115	self.collimation = Collimation()
116	self.aperture = Aperture()
117	self.process = Process()
118	self.trans_spectrum = TransmissionSpectrum()
119
120	def read_children(self, data, parent_list):
121	"""
122	A recursive method for stepping through the hierarchical data file.
123
124	:param data: h5py Group object of any kind
125	:param parent: h5py Group parent name
126	"""
127
128	# Loop through each element of the parent and process accordingly
129	for key in data.keys():
130	# Get all information for the current key
131	value = data.get(key)
132	if value.attrs.get(u'canSAS_class') is not None:
133	class_name = value.attrs.get(u'canSAS_class')
134	else:
135	class_name = value.attrs.get(u'NX_class')
136	if class_name is not None:
137	class_prog = re.compile(class_name)
138	else:
139	class_prog = re.compile(value.name)
140
141	if isinstance(value, h5py.Group):
142	# Set parent class before recursion
143	self.parent_class = class_name
144	parent_list.append(key)
145	# If a new sasentry, store the current data sets and create
146	# a fresh Data1D/2D object
147	if class_prog.match(u'SASentry'):
148	self.add_data_set(key)
149	elif class_prog.match(u'SASdata'):
150	self._initialize_new_data_set(parent_list)
151	# Recursion step to access data within the group
152	self.read_children(value, parent_list)
153	# Reset parent class when returning from recursive method
154	self.parent_class = class_name
155	self.add_intermediate()
156	parent_list.remove(key)
157
158	elif isinstance(value, h5py.Dataset):
159	# If this is a dataset, store the data appropriately
160	data_set = data[key][:]
161	unit = self._get_unit(value)
162
163	# I and Q Data
164	if key == u'I':
165	if isinstance(self.current_dataset, plottable_2D):
166	self.current_dataset.data = data_set
167	self.current_dataset.zaxis("Intensity", unit)
168	else:
169	self.current_dataset.y = data_set.flatten()
170	self.current_dataset.yaxis("Intensity", unit)
171	continue
172	elif key == u'Idev':
173	if isinstance(self.current_dataset, plottable_2D):
174	self.current_dataset.err_data = data_set.flatten()
175	else:
176	self.current_dataset.dy = data_set.flatten()
177	continue
178	elif key == u'Q':
179	self.current_dataset.xaxis("Q", unit)
180	if isinstance(self.current_dataset, plottable_2D):
181	self.current_dataset.q = data_set.flatten()
182	else:
183	self.current_dataset.x = data_set.flatten()
184	continue
185	elif key == u'Qdev':
186	self.current_dataset.dx = data_set.flatten()
187	continue
188	elif key == u'dQw':
189	self.current_dataset.dxw = data_set.flatten()
190	continue
191	elif key == u'dQl':
192	self.current_dataset.dxl = data_set.flatten()
193	continue
194	elif key == u'Qy':
195	self.current_dataset.yaxis("Q_y", unit)
196	self.current_dataset.qy_data = data_set.flatten()
197	continue
198	elif key == u'Qydev':
199	self.current_dataset.dqy_data = data_set.flatten()
200	continue
201	elif key == u'Qx':
202	self.current_dataset.xaxis("Q_x", unit)
203	self.current_dataset.qx_data = data_set.flatten()
204	continue
205	elif key == u'Qxdev':
206	self.current_dataset.dqx_data = data_set.flatten()
207	continue
208	elif key == u'Mask':
209	self.current_dataset.mask = data_set.flatten()
210	continue
211	# Transmission Spectrum
212	elif (key == u'T'
213	and self.parent_class == u'SAStransmission_spectrum'):
214	self.trans_spectrum.transmission = data_set.flatten()
215	continue
216	elif (key == u'Tdev'
217	and self.parent_class == u'SAStransmission_spectrum'):
218	self.trans_spectrum.transmission_deviation = \
219	data_set.flatten()
220	continue
221	elif (key == u'lambda'
222	and self.parent_class == u'SAStransmission_spectrum'):
223	self.trans_spectrum.wavelength = data_set.flatten()
224	continue
225
226	for data_point in data_set:
227	# Top Level Meta Data
228	if key == u'definition':
229	self.current_datainfo.meta_data['reader'] = data_point
230	elif key == u'run':
231	self.current_datainfo.run.append(data_point)
232	try:
233	run_name = value.attrs['name']
234	run_dict = {data_point: run_name}
235	self.current_datainfo.run_name = run_dict
236	except:
237	pass
238	elif key == u'title':
239	self.current_datainfo.title = data_point
240	elif key == u'SASnote':
241	self.current_datainfo.notes.append(data_point)
242
243	# Sample Information
244	# CanSAS 2.0 format
245	elif key == u'Title' and self.parent_class == u'SASsample':
246	self.current_datainfo.sample.name = data_point
247	# NXcanSAS format
248	elif key == u'name' and self.parent_class == u'SASsample':
249	self.current_datainfo.sample.name = data_point
250	# NXcanSAS format
251	elif key == u'ID' and self.parent_class == u'SASsample':
252	self.current_datainfo.sample.name = data_point
253	elif (key == u'thickness'
254	and self.parent_class == u'SASsample'):
255	self.current_datainfo.sample.thickness = data_point
256	elif (key == u'temperature'
257	and self.parent_class == u'SASsample'):
258	self.current_datainfo.sample.temperature = data_point
259	elif (key == u'transmission'
260	and self.parent_class == u'SASsample'):
261	self.current_datainfo.sample.transmission = data_point
262	elif (key == u'x_position'
263	and self.parent_class == u'SASsample'):
264	self.current_datainfo.sample.position.x = data_point
265	elif (key == u'y_position'
266	and self.parent_class == u'SASsample'):
267	self.current_datainfo.sample.position.y = data_point
268	elif key == u'pitch' and self.parent_class == u'SASsample':
269	self.current_datainfo.sample.orientation.x = data_point
270	elif key == u'yaw' and self.parent_class == u'SASsample':
271	self.current_datainfo.sample.orientation.y = data_point
272	elif key == u'roll' and self.parent_class == u'SASsample':
273	self.current_datainfo.sample.orientation.z = data_point
274	elif (key == u'details'
275	and self.parent_class == u'SASsample'):
276	self.current_datainfo.sample.details.append(data_point)
277
278	# Instrumental Information
279	elif (key == u'name'
280	and self.parent_class == u'SASinstrument'):
281	self.current_datainfo.instrument = data_point
282	elif key == u'name' and self.parent_class == u'SASdetector':
283	self.detector.name = data_point
284	elif key == u'SDD' and self.parent_class == u'SASdetector':
285	self.detector.distance = float(data_point)
286	self.detector.distance_unit = unit
287	elif (key == u'slit_length'
288	and self.parent_class == u'SASdetector'):
289	self.detector.slit_length = float(data_point)
290	self.detector.slit_length_unit = unit
291	elif (key == u'x_position'
292	and self.parent_class == u'SASdetector'):
293	self.detector.offset.x = float(data_point)
294	self.detector.offset_unit = unit
295	elif (key == u'y_position'
296	and self.parent_class == u'SASdetector'):
297	self.detector.offset.y = float(data_point)
298	self.detector.offset_unit = unit
299	elif (key == u'pitch'
300	and self.parent_class == u'SASdetector'):
301	self.detector.orientation.x = float(data_point)
302	self.detector.orientation_unit = unit
303	elif key == u'roll' and self.parent_class == u'SASdetector':
304	self.detector.orientation.z = float(data_point)
305	self.detector.orientation_unit = unit
306	elif key == u'yaw' and self.parent_class == u'SASdetector':
307	self.detector.orientation.y = float(data_point)
308	self.detector.orientation_unit = unit
309	elif (key == u'beam_center_x'
310	and self.parent_class == u'SASdetector'):
311	self.detector.beam_center.x = float(data_point)
312	self.detector.beam_center_unit = unit
313	elif (key == u'beam_center_y'
314	and self.parent_class == u'SASdetector'):
315	self.detector.beam_center.y = float(data_point)
316	self.detector.beam_center_unit = unit
317	elif (key == u'x_pixel_size'
318	and self.parent_class == u'SASdetector'):
319	self.detector.pixel_size.x = float(data_point)
320	self.detector.pixel_size_unit = unit
321	elif (key == u'y_pixel_size'
322	and self.parent_class == u'SASdetector'):
323	self.detector.pixel_size.y = float(data_point)
324	self.detector.pixel_size_unit = unit
325	elif (key == u'distance'
326	and self.parent_class == u'SAScollimation'):
327	self.collimation.length = data_point
328	self.collimation.length_unit = unit
329	elif (key == u'name'
330	and self.parent_class == u'SAScollimation'):
331	self.collimation.name = data_point
332	elif (key == u'shape'
333	and self.parent_class == u'SASaperture'):
334	self.aperture.shape = data_point
335	elif (key == u'x_gap'
336	and self.parent_class == u'SASaperture'):
337	self.aperture.size.x = data_point
338	elif (key == u'y_gap'
339	and self.parent_class == u'SASaperture'):
340	self.aperture.size.y = data_point
341
342	# Process Information
343	elif (key == u'Title'
344	and self.parent_class == u'SASprocess'): # CanSAS 2.0
345	self.process.name = data_point
346	elif (key == u'name'
347	and self.parent_class == u'SASprocess'): # NXcanSAS
348	self.process.name = data_point
349	elif (key == u'description'
350	and self.parent_class == u'SASprocess'):
351	self.process.description = data_point
352	elif key == u'date' and self.parent_class == u'SASprocess':
353	self.process.date = data_point
354	elif key == u'term' and self.parent_class == u'SASprocess':
355	self.process.term = data_point
356	elif self.parent_class == u'SASprocess':
357	self.process.notes.append(data_point)
358
359	# Source
360	elif (key == u'wavelength'
361	and self.parent_class == u'SASdata'):
362	self.current_datainfo.source.wavelength = data_point
363	self.current_datainfo.source.wavelength_unit = unit
364	elif (key == u'incident_wavelength'
365	and self.parent_class == 'SASsource'):
366	self.current_datainfo.source.wavelength = data_point
367	self.current_datainfo.source.wavelength_unit = unit
368	elif (key == u'wavelength_max'
369	and self.parent_class == u'SASsource'):
370	self.current_datainfo.source.wavelength_max = data_point
371	self.current_datainfo.source.wavelength_max_unit = unit
372	elif (key == u'wavelength_min'
373	and self.parent_class == u'SASsource'):
374	self.current_datainfo.source.wavelength_min = data_point
375	self.current_datainfo.source.wavelength_min_unit = unit
376	elif (key == u'incident_wavelength_spread'
377	and self.parent_class == u'SASsource'):
378	self.current_datainfo.source.wavelength_spread = \
379	data_point
380	self.current_datainfo.source.wavelength_spread_unit = \
381	unit
382	elif (key == u'beam_size_x'
383	and self.parent_class == u'SASsource'):
384	self.current_datainfo.source.beam_size.x = data_point
385	self.current_datainfo.source.beam_size_unit = unit
386	elif (key == u'beam_size_y'
387	and self.parent_class == u'SASsource'):
388	self.current_datainfo.source.beam_size.y = data_point
389	self.current_datainfo.source.beam_size_unit = unit
390	elif (key == u'beam_shape'
391	and self.parent_class == u'SASsource'):
392	self.current_datainfo.source.beam_shape = data_point
393	elif (key == u'radiation'
394	and self.parent_class == u'SASsource'):
395	self.current_datainfo.source.radiation = data_point
396	elif (key == u'transmission'
397	and self.parent_class == u'SASdata'):
398	self.current_datainfo.sample.transmission = data_point
399
400	# Everything else goes in meta_data
401	else:
402	new_key = self._create_unique_key(
403	self.current_datainfo.meta_data, key)
404	self.current_datainfo.meta_data[new_key] = data_point
405
406	else:
407	# I don't know if this reachable code
408	self.errors.add("ShouldNeverHappenException")
409
410	def add_intermediate(self):
411	"""
412	This method stores any intermediate objects within the final data set
413	after fully reading the set.
414
415	:param parent: The NXclass name for the h5py Group object that just
416	finished being processed
417	"""
418
419	if self.parent_class == u'SASprocess':
420	self.current_datainfo.process.append(self.process)
421	self.process = Process()
422	elif self.parent_class == u'SASdetector':
423	self.current_datainfo.detector.append(self.detector)
424	self.detector = Detector()
425	elif self.parent_class == u'SAStransmission_spectrum':
426	self.current_datainfo.trans_spectrum.append(self.trans_spectrum)
427	self.trans_spectrum = TransmissionSpectrum()
428	elif self.parent_class == u'SAScollimation':
429	self.current_datainfo.collimation.append(self.collimation)
430	self.collimation = Collimation()
431	elif self.parent_class == u'SASaperture':
432	self.collimation.aperture.append(self.aperture)
433	self.aperture = Aperture()
434	elif self.parent_class == u'SASdata':
435	if isinstance(self.current_dataset, plottable_2D):
436	self.data2d.append(self.current_dataset)
437	elif isinstance(self.current_dataset, plottable_1D):
438	self.data1d.append(self.current_dataset)
439
440	def final_data_cleanup(self):
441	"""
442	Does some final cleanup and formatting on self.current_datainfo and
443	all data1D and data2D objects and then combines the data and info into
444	Data1D and Data2D objects
445	"""
446	# Type cast data arrays to float64
447	if len(self.current_datainfo.trans_spectrum) > 0:
448	spectrum_list = []
449	for spectrum in self.current_datainfo.trans_spectrum:
450	spectrum.transmission = np.delete(spectrum.transmission, [0])
451	spectrum.transmission = spectrum.transmission.astype(np.float64)
452	spectrum.transmission_deviation = np.delete(
453	spectrum.transmission_deviation, [0])
454	spectrum.transmission_deviation = \
455	spectrum.transmission_deviation.astype(np.float64)
456	spectrum.wavelength = np.delete(spectrum.wavelength, [0])
457	spectrum.wavelength = spectrum.wavelength.astype(np.float64)
458	if len(spectrum.transmission) > 0:
459	spectrum_list.append(spectrum)
460	self.current_datainfo.trans_spectrum = spectrum_list
461
462	# Append errors to dataset and reset class errors
463	self.current_datainfo.errors = self.errors
464	self.errors.clear()
465
466	# Combine all plottables with datainfo and append each to output
467	# Type cast data arrays to float64 and find min/max as appropriate
468	for dataset in self.data2d:
469	zeros = np.ones(dataset.data.size, dtype=bool)
470	try:
471	for i in range(0, dataset.mask.size - 1):
472	zeros[i] = dataset.mask[i]
473	except:
474	self.errors.add(sys.exc_value)
475	dataset.mask = zeros
476	# Calculate the actual Q matrix
477	try:
478	if dataset.q_data.size <= 1:
479	dataset.q_data = np.sqrt(dataset.qx_data
480	* dataset.qx_data
481	+ dataset.qy_data
482	* dataset.qy_data)
483	except:
484	dataset.q_data = None
485
486	if dataset.data.ndim == 2:
487	(n_rows, n_cols) = dataset.data.shape
488	dataset.y_bins = dataset.qy_data[0::n_cols]
489	dataset.x_bins = dataset.qx_data[:n_cols]
490	dataset.data = dataset.data.flatten()
491	self.current_dataset = dataset
492	self.send_to_output()
493
494	for dataset in self.data1d:
495	self.current_dataset = dataset
496	self.send_to_output()
497
498	def add_data_set(self, key=""):
499	"""
500	Adds the current_dataset to the list of outputs after preforming final
501	processing on the data and then calls a private method to generate a
502	new data set.
503
504	:param key: NeXus group name for current tree level
505	"""
506
507	if self.current_datainfo and self.current_dataset:
508	self.final_data_cleanup()
509	self.data1d = []
510	self.data2d = []
511	self.current_datainfo = DataInfo()
512
513
514	def _initialize_new_data_set(self, parent_list=None):
515	"""
516	A private class method to generate a new 1D or 2D data object based on
517	the type of data within the set. Outside methods should call
518	add_data_set() to be sure any existing data is stored properly.
519
520	:param parent_list: List of names of parent elements
521	"""
522
523	if parent_list is None:
524	parent_list = []
525	if self._find_intermediate(parent_list, "Qx"):
526	self.current_dataset = plottable_2D()
527	else:
528	x = np.array(0)
529	y = np.array(0)
530	self.current_dataset = plottable_1D(x, y)
531	self.current_datainfo.filename = self.raw_data.filename
532
533	def _find_intermediate(self, parent_list, basename=""):
534	"""
535	A private class used to find an entry by either using a direct key or
536	knowing the approximate basename.
537
538	:param parent_list: List of parents nodes in the HDF5 file
539	:param basename: Approximate name of an entry to search for
540	:return:
541	"""
542
543	entry = False
544	key_prog = re.compile(basename)
545	top = self.raw_data
546	for parent in parent_list:
547	top = top.get(parent)
548	for key in top.keys():
549	if key_prog.match(key):
550	entry = True
551	break
552	return entry
553
554	def _create_unique_key(self, dictionary, name, numb=0):
555	"""
556	Create a unique key value for any dictionary to prevent overwriting
557	Recurses until a unique key value is found.
558
559	:param dictionary: A dictionary with any number of entries
560	:param name: The index of the item to be added to dictionary
561	:param numb: The number to be appended to the name, starts at 0
562	:return: The new name for the dictionary entry
563	"""
564	if dictionary.get(name) is not None:
565	numb += 1
566	name = name.split("_")[0]
567	name += "_{0}".format(numb)
568	name = self._create_unique_key(dictionary, name, numb)
569	return name
570
571	def _get_unit(self, value):
572	"""
573	Find the unit for a particular value within the h5py dictionary
574
575	:param value: attribute dictionary for a particular value set
576	:return: unit for the value passed to the method
577	"""
578	unit = value.attrs.get(u'units')
579	if unit is None:
580	unit = value.attrs.get(u'unit')
581	# Convert the unit formats
582	if unit == "1/A":
583	unit = "A^{-1}"
584	elif unit == "1/cm":
585	unit = "cm^{-1}"
586	return unit

Note: See TracBrowser for help on using the repository browser.

SasView

source: sasview/src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py @ cd57c7d4

Download in other formats: