Changeset 8db20a9 in sasview for src/sas/sascalc

Timestamp:

Jan 21, 2019 8:15:40 AM (6 years ago)

Author:

Piotr Rozyczko <piotr.rozyczko@…>

Branches:

ESS_GUI, ESS_GUI_bumps_abstraction, ESS_GUI_iss1116, ESS_GUI_opencl, ESS_GUI_sync_sascalc

Children:

d541324e

Parents:

3ca645bb

git-author:

Piotr Rozyczko <piotr.rozyczko@…> (01/21/19 08:12:08)

git-committer:

Piotr Rozyczko <piotr.rozyczko@…> (01/21/19 08:15:40)

Message:

Updated cansas read (cherrypicked and fixed from master).
Fixes: hdf5 returns byte strings so these need to be recasted properly.
​https://github.com/h5py/h5py/issues/379

Location:

src/sas/sascalc

Files:

• data_util/nxsunit.py (modified) (2 diffs)
• data_util/registry.py (modified) (1 diff)
• dataloader/data_info.py (modified) (4 diffs)
• dataloader/file_reader_base_class.py (modified) (10 diffs)
• dataloader/loader.py (modified) (1 diff)
• dataloader/readers/abs_reader.py (modified) (4 diffs)
• dataloader/readers/ascii_reader.py (modified) (1 diff)
• dataloader/readers/associations.py (modified) (1 diff)
• dataloader/readers/cansas_reader.py (modified) (1 diff)
• dataloader/readers/cansas_reader_HDF5.py (modified) (18 diffs)
• dataloader/readers/danse_reader.py (modified) (1 diff)
• dataloader/readers/red2d_reader.py (modified) (1 diff)
• file_converter/nxcansas_writer.py (modified) (12 diffs)

src/sas/sascalc/data_util/nxsunit.py

@@ -136 +136 @@
     sld = { '10^-6 Angstrom^-2': 1e-6, 'Angstrom^-2': 1 }
     Q = { 'invA': 1, 'invAng': 1, 'invAngstroms': 1, '1/A': 1,
+          '1/Angstrom': 1, '1/angstrom': 1, 'A^{-1}': 1, 'cm^{-1}': 1e-8,
           '10^-3 Angstrom^-1': 1e-3, '1/cm': 1e-8, '1/m': 1e-10,
-          'nm^-1': 0.1, '1/nm': 0.1, 'n_m^-1': 0.1 }
+          'nm^{-1}': 1, 'nm^-1': 0.1, '1/nm': 0.1, 'n_m^-1': 0.1 }
 
     _caret_optional(sld)
@@ -157 +158 @@
     # units for that particular dimension.
     # Note: don't have support for dimensionless units.
-    unknown = {None:1, '???':1, '': 1, 'a.u.': 1}
+    unknown = {None:1, '???':1, '': 1, 'a.u.': 1, 'Counts': 1, 'counts': 1}
 
     def __init__(self, name):

src/sas/sascalc/data_util/registry.py

@@ -111 +111 @@
             for L in loaders:
                 if L not in result: result.append(L)
-            loaders = L
+            loaders = result
+            #loaders = L
         # Raise an error if there are no matching extensions
         if len(loaders) == 0:

src/sas/sascalc/dataloader/data_info.py

@@ -954 +954 @@
         _str += "Data:\n"
         _str += "   Type:         %s\n" % self.__class__.__name__
-        _str += "   X- & Y-axis:  %s\t[%s]\n" % (self._yaxis, self._yunit)
+        _str += "   X-axis:       %s\t[%s]\n" % (self._xaxis, self._xunit)
+        _str += "   Y-axis:       %s\t[%s]\n" % (self._yaxis, self._yunit)
         _str += "   Z-axis:       %s\t[%s]\n" % (self._zaxis, self._zunit)
         _str += "   Length:       %g \n" % (len(self.data))
@@ -983 +984 @@
                            qx_data=qx_data, qy_data=qy_data,
                            q_data=q_data, mask=mask)
+
+        clone._xaxis = self._xaxis
+        clone._yaxis = self._yaxis
+        clone._zaxis = self._zaxis
+        clone._xunit = self._xunit
+        clone._yunit = self._yunit
+        clone._zunit = self._zunit
+        clone.x_bins = self.x_bins
+        clone.y_bins = self.y_bins
 
         clone.title = self.title
@@ -1153 +1163 @@
 def combine_data_info_with_plottable(data, datainfo):
     """
-    A function that combines the DataInfo data in self.current_datainto with a plottable_1D or 2D data object.
+    A function that combines the DataInfo data in self.current_datainto with a
+    plottable_1D or 2D data object.
 
     :param data: A plottable_1D or plottable_2D data object
@@ -1171 +1182 @@
         final_dataset.yaxis(data._yaxis, data._yunit)
     elif isinstance(data, plottable_2D):
-        final_dataset = Data2D(data.data, data.err_data, data.qx_data, data.qy_data, data.q_data,
-                               data.mask, data.dqx_data, data.dqy_data)
+        final_dataset = Data2D(data.data, data.err_data, data.qx_data,
+                               data.qy_data, data.q_data, data.mask,
+                               data.dqx_data, data.dqy_data)
         final_dataset.xaxis(data._xaxis, data._xunit)
         final_dataset.yaxis(data._yaxis, data._yunit)
         final_dataset.zaxis(data._zaxis, data._zunit)
-        if len(data.data.shape) == 2:
-            n_rows, n_cols = data.data.shape
-            final_dataset.y_bins = data.qy_data[0::int(n_cols)]
-            final_dataset.x_bins = data.qx_data[:int(n_cols)]
+        final_dataset.y_bins = data.y_bins
+        final_dataset.x_bins = data.x_bins
     else:
-        return_string = "Should Never Happen: _combine_data_info_with_plottable input is not a plottable1d or " + \
-                        "plottable2d data object"
+        return_string = ("Should Never Happen: _combine_data_info_with_plottabl"
+                         "e input is not a plottable1d or plottable2d data "
+                         "object")
         return return_string
 

src/sas/sascalc/dataloader/file_reader_base_class.py

@@ -16 +16 @@
 from .data_info import Data1D, Data2D, DataInfo, plottable_1D, plottable_2D,\
     combine_data_info_with_plottable
+from sas.sascalc.data_util.nxsunit import Converter
 
 logger = logging.getLogger(__name__)
@@ -37 +38 @@
                        "SasView cannot guarantee the accuracy of the data.")
 
+
 class FileReader(object):
     # String to describe the type of data this reader can load
@@ -45 +47 @@
     ext = ['.txt']
     # Deprecated extensions
-    deprecated_extensions = ['.asc', '.nxs']
+    deprecated_extensions = ['.asc']
     # Bypass extension check and try to load anyway
     allow_all = False
@@ -98 +100 @@
                     if len(self.output) > 0:
                         # Sort the data that's been loaded
-                        self.sort_one_d_data()
-                        self.sort_two_d_data()
+                        self.convert_data_units()
+                        self.sort_data()
         else:
             msg = "Unable to find file at: {}\n".format(filepath)
@@ -140 +142 @@
         Returns the entire file as a string.
         """
-        #return self.f_open.read()
         return decode(self.f_open.read())
 
@@ -166 +167 @@
         self.output.append(data_obj)
 
-    def sort_one_d_data(self):
+    def sort_data(self):
         """
         Sort 1D data along the X axis for consistency
@@ -174 +175 @@
                 # Normalize the units for
                 data.x_unit = self.format_unit(data.x_unit)
+                data._xunit = data.x_unit
                 data.y_unit = self.format_unit(data.y_unit)
+                data._yunit = data.y_unit
                 # Sort data by increasing x and remove 1st point
                 ind = np.lexsort((data.y, data.x))
@@ -203 +206 @@
                     data.ymin = np.min(data.y)
                     data.ymax = np.max(data.y)
+            elif isinstance(data, Data2D):
+                # Normalize the units for
+                data.Q_unit = self.format_unit(data.Q_unit)
+                data.I_unit = self.format_unit(data.I_unit)
+                data._xunit = data.Q_unit
+                data._yunit = data.Q_unit
+                data._zunit = data.I_unit
+                data.data = data.data.astype(np.float64)
+                data.qx_data = data.qx_data.astype(np.float64)
+                data.xmin = np.min(data.qx_data)
+                data.xmax = np.max(data.qx_data)
+                data.qy_data = data.qy_data.astype(np.float64)
+                data.ymin = np.min(data.qy_data)
+                data.ymax = np.max(data.qy_data)
+                data.q_data = np.sqrt(data.qx_data * data.qx_data
+                                         + data.qy_data * data.qy_data)
+                if data.err_data is not None:
+                    data.err_data = data.err_data.astype(np.float64)
+                if data.dqx_data is not None:
+                    data.dqx_data = data.dqx_data.astype(np.float64)
+                if data.dqy_data is not None:
+                    data.dqy_data = data.dqy_data.astype(np.float64)
+                if data.mask is not None:
+                    data.mask = data.mask.astype(dtype=bool)
+
+                if len(data.data.shape) == 2:
+                    n_rows, n_cols = data.data.shape
+                    data.y_bins = data.qy_data[0::int(n_cols)]
+                    data.x_bins = data.qx_data[:int(n_cols)]
+                    data.data = data.data.flatten()
+                data = self._remove_nans_in_data(data)
+                if len(data.data) > 0:
+                    data.xmin = np.min(data.qx_data)
+                    data.xmax = np.max(data.qx_data)
+                    data.ymin = np.min(data.qy_data)
+                    data.ymax = np.max(data.qy_data)
 
     @staticmethod
@@ -242 +281 @@
         return data
 
-    def sort_two_d_data(self):
-        for dataset in self.output:
-            if isinstance(dataset, Data2D):
-                # Normalize the units for
-                dataset.x_unit = self.format_unit(dataset.Q_unit)
-                dataset.y_unit = self.format_unit(dataset.I_unit)
-                dataset.data = dataset.data.astype(np.float64)
-                dataset.qx_data = dataset.qx_data.astype(np.float64)
-                dataset.xmin = np.min(dataset.qx_data)
-                dataset.xmax = np.max(dataset.qx_data)
-                dataset.qy_data = dataset.qy_data.astype(np.float64)
-                dataset.ymin = np.min(dataset.qy_data)
-                dataset.ymax = np.max(dataset.qy_data)
-                dataset.q_data = np.sqrt(dataset.qx_data * dataset.qx_data
-                                         + dataset.qy_data * dataset.qy_data)
-                if dataset.err_data is not None:
-                    dataset.err_data = dataset.err_data.astype(np.float64)
-                if dataset.dqx_data is not None:
-                    dataset.dqx_data = dataset.dqx_data.astype(np.float64)
-                if dataset.dqy_data is not None:
-                    dataset.dqy_data = dataset.dqy_data.astype(np.float64)
-                if dataset.mask is not None:
-                    dataset.mask = dataset.mask.astype(dtype=bool)
-
-                if len(dataset.data.shape) == 2:
-                    n_rows, n_cols = dataset.data.shape
-                    dataset.y_bins = dataset.qy_data[0::int(n_cols)]
-                    dataset.x_bins = dataset.qx_data[:int(n_cols)]
-                dataset.data = dataset.data.flatten()
-                dataset = self._remove_nans_in_data(dataset)
-                if len(dataset.data) > 0:
-                    dataset.xmin = np.min(dataset.qx_data)
-                    dataset.xmax = np.max(dataset.qx_data)
-                    dataset.ymin = np.min(dataset.qy_data)
-                    dataset.ymax = np.max(dataset.qy_data)
+    @staticmethod
+    def set_default_1d_units(data):
+        """
+        Set the x and y axes to the default 1D units
+        :param data: 1D data set
+        :return:
+        """
+        data.xaxis(r"\rm{Q}", '1/A')
+        data.yaxis(r"\rm{Intensity}", "1/cm")
+        return data
+
+    @staticmethod
+    def set_default_2d_units(data):
+        """
+        Set the x and y axes to the default 2D units
+        :param data: 2D data set
+        :return:
+        """
+        data.xaxis("\\rm{Q_{x}}", '1/A')
+        data.yaxis("\\rm{Q_{y}}", '1/A')
+        data.zaxis("\\rm{Intensity}", "1/cm")
+        return data
+
+    def convert_data_units(self, default_q_unit="1/A"):
+        """
+        Converts al; data to the sasview default of units of A^{-1} for Q and
+        cm^{-1} for I.
+        :param default_q_unit: The default Q unit used by Sasview
+        """
+        convert_q = True
+        new_output = []
+        for data in self.output:
+            if data.isSesans:
+                new_output.append(data)
+                continue
+            try:
+                file_x_unit = data._xunit
+                data_conv_x = Converter(file_x_unit)
+            except KeyError:
+                logger.info("Unrecognized Q units in data file. No data "
+                            "conversion attempted")
+                convert_q = False
+            try:
+
+                if isinstance(data, Data1D):
+                        if convert_q:
+                            data.x = data_conv_x(data.x, units=default_q_unit)
+                            data._xunit = default_q_unit
+                            data.x_unit = default_q_unit
+                            if data.dx is not None:
+                                data.dx = data_conv_x(data.dx,
+                                                      units=default_q_unit)
+                            if data.dxl is not None:
+                                data.dxl = data_conv_x(data.dxl,
+                                                       units=default_q_unit)
+                            if data.dxw is not None:
+                                data.dxw = data_conv_x(data.dxw,
+                                                       units=default_q_unit)
+                elif isinstance(data, Data2D):
+                    if convert_q:
+                        data.qx_data = data_conv_x(data.qx_data,
+                                                   units=default_q_unit)
+                        if data.dqx_data is not None:
+                            data.dqx_data = data_conv_x(data.dqx_data,
+                                                        units=default_q_unit)
+                        try:
+                            file_y_unit = data._yunit
+                            data_conv_y = Converter(file_y_unit)
+                            data.qy_data = data_conv_y(data.qy_data,
+                                                       units=default_q_unit)
+                            if data.dqy_data is not None:
+                                data.dqy_data = data_conv_y(data.dqy_data,
+                                                            units=default_q_unit)
+                        except KeyError:
+                            logger.info("Unrecognized Qy units in data file. No"
+                                        " data conversion attempted")
+            except KeyError:
+                message = "Unable to convert Q units from {0} to 1/A."
+                message.format(default_q_unit)
+                data.errors.append(message)
+            new_output.append(data)
+        self.output = new_output
 
     def format_unit(self, unit=None):
@@ -367 +453 @@
                     self.current_dataset.qy_data))
             if has_error_dy:
-                self.current_dataset.err_data = self.current_dataset.err_data[x != 0]
+                self.current_dataset.err_data = self.current_dataset.err_data[
+                    x != 0]
             if has_error_dqx:
-                self.current_dataset.dqx_data = self.current_dataset.dqx_data[x != 0]
+                self.current_dataset.dqx_data = self.current_dataset.dqx_data[
+                    x != 0]
             if has_error_dqy:
-                self.current_dataset.dqy_data = self.current_dataset.dqy_data[x != 0]
+                self.current_dataset.dqy_data = self.current_dataset.dqy_data[
+                    x != 0]
             if has_mask:
                 self.current_dataset.mask = self.current_dataset.mask[x != 0]

src/sas/sascalc/dataloader/loader.py

@@ -367 +367 @@
             try:
                 return fn(path, data)
-            except Exception:
-                pass  # give other loaders a chance to succeed
-        # If we get here it is because all loaders failed
-        raise  # reraises last exception
+            except Exception as exc:
+                msg = "Saving file {} using the {} writer failed.\n".format(
+                    path, type(fn).__name__)
+                msg += str(exc)
+                logger.exception(msg)  # give other loaders a chance to succeed
 
 

src/sas/sascalc/dataloader/readers/abs_reader.py

@@ -48 +48 @@
         detector = Detector()
         data_line = 0
+        x_index = 4
         self.reset_data_list(len(lines))
         self.current_datainfo.detector.append(detector)
@@ -63 +64 @@
         for line in lines:
             # Information line 1
+            if line.find(".bt5") > 0:
+                x_index = 0
             if is_info:
                 is_info = False
@@ -171 +174 @@
 
                 try:
-                    _x = float(toks[4])
+                    _x = float(toks[x_index])
                     _y = float(toks[1])
                     _dy = float(toks[2])
@@ -225 +228 @@
             raise ValueError("ascii_reader: could not load file")
 
+        self.current_dataset = self.set_default_1d_units(self.current_dataset)
         if data_conv_q is not None:
             self.current_dataset.xaxis("\\rm{Q}", base_q_unit)
-        else:
-            self.current_dataset.xaxis("\\rm{Q}", 'A^{-1}')
         if data_conv_i is not None:
             self.current_dataset.yaxis("\\rm{Intensity}", base_i_unit)
-        else:
-            self.current_dataset.yaxis("\\rm{Intensity}", "cm^{-1}")
 
         # Store loading process information

src/sas/sascalc/dataloader/readers/ascii_reader.py

@@ -157 +157 @@
 
         self.remove_empty_q_values()
-        self.current_dataset.xaxis("\\rm{Q}", 'A^{-1}')
-        self.current_dataset.yaxis("\\rm{Intensity}", "cm^{-1}")
+        self.current_dataset = self.set_default_1d_units(self.current_dataset)
 
         # Store loading process information

src/sas/sascalc/dataloader/readers/associations.py

@@ -23 +23 @@
     ".ses": "sesans_reader",
     ".h5": "cansas_reader_HDF5",
+    ".nxs": "cansas_reader_HDF5",
     ".txt": "ascii_reader",
     ".dat": "red2d_reader",

src/sas/sascalc/dataloader/readers/cansas_reader.py

@@ -812 +812 @@
             node.append(point)
             self.write_node(point, "Q", datainfo.x[i],
-                            {'unit': datainfo.x_unit})
+                            {'unit': datainfo._xunit})
             if len(datainfo.y) >= i:
                 self.write_node(point, "I", datainfo.y[i],
-                                {'unit': datainfo.y_unit})
+                                {'unit': datainfo._yunit})
             if datainfo.dy is not None and len(datainfo.dy) > i:
                 self.write_node(point, "Idev", datainfo.dy[i],
-                                {'unit': datainfo.y_unit})
+                                {'unit': datainfo._yunit})
             if datainfo.dx is not None and len(datainfo.dx) > i:
                 self.write_node(point, "Qdev", datainfo.dx[i],
-                                {'unit': datainfo.x_unit})
+                                {'unit': datainfo._xunit})
             if datainfo.dxw is not None and len(datainfo.dxw) > i:
                 self.write_node(point, "dQw", datainfo.dxw[i],
-                                {'unit': datainfo.x_unit})
+                                {'unit': datainfo._xunit})
             if datainfo.dxl is not None and len(datainfo.dxl) > i:
                 self.write_node(point, "dQl", datainfo.dxl[i],
-                                {'unit': datainfo.x_unit})
+                                {'unit': datainfo._xunit})
         if datainfo.isSesans:
             sesans_attrib = {'x_axis': datainfo._xaxis,

src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py

@@ -1 +1 @@
 """
-    CanSAS 2D data reader for reading HDF5 formatted CanSAS files.
+    NXcanSAS data reader for reading HDF5 formatted CanSAS files.
 """
 
@@ -12 +12 @@
     Data1D, Data2D, DataInfo, Process, Aperture, Collimation, \
     TransmissionSpectrum, Detector
-from ..data_info import combine_data_info_with_plottable
 from ..loader_exceptions import FileContentsException, DefaultReaderException
 from ..file_reader_base_class import FileReader, decode
 
+try:
+  basestring
+except NameError:  # CRUFT: python 2 support
+  basestring = str
+
+
 def h5attr(node, key, default=None):
     return decode(node.attrs.get(key, default))
 
+
 class Reader(FileReader):
     """
-    A class for reading in CanSAS v2.0 data files. The existing iteration opens
-    Mantid generated HDF5 formatted files with file extension .h5/.H5. Any
-    number of data sets may be present within the file and any dimensionality
-    of data may be used. Currently 1D and 2D SAS data sets are supported, but
-    future implementations will include 1D and 2D SESANS data.
-
-    Any number of SASdata sets may be present in a SASentry and the data within
-    can be either 1D I(Q) or 2D I(Qx, Qy).
-
-    Also supports reading NXcanSAS formatted HDF5 files
+    A class for reading in NXcanSAS data files. The current implementation has
+    been tested to load data generated by multiple facilities, all of which are
+    known to produce NXcanSAS standards compliant data. Any number of data sets
+    may be present within the file and any dimensionality of data may be used.
+    Currently 1D and 2D SAS data sets are supported, but should be immediately
+    extensible to SESANS data.
+
+    Any number of SASdata groups  may be present in a SASentry and the data
+    within each SASdata group can be a single 1D I(Q), multi-framed 1D I(Q),
+    2D I(Qx, Qy) or multi-framed 2D I(Qx, Qy).
 
     :Dependencies:
-        The CanSAS HDF5 reader requires h5py => v2.5.0 or later.
+        The NXcanSAS HDF5 reader requires h5py => v2.5.0 or later.
     """
 
     # CanSAS version
     cansas_version = 2.0
-    # Logged warnings or messages
-    logging = None
-    # List of errors for the current data set
-    errors = None
-    # Raw file contents to be processed
-    raw_data = None
-    # List of plottable1D objects that should be linked to the current_datainfo
-    data1d = None
-    # List of plottable2D objects that should be linked to the current_datainfo
-    data2d = None
     # Data type name
-    type_name = "CanSAS 2.0"
+    type_name = "NXcanSAS"
     # Wildcards
-    type = ["CanSAS 2.0 HDF5 Files (*.h5)|*.h5"]
+    type = ["NXcanSAS HDF5 Files (*.h5)|*.h5|"]
     # List of allowed extensions
     ext = ['.h5', '.H5']
@@ -81 +77 @@
                 except Exception as e:
                     if extension not in self.ext:
-                        msg = "CanSAS2.0 HDF5 Reader could not load file {}".format(basename + extension)
+                        msg = "NXcanSAS Reader could not load file {}".format(
+                            basename + extension)
                         raise DefaultReaderException(msg)
                     raise FileContentsException(e.message)
@@ -95 +92 @@
                     self.raw_data.close()
 
-                for dataset in self.output:
-                    if isinstance(dataset, Data1D):
-                        if dataset.x.size < 5:
-                            self.output = []
-                            raise FileContentsException("Fewer than 5 data points found.")
+                for data_set in self.output:
+                    if isinstance(data_set, Data1D):
+                        if data_set.x.size < 5:
+                            exception = FileContentsException(
+                                "Fewer than 5 data points found.")
+                            data_set.errors.append(exception)
 
     def reset_state(self):
@@ -109 +107 @@
         self.data2d = []
         self.raw_data = None
-        self.errors = set()
+        self.multi_frame = False
+        self.data_frames = []
+        self.data_uncertainty_frames = []
+        self.errors = []
         self.logging = []
+        self.q_names = []
+        self.mask_name = u''
+        self.i_name = u''
+        self.i_node = u''
+        self.i_uncertainties_name = u''
+        self.q_uncertainty_names = []
+        self.q_resolution_names = []
         self.parent_class = u''
         self.detector = Detector()
@@ -131 +139 @@
             value = data.get(key)
             class_name = h5attr(value, u'canSAS_class')
+            if isinstance(class_name, (list, tuple, np.ndarray)):
+                class_name = class_name[0]
             if class_name is None:
                 class_name = h5attr(value, u'NX_class')
@@ -140 +150 @@
             if isinstance(value, h5py.Group):
                 # Set parent class before recursion
+                last_parent_class = self.parent_class
                 self.parent_class = class_name
                 parent_list.append(key)
@@ -147 +158 @@
                     self.add_data_set(key)
                 elif class_prog.match(u'SASdata'):
-                    self._initialize_new_data_set(parent_list)
+                    self._find_data_attributes(value)
+                    self._initialize_new_data_set(value)
                 # Recursion step to access data within the group
                 self.read_children(value, parent_list)
+                self.add_intermediate()
                 # Reset parent class when returning from recursive method
-                self.parent_class = class_name
-                self.add_intermediate()
+                self.parent_class = last_parent_class
                 parent_list.remove(key)
 
             elif isinstance(value, h5py.Dataset):
                 # If this is a dataset, store the data appropriately
-                data_set = data[key][:]
+                data_set = value.value
                 unit = self._get_unit(value)
-
-                # I and Q Data
-                if key == u'I':
-                    if isinstance(self.current_dataset, plottable_2D):
-                        self.current_dataset.data = data_set
-                        self.current_dataset.zaxis("Intensity", unit)
-                    else:
-                        self.current_dataset.y = data_set.flatten()
-                        self.current_dataset.yaxis("Intensity", unit)
-                    continue
-                elif key == u'Idev':
-                    if isinstance(self.current_dataset, plottable_2D):
-                        self.current_dataset.err_data = data_set.flatten()
-                    else:
-                        self.current_dataset.dy = data_set.flatten()
-                    continue
-                elif key == u'Q':
-                    self.current_dataset.xaxis("Q", unit)
-                    if isinstance(self.current_dataset, plottable_2D):
-                        self.current_dataset.q = data_set.flatten()
-                    else:
-                        self.current_dataset.x = data_set.flatten()
-                    continue
-                elif key == u'Qdev':
-                    self.current_dataset.dx = data_set.flatten()
-                    continue
-                elif key == u'dQw':
-                    self.current_dataset.dxw = data_set.flatten()
-                    continue
-                elif key == u'dQl':
-                    self.current_dataset.dxl = data_set.flatten()
-                    continue
-                elif key == u'Qy':
-                    self.current_dataset.yaxis("Q_y", unit)
-                    self.current_dataset.qy_data = data_set.flatten()
-                    continue
-                elif key == u'Qydev':
-                    self.current_dataset.dqy_data = data_set.flatten()
-                    continue
-                elif key == u'Qx':
-                    self.current_dataset.xaxis("Q_x", unit)
-                    self.current_dataset.qx_data = data_set.flatten()
-                    continue
-                elif key == u'Qxdev':
-                    self.current_dataset.dqx_data = data_set.flatten()
-                    continue
-                elif key == u'Mask':
-                    self.current_dataset.mask = data_set.flatten()
-                    continue
-                # Transmission Spectrum
-                elif (key == u'T'
-                      and self.parent_class == u'SAStransmission_spectrum'):
-                    self.trans_spectrum.transmission = data_set.flatten()
-                    continue
-                elif (key == u'Tdev'
-                      and self.parent_class == u'SAStransmission_spectrum'):
-                    self.trans_spectrum.transmission_deviation = \
-                        data_set.flatten()
-                    continue
-                elif (key == u'lambda'
-                      and self.parent_class == u'SAStransmission_spectrum'):
-                    self.trans_spectrum.wavelength = data_set.flatten()
-                    continue
 
                 for data_point in data_set:
@@ -231 +180 @@
                     # Top Level Meta Data
                     if key == u'definition':
-                        self.current_datainfo.meta_data['reader'] = data_point
+                        if isinstance(data_set, basestring):
+                            self.current_datainfo.meta_data['reader'] = data_set
+                            break
+                        else:
+                            self.current_datainfo.meta_data[
+                                'reader'] = data_point
+                    # Run
                     elif key == u'run':
-                        self.current_datainfo.run.append(data_point)
                         try:
                             run_name = h5attr(value, 'name')
-                            run_dict = {data_point: run_name}
+                            run_dict = {data_set: run_name}
                             self.current_datainfo.run_name = run_dict
                         except Exception:
                             pass
+                        if isinstance(data_set, basestring):
+                            self.current_datainfo.run.append(data_set)
+                            break
+                        else:
+                            self.current_datainfo.run.append(data_point)
+                    # Title
                     elif key == u'title':
-                        self.current_datainfo.title = data_point
+                        if isinstance(data_set, basestring):
+                            self.current_datainfo.title = data_set
+                            break
+                        else:
+                            self.current_datainfo.title = data_point
+                    # Note
                     elif key == u'SASnote':
-                        self.current_datainfo.notes.append(data_point)
-
+                        self.current_datainfo.notes.append(data_set)
+                        break
                     # Sample Information
-                    # CanSAS 2.0 format
-                    elif key == u'Title' and self.parent_class == u'SASsample':
-                        self.current_datainfo.sample.name = data_point
-                    # NXcanSAS format
-                    elif key == u'name' and self.parent_class == u'SASsample':
-                        self.current_datainfo.sample.name = data_point
-                    # NXcanSAS format
-                    elif key == u'ID' and self.parent_class == u'SASsample':
-                        self.current_datainfo.sample.name = data_point
-                    elif (key == u'thickness'
-                          and self.parent_class == u'SASsample'):
-                        self.current_datainfo.sample.thickness = data_point
-                    elif (key == u'temperature'
-                          and self.parent_class == u'SASsample'):
-                        self.current_datainfo.sample.temperature = data_point
-                    elif (key == u'transmission'
-                          and self.parent_class == u'SASsample'):
-                        self.current_datainfo.sample.transmission = data_point
-                    elif (key == u'x_position'
-                          and self.parent_class == u'SASsample'):
-                        self.current_datainfo.sample.position.x = data_point
-                    elif (key == u'y_position'
-                          and self.parent_class == u'SASsample'):
-                        self.current_datainfo.sample.position.y = data_point
-                    elif key == u'pitch' and self.parent_class == u'SASsample':
-                        self.current_datainfo.sample.orientation.x = data_point
-                    elif key == u'yaw' and self.parent_class == u'SASsample':
-                        self.current_datainfo.sample.orientation.y = data_point
-                    elif key == u'roll' and self.parent_class == u'SASsample':
-                        self.current_datainfo.sample.orientation.z = data_point
-                    elif (key == u'details'
-                          and self.parent_class == u'SASsample'):
-                        self.current_datainfo.sample.details.append(data_point)
-
+                    elif self.parent_class == u'SASsample':
+                        self.process_sample(data_point, key)
                     # Instrumental Information
                     elif (key == u'name'
                           and self.parent_class == u'SASinstrument'):
                         self.current_datainfo.instrument = data_point
-                    elif key == u'name' and self.parent_class == u'SASdetector':
-                        self.detector.name = data_point
-                    elif key == u'SDD' and self.parent_class == u'SASdetector':
-                        self.detector.distance = float(data_point)
-                        self.detector.distance_unit = unit
-                    elif (key == u'slit_length'
-                          and self.parent_class == u'SASdetector'):
-                        self.detector.slit_length = float(data_point)
-                        self.detector.slit_length_unit = unit
-                    elif (key == u'x_position'
-                          and self.parent_class == u'SASdetector'):
-                        self.detector.offset.x = float(data_point)
-                        self.detector.offset_unit = unit
-                    elif (key == u'y_position'
-                          and self.parent_class == u'SASdetector'):
-                        self.detector.offset.y = float(data_point)
-                        self.detector.offset_unit = unit
-                    elif (key == u'pitch'
-                          and self.parent_class == u'SASdetector'):
-                        self.detector.orientation.x = float(data_point)
-                        self.detector.orientation_unit = unit
-                    elif key == u'roll' and self.parent_class == u'SASdetector':
-                        self.detector.orientation.z = float(data_point)
-                        self.detector.orientation_unit = unit
-                    elif key == u'yaw' and self.parent_class == u'SASdetector':
-                        self.detector.orientation.y = float(data_point)
-                        self.detector.orientation_unit = unit
-                    elif (key == u'beam_center_x'
-                          and self.parent_class == u'SASdetector'):
-                        self.detector.beam_center.x = float(data_point)
-                        self.detector.beam_center_unit = unit
-                    elif (key == u'beam_center_y'
-                          and self.parent_class == u'SASdetector'):
-                        self.detector.beam_center.y = float(data_point)
-                        self.detector.beam_center_unit = unit
-                    elif (key == u'x_pixel_size'
-                          and self.parent_class == u'SASdetector'):
-                        self.detector.pixel_size.x = float(data_point)
-                        self.detector.pixel_size_unit = unit
-                    elif (key == u'y_pixel_size'
-                          and self.parent_class == u'SASdetector'):
-                        self.detector.pixel_size.y = float(data_point)
-                        self.detector.pixel_size_unit = unit
-                    elif (key == u'distance'
-                          and self.parent_class == u'SAScollimation'):
-                        self.collimation.length = data_point
-                        self.collimation.length_unit = unit
-                    elif (key == u'name'
-                          and self.parent_class == u'SAScollimation'):
-                        self.collimation.name = data_point
-                    elif (key == u'shape'
-                          and self.parent_class == u'SASaperture'):
-                        self.aperture.shape = data_point
-                    elif (key == u'x_gap'
-                          and self.parent_class == u'SASaperture'):
-                        self.aperture.size.x = data_point
-                    elif (key == u'y_gap'
-                          and self.parent_class == u'SASaperture'):
-                        self.aperture.size.y = data_point
-
+                    # Detector
+                    elif self.parent_class == u'SASdetector':
+                        self.process_detector(data_point, key, unit)
+                    # Collimation
+                    elif self.parent_class == u'SAScollimation':
+                        self.process_collimation(data_point, key, unit)
+                    # Aperture
+                    elif self.parent_class == u'SASaperture':
+                        self.process_aperture(data_point, key)
                     # Process Information
-                    elif (key == u'Title'
-                          and self.parent_class == u'SASprocess'): # CanSAS 2.0
-                        self.process.name = data_point
-                    elif (key == u'name'
-                          and self.parent_class == u'SASprocess'): # NXcanSAS
-                        self.process.name = data_point
-                    elif (key == u'description'
-                          and self.parent_class == u'SASprocess'):
-                        self.process.description = data_point
-                    elif key == u'date' and self.parent_class == u'SASprocess':
-                        self.process.date = data_point
-                    elif key == u'term' and self.parent_class == u'SASprocess':
-                        self.process.term = data_point
-                    elif self.parent_class == u'SASprocess':
-                        self.process.notes.append(data_point)
-
+                    elif self.parent_class == u'SASprocess': # CanSAS 2.0
+                        self.process_process(data_point, key)
                     # Source
-                    elif (key == u'wavelength'
-                          and self.parent_class == u'SASdata'):
-                        self.current_datainfo.source.wavelength = data_point
-                        self.current_datainfo.source.wavelength_unit = unit
-                    elif (key == u'incident_wavelength'
-                          and self.parent_class == 'SASsource'):
-                        self.current_datainfo.source.wavelength = data_point
-                        self.current_datainfo.source.wavelength_unit = unit
-                    elif (key == u'wavelength_max'
-                          and self.parent_class == u'SASsource'):
-                        self.current_datainfo.source.wavelength_max = data_point
-                        self.current_datainfo.source.wavelength_max_unit = unit
-                    elif (key == u'wavelength_min'
-                          and self.parent_class == u'SASsource'):
-                        self.current_datainfo.source.wavelength_min = data_point
-                        self.current_datainfo.source.wavelength_min_unit = unit
-                    elif (key == u'incident_wavelength_spread'
-                          and self.parent_class == u'SASsource'):
-                        self.current_datainfo.source.wavelength_spread = \
-                            data_point
-                        self.current_datainfo.source.wavelength_spread_unit = \
-                            unit
-                    elif (key == u'beam_size_x'
-                          and self.parent_class == u'SASsource'):
-                        self.current_datainfo.source.beam_size.x = data_point
-                        self.current_datainfo.source.beam_size_unit = unit
-                    elif (key == u'beam_size_y'
-                          and self.parent_class == u'SASsource'):
-                        self.current_datainfo.source.beam_size.y = data_point
-                        self.current_datainfo.source.beam_size_unit = unit
-                    elif (key == u'beam_shape'
-                          and self.parent_class == u'SASsource'):
-                        self.current_datainfo.source.beam_shape = data_point
-                    elif (key == u'radiation'
-                          and self.parent_class == u'SASsource'):
-                        self.current_datainfo.source.radiation = data_point
-                    elif (key == u'transmission'
-                          and self.parent_class == u'SASdata'):
-                        self.current_datainfo.sample.transmission = data_point
-
+                    elif self.parent_class == u'SASsource':
+                        self.process_source(data_point, key, unit)
                     # Everything else goes in meta_data
+                    elif self.parent_class == u'SASdata':
+                        if isinstance(self.current_dataset, plottable_2D):
+                            self.process_2d_data_object(data_set, key, unit)
+                        else:
+                            self.process_1d_data_object(data_set, key, unit)
+
+                        break
+                    elif self.parent_class == u'SAStransmission_spectrum':
+                        self.process_trans_spectrum(data_set, key)
+                        break
                     else:
                         new_key = self._create_unique_key(
@@ -410 +250 @@
             else:
                 # I don't know if this reachable code
-                self.errors.add("ShouldNeverHappenException")
+                self.errors.append("ShouldNeverHappenException")
+
+    def process_1d_data_object(self, data_set, key, unit):
+        """
+        SASdata processor method for 1d data items
+        :param data_set: data from HDF5 file
+        :param key: canSAS_class attribute
+        :param unit: unit attribute
+        """
+        if key == self.i_name:
+            if self.multi_frame:
+                for x in range(0, data_set.shape[0]):
+                    self.data_frames.append(data_set[x].flatten())
+            else:
+                self.current_dataset.y = data_set.flatten()
+                self.current_dataset.yaxis("Intensity", unit)
+        elif key == self.i_uncertainties_name:
+            if self.multi_frame:
+                for x in range(0, data_set.shape[0]):
+                    self.data_uncertainty_frames.append(data_set[x].flatten())
+            self.current_dataset.dy = data_set.flatten()
+        elif key in self.q_names:
+            self.current_dataset.xaxis("Q", unit)
+            self.current_dataset.x = data_set.flatten()
+        elif key in self.q_resolution_names:
+            if (len(self.q_resolution_names) > 1
+                    and np.where(self.q_resolution_names == key)[0] == 0):
+                self.current_dataset.dxw = data_set.flatten()
+            elif (len(self.q_resolution_names) > 1
+                  and np.where(self.q_resolution_names == key)[0] == 1):
+                self.current_dataset.dxl = data_set.flatten()
+            else:
+                self.current_dataset.dx = data_set.flatten()
+        elif key in self.q_uncertainty_names:
+            if (len(self.q_uncertainty_names) > 1
+                    and np.where(self.q_uncertainty_names == key)[0] == 0):
+                self.current_dataset.dxw = data_set.flatten()
+            elif (len(self.q_uncertainty_names) > 1
+                  and np.where(self.q_uncertainty_names == key)[0] == 1):
+                self.current_dataset.dxl = data_set.flatten()
+            else:
+                self.current_dataset.dx = data_set.flatten()
+        elif key == self.mask_name:
+            self.current_dataset.mask = data_set.flatten()
+        elif key == u'wavelength':
+            self.current_datainfo.source.wavelength = data_set[0]
+            self.current_datainfo.source.wavelength_unit = unit
+
+    def process_2d_data_object(self, data_set, key, unit):
+        if key == self.i_name:
+            self.current_dataset.data = data_set
+            self.current_dataset.zaxis("Intensity", unit)
+        elif key == self.i_uncertainties_name:
+            self.current_dataset.err_data = data_set.flatten()
+        elif key in self.q_names:
+            self.current_dataset.xaxis("Q_x", unit)
+            self.current_dataset.yaxis("Q_y", unit)
+            if self.q_names[0] == self.q_names[1]:
+                # All q data in a single array
+                self.current_dataset.qx_data = data_set[0]
+                self.current_dataset.qy_data = data_set[1]
+            elif self.q_names.index(key) == 0:
+                self.current_dataset.qx_data = data_set
+            elif self.q_names.index(key) == 1:
+                self.current_dataset.qy_data = data_set
+        elif key in self.q_uncertainty_names or key in self.q_resolution_names:
+            if ((self.q_uncertainty_names[0] == self.q_uncertainty_names[1]) or
+                    (self.q_resolution_names[0] == self.q_resolution_names[1])):
+                # All q data in a single array
+                self.current_dataset.dqx_data = data_set[0].flatten()
+                self.current_dataset.dqy_data = data_set[1].flatten()
+            elif (self.q_uncertainty_names.index(key) == 0 or
+                  self.q_resolution_names.index(key) == 0):
+                self.current_dataset.dqx_data = data_set.flatten()
+            elif (self.q_uncertainty_names.index(key) == 1 or
+                  self.q_resolution_names.index(key) == 1):
+                self.current_dataset.dqy_data = data_set.flatten()
+                self.current_dataset.yaxis("Q_y", unit)
+        elif key == self.mask_name:
+            self.current_dataset.mask = data_set.flatten()
+        elif key == u'Qy':
+            self.current_dataset.yaxis("Q_y", unit)
+            self.current_dataset.qy_data = data_set.flatten()
+        elif key == u'Qydev':
+            self.current_dataset.dqy_data = data_set.flatten()
+        elif key == u'Qx':
+            self.current_dataset.xaxis("Q_x", unit)
+            self.current_dataset.qx_data = data_set.flatten()
+        elif key == u'Qxdev':
+            self.current_dataset.dqx_data = data_set.flatten()
+
+    def process_trans_spectrum(self, data_set, key):
+        """
+        SAStransmission_spectrum processor
+        :param data_set: data from HDF5 file
+        :param key: canSAS_class attribute
+        """
+        if key == u'T':
+            self.trans_spectrum.transmission = data_set.flatten()
+        elif key == u'Tdev':
+            self.trans_spectrum.transmission_deviation = data_set.flatten()
+        elif key == u'lambda':
+            self.trans_spectrum.wavelength = data_set.flatten()
+
+    def process_sample(self, data_point, key):
+        """
+        SASsample processor
+        :param data_point: Single point from an HDF5 data file
+        :param key: class name data_point was taken from
+        """
+        if key == u'Title':
+            self.current_datainfo.sample.name = data_point
+        elif key == u'name':
+            self.current_datainfo.sample.name = data_point
+        elif key == u'ID':
+            self.current_datainfo.sample.name = data_point
+        elif key == u'thickness':
+            self.current_datainfo.sample.thickness = data_point
+        elif key == u'temperature':
+            self.current_datainfo.sample.temperature = data_point
+        elif key == u'transmission':
+            self.current_datainfo.sample.transmission = data_point
+        elif key == u'x_position':
+            self.current_datainfo.sample.position.x = data_point
+        elif key == u'y_position':
+            self.current_datainfo.sample.position.y = data_point
+        elif key == u'pitch':
+            self.current_datainfo.sample.orientation.x = data_point
+        elif key == u'yaw':
+            self.current_datainfo.sample.orientation.y = data_point
+        elif key == u'roll':
+            self.current_datainfo.sample.orientation.z = data_point
+        elif key == u'details':
+            self.current_datainfo.sample.details.append(data_point)
+
+    def process_detector(self, data_point, key, unit):
+        """
+        SASdetector processor
+        :param data_point: Single point from an HDF5 data file
+        :param key: class name data_point was taken from
+        :param unit: unit attribute from data set
+        """
+        if key == u'name':
+            self.detector.name = data_point
+        elif key == u'SDD':
+            self.detector.distance = float(data_point)
+            self.detector.distance_unit = unit
+        elif key == u'slit_length':
+            self.detector.slit_length = float(data_point)
+            self.detector.slit_length_unit = unit
+        elif key == u'x_position':
+            self.detector.offset.x = float(data_point)
+            self.detector.offset_unit = unit
+        elif key == u'y_position':
+            self.detector.offset.y = float(data_point)
+            self.detector.offset_unit = unit
+        elif key == u'pitch':
+            self.detector.orientation.x = float(data_point)
+            self.detector.orientation_unit = unit
+        elif key == u'roll':
+            self.detector.orientation.z = float(data_point)
+            self.detector.orientation_unit = unit
+        elif key == u'yaw':
+            self.detector.orientation.y = float(data_point)
+            self.detector.orientation_unit = unit
+        elif key == u'beam_center_x':
+            self.detector.beam_center.x = float(data_point)
+            self.detector.beam_center_unit = unit
+        elif key == u'beam_center_y':
+            self.detector.beam_center.y = float(data_point)
+            self.detector.beam_center_unit = unit
+        elif key == u'x_pixel_size':
+            self.detector.pixel_size.x = float(data_point)
+            self.detector.pixel_size_unit = unit
+        elif key == u'y_pixel_size':
+            self.detector.pixel_size.y = float(data_point)
+            self.detector.pixel_size_unit = unit
+
+    def process_collimation(self, data_point, key, unit):
+        """
+        SAScollimation processor
+        :param data_point: Single point from an HDF5 data file
+        :param key: class name data_point was taken from
+        :param unit: unit attribute from data set
+        """
+        if key == u'distance':
+            self.collimation.length = data_point
+            self.collimation.length_unit = unit
+        elif key == u'name':
+            self.collimation.name = data_point
+
+    def process_aperture(self, data_point, key):
+        """
+        SASaperture processor
+        :param data_point: Single point from an HDF5 data file
+        :param key: class name data_point was taken from
+        """
+        if key == u'shape':
+            self.aperture.shape = data_point
+        elif key == u'x_gap':
+            self.aperture.size.x = data_point
+        elif key == u'y_gap':
+            self.aperture.size.y = data_point
+
+    def process_source(self, data_point, key, unit):
+        """
+        SASsource processor
+        :param data_point: Single point from an HDF5 data file
+        :param key: class name data_point was taken from
+        :param unit: unit attribute from data set
+        """
+        if key == u'incident_wavelength':
+            self.current_datainfo.source.wavelength = data_point
+            self.current_datainfo.source.wavelength_unit = unit
+        elif key == u'wavelength_max':
+            self.current_datainfo.source.wavelength_max = data_point
+            self.current_datainfo.source.wavelength_max_unit = unit
+        elif key == u'wavelength_min':
+            self.current_datainfo.source.wavelength_min = data_point
+            self.current_datainfo.source.wavelength_min_unit = unit
+        elif key == u'incident_wavelength_spread':
+            self.current_datainfo.source.wavelength_spread = data_point
+            self.current_datainfo.source.wavelength_spread_unit = unit
+        elif key == u'beam_size_x':
+            self.current_datainfo.source.beam_size.x = data_point
+            self.current_datainfo.source.beam_size_unit = unit
+        elif key == u'beam_size_y':
+            self.current_datainfo.source.beam_size.y = data_point
+            self.current_datainfo.source.beam_size_unit = unit
+        elif key == u'beam_shape':
+            self.current_datainfo.source.beam_shape = data_point
+        elif key == u'radiation':
+            self.current_datainfo.source.radiation = data_point
+
+    def process_process(self, data_point, key):
+        """
+        SASprocess processor
+        :param data_point: Single point from an HDF5 data file
+        :param key: class name data_point was taken from
+        """
+        term_match = re.compile(u'^term[0-9]+$')
+        if key == u'Title':  # CanSAS 2.0
+            self.process.name = data_point
+        elif key == u'name':  # NXcanSAS
+            self.process.name = data_point
+        elif key == u'description':
+            self.process.description = data_point
+        elif key == u'date':
+            self.process.date = data_point
+        elif term_match.match(key):
+            self.process.term.append(data_point)
+        else:
+            self.process.notes.append(data_point)
 
     def add_intermediate(self):
@@ -440 +532 @@
                 self.data2d.append(self.current_dataset)
             elif isinstance(self.current_dataset, plottable_1D):
-                self.data1d.append(self.current_dataset)
+                if self.multi_frame:
+                    for x in range(0, len(self.data_frames)):
+                        self.current_dataset.y = self.data_frames[x]
+                        if len(self.data_uncertainty_frames) > x:
+                            self.current_dataset.dy = \
+                                self.data_uncertainty_frames[x]
+                        self.data1d.append(self.current_dataset)
+                else:
+                    self.data1d.append(self.current_dataset)
 
     def final_data_cleanup(self):
@@ -452 +552 @@
             spectrum_list = []
             for spectrum in self.current_datainfo.trans_spectrum:
-                spectrum.transmission = np.delete(spectrum.transmission, [0])
                 spectrum.transmission = spectrum.transmission.astype(np.float64)
-                spectrum.transmission_deviation = np.delete(
-                    spectrum.transmission_deviation, [0])
                 spectrum.transmission_deviation = \
                     spectrum.transmission_deviation.astype(np.float64)
-                spectrum.wavelength = np.delete(spectrum.wavelength, [0])
                 spectrum.wavelength = spectrum.wavelength.astype(np.float64)
                 if len(spectrum.transmission) > 0:
@@ -466 +562 @@
         # Append errors to dataset and reset class errors
         self.current_datainfo.errors = self.errors
-        self.errors.clear()
+        self.errors = []
 
         # Combine all plottables with datainfo and append each to output
         # Type cast data arrays to float64 and find min/max as appropriate
         for dataset in self.data2d:
-            zeros = np.ones(dataset.data.size, dtype=bool)
-            try:
-                for i in range(0, dataset.mask.size - 1):
-                    zeros[i] = dataset.mask[i]
-            except:
-                self.errors.add(sys.exc_value)
-            dataset.mask = zeros
             # Calculate the actual Q matrix
             try:
@@ -484 +573 @@
                                              * dataset.qx_data
                                              + dataset.qy_data
-                                             * dataset.qy_data)
+                                             * dataset.qy_data).flatten()
             except:
                 dataset.q_data = None
 
             if dataset.data.ndim == 2:
-                (n_rows, n_cols) = dataset.data.shape
-                dataset.y_bins = dataset.qy_data[0::n_cols]
-                dataset.x_bins = dataset.qx_data[:n_cols]
+                dataset.y_bins = np.unique(dataset.qy_data.flatten())
+                dataset.x_bins = np.unique(dataset.qx_data.flatten())
                 dataset.data = dataset.data.flatten()
+                dataset.qx_data = dataset.qx_data.flatten()
+                dataset.qy_data = dataset.qy_data.flatten()
+
+            try:
+                iter(dataset.mask)
+                dataset.mask = np.invert(np.asarray(dataset.mask, dtype=bool))
+            except TypeError:
+                dataset.mask = np.ones(dataset.data.shape, dtype=bool)
             self.current_dataset = dataset
             self.send_to_output()
@@ -511 +607 @@
         if self.current_datainfo and self.current_dataset:
             self.final_data_cleanup()
+        self.data_frames = []
+        self.data_uncertainty_frames = []
         self.data1d = []
         self.data2d = []
         self.current_datainfo = DataInfo()
 
-
-    def _initialize_new_data_set(self, parent_list=None):
+    def _initialize_new_data_set(self, value=None):
         """
         A private class method to generate a new 1D or 2D data object based on
@@ -524 +621 @@
         :param parent_list: List of names of parent elements
         """
-
-        if parent_list is None:
-            parent_list = []
-        if self._find_intermediate(parent_list, "Qx"):
+        if self._is_2d_not_multi_frame(value):
             self.current_dataset = plottable_2D()
         else:
@@ -535 +629 @@
         self.current_datainfo.filename = self.raw_data.filename
 
-    def _find_intermediate(self, parent_list, basename=""):
-        """
-        A private class used to find an entry by either using a direct key or
-        knowing the approximate basename.
-
-        :param parent_list: List of parents nodes in the HDF5 file
+    @staticmethod
+    def as_list_or_array(iterable):
+        """
+        Return value as a list if not already a list or array.
+        :param iterable:
+        :return:
+        """
+        if not (isinstance(iterable, np.ndarray) or isinstance(iterable, list)):
+            iterable = iterable.split(",") if isinstance(iterable, basestring)\
+                else [iterable]
+        return iterable
+
+    def _find_data_attributes(self, value):
+        """
+        A class to find the indices for Q, the name of the Qdev and Idev, and
+        the name of the mask.
+        :param value: SASdata/NXdata HDF5 Group
+        """
+        # Initialize values to base types
+        self.mask_name = u''
+        self.i_name = u''
+        self.i_node = u''
+        self.i_uncertainties_name = u''
+        self.q_names = []
+        self.q_uncertainty_names = []
+        self.q_resolution_names = []
+        # Get attributes
+        attrs = value.attrs
+        signal = attrs.get("signal", "I")
+        if isinstance(signal, bytes):
+            signal = signal.decode()
+        i_axes = attrs.get("I_axes", ["Q"])
+        if isinstance(i_axes, bytes):
+            i_axes = i_axes.decode()
+        q_indices = attrs.get("Q_indices", [0])
+        if isinstance(q_indices, bytes):
+            q_indices = q_indices.decode()
+        i_axes = self.as_list_or_array(i_axes)
+        keys = value.keys()
+        # Assign attributes to appropriate class variables
+        self.q_names = [i_axes[int(v)] for v in self.as_list_or_array(q_indices)]
+        self.mask_name = attrs.get("mask")
+        if isinstance(self.mask_name, bytes):
+            self.mask_name = self.mask_name.decode()
+        self.i_name = signal
+        self.i_node = value.get(self.i_name)
+
+        for item in self.q_names:
+            if item in keys:
+                q_vals = value.get(item)
+                if q_vals.attrs.get("uncertainties") is not None:
+                    self.q_uncertainty_names = q_vals.attrs.get("uncertainties")
+                elif q_vals.attrs.get("uncertainty") is not None:
+                    self.q_uncertainty_names = q_vals.attrs.get("uncertainty")
+                if isinstance(self.q_uncertainty_names, basestring):
+                    self.q_uncertainty_names = self.q_uncertainty_names.split(",")
+                if q_vals.attrs.get("resolutions") is not None:
+                    self.q_resolution_names = q_vals.attrs.get("resolutions")
+                if isinstance(self.q_resolution_names, basestring):
+                    self.q_resolution_names = self.q_resolution_names.split(",")
+        if self.i_name in keys:
+            i_vals = value.get(self.i_name)
+            self.i_uncertainties_name = i_vals.attrs.get("uncertainties")
+            if self.i_uncertainties_name is None:
+                self.i_uncertainties_name = i_vals.attrs.get("uncertainty")
+            if isinstance(self.i_uncertainties_name, bytes):
+                key = self.i_uncertainties_name
+                self.i_uncertainties_name = key.decode() if isinstance(key, bytes) else key
+
+    def _is_2d_not_multi_frame(self, value, i_base="", q_base=""):
+        """
+        A private class to determine if the data set is 1d or 2d.
+
+        :param value: Nexus/NXcanSAS data group
         :param basename: Approximate name of an entry to search for
-        :return:
-        """
-
-        entry = False
-        key_prog = re.compile(basename)
-        top = self.raw_data
-        for parent in parent_list:
-            top = top.get(parent)
-        for key in top.keys():
-            if key_prog.match(key):
-                entry = True
-                break
-        return entry
+        :return: True if 2D, otherwise false
+        """
+        i_basename = i_base if i_base != "" else self.i_name
+        i_vals = value.get(i_basename)
+        q_basename = q_base if q_base != "" else self.q_names
+        q_vals = value.get(q_basename[0])
+        self.multi_frame = (i_vals is not None and q_vals is not None
+                            and len(i_vals.shape) != 1
+                            and len(q_vals.shape) == 1)
+        return (i_vals is not None and len(i_vals.shape) != 1
+                and not self.multi_frame)
 
     def _create_unique_key(self, dictionary, name, numb=0):
@@ -583 +743 @@
         if unit is None:
             unit = h5attr(value, u'unit')
-        # Convert the unit formats
-        if unit == "1/A":
-            unit = "A^{-1}"
-        elif unit == "1/cm":
-            unit = "cm^{-1}"
         return unit

src/sas/sascalc/dataloader/readers/danse_reader.py

@@ -180 +180 @@
         detector.beam_center.y = center_y * pixel
 
-
-        self.current_dataset.xaxis("\\rm{Q_{x}}", 'A^{-1}')
-        self.current_dataset.yaxis("\\rm{Q_{y}}", 'A^{-1}')
-        self.current_dataset.zaxis("\\rm{Intensity}", "cm^{-1}")
-
+        self.current_dataset = self.set_default_2d_units(self.current_dataset)
         self.current_dataset.x_bins = x_vals
         self.current_dataset.y_bins = y_vals

src/sas/sascalc/dataloader/readers/red2d_reader.py

@@ -317 +317 @@
 
         # Units of axes
-        self.current_dataset.xaxis(r"\rm{Q_{x}}", 'A^{-1}')
-        self.current_dataset.yaxis(r"\rm{Q_{y}}", 'A^{-1}')
-        self.current_dataset.zaxis(r"\rm{Intensity}", "cm^{-1}")
+        self.current_dataset = self.set_default_2d_units(self.current_dataset)
 
         # Store loading process information

src/sas/sascalc/file_converter/nxcansas_writer.py

@@ -8 +8 @@
 import os
 
-from sas.sascalc.dataloader.readers.cansas_reader_HDF5 import Reader as Cansas2Reader
+from sas.sascalc.dataloader.readers.cansas_reader_HDF5 import Reader
 from sas.sascalc.dataloader.data_info import Data1D, Data2D
 
-class NXcanSASWriter(Cansas2Reader):
+class NXcanSASWriter(Reader):
     """
     A class for writing in NXcanSAS data files. Any number of data sets may be
@@ -87 +87 @@
                     entry[names[2]].attrs['units'] = units
 
-        valid_data = all([issubclass(d.__class__, (Data1D, Data2D)) for d in dataset])
+        valid_data = all([isinstance(d, (Data1D, Data2D)) for d in dataset])
         if not valid_data:
-            raise ValueError("All entries of dataset must be Data1D or Data2D objects")
+            raise ValueError("All entries of dataset must be Data1D or Data2D"
+                             "objects")
 
         # Get run name and number from first Data object
@@ -109 +110 @@
         sasentry.attrs['version'] = '1.0'
 
-        i = 1
-
-        for data_obj in dataset:
-            data_entry = sasentry.create_group("sasdata{0:0=2d}".format(i))
+        for i, data_obj in enumerate(dataset):
+            data_entry = sasentry.create_group("sasdata{0:0=2d}".format(i+1))
             data_entry.attrs['canSAS_class'] = 'SASdata'
             if isinstance(data_obj, Data1D):
@@ -118 +117 @@
             elif isinstance(data_obj, Data2D):
                 self._write_2d_data(data_obj, data_entry)
-            i += 1
 
         data_info = dataset[0]
@@ -148 +146 @@
                 sample_entry.create_dataset('details', data=details)
 
-        # Instrumment metadata
+        # Instrument metadata
         instrument_entry = sasentry.create_group('sasinstrument')
         instrument_entry.attrs['canSAS_class'] = 'SASinstrument'
@@ -176 +174 @@
             units=data_info.source.beam_size_unit, write_fn=_write_h5_float)
 
-
         # Collimation metadata
         if len(data_info.collimation) > 0:
-            i = 1
-            for coll_info in data_info.collimation:
+            for i, coll_info in enumerate(data_info.collimation):
                 collimation_entry = instrument_entry.create_group(
-                    'sascollimation{0:0=2d}'.format(i))
+                    'sascollimation{0:0=2d}'.format(i + 1))
                 collimation_entry.attrs['canSAS_class'] = 'SAScollimation'
                 if coll_info.length is not None:
                     _write_h5_float(collimation_entry, coll_info.length, 'SDD')
-                    collimation_entry['SDD'].attrs['units'] = coll_info.length_unit
+                    collimation_entry['SDD'].attrs['units'] =\
+                        coll_info.length_unit
                 if coll_info.name is not None:
                     collimation_entry['name'] = _h5_string(coll_info.name)
         else:
-            # Create a blank one - at least 1 set of collimation metadata
-            # required by format
-            collimation_entry = instrument_entry.create_group('sascollimation01')
+            # Create a blank one - at least 1 collimation required by format
+            instrument_entry.create_group('sascollimation01')
 
         # Detector metadata
         if len(data_info.detector) > 0:
             i = 1
-            for det_info in data_info.detector:
+            for i, det_info in enumerate(data_info.detector):
                 detector_entry = instrument_entry.create_group(
-                    'sasdetector{0:0=2d}'.format(i))
+                    'sasdetector{0:0=2d}'.format(i + 1))
                 detector_entry.attrs['canSAS_class'] = 'SASdetector'
                 if det_info.distance is not None:
                     _write_h5_float(detector_entry, det_info.distance, 'SDD')
-                    detector_entry['SDD'].attrs['units'] = det_info.distance_unit
+                    detector_entry['SDD'].attrs['units'] =\
+                        det_info.distance_unit
                 if det_info.name is not None:
                     detector_entry['name'] = _h5_string(det_info.name)
@@ -209 +206 @@
                     detector_entry['name'] = _h5_string('')
                 if det_info.slit_length is not None:
-                    _write_h5_float(detector_entry, det_info.slit_length, 'slit_length')
-                    detector_entry['slit_length'].attrs['units'] = det_info.slit_length_unit
+                    _write_h5_float(detector_entry, det_info.slit_length,
+                                    'slit_length')
+                    detector_entry['slit_length'].attrs['units'] =\
+                        det_info.slit_length_unit
                 _write_h5_vector(detector_entry, det_info.offset)
                 # NXcanSAS doesn't save information about pitch, only roll
@@ -224 +223 @@
                     names=['x_pixel_size', 'y_pixel_size'],
                     write_fn=_write_h5_float, units=det_info.pixel_size_unit)
-
-                i += 1
         else:
             # Create a blank one - at least 1 detector required by format
@@ -231 +228 @@
             detector_entry.attrs['canSAS_class'] = 'SASdetector'
             detector_entry.attrs['name'] = ''
+
+        # Process meta data
+        for i, process in enumerate(data_info.process):
+            process_entry = sasentry.create_group('sasprocess{0:0=2d}'.format(
+                i + 1))
+            process_entry.attrs['canSAS_class'] = 'SASprocess'
+            if process.name:
+                name = _h5_string(process.name)
+                process_entry.create_dataset('name', data=name)
+            if process.date:
+                date = _h5_string(process.date)
+                process_entry.create_dataset('date', data=date)
+            if process.description:
+                desc = _h5_string(process.description)
+                process_entry.create_dataset('description', data=desc)
+            for j, term in enumerate(process.term):
+                # Don't save empty terms
+                if term:
+                    h5_term = _h5_string(term)
+                    process_entry.create_dataset('term{0:0=2d}'.format(
+                        j + 1), data=h5_term)
+            for j, note in enumerate(process.notes):
+                # Don't save empty notes
+                if note:
+                    h5_note = _h5_string(note)
+                    process_entry.create_dataset('note{0:0=2d}'.format(
+                        j + 1), data=h5_note)
+
+        # Transmission Spectrum
+        for i, trans in enumerate(data_info.trans_spectrum):
+            trans_entry = sasentry.create_group(
+                'sastransmission_spectrum{0:0=2d}'.format(i + 1))
+            trans_entry.attrs['canSAS_class'] = 'SAStransmission_spectrum'
+            trans_entry.attrs['signal'] = 'T'
+            trans_entry.attrs['T_axes'] = 'T'
+            trans_entry.attrs['name'] = trans.name
+            if trans.timestamp is not '':
+                trans_entry.attrs['timestamp'] = trans.timestamp
+            transmission = trans_entry.create_dataset('T',
+                                                      data=trans.transmission)
+            transmission.attrs['unertainties'] = 'Tdev'
+            trans_entry.create_dataset('Tdev',
+                                       data=trans.transmission_deviation)
+            trans_entry.create_dataset('lambda', data=trans.wavelength)
 
         note_entry = sasentry.create_group('sasnote'.format(i))
@@ -254 +295 @@
         data_entry.attrs['signal'] = 'I'
         data_entry.attrs['I_axes'] = 'Q'
-        data_entry.attrs['I_uncertainties'] = 'Idev'
-        data_entry.attrs['Q_indicies'] = 0
-
-        dI = data_obj.dy
-        if dI is None:
-            dI = np.zeros((data_obj.y.shape))
-
-        data_entry.create_dataset('Q', data=data_obj.x)
-        data_entry.create_dataset('I', data=data_obj.y)
-        data_entry.create_dataset('Idev', data=dI)
+        data_entry.attrs['Q_indices'] = [0]
+        q_entry = data_entry.create_dataset('Q', data=data_obj.x)
+        q_entry.attrs['units'] = data_obj.x_unit
+        i_entry = data_entry.create_dataset('I', data=data_obj.y)
+        i_entry.attrs['units'] = data_obj.y_unit
+        if data_obj.dy is not None:
+            i_entry.attrs['uncertainties'] = 'Idev'
+            i_dev_entry = data_entry.create_dataset('Idev', data=data_obj.dy)
+            i_dev_entry.attrs['units'] = data_obj.y_unit
+        if data_obj.dx is not None:
+            q_entry.attrs['resolutions'] = 'dQ'
+            dq_entry = data_entry.create_dataset('dQ', data=data_obj.dx)
+            dq_entry.attrs['units'] = data_obj.x_unit
+        elif data_obj.dxl is not None:
+            q_entry.attrs['resolutions'] = ['dQl','dQw']
+            dql_entry = data_entry.create_dataset('dQl', data=data_obj.dxl)
+            dql_entry.attrs['units'] = data_obj.x_unit
+            dqw_entry = data_entry.create_dataset('dQw', data=data_obj.dxw)
+            dqw_entry.attrs['units'] = data_obj.x_unit
 
     def _write_2d_data(self, data, data_entry):
@@ -273 +323 @@
         """
         data_entry.attrs['signal'] = 'I'
-        data_entry.attrs['I_axes'] = 'Q,Q'
-        data_entry.attrs['I_uncertainties'] = 'Idev'
-        data_entry.attrs['Q_indicies'] = [0,1]
+        data_entry.attrs['I_axes'] = 'Qx,Qy'
+        data_entry.attrs['Q_indices'] = [0,1]
 
         (n_rows, n_cols) = (len(data.y_bins), len(data.x_bins))
 
-        if n_rows == 0 and n_cols == 0:
+        if (n_rows == 0 and n_cols == 0) or (n_cols*n_rows != data.data.size):
             # Calculate rows and columns, assuming detector is square
             # Same logic as used in PlotPanel.py _get_bins
@@ -288 +337 @@
                 raise ValueError("Unable to calculate dimensions of 2D data")
 
-        I = np.reshape(data.data, (n_rows, n_cols))
-        dI = np.zeros((n_rows, n_cols))
-        if not all(data.err_data == [None]):
-            dI = np.reshape(data.err_data, (n_rows, n_cols))
-        qx =  np.reshape(data.qx_data, (n_rows, n_cols))
+        intensity = np.reshape(data.data, (n_rows, n_cols))
+        qx = np.reshape(data.qx_data, (n_rows, n_cols))
         qy = np.reshape(data.qy_data, (n_rows, n_cols))
 
-        I_entry = data_entry.create_dataset('I', data=I)
-        I_entry.attrs['units'] = data.I_unit
-        Qx_entry = data_entry.create_dataset('Qx', data=qx)
-        Qx_entry.attrs['units'] = data.Q_unit
-        Qy_entry = data_entry.create_dataset('Qy', data=qy)
-        Qy_entry.attrs['units'] = data.Q_unit
-        Idev_entry = data_entry.create_dataset('Idev', data=dI)
-        Idev_entry.attrs['units'] = data.I_unit
+        i_entry = data_entry.create_dataset('I', data=intensity)
+        i_entry.attrs['units'] = data.I_unit
+        qx_entry = data_entry.create_dataset('Qx', data=qx)
+        qx_entry.attrs['units'] = data.Q_unit
+        qy_entry = data_entry.create_dataset('Qy', data=qy)
+        qy_entry.attrs['units'] = data.Q_unit
+        if (data.err_data is not None
+                and not all(v is None for v in data.err_data)):
+            d_i = np.reshape(data.err_data, (n_rows, n_cols))
+            i_entry.attrs['uncertainties'] = 'Idev'
+            i_dev_entry = data_entry.create_dataset('Idev', data=d_i)
+            i_dev_entry.attrs['units'] = data.I_unit
+        if (data.dqx_data is not None
+                and not all(v is None for v in data.dqx_data)):
+            qx_entry.attrs['resolutions'] = 'dQx'
+            dqx_entry = data_entry.create_dataset('dQx', data=data.dqx_data)
+            dqx_entry.attrs['units'] = data.Q_unit
+        if (data.dqy_data is not None
+                and not all(v is None for v in data.dqy_data)):
+            qy_entry.attrs['resolutions'] = 'dQy'
+            dqy_entry = data_entry.create_dataset('dQy', data=data.dqy_data)
+            dqy_entry.attrs['units'] = data.Q_unit
+        if data.mask is not None and not all(v is None for v in data.mask):
+            data_entry.attrs['mask'] = "mask"
+            mask = np.invert(np.asarray(data.mask, dtype=bool))
+            data_entry.create_dataset('mask', data=mask)