← Previous Change
Next Change →

dataloader

Timestamp:

Aug 1, 2017 12:02:35 PM (7 years ago)

Author:

lewis

Branches:

master, ESS_GUI, ESS_GUI_Docs, ESS_GUI_batch_fitting, ESS_GUI_bumps_abstraction, ESS_GUI_iss1116, ESS_GUI_iss879, ESS_GUI_iss959, ESS_GUI_opencl, ESS_GUI_ordering, ESS_GUI_sync_sascalc, costrafo411, magnetic_scatt, release-4.2.2, ticket-1009, ticket-1094-headless, ticket-1242-2d-resolution, ticket-1243, ticket-1249, ticket885, unittest-saveload

Children:

511ccb2d

Parents:

248ff73 (diff), bc04647 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' into ticket-876

Location:

src/sas/sascalc/dataloader

Files:

: 2 added
: 1 deleted
: 15 edited

data_info.py (modified) (1 diff)
manipulations.py (modified) (27 diffs)
readers/IgorReader.py (modified) (1 diff)
readers/sesans_reader.py (modified) (3 diffs)
file_reader_base_class.py (added)
loader.py (modified) (5 diffs)
loader_exceptions.py (added)
readers/__init__.py (modified) (1 diff)
readers/abs_reader.py (modified) (1 diff)
readers/anton_paar_saxs_reader.py (modified) (8 diffs)
readers/ascii_reader.py (modified) (1 diff)
readers/associations.py (modified) (1 diff)
readers/cansas_reader.py (modified) (18 diffs)
readers/cansas_reader_HDF5.py (modified) (8 diffs)
readers/danse_reader.py (modified) (5 diffs)
readers/defaults.json (deleted)
readers/red2d_reader.py (modified) (18 diffs)
readers/xml_reader.py (modified) (6 diffs)

Legend:

: Unmodified
: Added
: Removed

src/sas/sascalc/dataloader/data_info.py

rbeba407	r5a8cdbb
16	16	######################################################################
17	17
	18	from __future__ import print_function
18	19
19	20	#TODO: Keep track of data manipulation in the 'process' data structure.

src/sas/sascalc/dataloader/manipulations.py

-                      r7432acb
+                      r324e0bf
+from __future__ import division
 """
 Data manipulations for 2D data sets.
 Using the meta data information, various types of averaging
 are performed in Q-space
+To test this module use:
+```
+cd test
+PYTHONPATH=../src/ python2  -m sasdataloader.test.utest_averaging DataInfoTests.test_sectorphi_quarter
+```
 """
 #####################################################################
 #This software was developed by the University of Tennessee as part of the
 #Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
 #project funded by the US National Science Foundation.
 #See the license text in license.txt
 #copyright 2008, University of Tennessee
+# This software was developed by the University of Tennessee as part of the
+# Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
+# project funded by the US National Science Foundation.
+# See the license text in license.txt
+# copyright 2008, University of Tennessee
 ######################################################################
+#TODO: copy the meta data from the 2D object to the resulting 1D object
+# TODO: copy the meta data from the 2D object to the resulting 1D object
 import math
+import numpy
+import numpy as np
+import sys
 #from data_info import plottable_2D
 …
     return phi_out
-def reader2D_converter(data2d=None):
-    """
-    convert old 2d format opened by IhorReader or danse_reader
-    to new Data2D format
-    :param data2d: 2d array of Data2D object
-    :return: 1d arrays of Data2D object
-    """
-    if data2d.data is None or data2d.x_bins is None or data2d.y_bins is None:
-        raise ValueError, "Can't convert this data: data=None..."
-    new_x = numpy.tile(data2d.x_bins, (len(data2d.y_bins), 1))
-    new_y = numpy.tile(data2d.y_bins, (len(data2d.x_bins), 1))
-    new_y = new_y.swapaxes(0, 1)
-    new_data = data2d.data.flatten()
-    qx_data = new_x.flatten()
-    qy_data = new_y.flatten()
-    q_data = numpy.sqrt(qx_data * qx_data + qy_data * qy_data)
-    if data2d.err_data is None or numpy.any(data2d.err_data <= 0):
-        new_err_data = numpy.sqrt(numpy.abs(new_data))
-    else:
-        new_err_data = data2d.err_data.flatten()
-    mask = numpy.ones(len(new_data), dtype=bool)
-    #TODO: make sense of the following two lines...
-    #from sas.sascalc.dataloader.data_info import Data2D
-    #output = Data2D()
-    output = data2d
-    output.data = new_data
-    output.err_data = new_err_data
-    output.qx_data = qx_data
-    output.qy_data = qy_data
-    output.q_data = q_data
-    output.mask = mask
-    return output
-class _Slab(object):
-    """
-    Compute average I(Q) for a region of interest
-    """
-    def __init__(self, x_min=0.0, x_max=0.0, y_min=0.0,
-                 y_max=0.0, bin_width=0.001):
-        # Minimum Qx value [A-1]
-        self.x_min = x_min
-        # Maximum Qx value [A-1]
-        self.x_max = x_max
-        # Minimum Qy value [A-1]
-        self.y_min = y_min
-        # Maximum Qy value [A-1]
-        self.y_max = y_max
-        # Bin width (step size) [A-1]
-        self.bin_width = bin_width
-        # If True, I(|Q|) will be return, otherwise,
-        # negative q-values are allowed
-        self.fold = False
-    def __call__(self, data2D):
-        return NotImplemented
-    def _avg(self, data2D, maj):
-        """
-        Compute average I(Q_maj) for a region of interest.
-        The major axis is defined as the axis of Q_maj.
-        The minor axis is the axis that we average over.
-        :param data2D: Data2D object
-        :param maj_min: min value on the major axis
-        :return: Data1D object
-        """
-        if len(data2D.detector) > 1:
-            msg = "_Slab._avg: invalid number of "
-            msg += " detectors: %g" % len(data2D.detector)
-            raise RuntimeError, msg
-        # Get data
-        data = data2D.data[numpy.isfinite(data2D.data)]
-        err_data = data2D.err_data[numpy.isfinite(data2D.data)]
-        qx_data = data2D.qx_data[numpy.isfinite(data2D.data)]
-        qy_data = data2D.qy_data[numpy.isfinite(data2D.data)]
-        # Build array of Q intervals
-        if maj == 'x':
-            if self.fold:
-                x_min = 0
-            else:
-                x_min = self.x_min
-            nbins = int(math.ceil((self.x_max - x_min) / self.bin_width))
-        elif maj == 'y':
-            if self.fold:
-                y_min = 0
-            else:
-                y_min = self.y_min
-            nbins = int(math.ceil((self.y_max - y_min) / self.bin_width))
-        else:
-            raise RuntimeError, "_Slab._avg: unrecognized axis %s" % str(maj)
-        x = numpy.zeros(nbins)
-        y = numpy.zeros(nbins)
-        err_y = numpy.zeros(nbins)
-        y_counts = numpy.zeros(nbins)
-        # Average pixelsize in q space
-        for npts in range(len(data)):
-            # default frac
-            frac_x = 0
-            frac_y = 0
-            # get ROI
-            if self.x_min <= qx_data[npts] and self.x_max > qx_data[npts]:
-                frac_x = 1
-            if self.y_min <= qy_data[npts] and self.y_max > qy_data[npts]:
-                frac_y = 1
-            frac = frac_x * frac_y
-            if frac == 0:
-                continue
-            # binning: find axis of q
-            if maj == 'x':
-                q_value = qx_data[npts]
-                min_value = x_min
-            if maj == 'y':
-                q_value = qy_data[npts]
-                min_value = y_min
-            if self.fold and q_value < 0:
-                q_value = -q_value
-            # bin
-            i_q = int(math.ceil((q_value - min_value) / self.bin_width)) - 1
-            # skip outside of max bins
-            if i_q < 0 or i_q >= nbins:
-                continue
-            #TODO: find better definition of x[i_q] based on q_data
-            # min_value + (i_q + 1) * self.bin_width / 2.0
-            x[i_q] += frac * q_value
-            y[i_q] += frac * data[npts]
-            if err_data is None or err_data[npts] == 0.0:
-                if data[npts] < 0:
-                    data[npts] = -data[npts]
-                err_y[i_q] += frac * frac * data[npts]
-            else:
-                err_y[i_q] += frac * frac * err_data[npts] * err_data[npts]
-            y_counts[i_q] += frac
-        # Average the sums
-        for n in range(nbins):
-            err_y[n] = math.sqrt(err_y[n])
-        err_y = err_y / y_counts
-        y = y / y_counts
-        x = x / y_counts
-        idx = (numpy.isfinite(y) & numpy.isfinite(x))
-        if not idx.any():
-            msg = "Average Error: No points inside ROI to average..."
-            raise ValueError, msg
-        return Data1D(x=x[idx], y=y[idx], dy=err_y[idx])
-class SlabY(_Slab):
-    """
-    Compute average I(Qy) for a region of interest
-    """
-    def __call__(self, data2D):
-        """
-        Compute average I(Qy) for a region of interest
-        :param data2D: Data2D object
-        :return: Data1D object
-        """
-        return self._avg(data2D, 'y')
-class SlabX(_Slab):
-    """
-    Compute average I(Qx) for a region of interest
-    """
-    def __call__(self, data2D):
-        """
-        Compute average I(Qx) for a region of interest
-        :param data2D: Data2D object
-        :return: Data1D object
-        """
-        return self._avg(data2D, 'x')
-class Boxsum(object):
-    """
-    Perform the sum of counts in a 2D region of interest.
-    """
-    def __init__(self, x_min=0.0, x_max=0.0, y_min=0.0, y_max=0.0):
-        # Minimum Qx value [A-1]
-        self.x_min = x_min
-        # Maximum Qx value [A-1]
-        self.x_max = x_max
-        # Minimum Qy value [A-1]
-        self.y_min = y_min
-        # Maximum Qy value [A-1]
-        self.y_max = y_max
-    def __call__(self, data2D):
-        """
-        Perform the sum in the region of interest
-        :param data2D: Data2D object
-        :return: number of counts, error on number of counts,
-            number of points summed
-        """
-        y, err_y, y_counts = self._sum(data2D)
-        # Average the sums
-        counts = 0 if y_counts == 0 else y
-        error = 0 if y_counts == 0 else math.sqrt(err_y)
-        # Added y_counts to return, SMK & PDB, 04/03/2013
-        return counts, error, y_counts
-    def _sum(self, data2D):
-        """
-        Perform the sum in the region of interest
-        :param data2D: Data2D object
-        :return: number of counts,
-            error on number of counts, number of entries summed
-        """
-        if len(data2D.detector) > 1:
-            msg = "Circular averaging: invalid number "
-            msg += "of detectors: %g" % len(data2D.detector)
-            raise RuntimeError, msg
-        # Get data
-        data = data2D.data[numpy.isfinite(data2D.data)]
-        err_data = data2D.err_data[numpy.isfinite(data2D.data)]
-        qx_data = data2D.qx_data[numpy.isfinite(data2D.data)]
-        qy_data = data2D.qy_data[numpy.isfinite(data2D.data)]
-        y = 0.0
-        err_y = 0.0
-        y_counts = 0.0
-        # Average pixelsize in q space
-        for npts in range(len(data)):
-            # default frac
-            frac_x = 0
-            frac_y = 0
-            # get min and max at each points
-            qx = qx_data[npts]
-            qy = qy_data[npts]
-            # get the ROI
-            if self.x_min <= qx and self.x_max > qx:
-                frac_x = 1
-            if self.y_min <= qy and self.y_max > qy:
-                frac_y = 1
-            #Find the fraction along each directions
-            frac = frac_x * frac_y
-            if frac == 0:
-                continue
-            y += frac * data[npts]
-            if err_data is None or err_data[npts] == 0.0:
-                if data[npts] < 0:
-                    data[npts] = -data[npts]
-                err_y += frac * frac * data[npts]
-            else:
-                err_y += frac * frac * err_data[npts] * err_data[npts]
-            y_counts += frac
-        return y, err_y, y_counts
-class Boxavg(Boxsum):
-    """
-    Perform the average of counts in a 2D region of interest.
-    """
-    def __init__(self, x_min=0.0, x_max=0.0, y_min=0.0, y_max=0.0):
-        super(Boxavg, self).__init__(x_min=x_min, x_max=x_max,
-                                     y_min=y_min, y_max=y_max)
-    def __call__(self, data2D):
-        """
-        Perform the sum in the region of interest
-        :param data2D: Data2D object
-        :return: average counts, error on average counts
-        """
-        y, err_y, y_counts = self._sum(data2D)
-        # Average the sums
-        counts = 0 if y_counts == 0 else y / y_counts
-        error = 0 if y_counts == 0 else math.sqrt(err_y) / y_counts
-        return counts, error
 def get_pixel_fraction_square(x, xmin, xmax):
     """
 …
         return 1.0
+class CircularAverage(object):
+    """
+    Perform circular averaging on 2D data
+    The data returned is the distribution of counts
+    as a function of Q
+    """
+    def __init__(self, r_min=0.0, r_max=0.0, bin_width=0.0005):
+        # Minimum radius included in the average [A-1]
+        self.r_min = r_min
+        # Maximum radius included in the average [A-1]
+        self.r_max = r_max
+        # Bin width (step size) [A-1]
+        self.bin_width = bin_width
+    def __call__(self, data2D, ismask=False):
+        """
+        Perform circular averaging on the data
+        :param data2D: Data2D object
+        :return: Data1D object
+        """
+        # Get data W/ finite values
+        data = data2D.data[numpy.isfinite(data2D.data)]
+        q_data = data2D.q_data[numpy.isfinite(data2D.data)]
+        err_data = data2D.err_data[numpy.isfinite(data2D.data)]
+        mask_data = data2D.mask[numpy.isfinite(data2D.data)]
+        dq_data = None
+        # Get the dq for resolution averaging
+        if data2D.dqx_data is not None and data2D.dqy_data is not None:
+            # The pinholes and det. pix contribution present
+            # in both direction of the 2D which must be subtracted when
+            # converting to 1D: dq_overlap should calculated ideally at
+            # q = 0. Note This method works on only pinhole geometry.
+            # Extrapolate dqx(r) and dqy(phi) at q = 0, and take an average.
+            z_max = max(data2D.q_data)
+            z_min = min(data2D.q_data)
+            x_max = data2D.dqx_data[data2D.q_data[z_max]]
+            x_min = data2D.dqx_data[data2D.q_data[z_min]]
+            y_max = data2D.dqy_data[data2D.q_data[z_max]]
+            y_min = data2D.dqy_data[data2D.q_data[z_min]]
+            # Find qdx at q = 0
+            dq_overlap_x = (x_min * z_max - x_max * z_min) / (z_max - z_min)
+            # when extrapolation goes wrong
+            if dq_overlap_x > min(data2D.dqx_data):
+                dq_overlap_x = min(data2D.dqx_data)
+            dq_overlap_x *= dq_overlap_x
+            # Find qdx at q = 0
+            dq_overlap_y = (y_min * z_max - y_max * z_min) / (z_max - z_min)
+            # when extrapolation goes wrong
+            if dq_overlap_y > min(data2D.dqy_data):
+                dq_overlap_y = min(data2D.dqy_data)
+            # get dq at q=0.
+            dq_overlap_y *= dq_overlap_y
+            dq_overlap = numpy.sqrt((dq_overlap_x + dq_overlap_y) / 2.0)
+            # Final protection of dq
+            if dq_overlap < 0:
+                dq_overlap = y_min
+            dqx_data = data2D.dqx_data[numpy.isfinite(data2D.data)]
+            dqy_data = data2D.dqy_data[numpy.isfinite(data2D.data)] - dq_overlap
+            # def; dqx_data = dq_r dqy_data = dq_phi
+            # Convert dq 2D to 1D here
+            dqx = dqx_data * dqx_data
+            dqy = dqy_data * dqy_data
+            dq_data = numpy.add(dqx, dqy)
+            dq_data = numpy.sqrt(dq_data)
+        #q_data_max = numpy.max(q_data)
+        if len(data2D.q_data) is None:
+            msg = "Circular averaging: invalid q_data: %g" % data2D.q_data
+            raise RuntimeError, msg
+        # Build array of Q intervals
+        nbins = int(math.ceil((self.r_max - self.r_min) / self.bin_width))
+        x = numpy.zeros(nbins)
+        y = numpy.zeros(nbins)
+        err_y = numpy.zeros(nbins)
+        err_x = numpy.zeros(nbins)
+        y_counts = numpy.zeros(nbins)
+        for npt in range(len(data)):
+            if ismask and not mask_data[npt]:
+                continue
+            frac = 0
+            # q-value at the pixel (j,i)
+            q_value = q_data[npt]
+            data_n = data[npt]
+            ## No need to calculate the frac when all data are within range
+            if self.r_min >= self.r_max:
+                raise ValueError, "Limit Error: min > max"
+            if self.r_min <= q_value and q_value <= self.r_max:
+                frac = 1
+            if frac == 0:
+                continue
+            i_q = int(math.floor((q_value - self.r_min) / self.bin_width))
+            # Take care of the edge case at phi = 2pi.
+            if i_q == nbins:
+                i_q = nbins - 1
+            y[i_q] += frac * data_n
+            # Take dqs from data to get the q_average
+            x[i_q] += frac * q_value
+            if err_data is None or err_data[npt] == 0.0:
+                if data_n < 0:
+                    data_n = -data_n
+                err_y[i_q] += frac * frac * data_n
+            else:
+                err_y[i_q] += frac * frac * err_data[npt] * err_data[npt]
+            if dq_data is not None:
+                # To be consistent with dq calculation in 1d reduction,
+                # we need just the averages (not quadratures) because
+                # it should not depend on the number of the q points
+                # in the qr bins.
+                err_x[i_q] += frac * dq_data[npt]
+            else:
+                err_x = None
+            y_counts[i_q] += frac
+        # Average the sums
+        for n in range(nbins):
+            if err_y[n] < 0:
+                err_y[n] = -err_y[n]
+            err_y[n] = math.sqrt(err_y[n])
+            #if err_x is not None:
+            #    err_x[n] = math.sqrt(err_x[n])
+        err_y = err_y / y_counts
+        err_y[err_y == 0] = numpy.average(err_y)
+        y = y / y_counts
+        x = x / y_counts
+        idx = (numpy.isfinite(y)) & (numpy.isfinite(x))
+        if err_x is not None:
+            d_x = err_x[idx] / y_counts[idx]
+        else:
+            d_x = None
+        if not idx.any():
+            msg = "Average Error: No points inside ROI to average..."
+            raise ValueError, msg
+        return Data1D(x=x[idx], y=y[idx], dy=err_y[idx], dx=d_x)
+class Ring(object):
+    """
+    Defines a ring on a 2D data set.
+    The ring is defined by r_min, r_max, and
+    the position of the center of the ring.
+    The data returned is the distribution of counts
+    around the ring as a function of phi.
+    Phi_min and phi_max should be defined between 0 and 2*pi
+    in anti-clockwise starting from the x- axis on the left-hand side
+    """
+    #Todo: remove center.
+    def __init__(self, r_min=0, r_max=0, center_x=0, center_y=0, nbins=36):
+        # Minimum radius
+        self.r_min = r_min
+        # Maximum radius
+        self.r_max = r_max
+        # Center of the ring in x
+        self.center_x = center_x
+        # Center of the ring in y
+        self.center_y = center_y
+        # Number of angular bins
+        self.nbins_phi = nbins
+    def __call__(self, data2D):
+        """
+        Apply the ring to the data set.
+        Returns the angular distribution for a given q range
+        :param data2D: Data2D object
+        :return: Data1D object
+        """
+        if data2D.__class__.__name__ not in ["Data2D", "plottable_2D"]:
+            raise RuntimeError, "Ring averaging only take plottable_2D objects"
+        Pi = math.pi
+        # Get data
+        data = data2D.data[numpy.isfinite(data2D.data)]
+        q_data = data2D.q_data[numpy.isfinite(data2D.data)]
+        err_data = data2D.err_data[numpy.isfinite(data2D.data)]
+        qx_data = data2D.qx_data[numpy.isfinite(data2D.data)]
+        qy_data = data2D.qy_data[numpy.isfinite(data2D.data)]
+        # Set space for 1d outputs
+        phi_bins = numpy.zeros(self.nbins_phi)
+        phi_counts = numpy.zeros(self.nbins_phi)
+        phi_values = numpy.zeros(self.nbins_phi)
+        phi_err = numpy.zeros(self.nbins_phi)
+        # Shift to apply to calculated phi values in order
+        # to center first bin at zero
+        phi_shift = Pi / self.nbins_phi
+        for npt in range(len(data)):
+            frac = 0
+            # q-value at the point (npt)
+            q_value = q_data[npt]
+            data_n = data[npt]
+            # phi-value at the point (npt)
+            phi_value = math.atan2(qy_data[npt], qx_data[npt]) + Pi
+            if self.r_min <= q_value and q_value <= self.r_max:
+                frac = 1
+            if frac == 0:
+                continue
+            # binning
+            i_phi = int(math.floor((self.nbins_phi) * \
+                                   (phi_value + phi_shift) / (2 * Pi)))
+            # Take care of the edge case at phi = 2pi.
+            if i_phi >= self.nbins_phi:
+                i_phi = 0
+            phi_bins[i_phi] += frac * data[npt]
+            if err_data is None or err_data[npt] == 0.0:
+                if data_n < 0:
+                    data_n = -data_n
+                phi_err[i_phi] += frac * frac * math.fabs(data_n)
+            else:
+                phi_err[i_phi] += frac * frac * err_data[npt] * err_data[npt]
+            phi_counts[i_phi] += frac
+        for i in range(self.nbins_phi):
+            phi_bins[i] = phi_bins[i] / phi_counts[i]
+            phi_err[i] = math.sqrt(phi_err[i]) / phi_counts[i]
+            phi_values[i] = 2.0 * math.pi / self.nbins_phi * (1.0 * i)
+        idx = (numpy.isfinite(phi_bins))
+        if not idx.any():
+            msg = "Average Error: No points inside ROI to average..."
+            raise ValueError, msg
+        #elif len(phi_bins[idx])!= self.nbins_phi:
+        #    print "resulted",self.nbins_phi- len(phi_bins[idx])
+        #,"empty bin(s) due to tight binning..."
+        return Data1D(x=phi_values[idx], y=phi_bins[idx], dy=phi_err[idx])
+def get_intercept(q, q_0, q_1):
+    """
+    Returns the fraction of the side at which the
+    q-value intercept the pixel, None otherwise.
+    The values returned is the fraction ON THE SIDE
+    OF THE LOWEST Q. ::
+            A           B
+        +-----------+--------+    <--- pixel size
+                    1
+        Q_0 -------- Q ----- Q_1   <--- equivalent Q range
+        if Q_1 > Q_0, A is returned
+        if Q_1 < Q_0, B is returned
+        if Q is outside the range of [Q_0, Q_1], None is returned
+    """
+    if q_1 > q_0:
+        if q > q_0 and q <= q_1:
+            return (q - q_0) / (q_1 - q_0)
+    else:
+        if q > q_1 and q <= q_0:
+            return (q - q_1) / (q_0 - q_1)
+    return None
 def get_pixel_fraction(qmax, q_00, q_01, q_10, q_11):
 …
     return frac_max
+def get_intercept(q, q_0, q_1):
+    """
+    Returns the fraction of the side at which the
+    q-value intercept the pixel, None otherwise.
+    The values returned is the fraction ON THE SIDE
+    OF THE LOWEST Q. ::
+            A           B
+        +-----------+--------+    <--- pixel size
+                    1
+        Q_0 -------- Q ----- Q_1   <--- equivalent Q range
+        if Q_1 > Q_0, A is returned
+        if Q_1 < Q_0, B is returned
+        if Q is outside the range of [Q_0, Q_1], None is returned
+    """
+    if q_1 > q_0:
+        if q > q_0 and q <= q_1:
+            return (q - q_0) / (q_1 - q_0)
+def get_dq_data(data2D):
+    '''
+    Get the dq for resolution averaging
+    The pinholes and det. pix contribution present
+    in both direction of the 2D which must be subtracted when
+    converting to 1D: dq_overlap should calculated ideally at
+    q = 0. Note This method works on only pinhole geometry.
+    Extrapolate dqx(r) and dqy(phi) at q = 0, and take an average.
+    '''
+    z_max = max(data2D.q_data)
+    z_min = min(data2D.q_data)
+    dqx_at_z_max = data2D.dqx_data[np.argmax(data2D.q_data)]
+    dqx_at_z_min = data2D.dqx_data[np.argmin(data2D.q_data)]
+    dqy_at_z_max = data2D.dqy_data[np.argmax(data2D.q_data)]
+    dqy_at_z_min = data2D.dqy_data[np.argmin(data2D.q_data)]
+    # Find qdx at q = 0
+    dq_overlap_x = (dqx_at_z_min * z_max - dqx_at_z_max * z_min) / (z_max - z_min)
+    # when extrapolation goes wrong
+    if dq_overlap_x > min(data2D.dqx_data):
+        dq_overlap_x = min(data2D.dqx_data)
+    dq_overlap_x *= dq_overlap_x
+    # Find qdx at q = 0
+    dq_overlap_y = (dqy_at_z_min * z_max - dqy_at_z_max * z_min) / (z_max - z_min)
+    # when extrapolation goes wrong
+    if dq_overlap_y > min(data2D.dqy_data):
+        dq_overlap_y = min(data2D.dqy_data)
+    # get dq at q=0.
+    dq_overlap_y *= dq_overlap_y
+    dq_overlap = np.sqrt((dq_overlap_x + dq_overlap_y) / 2.0)
+    # Final protection of dq
+    if dq_overlap < 0:
+        dq_overlap = dqy_at_z_min
+    dqx_data = data2D.dqx_data[np.isfinite(data2D.data)]
+    dqy_data = data2D.dqy_data[np.isfinite(
+        data2D.data)] - dq_overlap
+    # def; dqx_data = dq_r dqy_data = dq_phi
+    # Convert dq 2D to 1D here
+    dq_data = np.sqrt(dqx_data**2 + dqx_data**2)
+    return dq_data
+################################################################################
+def reader2D_converter(data2d=None):
+    """
+    convert old 2d format opened by IhorReader or danse_reader
+    to new Data2D format
+    This is mainly used by the Readers
+    :param data2d: 2d array of Data2D object
+    :return: 1d arrays of Data2D object
+    """
+    if data2d.data is None or data2d.x_bins is None or data2d.y_bins is None:
+        raise ValueError("Can't convert this data: data=None...")
+    new_x = np.tile(data2d.x_bins, (len(data2d.y_bins), 1))
+    new_y = np.tile(data2d.y_bins, (len(data2d.x_bins), 1))
+    new_y = new_y.swapaxes(0, 1)
+    new_data = data2d.data.flatten()
+    qx_data = new_x.flatten()
+    qy_data = new_y.flatten()
+    q_data = np.sqrt(qx_data * qx_data + qy_data * qy_data)
+    if data2d.err_data is None or np.any(data2d.err_data <= 0):
+        new_err_data = np.sqrt(np.abs(new_data))
     else:
+        if q > q_1 and q <= q_0:
+            return (q - q_1) / (q_0 - q_1)
+    return None
+        new_err_data = data2d.err_data.flatten()
+    mask = np.ones(len(new_data), dtype=bool)
+    # TODO: make sense of the following two lines...
+    #from sas.sascalc.dataloader.data_info import Data2D
+    #output = Data2D()
+    output = data2d
+    output.data = new_data
+    output.err_data = new_err_data
+    output.qx_data = qx_data
+    output.qy_data = qy_data
+    output.q_data = q_data
+    output.mask = mask
+    return output
+################################################################################
+class Binning(object):
+    '''
+    This class just creates a binning object
+    either linear or log
+    '''
+    def __init__(self, min_value, max_value, n_bins, base=None):
+        '''
+        if base is None: Linear binning
+        '''
+        self.min = min_value if min_value > 0 else 0.0001
+        self.max = max_value
+        self.n_bins = n_bins
+        self.base = base
+    def get_bin_index(self, value):
+        '''
+        The general formula logarithm binning is:
+        bin = floor(N * (log(x) - log(min)) / (log(max) - log(min)))
+        '''
+        if self.base:
+            temp_x = self.n_bins * (math.log(value, self.base) - math.log(self.min, self.base))
+            temp_y = math.log(self.max, self.base) - math.log(self.min, self.base)
+        else:
+            temp_x = self.n_bins * (value - self.min)
+            temp_y = self.max - self.min
+        # Bin index calulation
+        return int(math.floor(temp_x / temp_y))
+################################################################################
+class _Slab(object):
+    """
+    Compute average I(Q) for a region of interest
+    """
+    def __init__(self, x_min=0.0, x_max=0.0, y_min=0.0,
+                 y_max=0.0, bin_width=0.001):
+        # Minimum Qx value [A-1]
+        self.x_min = x_min
+        # Maximum Qx value [A-1]
+        self.x_max = x_max
+        # Minimum Qy value [A-1]
+        self.y_min = y_min
+        # Maximum Qy value [A-1]
+        self.y_max = y_max
+        # Bin width (step size) [A-1]
+        self.bin_width = bin_width
+        # If True, I(|Q|) will be return, otherwise,
+        # negative q-values are allowed
+        self.fold = False
+    def __call__(self, data2D):
+        return NotImplemented
+    def _avg(self, data2D, maj):
+        """
+        Compute average I(Q_maj) for a region of interest.
+        The major axis is defined as the axis of Q_maj.
+        The minor axis is the axis that we average over.
+        :param data2D: Data2D object
+        :param maj_min: min value on the major axis
+        :return: Data1D object
+        """
+        if len(data2D.detector) > 1:
+            msg = "_Slab._avg: invalid number of "
+            msg += " detectors: %g" % len(data2D.detector)
+            raise RuntimeError(msg)
+        # Get data
+        data = data2D.data[np.isfinite(data2D.data)]
+        err_data = data2D.err_data[np.isfinite(data2D.data)]
+        qx_data = data2D.qx_data[np.isfinite(data2D.data)]
+        qy_data = data2D.qy_data[np.isfinite(data2D.data)]
+        # Build array of Q intervals
+        if maj == 'x':
+            if self.fold:
+                x_min = 0
+            else:
+                x_min = self.x_min
+            nbins = int(math.ceil((self.x_max - x_min) / self.bin_width))
+        elif maj == 'y':
+            if self.fold:
+                y_min = 0
+            else:
+                y_min = self.y_min
+            nbins = int(math.ceil((self.y_max - y_min) / self.bin_width))
+        else:
+            raise RuntimeError("_Slab._avg: unrecognized axis %s" % str(maj))
+        x = np.zeros(nbins)
+        y = np.zeros(nbins)
+        err_y = np.zeros(nbins)
+        y_counts = np.zeros(nbins)
+        # Average pixelsize in q space
+        for npts in range(len(data)):
+            # default frac
+            frac_x = 0
+            frac_y = 0
+            # get ROI
+            if self.x_min <= qx_data[npts] and self.x_max > qx_data[npts]:
+                frac_x = 1
+            if self.y_min <= qy_data[npts] and self.y_max > qy_data[npts]:
+                frac_y = 1
+            frac = frac_x * frac_y
+            if frac == 0:
+                continue
+            # binning: find axis of q
+            if maj == 'x':
+                q_value = qx_data[npts]
+                min_value = x_min
+            if maj == 'y':
+                q_value = qy_data[npts]
+                min_value = y_min
+            if self.fold and q_value < 0:
+                q_value = -q_value
+            # bin
+            i_q = int(math.ceil((q_value - min_value) / self.bin_width)) - 1
+            # skip outside of max bins
+            if i_q < 0 or i_q >= nbins:
+                continue
+            # TODO: find better definition of x[i_q] based on q_data
+            # min_value + (i_q + 1) * self.bin_width / 2.0
+            x[i_q] += frac * q_value
+            y[i_q] += frac * data[npts]
+            if err_data is None or err_data[npts] == 0.0:
+                if data[npts] < 0:
+                    data[npts] = -data[npts]
+                err_y[i_q] += frac * frac * data[npts]
+            else:
+                err_y[i_q] += frac * frac * err_data[npts] * err_data[npts]
+            y_counts[i_q] += frac
+        # Average the sums
+        for n in range(nbins):
+            err_y[n] = math.sqrt(err_y[n])
+        err_y = err_y / y_counts
+        y = y / y_counts
+        x = x / y_counts
+        idx = (np.isfinite(y) & np.isfinite(x))
+        if not idx.any():
+            msg = "Average Error: No points inside ROI to average..."
+            raise ValueError(msg)
+        return Data1D(x=x[idx], y=y[idx], dy=err_y[idx])
+class SlabY(_Slab):
+    """
+    Compute average I(Qy) for a region of interest
+    """
+    def __call__(self, data2D):
+        """
+        Compute average I(Qy) for a region of interest
+        :param data2D: Data2D object
+        :return: Data1D object
+        """
+        return self._avg(data2D, 'y')
+class SlabX(_Slab):
+    """
+    Compute average I(Qx) for a region of interest
+    """
+    def __call__(self, data2D):
+        """
+        Compute average I(Qx) for a region of interest
+        :param data2D: Data2D object
+        :return: Data1D object
+        """
+        return self._avg(data2D, 'x')
+################################################################################
+class Boxsum(object):
+    """
+    Perform the sum of counts in a 2D region of interest.
+    """
+    def __init__(self, x_min=0.0, x_max=0.0, y_min=0.0, y_max=0.0):
+        # Minimum Qx value [A-1]
+        self.x_min = x_min
+        # Maximum Qx value [A-1]
+        self.x_max = x_max
+        # Minimum Qy value [A-1]
+        self.y_min = y_min
+        # Maximum Qy value [A-1]
+        self.y_max = y_max
+    def __call__(self, data2D):
+        """
+        Perform the sum in the region of interest
+        :param data2D: Data2D object
+        :return: number of counts, error on number of counts,
+            number of points summed
+        """
+        y, err_y, y_counts = self._sum(data2D)
+        # Average the sums
+        counts = 0 if y_counts == 0 else y
+        error = 0 if y_counts == 0 else math.sqrt(err_y)
+        # Added y_counts to return, SMK & PDB, 04/03/2013
+        return counts, error, y_counts
+    def _sum(self, data2D):
+        """
+        Perform the sum in the region of interest
+        :param data2D: Data2D object
+        :return: number of counts,
+            error on number of counts, number of entries summed
+        """
+        if len(data2D.detector) > 1:
+            msg = "Circular averaging: invalid number "
+            msg += "of detectors: %g" % len(data2D.detector)
+            raise RuntimeError(msg)
+        # Get data
+        data = data2D.data[np.isfinite(data2D.data)]
+        err_data = data2D.err_data[np.isfinite(data2D.data)]
+        qx_data = data2D.qx_data[np.isfinite(data2D.data)]
+        qy_data = data2D.qy_data[np.isfinite(data2D.data)]
+        y = 0.0
+        err_y = 0.0
+        y_counts = 0.0
+        # Average pixelsize in q space
+        for npts in range(len(data)):
+            # default frac
+            frac_x = 0
+            frac_y = 0
+            # get min and max at each points
+            qx = qx_data[npts]
+            qy = qy_data[npts]
+            # get the ROI
+            if self.x_min <= qx and self.x_max > qx:
+                frac_x = 1
+            if self.y_min <= qy and self.y_max > qy:
+                frac_y = 1
+            # Find the fraction along each directions
+            frac = frac_x * frac_y
+            if frac == 0:
+                continue
+            y += frac * data[npts]
+            if err_data is None or err_data[npts] == 0.0:
+                if data[npts] < 0:
+                    data[npts] = -data[npts]
+                err_y += frac * frac * data[npts]
+            else:
+                err_y += frac * frac * err_data[npts] * err_data[npts]
+            y_counts += frac
+        return y, err_y, y_counts
+class Boxavg(Boxsum):
+    """
+    Perform the average of counts in a 2D region of interest.
+    """
+    def __init__(self, x_min=0.0, x_max=0.0, y_min=0.0, y_max=0.0):
+        super(Boxavg, self).__init__(x_min=x_min, x_max=x_max,
+                                     y_min=y_min, y_max=y_max)
+    def __call__(self, data2D):
+        """
+        Perform the sum in the region of interest
+        :param data2D: Data2D object
+        :return: average counts, error on average counts
+        """
+        y, err_y, y_counts = self._sum(data2D)
+        # Average the sums
+        counts = 0 if y_counts == 0 else y / y_counts
+        error = 0 if y_counts == 0 else math.sqrt(err_y) / y_counts
+        return counts, error
+################################################################################
+class CircularAverage(object):
+    """
+    Perform circular averaging on 2D data
+    The data returned is the distribution of counts
+    as a function of Q
+    """
+    def __init__(self, r_min=0.0, r_max=0.0, bin_width=0.0005):
+        # Minimum radius included in the average [A-1]
+        self.r_min = r_min
+        # Maximum radius included in the average [A-1]
+        self.r_max = r_max
+        # Bin width (step size) [A-1]
+        self.bin_width = bin_width
+    def __call__(self, data2D, ismask=False):
+        """
+        Perform circular averaging on the data
+        :param data2D: Data2D object
+        :return: Data1D object
+        """
+        # Get data W/ finite values
+        data = data2D.data[np.isfinite(data2D.data)]
+        q_data = data2D.q_data[np.isfinite(data2D.data)]
+        err_data = data2D.err_data[np.isfinite(data2D.data)]
+        mask_data = data2D.mask[np.isfinite(data2D.data)]
+        dq_data = None
+        if data2D.dqx_data is not None and data2D.dqy_data is not None:
+            dq_data = get_dq_data(data2D)
+        if len(q_data) == 0:
+            msg = "Circular averaging: invalid q_data: %g" % data2D.q_data
+            raise RuntimeError(msg)
+        # Build array of Q intervals
+        nbins = int(math.ceil((self.r_max - self.r_min) / self.bin_width))
+        x = np.zeros(nbins)
+        y = np.zeros(nbins)
+        err_y = np.zeros(nbins)
+        err_x = np.zeros(nbins)
+        y_counts = np.zeros(nbins)
+        for npt in range(len(data)):
+            if ismask and not mask_data[npt]:
+                continue
+            frac = 0
+            # q-value at the pixel (j,i)
+            q_value = q_data[npt]
+            data_n = data[npt]
+            # No need to calculate the frac when all data are within range
+            if self.r_min >= self.r_max:
+                raise ValueError("Limit Error: min > max")
+            if self.r_min <= q_value and q_value <= self.r_max:
+                frac = 1
+            if frac == 0:
+                continue
+            i_q = int(math.floor((q_value - self.r_min) / self.bin_width))
+            # Take care of the edge case at phi = 2pi.
+            if i_q == nbins:
+                i_q = nbins - 1
+            y[i_q] += frac * data_n
+            # Take dqs from data to get the q_average
+            x[i_q] += frac * q_value
+            if err_data is None or err_data[npt] == 0.0:
+                if data_n < 0:
+                    data_n = -data_n
+                err_y[i_q] += frac * frac * data_n
+            else:
+                err_y[i_q] += frac * frac * err_data[npt] * err_data[npt]
+            if dq_data is not None:
+                # To be consistent with dq calculation in 1d reduction,
+                # we need just the averages (not quadratures) because
+                # it should not depend on the number of the q points
+                # in the qr bins.
+                err_x[i_q] += frac * dq_data[npt]
+            else:
+                err_x = None
+            y_counts[i_q] += frac
+        # Average the sums
+        for n in range(nbins):
+            if err_y[n] < 0:
+                err_y[n] = -err_y[n]
+            err_y[n] = math.sqrt(err_y[n])
+            # if err_x is not None:
+            #    err_x[n] = math.sqrt(err_x[n])
+        err_y = err_y / y_counts
+        err_y[err_y == 0] = np.average(err_y)
+        y = y / y_counts
+        x = x / y_counts
+        idx = (np.isfinite(y)) & (np.isfinite(x))
+        if err_x is not None:
+            d_x = err_x[idx] / y_counts[idx]
+        else:
+            d_x = None
+        if not idx.any():
+            msg = "Average Error: No points inside ROI to average..."
+            raise ValueError(msg)
+        return Data1D(x=x[idx], y=y[idx], dy=err_y[idx], dx=d_x)
+################################################################################
+class Ring(object):
+    """
+    Defines a ring on a 2D data set.
+    The ring is defined by r_min, r_max, and
+    the position of the center of the ring.
+    The data returned is the distribution of counts
+    around the ring as a function of phi.
+    Phi_min and phi_max should be defined between 0 and 2*pi
+    in anti-clockwise starting from the x- axis on the left-hand side
+    """
+    # Todo: remove center.
+    def __init__(self, r_min=0, r_max=0, center_x=0, center_y=0, nbins=36):
+        # Minimum radius
+        self.r_min = r_min
+        # Maximum radius
+        self.r_max = r_max
+        # Center of the ring in x
+        self.center_x = center_x
+        # Center of the ring in y
+        self.center_y = center_y
+        # Number of angular bins
+        self.nbins_phi = nbins
+    def __call__(self, data2D):
+        """
+        Apply the ring to the data set.
+        Returns the angular distribution for a given q range
+        :param data2D: Data2D object
+        :return: Data1D object
+        """
+        if data2D.__class__.__name__ not in ["Data2D", "plottable_2D"]:
+            raise RuntimeError("Ring averaging only take plottable_2D objects")
+        Pi = math.pi
+        # Get data
+        data = data2D.data[np.isfinite(data2D.data)]
+        q_data = data2D.q_data[np.isfinite(data2D.data)]
+        err_data = data2D.err_data[np.isfinite(data2D.data)]
+        qx_data = data2D.qx_data[np.isfinite(data2D.data)]
+        qy_data = data2D.qy_data[np.isfinite(data2D.data)]
+        # Set space for 1d outputs
+        phi_bins = np.zeros(self.nbins_phi)
+        phi_counts = np.zeros(self.nbins_phi)
+        phi_values = np.zeros(self.nbins_phi)
+        phi_err = np.zeros(self.nbins_phi)
+        # Shift to apply to calculated phi values in order
+        # to center first bin at zero
+        phi_shift = Pi / self.nbins_phi
+        for npt in range(len(data)):
+            frac = 0
+            # q-value at the point (npt)
+            q_value = q_data[npt]
+            data_n = data[npt]
+            # phi-value at the point (npt)
+            phi_value = math.atan2(qy_data[npt], qx_data[npt]) + Pi
+            if self.r_min <= q_value and q_value <= self.r_max:
+                frac = 1
+            if frac == 0:
+                continue
+            # binning
+            i_phi = int(math.floor((self.nbins_phi) *
+                                   (phi_value + phi_shift) / (2 * Pi)))
+            # Take care of the edge case at phi = 2pi.
+            if i_phi >= self.nbins_phi:
+                i_phi = 0
+            phi_bins[i_phi] += frac * data[npt]
+            if err_data is None or err_data[npt] == 0.0:
+                if data_n < 0:
+                    data_n = -data_n
+                phi_err[i_phi] += frac * frac * math.fabs(data_n)
+            else:
+                phi_err[i_phi] += frac * frac * err_data[npt] * err_data[npt]
+            phi_counts[i_phi] += frac
+        for i in range(self.nbins_phi):
+            phi_bins[i] = phi_bins[i] / phi_counts[i]
+            phi_err[i] = math.sqrt(phi_err[i]) / phi_counts[i]
+            phi_values[i] = 2.0 * math.pi / self.nbins_phi * (1.0 * i)
+        idx = (np.isfinite(phi_bins))
+        if not idx.any():
+            msg = "Average Error: No points inside ROI to average..."
+            raise ValueError(msg)
+        # elif len(phi_bins[idx])!= self.nbins_phi:
+        #    print "resulted",self.nbins_phi- len(phi_bins[idx])
+        #,"empty bin(s) due to tight binning..."
+        return Data1D(x=phi_values[idx], y=phi_bins[idx], dy=phi_err[idx])
 …
     starting from the x- axis on the left-hand side
     """
+    def __init__(self, r_min, r_max, phi_min=0, phi_max=2 * math.pi, nbins=20):
+    def __init__(self, r_min, r_max, phi_min=0, phi_max=2 * math.pi, nbins=20,
+                 base=None):
+        '''
+        :param base: must be a valid base for an algorithm, i.e.,
+        a positive number
+        '''
         self.r_min = r_min
         self.r_max = r_max
 …
         self.phi_max = phi_max
         self.nbins = nbins
+        self.base = base
     def _agv(self, data2D, run='phi'):
 …
         """
         if data2D.__class__.__name__ not in ["Data2D", "plottable_2D"]:
+            raise RuntimeError, "Ring averaging only take plottable_2D objects"
+        Pi = math.pi
+            raise RuntimeError("Ring averaging only take plottable_2D objects")
         # Get the all data & info
+        data = data2D.data[numpy.isfinite(data2D.data)]
+        q_data = data2D.q_data[numpy.isfinite(data2D.data)]
+        err_data = data2D.err_data[numpy.isfinite(data2D.data)]
+        qx_data = data2D.qx_data[numpy.isfinite(data2D.data)]
+        qy_data = data2D.qy_data[numpy.isfinite(data2D.data)]
+        data = data2D.data[np.isfinite(data2D.data)]
+        q_data = data2D.q_data[np.isfinite(data2D.data)]
+        err_data = data2D.err_data[np.isfinite(data2D.data)]
+        qx_data = data2D.qx_data[np.isfinite(data2D.data)]
+        qy_data = data2D.qy_data[np.isfinite(data2D.data)]
         dq_data = None
-        # Get the dq for resolution averaging
         if data2D.dqx_data is not None and data2D.dqy_data is not None:
+            # The pinholes and det. pix contribution present
+            # in both direction of the 2D which must be subtracted when
+            # converting to 1D: dq_overlap should calculated ideally at
+            # q = 0.
+            # Extrapolate dqy(perp) at q = 0
+            z_max = max(data2D.q_data)
+            z_min = min(data2D.q_data)
+            x_max = data2D.dqx_data[data2D.q_data[z_max]]
+            x_min = data2D.dqx_data[data2D.q_data[z_min]]
+            y_max = data2D.dqy_data[data2D.q_data[z_max]]
+            y_min = data2D.dqy_data[data2D.q_data[z_min]]
+            # Find qdx at q = 0
+            dq_overlap_x = (x_min * z_max - x_max * z_min) / (z_max - z_min)
+            # when extrapolation goes wrong
+            if dq_overlap_x > min(data2D.dqx_data):
+                dq_overlap_x = min(data2D.dqx_data)
+            dq_overlap_x *= dq_overlap_x
+            # Find qdx at q = 0
+            dq_overlap_y = (y_min * z_max - y_max * z_min) / (z_max - z_min)
+            # when extrapolation goes wrong
+            if dq_overlap_y > min(data2D.dqy_data):
+                dq_overlap_y = min(data2D.dqy_data)
+            # get dq at q=0.
+            dq_overlap_y *= dq_overlap_y
+            dq_overlap = numpy.sqrt((dq_overlap_x + dq_overlap_y) / 2.0)
+            if dq_overlap < 0:
+                dq_overlap = y_min
+            dqx_data = data2D.dqx_data[numpy.isfinite(data2D.data)]
+            dqy_data = data2D.dqy_data[numpy.isfinite(data2D.data)] - dq_overlap
+            # def; dqx_data = dq_r dqy_data = dq_phi
+            # Convert dq 2D to 1D here
+            dqx = dqx_data * dqx_data
+            dqy = dqy_data * dqy_data
+            dq_data = numpy.add(dqx, dqy)
+            dq_data = numpy.sqrt(dq_data)
+        #set space for 1d outputs
+        x = numpy.zeros(self.nbins)
+        y = numpy.zeros(self.nbins)
+        y_err = numpy.zeros(self.nbins)
+        x_err = numpy.zeros(self.nbins)
+        y_counts = numpy.zeros(self.nbins)
+            dq_data = get_dq_data(data2D)
+        # set space for 1d outputs
+        x = np.zeros(self.nbins)
+        y = np.zeros(self.nbins)
+        y_err = np.zeros(self.nbins)
+        x_err = np.zeros(self.nbins)
+        y_counts = np.zeros(self.nbins)  # Cycle counts (for the mean)
         # Get the min and max into the region: 0 <= phi < 2Pi
 …
         phi_max = flip_phi(self.phi_max)
+        #  binning object
+        if run.lower() == 'phi':
+            binning = Binning(self.phi_min, self.phi_max, self.nbins, self.base)
+        else:
+            binning = Binning(self.r_min, self.r_max, self.nbins, self.base)
         for n in range(len(data)):
-            frac = 0
             # q-value at the pixel (j,i)
 …
             # phi-value of the pixel (j,i)
+            phi_value = math.atan2(qy_data[n], qx_data[n]) + Pi
+            ## No need to calculate the frac when all data are within range
+            if self.r_min <= q_value and q_value <= self.r_max:
+                frac = 1
+            if frac == 0:
+            phi_value = math.atan2(qy_data[n], qx_data[n]) + math.pi
+            # No need to calculate: data outside of the radius
+            if self.r_min > q_value or q_value > self.r_max:
                 continue
+            #In case of two ROIs (symmetric major and minor regions)(for 'q2')
+            # In case of two ROIs (symmetric major and minor regions)(for 'q2')
             if run.lower() == 'q2':
                 ## For minor sector wing
+                # For minor sector wing
                 # Calculate the minor wing phis
                 phi_min_minor = flip_phi(phi_min - Pi)
                 phi_max_minor = flip_phi(phi_max - Pi)
+                phi_min_minor = flip_phi(phi_min - math.pi)
+                phi_max_minor = flip_phi(phi_max - math.pi)
                 # Check if phis of the minor ring is within 0 to 2pi
                 if phi_min_minor > phi_max_minor:
                     is_in = (phi_value > phi_min_minor or \
                               phi_value < phi_max_minor)
+                    is_in = (phi_value > phi_min_minor or
+                             phi_value < phi_max_minor)
                 else:
                     is_in = (phi_value > phi_min_minor and \
                               phi_value < phi_max_minor)
             #For all cases(i.e.,for 'q', 'q2', and 'phi')
             #Find pixels within ROI
+                    is_in = (phi_value > phi_min_minor and
+                             phi_value < phi_max_minor)
+            # For all cases(i.e.,for 'q', 'q2', and 'phi')
+            # Find pixels within ROI
             if phi_min > phi_max:
                 is_in = is_in or (phi_value > phi_min or \
                                    phi_value < phi_max)
+                is_in = is_in or (phi_value > phi_min or
+                                  phi_value < phi_max)
             else:
+                is_in = is_in or (phi_value >= phi_min  and \
+                                    phi_value < phi_max)
+                is_in = is_in or (phi_value >= phi_min and
+                                  phi_value < phi_max)
+            # data oustide of the phi range
             if not is_in:
-                frac = 0
-            if frac == 0:
                 continue
+            # Check which type of averaging we need
+            # Get the binning index
             if run.lower() == 'phi':
+                temp_x = (self.nbins) * (phi_value - self.phi_min)
+                temp_y = (self.phi_max - self.phi_min)
+                i_bin = int(math.floor(temp_x / temp_y))
+                i_bin = binning.get_bin_index(phi_value)
             else:
+                temp_x = (self.nbins) * (q_value - self.r_min)
+                temp_y = (self.r_max - self.r_min)
+                i_bin = int(math.floor(temp_x / temp_y))
+                i_bin = binning.get_bin_index(q_value)
             # Take care of the edge case at phi = 2pi.
 …
                 i_bin = self.nbins - 1
             ## Get the total y
             y[i_bin] += frac * data_n
             x[i_bin] += frac * q_value
+            # Get the total y
+            y[i_bin] += data_n
+            x[i_bin] += q_value
             if err_data[n] is None or err_data[n] == 0.0:
                 if data_n < 0:
                     data_n = -data_n
                 y_err[i_bin] += frac * frac * data_n
+                y_err[i_bin] += data_n
             else:
                 y_err[i_bin] += frac * frac * err_data[n] * err_data[n]
+                y_err[i_bin] += err_data[n]**2
             if dq_data is not None:
 …
                 # it should not depend on the number of the q points
                 # in the qr bins.
                 x_err[i_bin] += frac * dq_data[n]
+                x_err[i_bin] += dq_data[n]
             else:
                 x_err = None
             y_counts[i_bin] += frac
+            y_counts[i_bin] += 1
         # Organize the results
 …
                 # We take the center of ring area, not radius.
                 # This is more accurate than taking the radial center of ring.
                 #delta_r = (self.r_max - self.r_min) / self.nbins
                 #r_inner = self.r_min + delta_r * i
                 #r_outer = r_inner + delta_r
                 #x[i] = math.sqrt((r_inner * r_inner + r_outer * r_outer) / 2)
+                # delta_r = (self.r_max - self.r_min) / self.nbins
+                # r_inner = self.r_min + delta_r * i
+                # r_outer = r_inner + delta_r
+                # x[i] = math.sqrt((r_inner * r_inner + r_outer * r_outer) / 2)
                 x[i] = x[i] / y_counts[i]
         y_err[y_err == 0] = numpy.average(y_err)
         idx = (numpy.isfinite(y) & numpy.isfinite(y_err))
+        y_err[y_err == 0] = np.average(y_err)
+        idx = (np.isfinite(y) & np.isfinite(y_err))
         if x_err is not None:
             d_x = x_err[idx] / y_counts[idx]
 …
         if not idx.any():
             msg = "Average Error: No points inside sector of ROI to average..."
             raise ValueError, msg
         #elif len(y[idx])!= self.nbins:
+            raise ValueError(msg)
+        # elif len(y[idx])!= self.nbins:
         #    print "resulted",self.nbins- len(y[idx]),
         #"empty bin(s) due to tight binning..."
+        # "empty bin(s) due to tight binning..."
         return Data1D(x=x[idx], y=y[idx], dy=y_err[idx], dx=d_x)
 …
     The number of bin in phi also has to be defined.
     """
     def __call__(self, data2D):
         """
 …
     The number of bin in Q also has to be defined.
     """
     def __call__(self, data2D):
         """
 …
         return self._agv(data2D, 'q2')
+################################################################################
 class Ringcut(object):
 …
     in anti-clockwise starting from the x- axis on the left-hand side
     """
     def __init__(self, r_min=0, r_max=0, center_x=0, center_y=0):
         # Minimum radius
 …
         """
         if data2D.__class__.__name__ not in ["Data2D", "plottable_2D"]:
             raise RuntimeError, "Ring cut only take plottable_2D objects"
+            raise RuntimeError("Ring cut only take plottable_2D objects")
         # Get data
         qx_data = data2D.qx_data
         qy_data = data2D.qy_data
         q_data = numpy.sqrt(qx_data * qx_data + qy_data * qy_data)
+        q_data = np.sqrt(qx_data * qx_data + qy_data * qy_data)
         # check whether or not the data point is inside ROI
 …
         return out
+################################################################################
 class Boxcut(object):
 …
     Find a rectangular 2D region of interest.
     """
     def __init__(self, x_min=0.0, x_max=0.0, y_min=0.0, y_max=0.0):
         # Minimum Qx value [A-1]
 …
         """
         if data2D.__class__.__name__ not in ["Data2D", "plottable_2D"]:
             raise RuntimeError, "Boxcut take only plottable_2D objects"
+            raise RuntimeError("Boxcut take only plottable_2D objects")
         # Get qx_ and qy_data
         qx_data = data2D.qx_data
 …
         return outx & outy
+################################################################################
 class Sectorcut(object):
 …
     and (phi_max-phi_min) should not be larger than pi
     """
     def __init__(self, phi_min=0, phi_max=math.pi):
         self.phi_min = phi_min
 …
         """
         if data2D.__class__.__name__ not in ["Data2D", "plottable_2D"]:
             raise RuntimeError, "Sectorcut take only plottable_2D objects"
+            raise RuntimeError("Sectorcut take only plottable_2D objects")
         Pi = math.pi
         # Get data
 …
         # get phi from data
         phi_data = numpy.arctan2(qy_data, qx_data)
+        phi_data = np.arctan2(qy_data, qx_data)
         # Get the min and max into the region: -pi <= phi < Pi
 …
         # check for major sector
         if phi_min_major > phi_max_major:
+            out_major = (phi_min_major <= phi_data) + (phi_max_major > phi_data)
+            out_major = (phi_min_major <= phi_data) + \
+                (phi_max_major > phi_data)
         else:
+            out_major = (phi_min_major <= phi_data) & (phi_max_major > phi_data)
+            out_major = (phi_min_major <= phi_data) & (
+                phi_max_major > phi_data)
         # minor sector
 …
         if phi_min_minor > phi_max_minor:
             out_minor = (phi_min_minor <= phi_data) + \
                             (phi_max_minor >= phi_data)
+                (phi_max_minor >= phi_data)
         else:
             out_minor = (phi_min_minor <= phi_data) & \
                             (phi_max_minor >= phi_data)
+                (phi_max_minor >= phi_data)
         out = out_major + out_minor

src/sas/sascalc/dataloader/readers/IgorReader.py

r959eb01	ra1b8fee
12	12	#copyright 2008, University of Tennessee
13	13	#############################################################################
	14	from __future__ import print_function
	15
14	16	import os
15	17

src/sas/sascalc/dataloader/readers/sesans_reader.py

-                      rb2c28a5
+                      r5a8cdbb
     Class to load sesans files (6 columns).
     """
     ## File type
+    # File type
     type_name = "SESANS"
 …
     type = ["SESANS files (*.ses)|*.ses",
             "SESANS files (*..sesans)|*.sesans"]
     ## List of allowed extensions
+    # List of allowed extensions
     ext = ['.ses', '.SES', '.sesans', '.SESANS']
     ## Flag to bypass extension check
     allow_all = False
+    # Flag to bypass extension check
+    allow_all = True
     def get_file_contents(self):
 …
         self.output = []
+        error_message = ""
+        loaded_correctly = True
+        line = self.f_open.readline()
+        params = {}
+        while not line.startswith("BEGIN_DATA"):
+            terms = line.split()
+            if len(terms) >= 2:
+                params[terms[0]] = " ".join(terms[1:])
+            line = self.f_open.readline()
+        self.params = params
+        import pdb; pdb.set_trace()
+        if "FileFormatVersion" not in self.params:
+            raise FileContentsException("SES file missing FileFormatVersion")
+        if float(self.params["FileFormatVersion"]) >= 2.0:
+            raise FileContentsException("SASView only supports SES version 1")
+        buff = self.f_open.read()
+        lines = buff.splitlines()
+        if "SpinEchoLength_unit" not in self.params:
+            raise FileContentsException("SpinEchoLength has no units")
+        if "Wavelength_unit" not in self.params:
+            raise FileContentsException("Wavelength has no units")
+        if params["SpinEchoLength_unit"] != params["Wavelength_unit"]:
+            raise FileContentsException("The spin echo data has rudely used "
+                               "different units for the spin echo length "
+                               "and the wavelength.  While sasview could "
+                               "handle this instance, it is a violation "
+                               "of the file format and will not be "
+                               "handled by other software.")
         self.current_datainfo.filename = os.path.basename(self.f_open.name)
+        headers = self.f_open.readline().split()
+        paramnames=[]
+        paramvals=[]
+        zvals=[]
+        dzvals=[]
+        lamvals=[]
+        dlamvals=[]
+        Pvals=[]
+        dPvals=[]
+        self._insist_header(headers, "SpinEchoLength")
+        self._insist_header(headers, "Depolarisation")
+        self._insist_header(headers, "Depolarisation_error")
+        self._insist_header(headers, "Wavelength")
+        for line in lines:
+            # Initial try for CSV (split on ,)
+            line=line.strip()
+            toks = line.split('\t')
+            if len(toks)==2:
+                paramnames.append(toks[0])
+                paramvals.append(toks[1])
+            elif len(toks)>5:
+                zvals.append(toks[0])
+                dzvals.append(toks[3])
+                lamvals.append(toks[4])
+                dlamvals.append(toks[5])
+                Pvals.append(toks[1])
+                dPvals.append(toks[2])
+            data = np.loadtxt(self.f_open)
+            if data.shape[1] != len(headers):
+                raise FileContentsException(
+                    "File has {} headers, but {} columns".format(
+                        len(headers),
+                        data.shape[1]))
+            if not data.size:
+                raise FileContentsException("{} is empty".format(path))
+            x = data[:, headers.index("SpinEchoLength")]
+            if "SpinEchoLength_error" in headers:
+                dx = data[:, headers.index("SpinEchoLength_error")]
             else:
+                continue
+                dx = x * 0.05
+            lam = data[:, headers.index("Wavelength")]
+            if "Wavelength_error" in headers:
+                dlam = data[:, headers.index("Wavelength_error")]
+            else:
+                dlam = lam * 0.05
+            y = data[:, headers.index("Depolarisation")]
+            dy = data[:, headers.index("Depolarisation_error")]
+        x=[]
+        y=[]
+        lam=[]
+        dx=[]
+        dy=[]
+        dlam=[]
+        lam_header = lamvals[0].split()
+        data_conv_z = None
+        default_z_unit = "A"
+        data_conv_P = None
+        default_p_unit = " " # Adjust unit for axis (L^-3)
+        lam_unit = lam_header[1].replace("[","").replace("]","")
+        if lam_unit == 'AA':
+            lam_unit = 'A'
+        varheader=[zvals[0],dzvals[0],lamvals[0],dlamvals[0],Pvals[0],dPvals[0]]
+        valrange=range(1, len(zvals))
+        try:
+            for i in valrange:
+                x.append(float(zvals[i]))
+                y.append(float(Pvals[i]))
+                lam.append(float(lamvals[i]))
+                dy.append(float(dPvals[i]))
+                dx.append(float(dzvals[i]))
+                dlam.append(float(dlamvals[i]))
+        except ValueError as val_err:
+            err_msg = "Invalid float"
+            err_msg += ":".join(val_err.message.split(":")[1:])
+            raise FileContentsException(err_msg)
+            lam_unit = self._unit_fetch("Wavelength")
+            x, x_unit = self._unit_conversion(x, "A",
+                                              self._unit_fetch(
+                                                  "SpinEchoLength"))
+            dx, dx_unit = self._unit_conversion(
+                dx, lam_unit,
+                self._unit_fetch("SpinEchoLength"))
+            dlam, dlam_unit = self._unit_conversion(
+                dlam, lam_unit,
+                self._unit_fetch("Wavelength"))
+            y_unit = self._unit_fetch("Depolarisation")
+        x, y, lam, dy, dx, dlam = [
+            np.asarray(v, 'double')
+           for v in (x, y, lam, dy, dx, dlam)
+        ]
+            self.current_dataset.x = x
+            self.current_dataset.y = y
+            self.current_dataset.lam = lam
+            self.current_dataset.dy = dy
+            self.current_dataset.dx = dx
+            self.current_dataset.dlam = dlam
+            self.current_datainfo.isSesans = True
+        self.f_open.close()
+            self.current_datainfo._yunit = y_unit
+            self.current_datainfo._xunit = x_unit
+            self.current_datainfo.source.wavelength_unit = lam_unit
+            self.current_datainfo.source.wavelength = lam
+            self.current_datainfo.filename = os.basename(self.f_open.name)
+            self.current_dataset.xaxis(r"\rm{z}", x_unit)
+            # Adjust label to ln P/(lam^2 t), remove lam column refs
+            self.current_dataset.yaxis(r"\rm{ln(P)/(t \lambda^2)}", y_unit)
+            # Store loading process information
+            self.current_datainfo.meta_data['loader'] = self.type_name
+            self.current_datainfo.sample.name = params["Sample"]
+            self.current_datainfo.sample.ID = params["DataFileTitle"]
+            self.current_datainfo.sample.thickness = self._unit_conversion(
+                float(params["Thickness"]), "cm",
+                self._unit_fetch("Thickness"))[0]
+        self.current_dataset.x, self.current_dataset._xunit = self._unit_conversion(x, lam_unit, default_z_unit)
+        self.current_dataset.y = y
+        self.current_dataset._yunit = r'\AA^{-2} cm^{-1}'  # output y_unit added
+        self.current_dataset.dx, _ = self._unit_conversion(dx, lam_unit, default_z_unit)
+        self.current_dataset.dy = dy
+        self.current_dataset.lam, _ = self._unit_conversion(lam, lam_unit, default_z_unit)
+        self.current_dataset.dlam, _ = self._unit_conversion(dlam, lam_unit, default_z_unit)
+            self.current_datainfo.sample.zacceptance = (
+                float(params["Theta_zmax"]),
+                self._unit_fetch("Theta_zmax"))
+        self.current_dataset.xaxis(r"\rm{z}", self.current_dataset._xunit)
+        self.current_dataset.yaxis(r"\rm{ln(P)/(t \lambda^2)}", self.current_dataset._yunit)  # Adjust label to ln P/(lam^2 t), remove lam column refs
+            self.current_datainfo.sample.yacceptance = (
+                float(params["Theta_ymax"]),
+                self._unit_fetch("Theta_ymax"))
+        # Store loading process information
+        self.current_datainfo.meta_data['loader'] = self.type_name
+        try:
+            self.current_datainfo.sample.thickness = float(paramvals[6])
+        except ValueError as val_err:
+            loaded_correctly = False
+            error_message += "\nInvalid sample thickness '{}'".format(paramvals[6])
+            self.send_to_output()
+        self.current_datainfo.sample.name = paramvals[1]
+        self.current_datainfo.sample.ID = paramvals[0]
+        zaccept_unit_split = paramnames[7].split("[")
+        zaccept_unit = zaccept_unit_split[1].replace("]","")
+        if zaccept_unit.strip() == r'\AA^-1' or zaccept_unit.strip() == r'\A^-1':
+            zaccept_unit = "1/A"
+        self.current_datainfo.sample.zacceptance=(float(paramvals[7]),zaccept_unit)
+    @staticmethod
+    def _insist_header(headers, name):
+        if name not in headers:
+            raise FileContentsException(
+                "Missing {} column in spin echo data".format(name))
+        self.current_datainfo.vars = varheader
+    @staticmethod
+    def _unit_conversion(value, value_unit, default_unit):
+        """
+        Performs unit conversion on a measurement.
+        if len(self.current_dataset.x) < 1:
+            raise FileContentsException("No data points in file.")
+        self.send_to_output()
+        if not loaded_correctly:
+            raise DataReaderException(error_message)
+    def _unit_conversion(self, value, value_unit, default_unit):
+        if has_converter == True and value_unit != default_unit:
+            data_conv_q = Converter(value_unit)
+            value = data_conv_q(value, units=default_unit)
+        :param value: The magnitude of the measurement
+        :param value_unit: a string containing the final desired unit
+        :param default_unit: string with the units of the original measurement
+        :return: The magnitude of the measurement in the new units
+        """
+        # (float, string, string) -> float
+        if has_converter and value_unit != default_unit:
+            data_conv_q = Converter(default_unit)
+            value = data_conv_q(value, units=value_unit)
             new_unit = default_unit
         else:
             new_unit = value_unit
         return value, new_unit
+    def _unit_fetch(self, unit):
+        return self.params[unit+"_unit"]

src/sas/sascalc/dataloader/loader.py

-                      r463e7ffc
+                      r8dec7e7
 """
     File handler to support different file extensions.
     Uses reflectometry's registry utility.
+    Uses reflectometer registry utility.
     The default readers are found in the 'readers' sub-module
 …
 """
 #####################################################################
 #This software was developed by the University of Tennessee as part of the
 #Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
 #project funded by the US National Science Foundation.
 #See the license text in license.txt
 #copyright 2008, University of Tennessee
+# This software was developed by the University of Tennessee as part of the
+# Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
+# project funded by the US National Science Foundation.
+# See the license text in license.txt
+# copyright 2008, University of Tennessee
 ######################################################################
 …
 # Default readers are defined in the readers sub-module
 import readers
+from loader_exceptions import NoKnownLoaderException, FileContentsException,\
+    DefaultReaderException
 from readers import ascii_reader
 from readers import cansas_reader
+from readers import cansas_reader_HDF5
 logger = logging.getLogger(__name__)
 class Registry(ExtensionRegistry):
 …
     Readers and writers are supported.
     """
     def __init__(self):
         super(Registry, self).__init__()
         ## Writers
+        # Writers
         self.writers = {}
         ## List of wildcards
+        # List of wildcards
         self.wildcards = ['All (*.*)|*.*']
         ## Creation time, for testing
+        # Creation time, for testing
         self._created = time.time()
 …
             of a particular reader
+        Defaults to the ascii (multi-column) reader
+        if no reader was registered for the file's
+        extension.
+        Defaults to the ascii (multi-column), cansas XML, and cansas NeXuS
+        readers if no reader was registered for the file's extension.
         """
         try:
             return super(Registry, self).load(path, format=format)
+        except:
+            try:
+                # No reader was found. Default to the ascii reader.
+                ascii_loader = ascii_reader.Reader()
+                return ascii_loader.read(path)
+            except:
+                cansas_loader = cansas_reader.Reader()
+                return cansas_loader.read(path)
+        except NoKnownLoaderException as nkl_e:
+            pass  # try the ASCII reader
+        except FileContentsException as fc_exc:
+            raise RuntimeError(fc_exc.message)
+        except Exception:
+            pass
+        try:
+            ascii_loader = ascii_reader.Reader()
+            return ascii_loader.read(path)
+        except DefaultReaderException:
+            pass  # Loader specific error to try the cansas XML reader
+        except FileContentsException as e:
+            # TODO: handle error
+            raise RuntimeError(e)
+        try:
+            cansas_loader = cansas_reader.Reader()
+            return cansas_loader.read(path)
+        except DefaultReaderException:
+            pass  # Loader specific error to try the cansas NeXuS reader
+        except FileContentsException as e:
+            # TODO: Handle errors properly
+            raise RuntimeError(e)
+        except Exception as csr:
+            # TODO: Modify cansas reader to throw DefaultReaderException
+            pass
+        try:
+            cansas_nexus_loader = cansas_reader_HDF5.Reader()
+            return cansas_nexus_loader.read(path)
+        except DefaultReaderException as e:
+            logging.error("No default loader can load the data")
+            # No known reader available. Give up and throw an error
+            msg = "\n\tUnknown data format: %s.\n\tThe file is not a " % path
+            msg += "known format that can be loaded by SasView.\n"
+            raise NoKnownLoaderException, msg
+        except FileContentsException as e:
+            # TODO: Handle error(s) properly
+            raise RuntimeError(e)
     def find_plugins(self, dir):

src/sas/sascalc/dataloader/readers/init.py

-                      r959eb01
+                      r7a5d066
 # Backward compatibility with the previous implementation of the default readers
 from associations import register_readers
+# Method to associate extensions to default readers
+from associations import read_associations
-# Method to associate extensions to default readers
-from associations import read_associations
 # Method to return the location of the XML settings file

src/sas/sascalc/dataloader/readers/abs_reader.py

-                      r959eb01
+                      rad92c5a
 """
+    IGOR 1D data reader
 """
 #####################################################################
 #This software was developed by the University of Tennessee as part of the
 #Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
 #project funded by the US National Science Foundation.
 #See the license text in license.txt
 #copyright 2008, University of Tennessee
+# This software was developed by the University of Tennessee as part of the
+# Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
+# project funded by the US National Science Foundation.
+# See the license text in license.txt
+# copyright 2008, University of Tennessee
 ######################################################################
+import logging
 import numpy as np
+import os
+from sas.sascalc.dataloader.data_info import Data1D
+from sas.sascalc.dataloader.data_info import Detector
+has_converter = True
+try:
+    from sas.sascalc.data_util.nxsunit import Converter
+except:
+    has_converter = False
+class Reader:
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+from sas.sascalc.dataloader.data_info import DataInfo, plottable_1D, Data1D,\
+    Detector
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException,\
+    DefaultReaderException
+logger = logging.getLogger(__name__)
+class Reader(FileReader):
     """
     Class to load IGOR reduced .ABS files
     """
     ## File type
+    # File type
     type_name = "IGOR 1D"
     ## Wildcards
+    # Wildcards
     type = ["IGOR 1D files (*.abs)|*.abs"]
     ## List of allowed extensions
     ext = ['.abs', '.ABS']
+    # List of allowed extensions
+    ext = ['.abs']
     def read(self, path):
+    def get_file_contents(self):
         """
+        Load data file.
+        :param path: file path
+        :return: Data1D object, or None
+        Get the contents of the file
         :raise RuntimeError: when the file can't be opened
         :raise ValueError: when the length of the data vectors are inconsistent
         """
+        if os.path.isfile(path):
+            basename = os.path.basename(path)
+            root, extension = os.path.splitext(basename)
+            if extension.lower() in self.ext:
+                try:
+                    input_f = open(path,'r')
+        buff = self.f_open.read()
+        filepath = self.f_open.name
+        lines = buff.splitlines()
+        self.has_converter = True
+        try:
+            from sas.sascalc.data_util.nxsunit import Converter
+        except:
+            self.has_converter = False
+        self.output = []
+        self.current_datainfo = DataInfo()
+        self.current_datainfo.filename = filepath
+        self.reset_data_list(len(lines))
+        detector = Detector()
+        data_line = 0
+        self.reset_data_list(len(lines))
+        self.current_datainfo.detector.append(detector)
+        self.current_datainfo.filename = filepath
+        is_info = False
+        is_center = False
+        is_data_started = False
+        base_q_unit = '1/A'
+        base_i_unit = '1/cm'
+        data_conv_q = Converter(base_q_unit)
+        data_conv_i = Converter(base_i_unit)
+        for line in lines:
+            # Information line 1
+            if is_info:
+                is_info = False
+                line_toks = line.split()
+                # Wavelength in Angstrom
+                try:
+                    value = float(line_toks[1])
+                    if self.has_converter and \
+                            self.current_datainfo.source.wavelength_unit != 'A':
+                        conv = Converter('A')
+                        self.current_datainfo.source.wavelength = conv(value,
+                            units=self.current_datainfo.source.wavelength_unit)
+                    else:
+                        self.current_datainfo.source.wavelength = value
+                except KeyError:
+                    msg = "ABSReader cannot read wavelength from %s" % filepath
+                    self.current_datainfo.errors.append(msg)
+                # Detector distance in meters
+                try:
+                    value = float(line_toks[3])
+                    if self.has_converter and detector.distance_unit != 'm':
+                        conv = Converter('m')
+                        detector.distance = conv(value,
+                                        units=detector.distance_unit)
+                    else:
+                        detector.distance = value
                 except:
+                    raise  RuntimeError, "abs_reader: cannot open %s" % path
+                buff = input_f.read()
+                lines = buff.split('\n')
+                x  = np.zeros(0)
+                y  = np.zeros(0)
+                dy = np.zeros(0)
+                dx = np.zeros(0)
+                output = Data1D(x, y, dy=dy, dx=dx)
+                detector = Detector()
+                output.detector.append(detector)
+                output.filename = basename
+                is_info = False
+                    msg = "ABSReader cannot read SDD from %s" % filepath
+                    self.current_datainfo.errors.append(msg)
+                # Transmission
+                try:
+                    self.current_datainfo.sample.transmission = \
+                        float(line_toks[4])
+                except ValueError:
+                    # Transmission isn't always in the header
+                    pass
+                # Sample thickness in mm
+                try:
+                    value = float(line_toks[5])
+                    if self.has_converter and \
+                            self.current_datainfo.sample.thickness_unit != 'cm':
+                        conv = Converter('cm')
+                        self.current_datainfo.sample.thickness = conv(value,
+                            units=self.current_datainfo.sample.thickness_unit)
+                    else:
+                        self.current_datainfo.sample.thickness = value
+                except ValueError:
+                    # Thickness is not a mandatory entry
+                    pass
+            # MON CNT  LAMBDA  DET ANG  DET DIST  TRANS  THICK  AVE   STEP
+            if line.count("LAMBDA") > 0:
+                is_info = True
+            # Find center info line
+            if is_center:
                 is_center = False
+                is_data_started = False
+                data_conv_q = None
+                data_conv_i = None
+                if has_converter == True and output.x_unit != '1/A':
+                    data_conv_q = Converter('1/A')
+                    # Test it
+                    data_conv_q(1.0, output.x_unit)
+                if has_converter == True and output.y_unit != '1/cm':
+                    data_conv_i = Converter('1/cm')
+                    # Test it
+                    data_conv_i(1.0, output.y_unit)
+                for line in lines:
+                    # Information line 1
+                    if is_info == True:
+                        is_info = False
+                        line_toks = line.split()
+                        # Wavelength in Angstrom
+                        try:
+                            value = float(line_toks[1])
+                            if has_converter == True and \
+                                output.source.wavelength_unit != 'A':
+                                conv = Converter('A')
+                                output.source.wavelength = conv(value,
+                                        units=output.source.wavelength_unit)
+                            else:
+                                output.source.wavelength = value
+                        except:
+                            #goes to ASC reader
+                            msg = "abs_reader: cannot open %s" % path
+                            raise  RuntimeError, msg
+                        # Distance in meters
+                        try:
+                            value = float(line_toks[3])
+                            if has_converter == True and \
+                                detector.distance_unit != 'm':
+                                conv = Converter('m')
+                                detector.distance = conv(value,
+                                                units=detector.distance_unit)
+                            else:
+                                detector.distance = value
+                        except:
+                            #goes to ASC reader
+                            msg = "abs_reader: cannot open %s" % path
+                            raise  RuntimeError, msg
+                        # Transmission
+                        try:
+                            output.sample.transmission = float(line_toks[4])
+                        except:
+                            # Transmission is not a mandatory entry
+                            pass
+                        # Thickness in mm
+                        try:
+                            value = float(line_toks[5])
+                            if has_converter == True and \
+                                output.sample.thickness_unit != 'cm':
+                                conv = Converter('cm')
+                                output.sample.thickness = conv(value,
+                                            units=output.sample.thickness_unit)
+                            else:
+                                output.sample.thickness = value
+                        except:
+                            # Thickness is not a mandatory entry
+                            pass
+                    #MON CNT   LAMBDA   DET ANG   DET DIST   TRANS   THICK
+                    #  AVE   STEP
+                    if line.count("LAMBDA") > 0:
+                        is_info = True
+                    # Find center info line
+                    if is_center == True:
+                        is_center = False
+                        line_toks = line.split()
+                        # Center in bin number
+                        center_x = float(line_toks[0])
+                        center_y = float(line_toks[1])
+                        # Bin size
+                        if has_converter == True and \
+                            detector.pixel_size_unit != 'mm':
+                            conv = Converter('mm')
+                            detector.pixel_size.x = conv(5.0,
+                                                units=detector.pixel_size_unit)
+                            detector.pixel_size.y = conv(5.0,
+                                                units=detector.pixel_size_unit)
+                        else:
+                            detector.pixel_size.x = 5.0
+                            detector.pixel_size.y = 5.0
+                        # Store beam center in distance units
+                        # Det 640 x 640 mm
+                        if has_converter == True and \
+                            detector.beam_center_unit != 'mm':
+                            conv = Converter('mm')
+                            detector.beam_center.x = conv(center_x * 5.0,
+                                             units=detector.beam_center_unit)
+                            detector.beam_center.y = conv(center_y * 5.0,
+                                            units=detector.beam_center_unit)
+                        else:
+                            detector.beam_center.x = center_x * 5.0
+                            detector.beam_center.y = center_y * 5.0
+                        # Detector type
+                        try:
+                            detector.name = line_toks[7]
+                        except:
+                            # Detector name is not a mandatory entry
+                            pass
+                    #BCENT(X,Y)   A1(mm)   A2(mm)   A1A2DIST(m)   DL/L
+                    #  BSTOP(mm)   DET_TYP
+                    if line.count("BCENT") > 0:
+                        is_center = True
+                    # Parse the data
+                    if is_data_started == True:
+                        toks = line.split()
+                        try:
+                            _x  = float(toks[0])
+                            _y  = float(toks[1])
+                            _dy = float(toks[2])
+                            _dx = float(toks[3])
+                            if data_conv_q is not None:
+                                _x = data_conv_q(_x, units=output.x_unit)
+                                _dx = data_conv_i(_dx, units=output.x_unit)
+                            if data_conv_i is not None:
+                                _y = data_conv_i(_y, units=output.y_unit)
+                                _dy = data_conv_i(_dy, units=output.y_unit)
+                            x = np.append(x, _x)
+                            y = np.append(y, _y)
+                            dy = np.append(dy, _dy)
+                            dx = np.append(dx, _dx)
+                        except:
+                            # Could not read this data line. If we are here
+                            # it is because we are in the data section. Just
+                            # skip it.
+                            pass
+                    #The 6 columns are | Q (1/A) | I(Q) (1/cm) | std. dev.
+                    # I(Q) (1/cm) | sigmaQ | meanQ | ShadowFactor|
+                    if line.count("The 6 columns") > 0:
+                        is_data_started = True
+                # Sanity check
+                if not len(y) == len(dy):
+                    msg = "abs_reader: y and dy have different length"
+                    raise ValueError, msg
+                # If the data length is zero, consider this as
+                # though we were not able to read the file.
+                if len(x) == 0:
+                    raise ValueError, "ascii_reader: could not load file"
+                output.x = x[x != 0]
+                output.y = y[x != 0]
+                output.dy = dy[x != 0]
+                output.dx = dx[x != 0]
+                if data_conv_q is not None:
+                    output.xaxis("\\rm{Q}", output.x_unit)
+                line_toks = line.split()
+                # Center in bin number
+                center_x = float(line_toks[0])
+                center_y = float(line_toks[1])
+                # Bin size
+                if self.has_converter and detector.pixel_size_unit != 'mm':
+                    conv = Converter('mm')
+                    detector.pixel_size.x = conv(5.08,
+                                        units=detector.pixel_size_unit)
+                    detector.pixel_size.y = conv(5.08,
+                                        units=detector.pixel_size_unit)
                 else:
+                    output.xaxis("\\rm{Q}", 'A^{-1}')
+                if data_conv_i is not None:
+                    output.yaxis("\\rm{Intensity}", output.y_unit)
+                    detector.pixel_size.x = 5.08
+                    detector.pixel_size.y = 5.08
+                # Store beam center in distance units
+                # Det 640 x 640 mm
+                if self.has_converter and detector.beam_center_unit != 'mm':
+                    conv = Converter('mm')
+                    detector.beam_center.x = conv(center_x * 5.08,
+                                     units=detector.beam_center_unit)
+                    detector.beam_center.y = conv(center_y * 5.08,
+                                    units=detector.beam_center_unit)
                 else:
+                    output.yaxis("\\rm{Intensity}", "cm^{-1}")
+                # Store loading process information
+                output.meta_data['loader'] = self.type_name
+                return output
+                    detector.beam_center.x = center_x * 5.08
+                    detector.beam_center.y = center_y * 5.08
+                # Detector type
+                try:
+                    detector.name = line_toks[7]
+                except:
+                    # Detector name is not a mandatory entry
+                    pass
+            # BCENT(X,Y)  A1(mm)  A2(mm)  A1A2DIST(m)  DL/L  BSTOP(mm)  DET_TYP
+            if line.count("BCENT") > 0:
+                is_center = True
+            # Parse the data
+            if is_data_started:
+                toks = line.split()
+                try:
+                    _x = float(toks[0])
+                    _y = float(toks[1])
+                    _dy = float(toks[2])
+                    _dx = float(toks[3])
+                    if data_conv_q is not None:
+                        _x = data_conv_q(_x, units=base_q_unit)
+                        _dx = data_conv_q(_dx, units=base_q_unit)
+                    if data_conv_i is not None:
+                        _y = data_conv_i(_y, units=base_i_unit)
+                        _dy = data_conv_i(_dy, units=base_i_unit)
+                    self.current_dataset.x[data_line] = _x
+                    self.current_dataset.y[data_line] = _y
+                    self.current_dataset.dy[data_line] = _dy
+                    self.current_dataset.dx[data_line] = _dx
+                    data_line += 1
+                except ValueError:
+                    # Could not read this data line. If we are here
+                    # it is because we are in the data section. Just
+                    # skip it.
+                    pass
+            # The 6 columns are | Q (1/A) | I(Q) (1/cm) | std. dev.
+            # I(Q) (1/cm) | sigmaQ | meanQ | ShadowFactor|
+            if line.count("The 6 columns") > 0:
+                is_data_started = True
+        self.remove_empty_q_values(True, True)
+        # Sanity check
+        if not len(self.current_dataset.y) == len(self.current_dataset.dy):
+            self.set_all_to_none()
+            msg = "abs_reader: y and dy have different length"
+            raise ValueError(msg)
+        # If the data length is zero, consider this as
+        # though we were not able to read the file.
+        if len(self.current_dataset.x) == 0:
+            self.set_all_to_none()
+            raise ValueError("ascii_reader: could not load file")
+        if data_conv_q is not None:
+            self.current_dataset.xaxis("\\rm{Q}", base_q_unit)
         else:
+            raise RuntimeError, "%s is not a file" % path
+        return None
+            self.current_dataset.xaxis("\\rm{Q}", 'A^{-1}')
+        if data_conv_i is not None:
+            self.current_dataset.yaxis("\\rm{Intensity}", base_i_unit)
+        else:
+            self.current_dataset.yaxis("\\rm{Intensity}", "cm^{-1}")
+        # Store loading process information
+        self.current_datainfo.meta_data['loader'] = self.type_name
+        self.send_to_output()

src/sas/sascalc/dataloader/readers/anton_paar_saxs_reader.py

-                      ra235f715
+                      rfafe52a
 from sas.sascalc.dataloader.readers.xml_reader import XMLreader
 from sas.sascalc.dataloader.data_info import plottable_1D, Data1D, Sample, Source
+from sas.sascalc.dataloader.data_info import plottable_1D, Data1D, DataInfo, Sample, Source
 from sas.sascalc.dataloader.data_info import Process, Aperture, Collimation, TransmissionSpectrum, Detector
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException, DataReaderException
 class Reader(XMLreader):
     """
+    A class for reading in CanSAS v2.0 data files. The existing iteration opens Mantid generated HDF5 formatted files
+    with file extension .h5/.H5. Any number of data sets may be present within the file and any dimensionality of data
+    may be used. Currently 1D and 2D SAS data sets are supported, but future implementations will include 1D and 2D
+    SESANS data. This class assumes a single data set for each sasentry.
+    :Dependencies:
+        The CanSAS HDF5 reader requires h5py v2.5.0 or later.
+    A class for reading in Anton Paar .pdh files
     """
 …
     ## Raw file contents to be processed
     raw_data = None
-    ## Data set being modified
-    current_dataset = None
     ## For recursion and saving purposes, remember parent objects
     parent_list = None
 …
     ## Flag to bypass extension check
     allow_all = False
-    ## List of files to return
-    output = None
     def reset_state(self):
         self.current_dataset = Data1D(np.empty(0), np.empty(0),
                                             np.empty(0), np.empty(0))
+        self.current_dataset = plottable_1D(np.empty(0), np.empty(0), np.empty(0), np.empty(0))
+        self.current_datainfo = DataInfo()
         self.datasets = []
         self.raw_data = None
 …
         self.lower = 5
     def read(self, filename):
+    def get_file_contents(self):
         """
             This is the general read method that all SasView data_loaders must have.
 …
         ## Reinitialize the class when loading a new data file to reset all class variables
         self.reset_state()
+        ## Check that the file exists
+        if os.path.isfile(filename):
+            basename = os.path.basename(filename)
+            _, extension = os.path.splitext(basename)
+            # If the file type is not allowed, return empty list
+            if extension in self.ext or self.allow_all:
+                ## Load the data file
+                input_f = open(filename, 'r')
+                buff = input_f.read()
+                self.raw_data = buff.splitlines()
+                self.read_data()
+        return self.output
+        buff = self.f_open.read()
+        self.raw_data = buff.splitlines()
+        self.read_data()
     def read_data(self):
+        correctly_loaded = True
+        error_message = ""
         q_unit = "1/nm"
         i_unit = "1/um^2"
+        self.current_dataset.title = self.raw_data[0]
+        self.current_dataset.meta_data["Keywords"] = self.raw_data[1]
+        line3 = self.raw_data[2].split()
+        line4 = self.raw_data[3].split()
+        line5 = self.raw_data[4].split()
+        self.data_points = int(line3[0])
+        self.lower = 5
+        self.upper = self.lower + self.data_points
+        self.source.radiation = 'x-ray'
+        normal = float(line4[3])
+        self.current_dataset.source.radiation = "x-ray"
+        self.current_dataset.source.name = "Anton Paar SAXSess Instrument"
+        self.current_dataset.source.wavelength = float(line4[4])
+        xvals = []
+        yvals = []
+        dyvals = []
+        for i in range(self.lower, self.upper):
+            index = i - self.lower
+            data = self.raw_data[i].split()
+            xvals.insert(index, normal * float(data[0]))
+            yvals.insert(index, normal * float(data[1]))
+            dyvals.insert(index, normal * float(data[2]))
+        try:
+            self.current_datainfo.title = self.raw_data[0]
+            self.current_datainfo.meta_data["Keywords"] = self.raw_data[1]
+            line3 = self.raw_data[2].split()
+            line4 = self.raw_data[3].split()
+            line5 = self.raw_data[4].split()
+            self.data_points = int(line3[0])
+            self.lower = 5
+            self.upper = self.lower + self.data_points
+            self.source.radiation = 'x-ray'
+            normal = float(line4[3])
+            self.current_datainfo.source.radiation = "x-ray"
+            self.current_datainfo.source.name = "Anton Paar SAXSess Instrument"
+            self.current_datainfo.source.wavelength = float(line4[4])
+            xvals = []
+            yvals = []
+            dyvals = []
+            for i in range(self.lower, self.upper):
+                index = i - self.lower
+                data = self.raw_data[i].split()
+                xvals.insert(index, normal * float(data[0]))
+                yvals.insert(index, normal * float(data[1]))
+                dyvals.insert(index, normal * float(data[2]))
+        except Exception as e:
+            error_message = "Couldn't load {}.\n".format(self.f_open.name)
+            error_message += e.message
+            raise FileContentsException(error_message)
         self.current_dataset.x = np.append(self.current_dataset.x, xvals)
         self.current_dataset.y = np.append(self.current_dataset.y, yvals)
         self.current_dataset.dy = np.append(self.current_dataset.dy, dyvals)
         if self.data_points != self.current_dataset.x.size:
+            self.errors.add("Not all data was loaded properly.")
+        if self.current_dataset.dx.size != self.current_dataset.x.size:
+            dxvals = np.zeros(self.current_dataset.x.size)
+            self.current_dataset.dx = dxvals
+            error_message += "Not all data points could be loaded.\n"
+            correctly_loaded = False
         if self.current_dataset.x.size != self.current_dataset.y.size:
+            self.errors.add("The x and y data sets are not the same size.")
+            error_message += "The x and y data sets are not the same size.\n"
+            correctly_loaded = False
         if self.current_dataset.y.size != self.current_dataset.dy.size:
+            self.errors.add("The y and dy datasets are not the same size.")
+        self.current_dataset.errors = self.errors
+            error_message += "The y and dy datasets are not the same size.\n"
+            correctly_loaded = False
         self.current_dataset.xaxis("Q", q_unit)
         self.current_dataset.yaxis("Intensity", i_unit)
         xml_intermediate = self.raw_data[self.upper:]
         xml = ''.join(xml_intermediate)
+        self.set_xml_string(xml)
+        dom = self.xmlroot.xpath('/fileinfo')
+        self._parse_child(dom)
+        self.output.append(self.current_dataset)
+        try:
+            self.set_xml_string(xml)
+            dom = self.xmlroot.xpath('/fileinfo')
+            self._parse_child(dom)
+        except Exception as e:
+            # Data loaded but XML metadata has an error
+            error_message += "Data points have been loaded but there was an "
+            error_message += "error reading XML metadata: " + e.message
+            correctly_loaded = False
+        self.send_to_output()
+        if not correctly_loaded:
+            raise DataReaderException(error_message)
     def _parse_child(self, dom, parent=''):
 …
                 self._parse_child(node, key)
                 if key == "SampleDetector":
                     self.current_dataset.detector.append(self.detector)
+                    self.current_datainfo.detector.append(self.detector)
                     self.detector = Detector()
             else:
                 if key == "value":
                     if parent == "Wavelength":
                         self.current_dataset.source.wavelength = value
+                        self.current_datainfo.source.wavelength = value
                     elif parent == "SampleDetector":
                         self.detector.distance = value
                     elif parent == "Temperature":
                         self.current_dataset.sample.temperature = value
+                        self.current_datainfo.sample.temperature = value
                     elif parent == "CounterSlitLength":
                         self.detector.slit_length = value
 …
                     value = value.replace("_", "")
                     if parent == "Wavelength":
                         self.current_dataset.source.wavelength_unit = value
+                        self.current_datainfo.source.wavelength_unit = value
                     elif parent == "SampleDetector":
                         self.detector.distance_unit = value
 …
                         self.current_dataset.yaxis(self.current_dataset._yaxis, value)
                     elif parent == "Temperature":
                         self.current_dataset.sample.temperature_unit = value
+                        self.current_datainfo.sample.temperature_unit = value
                     elif parent == "CounterSlitLength":
                         self.detector.slit_length_unit = value

src/sas/sascalc/dataloader/readers/ascii_reader.py

-                      r235f514
+                      r080d88e
 """
     ASCII reader
+    Generic multi-column ASCII data reader
 """
 ############################################################################
 #This software was developed by the University of Tennessee as part of the
 #Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
 #project funded by the US National Science Foundation.
 #If you use DANSE applications to do scientific research that leads to
 #publication, we ask that you acknowledge the use of the software with the
 #following sentence:
 #This work benefited from DANSE software developed under NSF award DMR-0520547.
 #copyright 2008, University of Tennessee
+# This software was developed by the University of Tennessee as part of the
+# Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
+# project funded by the US National Science Foundation.
+# If you use DANSE applications to do scientific research that leads to
+# publication, we ask that you acknowledge the use of the software with the
+# following sentence:
+# This work benefited from DANSE software developed under NSF award DMR-0520547.
+# copyright 2008, University of Tennessee
 #############################################################################
+import logging
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+from sas.sascalc.dataloader.data_info import DataInfo, plottable_1D
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException,\
+    DefaultReaderException
+import numpy as np
+import os
+from sas.sascalc.dataloader.data_info import Data1D
+# Check whether we have a converter available
+has_converter = True
+try:
+    from sas.sascalc.data_util.nxsunit import Converter
+except:
+    has_converter = False
+_ZERO = 1e-16
+logger = logging.getLogger(__name__)
 class Reader:
+class Reader(FileReader):
     """
     Class to load ascii files (2, 3 or 4 columns).
     """
     ## File type
+    # File type
     type_name = "ASCII"
+    ## Wildcards
+    # Wildcards
     type = ["ASCII files (*.txt)|*.txt",
             "ASCII files (*.dat)|*.dat",
             "ASCII files (*.abs)|*.abs",
             "CSV files (*.csv)|*.csv"]
+    ## List of allowed extensions
+    ext = ['.txt', '.TXT', '.dat', '.DAT', '.abs', '.ABS', 'csv', 'CSV']
+    # List of allowed extensions
+    ext = ['.txt', '.dat', '.abs', '.csv']
+    # Flag to bypass extension check
+    allow_all = True
+    # data unless that is the only data
+    min_data_pts = 5
+    ## Flag to bypass extension check
+    allow_all = True
+    def get_file_contents(self):
+        """
+        Get the contents of the file
+        """
+    def read(self, path):
+        """
+        Load data file
+        buff = self.f_open.read()
+        filepath = self.f_open.name
+        lines = buff.splitlines()
+        self.output = []
+        self.current_datainfo = DataInfo()
+        self.current_datainfo.filename = filepath
+        self.reset_data_list(len(lines))
+        :param path: file path
+        :return: Data1D object, or None
+        # The first good line of data will define whether
+        # we have 2-column or 3-column ascii
+        has_error_dx = None
+        has_error_dy = None
+        :raise RuntimeError: when the file can't be opened
+        :raise ValueError: when the length of the data vectors are inconsistent
+        """
+        if os.path.isfile(path):
+            basename = os.path.basename(path)
+            _, extension = os.path.splitext(basename)
+            if self.allow_all or extension.lower() in self.ext:
+                try:
+                    # Read in binary mode since GRASP frequently has no-ascii
+                    # characters that breaks the open operation
+                    input_f = open(path,'rb')
+                except:
+                    raise  RuntimeError, "ascii_reader: cannot open %s" % path
+                buff = input_f.read()
+                lines = buff.splitlines()
+        # Initialize counters for data lines and header lines.
+        is_data = False
+        # More than "5" lines of data is considered as actual
+        # To count # of current data candidate lines
+        candidate_lines = 0
+        # To count total # of previous data candidate lines
+        candidate_lines_previous = 0
+        # Current line number
+        line_no = 0
+        # minimum required number of columns of data
+        lentoks = 2
+        for line in lines:
+            toks = self.splitline(line.strip())
+            # To remember the number of columns in the current line of data
+            new_lentoks = len(toks)
+            try:
+                if new_lentoks == 0:
+                    # If the line is blank, skip and continue on
+                    # In case of breaks within data sets.
+                    continue
+                elif new_lentoks != lentoks and is_data:
+                    # If a footer is found, break the loop and save the data
+                    break
+                elif new_lentoks != lentoks and not is_data:
+                    # If header lines are numerical
+                    candidate_lines = 0
+                    self.reset_data_list(len(lines) - line_no)
+                # Arrays for data storage
+                tx = np.zeros(0)
+                ty = np.zeros(0)
+                tdy = np.zeros(0)
+                tdx = np.zeros(0)
+                self.current_dataset.x[candidate_lines] = float(toks[0])
+                # The first good line of data will define whether
+                # we have 2-column or 3-column ascii
+                if new_lentoks > 1:
+                    self.current_dataset.y[candidate_lines] = float(toks[1])
+                # If a 3rd row is present, consider it dy
+                if new_lentoks > 2:
+                    self.current_dataset.dy[candidate_lines] = \
+                        float(toks[2])
+                    has_error_dy = True
+                # If a 4th row is present, consider it dx
+                if new_lentoks > 3:
+                    self.current_dataset.dx[candidate_lines] = \
+                        float(toks[3])
+                    has_error_dx = True
+                candidate_lines += 1
+                # If 5 or more lines, this is considering the set data
+                if candidate_lines >= self.min_data_pts:
+                    is_data = True
+                # To remember the # of columns on the current line
+                # for the next line of data
+                lentoks = new_lentoks
+                line_no += 1
+            except ValueError:
+                # ValueError is raised when non numeric strings conv. to float
+                # It is data and meet non - number, then stop reading
+                if is_data:
+                    break
+                # Delete the previously stored lines of data candidates if
+                # the list is not data
+                self.reset_data_list(len(lines) - line_no)
+                lentoks = 2
                 has_error_dx = None
                 has_error_dy = None
+                # Reset # of lines of data candidates
+                candidate_lines = 0
+                #Initialize counters for data lines and header lines.
+                is_data = False
+                # More than "5" lines of data is considered as actual
+                # data unless that is the only data
+                min_data_pts = 5
+                # To count # of current data candidate lines
+                candidate_lines = 0
+                # To count total # of previous data candidate lines
+                candidate_lines_previous = 0
+                #minimum required number of columns of data
+                lentoks = 2
+                for line in lines:
+                    toks = self.splitline(line)
+                    # To remember the # of columns in the current line of data
+                    new_lentoks = len(toks)
+                    try:
+                        if new_lentoks == 1 and not is_data:
+                            ## If only one item in list, no longer data
+                            raise ValueError
+                        elif new_lentoks == 0:
+                            ## If the line is blank, skip and continue on
+                            ## In case of breaks within data sets.
+                            continue
+                        elif new_lentoks != lentoks and is_data:
+                            ## If a footer is found, break the loop and save the data
+                            break
+                        elif new_lentoks != lentoks and not is_data:
+                            ## If header lines are numerical
+                            candidate_lines = 0
+                            candidate_lines_previous = 0
+        if not is_data:
+            self.set_all_to_none()
+            if self.extension in self.ext:
+                msg = "ASCII Reader error: Fewer than five Q data points found "
+                msg += "in {}.".format(filepath)
+                raise FileContentsException(msg)
+            else:
+                msg = "ASCII Reader could not load the file {}".format(filepath)
+                raise DefaultReaderException(msg)
+        # Sanity check
+        if has_error_dy and not len(self.current_dataset.y) == \
+                len(self.current_dataset.dy):
+            msg = "ASCII Reader error: Number of I and dI data points are"
+            msg += " different in {}.".format(filepath)
+            # TODO: Add error to self.current_datainfo.errors instead?
+            self.set_all_to_none()
+            raise FileContentsException(msg)
+        if has_error_dx and not len(self.current_dataset.x) == \
+                len(self.current_dataset.dx):
+            msg = "ASCII Reader error: Number of Q and dQ data points are"
+            msg += " different in {}.".format(filepath)
+            # TODO: Add error to self.current_datainfo.errors instead?
+            self.set_all_to_none()
+            raise FileContentsException(msg)
+                        #Make sure that all columns are numbers.
+                        for colnum in range(len(toks)):
+                            # Any non-floating point values throw ValueError
+                            float(toks[colnum])
+        self.remove_empty_q_values(has_error_dx, has_error_dy)
+        self.current_dataset.xaxis("\\rm{Q}", 'A^{-1}')
+        self.current_dataset.yaxis("\\rm{Intensity}", "cm^{-1}")
+                        candidate_lines += 1
+                        _x = float(toks[0])
+                        _y = float(toks[1])
+                        _dx = None
+                        _dy = None
+                        #If 5 or more lines, this is considering the set data
+                        if candidate_lines >= min_data_pts:
+                            is_data = True
+                        # If a 3rd row is present, consider it dy
+                        if new_lentoks > 2:
+                            _dy = float(toks[2])
+                        has_error_dy = False if _dy is None else True
+                        # If a 4th row is present, consider it dx
+                        if new_lentoks > 3:
+                            _dx = float(toks[3])
+                        has_error_dx = False if _dx is None else True
+                        # Delete the previously stored lines of data candidates if
+                        # the list is not data
+                        if candidate_lines == 1 and -1 < candidate_lines_previous < min_data_pts and \
+                            is_data == False:
+                            try:
+                                tx = np.zeros(0)
+                                ty = np.zeros(0)
+                                tdy = np.zeros(0)
+                                tdx = np.zeros(0)
+                            except:
+                                pass
+                        if has_error_dy == True:
+                            tdy = np.append(tdy, _dy)
+                        if has_error_dx == True:
+                            tdx = np.append(tdx, _dx)
+                        tx = np.append(tx, _x)
+                        ty = np.append(ty, _y)
+                        #To remember the # of columns on the current line
+                        # for the next line of data
+                        lentoks = new_lentoks
+                        candidate_lines_previous = candidate_lines
+                    except ValueError:
+                        # It is data and meet non - number, then stop reading
+                        if is_data == True:
+                            break
+                        lentoks = 2
+                        has_error_dx = None
+                        has_error_dy = None
+                        #Reset # of lines of data candidates
+                        candidate_lines = 0
+                    except:
+                        pass
+                input_f.close()
+                if not is_data:
+                    msg = "ascii_reader: x has no data"
+                    raise RuntimeError, msg
+                # Sanity check
+                if has_error_dy == True and not len(ty) == len(tdy):
+                    msg = "ascii_reader: y and dy have different length"
+                    raise RuntimeError, msg
+                if has_error_dx == True and not len(tx) == len(tdx):
+                    msg = "ascii_reader: y and dy have different length"
+                    raise RuntimeError, msg
+                # If the data length is zero, consider this as
+                # though we were not able to read the file.
+                if len(tx) == 0:
+                    raise RuntimeError, "ascii_reader: could not load file"
+                #Let's re-order the data to make cal.
+                # curve look better some cases
+                ind = np.lexsort((ty, tx))
+                x = np.zeros(len(tx))
+                y = np.zeros(len(ty))
+                dy = np.zeros(len(tdy))
+                dx = np.zeros(len(tdx))
+                output = Data1D(x, y, dy=dy, dx=dx)
+                self.filename = output.filename = basename
+                for i in ind:
+                    x[i] = tx[ind[i]]
+                    y[i] = ty[ind[i]]
+                    if has_error_dy == True:
+                        dy[i] = tdy[ind[i]]
+                    if has_error_dx == True:
+                        dx[i] = tdx[ind[i]]
+                # Zeros in dx, dy
+                if has_error_dx:
+                    dx[dx == 0] = _ZERO
+                if has_error_dy:
+                    dy[dy == 0] = _ZERO
+                #Data
+                output.x = x[x != 0]
+                output.y = y[x != 0]
+                output.dy = dy[x != 0] if has_error_dy == True\
+                    else np.zeros(len(output.y))
+                output.dx = dx[x != 0] if has_error_dx == True\
+                    else np.zeros(len(output.x))
+                output.xaxis("\\rm{Q}", 'A^{-1}')
+                output.yaxis("\\rm{Intensity}", "cm^{-1}")
+                # Store loading process information
+                output.meta_data['loader'] = self.type_name
+                if len(output.x) < 1:
+                    raise RuntimeError, "%s is empty" % path
+                return output
+        else:
+            raise RuntimeError, "%s is not a file" % path
+        return None
+    def splitline(self, line):
+        """
+        Splits a line into pieces based on common delimeters
+        :param line: A single line of text
+        :return: list of values
+        """
+        # Initial try for CSV (split on ,)
+        toks = line.split(',')
+        # Now try SCSV (split on ;)
+        if len(toks) < 2:
+            toks = line.split(';')
+        # Now go for whitespace
+        if len(toks) < 2:
+            toks = line.split()
+        return toks
+        # Store loading process information
+        self.current_datainfo.meta_data['loader'] = self.type_name
+        self.send_to_output()

src/sas/sascalc/dataloader/readers/associations.py

-                      ra1b8fee
+                      r248ff73
 #copyright 2009, University of Tennessee
 #############################################################################
-from __future__ import print_function
-import os
 import sys
 import logging
-import json
 logger = logging.getLogger(__name__)
+FILE_NAME = 'defaults.json'
+FILE_ASSOCIATIONS = {
+    ".xml": "cansas_reader",
+    ".ses": "sesans_reader",
+    ".h5": "cansas_reader_HDF5",
+    ".txt": "ascii_reader",
+    ".dat": "red2d_reader",
+    ".abs": "abs_reader",
+    ".d1d": "hfir1d_reader",
+    ".sans": "danse_reader",
+    ".nxs": "nexus_reader",
+    ".pdh": "anton_paar_saxs_reader"
+}
+def read_associations(loader, settings=FILE_NAME):
+def read_associations(loader, settings=FILE_ASSOCIATIONS):
     """
     Read the specified settings file to associate
     default readers to file extension.
     :param loader: Loader object
     :param settings: path to the json settings file [string]
     """
+    reader_dir = os.path.dirname(__file__)
+    path = os.path.join(reader_dir, settings)
+    # If we can't find the file in the installation
+    # directory, look into the execution directory.
+    if not os.path.isfile(path):
+        path = os.path.join(os.getcwd(), settings)
+    if not os.path.isfile(path):
+        path = os.path.join(sys.path[0], settings)
+    if not os.path.isfile(path):
+        path = settings
+    if not os.path.isfile(path):
+        path = "./%s" % settings
+    if os.path.isfile(path):
+        with open(path) as fh:
+            json_tree = json.load(fh)
+        # Read in the file extension associations
+        entry_list = json_tree['SasLoader']['FileType']
+        # For each FileType entry, get the associated reader and extension
+        for entry in entry_list:
+            reader = entry['-reader']
+            ext = entry['-extension']
+            if reader is not None and ext is not None:
+                # Associate the extension with a particular reader
+                # TODO: Modify the Register code to be case-insensitive
+                # and remove the extra line below.
+                try:
+                    exec "import %s" % reader
+                    exec "loader.associate_file_type('%s', %s)" % (ext.lower(),
+                                                                    reader)
+                    exec "loader.associate_file_type('%s', %s)" % (ext.upper(),
+                                                                    reader)
+                except:
+                    msg = "read_associations: skipping association"
+                    msg += " for %s\n  %s" % (ext.lower(), sys.exc_value)
+                    logger.error(msg)
+    else:
+        print("Could not find reader association settings\n  %s [%s]" % (__file__, os.getcwd()))
+def register_readers(registry_function):
+    """
+    Function called by the registry/loader object to register
+    all default readers using a call back function.
+    :WARNING: this method is now obsolete
+    :param registry_function: function to be called to register each reader
+    """
+    logger.info("register_readers is now obsolete: use read_associations()")
+    import abs_reader
+    import ascii_reader
+    import cansas_reader
+    import danse_reader
+    import hfir1d_reader
+    import IgorReader
+    import red2d_reader
+    #import tiff_reader
+    import nexus_reader
+    import sesans_reader
+    import cansas_reader_HDF5
+    import anton_paar_saxs_reader
+    registry_function(sesans_reader)
+    registry_function(abs_reader)
+    registry_function(ascii_reader)
+    registry_function(cansas_reader)
+    registry_function(danse_reader)
+    registry_function(hfir1d_reader)
+    registry_function(IgorReader)
+    registry_function(red2d_reader)
+    #registry_function(tiff_reader)
+    registry_function(nexus_reader)
+    registry_function(cansas_reader_HDF5)
+    registry_function(anton_paar_saxs_reader)
+    return True
+    # For each FileType entry, get the associated reader and extension
+    for ext, reader in settings.iteritems():
+        if reader is not None and ext is not None:
+            # Associate the extension with a particular reader
+            # TODO: Modify the Register code to be case-insensitive
+            # FIXME: Remove exec statements
+            # and remove the extra line below.
+            try:
+                exec "import %s" % reader
+                exec "loader.associate_file_type('%s', %s)" % (ext.lower(),
+                                                                reader)
+                exec "loader.associate_file_type('%s', %s)" % (ext.upper(),
+                                                                reader)
+            except:
+                msg = "read_associations: skipping association"
+                msg += " for %s\n  %s" % (ext.lower(), sys.exc_value)
+                logger.error(msg)

src/sas/sascalc/dataloader/readers/cansas_reader.py

-                      r7432acb
+                      r248ff73
-"""
-    CanSAS data reader - new recursive cansas_version.
-"""
-############################################################################
-#This software was developed by the University of Tennessee as part of the
-#Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
-#project funded by the US National Science Foundation.
-#If you use DANSE applications to do scientific research that leads to
-#publication, we ask that you acknowledge the use of the software with the
-#following sentence:
-#This work benefited from DANSE software developed under NSF award DMR-0520547.
-#copyright 2008,2009 University of Tennessee
-#############################################################################
 import logging
 import numpy as np
 …
 from sas.sascalc.dataloader.readers.xml_reader import XMLreader
 from sas.sascalc.dataloader.readers.cansas_constants import CansasConstants, CurrentLevel
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException, \
+    DefaultReaderException, DataReaderException
 # The following 2 imports *ARE* used. Do not remove either.
 import xml.dom.minidom
 from xml.dom.minidom import parseString
+from lxml import etree
 logger = logging.getLogger(__name__)
 …
 class Reader(XMLreader):
-    """
-    Class to load cansas 1D XML files
-    :Dependencies:
-        The CanSAS reader requires PyXML 0.8.4 or later.
-    """
-    # CanSAS version - defaults to version 1.0
     cansas_version = "1.0"
     base_ns = "{cansas1d/1.0}"
 …
     ns_list = None
     # Temporary storage location for loading multiple data sets in a single file
-    current_datainfo = None
-    current_dataset = None
     current_data1d = None
     data = None
-    # List of data1D objects to be sent back to SasView
-    output = None
     # Wildcards
     type = ["XML files (*.xml)|*.xml", "SasView Save Files (*.svs)|*.svs"]
 …
     def read(self, xml_file, schema_path="", invalid=True):
+        """
+        Validate and read in an xml_file file in the canSAS format.
+        :param xml_file: A canSAS file path in proper XML format
+        :param schema_path: A file path to an XML schema to validate the xml_file against
+        """
+        # For every file loaded, reset everything to a base state
+        if schema_path != "" or invalid != True:
+            # read has been called from self.get_file_contents because xml file doens't conform to schema
+            _, self.extension = os.path.splitext(os.path.basename(xml_file))
+            return self.get_file_contents(xml_file=xml_file, schema_path=schema_path, invalid=invalid)
+        # Otherwise, read has been called by the data loader - file_reader_base_class handles this
+        return super(XMLreader, self).read(xml_file)
+    def get_file_contents(self, xml_file=None, schema_path="", invalid=True):
+        # Reset everything since we're loading a new file
         self.reset_state()
         self.invalid = invalid
+        # Check that the file exists
+        if os.path.isfile(xml_file):
+            basename, extension = os.path.splitext(os.path.basename(xml_file))
+            # If the file type is not allowed, return nothing
+            if extension in self.ext or self.allow_all:
+                # Get the file location of
+                self.load_file_and_schema(xml_file, schema_path)
+                self.add_data_set()
+                # Try to load the file, but raise an error if unable to.
+                # Check the file matches the XML schema
+        if xml_file is None:
+            xml_file = self.f_open.name
+        # We don't sure f_open since lxml handles opnening/closing files
+        if not self.f_open.closed:
+            self.f_open.close()
+        basename, _ = os.path.splitext(os.path.basename(xml_file))
+        try:
+            # Raises FileContentsException
+            self.load_file_and_schema(xml_file, schema_path)
+            self.current_datainfo = DataInfo()
+            # Raises FileContentsException if file doesn't meet CanSAS schema
+            self.is_cansas(self.extension)
+            self.invalid = False # If we reach this point then file must be valid CanSAS
+            # Parse each SASentry
+            entry_list = self.xmlroot.xpath('/ns:SASroot/ns:SASentry', namespaces={
+                'ns': self.cansas_defaults.get("ns")
+            })
+            # Look for a SASentry
+            self.names.append("SASentry")
+            self.set_processing_instructions()
+            for entry in entry_list:
+                self.current_datainfo.filename = basename + self.extension
+                self.current_datainfo.meta_data["loader"] = "CanSAS XML 1D"
+                self.current_datainfo.meta_data[PREPROCESS] = self.processing_instructions
+                self._parse_entry(entry)
+                has_error_dx = self.current_dataset.dx is not None
+                has_error_dy = self.current_dataset.dy is not None
+                self.remove_empty_q_values(has_error_dx=has_error_dx,
+                    has_error_dy=has_error_dy)
+                self.send_to_output() # Combine datasets with DataInfo
+                self.current_datainfo = DataInfo() # Reset DataInfo
+        except FileContentsException as fc_exc:
+            # File doesn't meet schema - try loading with a less strict schema
+            base_name = xml_reader.__file__
+            base_name = base_name.replace("\\", "/")
+            base = base_name.split("/sas/")[0]
+            if self.cansas_version == "1.1":
+                invalid_schema = INVALID_SCHEMA_PATH_1_1.format(base, self.cansas_defaults.get("schema"))
+            else:
+                invalid_schema = INVALID_SCHEMA_PATH_1_0.format(base, self.cansas_defaults.get("schema"))
+            self.set_schema(invalid_schema)
+            if self.invalid:
                 try:
+                    self.is_cansas(extension)
+                    self.invalid = False
+                    # Get each SASentry from XML file and add it to a list.
+                    entry_list = self.xmlroot.xpath(
+                            '/ns:SASroot/ns:SASentry',
+                            namespaces={'ns': self.cansas_defaults.get("ns")})
+                    self.names.append("SASentry")
+                    # Get all preprocessing events and encoding
+                    self.set_processing_instructions()
+                    # Parse each <SASentry> item
+                    for entry in entry_list:
+                        # Create a new DataInfo object for every <SASentry>
+                        # Set the file name and then parse the entry.
+                        self.current_datainfo.filename = basename + extension
+                        self.current_datainfo.meta_data["loader"] = "CanSAS XML 1D"
+                        self.current_datainfo.meta_data[PREPROCESS] = \
+                            self.processing_instructions
+                        # Parse the XML SASentry
+                        self._parse_entry(entry)
+                        # Combine datasets with datainfo
+                        self.add_data_set()
+                except RuntimeError:
+                    # If the file does not match the schema, raise this error
+                    # Load data with less strict schema
+                    self.read(xml_file, invalid_schema, False)
+                    # File can still be read but doesn't match schema, so raise exception
+                    self.load_file_and_schema(xml_file) # Reload strict schema so we can find where error are in file
                     invalid_xml = self.find_invalid_xml()
+                    invalid_xml = INVALID_XML.format(basename + extension) + invalid_xml
+                    self.errors.add(invalid_xml)
+                    # Try again with an invalid CanSAS schema, that requires only a data set in each
+                    base_name = xml_reader.__file__
+                    base_name = base_name.replace("\\", "/")
+                    base = base_name.split("/sas/")[0]
+                    if self.cansas_version == "1.1":
+                        invalid_schema = INVALID_SCHEMA_PATH_1_1.format(base, self.cansas_defaults.get("schema"))
+                    else:
+                        invalid_schema = INVALID_SCHEMA_PATH_1_0.format(base, self.cansas_defaults.get("schema"))
+                    self.set_schema(invalid_schema)
+                    try:
+                        if self.invalid:
+                            if self.is_cansas():
+                                self.output = self.read(xml_file, invalid_schema, False)
+                            else:
+                                raise RuntimeError
+                        else:
+                            raise RuntimeError
+                    except RuntimeError:
+                        x = np.zeros(1)
+                        y = np.zeros(1)
+                        self.current_data1d = Data1D(x,y)
+                        self.current_data1d.errors = self.errors
+                        return [self.current_data1d]
+        else:
+            self.output.append("Not a valid file path.")
+        # Return a list of parsed entries that dataloader can manage
+        return self.output
+                    invalid_xml = INVALID_XML.format(basename + self.extension) + invalid_xml
+                    raise DataReaderException(invalid_xml) # Handled by base class
+                except FileContentsException as fc_exc:
+                    if not self.extension in self.ext: # If the file has no associated loader
+                        raise DefaultReaderException(msg)
+                    msg = "CanSAS Reader could not load the file {}".format(xml_file)
+                    if fc_exc.message is not None: # Propagate error messages from earlier
+                        msg = fc_exc.message
+                    raise FileContentsException(msg)
+                    pass
+            else:
+                raise fc_exc
+        except Exception as e: # Convert all other exceptions to FileContentsExceptions
+            raise FileContentsException(e.message)
+    def load_file_and_schema(self, xml_file, schema_path=""):
+        base_name = xml_reader.__file__
+        base_name = base_name.replace("\\", "/")
+        base = base_name.split("/sas/")[0]
+        # Try and parse the XML file
+        try:
+            self.set_xml_file(xml_file)
+        except etree.XMLSyntaxError: # File isn't valid XML so can't be loaded
+            msg = "Cansas cannot load {}.\n Invalid XML syntax".format(xml_file)
+            raise FileContentsException(msg)
+        self.cansas_version = self.xmlroot.get("version", "1.0")
+        self.cansas_defaults = CANSAS_NS.get(self.cansas_version, "1.0")
+        if schema_path == "":
+            schema_path = "{}/sas/sascalc/dataloader/readers/schema/{}".format(
+                base, self.cansas_defaults.get("schema").replace("\\", "/")
+            )
+        self.set_schema(schema_path)
+    def is_cansas(self, ext="xml"):
+        """
+        Checks to see if the XML file is a CanSAS file
+        :param ext: The file extension of the data file
+        :raises FileContentsException: Raised if XML file isn't valid CanSAS
+        """
+        if self.validate_xml(): # Check file is valid XML
+            name = "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation"
+            value = self.xmlroot.get(name)
+            # Check schema CanSAS version matches file CanSAS version
+            if CANSAS_NS.get(self.cansas_version).get("ns") == value.rsplit(" ")[0]:
+                return True
+        if ext == "svs":
+            return True # Why is this required?
+        # If we get to this point then file isn't valid CanSAS
+        raise FileContentsException("The file is not valid CanSAS")
     def _parse_entry(self, dom, recurse=False):
-        """
-        Parse a SASEntry - new recursive method for parsing the dom of
-            the CanSAS data format. This will allow multiple data files
-            and extra nodes to be read in simultaneously.
-        :param dom: dom object with a namespace base of names
-        """
         if not self._is_call_local() and not recurse:
             self.reset_state()
+            self.add_data_set()
+            self.data = []
+            self.current_datainfo = DataInfo()
             self.names.append("SASentry")
             self.parent_class = "SASentry"
+        self._check_for_empty_data()
+        self.base_ns = "{0}{1}{2}".format("{", \
+                            CANSAS_NS.get(self.cansas_version).get("ns"), "}")
+        # Go through each child in the parent element
+        # Create an empty dataset if no data has been passed to the reader
+        if self.current_dataset is None:
+            self.current_dataset = plottable_1D(np.empty(0), np.empty(0),
+                np.empty(0), np.empty(0))
+        self.base_ns = "{" + CANSAS_NS.get(self.cansas_version).get("ns") + "}"
+        # Loop through each child in the parent element
         for node in dom:
             attr = node.attrib
 …
             if tagname == "fitting_plug_in" or tagname == "pr_inversion" or tagname == "invariant":
                 continue
             # Get where to store content
             self.names.append(tagname_original)
 …
                         else:
                             self.current_dataset.shape = ()
                 # Recursion step to access data within the group
                 self._parse_entry(node, True)
+                # Recurse to access data within the group
+                self._parse_entry(node, recurse=True)
                 if tagname == "SASsample":
                     self.current_datainfo.sample.name = name
 …
                     self.aperture.name = name
                     self.aperture.type = type
                 self.add_intermediate()
+                self._add_intermediate()
             else:
                 if isinstance(self.current_dataset, plottable_2D):
 …
                     self.current_datainfo.notes.append(data_point)
                 # I and Q - 1D data
+                # I and Q points
                 elif tagname == 'I' and isinstance(self.current_dataset, plottable_1D):
                     unit_list = unit.split("|")
 …
                     self.current_dataset.dx = np.append(self.current_dataset.dx, data_point)
                 elif tagname == 'dQw':
+                    if self.current_dataset.dqw is None: self.current_dataset.dqw = np.empty(0)
                     self.current_dataset.dxw = np.append(self.current_dataset.dxw, data_point)
                 elif tagname == 'dQl':
+                    if self.current_dataset.dxl is None: self.current_dataset.dxl = np.empty(0)
                     self.current_dataset.dxl = np.append(self.current_dataset.dxl, data_point)
                 elif tagname == 'Qmean':
 …
                 elif tagname == 'name' and self.parent_class == 'SASinstrument':
                     self.current_datainfo.instrument = data_point
                 # Detector Information
                 elif tagname == 'name' and self.parent_class == 'SASdetector':
 …
                     self.detector.orientation.z = data_point
                     self.detector.orientation_unit = unit
                 # Collimation and Aperture
                 elif tagname == 'length' and self.parent_class == 'SAScollimation':
 …
                 elif tagname == 'term' and self.parent_class == 'SASprocess':
                     unit = attr.get("unit", "")
+                    dic = {}
+                    dic["name"] = name
+                    dic["value"] = data_point
+                    dic["unit"] = unit
+                    dic = { "name": name, "value": data_point, "unit": unit }
                     self.process.term.append(dic)
 …
         if not self._is_call_local() and not recurse:
             self.frm = ""
+            self.add_data_set()
+            self.current_datainfo.errors = set()
+            for error in self.errors:
+                self.current_datainfo.errors.add(error)
+            self.errors.clear()
+            self.send_to_output()
             empty = None
             return self.output[0], empty
     def _is_call_local(self):
-        """
-        """
         if self.frm == "":
             inter = inspect.stack()
 …
         return True
+    def is_cansas(self, ext="xml"):
+        """
+        Checks to see if the xml file is a CanSAS file
+        :param ext: The file extension of the data file
+        """
+        if self.validate_xml():
+            name = "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation"
+            value = self.xmlroot.get(name)
+            if CANSAS_NS.get(self.cansas_version).get("ns") == \
+                    value.rsplit(" ")[0]:
+                return True
+        if ext == "svs":
+            return True
+        raise RuntimeError
+    def load_file_and_schema(self, xml_file, schema_path=""):
+        """
+        Loads the file and associates a schema, if a schema is passed in or if one already exists
+        :param xml_file: The xml file path sent to Reader.read
+        :param schema_path: The path to a schema associated with the xml_file, or find one based on the file
+        """
+        base_name = xml_reader.__file__
+        base_name = base_name.replace("\\", "/")
+        base = base_name.split("/sas/")[0]
+        # Load in xml file and get the cansas version from the header
+        self.set_xml_file(xml_file)
+        self.cansas_version = self.xmlroot.get("version", "1.0")
+        # Generic values for the cansas file based on the version
+        self.cansas_defaults = CANSAS_NS.get(self.cansas_version, "1.0")
+        if schema_path == "":
+            schema_path = "{0}/sas/sascalc/dataloader/readers/schema/{1}".format \
+                (base, self.cansas_defaults.get("schema")).replace("\\", "/")
+        # Link a schema to the XML file.
+        self.set_schema(schema_path)
+    def add_data_set(self):
+        """
+        Adds the current_dataset to the list of outputs after preforming final processing on the data and then calls a
+        private method to generate a new data set.
+        :param key: NeXus group name for current tree level
+        """
+        if self.current_datainfo and self.current_dataset:
+            self._final_cleanup()
+        self.data = []
+        self.current_datainfo = DataInfo()
+    def _initialize_new_data_set(self, node=None):
+        """
+        A private class method to generate a new 1D data object.
+        Outside methods should call add_data_set() to be sure any existing data is stored properly.
+        :param node: XML node to determine if 1D or 2D data
+        """
+        x = np.array(0)
+        y = np.array(0)
+        for child in node:
+            if child.tag.replace(self.base_ns, "") == "Idata":
+                for i_child in child:
+                    if i_child.tag.replace(self.base_ns, "") == "Qx":
+                        self.current_dataset = plottable_2D()
+                        return
+        self.current_dataset = plottable_1D(x, y)
+    def add_intermediate(self):
+    def _add_intermediate(self):
         """
         This method stores any intermediate objects within the final data set after fully reading the set.
+        :param parent: The NXclass name for the h5py Group object that just finished being processed
+        """
+        """
         if self.parent_class == 'SASprocess':
             self.current_datainfo.process.append(self.process)
 …
             self._check_for_empty_resolution()
             self.data.append(self.current_dataset)
-    def _final_cleanup(self):
-        """
-        Final cleanup of the Data1D object to be sure it has all the
-        appropriate information needed for perspectives
-        """
-        # Append errors to dataset and reset class errors
-        self.current_datainfo.errors = set()
-        for error in self.errors:
-            self.current_datainfo.errors.add(error)
-        self.errors.clear()
-        # Combine all plottables with datainfo and append each to output
-        # Type cast data arrays to float64 and find min/max as appropriate
-        for dataset in self.data:
-            if isinstance(dataset, plottable_1D):
-                if dataset.x is not None:
-                    dataset.x = np.delete(dataset.x, [0])
-                    dataset.x = dataset.x.astype(np.float64)
-                    dataset.xmin = np.min(dataset.x)
-                    dataset.xmax = np.max(dataset.x)
-                if dataset.y is not None:
-                    dataset.y = np.delete(dataset.y, [0])
-                    dataset.y = dataset.y.astype(np.float64)
-                    dataset.ymin = np.min(dataset.y)
-                    dataset.ymax = np.max(dataset.y)
-                if dataset.dx is not None:
-                    dataset.dx = np.delete(dataset.dx, [0])
-                    dataset.dx = dataset.dx.astype(np.float64)
-                if dataset.dxl is not None:
-                    dataset.dxl = np.delete(dataset.dxl, [0])
-                    dataset.dxl = dataset.dxl.astype(np.float64)
-                if dataset.dxw is not None:
-                    dataset.dxw = np.delete(dataset.dxw, [0])
-                    dataset.dxw = dataset.dxw.astype(np.float64)
-                if dataset.dy is not None:
-                    dataset.dy = np.delete(dataset.dy, [0])
-                    dataset.dy = dataset.dy.astype(np.float64)
-                np.trim_zeros(dataset.x)
-                np.trim_zeros(dataset.y)
-                np.trim_zeros(dataset.dy)
-            elif isinstance(dataset, plottable_2D):
-                dataset.data = dataset.data.astype(np.float64)
-                dataset.qx_data = dataset.qx_data.astype(np.float64)
-                dataset.xmin = np.min(dataset.qx_data)
-                dataset.xmax = np.max(dataset.qx_data)
-                dataset.qy_data = dataset.qy_data.astype(np.float64)
-                dataset.ymin = np.min(dataset.qy_data)
-                dataset.ymax = np.max(dataset.qy_data)
-                dataset.q_data = np.sqrt(dataset.qx_data * dataset.qx_data
-                                         + dataset.qy_data * dataset.qy_data)
-                if dataset.err_data is not None:
-                    dataset.err_data = dataset.err_data.astype(np.float64)
-                if dataset.dqx_data is not None:
-                    dataset.dqx_data = dataset.dqx_data.astype(np.float64)
-                if dataset.dqy_data is not None:
-                    dataset.dqy_data = dataset.dqy_data.astype(np.float64)
-                if dataset.mask is not None:
-                    dataset.mask = dataset.mask.astype(dtype=bool)
-                if len(dataset.shape) == 2:
-                    n_rows, n_cols = dataset.shape
-                    dataset.y_bins = dataset.qy_data[0::int(n_cols)]
-                    dataset.x_bins = dataset.qx_data[:int(n_cols)]
-                    dataset.data = dataset.data.flatten()
-                else:
-                    dataset.y_bins = []
-                    dataset.x_bins = []
-                    dataset.data = dataset.data.flatten()
-            final_dataset = combine_data(dataset, self.current_datainfo)
-            self.output.append(final_dataset)
-    def _create_unique_key(self, dictionary, name, numb=0):
-        """
-        Create a unique key value for any dictionary to prevent overwriting
-        Recurse until a unique key value is found.
-        :param dictionary: A dictionary with any number of entries
-        :param name: The index of the item to be added to dictionary
-        :param numb: The number to be appended to the name, starts at 0
-        """
-        if dictionary.get(name) is not None:
-            numb += 1
-            name = name.split("_")[0]
-            name += "_{0}".format(numb)
-            name = self._create_unique_key(dictionary, name, numb)
-        return name
     def _get_node_value(self, node, tagname):
 …
         return node_value, value_unit
-    def _check_for_empty_data(self):
-        """
-        Creates an empty data set if no data is passed to the reader
-        :param data1d: presumably a Data1D object
-        """
-        if self.current_dataset is None:
-            x_vals = np.empty(0)
-            y_vals = np.empty(0)
-            dx_vals = np.empty(0)
-            dy_vals = np.empty(0)
-            dxl = np.empty(0)
-            dxw = np.empty(0)
-            self.current_dataset = plottable_1D(x_vals, y_vals, dx_vals, dy_vals)
-            self.current_dataset.dxl = dxl
-            self.current_dataset.dxw = dxw
     def _check_for_empty_resolution(self):
         """
+        A method to check all resolution data sets are the same size as I and Q
+        """
+        if isinstance(self.current_dataset, plottable_1D):
+            dql_exists = False
+            dqw_exists = False
+            dq_exists = False
+            di_exists = False
+            if self.current_dataset.dxl is not None:
+                dql_exists = True
+            if self.current_dataset.dxw is not None:
+                dqw_exists = True
+            if self.current_dataset.dx is not None:
+                dq_exists = True
+            if self.current_dataset.dy is not None:
+                di_exists = True
+            if dqw_exists and not dql_exists:
+                array_size = self.current_dataset.dxw.size - 1
+                self.current_dataset.dxl = np.append(self.current_dataset.dxl,
+                                                     np.zeros([array_size]))
+            elif dql_exists and not dqw_exists:
+                array_size = self.current_dataset.dxl.size - 1
+                self.current_dataset.dxw = np.append(self.current_dataset.dxw,
+                                                     np.zeros([array_size]))
+            elif not dql_exists and not dqw_exists and not dq_exists:
+                array_size = self.current_dataset.x.size - 1
+                self.current_dataset.dx = np.append(self.current_dataset.dx,
+                                                    np.zeros([array_size]))
+            if not di_exists:
+                array_size = self.current_dataset.y.size - 1
+                self.current_dataset.dy = np.append(self.current_dataset.dy,
+                                                    np.zeros([array_size]))
+        elif isinstance(self.current_dataset, plottable_2D):
+            dqx_exists = False
+            dqy_exists = False
+            di_exists = False
+            mask_exists = False
+            if self.current_dataset.dqx_data is not None:
+                dqx_exists = True
+            if self.current_dataset.dqy_data is not None:
+                dqy_exists = True
+            if self.current_dataset.err_data is not None:
+                di_exists = True
+            if self.current_dataset.mask is not None:
+                mask_exists = True
+            if not dqy_exists:
+                array_size = self.current_dataset.qy_data.size - 1
+                self.current_dataset.dqy_data = np.append(
+                    self.current_dataset.dqy_data, np.zeros([array_size]))
+            if not dqx_exists:
+                array_size = self.current_dataset.qx_data.size - 1
+                self.current_dataset.dqx_data = np.append(
+                    self.current_dataset.dqx_data, np.zeros([array_size]))
+            if not di_exists:
+                array_size = self.current_dataset.data.size - 1
+                self.current_dataset.err_data = np.append(
+                    self.current_dataset.err_data, np.zeros([array_size]))
+            if not mask_exists:
+                array_size = self.current_dataset.data.size - 1
+                self.current_dataset.mask = np.append(
+                    self.current_dataset.mask,
+                    np.ones([array_size] ,dtype=bool))
+    ####### All methods below are for writing CanSAS XML files #######
+        a method to check all resolution data sets are the same size as I and q
+        """
+        dql_exists = False
+        dqw_exists = False
+        dq_exists = False
+        di_exists = False
+        if self.current_dataset.dxl is not None:
+            dql_exists = True
+        if self.current_dataset.dxw is not None:
+            dqw_exists = True
+        if self.current_dataset.dx is not None:
+            dq_exists = True
+        if self.current_dataset.dy is not None:
+            di_exists = True
+        if dqw_exists and not dql_exists:
+            array_size = self.current_dataset.dxw.size - 1
+            self.current_dataset.dxl = np.append(self.current_dataset.dxl,
+                                                 np.zeros([array_size]))
+        elif dql_exists and not dqw_exists:
+            array_size = self.current_dataset.dxl.size - 1
+            self.current_dataset.dxw = np.append(self.current_dataset.dxw,
+                                                 np.zeros([array_size]))
+        elif not dql_exists and not dqw_exists and not dq_exists:
+            array_size = self.current_dataset.x.size - 1
+            self.current_dataset.dx = np.append(self.current_dataset.dx,
+                                                np.zeros([array_size]))
+        if not di_exists:
+            array_size = self.current_dataset.y.size - 1
+            self.current_dataset.dy = np.append(self.current_dataset.dy,
+                                                np.zeros([array_size]))
+    def _initialize_new_data_set(self, node=None):
+        if node is not None:
+            for child in node:
+                if child.tag.replace(self.base_ns, "") == "Idata":
+                    for i_child in child:
+                        if i_child.tag.replace(self.base_ns, "") == "Qx":
+                            self.current_dataset = plottable_2D()
+                            return
+        self.current_dataset = plottable_1D(np.array(0), np.array(0))
+    ## Writing Methods
     def write(self, filename, datainfo):
         """
 …
             exec "storage.%s = entry.text.strip()" % variable
 # DO NOT REMOVE Called by outside packages:
 #    sas.sasgui.perspectives.invariant.invariant_state

src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py

-                      rc94280c
+                      r8dec7e7
     TransmissionSpectrum, Detector
 from sas.sascalc.dataloader.data_info import combine_data_info_with_plottable
+class Reader():
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException, DefaultReaderException
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+class Reader(FileReader):
     """
     A class for reading in CanSAS v2.0 data files. The existing iteration opens
 …
     # Raw file contents to be processed
     raw_data = None
-    # Data info currently being read in
-    current_datainfo = None
-    # SASdata set currently being read in
-    current_dataset = None
     # List of plottable1D objects that should be linked to the current_datainfo
     data1d = None
 …
     # Flag to bypass extension check
     allow_all = True
+    # List of files to return
+    output = None
+    def read(self, filename):
+    def get_file_contents(self):
         """
         This is the general read method that all SasView data_loaders must have.
 …
         # Reinitialize when loading a new data file to reset all class variables
         self.reset_class_variables()
+        filename = self.f_open.name
+        self.f_open.close() # IO handled by h5py
         # Check that the file exists
         if os.path.isfile(filename):
 …
             if extension in self.ext or self.allow_all:
                 # Load the data file
+                self.raw_data = h5py.File(filename, 'r')
+                try:
+                    self.raw_data = h5py.File(filename, 'r')
+                except Exception as e:
+                    if extension not in self.ext:
+                        msg = "CanSAS2.0 HDF5 Reader could not load file {}".format(basename + extension)
+                        raise DefaultReaderException(msg)
+                    raise FileContentsException(e.message)
                 # Read in all child elements of top level SASroot
                 self.read_children(self.raw_data, [])
 …
         Data1D and Data2D objects
         """
         # Type cast data arrays to float64
         if len(self.current_datainfo.trans_spectrum) > 0:
 …
         # Type cast data arrays to float64 and find min/max as appropriate
         for dataset in self.data2d:
-            dataset.data = dataset.data.astype(np.float64)
-            dataset.err_data = dataset.err_data.astype(np.float64)
-            if dataset.qx_data is not None:
-                dataset.xmin = np.min(dataset.qx_data)
-                dataset.xmax = np.max(dataset.qx_data)
-                dataset.qx_data = dataset.qx_data.astype(np.float64)
-            if dataset.dqx_data is not None:
-                dataset.dqx_data = dataset.dqx_data.astype(np.float64)
-            if dataset.qy_data is not None:
-                dataset.ymin = np.min(dataset.qy_data)
-                dataset.ymax = np.max(dataset.qy_data)
-                dataset.qy_data = dataset.qy_data.astype(np.float64)
-            if dataset.dqy_data is not None:
-                dataset.dqy_data = dataset.dqy_data.astype(np.float64)
-            if dataset.q_data is not None:
-                dataset.q_data = dataset.q_data.astype(np.float64)
             zeros = np.ones(dataset.data.size, dtype=bool)
             try:
 …
                 dataset.x_bins = dataset.qx_data[:n_cols]
                 dataset.data = dataset.data.flatten()
+            final_dataset = combine_data_info_with_plottable(
+                dataset, self.current_datainfo)
+            self.output.append(final_dataset)
+            self.current_dataset = dataset
+            self.send_to_output()
         for dataset in self.data1d:
+            if dataset.x is not None:
+                dataset.x = dataset.x.astype(np.float64)
+                dataset.xmin = np.min(dataset.x)
+                dataset.xmax = np.max(dataset.x)
+            if dataset.y is not None:
+                dataset.y = dataset.y.astype(np.float64)
+                dataset.ymin = np.min(dataset.y)
+                dataset.ymax = np.max(dataset.y)
+            if dataset.dx is not None:
+                dataset.dx = dataset.dx.astype(np.float64)
+            if dataset.dxl is not None:
+                dataset.dxl = dataset.dxl.astype(np.float64)
+            if dataset.dxw is not None:
+                dataset.dxw = dataset.dxw.astype(np.float64)
+            if dataset.dy is not None:
+                dataset.dy = dataset.dy.astype(np.float64)
+            final_dataset = combine_data_info_with_plottable(
+                dataset, self.current_datainfo)
+            self.output.append(final_dataset)
+            self.current_dataset = dataset
+            self.send_to_output()
     def add_data_set(self, key=""):

src/sas/sascalc/dataloader/readers/danse_reader.py

-                      r235f514
+                      r713a047
 #This software was developed by the University of Tennessee as part of the
 #Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
 #project funded by the US National Science Foundation.
+#project funded by the US National Science Foundation.
 #If you use DANSE applications to do scientific research that leads to
 #publication, we ask that you acknowledge the use of the software with the
 …
 import math
 import os
-import sys
 import numpy as np
 import logging
 from sas.sascalc.dataloader.data_info import Data2D, Detector
+from sas.sascalc.dataloader.data_info import plottable_2D, DataInfo, Detector
 from sas.sascalc.dataloader.manipulations import reader2D_converter
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException, DataReaderException
 logger = logging.getLogger(__name__)
 …
 class Reader:
+class Reader(FileReader):
     """
     Example data manipulation
 …
     ## Extension
     ext  = ['.sans', '.SANS']
+    def read(self, filename=None):
+        """
+        Open and read the data in a file
+        @param file: path of the file
+        """
+        read_it = False
+        for item in self.ext:
+            if filename.lower().find(item) >= 0:
+                read_it = True
+        if read_it:
+    def get_file_contents(self):
+        self.current_datainfo = DataInfo()
+        self.current_dataset = plottable_2D()
+        self.output = []
+        loaded_correctly = True
+        error_message = ""
+        # defaults
+        # wavelength in Angstrom
+        wavelength = 10.0
+        # Distance in meter
+        distance   = 11.0
+        # Pixel number of center in x
+        center_x   = 65
+        # Pixel number of center in y
+        center_y   = 65
+        # Pixel size [mm]
+        pixel      = 5.0
+        # Size in x, in pixels
+        size_x     = 128
+        # Size in y, in pixels
+        size_y     = 128
+        # Format version
+        fversion   = 1.0
+        self.current_datainfo.filename = os.path.basename(self.f_open.name)
+        detector = Detector()
+        self.current_datainfo.detector.append(detector)
+        self.current_dataset.data = np.zeros([size_x, size_y])
+        self.current_dataset.err_data = np.zeros([size_x, size_y])
+        read_on = True
+        data_start_line = 1
+        while read_on:
+            line = self.f_open.readline()
+            data_start_line += 1
+            if line.find("DATA:") >= 0:
+                read_on = False
+                break
+            toks = line.split(':')
             try:
-                datafile = open(filename, 'r')
-            except:
-                raise  RuntimeError,"danse_reader cannot open %s" % (filename)
-            # defaults
-            # wavelength in Angstrom
-            wavelength = 10.0
-            # Distance in meter
-            distance   = 11.0
-            # Pixel number of center in x
-            center_x   = 65
-            # Pixel number of center in y
-            center_y   = 65
-            # Pixel size [mm]
-            pixel      = 5.0
-            # Size in x, in pixels
-            size_x     = 128
-            # Size in y, in pixels
-            size_y     = 128
-            # Format version
-            fversion   = 1.0
-            output = Data2D()
-            output.filename = os.path.basename(filename)
-            detector = Detector()
-            output.detector.append(detector)
-            output.data = np.zeros([size_x,size_y])
-            output.err_data = np.zeros([size_x, size_y])
-            data_conv_q = None
-            data_conv_i = None
-            if has_converter == True and output.Q_unit != '1/A':
-                data_conv_q = Converter('1/A')
-                # Test it
-                data_conv_q(1.0, output.Q_unit)
-            if has_converter == True and output.I_unit != '1/cm':
-                data_conv_i = Converter('1/cm')
-                # Test it
-                data_conv_i(1.0, output.I_unit)
-            read_on = True
-            while read_on:
-                line = datafile.readline()
-                if line.find("DATA:") >= 0:
-                    read_on = False
-                    break
-                toks = line.split(':')
                 if toks[0] == "FORMATVERSION":
                     fversion = float(toks[1])
                 if toks[0] == "WAVELENGTH":
+                elif toks[0] == "WAVELENGTH":
                     wavelength = float(toks[1])
                 elif toks[0] == "DISTANCE":
 …
                 elif toks[0] == "SIZE_Y":
                     size_y = int(toks[1])
+            # Read the data
+            data = []
+            error = []
+            if fversion == 1.0:
+                data_str = datafile.readline()
+                data = data_str.split(' ')
+            else:
+                read_on = True
+                while read_on:
+                    data_str = datafile.readline()
+                    if len(data_str) == 0:
+                        read_on = False
+                    else:
+                        toks = data_str.split()
+                        try:
+                            val = float(toks[0])
+                            err = float(toks[1])
+                            if data_conv_i is not None:
+                                val = data_conv_i(val, units=output._yunit)
+                                err = data_conv_i(err, units=output._yunit)
+                            data.append(val)
+                            error.append(err)
+                        except:
+                            logger.info("Skipping line:%s,%s" %(data_str,
+                                                                sys.exc_value))
+            # Initialize
+            x_vals = []
+            y_vals = []
+            ymin = None
+            ymax = None
+            xmin = None
+            xmax = None
+            # Qx and Qy vectors
+            theta = pixel / distance / 100.0
+            stepq = 4.0 * math.pi / wavelength * math.sin(theta / 2.0)
+            for i_x in range(size_x):
+                theta = (i_x - center_x + 1) * pixel / distance / 100.0
+                qx = 4.0 * math.pi / wavelength * math.sin(theta / 2.0)
+                if has_converter == True and output.Q_unit != '1/A':
+                    qx = data_conv_q(qx, units=output.Q_unit)
+                x_vals.append(qx)
+                if xmin is None or qx < xmin:
+                    xmin = qx
+                if xmax is None or qx > xmax:
+                    xmax = qx
+            ymin = None
+            ymax = None
+            for i_y in range(size_y):
+                theta = (i_y - center_y + 1) * pixel / distance / 100.0
+                qy = 4.0 * math.pi / wavelength * math.sin(theta/2.0)
+                if has_converter == True and output.Q_unit != '1/A':
+                    qy = data_conv_q(qy, units=output.Q_unit)
+                y_vals.append(qy)
+                if ymin is None or qy < ymin:
+                    ymin = qy
+                if ymax is None or qy > ymax:
+                    ymax = qy
+            # Store the data in the 2D array
+            i_x = 0
+            i_y = -1
+            for i_pt in range(len(data)):
+                try:
+                    value = float(data[i_pt])
+                except:
+                    # For version 1.0, the data were still
+                    # stored as strings at this point.
+                    msg = "Skipping entry (v1.0):%s,%s" % (str(data[i_pt]),
+                                                           sys.exc_value)
+                    logger.info(msg)
+                # Get bin number
+                if math.fmod(i_pt, size_x) == 0:
+                    i_x = 0
+                    i_y += 1
+                else:
+                    i_x += 1
+                output.data[i_y][i_x] = value
+                if fversion>1.0:
+                    output.err_data[i_y][i_x] = error[i_pt]
+            # Store all data
+            # Store wavelength
+            if has_converter == True and output.source.wavelength_unit != 'A':
+                conv = Converter('A')
+                wavelength = conv(wavelength,
+                                  units=output.source.wavelength_unit)
+            output.source.wavelength = wavelength
+            # Store distance
+            if has_converter == True and detector.distance_unit != 'm':
+                conv = Converter('m')
+                distance = conv(distance, units=detector.distance_unit)
+            detector.distance = distance
+            # Store pixel size
+            if has_converter == True and detector.pixel_size_unit != 'mm':
+                conv = Converter('mm')
+                pixel = conv(pixel, units=detector.pixel_size_unit)
+            detector.pixel_size.x = pixel
+            detector.pixel_size.y = pixel
+            # Store beam center in distance units
+            detector.beam_center.x = center_x * pixel
+            detector.beam_center.y = center_y * pixel
+            # Store limits of the image (2D array)
+            xmin = xmin - stepq / 2.0
+            xmax = xmax + stepq / 2.0
+            ymin = ymin - stepq /2.0
+            ymax = ymax + stepq / 2.0
+            if has_converter == True and output.Q_unit != '1/A':
+                xmin = data_conv_q(xmin, units=output.Q_unit)
+                xmax = data_conv_q(xmax, units=output.Q_unit)
+                ymin = data_conv_q(ymin, units=output.Q_unit)
+                ymax = data_conv_q(ymax, units=output.Q_unit)
+            output.xmin = xmin
+            output.xmax = xmax
+            output.ymin = ymin
+            output.ymax = ymax
+            # Store x and y axis bin centers
+            output.x_bins = x_vals
+            output.y_bins = y_vals
+            # Units
+            if data_conv_q is not None:
+                output.xaxis("\\rm{Q_{x}}", output.Q_unit)
+                output.yaxis("\\rm{Q_{y}}", output.Q_unit)
+            else:
+                output.xaxis("\\rm{Q_{x}}", 'A^{-1}')
+                output.yaxis("\\rm{Q_{y}}", 'A^{-1}')
+            if data_conv_i is not None:
+                output.zaxis("\\rm{Intensity}", output.I_unit)
+            else:
+                output.zaxis("\\rm{Intensity}", "cm^{-1}")
+            if not fversion >= 1.0:
+                msg = "Danse_reader can't read this file %s" % filename
+                raise ValueError, msg
+            else:
+                logger.info("Danse_reader Reading %s \n" % filename)
+            # Store loading process information
+            output.meta_data['loader'] = self.type_name
+            output = reader2D_converter(output)
+            return output
+        return None
+            except ValueError as e:
+                error_message += "Unable to parse {}. Default value used.\n".format(toks[0])
+                loaded_correctly = False
+        # Read the data
+        data = []
+        error = []
+        if not fversion >= 1.0:
+            msg = "danse_reader can't read this file {}".format(self.f_open.name)
+            raise FileContentsException(msg)
+        for line_num, data_str in enumerate(self.f_open.readlines()):
+            toks = data_str.split()
+            try:
+                val = float(toks[0])
+                err = float(toks[1])
+                data.append(val)
+                error.append(err)
+            except ValueError as exc:
+                msg = "Unable to parse line {}: {}".format(line_num + data_start_line, data_str.strip())
+                raise FileContentsException(msg)
+        num_pts = size_x * size_y
+        if len(data) < num_pts:
+            msg = "Not enough data points provided. Expected {} but got {}".format(
+                size_x * size_y, len(data))
+            raise FileContentsException(msg)
+        elif len(data) > num_pts:
+            error_message += ("Too many data points provided. Expected {0} but"
+                " got {1}. Only the first {0} will be used.\n").format(num_pts, len(data))
+            loaded_correctly = False
+            data = data[:num_pts]
+            error = error[:num_pts]
+        # Qx and Qy vectors
+        theta = pixel / distance / 100.0
+        i_x = np.arange(size_x)
+        theta = (i_x - center_x + 1) * pixel / distance / 100.0
+        x_vals = 4.0 * np.pi / wavelength * np.sin(theta / 2.0)
+        xmin = x_vals.min()
+        xmax = x_vals.max()
+        i_y = np.arange(size_y)
+        theta = (i_y - center_y + 1) * pixel / distance / 100.0
+        y_vals = 4.0 * np.pi / wavelength * np.sin(theta / 2.0)
+        ymin = y_vals.min()
+        ymax = y_vals.max()
+        self.current_dataset.data = np.array(data, dtype=np.float64).reshape((size_y, size_x))
+        if fversion > 1.0:
+            self.current_dataset.err_data = np.array(error, dtype=np.float64).reshape((size_y, size_x))
+        # Store all data
+        # Store wavelength
+        if has_converter == True and self.current_datainfo.source.wavelength_unit != 'A':
+            conv = Converter('A')
+            wavelength = conv(wavelength,
+                              units=self.current_datainfo.source.wavelength_unit)
+        self.current_datainfo.source.wavelength = wavelength
+        # Store distance
+        if has_converter == True and detector.distance_unit != 'm':
+            conv = Converter('m')
+            distance = conv(distance, units=detector.distance_unit)
+        detector.distance = distance
+        # Store pixel size
+        if has_converter == True and detector.pixel_size_unit != 'mm':
+            conv = Converter('mm')
+            pixel = conv(pixel, units=detector.pixel_size_unit)
+        detector.pixel_size.x = pixel
+        detector.pixel_size.y = pixel
+        # Store beam center in distance units
+        detector.beam_center.x = center_x * pixel
+        detector.beam_center.y = center_y * pixel
+        self.current_dataset.xaxis("\\rm{Q_{x}}", 'A^{-1}')
+        self.current_dataset.yaxis("\\rm{Q_{y}}", 'A^{-1}')
+        self.current_dataset.zaxis("\\rm{Intensity}", "cm^{-1}")
+        self.current_dataset.x_bins = x_vals
+        self.current_dataset.y_bins = y_vals
+        # Reshape data
+        x_vals = np.tile(x_vals, (size_y, 1)).flatten()
+        y_vals = np.tile(y_vals, (size_x, 1)).T.flatten()
+        if self.current_dataset.err_data == np.all(np.array(None)) or np.any(self.current_dataset.err_data <= 0):
+            new_err_data = np.sqrt(np.abs(self.current_dataset.data))
+        else:
+            new_err_data = self.current_dataset.err_data.flatten()
+        self.current_dataset.err_data = new_err_data
+        self.current_dataset.qx_data = x_vals
+        self.current_dataset.qy_data = y_vals
+        self.current_dataset.q_data = np.sqrt(x_vals**2 + y_vals**2)
+        self.current_dataset.mask = np.ones(len(x_vals), dtype=bool)
+        # Store loading process information
+        self.current_datainfo.meta_data['loader'] = self.type_name
+        self.send_to_output()
+        if not loaded_correctly:
+            raise DataReaderException(error_message)

src/sas/sascalc/dataloader/readers/red2d_reader.py

-                      ra1b8fee
+                      r2f85af7
 #This software was developed by the University of Tennessee as part of the
 #Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
 #project funded by the US National Science Foundation.
+#project funded by the US National Science Foundation.
 #See the license text in license.txt
 #copyright 2008, University of Tennessee
 ######################################################################
-from __future__ import print_function
 import os
 import numpy as np
 import math
+from sas.sascalc.dataloader.data_info import Data2D, Detector
+from sas.sascalc.dataloader.data_info import plottable_2D, DataInfo, Detector
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException
 # Look for unit converter
 …
 except:
     has_converter = False
 def check_point(x_point):
     """
 …
     except:
         return 0
 class Reader:
+class Reader(FileReader):
     """ Simple data reader for Igor data files """
     ## File type
 …
     ## Extension
     ext = ['.DAT', '.dat']
     def write(self, filename, data):
         """
         Write to .dat
         :param filename: file name to write
         :param data: data2D
 …
         import time
         # Write the file
+        fd = open(filename, 'w')
+        t = time.localtime()
+        time_str = time.strftime("%H:%M on %b %d %y", t)
+        header_str = "Data columns are Qx - Qy - I(Qx,Qy)\n\nASCII data"
+        header_str += " created at %s \n\n" % time_str
+        # simple 2D header
+        fd.write(header_str)
+        # write qx qy I values
+        for i in range(len(data.data)):
+            fd.write("%g  %g  %g\n" % (data.qx_data[i],
+                                        data.qy_data[i],
+                                       data.data[i]))
+        # close
+        fd.close()
+    def read(self, filename=None):
+        """ Read file """
+        if not os.path.isfile(filename):
+            raise ValueError, \
+            "Specified file %s is not a regular file" % filename
+        try:
+            fd = open(filename, 'w')
+            t = time.localtime()
+            time_str = time.strftime("%H:%M on %b %d %y", t)
+            header_str = "Data columns are Qx - Qy - I(Qx,Qy)\n\nASCII data"
+            header_str += " created at %s \n\n" % time_str
+            # simple 2D header
+            fd.write(header_str)
+            # write qx qy I values
+            for i in range(len(data.data)):
+                fd.write("%g  %g  %g\n" % (data.qx_data[i],
+                                            data.qy_data[i],
+                                           data.data[i]))
+        finally:
+            fd.close()
+    def get_file_contents(self):
         # Read file
+        f = open(filename, 'r')
+        buf = f.read()
+        f.close()
+        buf = self.f_open.read()
+        self.f_open.close()
         # Instantiate data object
+        output = Data2D()
+        output.filename = os.path.basename(filename)
+        detector = Detector()
+        if len(output.detector) > 0:
+            print(str(output.detector[0]))
+        output.detector.append(detector)
+        self.current_dataset = plottable_2D()
+        self.current_datainfo = DataInfo()
+        self.current_datainfo.filename = os.path.basename(self.f_open.name)
+        self.current_datainfo.detector.append(Detector())
         # Get content
         dataStarted = False
+        data_started = False
         ## Defaults
         lines = buf.split('\n')
         x = []
         y = []
         wavelength = None
         distance = None
         transmission = None
         pixel_x = None
         pixel_y = None
+        isInfo = False
+        isCenter = False
+        data_conv_q = None
+        data_conv_i = None
+        # Set units: This is the unit assumed for Q and I in the data file.
+        if has_converter == True and output.Q_unit != '1/A':
+            data_conv_q = Converter('1/A')
+            # Test it
+            data_conv_q(1.0, output.Q_unit)
+        if has_converter == True and output.I_unit != '1/cm':
+            data_conv_i = Converter('1/cm')
+            # Test it
+            data_conv_i(1.0, output.I_unit)
+        is_info = False
+        is_center = False
         # Remove the last lines before the for loop if the lines are empty
         # to calculate the exact number of data points
 …
             ## Reading the header applies only to IGOR/NIST 2D q_map data files
             # Find setup info line
             if isInfo:
                 isInfo = False
+            if is_info:
+                is_info = False
                 line_toks = line.split()
                 # Wavelength in Angstrom
 …
                     # Units
                     if has_converter == True and \
                     output.source.wavelength_unit != 'A':
+                    self.current_datainfo.source.wavelength_unit != 'A':
                         conv = Converter('A')
                         wavelength = conv(wavelength,
                                           units=output.source.wavelength_unit)
+                                          units=self.current_datainfo.source.wavelength_unit)
                 except:
                     #Not required
 …
                     distance = float(line_toks[3])
                     # Units
                     if has_converter == True and detector.distance_unit != 'm':
+                    if has_converter == True and self.current_datainfo.detector[0].distance_unit != 'm':
                         conv = Converter('m')
+                        distance = conv(distance, units=detector.distance_unit)
+                        distance = conv(distance,
+                            units=self.current_datainfo.detector[0].distance_unit)
                 except:
                     #Not required
                     pass
                 # Distance in meters
                 try:
 …
                     #Not required
                     pass
             if line.count("LAMBDA") > 0:
                 isInfo = True
+                is_info = True
             # Find center info line
             if isCenter:
                 isCenter = False
+            if is_center:
+                is_center = False
                 line_toks = line.split()
                 # Center in bin number
 …
             if line.count("BCENT") > 0:
                 isCenter = True
+                is_center = True
             # Check version
             if line.count("Data columns") > 0:
 …
             # Find data start
             if line.count("ASCII data") > 0:
                 dataStarted = True
+                data_started = True
                 continue
             ## Read and get data.
             if dataStarted == True:
+            if data_started == True:
                 line_toks = line.split()
                 if len(line_toks) == 0:
                     #empty line
                     continue
                 # the number of columns must be stayed same
+                # the number of columns must be stayed same
                 col_num = len(line_toks)
                 break
 …
         # index for lines_array
         lines_index = np.arange(len(lines))
         # get the data lines
         data_lines = lines_array[lines_index >= (line_num - 1)]
 …
         # split all data to one big list w/" "separator
         data_list = data_list.split()
         # Check if the size is consistent with data, otherwise
         #try the tab(\t) separator
 …
             data_point = data_array.reshape(row_num, col_num).transpose()
         except:
             msg = "red2d_reader: Can't read this file: Not a proper file format"
             raise ValueError, msg
+            msg = "red2d_reader can't read this file: Incorrect number of data points provided."
+            raise FileContentsException(msg)
         ## Get the all data: Let's HARDcoding; Todo find better way
         # Defaults
 …
         #if col_num > (6 + ver): mask[data_point[(6 + ver)] < 1] = False
         q_data = np.sqrt(qx_data*qx_data+qy_data*qy_data+qz_data*qz_data)
         # Extra protection(it is needed for some data files):
+        # Extra protection(it is needed for some data files):
         # If all mask elements are False, put all True
         if not mask.any():
             mask[mask == False] = True
         # Store limits of the image in q space
         xmin = np.min(qx_data)
 …
         ymax = np.max(qy_data)
-        # units
-        if has_converter == True and output.Q_unit != '1/A':
-            xmin = data_conv_q(xmin, units=output.Q_unit)
-            xmax = data_conv_q(xmax, units=output.Q_unit)
-            ymin = data_conv_q(ymin, units=output.Q_unit)
-            ymax = data_conv_q(ymax, units=output.Q_unit)
         ## calculate the range of the qx and qy_data
         x_size = math.fabs(xmax - xmin)
         y_size = math.fabs(ymax - ymin)
         # calculate the number of pixels in the each axes
         npix_y = math.floor(math.sqrt(len(data)))
         npix_x = math.floor(len(data) / npix_y)
         # calculate the size of bins
         xstep = x_size / (npix_x - 1)
         ystep = y_size / (npix_y - 1)
         # store x and y axis bin centers in q space
         x_bins = np.arange(xmin, xmax + xstep, xstep)
         y_bins = np.arange(ymin, ymax + ystep, ystep)
         # get the limits of q values
         xmin = xmin - xstep / 2
 …
         ymin = ymin - ystep / 2
         ymax = ymax + ystep / 2
         #Store data in outputs
         #TODO: Check the lengths
         output.data = data
+        self.current_dataset.data = data
         if (err_data == 1).all():
             output.err_data = np.sqrt(np.abs(data))
             output.err_data[output.err_data == 0.0] = 1.0
+            self.current_dataset.err_data = np.sqrt(np.abs(data))
+            self.current_dataset.err_data[self.current_dataset.err_data == 0.0] = 1.0
         else:
             output.err_data = err_data
         output.qx_data = qx_data
         output.qy_data = qy_data
         output.q_data = q_data
         output.mask = mask
         output.x_bins = x_bins
         output.y_bins = y_bins
         output.xmin = xmin
         output.xmax = xmax
         output.ymin = ymin
         output.ymax = ymax
         output.source.wavelength = wavelength
+            self.current_dataset.err_data = err_data
+        self.current_dataset.qx_data = qx_data
+        self.current_dataset.qy_data = qy_data
+        self.current_dataset.q_data = q_data
+        self.current_dataset.mask = mask
+        self.current_dataset.x_bins = x_bins
+        self.current_dataset.y_bins = y_bins
+        self.current_dataset.xmin = xmin
+        self.current_dataset.xmax = xmax
+        self.current_dataset.ymin = ymin
+        self.current_dataset.ymax = ymax
+        self.current_datainfo.source.wavelength = wavelength
         # Store pixel size in mm
         detector.pixel_size.x = pixel_x
         detector.pixel_size.y = pixel_y
+        self.current_datainfo.detector[0].pixel_size.x = pixel_x
+        self.current_datainfo.detector[0].pixel_size.y = pixel_y
         # Store the sample to detector distance
         detector.distance = distance
+        self.current_datainfo.detector[0].distance = distance
         # optional data: if all of dq data == 0, do not pass to output
         if len(dqx_data) == len(qx_data) and dqx_data.any() != 0:
 …
                     cos_th = qx_data / diag
                     sin_th = qy_data / diag
                     output.dqx_data = np.sqrt((dqx_data * cos_th) * \
+                    self.current_dataset.dqx_data = np.sqrt((dqx_data * cos_th) * \
                                                  (dqx_data * cos_th) \
                                                  + (dqy_data * sin_th) * \
                                                   (dqy_data * sin_th))
                     output.dqy_data = np.sqrt((dqx_data * sin_th) * \
+                    self.current_dataset.dqy_data = np.sqrt((dqx_data * sin_th) * \
                                                  (dqx_data * sin_th) \
                                                  + (dqy_data * cos_th) * \
                                                   (dqy_data * cos_th))
                 else:
                     output.dqx_data = dqx_data
                     output.dqy_data = dqy_data
+                    self.current_dataset.dqx_data = dqx_data
+                    self.current_dataset.dqy_data = dqy_data
         # Units of axes
+        if data_conv_q is not None:
+            output.xaxis("\\rm{Q_{x}}", output.Q_unit)
+            output.yaxis("\\rm{Q_{y}}", output.Q_unit)
+        else:
+            output.xaxis("\\rm{Q_{x}}", 'A^{-1}')
+            output.yaxis("\\rm{Q_{y}}", 'A^{-1}')
+        if data_conv_i is not None:
+            output.zaxis("\\rm{Intensity}", output.I_unit)
+        else:
+            output.zaxis("\\rm{Intensity}", "cm^{-1}")
+        self.current_dataset.xaxis("\\rm{Q_{x}}", 'A^{-1}')
+        self.current_dataset.yaxis("\\rm{Q_{y}}", 'A^{-1}')
+        self.current_dataset.zaxis("\\rm{Intensity}", "cm^{-1}")
         # Store loading process information
         output.meta_data['loader'] = self.type_name
         return output
+        self.current_datainfo.meta_data['loader'] = self.type_name
+        self.send_to_output()

src/sas/sascalc/dataloader/readers/xml_reader.py

-                      r235f514
+                      rfafe52a
 from lxml import etree
 from lxml.builder import E
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
 logger = logging.getLogger(__name__)
 …
 PARSER = etree.ETCompatXMLParser(remove_comments=True, remove_pis=False)
 class XMLreader():
+class XMLreader(FileReader):
     """
     Generic XML read and write class. Mostly helper functions.
 …
         except etree.XMLSyntaxError as xml_error:
             logger.info(xml_error)
+            raise xml_error
         except Exception:
             self.xml = None
 …
         except etree.XMLSyntaxError as xml_error:
             logger.info(xml_error)
+        except Exception:
+            raise xml_error
+        except Exception as exc:
             self.xml = None
             self.xmldoc = None
             self.xmlroot = None
+            raise exc
     def set_schema(self, schema):
 …
         Create a unique key value for any dictionary to prevent overwriting
         Recurses until a unique key value is found.
         :param dictionary: A dictionary with any number of entries
         :param name: The index of the item to be added to dictionary
 …
         Create an element tree for processing from an etree element
         :param root: etree Element(s)
+        :param root: etree Element(s)
         """
         return etree.ElementTree(root)

Note: See TracChangeset for help on using the changeset viewer.

Changeset 5a8cdbb in sasview for src/sas/sascalc/dataloader

Legend:

Download in other formats: