Changeset 44daa56 for src/sas/sascalc – SasView

src/sas/sascalc/pr/c_extensions/Cinvertor.c

-                      r959eb01
+                      rcb62bd5
+}
 const char set_has_bck_doc[] =
+const char set_est_bck_doc[] =
         "Sets background flag\n";
 …
  * Sets the maximum distance
  */
 static PyObject * set_has_bck(Cinvertor *self, PyObject *args) {
         int has_bck;
         if (!PyArg_ParseTuple(args, "i", &has_bck)) return NULL;
         self->params.has_bck = has_bck;
         return Py_BuildValue("i", self->params.has_bck);
+}
 const char get_has_bck_doc[] =
+static PyObject * set_est_bck(Cinvertor *self, PyObject *args) {
+        int est_bck;
+        if (!PyArg_ParseTuple(args, "i", &est_bck)) return NULL;
+        self->params.est_bck = est_bck;
+        return Py_BuildValue("i", self->params.est_bck);
+}
+const char get_est_bck_doc[] =
         "Gets background flag\n";
 …
  * Gets the maximum distance
  */
 static PyObject * get_has_bck(Cinvertor *self, PyObject *args) {
         return Py_BuildValue("i", self->params.has_bck);
+static PyObject * get_est_bck(Cinvertor *self, PyObject *args) {
+        return Py_BuildValue("i", self->params.est_bck);
+}
 …
         sqrt_alpha = sqrt(self->params.alpha);
         pi = acos(-1.0);
         offset = (self->params.has_bck==1) ? 0 : 1;
+        offset = (self->params.est_bck==1) ? 0 : 1;
     for (j=0; j<nfunc; j++) {
 …
+            }
             if (accept_q(self, self->params.x[i])){
                 if (self->params.has_bck==1 && j==0) {
+                if (self->params.est_bck==1 && j==0) {
                     a[i*nfunc+j] = 1.0/self->params.err[i];
                 } else {
 …
+        }
         for (i_r=0; i_r<nr; i_r++){
             if (self->params.has_bck==1 && j==0) {
+            if (self->params.est_bck==1 && j==0) {
                 a[(i_r+self->params.npoints)*nfunc+j] = 0.0;
             } else {
 …
                    {"set_slit_height", (PyCFunction)set_slit_height, METH_VARARGS, set_slit_height_doc},
                    {"get_slit_height", (PyCFunction)get_slit_height, METH_VARARGS, get_slit_height_doc},
                    {"set_has_bck", (PyCFunction)set_has_bck, METH_VARARGS, set_has_bck_doc},
                    {"get_has_bck", (PyCFunction)get_has_bck, METH_VARARGS, get_has_bck_doc},
+                   {"set_est_bck", (PyCFunction)set_est_bck, METH_VARARGS, set_est_bck_doc},
+                   {"get_est_bck", (PyCFunction)get_est_bck, METH_VARARGS, get_est_bck_doc},
                    {"get_nx", (PyCFunction)get_nx, METH_VARARGS, get_nx_doc},
                    {"get_ny", (PyCFunction)get_ny, METH_VARARGS, get_ny_doc},

src/sas/sascalc/pr/c_extensions/invertor.c

r959eb01	rcb62bd5
20	20	pars->q_min = -1.0;
21	21	pars->q_max = -1.0;
22		pars->~~has~~_bck = 0;
	22	pars->est_bck = 0;
23	23	}
24	24
…	…
313	313	return sqrt(sum_r2/(2.0*sum));
314	314	}
315

src/sas/sascalc/pr/c_extensions/invertor.h

r959eb01	rcb62bd5
27	27	double q_max;
28	28	/// Flag for whether or not to evalute a constant background while inverting
29		int ~~has~~_bck;
	29	int est_bck;
30	30	/// Slit height in units of q [A-1]
31	31	double slit_height;

src/sas/sascalc/pr/invertor.py

-                      r45dffa69
+                      rcb62bd5
          self.q_min, self.q_max,
          self.x, self.y,
          self.err, self.has_bck,
+         self.err, self.est_bck,
          self.slit_height, self.slit_width) = state
 …
                  self.q_min, self.q_max,
                  self.x, self.y,
                  self.err, self.has_bck,
+                 self.err, self.est_bck,
                  self.slit_height, self.slit_width,
+                )
 …
         elif name == 'slit_width':
             return self.set_slit_width(value)
         elif name == 'has_bck':
+        elif name == 'est_bck':
             if value == True:
                 return self.set_has_bck(1)
+                return self.set_est_bck(1)
             elif value == False:
                 return self.set_has_bck(0)
+                return self.set_est_bck(0)
             else:
                 raise ValueError, "Invertor: has_bck can only be True or False"
+                raise ValueError, "Invertor: est_bck can only be True or False"
         return Cinvertor.__setattr__(self, name, value)
 …
         elif name == 'slit_width':
             return self.get_slit_width()
         elif name == 'has_bck':
             value = self.get_has_bck()
+        elif name == 'est_bck':
+            value = self.get_est_bck()
             if value == 1:
                 return True
 …
         invertor.y = self.y
         invertor.err = self.err
+        invertor.has_bck = self.has_bck
+        invertor.est_bck = self.est_bck
+        invertor.background = self.background
         invertor.slit_height = self.slit_height
         invertor.slit_width = self.slit_width
 …
         """
         # Reset the background value before proceeding
+        self.background = 0.0
+        return self.lstsq(nfunc, nr=nr)
+        # self.background = 0.0
+        if not self.est_bck:
+            self.y -= self.background
+        out, cov = self.lstsq(nfunc, nr=nr)
+        if not self.est_bck:
+            self.y += self.background
+        return out, cov
     def iq(self, out, q):
 …
         # If we need to fit the background, add a term
         if self.has_bck == True:
+        if self.est_bck == True:
             nfunc_0 = nfunc
             nfunc += 1
 …
         # Keep a copy of the last output
+        if self.has_bck == False:
+            self.background = 0
+        if self.est_bck == False:
             self.out = c
             self.cov = err
 …
         file.write("#slit_width=%g\n" % self.slit_width)
         file.write("#background=%g\n" % self.background)
         if self.has_bck == True:
+        if self.est_bck == True:
             file.write("#has_bck=1\n")
         else:
 …
                         toks = line.split('=')
                         if int(toks[1]) == 1:
                             self.has_bck = True
+                            self.est_bck = True
                         else:
                             self.has_bck = False
+                            self.est_bck = False
                     # Now read in the parameters

src/sas/sascalc/data_util/registry.py

-                      ra1b8fee
+                      r5a8cdbb
-# This program is public domain
 """
 File extension registry.
 …
 from __future__ import print_function
+import os.path
+from sas.sascalc.dataloader.loader_exceptions import NoKnownLoaderException
 class ExtensionRegistry(object):
 …
         # Add an association by setting an element
         registry['.zip'] = unzip
         # Multiple extensions for one loader
         registry['.tgz'] = untar
         registry['.tar.gz'] = untar
         # Generic extensions to use after trying more specific extensions;
+        # Generic extensions to use after trying more specific extensions;
         # these will be checked after the more specific extensions fail.
         registry['.gz'] = gunzip
 …
         # Show registered extensions
         print registry.extensions()
         # Can also register a format name for explicit control from caller
         registry['cx3'] = cx3
 …
     def __init__(self, **kw):
         self.loaders = {}
     def __setitem__(self, ext, loader):
         if ext not in self.loaders:
             self.loaders[ext] = []
         self.loaders[ext].insert(0,loader)
     def __getitem__(self, ext):
         return self.loaders[ext]
     def __contains__(self, ext):
         return ext in self.loaders
     def formats(self):
         """
 …
         names.sort()
         return names
     def extensions(self):
         """
 …
         exts.sort()
         return exts
     def lookup(self, path):
         """
         Return the loader associated with the file type of path.
+        Raises ValueError if file type is not known.
+        """
+        :param path: Data file path
+        :raises ValueError: When no loaders are found for the file.
+        :return: List of available readers for the file extension
+        """
         # Find matching extensions
         extlist = [ext for ext in self.extensions() if path.endswith(ext)]
 …
         # Raise an error if there are no matching extensions
         if len(loaders) == 0:
+            raise ValueError, "Unknown file type for "+path
+        # All done
+            raise ValueError("Unknown file type for "+path)
         return loaders
     def load(self, path, format=None):
         """
         Call the loader for the file type of path.
         Raises ValueError if no loader is available.
         Raises KeyError if format is not available.
         May raise a loader-defined exception if loader fails.
+        :raise ValueError: if no loader is available.
+        :raise KeyError: if format is not available.
+        May raise a loader-defined exception if loader fails.
         """
+        loaders = []
         if format is None:
+            loaders = self.lookup(path)
+            try:
+                loaders = self.lookup(path)
+            except ValueError as e:
+                pass
         else:
+            loaders = self.loaders[format]
+            try:
+                loaders = self.loaders[format]
+            except KeyError as e:
+                pass
+        last_exc = None
         for fn in loaders:
             try:
                 return fn(path)
+            except:
+                pass # give other loaders a chance to succeed
+            except Exception as e:
+                last_exc = e
+                pass  # give other loaders a chance to succeed
         # If we get here it is because all loaders failed
+        raise # reraises last exception
+def test():
+    reg = ExtensionRegistry()
+    class CxError(Exception): pass
+    def cx(file): return 'cx'
+    def new_cx(file): return 'new_cx'
+    def fail_cx(file): raise CxError
+    def cat(file): return 'cat'
+    def gunzip(file): return 'gunzip'
+    reg['.cx'] = cx
+    reg['.cx1'] = cx
+    reg['.cx'] = new_cx
+    reg['.gz'] = gunzip
+    reg['.cx.gz'] = new_cx
+    reg['.cx1.gz'] = fail_cx
+    reg['.cx1'] = fail_cx
+    reg['.cx2'] = fail_cx
+    reg['new_cx'] = new_cx
+    # Two loaders associated with .cx
+    assert reg.lookup('hello.cx') == [new_cx,cx]
+    # Make sure the last loader applies first
+    assert reg.load('hello.cx') == 'new_cx'
+    # Make sure the next loader applies if the first fails
+    assert reg.load('hello.cx1') == 'cx'
+    # Make sure the format override works
+    assert reg.load('hello.cx1',format='.cx.gz') == 'new_cx'
+    # Make sure the format override works
+    assert reg.load('hello.cx1',format='new_cx') == 'new_cx'
+    # Make sure the case of all loaders failing is correct
+    try:  reg.load('hello.cx2')
+    except CxError: pass # correct failure
+    else: raise AssertError,"Incorrect error on load failure"
+    # Make sure the case of no loaders fails correctly
+    try: reg.load('hello.missing')
+    except ValueError,msg:
+        assert str(msg)=="Unknown file type for hello.missing",'Message: <%s>'%(msg)
+    else: raise AssertError,"No error raised for missing extension"
+    assert reg.formats() == ['new_cx']
+    assert reg.extensions() == ['.cx','.cx.gz','.cx1','.cx1.gz','.cx2','.gz']
+    # make sure that it supports multiple '.' in filename
+    assert reg.load('hello.extra.cx1') == 'cx'
+    assert reg.load('hello.gz') == 'gunzip'
+    assert reg.load('hello.cx1.gz') == 'gunzip' # Since .cx1.gz fails
+if __name__ == "__main__": test()
+        if last_exc is not None and len(loaders) != 0:
+            # If file has associated loader(s) and they;ve failed
+            raise last_exc
+        raise NoKnownLoaderException(e.message)  # raise generic exception

src/sas/sascalc/dataloader/data_info.py

-                      ra1b8fee
+                      r5a8cdbb
         return return_string
+    final_dataset.xmax = data.xmax
+    final_dataset.ymax = data.ymax
+    final_dataset.xmin = data.xmin
+    final_dataset.ymin = data.ymin
+    if hasattr(data, "xmax"):
+        final_dataset.xmax = data.xmax
+    if hasattr(data, "ymax"):
+        final_dataset.ymax = data.ymax
+    if hasattr(data, "xmin"):
+        final_dataset.xmin = data.xmin
+    if hasattr(data, "ymin"):
+        final_dataset.ymin = data.ymin
     final_dataset.isSesans = datainfo.isSesans
     final_dataset.title = datainfo.title

src/sas/sascalc/dataloader/loader.py

-                      r463e7ffc
+                      r3a81cd9
 """
     File handler to support different file extensions.
     Uses reflectometry's registry utility.
+    Uses reflectometer registry utility.
     The default readers are found in the 'readers' sub-module
 …
 """
 #####################################################################
 #This software was developed by the University of Tennessee as part of the
 #Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
 #project funded by the US National Science Foundation.
 #See the license text in license.txt
 #copyright 2008, University of Tennessee
+# This software was developed by the University of Tennessee as part of the
+# Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
+# project funded by the US National Science Foundation.
+# See the license text in license.txt
+# copyright 2008, University of Tennessee
 ######################################################################
 …
 # Default readers are defined in the readers sub-module
 import readers
+from loader_exceptions import NoKnownLoaderException, FileContentsException,\
+    DefaultReaderException
 from readers import ascii_reader
 from readers import cansas_reader
+from readers import cansas_reader_HDF5
 logger = logging.getLogger(__name__)
 class Registry(ExtensionRegistry):
 …
     Readers and writers are supported.
     """
     def __init__(self):
         super(Registry, self).__init__()
         ## Writers
+        # Writers
         self.writers = {}
         ## List of wildcards
+        # List of wildcards
         self.wildcards = ['All (*.*)|*.*']
         ## Creation time, for testing
+        # Creation time, for testing
         self._created = time.time()
 …
             of a particular reader
+        Defaults to the ascii (multi-column) reader
+        if no reader was registered for the file's
+        extension.
+        Defaults to the ascii (multi-column), cansas XML, and cansas NeXuS
+        readers if no reader was registered for the file's extension.
         """
         try:
             return super(Registry, self).load(path, format=format)
+        except:
+            try:
+                # No reader was found. Default to the ascii reader.
+                ascii_loader = ascii_reader.Reader()
+                return ascii_loader.read(path)
+            except:
+                cansas_loader = cansas_reader.Reader()
+                return cansas_loader.read(path)
+        except NoKnownLoaderException as nkl_e:
+            pass  # try the ASCII reader
+        except FileContentsException as fc_exc:
+            # File has an associated reader but it failed
+            raise RuntimeError(fc_exc.message)
+        except Exception:
+            pass
+        # File has no associated reader - try the ASCII reader
+        try:
+            ascii_loader = ascii_reader.Reader()
+            return ascii_loader.read(path)
+        except DefaultReaderException:
+            pass  # Loader specific error to try the cansas XML reader
+        except FileContentsException as e:
+            raise RuntimeError(e.message)
+        # ASCII reader failed - try CanSAS xML reader
+        try:
+            cansas_loader = cansas_reader.Reader()
+            return cansas_loader.read(path)
+        except DefaultReaderException:
+            pass  # Loader specific error to try the NXcanSAS reader
+        except FileContentsException as e:
+            raise RuntimeError(e.message)
+        except Exception as csr:
+            pass
+        # CanSAS XML reader failed - try NXcanSAS reader
+        try:
+            cansas_nexus_loader = cansas_reader_HDF5.Reader()
+            return cansas_nexus_loader.read(path)
+        except DefaultReaderException as e:
+            logging.error("No default loader can load the data")
+            # No known reader available. Give up and throw an error
+            msg = "\n\tUnknown data format: %s.\n\tThe file is not a " % path
+            msg += "known format that can be loaded by SasView.\n"
+            raise NoKnownLoaderException(msg)
+        except FileContentsException as e:
+            raise RuntimeError(e.message)
     def find_plugins(self, dir):

src/sas/sascalc/dataloader/readers/init.py

-                      r959eb01
+                      r7a5d066
 # Backward compatibility with the previous implementation of the default readers
 from associations import register_readers
+# Method to associate extensions to default readers
+from associations import read_associations
-# Method to associate extensions to default readers
-from associations import read_associations
 # Method to return the location of the XML settings file

src/sas/sascalc/dataloader/readers/abs_reader.py

-                      r959eb01
+                      rad92c5a
 """
+    IGOR 1D data reader
 """
 #####################################################################
 #This software was developed by the University of Tennessee as part of the
 #Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
 #project funded by the US National Science Foundation.
 #See the license text in license.txt
 #copyright 2008, University of Tennessee
+# This software was developed by the University of Tennessee as part of the
+# Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
+# project funded by the US National Science Foundation.
+# See the license text in license.txt
+# copyright 2008, University of Tennessee
 ######################################################################
+import logging
 import numpy as np
+import os
+from sas.sascalc.dataloader.data_info import Data1D
+from sas.sascalc.dataloader.data_info import Detector
+has_converter = True
+try:
+    from sas.sascalc.data_util.nxsunit import Converter
+except:
+    has_converter = False
+class Reader:
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+from sas.sascalc.dataloader.data_info import DataInfo, plottable_1D, Data1D,\
+    Detector
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException,\
+    DefaultReaderException
+logger = logging.getLogger(__name__)
+class Reader(FileReader):
     """
     Class to load IGOR reduced .ABS files
     """
     ## File type
+    # File type
     type_name = "IGOR 1D"
     ## Wildcards
+    # Wildcards
     type = ["IGOR 1D files (*.abs)|*.abs"]
     ## List of allowed extensions
     ext = ['.abs', '.ABS']
+    # List of allowed extensions
+    ext = ['.abs']
     def read(self, path):
+    def get_file_contents(self):
         """
+        Load data file.
+        :param path: file path
+        :return: Data1D object, or None
+        Get the contents of the file
         :raise RuntimeError: when the file can't be opened
         :raise ValueError: when the length of the data vectors are inconsistent
         """
+        if os.path.isfile(path):
+            basename = os.path.basename(path)
+            root, extension = os.path.splitext(basename)
+            if extension.lower() in self.ext:
+                try:
+                    input_f = open(path,'r')
+        buff = self.f_open.read()
+        filepath = self.f_open.name
+        lines = buff.splitlines()
+        self.has_converter = True
+        try:
+            from sas.sascalc.data_util.nxsunit import Converter
+        except:
+            self.has_converter = False
+        self.output = []
+        self.current_datainfo = DataInfo()
+        self.current_datainfo.filename = filepath
+        self.reset_data_list(len(lines))
+        detector = Detector()
+        data_line = 0
+        self.reset_data_list(len(lines))
+        self.current_datainfo.detector.append(detector)
+        self.current_datainfo.filename = filepath
+        is_info = False
+        is_center = False
+        is_data_started = False
+        base_q_unit = '1/A'
+        base_i_unit = '1/cm'
+        data_conv_q = Converter(base_q_unit)
+        data_conv_i = Converter(base_i_unit)
+        for line in lines:
+            # Information line 1
+            if is_info:
+                is_info = False
+                line_toks = line.split()
+                # Wavelength in Angstrom
+                try:
+                    value = float(line_toks[1])
+                    if self.has_converter and \
+                            self.current_datainfo.source.wavelength_unit != 'A':
+                        conv = Converter('A')
+                        self.current_datainfo.source.wavelength = conv(value,
+                            units=self.current_datainfo.source.wavelength_unit)
+                    else:
+                        self.current_datainfo.source.wavelength = value
+                except KeyError:
+                    msg = "ABSReader cannot read wavelength from %s" % filepath
+                    self.current_datainfo.errors.append(msg)
+                # Detector distance in meters
+                try:
+                    value = float(line_toks[3])
+                    if self.has_converter and detector.distance_unit != 'm':
+                        conv = Converter('m')
+                        detector.distance = conv(value,
+                                        units=detector.distance_unit)
+                    else:
+                        detector.distance = value
                 except:
+                    raise  RuntimeError, "abs_reader: cannot open %s" % path
+                buff = input_f.read()
+                lines = buff.split('\n')
+                x  = np.zeros(0)
+                y  = np.zeros(0)
+                dy = np.zeros(0)
+                dx = np.zeros(0)
+                output = Data1D(x, y, dy=dy, dx=dx)
+                detector = Detector()
+                output.detector.append(detector)
+                output.filename = basename
+                is_info = False
+                    msg = "ABSReader cannot read SDD from %s" % filepath
+                    self.current_datainfo.errors.append(msg)
+                # Transmission
+                try:
+                    self.current_datainfo.sample.transmission = \
+                        float(line_toks[4])
+                except ValueError:
+                    # Transmission isn't always in the header
+                    pass
+                # Sample thickness in mm
+                try:
+                    value = float(line_toks[5])
+                    if self.has_converter and \
+                            self.current_datainfo.sample.thickness_unit != 'cm':
+                        conv = Converter('cm')
+                        self.current_datainfo.sample.thickness = conv(value,
+                            units=self.current_datainfo.sample.thickness_unit)
+                    else:
+                        self.current_datainfo.sample.thickness = value
+                except ValueError:
+                    # Thickness is not a mandatory entry
+                    pass
+            # MON CNT  LAMBDA  DET ANG  DET DIST  TRANS  THICK  AVE   STEP
+            if line.count("LAMBDA") > 0:
+                is_info = True
+            # Find center info line
+            if is_center:
                 is_center = False
+                is_data_started = False
+                data_conv_q = None
+                data_conv_i = None
+                if has_converter == True and output.x_unit != '1/A':
+                    data_conv_q = Converter('1/A')
+                    # Test it
+                    data_conv_q(1.0, output.x_unit)
+                if has_converter == True and output.y_unit != '1/cm':
+                    data_conv_i = Converter('1/cm')
+                    # Test it
+                    data_conv_i(1.0, output.y_unit)
+                for line in lines:
+                    # Information line 1
+                    if is_info == True:
+                        is_info = False
+                        line_toks = line.split()
+                        # Wavelength in Angstrom
+                        try:
+                            value = float(line_toks[1])
+                            if has_converter == True and \
+                                output.source.wavelength_unit != 'A':
+                                conv = Converter('A')
+                                output.source.wavelength = conv(value,
+                                        units=output.source.wavelength_unit)
+                            else:
+                                output.source.wavelength = value
+                        except:
+                            #goes to ASC reader
+                            msg = "abs_reader: cannot open %s" % path
+                            raise  RuntimeError, msg
+                        # Distance in meters
+                        try:
+                            value = float(line_toks[3])
+                            if has_converter == True and \
+                                detector.distance_unit != 'm':
+                                conv = Converter('m')
+                                detector.distance = conv(value,
+                                                units=detector.distance_unit)
+                            else:
+                                detector.distance = value
+                        except:
+                            #goes to ASC reader
+                            msg = "abs_reader: cannot open %s" % path
+                            raise  RuntimeError, msg
+                        # Transmission
+                        try:
+                            output.sample.transmission = float(line_toks[4])
+                        except:
+                            # Transmission is not a mandatory entry
+                            pass
+                        # Thickness in mm
+                        try:
+                            value = float(line_toks[5])
+                            if has_converter == True and \
+                                output.sample.thickness_unit != 'cm':
+                                conv = Converter('cm')
+                                output.sample.thickness = conv(value,
+                                            units=output.sample.thickness_unit)
+                            else:
+                                output.sample.thickness = value
+                        except:
+                            # Thickness is not a mandatory entry
+                            pass
+                    #MON CNT   LAMBDA   DET ANG   DET DIST   TRANS   THICK
+                    #  AVE   STEP
+                    if line.count("LAMBDA") > 0:
+                        is_info = True
+                    # Find center info line
+                    if is_center == True:
+                        is_center = False
+                        line_toks = line.split()
+                        # Center in bin number
+                        center_x = float(line_toks[0])
+                        center_y = float(line_toks[1])
+                        # Bin size
+                        if has_converter == True and \
+                            detector.pixel_size_unit != 'mm':
+                            conv = Converter('mm')
+                            detector.pixel_size.x = conv(5.0,
+                                                units=detector.pixel_size_unit)
+                            detector.pixel_size.y = conv(5.0,
+                                                units=detector.pixel_size_unit)
+                        else:
+                            detector.pixel_size.x = 5.0
+                            detector.pixel_size.y = 5.0
+                        # Store beam center in distance units
+                        # Det 640 x 640 mm
+                        if has_converter == True and \
+                            detector.beam_center_unit != 'mm':
+                            conv = Converter('mm')
+                            detector.beam_center.x = conv(center_x * 5.0,
+                                             units=detector.beam_center_unit)
+                            detector.beam_center.y = conv(center_y * 5.0,
+                                            units=detector.beam_center_unit)
+                        else:
+                            detector.beam_center.x = center_x * 5.0
+                            detector.beam_center.y = center_y * 5.0
+                        # Detector type
+                        try:
+                            detector.name = line_toks[7]
+                        except:
+                            # Detector name is not a mandatory entry
+                            pass
+                    #BCENT(X,Y)   A1(mm)   A2(mm)   A1A2DIST(m)   DL/L
+                    #  BSTOP(mm)   DET_TYP
+                    if line.count("BCENT") > 0:
+                        is_center = True
+                    # Parse the data
+                    if is_data_started == True:
+                        toks = line.split()
+                        try:
+                            _x  = float(toks[0])
+                            _y  = float(toks[1])
+                            _dy = float(toks[2])
+                            _dx = float(toks[3])
+                            if data_conv_q is not None:
+                                _x = data_conv_q(_x, units=output.x_unit)
+                                _dx = data_conv_i(_dx, units=output.x_unit)
+                            if data_conv_i is not None:
+                                _y = data_conv_i(_y, units=output.y_unit)
+                                _dy = data_conv_i(_dy, units=output.y_unit)
+                            x = np.append(x, _x)
+                            y = np.append(y, _y)
+                            dy = np.append(dy, _dy)
+                            dx = np.append(dx, _dx)
+                        except:
+                            # Could not read this data line. If we are here
+                            # it is because we are in the data section. Just
+                            # skip it.
+                            pass
+                    #The 6 columns are | Q (1/A) | I(Q) (1/cm) | std. dev.
+                    # I(Q) (1/cm) | sigmaQ | meanQ | ShadowFactor|
+                    if line.count("The 6 columns") > 0:
+                        is_data_started = True
+                # Sanity check
+                if not len(y) == len(dy):
+                    msg = "abs_reader: y and dy have different length"
+                    raise ValueError, msg
+                # If the data length is zero, consider this as
+                # though we were not able to read the file.
+                if len(x) == 0:
+                    raise ValueError, "ascii_reader: could not load file"
+                output.x = x[x != 0]
+                output.y = y[x != 0]
+                output.dy = dy[x != 0]
+                output.dx = dx[x != 0]
+                if data_conv_q is not None:
+                    output.xaxis("\\rm{Q}", output.x_unit)
+                line_toks = line.split()
+                # Center in bin number
+                center_x = float(line_toks[0])
+                center_y = float(line_toks[1])
+                # Bin size
+                if self.has_converter and detector.pixel_size_unit != 'mm':
+                    conv = Converter('mm')
+                    detector.pixel_size.x = conv(5.08,
+                                        units=detector.pixel_size_unit)
+                    detector.pixel_size.y = conv(5.08,
+                                        units=detector.pixel_size_unit)
                 else:
+                    output.xaxis("\\rm{Q}", 'A^{-1}')
+                if data_conv_i is not None:
+                    output.yaxis("\\rm{Intensity}", output.y_unit)
+                    detector.pixel_size.x = 5.08
+                    detector.pixel_size.y = 5.08
+                # Store beam center in distance units
+                # Det 640 x 640 mm
+                if self.has_converter and detector.beam_center_unit != 'mm':
+                    conv = Converter('mm')
+                    detector.beam_center.x = conv(center_x * 5.08,
+                                     units=detector.beam_center_unit)
+                    detector.beam_center.y = conv(center_y * 5.08,
+                                    units=detector.beam_center_unit)
                 else:
+                    output.yaxis("\\rm{Intensity}", "cm^{-1}")
+                # Store loading process information
+                output.meta_data['loader'] = self.type_name
+                return output
+                    detector.beam_center.x = center_x * 5.08
+                    detector.beam_center.y = center_y * 5.08
+                # Detector type
+                try:
+                    detector.name = line_toks[7]
+                except:
+                    # Detector name is not a mandatory entry
+                    pass
+            # BCENT(X,Y)  A1(mm)  A2(mm)  A1A2DIST(m)  DL/L  BSTOP(mm)  DET_TYP
+            if line.count("BCENT") > 0:
+                is_center = True
+            # Parse the data
+            if is_data_started:
+                toks = line.split()
+                try:
+                    _x = float(toks[0])
+                    _y = float(toks[1])
+                    _dy = float(toks[2])
+                    _dx = float(toks[3])
+                    if data_conv_q is not None:
+                        _x = data_conv_q(_x, units=base_q_unit)
+                        _dx = data_conv_q(_dx, units=base_q_unit)
+                    if data_conv_i is not None:
+                        _y = data_conv_i(_y, units=base_i_unit)
+                        _dy = data_conv_i(_dy, units=base_i_unit)
+                    self.current_dataset.x[data_line] = _x
+                    self.current_dataset.y[data_line] = _y
+                    self.current_dataset.dy[data_line] = _dy
+                    self.current_dataset.dx[data_line] = _dx
+                    data_line += 1
+                except ValueError:
+                    # Could not read this data line. If we are here
+                    # it is because we are in the data section. Just
+                    # skip it.
+                    pass
+            # The 6 columns are | Q (1/A) | I(Q) (1/cm) | std. dev.
+            # I(Q) (1/cm) | sigmaQ | meanQ | ShadowFactor|
+            if line.count("The 6 columns") > 0:
+                is_data_started = True
+        self.remove_empty_q_values(True, True)
+        # Sanity check
+        if not len(self.current_dataset.y) == len(self.current_dataset.dy):
+            self.set_all_to_none()
+            msg = "abs_reader: y and dy have different length"
+            raise ValueError(msg)
+        # If the data length is zero, consider this as
+        # though we were not able to read the file.
+        if len(self.current_dataset.x) == 0:
+            self.set_all_to_none()
+            raise ValueError("ascii_reader: could not load file")
+        if data_conv_q is not None:
+            self.current_dataset.xaxis("\\rm{Q}", base_q_unit)
         else:
+            raise RuntimeError, "%s is not a file" % path
+        return None
+            self.current_dataset.xaxis("\\rm{Q}", 'A^{-1}')
+        if data_conv_i is not None:
+            self.current_dataset.yaxis("\\rm{Intensity}", base_i_unit)
+        else:
+            self.current_dataset.yaxis("\\rm{Intensity}", "cm^{-1}")
+        # Store loading process information
+        self.current_datainfo.meta_data['loader'] = self.type_name
+        self.send_to_output()

src/sas/sascalc/dataloader/readers/anton_paar_saxs_reader.py

-                      ra235f715
+                      rfafe52a
 from sas.sascalc.dataloader.readers.xml_reader import XMLreader
 from sas.sascalc.dataloader.data_info import plottable_1D, Data1D, Sample, Source
+from sas.sascalc.dataloader.data_info import plottable_1D, Data1D, DataInfo, Sample, Source
 from sas.sascalc.dataloader.data_info import Process, Aperture, Collimation, TransmissionSpectrum, Detector
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException, DataReaderException
 class Reader(XMLreader):
     """
+    A class for reading in CanSAS v2.0 data files. The existing iteration opens Mantid generated HDF5 formatted files
+    with file extension .h5/.H5. Any number of data sets may be present within the file and any dimensionality of data
+    may be used. Currently 1D and 2D SAS data sets are supported, but future implementations will include 1D and 2D
+    SESANS data. This class assumes a single data set for each sasentry.
+    :Dependencies:
+        The CanSAS HDF5 reader requires h5py v2.5.0 or later.
+    A class for reading in Anton Paar .pdh files
     """
 …
     ## Raw file contents to be processed
     raw_data = None
-    ## Data set being modified
-    current_dataset = None
     ## For recursion and saving purposes, remember parent objects
     parent_list = None
 …
     ## Flag to bypass extension check
     allow_all = False
-    ## List of files to return
-    output = None
     def reset_state(self):
         self.current_dataset = Data1D(np.empty(0), np.empty(0),
                                             np.empty(0), np.empty(0))
+        self.current_dataset = plottable_1D(np.empty(0), np.empty(0), np.empty(0), np.empty(0))
+        self.current_datainfo = DataInfo()
         self.datasets = []
         self.raw_data = None
 …
         self.lower = 5
     def read(self, filename):
+    def get_file_contents(self):
         """
             This is the general read method that all SasView data_loaders must have.
 …
         ## Reinitialize the class when loading a new data file to reset all class variables
         self.reset_state()
+        ## Check that the file exists
+        if os.path.isfile(filename):
+            basename = os.path.basename(filename)
+            _, extension = os.path.splitext(basename)
+            # If the file type is not allowed, return empty list
+            if extension in self.ext or self.allow_all:
+                ## Load the data file
+                input_f = open(filename, 'r')
+                buff = input_f.read()
+                self.raw_data = buff.splitlines()
+                self.read_data()
+        return self.output
+        buff = self.f_open.read()
+        self.raw_data = buff.splitlines()
+        self.read_data()
     def read_data(self):
+        correctly_loaded = True
+        error_message = ""
         q_unit = "1/nm"
         i_unit = "1/um^2"
+        self.current_dataset.title = self.raw_data[0]
+        self.current_dataset.meta_data["Keywords"] = self.raw_data[1]
+        line3 = self.raw_data[2].split()
+        line4 = self.raw_data[3].split()
+        line5 = self.raw_data[4].split()
+        self.data_points = int(line3[0])
+        self.lower = 5
+        self.upper = self.lower + self.data_points
+        self.source.radiation = 'x-ray'
+        normal = float(line4[3])
+        self.current_dataset.source.radiation = "x-ray"
+        self.current_dataset.source.name = "Anton Paar SAXSess Instrument"
+        self.current_dataset.source.wavelength = float(line4[4])
+        xvals = []
+        yvals = []
+        dyvals = []
+        for i in range(self.lower, self.upper):
+            index = i - self.lower
+            data = self.raw_data[i].split()
+            xvals.insert(index, normal * float(data[0]))
+            yvals.insert(index, normal * float(data[1]))
+            dyvals.insert(index, normal * float(data[2]))
+        try:
+            self.current_datainfo.title = self.raw_data[0]
+            self.current_datainfo.meta_data["Keywords"] = self.raw_data[1]
+            line3 = self.raw_data[2].split()
+            line4 = self.raw_data[3].split()
+            line5 = self.raw_data[4].split()
+            self.data_points = int(line3[0])
+            self.lower = 5
+            self.upper = self.lower + self.data_points
+            self.source.radiation = 'x-ray'
+            normal = float(line4[3])
+            self.current_datainfo.source.radiation = "x-ray"
+            self.current_datainfo.source.name = "Anton Paar SAXSess Instrument"
+            self.current_datainfo.source.wavelength = float(line4[4])
+            xvals = []
+            yvals = []
+            dyvals = []
+            for i in range(self.lower, self.upper):
+                index = i - self.lower
+                data = self.raw_data[i].split()
+                xvals.insert(index, normal * float(data[0]))
+                yvals.insert(index, normal * float(data[1]))
+                dyvals.insert(index, normal * float(data[2]))
+        except Exception as e:
+            error_message = "Couldn't load {}.\n".format(self.f_open.name)
+            error_message += e.message
+            raise FileContentsException(error_message)
         self.current_dataset.x = np.append(self.current_dataset.x, xvals)
         self.current_dataset.y = np.append(self.current_dataset.y, yvals)
         self.current_dataset.dy = np.append(self.current_dataset.dy, dyvals)
         if self.data_points != self.current_dataset.x.size:
+            self.errors.add("Not all data was loaded properly.")
+        if self.current_dataset.dx.size != self.current_dataset.x.size:
+            dxvals = np.zeros(self.current_dataset.x.size)
+            self.current_dataset.dx = dxvals
+            error_message += "Not all data points could be loaded.\n"
+            correctly_loaded = False
         if self.current_dataset.x.size != self.current_dataset.y.size:
+            self.errors.add("The x and y data sets are not the same size.")
+            error_message += "The x and y data sets are not the same size.\n"
+            correctly_loaded = False
         if self.current_dataset.y.size != self.current_dataset.dy.size:
+            self.errors.add("The y and dy datasets are not the same size.")
+        self.current_dataset.errors = self.errors
+            error_message += "The y and dy datasets are not the same size.\n"
+            correctly_loaded = False
         self.current_dataset.xaxis("Q", q_unit)
         self.current_dataset.yaxis("Intensity", i_unit)
         xml_intermediate = self.raw_data[self.upper:]
         xml = ''.join(xml_intermediate)
+        self.set_xml_string(xml)
+        dom = self.xmlroot.xpath('/fileinfo')
+        self._parse_child(dom)
+        self.output.append(self.current_dataset)
+        try:
+            self.set_xml_string(xml)
+            dom = self.xmlroot.xpath('/fileinfo')
+            self._parse_child(dom)
+        except Exception as e:
+            # Data loaded but XML metadata has an error
+            error_message += "Data points have been loaded but there was an "
+            error_message += "error reading XML metadata: " + e.message
+            correctly_loaded = False
+        self.send_to_output()
+        if not correctly_loaded:
+            raise DataReaderException(error_message)
     def _parse_child(self, dom, parent=''):
 …
                 self._parse_child(node, key)
                 if key == "SampleDetector":
                     self.current_dataset.detector.append(self.detector)
+                    self.current_datainfo.detector.append(self.detector)
                     self.detector = Detector()
             else:
                 if key == "value":
                     if parent == "Wavelength":
                         self.current_dataset.source.wavelength = value
+                        self.current_datainfo.source.wavelength = value
                     elif parent == "SampleDetector":
                         self.detector.distance = value
                     elif parent == "Temperature":
                         self.current_dataset.sample.temperature = value
+                        self.current_datainfo.sample.temperature = value
                     elif parent == "CounterSlitLength":
                         self.detector.slit_length = value
 …
                     value = value.replace("_", "")
                     if parent == "Wavelength":
                         self.current_dataset.source.wavelength_unit = value
+                        self.current_datainfo.source.wavelength_unit = value
                     elif parent == "SampleDetector":
                         self.detector.distance_unit = value
 …
                         self.current_dataset.yaxis(self.current_dataset._yaxis, value)
                     elif parent == "Temperature":
                         self.current_dataset.sample.temperature_unit = value
+                        self.current_datainfo.sample.temperature_unit = value
                     elif parent == "CounterSlitLength":
                         self.detector.slit_length_unit = value

src/sas/sascalc/dataloader/readers/ascii_reader.py

-                      r235f514
+                      rf994e8b1
 """
     ASCII reader
+    Generic multi-column ASCII data reader
 """
 ############################################################################
 #This software was developed by the University of Tennessee as part of the
 #Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
 #project funded by the US National Science Foundation.
 #If you use DANSE applications to do scientific research that leads to
 #publication, we ask that you acknowledge the use of the software with the
 #following sentence:
 #This work benefited from DANSE software developed under NSF award DMR-0520547.
 #copyright 2008, University of Tennessee
+# This software was developed by the University of Tennessee as part of the
+# Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
+# project funded by the US National Science Foundation.
+# If you use DANSE applications to do scientific research that leads to
+# publication, we ask that you acknowledge the use of the software with the
+# following sentence:
+# This work benefited from DANSE software developed under NSF award DMR-0520547.
+# copyright 2008, University of Tennessee
 #############################################################################
+import logging
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+from sas.sascalc.dataloader.data_info import DataInfo, plottable_1D
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException,\
+    DefaultReaderException
+import numpy as np
+import os
+from sas.sascalc.dataloader.data_info import Data1D
+# Check whether we have a converter available
+has_converter = True
+try:
+    from sas.sascalc.data_util.nxsunit import Converter
+except:
+    has_converter = False
+_ZERO = 1e-16
+logger = logging.getLogger(__name__)
 class Reader:
+class Reader(FileReader):
     """
     Class to load ascii files (2, 3 or 4 columns).
     """
     ## File type
+    # File type
     type_name = "ASCII"
+    ## Wildcards
+    # Wildcards
     type = ["ASCII files (*.txt)|*.txt",
             "ASCII files (*.dat)|*.dat",
             "ASCII files (*.abs)|*.abs",
             "CSV files (*.csv)|*.csv"]
+    ## List of allowed extensions
+    ext = ['.txt', '.TXT', '.dat', '.DAT', '.abs', '.ABS', 'csv', 'CSV']
+    # List of allowed extensions
+    ext = ['.txt', '.dat', '.abs', '.csv']
+    # Flag to bypass extension check
+    allow_all = True
+    # data unless that is the only data
+    min_data_pts = 5
+    ## Flag to bypass extension check
+    allow_all = True
+    def get_file_contents(self):
+        """
+        Get the contents of the file
+        """
+    def read(self, path):
+        """
+        Load data file
+        buff = self.f_open.read()
+        filepath = self.f_open.name
+        lines = buff.splitlines()
+        self.output = []
+        self.current_datainfo = DataInfo()
+        self.current_datainfo.filename = filepath
+        self.reset_data_list(len(lines))
+        :param path: file path
+        :return: Data1D object, or None
+        # The first good line of data will define whether
+        # we have 2-column or 3-column ascii
+        has_error_dx = None
+        has_error_dy = None
+        :raise RuntimeError: when the file can't be opened
+        :raise ValueError: when the length of the data vectors are inconsistent
+        """
+        if os.path.isfile(path):
+            basename = os.path.basename(path)
+            _, extension = os.path.splitext(basename)
+            if self.allow_all or extension.lower() in self.ext:
+                try:
+                    # Read in binary mode since GRASP frequently has no-ascii
+                    # characters that breaks the open operation
+                    input_f = open(path,'rb')
+                except:
+                    raise  RuntimeError, "ascii_reader: cannot open %s" % path
+                buff = input_f.read()
+                lines = buff.splitlines()
+        # Initialize counters for data lines and header lines.
+        is_data = False
+        # More than "5" lines of data is considered as actual
+        # To count # of current data candidate lines
+        candidate_lines = 0
+        # To count total # of previous data candidate lines
+        candidate_lines_previous = 0
+        # Current line number
+        line_no = 0
+        # minimum required number of columns of data
+        lentoks = 2
+        for line in lines:
+            toks = self.splitline(line.strip())
+            # To remember the number of columns in the current line of data
+            new_lentoks = len(toks)
+            try:
+                if new_lentoks == 0:
+                    # If the line is blank, skip and continue on
+                    # In case of breaks within data sets.
+                    continue
+                elif new_lentoks != lentoks and is_data:
+                    # If a footer is found, break the loop and save the data
+                    break
+                elif new_lentoks != lentoks and not is_data:
+                    # If header lines are numerical
+                    candidate_lines = 0
+                    self.reset_data_list(len(lines) - line_no)
+                # Arrays for data storage
+                tx = np.zeros(0)
+                ty = np.zeros(0)
+                tdy = np.zeros(0)
+                tdx = np.zeros(0)
+                self.current_dataset.x[candidate_lines] = float(toks[0])
+                # The first good line of data will define whether
+                # we have 2-column or 3-column ascii
+                if new_lentoks > 1:
+                    self.current_dataset.y[candidate_lines] = float(toks[1])
+                # If a 3rd row is present, consider it dy
+                if new_lentoks > 2:
+                    self.current_dataset.dy[candidate_lines] = \
+                        float(toks[2])
+                    has_error_dy = True
+                # If a 4th row is present, consider it dx
+                if new_lentoks > 3:
+                    self.current_dataset.dx[candidate_lines] = \
+                        float(toks[3])
+                    has_error_dx = True
+                candidate_lines += 1
+                # If 5 or more lines, this is considering the set data
+                if candidate_lines >= self.min_data_pts:
+                    is_data = True
+                if is_data and new_lentoks >= 8:
+                    msg = "This data looks like 2D ASCII data. Use the file "
+                    msg += "converter tool to convert it to NXcanSAS."
+                    raise FileContentsException(msg)
+                # To remember the # of columns on the current line
+                # for the next line of data
+                lentoks = new_lentoks
+                line_no += 1
+            except ValueError:
+                # ValueError is raised when non numeric strings conv. to float
+                # It is data and meet non - number, then stop reading
+                if is_data:
+                    break
+                # Delete the previously stored lines of data candidates if
+                # the list is not data
+                self.reset_data_list(len(lines) - line_no)
+                lentoks = 2
                 has_error_dx = None
                 has_error_dy = None
+                # Reset # of lines of data candidates
+                candidate_lines = 0
+        if not is_data:
+            self.set_all_to_none()
+            if self.extension in self.ext:
+                msg = "ASCII Reader error: Fewer than five Q data points found "
+                msg += "in {}.".format(filepath)
+                raise FileContentsException(msg)
+            else:
+                msg = "ASCII Reader could not load the file {}".format(filepath)
+                raise DefaultReaderException(msg)
+        # Sanity check
+        if has_error_dy and not len(self.current_dataset.y) == \
+                len(self.current_dataset.dy):
+            msg = "ASCII Reader error: Number of I and dI data points are"
+            msg += " different in {}.".format(filepath)
+            # TODO: Add error to self.current_datainfo.errors instead?
+            self.set_all_to_none()
+            raise FileContentsException(msg)
+        if has_error_dx and not len(self.current_dataset.x) == \
+                len(self.current_dataset.dx):
+            msg = "ASCII Reader error: Number of Q and dQ data points are"
+            msg += " different in {}.".format(filepath)
+            # TODO: Add error to self.current_datainfo.errors instead?
+            self.set_all_to_none()
+            raise FileContentsException(msg)
+                #Initialize counters for data lines and header lines.
+                is_data = False
+                # More than "5" lines of data is considered as actual
+                # data unless that is the only data
+                min_data_pts = 5
+                # To count # of current data candidate lines
+                candidate_lines = 0
+                # To count total # of previous data candidate lines
+                candidate_lines_previous = 0
+                #minimum required number of columns of data
+                lentoks = 2
+                for line in lines:
+                    toks = self.splitline(line)
+                    # To remember the # of columns in the current line of data
+                    new_lentoks = len(toks)
+                    try:
+                        if new_lentoks == 1 and not is_data:
+                            ## If only one item in list, no longer data
+                            raise ValueError
+                        elif new_lentoks == 0:
+                            ## If the line is blank, skip and continue on
+                            ## In case of breaks within data sets.
+                            continue
+                        elif new_lentoks != lentoks and is_data:
+                            ## If a footer is found, break the loop and save the data
+                            break
+                        elif new_lentoks != lentoks and not is_data:
+                            ## If header lines are numerical
+                            candidate_lines = 0
+                            candidate_lines_previous = 0
+        self.remove_empty_q_values(has_error_dx, has_error_dy)
+        self.current_dataset.xaxis("\\rm{Q}", 'A^{-1}')
+        self.current_dataset.yaxis("\\rm{Intensity}", "cm^{-1}")
+                        #Make sure that all columns are numbers.
+                        for colnum in range(len(toks)):
+                            # Any non-floating point values throw ValueError
+                            float(toks[colnum])
+                        candidate_lines += 1
+                        _x = float(toks[0])
+                        _y = float(toks[1])
+                        _dx = None
+                        _dy = None
+                        #If 5 or more lines, this is considering the set data
+                        if candidate_lines >= min_data_pts:
+                            is_data = True
+                        # If a 3rd row is present, consider it dy
+                        if new_lentoks > 2:
+                            _dy = float(toks[2])
+                        has_error_dy = False if _dy is None else True
+                        # If a 4th row is present, consider it dx
+                        if new_lentoks > 3:
+                            _dx = float(toks[3])
+                        has_error_dx = False if _dx is None else True
+                        # Delete the previously stored lines of data candidates if
+                        # the list is not data
+                        if candidate_lines == 1 and -1 < candidate_lines_previous < min_data_pts and \
+                            is_data == False:
+                            try:
+                                tx = np.zeros(0)
+                                ty = np.zeros(0)
+                                tdy = np.zeros(0)
+                                tdx = np.zeros(0)
+                            except:
+                                pass
+                        if has_error_dy == True:
+                            tdy = np.append(tdy, _dy)
+                        if has_error_dx == True:
+                            tdx = np.append(tdx, _dx)
+                        tx = np.append(tx, _x)
+                        ty = np.append(ty, _y)
+                        #To remember the # of columns on the current line
+                        # for the next line of data
+                        lentoks = new_lentoks
+                        candidate_lines_previous = candidate_lines
+                    except ValueError:
+                        # It is data and meet non - number, then stop reading
+                        if is_data == True:
+                            break
+                        lentoks = 2
+                        has_error_dx = None
+                        has_error_dy = None
+                        #Reset # of lines of data candidates
+                        candidate_lines = 0
+                    except:
+                        pass
+                input_f.close()
+                if not is_data:
+                    msg = "ascii_reader: x has no data"
+                    raise RuntimeError, msg
+                # Sanity check
+                if has_error_dy == True and not len(ty) == len(tdy):
+                    msg = "ascii_reader: y and dy have different length"
+                    raise RuntimeError, msg
+                if has_error_dx == True and not len(tx) == len(tdx):
+                    msg = "ascii_reader: y and dy have different length"
+                    raise RuntimeError, msg
+                # If the data length is zero, consider this as
+                # though we were not able to read the file.
+                if len(tx) == 0:
+                    raise RuntimeError, "ascii_reader: could not load file"
+                #Let's re-order the data to make cal.
+                # curve look better some cases
+                ind = np.lexsort((ty, tx))
+                x = np.zeros(len(tx))
+                y = np.zeros(len(ty))
+                dy = np.zeros(len(tdy))
+                dx = np.zeros(len(tdx))
+                output = Data1D(x, y, dy=dy, dx=dx)
+                self.filename = output.filename = basename
+                for i in ind:
+                    x[i] = tx[ind[i]]
+                    y[i] = ty[ind[i]]
+                    if has_error_dy == True:
+                        dy[i] = tdy[ind[i]]
+                    if has_error_dx == True:
+                        dx[i] = tdx[ind[i]]
+                # Zeros in dx, dy
+                if has_error_dx:
+                    dx[dx == 0] = _ZERO
+                if has_error_dy:
+                    dy[dy == 0] = _ZERO
+                #Data
+                output.x = x[x != 0]
+                output.y = y[x != 0]
+                output.dy = dy[x != 0] if has_error_dy == True\
+                    else np.zeros(len(output.y))
+                output.dx = dx[x != 0] if has_error_dx == True\
+                    else np.zeros(len(output.x))
+                output.xaxis("\\rm{Q}", 'A^{-1}')
+                output.yaxis("\\rm{Intensity}", "cm^{-1}")
+                # Store loading process information
+                output.meta_data['loader'] = self.type_name
+                if len(output.x) < 1:
+                    raise RuntimeError, "%s is empty" % path
+                return output
+        else:
+            raise RuntimeError, "%s is not a file" % path
+        return None
+    def splitline(self, line):
+        """
+        Splits a line into pieces based on common delimeters
+        :param line: A single line of text
+        :return: list of values
+        """
+        # Initial try for CSV (split on ,)
+        toks = line.split(',')
+        # Now try SCSV (split on ;)
+        if len(toks) < 2:
+            toks = line.split(';')
+        # Now go for whitespace
+        if len(toks) < 2:
+            toks = line.split()
+        return toks
+        # Store loading process information
+        self.current_datainfo.meta_data['loader'] = self.type_name
+        self.send_to_output()

src/sas/sascalc/dataloader/readers/associations.py

-                      ra1b8fee
+                      rce8c7bd
 #copyright 2009, University of Tennessee
 #############################################################################
-from __future__ import print_function
-import os
 import sys
 import logging
-import json
 logger = logging.getLogger(__name__)
+FILE_NAME = 'defaults.json'
+FILE_ASSOCIATIONS = {
+    ".xml": "cansas_reader",
+    ".ses": "sesans_reader",
+    ".h5": "cansas_reader_HDF5",
+    ".txt": "ascii_reader",
+    ".dat": "red2d_reader",
+    ".abs": "abs_reader",
+    ".sans": "danse_reader",
+    ".pdh": "anton_paar_saxs_reader"
+}
+def read_associations(loader, settings=FILE_NAME):
+def read_associations(loader, settings=FILE_ASSOCIATIONS):
     """
     Read the specified settings file to associate
     default readers to file extension.
     :param loader: Loader object
     :param settings: path to the json settings file [string]
     """
+    reader_dir = os.path.dirname(__file__)
+    path = os.path.join(reader_dir, settings)
+    # If we can't find the file in the installation
+    # directory, look into the execution directory.
+    if not os.path.isfile(path):
+        path = os.path.join(os.getcwd(), settings)
+    if not os.path.isfile(path):
+        path = os.path.join(sys.path[0], settings)
+    if not os.path.isfile(path):
+        path = settings
+    if not os.path.isfile(path):
+        path = "./%s" % settings
+    if os.path.isfile(path):
+        with open(path) as fh:
+            json_tree = json.load(fh)
+        # Read in the file extension associations
+        entry_list = json_tree['SasLoader']['FileType']
+        # For each FileType entry, get the associated reader and extension
+        for entry in entry_list:
+            reader = entry['-reader']
+            ext = entry['-extension']
+            if reader is not None and ext is not None:
+                # Associate the extension with a particular reader
+                # TODO: Modify the Register code to be case-insensitive
+                # and remove the extra line below.
+                try:
+                    exec "import %s" % reader
+                    exec "loader.associate_file_type('%s', %s)" % (ext.lower(),
+                                                                    reader)
+                    exec "loader.associate_file_type('%s', %s)" % (ext.upper(),
+                                                                    reader)
+                except:
+                    msg = "read_associations: skipping association"
+                    msg += " for %s\n  %s" % (ext.lower(), sys.exc_value)
+                    logger.error(msg)
+    else:
+        print("Could not find reader association settings\n  %s [%s]" % (__file__, os.getcwd()))
+def register_readers(registry_function):
+    """
+    Function called by the registry/loader object to register
+    all default readers using a call back function.
+    :WARNING: this method is now obsolete
+    :param registry_function: function to be called to register each reader
+    """
+    logger.info("register_readers is now obsolete: use read_associations()")
+    import abs_reader
+    import ascii_reader
+    import cansas_reader
+    import danse_reader
+    import hfir1d_reader
+    import IgorReader
+    import red2d_reader
+    #import tiff_reader
+    import nexus_reader
+    import sesans_reader
+    import cansas_reader_HDF5
+    import anton_paar_saxs_reader
+    registry_function(sesans_reader)
+    registry_function(abs_reader)
+    registry_function(ascii_reader)
+    registry_function(cansas_reader)
+    registry_function(danse_reader)
+    registry_function(hfir1d_reader)
+    registry_function(IgorReader)
+    registry_function(red2d_reader)
+    #registry_function(tiff_reader)
+    registry_function(nexus_reader)
+    registry_function(cansas_reader_HDF5)
+    registry_function(anton_paar_saxs_reader)
+    return True
+    # For each FileType entry, get the associated reader and extension
+    for ext, reader in settings.iteritems():
+        if reader is not None and ext is not None:
+            # Associate the extension with a particular reader
+            # TODO: Modify the Register code to be case-insensitive
+            # FIXME: Remove exec statements
+            # and remove the extra line below.
+            try:
+                exec "import %s" % reader
+                exec "loader.associate_file_type('%s', %s)" % (ext.lower(),
+                                                                reader)
+                exec "loader.associate_file_type('%s', %s)" % (ext.upper(),
+                                                                reader)
+            except:
+                msg = "read_associations: skipping association"
+                msg += " for %s\n  %s" % (ext.lower(), sys.exc_value)
+                logger.error(msg)

src/sas/sascalc/dataloader/readers/cansas_reader.py

-                      r7432acb
+                      r248ff73
-"""
-    CanSAS data reader - new recursive cansas_version.
-"""
-############################################################################
-#This software was developed by the University of Tennessee as part of the
-#Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
-#project funded by the US National Science Foundation.
-#If you use DANSE applications to do scientific research that leads to
-#publication, we ask that you acknowledge the use of the software with the
-#following sentence:
-#This work benefited from DANSE software developed under NSF award DMR-0520547.
-#copyright 2008,2009 University of Tennessee
-#############################################################################
 import logging
 import numpy as np
 …
 from sas.sascalc.dataloader.readers.xml_reader import XMLreader
 from sas.sascalc.dataloader.readers.cansas_constants import CansasConstants, CurrentLevel
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException, \
+    DefaultReaderException, DataReaderException
 # The following 2 imports *ARE* used. Do not remove either.
 import xml.dom.minidom
 from xml.dom.minidom import parseString
+from lxml import etree
 logger = logging.getLogger(__name__)
 …
 class Reader(XMLreader):
-    """
-    Class to load cansas 1D XML files
-    :Dependencies:
-        The CanSAS reader requires PyXML 0.8.4 or later.
-    """
-    # CanSAS version - defaults to version 1.0
     cansas_version = "1.0"
     base_ns = "{cansas1d/1.0}"
 …
     ns_list = None
     # Temporary storage location for loading multiple data sets in a single file
-    current_datainfo = None
-    current_dataset = None
     current_data1d = None
     data = None
-    # List of data1D objects to be sent back to SasView
-    output = None
     # Wildcards
     type = ["XML files (*.xml)|*.xml", "SasView Save Files (*.svs)|*.svs"]
 …
     def read(self, xml_file, schema_path="", invalid=True):
+        """
+        Validate and read in an xml_file file in the canSAS format.
+        :param xml_file: A canSAS file path in proper XML format
+        :param schema_path: A file path to an XML schema to validate the xml_file against
+        """
+        # For every file loaded, reset everything to a base state
+        if schema_path != "" or invalid != True:
+            # read has been called from self.get_file_contents because xml file doens't conform to schema
+            _, self.extension = os.path.splitext(os.path.basename(xml_file))
+            return self.get_file_contents(xml_file=xml_file, schema_path=schema_path, invalid=invalid)
+        # Otherwise, read has been called by the data loader - file_reader_base_class handles this
+        return super(XMLreader, self).read(xml_file)
+    def get_file_contents(self, xml_file=None, schema_path="", invalid=True):
+        # Reset everything since we're loading a new file
         self.reset_state()
         self.invalid = invalid
+        # Check that the file exists
+        if os.path.isfile(xml_file):
+            basename, extension = os.path.splitext(os.path.basename(xml_file))
+            # If the file type is not allowed, return nothing
+            if extension in self.ext or self.allow_all:
+                # Get the file location of
+                self.load_file_and_schema(xml_file, schema_path)
+                self.add_data_set()
+                # Try to load the file, but raise an error if unable to.
+                # Check the file matches the XML schema
+        if xml_file is None:
+            xml_file = self.f_open.name
+        # We don't sure f_open since lxml handles opnening/closing files
+        if not self.f_open.closed:
+            self.f_open.close()
+        basename, _ = os.path.splitext(os.path.basename(xml_file))
+        try:
+            # Raises FileContentsException
+            self.load_file_and_schema(xml_file, schema_path)
+            self.current_datainfo = DataInfo()
+            # Raises FileContentsException if file doesn't meet CanSAS schema
+            self.is_cansas(self.extension)
+            self.invalid = False # If we reach this point then file must be valid CanSAS
+            # Parse each SASentry
+            entry_list = self.xmlroot.xpath('/ns:SASroot/ns:SASentry', namespaces={
+                'ns': self.cansas_defaults.get("ns")
+            })
+            # Look for a SASentry
+            self.names.append("SASentry")
+            self.set_processing_instructions()
+            for entry in entry_list:
+                self.current_datainfo.filename = basename + self.extension
+                self.current_datainfo.meta_data["loader"] = "CanSAS XML 1D"
+                self.current_datainfo.meta_data[PREPROCESS] = self.processing_instructions
+                self._parse_entry(entry)
+                has_error_dx = self.current_dataset.dx is not None
+                has_error_dy = self.current_dataset.dy is not None
+                self.remove_empty_q_values(has_error_dx=has_error_dx,
+                    has_error_dy=has_error_dy)
+                self.send_to_output() # Combine datasets with DataInfo
+                self.current_datainfo = DataInfo() # Reset DataInfo
+        except FileContentsException as fc_exc:
+            # File doesn't meet schema - try loading with a less strict schema
+            base_name = xml_reader.__file__
+            base_name = base_name.replace("\\", "/")
+            base = base_name.split("/sas/")[0]
+            if self.cansas_version == "1.1":
+                invalid_schema = INVALID_SCHEMA_PATH_1_1.format(base, self.cansas_defaults.get("schema"))
+            else:
+                invalid_schema = INVALID_SCHEMA_PATH_1_0.format(base, self.cansas_defaults.get("schema"))
+            self.set_schema(invalid_schema)
+            if self.invalid:
                 try:
+                    self.is_cansas(extension)
+                    self.invalid = False
+                    # Get each SASentry from XML file and add it to a list.
+                    entry_list = self.xmlroot.xpath(
+                            '/ns:SASroot/ns:SASentry',
+                            namespaces={'ns': self.cansas_defaults.get("ns")})
+                    self.names.append("SASentry")
+                    # Get all preprocessing events and encoding
+                    self.set_processing_instructions()
+                    # Parse each <SASentry> item
+                    for entry in entry_list:
+                        # Create a new DataInfo object for every <SASentry>
+                        # Set the file name and then parse the entry.
+                        self.current_datainfo.filename = basename + extension
+                        self.current_datainfo.meta_data["loader"] = "CanSAS XML 1D"
+                        self.current_datainfo.meta_data[PREPROCESS] = \
+                            self.processing_instructions
+                        # Parse the XML SASentry
+                        self._parse_entry(entry)
+                        # Combine datasets with datainfo
+                        self.add_data_set()
+                except RuntimeError:
+                    # If the file does not match the schema, raise this error
+                    # Load data with less strict schema
+                    self.read(xml_file, invalid_schema, False)
+                    # File can still be read but doesn't match schema, so raise exception
+                    self.load_file_and_schema(xml_file) # Reload strict schema so we can find where error are in file
                     invalid_xml = self.find_invalid_xml()
+                    invalid_xml = INVALID_XML.format(basename + extension) + invalid_xml
+                    self.errors.add(invalid_xml)
+                    # Try again with an invalid CanSAS schema, that requires only a data set in each
+                    base_name = xml_reader.__file__
+                    base_name = base_name.replace("\\", "/")
+                    base = base_name.split("/sas/")[0]
+                    if self.cansas_version == "1.1":
+                        invalid_schema = INVALID_SCHEMA_PATH_1_1.format(base, self.cansas_defaults.get("schema"))
+                    else:
+                        invalid_schema = INVALID_SCHEMA_PATH_1_0.format(base, self.cansas_defaults.get("schema"))
+                    self.set_schema(invalid_schema)
+                    try:
+                        if self.invalid:
+                            if self.is_cansas():
+                                self.output = self.read(xml_file, invalid_schema, False)
+                            else:
+                                raise RuntimeError
+                        else:
+                            raise RuntimeError
+                    except RuntimeError:
+                        x = np.zeros(1)
+                        y = np.zeros(1)
+                        self.current_data1d = Data1D(x,y)
+                        self.current_data1d.errors = self.errors
+                        return [self.current_data1d]
+        else:
+            self.output.append("Not a valid file path.")
+        # Return a list of parsed entries that dataloader can manage
+        return self.output
+                    invalid_xml = INVALID_XML.format(basename + self.extension) + invalid_xml
+                    raise DataReaderException(invalid_xml) # Handled by base class
+                except FileContentsException as fc_exc:
+                    if not self.extension in self.ext: # If the file has no associated loader
+                        raise DefaultReaderException(msg)
+                    msg = "CanSAS Reader could not load the file {}".format(xml_file)
+                    if fc_exc.message is not None: # Propagate error messages from earlier
+                        msg = fc_exc.message
+                    raise FileContentsException(msg)
+                    pass
+            else:
+                raise fc_exc
+        except Exception as e: # Convert all other exceptions to FileContentsExceptions
+            raise FileContentsException(e.message)
+    def load_file_and_schema(self, xml_file, schema_path=""):
+        base_name = xml_reader.__file__
+        base_name = base_name.replace("\\", "/")
+        base = base_name.split("/sas/")[0]
+        # Try and parse the XML file
+        try:
+            self.set_xml_file(xml_file)
+        except etree.XMLSyntaxError: # File isn't valid XML so can't be loaded
+            msg = "Cansas cannot load {}.\n Invalid XML syntax".format(xml_file)
+            raise FileContentsException(msg)
+        self.cansas_version = self.xmlroot.get("version", "1.0")
+        self.cansas_defaults = CANSAS_NS.get(self.cansas_version, "1.0")
+        if schema_path == "":
+            schema_path = "{}/sas/sascalc/dataloader/readers/schema/{}".format(
+                base, self.cansas_defaults.get("schema").replace("\\", "/")
+            )
+        self.set_schema(schema_path)
+    def is_cansas(self, ext="xml"):
+        """
+        Checks to see if the XML file is a CanSAS file
+        :param ext: The file extension of the data file
+        :raises FileContentsException: Raised if XML file isn't valid CanSAS
+        """
+        if self.validate_xml(): # Check file is valid XML
+            name = "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation"
+            value = self.xmlroot.get(name)
+            # Check schema CanSAS version matches file CanSAS version
+            if CANSAS_NS.get(self.cansas_version).get("ns") == value.rsplit(" ")[0]:
+                return True
+        if ext == "svs":
+            return True # Why is this required?
+        # If we get to this point then file isn't valid CanSAS
+        raise FileContentsException("The file is not valid CanSAS")
     def _parse_entry(self, dom, recurse=False):
-        """
-        Parse a SASEntry - new recursive method for parsing the dom of
-            the CanSAS data format. This will allow multiple data files
-            and extra nodes to be read in simultaneously.
-        :param dom: dom object with a namespace base of names
-        """
         if not self._is_call_local() and not recurse:
             self.reset_state()
+            self.add_data_set()
+            self.data = []
+            self.current_datainfo = DataInfo()
             self.names.append("SASentry")
             self.parent_class = "SASentry"
+        self._check_for_empty_data()
+        self.base_ns = "{0}{1}{2}".format("{", \
+                            CANSAS_NS.get(self.cansas_version).get("ns"), "}")
+        # Go through each child in the parent element
+        # Create an empty dataset if no data has been passed to the reader
+        if self.current_dataset is None:
+            self.current_dataset = plottable_1D(np.empty(0), np.empty(0),
+                np.empty(0), np.empty(0))
+        self.base_ns = "{" + CANSAS_NS.get(self.cansas_version).get("ns") + "}"
+        # Loop through each child in the parent element
         for node in dom:
             attr = node.attrib
 …
             if tagname == "fitting_plug_in" or tagname == "pr_inversion" or tagname == "invariant":
                 continue
             # Get where to store content
             self.names.append(tagname_original)
 …
                         else:
                             self.current_dataset.shape = ()
                 # Recursion step to access data within the group
                 self._parse_entry(node, True)
+                # Recurse to access data within the group
+                self._parse_entry(node, recurse=True)
                 if tagname == "SASsample":
                     self.current_datainfo.sample.name = name
 …
                     self.aperture.name = name
                     self.aperture.type = type
                 self.add_intermediate()
+                self._add_intermediate()
             else:
                 if isinstance(self.current_dataset, plottable_2D):
 …
                     self.current_datainfo.notes.append(data_point)
                 # I and Q - 1D data
+                # I and Q points
                 elif tagname == 'I' and isinstance(self.current_dataset, plottable_1D):
                     unit_list = unit.split("|")
 …
                     self.current_dataset.dx = np.append(self.current_dataset.dx, data_point)
                 elif tagname == 'dQw':
+                    if self.current_dataset.dqw is None: self.current_dataset.dqw = np.empty(0)
                     self.current_dataset.dxw = np.append(self.current_dataset.dxw, data_point)
                 elif tagname == 'dQl':
+                    if self.current_dataset.dxl is None: self.current_dataset.dxl = np.empty(0)
                     self.current_dataset.dxl = np.append(self.current_dataset.dxl, data_point)
                 elif tagname == 'Qmean':
 …
                 elif tagname == 'name' and self.parent_class == 'SASinstrument':
                     self.current_datainfo.instrument = data_point
                 # Detector Information
                 elif tagname == 'name' and self.parent_class == 'SASdetector':
 …
                     self.detector.orientation.z = data_point
                     self.detector.orientation_unit = unit
                 # Collimation and Aperture
                 elif tagname == 'length' and self.parent_class == 'SAScollimation':
 …
                 elif tagname == 'term' and self.parent_class == 'SASprocess':
                     unit = attr.get("unit", "")
+                    dic = {}
+                    dic["name"] = name
+                    dic["value"] = data_point
+                    dic["unit"] = unit
+                    dic = { "name": name, "value": data_point, "unit": unit }
                     self.process.term.append(dic)
 …
         if not self._is_call_local() and not recurse:
             self.frm = ""
+            self.add_data_set()
+            self.current_datainfo.errors = set()
+            for error in self.errors:
+                self.current_datainfo.errors.add(error)
+            self.errors.clear()
+            self.send_to_output()
             empty = None
             return self.output[0], empty
     def _is_call_local(self):
-        """
-        """
         if self.frm == "":
             inter = inspect.stack()
 …
         return True
+    def is_cansas(self, ext="xml"):
+        """
+        Checks to see if the xml file is a CanSAS file
+        :param ext: The file extension of the data file
+        """
+        if self.validate_xml():
+            name = "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation"
+            value = self.xmlroot.get(name)
+            if CANSAS_NS.get(self.cansas_version).get("ns") == \
+                    value.rsplit(" ")[0]:
+                return True
+        if ext == "svs":
+            return True
+        raise RuntimeError
+    def load_file_and_schema(self, xml_file, schema_path=""):
+        """
+        Loads the file and associates a schema, if a schema is passed in or if one already exists
+        :param xml_file: The xml file path sent to Reader.read
+        :param schema_path: The path to a schema associated with the xml_file, or find one based on the file
+        """
+        base_name = xml_reader.__file__
+        base_name = base_name.replace("\\", "/")
+        base = base_name.split("/sas/")[0]
+        # Load in xml file and get the cansas version from the header
+        self.set_xml_file(xml_file)
+        self.cansas_version = self.xmlroot.get("version", "1.0")
+        # Generic values for the cansas file based on the version
+        self.cansas_defaults = CANSAS_NS.get(self.cansas_version, "1.0")
+        if schema_path == "":
+            schema_path = "{0}/sas/sascalc/dataloader/readers/schema/{1}".format \
+                (base, self.cansas_defaults.get("schema")).replace("\\", "/")
+        # Link a schema to the XML file.
+        self.set_schema(schema_path)
+    def add_data_set(self):
+        """
+        Adds the current_dataset to the list of outputs after preforming final processing on the data and then calls a
+        private method to generate a new data set.
+        :param key: NeXus group name for current tree level
+        """
+        if self.current_datainfo and self.current_dataset:
+            self._final_cleanup()
+        self.data = []
+        self.current_datainfo = DataInfo()
+    def _initialize_new_data_set(self, node=None):
+        """
+        A private class method to generate a new 1D data object.
+        Outside methods should call add_data_set() to be sure any existing data is stored properly.
+        :param node: XML node to determine if 1D or 2D data
+        """
+        x = np.array(0)
+        y = np.array(0)
+        for child in node:
+            if child.tag.replace(self.base_ns, "") == "Idata":
+                for i_child in child:
+                    if i_child.tag.replace(self.base_ns, "") == "Qx":
+                        self.current_dataset = plottable_2D()
+                        return
+        self.current_dataset = plottable_1D(x, y)
+    def add_intermediate(self):
+    def _add_intermediate(self):
         """
         This method stores any intermediate objects within the final data set after fully reading the set.
+        :param parent: The NXclass name for the h5py Group object that just finished being processed
+        """
+        """
         if self.parent_class == 'SASprocess':
             self.current_datainfo.process.append(self.process)
 …
             self._check_for_empty_resolution()
             self.data.append(self.current_dataset)
-    def _final_cleanup(self):
-        """
-        Final cleanup of the Data1D object to be sure it has all the
-        appropriate information needed for perspectives
-        """
-        # Append errors to dataset and reset class errors
-        self.current_datainfo.errors = set()
-        for error in self.errors:
-            self.current_datainfo.errors.add(error)
-        self.errors.clear()
-        # Combine all plottables with datainfo and append each to output
-        # Type cast data arrays to float64 and find min/max as appropriate
-        for dataset in self.data:
-            if isinstance(dataset, plottable_1D):
-                if dataset.x is not None:
-                    dataset.x = np.delete(dataset.x, [0])
-                    dataset.x = dataset.x.astype(np.float64)
-                    dataset.xmin = np.min(dataset.x)
-                    dataset.xmax = np.max(dataset.x)
-                if dataset.y is not None:
-                    dataset.y = np.delete(dataset.y, [0])
-                    dataset.y = dataset.y.astype(np.float64)
-                    dataset.ymin = np.min(dataset.y)
-                    dataset.ymax = np.max(dataset.y)
-                if dataset.dx is not None:
-                    dataset.dx = np.delete(dataset.dx, [0])
-                    dataset.dx = dataset.dx.astype(np.float64)
-                if dataset.dxl is not None:
-                    dataset.dxl = np.delete(dataset.dxl, [0])
-                    dataset.dxl = dataset.dxl.astype(np.float64)
-                if dataset.dxw is not None:
-                    dataset.dxw = np.delete(dataset.dxw, [0])
-                    dataset.dxw = dataset.dxw.astype(np.float64)
-                if dataset.dy is not None:
-                    dataset.dy = np.delete(dataset.dy, [0])
-                    dataset.dy = dataset.dy.astype(np.float64)
-                np.trim_zeros(dataset.x)
-                np.trim_zeros(dataset.y)
-                np.trim_zeros(dataset.dy)
-            elif isinstance(dataset, plottable_2D):
-                dataset.data = dataset.data.astype(np.float64)
-                dataset.qx_data = dataset.qx_data.astype(np.float64)
-                dataset.xmin = np.min(dataset.qx_data)
-                dataset.xmax = np.max(dataset.qx_data)
-                dataset.qy_data = dataset.qy_data.astype(np.float64)
-                dataset.ymin = np.min(dataset.qy_data)
-                dataset.ymax = np.max(dataset.qy_data)
-                dataset.q_data = np.sqrt(dataset.qx_data * dataset.qx_data
-                                         + dataset.qy_data * dataset.qy_data)
-                if dataset.err_data is not None:
-                    dataset.err_data = dataset.err_data.astype(np.float64)
-                if dataset.dqx_data is not None:
-                    dataset.dqx_data = dataset.dqx_data.astype(np.float64)
-                if dataset.dqy_data is not None:
-                    dataset.dqy_data = dataset.dqy_data.astype(np.float64)
-                if dataset.mask is not None:
-                    dataset.mask = dataset.mask.astype(dtype=bool)
-                if len(dataset.shape) == 2:
-                    n_rows, n_cols = dataset.shape
-                    dataset.y_bins = dataset.qy_data[0::int(n_cols)]
-                    dataset.x_bins = dataset.qx_data[:int(n_cols)]
-                    dataset.data = dataset.data.flatten()
-                else:
-                    dataset.y_bins = []
-                    dataset.x_bins = []
-                    dataset.data = dataset.data.flatten()
-            final_dataset = combine_data(dataset, self.current_datainfo)
-            self.output.append(final_dataset)
-    def _create_unique_key(self, dictionary, name, numb=0):
-        """
-        Create a unique key value for any dictionary to prevent overwriting
-        Recurse until a unique key value is found.
-        :param dictionary: A dictionary with any number of entries
-        :param name: The index of the item to be added to dictionary
-        :param numb: The number to be appended to the name, starts at 0
-        """
-        if dictionary.get(name) is not None:
-            numb += 1
-            name = name.split("_")[0]
-            name += "_{0}".format(numb)
-            name = self._create_unique_key(dictionary, name, numb)
-        return name
     def _get_node_value(self, node, tagname):
 …
         return node_value, value_unit
-    def _check_for_empty_data(self):
-        """
-        Creates an empty data set if no data is passed to the reader
-        :param data1d: presumably a Data1D object
-        """
-        if self.current_dataset is None:
-            x_vals = np.empty(0)
-            y_vals = np.empty(0)
-            dx_vals = np.empty(0)
-            dy_vals = np.empty(0)
-            dxl = np.empty(0)
-            dxw = np.empty(0)
-            self.current_dataset = plottable_1D(x_vals, y_vals, dx_vals, dy_vals)
-            self.current_dataset.dxl = dxl
-            self.current_dataset.dxw = dxw
     def _check_for_empty_resolution(self):
         """
+        A method to check all resolution data sets are the same size as I and Q
+        """
+        if isinstance(self.current_dataset, plottable_1D):
+            dql_exists = False
+            dqw_exists = False
+            dq_exists = False
+            di_exists = False
+            if self.current_dataset.dxl is not None:
+                dql_exists = True
+            if self.current_dataset.dxw is not None:
+                dqw_exists = True
+            if self.current_dataset.dx is not None:
+                dq_exists = True
+            if self.current_dataset.dy is not None:
+                di_exists = True
+            if dqw_exists and not dql_exists:
+                array_size = self.current_dataset.dxw.size - 1
+                self.current_dataset.dxl = np.append(self.current_dataset.dxl,
+                                                     np.zeros([array_size]))
+            elif dql_exists and not dqw_exists:
+                array_size = self.current_dataset.dxl.size - 1
+                self.current_dataset.dxw = np.append(self.current_dataset.dxw,
+                                                     np.zeros([array_size]))
+            elif not dql_exists and not dqw_exists and not dq_exists:
+                array_size = self.current_dataset.x.size - 1
+                self.current_dataset.dx = np.append(self.current_dataset.dx,
+                                                    np.zeros([array_size]))
+            if not di_exists:
+                array_size = self.current_dataset.y.size - 1
+                self.current_dataset.dy = np.append(self.current_dataset.dy,
+                                                    np.zeros([array_size]))
+        elif isinstance(self.current_dataset, plottable_2D):
+            dqx_exists = False
+            dqy_exists = False
+            di_exists = False
+            mask_exists = False
+            if self.current_dataset.dqx_data is not None:
+                dqx_exists = True
+            if self.current_dataset.dqy_data is not None:
+                dqy_exists = True
+            if self.current_dataset.err_data is not None:
+                di_exists = True
+            if self.current_dataset.mask is not None:
+                mask_exists = True
+            if not dqy_exists:
+                array_size = self.current_dataset.qy_data.size - 1
+                self.current_dataset.dqy_data = np.append(
+                    self.current_dataset.dqy_data, np.zeros([array_size]))
+            if not dqx_exists:
+                array_size = self.current_dataset.qx_data.size - 1
+                self.current_dataset.dqx_data = np.append(
+                    self.current_dataset.dqx_data, np.zeros([array_size]))
+            if not di_exists:
+                array_size = self.current_dataset.data.size - 1
+                self.current_dataset.err_data = np.append(
+                    self.current_dataset.err_data, np.zeros([array_size]))
+            if not mask_exists:
+                array_size = self.current_dataset.data.size - 1
+                self.current_dataset.mask = np.append(
+                    self.current_dataset.mask,
+                    np.ones([array_size] ,dtype=bool))
+    ####### All methods below are for writing CanSAS XML files #######
+        a method to check all resolution data sets are the same size as I and q
+        """
+        dql_exists = False
+        dqw_exists = False
+        dq_exists = False
+        di_exists = False
+        if self.current_dataset.dxl is not None:
+            dql_exists = True
+        if self.current_dataset.dxw is not None:
+            dqw_exists = True
+        if self.current_dataset.dx is not None:
+            dq_exists = True
+        if self.current_dataset.dy is not None:
+            di_exists = True
+        if dqw_exists and not dql_exists:
+            array_size = self.current_dataset.dxw.size - 1
+            self.current_dataset.dxl = np.append(self.current_dataset.dxl,
+                                                 np.zeros([array_size]))
+        elif dql_exists and not dqw_exists:
+            array_size = self.current_dataset.dxl.size - 1
+            self.current_dataset.dxw = np.append(self.current_dataset.dxw,
+                                                 np.zeros([array_size]))
+        elif not dql_exists and not dqw_exists and not dq_exists:
+            array_size = self.current_dataset.x.size - 1
+            self.current_dataset.dx = np.append(self.current_dataset.dx,
+                                                np.zeros([array_size]))
+        if not di_exists:
+            array_size = self.current_dataset.y.size - 1
+            self.current_dataset.dy = np.append(self.current_dataset.dy,
+                                                np.zeros([array_size]))
+    def _initialize_new_data_set(self, node=None):
+        if node is not None:
+            for child in node:
+                if child.tag.replace(self.base_ns, "") == "Idata":
+                    for i_child in child:
+                        if i_child.tag.replace(self.base_ns, "") == "Qx":
+                            self.current_dataset = plottable_2D()
+                            return
+        self.current_dataset = plottable_1D(np.array(0), np.array(0))
+    ## Writing Methods
     def write(self, filename, datainfo):
         """
 …
             exec "storage.%s = entry.text.strip()" % variable
 # DO NOT REMOVE Called by outside packages:
 #    sas.sasgui.perspectives.invariant.invariant_state

src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py

-                      rc94280c
+                      r8dec7e7
     TransmissionSpectrum, Detector
 from sas.sascalc.dataloader.data_info import combine_data_info_with_plottable
+class Reader():
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException, DefaultReaderException
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+class Reader(FileReader):
     """
     A class for reading in CanSAS v2.0 data files. The existing iteration opens
 …
     # Raw file contents to be processed
     raw_data = None
-    # Data info currently being read in
-    current_datainfo = None
-    # SASdata set currently being read in
-    current_dataset = None
     # List of plottable1D objects that should be linked to the current_datainfo
     data1d = None
 …
     # Flag to bypass extension check
     allow_all = True
+    # List of files to return
+    output = None
+    def read(self, filename):
+    def get_file_contents(self):
         """
         This is the general read method that all SasView data_loaders must have.
 …
         # Reinitialize when loading a new data file to reset all class variables
         self.reset_class_variables()
+        filename = self.f_open.name
+        self.f_open.close() # IO handled by h5py
         # Check that the file exists
         if os.path.isfile(filename):
 …
             if extension in self.ext or self.allow_all:
                 # Load the data file
+                self.raw_data = h5py.File(filename, 'r')
+                try:
+                    self.raw_data = h5py.File(filename, 'r')
+                except Exception as e:
+                    if extension not in self.ext:
+                        msg = "CanSAS2.0 HDF5 Reader could not load file {}".format(basename + extension)
+                        raise DefaultReaderException(msg)
+                    raise FileContentsException(e.message)
                 # Read in all child elements of top level SASroot
                 self.read_children(self.raw_data, [])
 …
         Data1D and Data2D objects
         """
         # Type cast data arrays to float64
         if len(self.current_datainfo.trans_spectrum) > 0:
 …
         # Type cast data arrays to float64 and find min/max as appropriate
         for dataset in self.data2d:
-            dataset.data = dataset.data.astype(np.float64)
-            dataset.err_data = dataset.err_data.astype(np.float64)
-            if dataset.qx_data is not None:
-                dataset.xmin = np.min(dataset.qx_data)
-                dataset.xmax = np.max(dataset.qx_data)
-                dataset.qx_data = dataset.qx_data.astype(np.float64)
-            if dataset.dqx_data is not None:
-                dataset.dqx_data = dataset.dqx_data.astype(np.float64)
-            if dataset.qy_data is not None:
-                dataset.ymin = np.min(dataset.qy_data)
-                dataset.ymax = np.max(dataset.qy_data)
-                dataset.qy_data = dataset.qy_data.astype(np.float64)
-            if dataset.dqy_data is not None:
-                dataset.dqy_data = dataset.dqy_data.astype(np.float64)
-            if dataset.q_data is not None:
-                dataset.q_data = dataset.q_data.astype(np.float64)
             zeros = np.ones(dataset.data.size, dtype=bool)
             try:
 …
                 dataset.x_bins = dataset.qx_data[:n_cols]
                 dataset.data = dataset.data.flatten()
+            final_dataset = combine_data_info_with_plottable(
+                dataset, self.current_datainfo)
+            self.output.append(final_dataset)
+            self.current_dataset = dataset
+            self.send_to_output()
         for dataset in self.data1d:
+            if dataset.x is not None:
+                dataset.x = dataset.x.astype(np.float64)
+                dataset.xmin = np.min(dataset.x)
+                dataset.xmax = np.max(dataset.x)
+            if dataset.y is not None:
+                dataset.y = dataset.y.astype(np.float64)
+                dataset.ymin = np.min(dataset.y)
+                dataset.ymax = np.max(dataset.y)
+            if dataset.dx is not None:
+                dataset.dx = dataset.dx.astype(np.float64)
+            if dataset.dxl is not None:
+                dataset.dxl = dataset.dxl.astype(np.float64)
+            if dataset.dxw is not None:
+                dataset.dxw = dataset.dxw.astype(np.float64)
+            if dataset.dy is not None:
+                dataset.dy = dataset.dy.astype(np.float64)
+            final_dataset = combine_data_info_with_plottable(
+                dataset, self.current_datainfo)
+            self.output.append(final_dataset)
+            self.current_dataset = dataset
+            self.send_to_output()
     def add_data_set(self, key=""):

src/sas/sascalc/dataloader/readers/danse_reader.py

-                      r235f514
+                      r713a047
 #This software was developed by the University of Tennessee as part of the
 #Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
 #project funded by the US National Science Foundation.
+#project funded by the US National Science Foundation.
 #If you use DANSE applications to do scientific research that leads to
 #publication, we ask that you acknowledge the use of the software with the
 …
 import math
 import os
-import sys
 import numpy as np
 import logging
 from sas.sascalc.dataloader.data_info import Data2D, Detector
+from sas.sascalc.dataloader.data_info import plottable_2D, DataInfo, Detector
 from sas.sascalc.dataloader.manipulations import reader2D_converter
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException, DataReaderException
 logger = logging.getLogger(__name__)
 …
 class Reader:
+class Reader(FileReader):
     """
     Example data manipulation
 …
     ## Extension
     ext  = ['.sans', '.SANS']
+    def read(self, filename=None):
+        """
+        Open and read the data in a file
+        @param file: path of the file
+        """
+        read_it = False
+        for item in self.ext:
+            if filename.lower().find(item) >= 0:
+                read_it = True
+        if read_it:
+    def get_file_contents(self):
+        self.current_datainfo = DataInfo()
+        self.current_dataset = plottable_2D()
+        self.output = []
+        loaded_correctly = True
+        error_message = ""
+        # defaults
+        # wavelength in Angstrom
+        wavelength = 10.0
+        # Distance in meter
+        distance   = 11.0
+        # Pixel number of center in x
+        center_x   = 65
+        # Pixel number of center in y
+        center_y   = 65
+        # Pixel size [mm]
+        pixel      = 5.0
+        # Size in x, in pixels
+        size_x     = 128
+        # Size in y, in pixels
+        size_y     = 128
+        # Format version
+        fversion   = 1.0
+        self.current_datainfo.filename = os.path.basename(self.f_open.name)
+        detector = Detector()
+        self.current_datainfo.detector.append(detector)
+        self.current_dataset.data = np.zeros([size_x, size_y])
+        self.current_dataset.err_data = np.zeros([size_x, size_y])
+        read_on = True
+        data_start_line = 1
+        while read_on:
+            line = self.f_open.readline()
+            data_start_line += 1
+            if line.find("DATA:") >= 0:
+                read_on = False
+                break
+            toks = line.split(':')
             try:
-                datafile = open(filename, 'r')
-            except:
-                raise  RuntimeError,"danse_reader cannot open %s" % (filename)
-            # defaults
-            # wavelength in Angstrom
-            wavelength = 10.0
-            # Distance in meter
-            distance   = 11.0
-            # Pixel number of center in x
-            center_x   = 65
-            # Pixel number of center in y
-            center_y   = 65
-            # Pixel size [mm]
-            pixel      = 5.0
-            # Size in x, in pixels
-            size_x     = 128
-            # Size in y, in pixels
-            size_y     = 128
-            # Format version
-            fversion   = 1.0
-            output = Data2D()
-            output.filename = os.path.basename(filename)
-            detector = Detector()
-            output.detector.append(detector)
-            output.data = np.zeros([size_x,size_y])
-            output.err_data = np.zeros([size_x, size_y])
-            data_conv_q = None
-            data_conv_i = None
-            if has_converter == True and output.Q_unit != '1/A':
-                data_conv_q = Converter('1/A')
-                # Test it
-                data_conv_q(1.0, output.Q_unit)
-            if has_converter == True and output.I_unit != '1/cm':
-                data_conv_i = Converter('1/cm')
-                # Test it
-                data_conv_i(1.0, output.I_unit)
-            read_on = True
-            while read_on:
-                line = datafile.readline()
-                if line.find("DATA:") >= 0:
-                    read_on = False
-                    break
-                toks = line.split(':')
                 if toks[0] == "FORMATVERSION":
                     fversion = float(toks[1])
                 if toks[0] == "WAVELENGTH":
+                elif toks[0] == "WAVELENGTH":
                     wavelength = float(toks[1])
                 elif toks[0] == "DISTANCE":
 …
                 elif toks[0] == "SIZE_Y":
                     size_y = int(toks[1])
+            # Read the data
+            data = []
+            error = []
+            if fversion == 1.0:
+                data_str = datafile.readline()
+                data = data_str.split(' ')
+            else:
+                read_on = True
+                while read_on:
+                    data_str = datafile.readline()
+                    if len(data_str) == 0:
+                        read_on = False
+                    else:
+                        toks = data_str.split()
+                        try:
+                            val = float(toks[0])
+                            err = float(toks[1])
+                            if data_conv_i is not None:
+                                val = data_conv_i(val, units=output._yunit)
+                                err = data_conv_i(err, units=output._yunit)
+                            data.append(val)
+                            error.append(err)
+                        except:
+                            logger.info("Skipping line:%s,%s" %(data_str,
+                                                                sys.exc_value))
+            # Initialize
+            x_vals = []
+            y_vals = []
+            ymin = None
+            ymax = None
+            xmin = None
+            xmax = None
+            # Qx and Qy vectors
+            theta = pixel / distance / 100.0
+            stepq = 4.0 * math.pi / wavelength * math.sin(theta / 2.0)
+            for i_x in range(size_x):
+                theta = (i_x - center_x + 1) * pixel / distance / 100.0
+                qx = 4.0 * math.pi / wavelength * math.sin(theta / 2.0)
+                if has_converter == True and output.Q_unit != '1/A':
+                    qx = data_conv_q(qx, units=output.Q_unit)
+                x_vals.append(qx)
+                if xmin is None or qx < xmin:
+                    xmin = qx
+                if xmax is None or qx > xmax:
+                    xmax = qx
+            ymin = None
+            ymax = None
+            for i_y in range(size_y):
+                theta = (i_y - center_y + 1) * pixel / distance / 100.0
+                qy = 4.0 * math.pi / wavelength * math.sin(theta/2.0)
+                if has_converter == True and output.Q_unit != '1/A':
+                    qy = data_conv_q(qy, units=output.Q_unit)
+                y_vals.append(qy)
+                if ymin is None or qy < ymin:
+                    ymin = qy
+                if ymax is None or qy > ymax:
+                    ymax = qy
+            # Store the data in the 2D array
+            i_x = 0
+            i_y = -1
+            for i_pt in range(len(data)):
+                try:
+                    value = float(data[i_pt])
+                except:
+                    # For version 1.0, the data were still
+                    # stored as strings at this point.
+                    msg = "Skipping entry (v1.0):%s,%s" % (str(data[i_pt]),
+                                                           sys.exc_value)
+                    logger.info(msg)
+                # Get bin number
+                if math.fmod(i_pt, size_x) == 0:
+                    i_x = 0
+                    i_y += 1
+                else:
+                    i_x += 1
+                output.data[i_y][i_x] = value
+                if fversion>1.0:
+                    output.err_data[i_y][i_x] = error[i_pt]
+            # Store all data
+            # Store wavelength
+            if has_converter == True and output.source.wavelength_unit != 'A':
+                conv = Converter('A')
+                wavelength = conv(wavelength,
+                                  units=output.source.wavelength_unit)
+            output.source.wavelength = wavelength
+            # Store distance
+            if has_converter == True and detector.distance_unit != 'm':
+                conv = Converter('m')
+                distance = conv(distance, units=detector.distance_unit)
+            detector.distance = distance
+            # Store pixel size
+            if has_converter == True and detector.pixel_size_unit != 'mm':
+                conv = Converter('mm')
+                pixel = conv(pixel, units=detector.pixel_size_unit)
+            detector.pixel_size.x = pixel
+            detector.pixel_size.y = pixel
+            # Store beam center in distance units
+            detector.beam_center.x = center_x * pixel
+            detector.beam_center.y = center_y * pixel
+            # Store limits of the image (2D array)
+            xmin = xmin - stepq / 2.0
+            xmax = xmax + stepq / 2.0
+            ymin = ymin - stepq /2.0
+            ymax = ymax + stepq / 2.0
+            if has_converter == True and output.Q_unit != '1/A':
+                xmin = data_conv_q(xmin, units=output.Q_unit)
+                xmax = data_conv_q(xmax, units=output.Q_unit)
+                ymin = data_conv_q(ymin, units=output.Q_unit)
+                ymax = data_conv_q(ymax, units=output.Q_unit)
+            output.xmin = xmin
+            output.xmax = xmax
+            output.ymin = ymin
+            output.ymax = ymax
+            # Store x and y axis bin centers
+            output.x_bins = x_vals
+            output.y_bins = y_vals
+            # Units
+            if data_conv_q is not None:
+                output.xaxis("\\rm{Q_{x}}", output.Q_unit)
+                output.yaxis("\\rm{Q_{y}}", output.Q_unit)
+            else:
+                output.xaxis("\\rm{Q_{x}}", 'A^{-1}')
+                output.yaxis("\\rm{Q_{y}}", 'A^{-1}')
+            if data_conv_i is not None:
+                output.zaxis("\\rm{Intensity}", output.I_unit)
+            else:
+                output.zaxis("\\rm{Intensity}", "cm^{-1}")
+            if not fversion >= 1.0:
+                msg = "Danse_reader can't read this file %s" % filename
+                raise ValueError, msg
+            else:
+                logger.info("Danse_reader Reading %s \n" % filename)
+            # Store loading process information
+            output.meta_data['loader'] = self.type_name
+            output = reader2D_converter(output)
+            return output
+        return None
+            except ValueError as e:
+                error_message += "Unable to parse {}. Default value used.\n".format(toks[0])
+                loaded_correctly = False
+        # Read the data
+        data = []
+        error = []
+        if not fversion >= 1.0:
+            msg = "danse_reader can't read this file {}".format(self.f_open.name)
+            raise FileContentsException(msg)
+        for line_num, data_str in enumerate(self.f_open.readlines()):
+            toks = data_str.split()
+            try:
+                val = float(toks[0])
+                err = float(toks[1])
+                data.append(val)
+                error.append(err)
+            except ValueError as exc:
+                msg = "Unable to parse line {}: {}".format(line_num + data_start_line, data_str.strip())
+                raise FileContentsException(msg)
+        num_pts = size_x * size_y
+        if len(data) < num_pts:
+            msg = "Not enough data points provided. Expected {} but got {}".format(
+                size_x * size_y, len(data))
+            raise FileContentsException(msg)
+        elif len(data) > num_pts:
+            error_message += ("Too many data points provided. Expected {0} but"
+                " got {1}. Only the first {0} will be used.\n").format(num_pts, len(data))
+            loaded_correctly = False
+            data = data[:num_pts]
+            error = error[:num_pts]
+        # Qx and Qy vectors
+        theta = pixel / distance / 100.0
+        i_x = np.arange(size_x)
+        theta = (i_x - center_x + 1) * pixel / distance / 100.0
+        x_vals = 4.0 * np.pi / wavelength * np.sin(theta / 2.0)
+        xmin = x_vals.min()
+        xmax = x_vals.max()
+        i_y = np.arange(size_y)
+        theta = (i_y - center_y + 1) * pixel / distance / 100.0
+        y_vals = 4.0 * np.pi / wavelength * np.sin(theta / 2.0)
+        ymin = y_vals.min()
+        ymax = y_vals.max()
+        self.current_dataset.data = np.array(data, dtype=np.float64).reshape((size_y, size_x))
+        if fversion > 1.0:
+            self.current_dataset.err_data = np.array(error, dtype=np.float64).reshape((size_y, size_x))
+        # Store all data
+        # Store wavelength
+        if has_converter == True and self.current_datainfo.source.wavelength_unit != 'A':
+            conv = Converter('A')
+            wavelength = conv(wavelength,
+                              units=self.current_datainfo.source.wavelength_unit)
+        self.current_datainfo.source.wavelength = wavelength
+        # Store distance
+        if has_converter == True and detector.distance_unit != 'm':
+            conv = Converter('m')
+            distance = conv(distance, units=detector.distance_unit)
+        detector.distance = distance
+        # Store pixel size
+        if has_converter == True and detector.pixel_size_unit != 'mm':
+            conv = Converter('mm')
+            pixel = conv(pixel, units=detector.pixel_size_unit)
+        detector.pixel_size.x = pixel
+        detector.pixel_size.y = pixel
+        # Store beam center in distance units
+        detector.beam_center.x = center_x * pixel
+        detector.beam_center.y = center_y * pixel
+        self.current_dataset.xaxis("\\rm{Q_{x}}", 'A^{-1}')
+        self.current_dataset.yaxis("\\rm{Q_{y}}", 'A^{-1}')
+        self.current_dataset.zaxis("\\rm{Intensity}", "cm^{-1}")
+        self.current_dataset.x_bins = x_vals
+        self.current_dataset.y_bins = y_vals
+        # Reshape data
+        x_vals = np.tile(x_vals, (size_y, 1)).flatten()
+        y_vals = np.tile(y_vals, (size_x, 1)).T.flatten()
+        if self.current_dataset.err_data == np.all(np.array(None)) or np.any(self.current_dataset.err_data <= 0):
+            new_err_data = np.sqrt(np.abs(self.current_dataset.data))
+        else:
+            new_err_data = self.current_dataset.err_data.flatten()
+        self.current_dataset.err_data = new_err_data
+        self.current_dataset.qx_data = x_vals
+        self.current_dataset.qy_data = y_vals
+        self.current_dataset.q_data = np.sqrt(x_vals**2 + y_vals**2)
+        self.current_dataset.mask = np.ones(len(x_vals), dtype=bool)
+        # Store loading process information
+        self.current_datainfo.meta_data['loader'] = self.type_name
+        self.send_to_output()
+        if not loaded_correctly:
+            raise DataReaderException(error_message)

src/sas/sascalc/dataloader/readers/red2d_reader.py

-                      ra1b8fee
+                      r2f85af7
 #This software was developed by the University of Tennessee as part of the
 #Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
 #project funded by the US National Science Foundation.
+#project funded by the US National Science Foundation.
 #See the license text in license.txt
 #copyright 2008, University of Tennessee
 ######################################################################
-from __future__ import print_function
 import os
 import numpy as np
 import math
+from sas.sascalc.dataloader.data_info import Data2D, Detector
+from sas.sascalc.dataloader.data_info import plottable_2D, DataInfo, Detector
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException
 # Look for unit converter
 …
 except:
     has_converter = False
 def check_point(x_point):
     """
 …
     except:
         return 0
 class Reader:
+class Reader(FileReader):
     """ Simple data reader for Igor data files """
     ## File type
 …
     ## Extension
     ext = ['.DAT', '.dat']
     def write(self, filename, data):
         """
         Write to .dat
         :param filename: file name to write
         :param data: data2D
 …
         import time
         # Write the file
+        fd = open(filename, 'w')
+        t = time.localtime()
+        time_str = time.strftime("%H:%M on %b %d %y", t)
+        header_str = "Data columns are Qx - Qy - I(Qx,Qy)\n\nASCII data"
+        header_str += " created at %s \n\n" % time_str
+        # simple 2D header
+        fd.write(header_str)
+        # write qx qy I values
+        for i in range(len(data.data)):
+            fd.write("%g  %g  %g\n" % (data.qx_data[i],
+                                        data.qy_data[i],
+                                       data.data[i]))
+        # close
+        fd.close()
+    def read(self, filename=None):
+        """ Read file """
+        if not os.path.isfile(filename):
+            raise ValueError, \
+            "Specified file %s is not a regular file" % filename
+        try:
+            fd = open(filename, 'w')
+            t = time.localtime()
+            time_str = time.strftime("%H:%M on %b %d %y", t)
+            header_str = "Data columns are Qx - Qy - I(Qx,Qy)\n\nASCII data"
+            header_str += " created at %s \n\n" % time_str
+            # simple 2D header
+            fd.write(header_str)
+            # write qx qy I values
+            for i in range(len(data.data)):
+                fd.write("%g  %g  %g\n" % (data.qx_data[i],
+                                            data.qy_data[i],
+                                           data.data[i]))
+        finally:
+            fd.close()
+    def get_file_contents(self):
         # Read file
+        f = open(filename, 'r')
+        buf = f.read()
+        f.close()
+        buf = self.f_open.read()
+        self.f_open.close()
         # Instantiate data object
+        output = Data2D()
+        output.filename = os.path.basename(filename)
+        detector = Detector()
+        if len(output.detector) > 0:
+            print(str(output.detector[0]))
+        output.detector.append(detector)
+        self.current_dataset = plottable_2D()
+        self.current_datainfo = DataInfo()
+        self.current_datainfo.filename = os.path.basename(self.f_open.name)
+        self.current_datainfo.detector.append(Detector())
         # Get content
         dataStarted = False
+        data_started = False
         ## Defaults
         lines = buf.split('\n')
         x = []
         y = []
         wavelength = None
         distance = None
         transmission = None
         pixel_x = None
         pixel_y = None
+        isInfo = False
+        isCenter = False
+        data_conv_q = None
+        data_conv_i = None
+        # Set units: This is the unit assumed for Q and I in the data file.
+        if has_converter == True and output.Q_unit != '1/A':
+            data_conv_q = Converter('1/A')
+            # Test it
+            data_conv_q(1.0, output.Q_unit)
+        if has_converter == True and output.I_unit != '1/cm':
+            data_conv_i = Converter('1/cm')
+            # Test it
+            data_conv_i(1.0, output.I_unit)
+        is_info = False
+        is_center = False
         # Remove the last lines before the for loop if the lines are empty
         # to calculate the exact number of data points
 …
             ## Reading the header applies only to IGOR/NIST 2D q_map data files
             # Find setup info line
             if isInfo:
                 isInfo = False
+            if is_info:
+                is_info = False
                 line_toks = line.split()
                 # Wavelength in Angstrom
 …
                     # Units
                     if has_converter == True and \
                     output.source.wavelength_unit != 'A':
+                    self.current_datainfo.source.wavelength_unit != 'A':
                         conv = Converter('A')
                         wavelength = conv(wavelength,
                                           units=output.source.wavelength_unit)
+                                          units=self.current_datainfo.source.wavelength_unit)
                 except:
                     #Not required
 …
                     distance = float(line_toks[3])
                     # Units
                     if has_converter == True and detector.distance_unit != 'm':
+                    if has_converter == True and self.current_datainfo.detector[0].distance_unit != 'm':
                         conv = Converter('m')
+                        distance = conv(distance, units=detector.distance_unit)
+                        distance = conv(distance,
+                            units=self.current_datainfo.detector[0].distance_unit)
                 except:
                     #Not required
                     pass
                 # Distance in meters
                 try:
 …
                     #Not required
                     pass
             if line.count("LAMBDA") > 0:
                 isInfo = True
+                is_info = True
             # Find center info line
             if isCenter:
                 isCenter = False
+            if is_center:
+                is_center = False
                 line_toks = line.split()
                 # Center in bin number
 …
             if line.count("BCENT") > 0:
                 isCenter = True
+                is_center = True
             # Check version
             if line.count("Data columns") > 0:
 …
             # Find data start
             if line.count("ASCII data") > 0:
                 dataStarted = True
+                data_started = True
                 continue
             ## Read and get data.
             if dataStarted == True:
+            if data_started == True:
                 line_toks = line.split()
                 if len(line_toks) == 0:
                     #empty line
                     continue
                 # the number of columns must be stayed same
+                # the number of columns must be stayed same
                 col_num = len(line_toks)
                 break
 …
         # index for lines_array
         lines_index = np.arange(len(lines))
         # get the data lines
         data_lines = lines_array[lines_index >= (line_num - 1)]
 …
         # split all data to one big list w/" "separator
         data_list = data_list.split()
         # Check if the size is consistent with data, otherwise
         #try the tab(\t) separator
 …
             data_point = data_array.reshape(row_num, col_num).transpose()
         except:
             msg = "red2d_reader: Can't read this file: Not a proper file format"
             raise ValueError, msg
+            msg = "red2d_reader can't read this file: Incorrect number of data points provided."
+            raise FileContentsException(msg)
         ## Get the all data: Let's HARDcoding; Todo find better way
         # Defaults
 …
         #if col_num > (6 + ver): mask[data_point[(6 + ver)] < 1] = False
         q_data = np.sqrt(qx_data*qx_data+qy_data*qy_data+qz_data*qz_data)
         # Extra protection(it is needed for some data files):
+        # Extra protection(it is needed for some data files):
         # If all mask elements are False, put all True
         if not mask.any():
             mask[mask == False] = True
         # Store limits of the image in q space
         xmin = np.min(qx_data)
 …
         ymax = np.max(qy_data)
-        # units
-        if has_converter == True and output.Q_unit != '1/A':
-            xmin = data_conv_q(xmin, units=output.Q_unit)
-            xmax = data_conv_q(xmax, units=output.Q_unit)
-            ymin = data_conv_q(ymin, units=output.Q_unit)
-            ymax = data_conv_q(ymax, units=output.Q_unit)
         ## calculate the range of the qx and qy_data
         x_size = math.fabs(xmax - xmin)
         y_size = math.fabs(ymax - ymin)
         # calculate the number of pixels in the each axes
         npix_y = math.floor(math.sqrt(len(data)))
         npix_x = math.floor(len(data) / npix_y)
         # calculate the size of bins
         xstep = x_size / (npix_x - 1)
         ystep = y_size / (npix_y - 1)
         # store x and y axis bin centers in q space
         x_bins = np.arange(xmin, xmax + xstep, xstep)
         y_bins = np.arange(ymin, ymax + ystep, ystep)
         # get the limits of q values
         xmin = xmin - xstep / 2
 …
         ymin = ymin - ystep / 2
         ymax = ymax + ystep / 2
         #Store data in outputs
         #TODO: Check the lengths
         output.data = data
+        self.current_dataset.data = data
         if (err_data == 1).all():
             output.err_data = np.sqrt(np.abs(data))
             output.err_data[output.err_data == 0.0] = 1.0
+            self.current_dataset.err_data = np.sqrt(np.abs(data))
+            self.current_dataset.err_data[self.current_dataset.err_data == 0.0] = 1.0
         else:
             output.err_data = err_data
         output.qx_data = qx_data
         output.qy_data = qy_data
         output.q_data = q_data
         output.mask = mask
         output.x_bins = x_bins
         output.y_bins = y_bins
         output.xmin = xmin
         output.xmax = xmax
         output.ymin = ymin
         output.ymax = ymax
         output.source.wavelength = wavelength
+            self.current_dataset.err_data = err_data
+        self.current_dataset.qx_data = qx_data
+        self.current_dataset.qy_data = qy_data
+        self.current_dataset.q_data = q_data
+        self.current_dataset.mask = mask
+        self.current_dataset.x_bins = x_bins
+        self.current_dataset.y_bins = y_bins
+        self.current_dataset.xmin = xmin
+        self.current_dataset.xmax = xmax
+        self.current_dataset.ymin = ymin
+        self.current_dataset.ymax = ymax
+        self.current_datainfo.source.wavelength = wavelength
         # Store pixel size in mm
         detector.pixel_size.x = pixel_x
         detector.pixel_size.y = pixel_y
+        self.current_datainfo.detector[0].pixel_size.x = pixel_x
+        self.current_datainfo.detector[0].pixel_size.y = pixel_y
         # Store the sample to detector distance
         detector.distance = distance
+        self.current_datainfo.detector[0].distance = distance
         # optional data: if all of dq data == 0, do not pass to output
         if len(dqx_data) == len(qx_data) and dqx_data.any() != 0:
 …
                     cos_th = qx_data / diag
                     sin_th = qy_data / diag
                     output.dqx_data = np.sqrt((dqx_data * cos_th) * \
+                    self.current_dataset.dqx_data = np.sqrt((dqx_data * cos_th) * \
                                                  (dqx_data * cos_th) \
                                                  + (dqy_data * sin_th) * \
                                                   (dqy_data * sin_th))
                     output.dqy_data = np.sqrt((dqx_data * sin_th) * \
+                    self.current_dataset.dqy_data = np.sqrt((dqx_data * sin_th) * \
                                                  (dqx_data * sin_th) \
                                                  + (dqy_data * cos_th) * \
                                                   (dqy_data * cos_th))
                 else:
                     output.dqx_data = dqx_data
                     output.dqy_data = dqy_data
+                    self.current_dataset.dqx_data = dqx_data
+                    self.current_dataset.dqy_data = dqy_data
         # Units of axes
+        if data_conv_q is not None:
+            output.xaxis("\\rm{Q_{x}}", output.Q_unit)
+            output.yaxis("\\rm{Q_{y}}", output.Q_unit)
+        else:
+            output.xaxis("\\rm{Q_{x}}", 'A^{-1}')
+            output.yaxis("\\rm{Q_{y}}", 'A^{-1}')
+        if data_conv_i is not None:
+            output.zaxis("\\rm{Intensity}", output.I_unit)
+        else:
+            output.zaxis("\\rm{Intensity}", "cm^{-1}")
+        self.current_dataset.xaxis("\\rm{Q_{x}}", 'A^{-1}')
+        self.current_dataset.yaxis("\\rm{Q_{y}}", 'A^{-1}')
+        self.current_dataset.zaxis("\\rm{Intensity}", "cm^{-1}")
         # Store loading process information
         output.meta_data['loader'] = self.type_name
         return output
+        self.current_datainfo.meta_data['loader'] = self.type_name
+        self.send_to_output()

src/sas/sascalc/dataloader/readers/sesans_reader.py

-                      r149b8f6
+                      rbe43448
 import numpy as np
 import os
+from sas.sascalc.dataloader.data_info import Data1D
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+from sas.sascalc.dataloader.data_info import plottable_1D, DataInfo
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException, DataReaderException
 # Check whether we have a converter available
 …
 _ZERO = 1e-16
+class Reader:
+class Reader(FileReader):
     """
     Class to load sesans files (6 columns).
 …
     type_name = "SESANS"
     # Wildcards
+    ## Wildcards
     type = ["SESANS files (*.ses)|*.ses",
             "SESANS files (*..sesans)|*.sesans"]
 …
     allow_all = True
+    def read(self, path):
+        """
+        Load data file
+    def get_file_contents(self):
+        self.current_datainfo = DataInfo()
+        self.current_dataset = plottable_1D(np.array([]), np.array([]))
+        self.current_datainfo.isSesans = True
+        self.output = []
+        :param path: file path
+        line = self.f_open.readline()
+        params = {}
+        while not line.startswith("BEGIN_DATA"):
+            terms = line.split()
+            if len(terms) >= 2:
+                params[terms[0]] = " ".join(terms[1:])
+            line = self.f_open.readline()
+        self.params = params
+        :return: SESANSData1D object, or None
+        if "FileFormatVersion" not in self.params:
+            raise FileContentsException("SES file missing FileFormatVersion")
+        if float(self.params["FileFormatVersion"]) >= 2.0:
+            raise FileContentsException("SASView only supports SES version 1")
+        :raise RuntimeError: when the file can't be opened
+        :raise ValueError: when the length of the data vectors are inconsistent
+        """
+        if os.path.isfile(path):
+            basename = os.path.basename(path)
+            _, extension = os.path.splitext(basename)
+            if not (self.allow_all or extension.lower() in self.ext):
+                raise RuntimeError(
+                    "{} has an unrecognized file extension".format(path))
+        if "SpinEchoLength_unit" not in self.params:
+            raise FileContentsException("SpinEchoLength has no units")
+        if "Wavelength_unit" not in self.params:
+            raise FileContentsException("Wavelength has no units")
+        if params["SpinEchoLength_unit"] != params["Wavelength_unit"]:
+            raise FileContentsException("The spin echo data has rudely used "
+                               "different units for the spin echo length "
+                               "and the wavelength.  While sasview could "
+                               "handle this instance, it is a violation "
+                               "of the file format and will not be "
+                               "handled by other software.")
+        headers = self.f_open.readline().split()
+        self._insist_header(headers, "SpinEchoLength")
+        self._insist_header(headers, "Depolarisation")
+        self._insist_header(headers, "Depolarisation_error")
+        self._insist_header(headers, "Wavelength")
+        data = np.loadtxt(self.f_open)
+        if data.shape[1] != len(headers):
+            raise FileContentsException(
+                "File has {} headers, but {} columns".format(
+                    len(headers),
+                    data.shape[1]))
+        if not data.size:
+            raise FileContentsException("{} is empty".format(path))
+        x = data[:, headers.index("SpinEchoLength")]
+        if "SpinEchoLength_error" in headers:
+            dx = data[:, headers.index("SpinEchoLength_error")]
         else:
+            raise RuntimeError("{} is not a file".format(path))
+        with open(path, 'r') as input_f:
+            line = input_f.readline()
+            params = {}
+            while not line.startswith("BEGIN_DATA"):
+                terms = line.split()
+                if len(terms) >= 2:
+                    params[terms[0]] = " ".join(terms[1:])
+                line = input_f.readline()
+            self.params = params
+            dx = x * 0.05
+        lam = data[:, headers.index("Wavelength")]
+        if "Wavelength_error" in headers:
+            dlam = data[:, headers.index("Wavelength_error")]
+        else:
+            dlam = lam * 0.05
+        y = data[:, headers.index("Depolarisation")]
+        dy = data[:, headers.index("Depolarisation_error")]
+            if "FileFormatVersion" not in self.params:
+                raise RuntimeError("SES file missing FileFormatVersion")
+            if float(self.params["FileFormatVersion"]) >= 2.0:
+                raise RuntimeError("SASView only supports SES version 1")
+        lam_unit = self._unit_fetch("Wavelength")
+        x, x_unit = self._unit_conversion(x, "A",
+                                          self._unit_fetch(
+                                              "SpinEchoLength"))
+        dx, dx_unit = self._unit_conversion(
+            dx, lam_unit,
+            self._unit_fetch("SpinEchoLength"))
+        dlam, dlam_unit = self._unit_conversion(
+            dlam, lam_unit,
+            self._unit_fetch("Wavelength"))
+        y_unit = self._unit_fetch("Depolarisation")
+            if "SpinEchoLength_unit" not in self.params:
+                raise RuntimeError("SpinEchoLength has no units")
+            if "Wavelength_unit" not in self.params:
+                raise RuntimeError("Wavelength has no units")
+            if params["SpinEchoLength_unit"] != params["Wavelength_unit"]:
+                raise RuntimeError("The spin echo data has rudely used "
+                                   "different units for the spin echo length "
+                                   "and the wavelength.  While sasview could "
+                                   "handle this instance, it is a violation "
+                                   "of the file format and will not be "
+                                   "handled by other software.")
+        self.current_dataset.x = x
+        self.current_dataset.y = y
+        self.current_dataset.lam = lam
+        self.current_dataset.dy = dy
+        self.current_dataset.dx = dx
+        self.current_dataset.dlam = dlam
+        self.current_datainfo.isSesans = True
+            headers = input_f.readline().split()
+        self.current_datainfo._yunit = y_unit
+        self.current_datainfo._xunit = x_unit
+        self.current_datainfo.source.wavelength_unit = lam_unit
+        self.current_datainfo.source.wavelength = lam
+        self.current_datainfo.filename = os.path.basename(self.f_open.name)
+        self.current_dataset.xaxis(r"\rm{z}", x_unit)
+        # Adjust label to ln P/(lam^2 t), remove lam column refs
+        self.current_dataset.yaxis(r"\rm{ln(P)/(t \lambda^2)}", y_unit)
+        # Store loading process information
+        self.current_datainfo.meta_data['loader'] = self.type_name
+        self.current_datainfo.sample.name = params["Sample"]
+        self.current_datainfo.sample.ID = params["DataFileTitle"]
+        self.current_datainfo.sample.thickness = self._unit_conversion(
+            float(params["Thickness"]), "cm",
+            self._unit_fetch("Thickness"))[0]
+            self._insist_header(headers, "SpinEchoLength")
+            self._insist_header(headers, "Depolarisation")
+            self._insist_header(headers, "Depolarisation_error")
+            self._insist_header(headers, "Wavelength")
+        self.current_datainfo.sample.zacceptance = (
+            float(params["Theta_zmax"]),
+            self._unit_fetch("Theta_zmax"))
+            data = np.loadtxt(input_f)
+        self.current_datainfo.sample.yacceptance = (
+            float(params["Theta_ymax"]),
+            self._unit_fetch("Theta_ymax"))
+            if data.shape[1] != len(headers):
+                raise RuntimeError(
+                    "File has {} headers, but {} columns".format(
+                        len(headers),
+                        data.shape[1]))
+            if not data.size:
+                raise RuntimeError("{} is empty".format(path))
+            x = data[:, headers.index("SpinEchoLength")]
+            if "SpinEchoLength_error" in headers:
+                dx = data[:, headers.index("SpinEchoLength_error")]
+            else:
+                dx = x * 0.05
+            lam = data[:, headers.index("Wavelength")]
+            if "Wavelength_error" in headers:
+                dlam = data[:, headers.index("Wavelength_error")]
+            else:
+                dlam = lam * 0.05
+            y = data[:, headers.index("Depolarisation")]
+            dy = data[:, headers.index("Depolarisation_error")]
+            lam_unit = self._unit_fetch("Wavelength")
+            x, x_unit = self._unit_conversion(x, "A",
+                                              self._unit_fetch(
+                                                  "SpinEchoLength"))
+            dx, dx_unit = self._unit_conversion(
+                dx, lam_unit,
+                self._unit_fetch("SpinEchoLength"))
+            dlam, dlam_unit = self._unit_conversion(
+                dlam, lam_unit,
+                self._unit_fetch("Wavelength"))
+            y_unit = self._unit_fetch("Depolarisation")
+            output = Data1D(x=x, y=y, lam=lam, dy=dy, dx=dx, dlam=dlam,
+                            isSesans=True)
+            output.y_unit = y_unit
+            output.x_unit = x_unit
+            output.source.wavelength_unit = lam_unit
+            output.source.wavelength = lam
+            self.filename = output.filename = basename
+            output.xaxis(r"\rm{z}", x_unit)
+            # Adjust label to ln P/(lam^2 t), remove lam column refs
+            output.yaxis(r"\rm{ln(P)/(t \lambda^2)}", y_unit)
+            # Store loading process information
+            output.meta_data['loader'] = self.type_name
+            output.sample.name = params["Sample"]
+            output.sample.ID = params["DataFileTitle"]
+            output.sample.thickness = self._unit_conversion(
+                float(params["Thickness"]), "cm",
+                self._unit_fetch("Thickness"))[0]
+            output.sample.zacceptance = (
+                float(params["Theta_zmax"]),
+                self._unit_fetch("Theta_zmax"))
+            output.sample.yacceptance = (
+                float(params["Theta_ymax"]),
+                self._unit_fetch("Theta_ymax"))
+            return output
+        self.send_to_output()
     @staticmethod
     def _insist_header(headers, name):
         if name not in headers:
             raise RuntimeError(
+            raise FileContentsException(
                 "Missing {} column in spin echo data".format(name))

src/sas/sascalc/dataloader/readers/xml_reader.py

-                      r235f514
+                      rfafe52a
 from lxml import etree
 from lxml.builder import E
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
 logger = logging.getLogger(__name__)
 …
 PARSER = etree.ETCompatXMLParser(remove_comments=True, remove_pis=False)
 class XMLreader():
+class XMLreader(FileReader):
     """
     Generic XML read and write class. Mostly helper functions.
 …
         except etree.XMLSyntaxError as xml_error:
             logger.info(xml_error)
+            raise xml_error
         except Exception:
             self.xml = None
 …
         except etree.XMLSyntaxError as xml_error:
             logger.info(xml_error)
+        except Exception:
+            raise xml_error
+        except Exception as exc:
             self.xml = None
             self.xmldoc = None
             self.xmlroot = None
+            raise exc
     def set_schema(self, schema):
 …
         Create a unique key value for any dictionary to prevent overwriting
         Recurses until a unique key value is found.
         :param dictionary: A dictionary with any number of entries
         :param name: The index of the item to be added to dictionary
 …
         Create an element tree for processing from an etree element
         :param root: etree Element(s)
+        :param root: etree Element(s)
         """
         return etree.ElementTree(root)

SasView

Changeset 44daa56 in sasview for src/sas/sascalc

Legend:

src/sas/sascalc/pr/c_extensions/Cinvertor.c

src/sas/sascalc/pr/c_extensions/invertor.c

src/sas/sascalc/pr/c_extensions/invertor.h

src/sas/sascalc/pr/invertor.py

src/sas/sascalc/data_util/registry.py

src/sas/sascalc/dataloader/data_info.py

src/sas/sascalc/dataloader/loader.py

src/sas/sascalc/dataloader/readers/init.py

src/sas/sascalc/dataloader/readers/abs_reader.py

src/sas/sascalc/dataloader/readers/anton_paar_saxs_reader.py

src/sas/sascalc/dataloader/readers/ascii_reader.py

src/sas/sascalc/dataloader/readers/associations.py

src/sas/sascalc/dataloader/readers/cansas_reader.py

src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py

src/sas/sascalc/dataloader/readers/danse_reader.py

src/sas/sascalc/dataloader/readers/red2d_reader.py

src/sas/sascalc/dataloader/readers/sesans_reader.py

src/sas/sascalc/dataloader/readers/xml_reader.py

Download in other formats: