Changeset 44daa56 in sasview for src/sas

Timestamp:

Aug 21, 2017 4:51:20 AM (7 years ago)

Author:

lewis

Branches:

master, ESS_GUI, ESS_GUI_Docs, ESS_GUI_batch_fitting, ESS_GUI_bumps_abstraction, ESS_GUI_iss1116, ESS_GUI_iss879, ESS_GUI_iss959, ESS_GUI_opencl, ESS_GUI_ordering, ESS_GUI_sync_sascalc, costrafo411, magnetic_scatt, release-4.2.2, ticket-1009, ticket-1094-headless, ticket-1242-2d-resolution, ticket-1243, ticket-1249, ticket885, unittest-saveload

Children:

dcb91cf

Parents:

ce8c7bd (diff), a06ee7e (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' into ticket-876

Location:

src/sas

Files:

• sascalc/pr/c_extensions/Cinvertor.c (modified) (7 diffs)
• sascalc/pr/c_extensions/invertor.c (modified) (2 diffs)
• sascalc/pr/c_extensions/invertor.h (modified) (1 diff)
• sascalc/pr/invertor.py (modified) (10 diffs)
• sasgui/perspectives/calculator/model_editor.py (modified) (16 diffs)
• sasgui/perspectives/calculator/pyconsole.py (modified) (2 diffs)
• sasgui/perspectives/fitting/fitting.py (modified) (10 diffs)
• sasgui/perspectives/fitting/models.py (modified) (2 diffs)
• sasgui/perspectives/pr/inversion_panel.py (modified) (12 diffs)
• sasgui/perspectives/pr/inversion_state.py (modified) (6 diffs)
• sasgui/perspectives/pr/media/pr_help.rst (modified) (1 diff)
• sasgui/perspectives/pr/pr.py (modified) (10 diffs)
• sascalc/data_util/registry.py (modified) (8 diffs)
• sascalc/dataloader/data_info.py (modified) (1 diff)
• sascalc/dataloader/file_reader_base_class.py (added)
• sascalc/dataloader/loader.py (modified) (5 diffs)
• sascalc/dataloader/loader_exceptions.py (added)
• sascalc/dataloader/readers/IgorReader.py (deleted)
• sascalc/dataloader/readers/__init__.py (modified) (1 diff)
• sascalc/dataloader/readers/abs_reader.py (modified) (1 diff)
• sascalc/dataloader/readers/anton_paar_saxs_reader.py (modified) (8 diffs)
• sascalc/dataloader/readers/ascii_reader.py (modified) (1 diff)
• sascalc/dataloader/readers/associations.py (modified) (1 diff)
• sascalc/dataloader/readers/cansas_reader.py (modified) (18 diffs)
• sascalc/dataloader/readers/cansas_reader_HDF5.py (modified) (8 diffs)
• sascalc/dataloader/readers/danse_reader.py (modified) (5 diffs)
• sascalc/dataloader/readers/defaults.json (deleted)
• sascalc/dataloader/readers/hfir1d_reader.py (deleted)
• sascalc/dataloader/readers/nexus_reader.py (deleted)
• sascalc/dataloader/readers/red2d_reader.py (modified) (18 diffs)
• sascalc/dataloader/readers/sesans_reader.py (modified) (4 diffs)
• sascalc/dataloader/readers/xml_reader.py (modified) (6 diffs)
• sascalc/file_converter/ascii2d_loader.py (added)
• sasgui/guiframe/local_perspectives/data_loader/data_loader.py (modified) (11 diffs)
• sasgui/perspectives/file_converter/converter_panel.py (modified) (12 diffs)
• sasgui/perspectives/file_converter/file_converter.py (modified) (1 diff)
• sasgui/perspectives/file_converter/media/file_converter_help.rst (modified) (4 diffs)

src/sas/sascalc/pr/c_extensions/Cinvertor.c

@@ -294 +294 @@
 }
 
-const char set_has_bck_doc[] =
+const char set_est_bck_doc[] =
         "Sets background flag\n";
 
@@ -300 +300 @@
  * Sets the maximum distance
  */
-static PyObject * set_has_bck(Cinvertor *self, PyObject *args) {
-        int has_bck;
-
-        if (!PyArg_ParseTuple(args, "i", &has_bck)) return NULL;
-        self->params.has_bck = has_bck;
-        return Py_BuildValue("i", self->params.has_bck);
-}
-
-const char get_has_bck_doc[] =
+static PyObject * set_est_bck(Cinvertor *self, PyObject *args) {
+        int est_bck;
+
+        if (!PyArg_ParseTuple(args, "i", &est_bck)) return NULL;
+        self->params.est_bck = est_bck;
+        return Py_BuildValue("i", self->params.est_bck);
+}
+
+const char get_est_bck_doc[] =
         "Gets background flag\n";
 
@@ -314 +314 @@
  * Gets the maximum distance
  */
-static PyObject * get_has_bck(Cinvertor *self, PyObject *args) {
-        return Py_BuildValue("i", self->params.has_bck);
+static PyObject * get_est_bck(Cinvertor *self, PyObject *args) {
+        return Py_BuildValue("i", self->params.est_bck);
 }
 
@@ -882 +882 @@
         sqrt_alpha = sqrt(self->params.alpha);
         pi = acos(-1.0);
-        offset = (self->params.has_bck==1) ? 0 : 1;
+        offset = (self->params.est_bck==1) ? 0 : 1;
 
     for (j=0; j<nfunc; j++) {
@@ -892 +892 @@
             }
             if (accept_q(self, self->params.x[i])){
-                if (self->params.has_bck==1 && j==0) {
+                if (self->params.est_bck==1 && j==0) {
                     a[i*nfunc+j] = 1.0/self->params.err[i];
                 } else {
@@ -906 +906 @@
         }
         for (i_r=0; i_r<nr; i_r++){
-            if (self->params.has_bck==1 && j==0) {
+            if (self->params.est_bck==1 && j==0) {
                 a[(i_r+self->params.npoints)*nfunc+j] = 0.0;
             } else {
@@ -1029 +1029 @@
                    {"set_slit_height", (PyCFunction)set_slit_height, METH_VARARGS, set_slit_height_doc},
                    {"get_slit_height", (PyCFunction)get_slit_height, METH_VARARGS, get_slit_height_doc},
-                   {"set_has_bck", (PyCFunction)set_has_bck, METH_VARARGS, set_has_bck_doc},
-                   {"get_has_bck", (PyCFunction)get_has_bck, METH_VARARGS, get_has_bck_doc},
+                   {"set_est_bck", (PyCFunction)set_est_bck, METH_VARARGS, set_est_bck_doc},
+                   {"get_est_bck", (PyCFunction)get_est_bck, METH_VARARGS, get_est_bck_doc},
                    {"get_nx", (PyCFunction)get_nx, METH_VARARGS, get_nx_doc},
                    {"get_ny", (PyCFunction)get_ny, METH_VARARGS, get_ny_doc},

src/sas/sascalc/pr/c_extensions/invertor.c

@@ -20 +20 @@
         pars->q_min = -1.0;
         pars->q_max = -1.0;
-        pars->has_bck = 0;
+        pars->est_bck = 0;
 }
 
@@ -313 +313 @@
     return sqrt(sum_r2/(2.0*sum));
 }
-

src/sas/sascalc/pr/c_extensions/invertor.h

@@ -27 +27 @@
     double q_max;
     /// Flag for whether or not to evalute a constant background while inverting
-    int has_bck;
+    int est_bck;
     /// Slit height in units of q [A-1]
     double slit_height;

src/sas/sascalc/pr/invertor.py

@@ -121 +121 @@
          self.q_min, self.q_max,
          self.x, self.y,
-         self.err, self.has_bck,
+         self.err, self.est_bck,
          self.slit_height, self.slit_width) = state
 
@@ -133 +133 @@
                  self.q_min, self.q_max,
                  self.x, self.y,
-                 self.err, self.has_bck,
+                 self.err, self.est_bck,
                  self.slit_height, self.slit_width,
                 )
@@ -175 +175 @@
         elif name == 'slit_width':
             return self.set_slit_width(value)
-        elif name == 'has_bck':
+        elif name == 'est_bck':
             if value == True:
-                return self.set_has_bck(1)
+                return self.set_est_bck(1)
             elif value == False:
-                return self.set_has_bck(0)
+                return self.set_est_bck(0)
             else:
-                raise ValueError, "Invertor: has_bck can only be True or False"
+                raise ValueError, "Invertor: est_bck can only be True or False"
 
         return Cinvertor.__setattr__(self, name, value)
@@ -220 +220 @@
         elif name == 'slit_width':
             return self.get_slit_width()
-        elif name == 'has_bck':
-            value = self.get_has_bck()
+        elif name == 'est_bck':
+            value = self.get_est_bck()
             if value == 1:
                 return True
@@ -248 +248 @@
         invertor.y = self.y
         invertor.err = self.err
-        invertor.has_bck = self.has_bck
+        invertor.est_bck = self.est_bck
+        invertor.background = self.background
         invertor.slit_height = self.slit_height
         invertor.slit_width = self.slit_width
@@ -290 +291 @@
         """
         # Reset the background value before proceeding
-        self.background = 0.0
-        return self.lstsq(nfunc, nr=nr)
+        # self.background = 0.0
+        if not self.est_bck:
+            self.y -= self.background
+        out, cov = self.lstsq(nfunc, nr=nr)
+        if not self.est_bck:
+            self.y += self.background
+        return out, cov
 
     def iq(self, out, q):
@@ -454 +460 @@
 
         # If we need to fit the background, add a term
-        if self.has_bck == True:
+        if self.est_bck == True:
             nfunc_0 = nfunc
             nfunc += 1
@@ -500 +506 @@
 
         # Keep a copy of the last output
-        if self.has_bck == False:
-            self.background = 0
+        if self.est_bck == False:
             self.out = c
             self.cov = err
@@ -653 +658 @@
         file.write("#slit_width=%g\n" % self.slit_width)
         file.write("#background=%g\n" % self.background)
-        if self.has_bck == True:
+        if self.est_bck == True:
             file.write("#has_bck=1\n")
         else:
@@ -734 +739 @@
                         toks = line.split('=')
                         if int(toks[1]) == 1:
-                            self.has_bck = True
+                            self.est_bck = True
                         else:
-                            self.has_bck = False
+                            self.est_bck = False
 
                     # Now read in the parameters

src/sas/sasgui/perspectives/calculator/model_editor.py

@@ -643 +643 @@
         self.name_hsizer = None
         self.name_tcl = None
+        self.overwrite_cb = None
         self.desc_sizer = None
         self.desc_tcl = None
@@ -657 +658 @@
         self.warning = ""
         #This does not seem to be used anywhere so commenting out for now
-        #    -- PDB 2/26/17 
+        #    -- PDB 2/26/17
         #self._description = "New Plugin Model"
         self.function_tcl = None
@@ -689 +690 @@
         #title name [string]
         name_txt = wx.StaticText(self, -1, 'Function Name : ')
-        overwrite_cb = wx.CheckBox(self, -1, "Overwrite existing plugin model of this name?", (10, 10))
-        overwrite_cb.SetValue(False)
-        overwrite_cb.SetToolTipString("Overwrite it if already exists?")
-        wx.EVT_CHECKBOX(self, overwrite_cb.GetId(), self.on_over_cb)
+        self.overwrite_cb = wx.CheckBox(self, -1, "Overwrite existing plugin model of this name?", (10, 10))
+        self.overwrite_cb.SetValue(False)
+        self.overwrite_cb.SetToolTipString("Overwrite it if already exists?")
+        wx.EVT_CHECKBOX(self, self.overwrite_cb.GetId(), self.on_over_cb)
         self.name_tcl = wx.TextCtrl(self, -1, size=(PANEL_WIDTH * 3 / 5, -1))
         self.name_tcl.Bind(wx.EVT_TEXT_ENTER, self.on_change_name)
@@ -700 +701 @@
         self.name_tcl.SetToolTipString(hint_name)
         self.name_hsizer.AddMany([(self.name_tcl, 0, wx.LEFT | wx.TOP, 0),
-                                  (overwrite_cb, 0, wx.LEFT, 20)])
+                                  (self.overwrite_cb, 0, wx.LEFT, 20)])
         self.name_sizer.AddMany([(name_txt, 0, wx.LEFT | wx.TOP, 10),
                                  (self.name_hsizer, 0,
@@ -740 +741 @@
         self.param_sizer.AddMany([(param_txt, 0, wx.LEFT, 10),
                                   (self.param_tcl, 1, wx.EXPAND | wx.ALL, 10)])
-        
+
         # Parameters with polydispersity
         pd_param_txt = wx.StaticText(self, -1, 'Fit Parameters requiring ' + \
@@ -755 +756 @@
         self.pd_param_tcl.setDisplayLineNumbers(True)
         self.pd_param_tcl.SetToolTipString(pd_param_tip)
-        
+
         self.param_sizer.AddMany([(pd_param_txt, 0, wx.LEFT, 10),
                                   (self.pd_param_tcl, 1, wx.EXPAND | wx.ALL, 10)])
@@ -995 +996 @@
             info = 'Error'
             color = 'red'
+            self.overwrite_cb.SetValue(True)
+            self.overwrite_name = True
         else:
             self._notes = result
@@ -1030 +1033 @@
         if has_scipy:
             lines.insert(0, 'import scipy')
-        
-        # Think about 2D later        
+
+        # Think about 2D later
         #self.is_2d = func_str.count("#self.ndim = 2")
         #line_2d = ''
         #if self.is_2d:
         #    line_2d = CUSTOM_2D_TEMP.split('\n')
-        
-        # Also think about test later        
+
+        # Also think about test later
         #line_test = TEST_TEMPLATE.split('\n')
         #local_params = ''
@@ -1043 +1046 @@
         spaces4  = ' '*4
         spaces13 = ' '*13
-        spaces16 = ' '*16     
+        spaces16 = ' '*16
         param_names = []    # to store parameter names
         has_scipy = func_str.count("scipy.")
@@ -1055 +1058 @@
             out_f.write(line + '\n')
             if line.count('#name'):
-                out_f.write('name = "%s" \n' % name)               
+                out_f.write('name = "%s" \n' % name)
             elif line.count('#title'):
-                out_f.write('title = "User model for %s"\n' % name)               
+                out_f.write('title = "User model for %s"\n' % name)
             elif line.count('#description'):
-                out_f.write('description = "%s"\n' % desc_str)               
+                out_f.write('description = "%s"\n' % desc_str)
             elif line.count('#parameters'):
                 out_f.write('parameters = [ \n')
@@ -1065 +1068 @@
                     p_line = param_line.lstrip().rstrip()
                     if p_line:
-                        pname, pvalue = self.get_param_helper(p_line)
+                        pname, pvalue, desc = self.get_param_helper(p_line)
                         param_names.append(pname)
-                        out_f.write("%s['%s', '', %s, [-numpy.inf, numpy.inf], '', ''],\n" % (spaces16, pname, pvalue))
+                        out_f.write("%s['%s', '', %s, [-numpy.inf, numpy.inf], '', '%s'],\n" % (spaces16, pname, pvalue, desc))
                 for param_line in pd_param_str.split('\n'):
                     p_line = param_line.lstrip().rstrip()
                     if p_line:
-                        pname, pvalue = self.get_param_helper(p_line)
+                        pname, pvalue, desc = self.get_param_helper(p_line)
                         param_names.append(pname)
-                        out_f.write("%s['%s', '', %s, [-numpy.inf, numpy.inf], 'volume', ''],\n" % (spaces16, pname, pvalue))
+                        out_f.write("%s['%s', '', %s, [-numpy.inf, numpy.inf], 'volume', '%s'],\n" % (spaces16, pname, pvalue, desc))
                 out_f.write('%s]\n' % spaces13)
-            
+
         # No form_volume or ER available in simple model editor
         out_f.write('def form_volume(*arg): \n')
@@ -1082 +1085 @@
         out_f.write('def ER(*arg): \n')
         out_f.write('    return 1.0 \n')
-        
+
         # function to compute
         out_f.write('\n')
@@ -1091 +1094 @@
         for func_line in func_str.split('\n'):
             out_f.write('%s%s\n' % (spaces4, func_line))
-        
+
         Iqxy_string = 'return Iq(numpy.sqrt(x**2+y**2) '
-            
+
         out_f.write('\n')
         out_f.write('def Iqxy(x, y ')
@@ -1113 +1116 @@
         items = line.split(";")
         for item in items:
-            name = item.split("=")[0].lstrip().rstrip()
+            name = item.split("=")[0].strip()
+            description = ""
             try:
-                value = item.split("=")[1].lstrip().rstrip()
+                value = item.split("=")[1].strip()
+                if value.count("#"):
+                    # If line ends in a comment, remove it before parsing float
+                    index = value.index("#")
+                    description = value[(index + 1):].strip()
+                    value = value[:value.index("#")].strip()
                 float(value)
-            except:
+            except ValueError:
                 value = 1.0 # default
 
-        return name, value
+        return name, value, description
 
     def set_function_helper(self, line):
@@ -1204 +1213 @@
 import numpy
 
-#name 
+#name
 
 #title
@@ -1210 +1219 @@
 #description
 
-#parameters 
+#parameters
 
 """

src/sas/sasgui/perspectives/calculator/pyconsole.py

@@ -37 +37 @@
     Iqxy = model.evalDistribution([qx, qy])
 
-    result = """
-    Iq(%s) = %s
-    Iqxy(%s, %s) = %s
-    """%(q, Iq, qx, qy, Iqxy)
+    # check the model's unit tests run
+    from sasmodels.model_test import run_one
+    result = run_one(path)
 
     return result
@@ -89 +88 @@
         ok = wx.Button(self, wx.ID_OK, "OK")
 
-        # Mysterious constraint layouts from 
+        # Mysterious constraint layouts from
         # https://www.wxpython.org/docs/api/wx.lib.layoutf.Layoutf-class.html
         lc = layoutf.Layoutf('t=t5#1;b=t5#2;l=l5#1;r=r5#1', (self,ok))

src/sas/sasgui/perspectives/fitting/fitting.py

@@ -257 +257 @@
         toks = os.path.splitext(label)
         path = os.path.join(models.find_plugins_dir(), toks[0])
+        message = "Are you sure you want to delete the file {}?".format(path)
+        dlg = wx.MessageDialog(self.frame, message, '', wx.YES_NO | wx.ICON_QUESTION)
+        if not dlg.ShowModal() == wx.ID_YES:
+            return
         try:
             for ext in ['.py', '.pyc']:
                 p_path = path + ext
+                if ext == '.pyc' and not os.path.isfile(path + ext):
+                    # If model is invalid, .pyc file may not exist as model has
+                    # never been compiled. Don't try and delete it
+                    continue
                 os.remove(p_path)
             self.update_custom_combo()
@@ -361 +369 @@
                                    'Add a new model function')
         wx.EVT_MENU(owner, wx_id, self.make_new_model)
-        
+
         wx_id = wx.NewId()
         self.edit_model_menu.Append(wx_id, 'Sum|Multi(p1, p2)',
@@ -383 +391 @@
           '(Re)Load all models present in user plugin_models folder')
         wx.EVT_MENU(owner, wx_id, self.load_plugin_models)
-                
+
     def set_edit_menu_helper(self, owner=None, menu=None):
         """
@@ -1734 +1742 @@
             @param unsmeared_error: data error, rescaled to unsmeared model
         """
-            
-        number_finite = np.count_nonzero(np.isfinite(y)) 
+
+        number_finite = np.count_nonzero(np.isfinite(y))
         np.nan_to_num(y)
         new_plot = self.create_theory_1D(x, y, page_id, model, data, state,
@@ -1794 +1802 @@
             msg = "Computing Error: Model did not return any finite value."
             wx.PostEvent(self.parent, StatusEvent(status = msg, info="error"))
-        else:                 
+        else:
             msg = "Computation  completed!"
             if number_finite != y.size:
@@ -1824 +1832 @@
         that can be plot.
         """
-        number_finite = np.count_nonzero(np.isfinite(image)) 
+        number_finite = np.count_nonzero(np.isfinite(image))
         np.nan_to_num(image)
         new_plot = Data2D(image=image, err_image=data.err_data)
@@ -1927 +1935 @@
                 ## and may be the cause of other noted instabilities
                 ##
-                ##    -PDB August 12, 2014 
+                ##    -PDB August 12, 2014
                 while self.calc_2D.isrunning():
                     time.sleep(0.1)
@@ -1969 +1977 @@
             if (self.calc_1D is not None) and self.calc_1D.isrunning():
                 self.calc_1D.stop()
-                ## stop just raises the flag -- the thread is supposed to 
+                ## stop just raises the flag -- the thread is supposed to
                 ## then kill itself but cannot.  Paul Kienzle came up with
                 ## this fix to prevent threads from stepping on each other
@@ -1981 +1989 @@
                 ## a request to stop the computation.
                 ## It seems thus that the whole thread approach used here
-                ## May need rethinking  
+                ## May need rethinking
                 ##
                 ##    -PDB August 12, 2014
@@ -2146 +2154 @@
             residuals.dxw = None
             residuals.ytransform = 'y'
-            # For latter scale changes 
+            # For latter scale changes
             residuals.xaxis('\\rm{Q} ', 'A^{-1}')
             residuals.yaxis('\\rm{Residuals} ', 'normalized')

src/sas/sasgui/perspectives/fitting/models.py

@@ -186 +186 @@
             try:
                 model = load_custom_model(path)
-                model.name = PLUGIN_NAME_BASE + model.name
+                if not model.name.count(PLUGIN_NAME_BASE):
+                    model.name = PLUGIN_NAME_BASE + model.name
                 plugins[model.name] = model
             except Exception:
@@ -297 +298 @@
         for name, plug in self.stored_plugins.iteritems():
             self.model_dictionary[name] = plug
-        
+
         self._get_multifunc_models()
 

src/sas/sasgui/perspectives/pr/inversion_panel.py

@@ -70 +70 @@
         self.rg_ctl = None
         self.iq0_ctl = None
-        self.bck_chk = None
+        self.bck_value = None
+        self.bck_est_ctl = None
+        self.bck_man_ctl = None
+        self.est_bck = True
+        self.bck_input = None
         self.bck_ctl = None
 
@@ -312 +316 @@
         # Read the panel's parameters
         flag, alpha, dmax, nfunc, qmin, \
-        qmax, height, width = self._read_pars()
+        qmax, height, width, bck = self._read_pars()
 
         state.nfunc = nfunc
@@ -326 +330 @@
 
         # Background evaluation checkbox
-        state.estimate_bck = self.bck_chk.IsChecked()
+        state.estimate_bck = self.est_bck
+        state.bck_value = bck
 
         # Estimates
@@ -371 +376 @@
         self.plot_data.SetValue(str(state.file))
 
-        # Background evaluation checkbox
-        self.bck_chk.SetValue(state.estimate_bck)
+        # Background value
+        self.bck_est_ctl.SetValue(state.estimate_bck)
+        self.bck_man_ctl.SetValue(not state.estimate_bck)
+        if not state.estimate_bck:
+            self.bck_input.Enable()
+            self.bck_input.SetValue(str(state.bck_value))
+        self.est_bck = state.estimate_bck
+        self.bck_value = state.bck_value
 
         # Estimates
@@ -431 +442 @@
                        wx.EXPAND | wx.LEFT | wx.RIGHT | wx.ADJUST_MINSIZE, 15)
 
-        self.bck_chk = wx.CheckBox(self, -1, "Estimate background level")
-        hint_msg = "Check box to let the fit estimate "
-        hint_msg += "the constant background level."
-        self.bck_chk.SetToolTipString(hint_msg)
-        self.bck_chk.Bind(wx.EVT_CHECKBOX, self._on_pars_changed)
+        radio_sizer = wx.GridBagSizer(5, 5)
+
+        self.bck_est_ctl = wx.RadioButton(self, -1, "Estimate background level",
+            name="estimate_bck", style=wx.RB_GROUP)
+        self.bck_man_ctl = wx.RadioButton(self, -1, "Input manual background level",
+            name="manual_bck")
+
+        self.bck_est_ctl.Bind(wx.EVT_RADIOBUTTON, self._on_bck_changed)
+        self.bck_man_ctl.Bind(wx.EVT_RADIOBUTTON, self._on_bck_changed)
+
+        radio_sizer.Add(self.bck_est_ctl, (0,0), (1,1), wx.LEFT | wx.EXPAND)
+        radio_sizer.Add(self.bck_man_ctl, (0,1), (1,1), wx.RIGHT | wx.EXPAND)
+
         iy += 1
-        pars_sizer.Add(self.bck_chk, (iy, 0), (1, 2),
+        pars_sizer.Add(radio_sizer, (iy, 0), (1, 2),
                        wx.LEFT | wx.EXPAND | wx.ADJUST_MINSIZE, 15)
+
+        background_label = wx.StaticText(self, -1, "Background: ")
+        self.bck_input = PrTextCtrl(self, -1, style=wx.TE_PROCESS_ENTER,
+            size=(60, 20), value="0.0")
+        self.bck_input.Disable()
+        self.bck_input.Bind(wx.EVT_TEXT, self._read_pars)
+        background_units = wx.StaticText(self, -1, "[A^(-1)]", size=(55, 20))
+        iy += 1
+
+        background_sizer = wx.GridBagSizer(5, 5)
+
+        background_sizer.Add(background_label, (0, 0), (1,1),
+            wx.LEFT | wx.EXPAND | wx.ADJUST_MINSIZE, 23)
+        background_sizer.Add(self.bck_input, (0, 1), (1,1),
+            wx.LEFT | wx.ADJUST_MINSIZE, 5)
+        background_sizer.Add(background_units, (0, 2), (1,1),
+            wx.LEFT | wx.ADJUST_MINSIZE, 5)
+        pars_sizer.Add(background_sizer, (iy, 0), (1, 2),
+            wx.LEFT | wx.EXPAND | wx.ADJUST_MINSIZE, 15)
+
         boxsizer1.Add(pars_sizer, 0, wx.EXPAND)
         vbox.Add(boxsizer1, (iy_vb, 0), (1, 1),
@@ -764 +803 @@
         self._on_pars_changed()
 
+    def _on_bck_changed(self, evt=None):
+        self.est_bck = self.bck_est_ctl.GetValue()
+        if self.est_bck:
+            self.bck_input.Disable()
+        else:
+            self.bck_input.Enable()
+
     def _on_pars_changed(self, evt=None):
         """
@@ -770 +816 @@
         scenes.
         """
-        flag, alpha, dmax, nfunc, qmin, qmax, height, width = self._read_pars()
-        has_bck = self.bck_chk.IsChecked()
+        flag, alpha, dmax, nfunc, qmin, qmax, height, width, bck = self._read_pars()
 
         # If the pars are valid, estimate alpha
@@ -783 +828 @@
                                                       d_max=dmax,
                                                       q_min=qmin, q_max=qmax,
-                                                      bck=has_bck,
+                                                      est_bck=self.est_bck,
+                                                      bck_val=bck,
                                                       height=height,
                                                       width=width)
@@ -797 +843 @@
         height = 0
         width = 0
+        background = 0
         flag = True
         # Read slit height
@@ -890 +937 @@
             self.qmax_ctl.Refresh()
 
-        return flag, alpha, dmax, nfunc, qmin, qmax, height, width
+        # Read background
+        if not self.est_bck:
+            try:
+                bck_str = self.bck_input.GetValue()
+                if len(bck_str.strip()) == 0:
+                    background = 0.0
+                else:
+                    background = float(bck_str)
+                    self.bck_input.SetBackgroundColour(wx.WHITE)
+            except ValueError:
+                background = 0.0
+                self.bck_input.SetBackgroundColour("pink")
+            self.bck_input.Refresh()
+
+        return flag, alpha, dmax, nfunc, qmin, qmax, height, width, background
 
     def _on_explore(self, evt):
@@ -915 +976 @@
         # Push it to the manager
 
-        flag, alpha, dmax, nfunc, qmin, qmax, height, width = self._read_pars()
-        has_bck = self.bck_chk.IsChecked()
+        flag, alpha, dmax, nfunc, qmin, qmax, height, width, bck = self._read_pars()
 
         if flag:
@@ -928 +988 @@
                                                    d_max=dmax,
                                                    q_min=qmin, q_max=qmax,
-                                                   bck=has_bck,
+                                                   est_bck=self.est_bck,
+                                                   bck_val = bck,
                                                    height=height,
                                                    width=width)

src/sas/sasgui/perspectives/pr/inversion_state.py

@@ -36 +36 @@
            ["qmin", "qmin"],
            ["qmax", "qmax"],
-           ["estimate_bck", "estimate_bck"]]
+           ["estimate_bck", "estimate_bck"],
+           ["bck_value", "bck_value"]]
 
 ## List of P(r) inversion outputs
@@ -62 +63 @@
         self.estimate_bck = False
         self.timestamp = time.time()
+        self.bck_value = 0.0
 
         # Inversion parameters
@@ -109 +111 @@
         state += "Timestamp:    %s\n" % self.timestamp
         state += "Estimate bck: %s\n" % str(self.estimate_bck)
+        state += "Bck Value:    %s\n" % str(self.bck_value)
         state += "No. terms:    %s\n" % str(self.nfunc)
         state += "D_max:        %s\n" % str(self.d_max)
@@ -296 +299 @@
                             self.coefficients.append(float(c))
                         except:
-                            # Bad data, skip. We will count the number of 
-                            # coefficients at the very end and deal with 
+                            # Bad data, skip. We will count the number of
+                            # coefficients at the very end and deal with
                             # inconsistencies then.
                             pass
@@ -329 +332 @@
                                 cov_row.append(float(c))
                             except:
-                                # Bad data, skip. We will count the number of 
-                                # coefficients at the very end and deal with 
+                                # Bad data, skip. We will count the number of
+                                # coefficients at the very end and deal with
                                 # inconsistencies then.
                                 pass
@@ -461 +464 @@
                 tree = etree.parse(path, parser=etree.ETCompatXMLParser())
                 # Check the format version number
-                # Specifying the namespace will take care of the file 
-                #format version 
+                # Specifying the namespace will take care of the file
+                #format version
                 root = tree.getroot()
 

src/sas/sasgui/perspectives/pr/media/pr_help.rst

@@ -49 +49 @@
 P(r) inversion requires that the background be perfectly subtracted.  This is
 often difficult to do well and thus many data sets will include a background.
-For those cases, the user should check the "estimate background" box and the
-module will do its best to estimate it.
+For those cases, the user should check the "Estimate background level" option
+and the module will do its best to estimate it. If you know the background value
+for your data, select the "Input manual background level" option. Note that
+this value will be treated as having 0 error.
 
 The P(r) module is constantly computing in the background what the optimum

src/sas/sasgui/perspectives/pr/pr.py

@@ -68 +68 @@
         self.q_min = None
         self.q_max = None
-        self.has_bck = False
+        self.est_bck = False
+        self.bck_val = 0
         self.slit_height = 0
         self.slit_width = 0
@@ -828 +829 @@
         self.control_panel.iq0 = pr.iq0(out)
         self.control_panel.bck = pr.background
+        self.control_panel.bck_input.SetValue("{:.2g}".format(pr.background))
 
         # Show I(q) fit
@@ -907 +909 @@
 
     def setup_plot_inversion(self, alpha, nfunc, d_max, q_min=None, q_max=None,
-                             bck=False, height=0, width=0):
+                             est_bck=False, bck_val=0, height=0, width=0):
         """
             Set up inversion from plotted data
@@ -916 +918 @@
         self.q_min = q_min
         self.q_max = q_max
-        self.has_bck = bck
+        self.est_bck = est_bck
+        self.bck_val = bck_val
         self.slit_height = height
         self.slit_width = width
@@ -930 +933 @@
     def estimate_plot_inversion(self, alpha, nfunc, d_max,
                                 q_min=None, q_max=None,
-                                bck=False, height=0, width=0):
+                                est_bck=False, bck_val=0, height=0, width=0):
         """
             Estimate parameters from plotted data
@@ -939 +942 @@
         self.q_min = q_min
         self.q_max = q_max
-        self.has_bck = bck
+        self.est_bck = est_bck
+        self.bck_val = bck_val
         self.slit_height = height
         self.slit_width = width
@@ -973 +977 @@
         pr.x = self.current_plottable.x
         pr.y = self.current_plottable.y
-        pr.has_bck = self.has_bck
+        pr.est_bck = self.est_bck
         pr.slit_height = self.slit_height
         pr.slit_width = self.slit_width
+        pr.background = self.bck_val
 
         # Keep track of the plot window title to ensure that
@@ -1019 +1024 @@
         self.q_min = q_min
         self.q_max = q_max
-        self.has_bck = bck
+        self.est_bck = bck
         self.slit_height = height
         self.slit_width = width
@@ -1042 +1047 @@
         self.q_min = q_min
         self.q_max = q_max
-        self.has_bck = bck
+        self.est_bck = bck
         self.slit_height = height
         self.slit_width = width
@@ -1115 +1120 @@
             pr.y = y
             pr.err = err
-            pr.has_bck = self.has_bck
+            pr.est_bck = self.est_bck
             pr.slit_height = self.slit_height
             pr.slit_width = self.slit_width

src/sas/sascalc/data_util/registry.py

@@ -1 @@
-# This program is public domain
 """
 File extension registry.
@@ -8 +7 @@
 from __future__ import print_function
 
-import os.path
+from sas.sascalc.dataloader.loader_exceptions import NoKnownLoaderException
+
 
 class ExtensionRegistry(object):
@@ -22 +22 @@
         # Add an association by setting an element
         registry['.zip'] = unzip
-        
+
         # Multiple extensions for one loader
         registry['.tgz'] = untar
         registry['.tar.gz'] = untar
 
-        # Generic extensions to use after trying more specific extensions; 
+        # Generic extensions to use after trying more specific extensions;
         # these will be checked after the more specific extensions fail.
         registry['.gz'] = gunzip
@@ -38 +38 @@
         # Show registered extensions
         print registry.extensions()
-        
+
         # Can also register a format name for explicit control from caller
         registry['cx3'] = cx3
@@ -62 +62 @@
     def __init__(self, **kw):
         self.loaders = {}
+
     def __setitem__(self, ext, loader):
         if ext not in self.loaders:
             self.loaders[ext] = []
         self.loaders[ext].insert(0,loader)
+
     def __getitem__(self, ext):
         return self.loaders[ext]
+
     def __contains__(self, ext):
         return ext in self.loaders
+
     def formats(self):
         """
@@ -77 +81 @@
         names.sort()
         return names
+
     def extensions(self):
         """
@@ -84 +89 @@
         exts.sort()
         return exts
+
     def lookup(self, path):
         """
         Return the loader associated with the file type of path.
-        
-        Raises ValueError if file type is not known.
-        """        
+
+        :param path: Data file path
+        :raises ValueError: When no loaders are found for the file.
+        :return: List of available readers for the file extension
+        """
         # Find matching extensions
         extlist = [ext for ext in self.extensions() if path.endswith(ext)]
@@ -106 +114 @@
         # Raise an error if there are no matching extensions
         if len(loaders) == 0:
-            raise ValueError, "Unknown file type for "+path
-        # All done
+            raise ValueError("Unknown file type for "+path)
         return loaders
+
     def load(self, path, format=None):
         """
         Call the loader for the file type of path.
 
-        Raises ValueError if no loader is available.
-        Raises KeyError if format is not available.
-        May raise a loader-defined exception if loader fails.        
+        :raise ValueError: if no loader is available.
+        :raise KeyError: if format is not available.
+        May raise a loader-defined exception if loader fails.
         """
+        loaders = []
         if format is None:
-            loaders = self.lookup(path)
+            try:
+                loaders = self.lookup(path)
+            except ValueError as e:
+                pass
         else:
-            loaders = self.loaders[format]
+            try:
+                loaders = self.loaders[format]
+            except KeyError as e:
+                pass
+        last_exc = None
         for fn in loaders:
             try:
                 return fn(path)
-            except:
-                pass # give other loaders a chance to succeed
+            except Exception as e:
+                last_exc = e
+                pass  # give other loaders a chance to succeed
         # If we get here it is because all loaders failed
-        raise # reraises last exception
-
-def test():
-    reg = ExtensionRegistry()
-    class CxError(Exception): pass
-    def cx(file): return 'cx'
-    def new_cx(file): return 'new_cx'
-    def fail_cx(file): raise CxError
-    def cat(file): return 'cat'
-    def gunzip(file): return 'gunzip'
-    reg['.cx'] = cx
-    reg['.cx1'] = cx
-    reg['.cx'] = new_cx
-    reg['.gz'] = gunzip
-    reg['.cx.gz'] = new_cx
-    reg['.cx1.gz'] = fail_cx
-    reg['.cx1'] = fail_cx
-    reg['.cx2'] = fail_cx
-    reg['new_cx'] = new_cx
-
-    # Two loaders associated with .cx
-    assert reg.lookup('hello.cx') == [new_cx,cx]
-    # Make sure the last loader applies first
-    assert reg.load('hello.cx') == 'new_cx'
-    # Make sure the next loader applies if the first fails
-    assert reg.load('hello.cx1') == 'cx'
-    # Make sure the format override works
-    assert reg.load('hello.cx1',format='.cx.gz') == 'new_cx'
-    # Make sure the format override works
-    assert reg.load('hello.cx1',format='new_cx') == 'new_cx'
-    # Make sure the case of all loaders failing is correct
-    try:  reg.load('hello.cx2')
-    except CxError: pass # correct failure
-    else: raise AssertError,"Incorrect error on load failure"
-    # Make sure the case of no loaders fails correctly
-    try: reg.load('hello.missing')
-    except ValueError,msg:
-        assert str(msg)=="Unknown file type for hello.missing",'Message: <%s>'%(msg)
-    else: raise AssertError,"No error raised for missing extension"
-    assert reg.formats() == ['new_cx']
-    assert reg.extensions() == ['.cx','.cx.gz','.cx1','.cx1.gz','.cx2','.gz']
-    # make sure that it supports multiple '.' in filename
-    assert reg.load('hello.extra.cx1') == 'cx'
-    assert reg.load('hello.gz') == 'gunzip'
-    assert reg.load('hello.cx1.gz') == 'gunzip' # Since .cx1.gz fails
-
-if __name__ == "__main__": test()
+        if last_exc is not None and len(loaders) != 0:
+            # If file has associated loader(s) and they;ve failed
+            raise last_exc
+        raise NoKnownLoaderException(e.message)  # raise generic exception

src/sas/sascalc/dataloader/data_info.py

@@ -1181 +1181 @@
         return return_string
 
-    final_dataset.xmax = data.xmax
-    final_dataset.ymax = data.ymax
-    final_dataset.xmin = data.xmin
-    final_dataset.ymin = data.ymin
+    if hasattr(data, "xmax"):
+        final_dataset.xmax = data.xmax
+    if hasattr(data, "ymax"):
+        final_dataset.ymax = data.ymax
+    if hasattr(data, "xmin"):
+        final_dataset.xmin = data.xmin
+    if hasattr(data, "ymin"):
+        final_dataset.ymin = data.ymin
     final_dataset.isSesans = datainfo.isSesans
     final_dataset.title = datainfo.title

src/sas/sascalc/dataloader/loader.py

@@ -1 +1 @@
 """
     File handler to support different file extensions.
-    Uses reflectometry's registry utility.
+    Uses reflectometer registry utility.
 
     The default readers are found in the 'readers' sub-module
@@ -14 +14 @@
 """
 #####################################################################
-#This software was developed by the University of Tennessee as part of the
-#Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
-#project funded by the US National Science Foundation.
-#See the license text in license.txt
-#copyright 2008, University of Tennessee
+# This software was developed by the University of Tennessee as part of the
+# Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
+# project funded by the US National Science Foundation.
+# See the license text in license.txt
+# copyright 2008, University of Tennessee
 ######################################################################
 
@@ -29 +29 @@
 # Default readers are defined in the readers sub-module
 import readers
+from loader_exceptions import NoKnownLoaderException, FileContentsException,\
+    DefaultReaderException
 from readers import ascii_reader
 from readers import cansas_reader
+from readers import cansas_reader_HDF5
 
 logger = logging.getLogger(__name__)
+
 
 class Registry(ExtensionRegistry):
@@ -39 +43 @@
     Readers and writers are supported.
     """
-
     def __init__(self):
         super(Registry, self).__init__()
 
-        ## Writers
+        # Writers
         self.writers = {}
 
-        ## List of wildcards
+        # List of wildcards
         self.wildcards = ['All (*.*)|*.*']
 
-        ## Creation time, for testing
+        # Creation time, for testing
         self._created = time.time()
 
@@ -63 +66 @@
             of a particular reader
 
-        Defaults to the ascii (multi-column) reader
-        if no reader was registered for the file's
-        extension.
+        Defaults to the ascii (multi-column), cansas XML, and cansas NeXuS
+        readers if no reader was registered for the file's extension.
         """
         try:
             return super(Registry, self).load(path, format=format)
-        except:
-            try:
-                # No reader was found. Default to the ascii reader.
-                ascii_loader = ascii_reader.Reader()
-                return ascii_loader.read(path)
-            except:
-                cansas_loader = cansas_reader.Reader()
-                return cansas_loader.read(path)
+        except NoKnownLoaderException as nkl_e:
+            pass  # try the ASCII reader
+        except FileContentsException as fc_exc:
+            # File has an associated reader but it failed
+            raise RuntimeError(fc_exc.message)
+        except Exception:
+            pass
+
+        # File has no associated reader - try the ASCII reader
+        try:
+            ascii_loader = ascii_reader.Reader()
+            return ascii_loader.read(path)
+        except DefaultReaderException:
+            pass  # Loader specific error to try the cansas XML reader
+        except FileContentsException as e:
+            raise RuntimeError(e.message)
+
+        # ASCII reader failed - try CanSAS xML reader
+        try:
+            cansas_loader = cansas_reader.Reader()
+            return cansas_loader.read(path)
+        except DefaultReaderException:
+            pass  # Loader specific error to try the NXcanSAS reader
+        except FileContentsException as e:
+            raise RuntimeError(e.message)
+        except Exception as csr:
+            pass
+
+        # CanSAS XML reader failed - try NXcanSAS reader
+        try:
+            cansas_nexus_loader = cansas_reader_HDF5.Reader()
+            return cansas_nexus_loader.read(path)
+        except DefaultReaderException as e:
+            logging.error("No default loader can load the data")
+            # No known reader available. Give up and throw an error
+            msg = "\n\tUnknown data format: %s.\n\tThe file is not a " % path
+            msg += "known format that can be loaded by SasView.\n"
+            raise NoKnownLoaderException(msg)
+        except FileContentsException as e:
+            raise RuntimeError(e.message)
 
     def find_plugins(self, dir):

src/sas/sascalc/dataloader/readers/__init__.py

@@ -1 @@
-# Backward compatibility with the previous implementation of the default readers
-from associations import register_readers
+# Method to associate extensions to default readers
+from associations import read_associations
 
-# Method to associate extensions to default readers 
-from associations import read_associations
 
 # Method to return the location of the XML settings file

src/sas/sascalc/dataloader/readers/abs_reader.py

@@ -1 +1 @@
 """
+    IGOR 1D data reader
 """
 #####################################################################
-#This software was developed by the University of Tennessee as part of the
-#Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
-#project funded by the US National Science Foundation.
-#See the license text in license.txt
-#copyright 2008, University of Tennessee
+# This software was developed by the University of Tennessee as part of the
+# Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
+# project funded by the US National Science Foundation.
+# See the license text in license.txt
+# copyright 2008, University of Tennessee
 ######################################################################
 
+import logging
 import numpy as np
-import os
-from sas.sascalc.dataloader.data_info import Data1D
-from sas.sascalc.dataloader.data_info import Detector
-
-has_converter = True
-try:
-    from sas.sascalc.data_util.nxsunit import Converter
-except:
-    has_converter = False
-    
-    
-class Reader:
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+from sas.sascalc.dataloader.data_info import DataInfo, plottable_1D, Data1D,\
+    Detector
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException,\
+    DefaultReaderException
+
+logger = logging.getLogger(__name__)
+
+
+class Reader(FileReader):
     """
     Class to load IGOR reduced .ABS files
     """
-    ## File type
+    # File type
     type_name = "IGOR 1D"
-    ## Wildcards
+    # Wildcards
     type = ["IGOR 1D files (*.abs)|*.abs"]
-    ## List of allowed extensions
-    ext = ['.abs', '.ABS']
+    # List of allowed extensions
+    ext = ['.abs']
     
-    def read(self, path):
+    def get_file_contents(self):
         """ 
-        Load data file.
-        
-        :param path: file path
-        
-        :return: Data1D object, or None
+        Get the contents of the file
         
         :raise RuntimeError: when the file can't be opened
         :raise ValueError: when the length of the data vectors are inconsistent
         """
-        if os.path.isfile(path):
-            basename = os.path.basename(path)
-            root, extension = os.path.splitext(basename)
-            if extension.lower() in self.ext:
-                try:
-                    input_f = open(path,'r')
+        buff = self.f_open.read()
+        filepath = self.f_open.name
+        lines = buff.splitlines()
+        self.has_converter = True
+        try:
+            from sas.sascalc.data_util.nxsunit import Converter
+        except:
+            self.has_converter = False
+        self.output = []
+        self.current_datainfo = DataInfo()
+        self.current_datainfo.filename = filepath
+        self.reset_data_list(len(lines))
+        detector = Detector()
+        data_line = 0
+        self.reset_data_list(len(lines))
+        self.current_datainfo.detector.append(detector)
+        self.current_datainfo.filename = filepath
+
+        is_info = False
+        is_center = False
+        is_data_started = False
+
+        base_q_unit = '1/A'
+        base_i_unit = '1/cm'
+        data_conv_q = Converter(base_q_unit)
+        data_conv_i = Converter(base_i_unit)
+
+        for line in lines:
+            # Information line 1
+            if is_info:
+                is_info = False
+                line_toks = line.split()
+
+                # Wavelength in Angstrom
+                try:
+                    value = float(line_toks[1])
+                    if self.has_converter and \
+                            self.current_datainfo.source.wavelength_unit != 'A':
+                        conv = Converter('A')
+                        self.current_datainfo.source.wavelength = conv(value,
+                            units=self.current_datainfo.source.wavelength_unit)
+                    else:
+                        self.current_datainfo.source.wavelength = value
+                except KeyError:
+                    msg = "ABSReader cannot read wavelength from %s" % filepath
+                    self.current_datainfo.errors.append(msg)
+
+                # Detector distance in meters
+                try:
+                    value = float(line_toks[3])
+                    if self.has_converter and detector.distance_unit != 'm':
+                        conv = Converter('m')
+                        detector.distance = conv(value,
+                                        units=detector.distance_unit)
+                    else:
+                        detector.distance = value
                 except:
-                    raise  RuntimeError, "abs_reader: cannot open %s" % path
-                buff = input_f.read()
-                lines = buff.split('\n')
-                x  = np.zeros(0)
-                y  = np.zeros(0)
-                dy = np.zeros(0)
-                dx = np.zeros(0)
-                output = Data1D(x, y, dy=dy, dx=dx)
-                detector = Detector()
-                output.detector.append(detector)
-                output.filename = basename
-                
-                is_info = False
+                    msg = "ABSReader cannot read SDD from %s" % filepath
+                    self.current_datainfo.errors.append(msg)
+
+                # Transmission
+                try:
+                    self.current_datainfo.sample.transmission = \
+                        float(line_toks[4])
+                except ValueError:
+                    # Transmission isn't always in the header
+                    pass
+
+                # Sample thickness in mm
+                try:
+                    value = float(line_toks[5])
+                    if self.has_converter and \
+                            self.current_datainfo.sample.thickness_unit != 'cm':
+                        conv = Converter('cm')
+                        self.current_datainfo.sample.thickness = conv(value,
+                            units=self.current_datainfo.sample.thickness_unit)
+                    else:
+                        self.current_datainfo.sample.thickness = value
+                except ValueError:
+                    # Thickness is not a mandatory entry
+                    pass
+
+            # MON CNT  LAMBDA  DET ANG  DET DIST  TRANS  THICK  AVE   STEP
+            if line.count("LAMBDA") > 0:
+                is_info = True
+
+            # Find center info line
+            if is_center:
                 is_center = False
-                is_data_started = False
-                
-                data_conv_q = None
-                data_conv_i = None
-                
-                if has_converter == True and output.x_unit != '1/A':
-                    data_conv_q = Converter('1/A')
-                    # Test it
-                    data_conv_q(1.0, output.x_unit)
-                    
-                if has_converter == True and output.y_unit != '1/cm':
-                    data_conv_i = Converter('1/cm')
-                    # Test it
-                    data_conv_i(1.0, output.y_unit)
-                
-                for line in lines:
-                    
-                    # Information line 1
-                    if is_info == True:
-                        is_info = False
-                        line_toks = line.split()
-                        
-                        # Wavelength in Angstrom
-                        try:
-                            value = float(line_toks[1])
-                            if has_converter == True and \
-                                output.source.wavelength_unit != 'A':
-                                conv = Converter('A')
-                                output.source.wavelength = conv(value,
-                                        units=output.source.wavelength_unit)
-                            else:
-                                output.source.wavelength = value
-                        except:
-                            #goes to ASC reader
-                            msg = "abs_reader: cannot open %s" % path
-                            raise  RuntimeError, msg
-                        
-                        # Distance in meters
-                        try:
-                            value = float(line_toks[3])
-                            if has_converter == True and \
-                                detector.distance_unit != 'm':
-                                conv = Converter('m')
-                                detector.distance = conv(value,
-                                                units=detector.distance_unit)
-                            else:
-                                detector.distance = value
-                        except:
-                            #goes to ASC reader
-                            msg = "abs_reader: cannot open %s" % path
-                            raise  RuntimeError, msg
-                        # Transmission
-                        try:
-                            output.sample.transmission = float(line_toks[4])
-                        except:
-                            # Transmission is not a mandatory entry
-                            pass
-                    
-                        # Thickness in mm
-                        try:
-                            value = float(line_toks[5])
-                            if has_converter == True and \
-                                output.sample.thickness_unit != 'cm':
-                                conv = Converter('cm')
-                                output.sample.thickness = conv(value,
-                                            units=output.sample.thickness_unit)
-                            else:
-                                output.sample.thickness = value
-                        except:
-                            # Thickness is not a mandatory entry
-                            pass
-                    
-                    #MON CNT   LAMBDA   DET ANG   DET DIST   TRANS   THICK 
-                    #  AVE   STEP
-                    if line.count("LAMBDA") > 0:
-                        is_info = True
-                        
-                    # Find center info line
-                    if is_center == True:
-                        is_center = False
-                        line_toks = line.split()
-                        # Center in bin number
-                        center_x = float(line_toks[0])
-                        center_y = float(line_toks[1])
-                        
-                        # Bin size
-                        if has_converter == True and \
-                            detector.pixel_size_unit != 'mm':
-                            conv = Converter('mm')
-                            detector.pixel_size.x = conv(5.0,
-                                                units=detector.pixel_size_unit)
-                            detector.pixel_size.y = conv(5.0,
-                                                units=detector.pixel_size_unit)
-                        else:
-                            detector.pixel_size.x = 5.0
-                            detector.pixel_size.y = 5.0
-                        
-                        # Store beam center in distance units
-                        # Det 640 x 640 mm
-                        if has_converter == True and \
-                            detector.beam_center_unit != 'mm':
-                            conv = Converter('mm')
-                            detector.beam_center.x = conv(center_x * 5.0,
-                                             units=detector.beam_center_unit)
-                            detector.beam_center.y = conv(center_y * 5.0,
-                                            units=detector.beam_center_unit)
-                        else:
-                            detector.beam_center.x = center_x * 5.0
-                            detector.beam_center.y = center_y * 5.0
-                        
-                        # Detector type
-                        try:
-                            detector.name = line_toks[7]
-                        except:
-                            # Detector name is not a mandatory entry
-                            pass
-                    
-                    #BCENT(X,Y)   A1(mm)   A2(mm)   A1A2DIST(m)   DL/L
-                    #  BSTOP(mm)   DET_TYP
-                    if line.count("BCENT") > 0:
-                        is_center = True
-                        
-                    # Parse the data
-                    if is_data_started == True:
-                        toks = line.split()
-
-                        try:
-                            _x  = float(toks[0])
-                            _y  = float(toks[1])
-                            _dy = float(toks[2])
-                            _dx = float(toks[3])
-                            
-                            if data_conv_q is not None:
-                                _x = data_conv_q(_x, units=output.x_unit)
-                                _dx = data_conv_i(_dx, units=output.x_unit)
-                                
-                            if data_conv_i is not None:
-                                _y = data_conv_i(_y, units=output.y_unit)
-                                _dy = data_conv_i(_dy, units=output.y_unit)
-                           
-                            x = np.append(x, _x)
-                            y = np.append(y, _y)
-                            dy = np.append(dy, _dy)
-                            dx = np.append(dx, _dx)
-                            
-                        except:
-                            # Could not read this data line. If we are here
-                            # it is because we are in the data section. Just
-                            # skip it.
-                            pass
-                            
-                    #The 6 columns are | Q (1/A) | I(Q) (1/cm) | std. dev.
-                    # I(Q) (1/cm) | sigmaQ | meanQ | ShadowFactor|
-                    if line.count("The 6 columns") > 0:
-                        is_data_started = True
-            
-                # Sanity check
-                if not len(y) == len(dy):
-                    msg = "abs_reader: y and dy have different length"
-                    raise ValueError, msg
-                # If the data length is zero, consider this as
-                # though we were not able to read the file.
-                if len(x) == 0:
-                    raise ValueError, "ascii_reader: could not load file"
-                
-                output.x = x[x != 0]
-                output.y = y[x != 0]
-                output.dy = dy[x != 0]
-                output.dx = dx[x != 0]
-                if data_conv_q is not None:
-                    output.xaxis("\\rm{Q}", output.x_unit)
+                line_toks = line.split()
+                # Center in bin number
+                center_x = float(line_toks[0])
+                center_y = float(line_toks[1])
+
+                # Bin size
+                if self.has_converter and detector.pixel_size_unit != 'mm':
+                    conv = Converter('mm')
+                    detector.pixel_size.x = conv(5.08,
+                                        units=detector.pixel_size_unit)
+                    detector.pixel_size.y = conv(5.08,
+                                        units=detector.pixel_size_unit)
                 else:
-                    output.xaxis("\\rm{Q}", 'A^{-1}')
-                if data_conv_i is not None:
-                    output.yaxis("\\rm{Intensity}", output.y_unit)
+                    detector.pixel_size.x = 5.08
+                    detector.pixel_size.y = 5.08
+
+                # Store beam center in distance units
+                # Det 640 x 640 mm
+                if self.has_converter and detector.beam_center_unit != 'mm':
+                    conv = Converter('mm')
+                    detector.beam_center.x = conv(center_x * 5.08,
+                                     units=detector.beam_center_unit)
+                    detector.beam_center.y = conv(center_y * 5.08,
+                                    units=detector.beam_center_unit)
                 else:
-                    output.yaxis("\\rm{Intensity}", "cm^{-1}")
-                    
-                # Store loading process information
-                output.meta_data['loader'] = self.type_name
-                return output
+                    detector.beam_center.x = center_x * 5.08
+                    detector.beam_center.y = center_y * 5.08
+
+                # Detector type
+                try:
+                    detector.name = line_toks[7]
+                except:
+                    # Detector name is not a mandatory entry
+                    pass
+
+            # BCENT(X,Y)  A1(mm)  A2(mm)  A1A2DIST(m)  DL/L  BSTOP(mm)  DET_TYP
+            if line.count("BCENT") > 0:
+                is_center = True
+
+            # Parse the data
+            if is_data_started:
+                toks = line.split()
+
+                try:
+                    _x = float(toks[0])
+                    _y = float(toks[1])
+                    _dy = float(toks[2])
+                    _dx = float(toks[3])
+
+                    if data_conv_q is not None:
+                        _x = data_conv_q(_x, units=base_q_unit)
+                        _dx = data_conv_q(_dx, units=base_q_unit)
+
+                    if data_conv_i is not None:
+                        _y = data_conv_i(_y, units=base_i_unit)
+                        _dy = data_conv_i(_dy, units=base_i_unit)
+
+                    self.current_dataset.x[data_line] = _x
+                    self.current_dataset.y[data_line] = _y
+                    self.current_dataset.dy[data_line] = _dy
+                    self.current_dataset.dx[data_line] = _dx
+                    data_line += 1
+
+                except ValueError:
+                    # Could not read this data line. If we are here
+                    # it is because we are in the data section. Just
+                    # skip it.
+                    pass
+
+            # The 6 columns are | Q (1/A) | I(Q) (1/cm) | std. dev.
+            # I(Q) (1/cm) | sigmaQ | meanQ | ShadowFactor|
+            if line.count("The 6 columns") > 0:
+                is_data_started = True
+
+        self.remove_empty_q_values(True, True)
+
+        # Sanity check
+        if not len(self.current_dataset.y) == len(self.current_dataset.dy):
+            self.set_all_to_none()
+            msg = "abs_reader: y and dy have different length"
+            raise ValueError(msg)
+        # If the data length is zero, consider this as
+        # though we were not able to read the file.
+        if len(self.current_dataset.x) == 0:
+            self.set_all_to_none()
+            raise ValueError("ascii_reader: could not load file")
+
+        if data_conv_q is not None:
+            self.current_dataset.xaxis("\\rm{Q}", base_q_unit)
         else:
-            raise RuntimeError, "%s is not a file" % path
-        return None
+            self.current_dataset.xaxis("\\rm{Q}", 'A^{-1}')
+        if data_conv_i is not None:
+            self.current_dataset.yaxis("\\rm{Intensity}", base_i_unit)
+        else:
+            self.current_dataset.yaxis("\\rm{Intensity}", "cm^{-1}")
+
+        # Store loading process information
+        self.current_datainfo.meta_data['loader'] = self.type_name
+        self.send_to_output()

src/sas/sascalc/dataloader/readers/anton_paar_saxs_reader.py

@@ -9 +9 @@
 
 from sas.sascalc.dataloader.readers.xml_reader import XMLreader
-from sas.sascalc.dataloader.data_info import plottable_1D, Data1D, Sample, Source
+from sas.sascalc.dataloader.data_info import plottable_1D, Data1D, DataInfo, Sample, Source
 from sas.sascalc.dataloader.data_info import Process, Aperture, Collimation, TransmissionSpectrum, Detector
-
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException, DataReaderException
 
 class Reader(XMLreader):
     """
-    A class for reading in CanSAS v2.0 data files. The existing iteration opens Mantid generated HDF5 formatted files
-    with file extension .h5/.H5. Any number of data sets may be present within the file and any dimensionality of data
-    may be used. Currently 1D and 2D SAS data sets are supported, but future implementations will include 1D and 2D
-    SESANS data. This class assumes a single data set for each sasentry.
-
-    :Dependencies:
-        The CanSAS HDF5 reader requires h5py v2.5.0 or later.
+    A class for reading in Anton Paar .pdh files
     """
 
@@ -30 +24 @@
     ## Raw file contents to be processed
     raw_data = None
-    ## Data set being modified
-    current_dataset = None
     ## For recursion and saving purposes, remember parent objects
     parent_list = None
@@ -42 +34 @@
     ## Flag to bypass extension check
     allow_all = False
-    ## List of files to return
-    output = None
 
     def reset_state(self):
-        self.current_dataset = Data1D(np.empty(0), np.empty(0),
-                                            np.empty(0), np.empty(0))
+        self.current_dataset = plottable_1D(np.empty(0), np.empty(0), np.empty(0), np.empty(0))
+        self.current_datainfo = DataInfo()
         self.datasets = []
         self.raw_data = None
@@ -63 +53 @@
         self.lower = 5
 
-    def read(self, filename):
+    def get_file_contents(self):
         """
             This is the general read method that all SasView data_loaders must have.
@@ -73 +63 @@
         ## Reinitialize the class when loading a new data file to reset all class variables
         self.reset_state()
-        ## Check that the file exists
-        if os.path.isfile(filename):
-            basename = os.path.basename(filename)
-            _, extension = os.path.splitext(basename)
-            # If the file type is not allowed, return empty list
-            if extension in self.ext or self.allow_all:
-                ## Load the data file
-                input_f = open(filename, 'r')
-                buff = input_f.read()
-                self.raw_data = buff.splitlines()
-                self.read_data()
-        return self.output
+        buff = self.f_open.read()
+        self.raw_data = buff.splitlines()
+        self.read_data()
 
     def read_data(self):
+        correctly_loaded = True
+        error_message = ""
+
         q_unit = "1/nm"
         i_unit = "1/um^2"
-        self.current_dataset.title = self.raw_data[0]
-        self.current_dataset.meta_data["Keywords"] = self.raw_data[1]
-        line3 = self.raw_data[2].split()
-        line4 = self.raw_data[3].split()
-        line5 = self.raw_data[4].split()
-        self.data_points = int(line3[0])
-        self.lower = 5
-        self.upper = self.lower + self.data_points
-        self.source.radiation = 'x-ray'
-        normal = float(line4[3])
-        self.current_dataset.source.radiation = "x-ray"
-        self.current_dataset.source.name = "Anton Paar SAXSess Instrument"
-        self.current_dataset.source.wavelength = float(line4[4])
-        xvals = []
-        yvals = []
-        dyvals = []
-        for i in range(self.lower, self.upper):
-            index = i - self.lower
-            data = self.raw_data[i].split()
-            xvals.insert(index, normal * float(data[0]))
-            yvals.insert(index, normal * float(data[1]))
-            dyvals.insert(index, normal * float(data[2]))
+        try:
+            self.current_datainfo.title = self.raw_data[0]
+            self.current_datainfo.meta_data["Keywords"] = self.raw_data[1]
+            line3 = self.raw_data[2].split()
+            line4 = self.raw_data[3].split()
+            line5 = self.raw_data[4].split()
+            self.data_points = int(line3[0])
+            self.lower = 5
+            self.upper = self.lower + self.data_points
+            self.source.radiation = 'x-ray'
+            normal = float(line4[3])
+            self.current_datainfo.source.radiation = "x-ray"
+            self.current_datainfo.source.name = "Anton Paar SAXSess Instrument"
+            self.current_datainfo.source.wavelength = float(line4[4])
+            xvals = []
+            yvals = []
+            dyvals = []
+            for i in range(self.lower, self.upper):
+                index = i - self.lower
+                data = self.raw_data[i].split()
+                xvals.insert(index, normal * float(data[0]))
+                yvals.insert(index, normal * float(data[1]))
+                dyvals.insert(index, normal * float(data[2]))
+        except Exception as e:
+            error_message = "Couldn't load {}.\n".format(self.f_open.name)
+            error_message += e.message
+            raise FileContentsException(error_message)
         self.current_dataset.x = np.append(self.current_dataset.x, xvals)
         self.current_dataset.y = np.append(self.current_dataset.y, yvals)
         self.current_dataset.dy = np.append(self.current_dataset.dy, dyvals)
         if self.data_points != self.current_dataset.x.size:
-            self.errors.add("Not all data was loaded properly.")
-        if self.current_dataset.dx.size != self.current_dataset.x.size:
-            dxvals = np.zeros(self.current_dataset.x.size)
-            self.current_dataset.dx = dxvals
+            error_message += "Not all data points could be loaded.\n"
+            correctly_loaded = False
         if self.current_dataset.x.size != self.current_dataset.y.size:
-            self.errors.add("The x and y data sets are not the same size.")
+            error_message += "The x and y data sets are not the same size.\n"
+            correctly_loaded = False
         if self.current_dataset.y.size != self.current_dataset.dy.size:
-            self.errors.add("The y and dy datasets are not the same size.")
-        self.current_dataset.errors = self.errors
+            error_message += "The y and dy datasets are not the same size.\n"
+            correctly_loaded = False
+
         self.current_dataset.xaxis("Q", q_unit)
         self.current_dataset.yaxis("Intensity", i_unit)
         xml_intermediate = self.raw_data[self.upper:]
         xml = ''.join(xml_intermediate)
-        self.set_xml_string(xml)
-        dom = self.xmlroot.xpath('/fileinfo')
-        self._parse_child(dom)
-        self.output.append(self.current_dataset)
+        try:
+            self.set_xml_string(xml)
+            dom = self.xmlroot.xpath('/fileinfo')
+            self._parse_child(dom)
+        except Exception as e:
+            # Data loaded but XML metadata has an error
+            error_message += "Data points have been loaded but there was an "
+            error_message += "error reading XML metadata: " + e.message
+            correctly_loaded = False
+        self.send_to_output()
+        if not correctly_loaded:
+            raise DataReaderException(error_message)
 
     def _parse_child(self, dom, parent=''):
@@ -146 +143 @@
                 self._parse_child(node, key)
                 if key == "SampleDetector":
-                    self.current_dataset.detector.append(self.detector)
+                    self.current_datainfo.detector.append(self.detector)
                     self.detector = Detector()
             else:
                 if key == "value":
                     if parent == "Wavelength":
-                        self.current_dataset.source.wavelength = value
+                        self.current_datainfo.source.wavelength = value
                     elif parent == "SampleDetector":
                         self.detector.distance = value
                     elif parent == "Temperature":
-                        self.current_dataset.sample.temperature = value
+                        self.current_datainfo.sample.temperature = value
                     elif parent == "CounterSlitLength":
                         self.detector.slit_length = value
@@ -161 +158 @@
                     value = value.replace("_", "")
                     if parent == "Wavelength":
-                        self.current_dataset.source.wavelength_unit = value
+                        self.current_datainfo.source.wavelength_unit = value
                     elif parent == "SampleDetector":
                         self.detector.distance_unit = value
@@ -169 +166 @@
                         self.current_dataset.yaxis(self.current_dataset._yaxis, value)
                     elif parent == "Temperature":
-                        self.current_dataset.sample.temperature_unit = value
+                        self.current_datainfo.sample.temperature_unit = value
                     elif parent == "CounterSlitLength":
                         self.detector.slit_length_unit = value

src/sas/sascalc/dataloader/readers/ascii_reader.py

@@ -1 +1 @@
 """
-    ASCII reader
+    Generic multi-column ASCII data reader
 """
 ############################################################################
-#This software was developed by the University of Tennessee as part of the
-#Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
-#project funded by the US National Science Foundation. 
-#If you use DANSE applications to do scientific research that leads to
-#publication, we ask that you acknowledge the use of the software with the
-#following sentence:
-#This work benefited from DANSE software developed under NSF award DMR-0520547.
-#copyright 2008, University of Tennessee
+# This software was developed by the University of Tennessee as part of the
+# Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
+# project funded by the US National Science Foundation.
+# If you use DANSE applications to do scientific research that leads to
+# publication, we ask that you acknowledge the use of the software with the
+# following sentence:
+# This work benefited from DANSE software developed under NSF award DMR-0520547.
+# copyright 2008, University of Tennessee
 #############################################################################
 
+import logging
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+from sas.sascalc.dataloader.data_info import DataInfo, plottable_1D
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException,\
+    DefaultReaderException
 
-import numpy as np
-import os
-from sas.sascalc.dataloader.data_info import Data1D
-
-# Check whether we have a converter available
-has_converter = True
-try:
-    from sas.sascalc.data_util.nxsunit import Converter
-except:
-    has_converter = False
-_ZERO = 1e-16
+logger = logging.getLogger(__name__)
 
 
-class Reader:
+class Reader(FileReader):
     """
     Class to load ascii files (2, 3 or 4 columns).
     """
-    ## File type
+    # File type
     type_name = "ASCII"
-
-    ## Wildcards
+    # Wildcards
     type = ["ASCII files (*.txt)|*.txt",
             "ASCII files (*.dat)|*.dat",
             "ASCII files (*.abs)|*.abs",
             "CSV files (*.csv)|*.csv"]
-    ## List of allowed extensions
-    ext = ['.txt', '.TXT', '.dat', '.DAT', '.abs', '.ABS', 'csv', 'CSV']
+    # List of allowed extensions
+    ext = ['.txt', '.dat', '.abs', '.csv']
+    # Flag to bypass extension check
+    allow_all = True
+    # data unless that is the only data
+    min_data_pts = 5
 
-    ## Flag to bypass extension check
-    allow_all = True
+    def get_file_contents(self):
+        """
+        Get the contents of the file
+        """
 
-    def read(self, path):
-        """
-        Load data file
+        buff = self.f_open.read()
+        filepath = self.f_open.name
+        lines = buff.splitlines()
+        self.output = []
+        self.current_datainfo = DataInfo()
+        self.current_datainfo.filename = filepath
+        self.reset_data_list(len(lines))
 
-        :param path: file path
-        :return: Data1D object, or None
+        # The first good line of data will define whether
+        # we have 2-column or 3-column ascii
+        has_error_dx = None
+        has_error_dy = None
 
-        :raise RuntimeError: when the file can't be opened
-        :raise ValueError: when the length of the data vectors are inconsistent
-        """
-        if os.path.isfile(path):
-            basename = os.path.basename(path)
-            _, extension = os.path.splitext(basename)
-            if self.allow_all or extension.lower() in self.ext:
-                try:
-                    # Read in binary mode since GRASP frequently has no-ascii
-                    # characters that breaks the open operation
-                    input_f = open(path,'rb')
-                except:
-                    raise  RuntimeError, "ascii_reader: cannot open %s" % path
-                buff = input_f.read()
-                lines = buff.splitlines()
+        # Initialize counters for data lines and header lines.
+        is_data = False
+        # More than "5" lines of data is considered as actual
+        # To count # of current data candidate lines
+        candidate_lines = 0
+        # To count total # of previous data candidate lines
+        candidate_lines_previous = 0
+        # Current line number
+        line_no = 0
+        # minimum required number of columns of data
+        lentoks = 2
+        for line in lines:
+            toks = self.splitline(line.strip())
+            # To remember the number of columns in the current line of data
+            new_lentoks = len(toks)
+            try:
+                if new_lentoks == 0:
+                    # If the line is blank, skip and continue on
+                    # In case of breaks within data sets.
+                    continue
+                elif new_lentoks != lentoks and is_data:
+                    # If a footer is found, break the loop and save the data
+                    break
+                elif new_lentoks != lentoks and not is_data:
+                    # If header lines are numerical
+                    candidate_lines = 0
+                    self.reset_data_list(len(lines) - line_no)
 
-                # Arrays for data storage
-                tx = np.zeros(0)
-                ty = np.zeros(0)
-                tdy = np.zeros(0)
-                tdx = np.zeros(0)
+                self.current_dataset.x[candidate_lines] = float(toks[0])
 
-                # The first good line of data will define whether
-                # we have 2-column or 3-column ascii
+                if new_lentoks > 1:
+                    self.current_dataset.y[candidate_lines] = float(toks[1])
+
+                # If a 3rd row is present, consider it dy
+                if new_lentoks > 2:
+                    self.current_dataset.dy[candidate_lines] = \
+                        float(toks[2])
+                    has_error_dy = True
+
+                # If a 4th row is present, consider it dx
+                if new_lentoks > 3:
+                    self.current_dataset.dx[candidate_lines] = \
+                        float(toks[3])
+                    has_error_dx = True
+
+                candidate_lines += 1
+                # If 5 or more lines, this is considering the set data
+                if candidate_lines >= self.min_data_pts:
+                    is_data = True
+
+                if is_data and new_lentoks >= 8:
+                    msg = "This data looks like 2D ASCII data. Use the file "
+                    msg += "converter tool to convert it to NXcanSAS."
+                    raise FileContentsException(msg)
+
+                # To remember the # of columns on the current line
+                # for the next line of data
+                lentoks = new_lentoks
+                line_no += 1
+            except ValueError:
+                # ValueError is raised when non numeric strings conv. to float
+                # It is data and meet non - number, then stop reading
+                if is_data:
+                    break
+                # Delete the previously stored lines of data candidates if
+                # the list is not data
+                self.reset_data_list(len(lines) - line_no)
+                lentoks = 2
                 has_error_dx = None
                 has_error_dy = None
+                # Reset # of lines of data candidates
+                candidate_lines = 0
+        
+        if not is_data:
+            self.set_all_to_none()
+            if self.extension in self.ext:
+                msg = "ASCII Reader error: Fewer than five Q data points found "
+                msg += "in {}.".format(filepath)
+                raise FileContentsException(msg)
+            else:
+                msg = "ASCII Reader could not load the file {}".format(filepath)
+                raise DefaultReaderException(msg)
+        # Sanity check
+        if has_error_dy and not len(self.current_dataset.y) == \
+                len(self.current_dataset.dy):
+            msg = "ASCII Reader error: Number of I and dI data points are"
+            msg += " different in {}.".format(filepath)
+            # TODO: Add error to self.current_datainfo.errors instead?
+            self.set_all_to_none()
+            raise FileContentsException(msg)
+        if has_error_dx and not len(self.current_dataset.x) == \
+                len(self.current_dataset.dx):
+            msg = "ASCII Reader error: Number of Q and dQ data points are"
+            msg += " different in {}.".format(filepath)
+            # TODO: Add error to self.current_datainfo.errors instead?
+            self.set_all_to_none()
+            raise FileContentsException(msg)
 
-                #Initialize counters for data lines and header lines.
-                is_data = False
-                # More than "5" lines of data is considered as actual
-                # data unless that is the only data
-                min_data_pts = 5
-                # To count # of current data candidate lines
-                candidate_lines = 0
-                # To count total # of previous data candidate lines
-                candidate_lines_previous = 0
-                #minimum required number of columns of data
-                lentoks = 2
-                for line in lines:
-                    toks = self.splitline(line)
-                    # To remember the # of columns in the current line of data
-                    new_lentoks = len(toks)
-                    try:
-                        if new_lentoks == 1 and not is_data:
-                            ## If only one item in list, no longer data
-                            raise ValueError
-                        elif new_lentoks == 0:
-                            ## If the line is blank, skip and continue on
-                            ## In case of breaks within data sets.
-                            continue
-                        elif new_lentoks != lentoks and is_data:
-                            ## If a footer is found, break the loop and save the data
-                            break
-                        elif new_lentoks != lentoks and not is_data:
-                            ## If header lines are numerical
-                            candidate_lines = 0
-                            candidate_lines_previous = 0
+        self.remove_empty_q_values(has_error_dx, has_error_dy)
+        self.current_dataset.xaxis("\\rm{Q}", 'A^{-1}')
+        self.current_dataset.yaxis("\\rm{Intensity}", "cm^{-1}")
 
-                        #Make sure that all columns are numbers.
-                        for colnum in range(len(toks)):
-                            # Any non-floating point values throw ValueError
-                            float(toks[colnum])
-
-                        candidate_lines += 1
-                        _x = float(toks[0])
-                        _y = float(toks[1])
-                        _dx = None
-                        _dy = None
-
-                        #If 5 or more lines, this is considering the set data
-                        if candidate_lines >= min_data_pts:
-                            is_data = True
-
-                        # If a 3rd row is present, consider it dy
-                        if new_lentoks > 2:
-                            _dy = float(toks[2])
-                        has_error_dy = False if _dy is None else True
-
-                        # If a 4th row is present, consider it dx
-                        if new_lentoks > 3:
-                            _dx = float(toks[3])
-                        has_error_dx = False if _dx is None else True
-
-                        # Delete the previously stored lines of data candidates if
-                        # the list is not data
-                        if candidate_lines == 1 and -1 < candidate_lines_previous < min_data_pts and \
-                            is_data == False:
-                            try:
-                                tx = np.zeros(0)
-                                ty = np.zeros(0)
-                                tdy = np.zeros(0)
-                                tdx = np.zeros(0)
-                            except:
-                                pass
-
-                        if has_error_dy == True:
-                            tdy = np.append(tdy, _dy)
-                        if has_error_dx == True:
-                            tdx = np.append(tdx, _dx)
-                        tx = np.append(tx, _x)
-                        ty = np.append(ty, _y)
-
-                        #To remember the # of columns on the current line
-                        # for the next line of data
-                        lentoks = new_lentoks
-                        candidate_lines_previous = candidate_lines
-                    except ValueError:
-                        # It is data and meet non - number, then stop reading
-                        if is_data == True:
-                            break
-                        lentoks = 2
-                        has_error_dx = None
-                        has_error_dy = None
-                        #Reset # of lines of data candidates
-                        candidate_lines = 0
-                    except:
-                        pass
-
-                input_f.close()
-                if not is_data:
-                    msg = "ascii_reader: x has no data"
-                    raise RuntimeError, msg
-                # Sanity check
-                if has_error_dy == True and not len(ty) == len(tdy):
-                    msg = "ascii_reader: y and dy have different length"
-                    raise RuntimeError, msg
-                if has_error_dx == True and not len(tx) == len(tdx):
-                    msg = "ascii_reader: y and dy have different length"
-                    raise RuntimeError, msg
-                # If the data length is zero, consider this as
-                # though we were not able to read the file.
-                if len(tx) == 0:
-                    raise RuntimeError, "ascii_reader: could not load file"
-
-                #Let's re-order the data to make cal.
-                # curve look better some cases
-                ind = np.lexsort((ty, tx))
-                x = np.zeros(len(tx))
-                y = np.zeros(len(ty))
-                dy = np.zeros(len(tdy))
-                dx = np.zeros(len(tdx))
-                output = Data1D(x, y, dy=dy, dx=dx)
-                self.filename = output.filename = basename
-
-                for i in ind:
-                    x[i] = tx[ind[i]]
-                    y[i] = ty[ind[i]]
-                    if has_error_dy == True:
-                        dy[i] = tdy[ind[i]]
-                    if has_error_dx == True:
-                        dx[i] = tdx[ind[i]]
-                # Zeros in dx, dy
-                if has_error_dx:
-                    dx[dx == 0] = _ZERO
-                if has_error_dy:
-                    dy[dy == 0] = _ZERO
-                #Data
-                output.x = x[x != 0]
-                output.y = y[x != 0]
-                output.dy = dy[x != 0] if has_error_dy == True\
-                    else np.zeros(len(output.y))
-                output.dx = dx[x != 0] if has_error_dx == True\
-                    else np.zeros(len(output.x))
-
-                output.xaxis("\\rm{Q}", 'A^{-1}')
-                output.yaxis("\\rm{Intensity}", "cm^{-1}")
-
-                # Store loading process information
-                output.meta_data['loader'] = self.type_name
-                if len(output.x) < 1:
-                    raise RuntimeError, "%s is empty" % path
-                return output
-
-        else:
-            raise RuntimeError, "%s is not a file" % path
-        return None
-
-    def splitline(self, line):
-        """
-        Splits a line into pieces based on common delimeters
-        :param line: A single line of text
-        :return: list of values
-        """
-        # Initial try for CSV (split on ,)
-        toks = line.split(',')
-        # Now try SCSV (split on ;)
-        if len(toks) < 2:
-            toks = line.split(';')
-        # Now go for whitespace
-        if len(toks) < 2:
-            toks = line.split()
-        return toks
+        # Store loading process information
+        self.current_datainfo.meta_data['loader'] = self.type_name
+        self.send_to_output()

src/sas/sascalc/dataloader/readers/associations.py

@@ -14 +14 @@
 #copyright 2009, University of Tennessee
 #############################################################################
-from __future__ import print_function
-
-import os
 import sys
 import logging
-import json
 
 logger = logging.getLogger(__name__)
 
-FILE_NAME = 'defaults.json'
+FILE_ASSOCIATIONS = {
+    ".xml": "cansas_reader",
+    ".ses": "sesans_reader",
+    ".h5": "cansas_reader_HDF5",
+    ".txt": "ascii_reader",
+    ".dat": "red2d_reader",
+    ".abs": "abs_reader",
+    ".sans": "danse_reader",
+    ".pdh": "anton_paar_saxs_reader"
+}
 
-def read_associations(loader, settings=FILE_NAME):
+
+def read_associations(loader, settings=FILE_ASSOCIATIONS):
     """
     Read the specified settings file to associate
     default readers to file extension.
-    
+
     :param loader: Loader object
     :param settings: path to the json settings file [string]
     """
-    reader_dir = os.path.dirname(__file__)
-    path = os.path.join(reader_dir, settings)
-    
-    # If we can't find the file in the installation
-    # directory, look into the execution directory.
-    if not os.path.isfile(path):
-        path = os.path.join(os.getcwd(), settings)
-    if not os.path.isfile(path):
-        path = os.path.join(sys.path[0], settings)
-    if not os.path.isfile(path):
-        path = settings
-    if not os.path.isfile(path):
-        path = "./%s" % settings
-    if os.path.isfile(path):
-        with open(path) as fh:
-            json_tree = json.load(fh)
-        
-        # Read in the file extension associations
-        entry_list = json_tree['SasLoader']['FileType']
-
-        # For each FileType entry, get the associated reader and extension
-        for entry in entry_list:
-            reader = entry['-reader']
-            ext = entry['-extension']
-            
-            if reader is not None and ext is not None:
-                # Associate the extension with a particular reader
-                # TODO: Modify the Register code to be case-insensitive
-                # and remove the extra line below.
-                try:
-                    exec "import %s" % reader
-                    exec "loader.associate_file_type('%s', %s)" % (ext.lower(),
-                                                                    reader)
-                    exec "loader.associate_file_type('%s', %s)" % (ext.upper(),
-                                                                    reader)
-                except:
-                    msg = "read_associations: skipping association"
-                    msg += " for %s\n  %s" % (ext.lower(), sys.exc_value)
-                    logger.error(msg)
-    else:
-        print("Could not find reader association settings\n  %s [%s]" % (__file__, os.getcwd()))
-         
-         
-def register_readers(registry_function):
-    """
-    Function called by the registry/loader object to register
-    all default readers using a call back function.
-    
-    :WARNING: this method is now obsolete
-
-    :param registry_function: function to be called to register each reader
-    """
-    logger.info("register_readers is now obsolete: use read_associations()")
-    import abs_reader
-    import ascii_reader
-    import cansas_reader
-    import danse_reader
-    import hfir1d_reader
-    import IgorReader
-    import red2d_reader
-    #import tiff_reader
-    import nexus_reader
-    import sesans_reader
-    import cansas_reader_HDF5
-    import anton_paar_saxs_reader
-    registry_function(sesans_reader)
-    registry_function(abs_reader)
-    registry_function(ascii_reader)
-    registry_function(cansas_reader)
-    registry_function(danse_reader)
-    registry_function(hfir1d_reader)
-    registry_function(IgorReader)
-    registry_function(red2d_reader)
-    #registry_function(tiff_reader)
-    registry_function(nexus_reader)
-    registry_function(cansas_reader_HDF5)
-    registry_function(anton_paar_saxs_reader)
-    return True
+    # For each FileType entry, get the associated reader and extension
+    for ext, reader in settings.iteritems():
+        if reader is not None and ext is not None:
+            # Associate the extension with a particular reader
+            # TODO: Modify the Register code to be case-insensitive
+            # FIXME: Remove exec statements
+            # and remove the extra line below.
+            try:
+                exec "import %s" % reader
+                exec "loader.associate_file_type('%s', %s)" % (ext.lower(),
+                                                                reader)
+                exec "loader.associate_file_type('%s', %s)" % (ext.upper(),
+                                                                reader)
+            except:
+                msg = "read_associations: skipping association"
+                msg += " for %s\n  %s" % (ext.lower(), sys.exc_value)
+                logger.error(msg)

src/sas/sascalc/dataloader/readers/cansas_reader.py

@@ -1 @@
-"""
-    CanSAS data reader - new recursive cansas_version.
-"""
-############################################################################
-#This software was developed by the University of Tennessee as part of the
-#Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
-#project funded by the US National Science Foundation.
-#If you use DANSE applications to do scientific research that leads to
-#publication, we ask that you acknowledge the use of the software with the
-#following sentence:
-#This work benefited from DANSE software developed under NSF award DMR-0520547.
-#copyright 2008,2009 University of Tennessee
-#############################################################################
-
 import logging
 import numpy as np
@@ -29 +15 @@
 from sas.sascalc.dataloader.readers.xml_reader import XMLreader
 from sas.sascalc.dataloader.readers.cansas_constants import CansasConstants, CurrentLevel
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException, \
+    DefaultReaderException, DataReaderException
 
 # The following 2 imports *ARE* used. Do not remove either.
 import xml.dom.minidom
 from xml.dom.minidom import parseString
+
+from lxml import etree
 
 logger = logging.getLogger(__name__)
@@ -55 +45 @@
 
 class Reader(XMLreader):
-    """
-    Class to load cansas 1D XML files
-
-    :Dependencies:
-        The CanSAS reader requires PyXML 0.8.4 or later.
-    """
-    # CanSAS version - defaults to version 1.0
     cansas_version = "1.0"
     base_ns = "{cansas1d/1.0}"
@@ -75 +58 @@
     ns_list = None
     # Temporary storage location for loading multiple data sets in a single file
-    current_datainfo = None
-    current_dataset = None
     current_data1d = None
     data = None
-    # List of data1D objects to be sent back to SasView
-    output = None
     # Wildcards
     type = ["XML files (*.xml)|*.xml", "SasView Save Files (*.svs)|*.svs"]
@@ -110 +89 @@
 
     def read(self, xml_file, schema_path="", invalid=True):
-        """
-        Validate and read in an xml_file file in the canSAS format.
-
-        :param xml_file: A canSAS file path in proper XML format
-        :param schema_path: A file path to an XML schema to validate the xml_file against
-        """
-        # For every file loaded, reset everything to a base state
+        if schema_path != "" or invalid != True:
+            # read has been called from self.get_file_contents because xml file doens't conform to schema
+            _, self.extension = os.path.splitext(os.path.basename(xml_file))
+            return self.get_file_contents(xml_file=xml_file, schema_path=schema_path, invalid=invalid)
+
+        # Otherwise, read has been called by the data loader - file_reader_base_class handles this
+        return super(XMLreader, self).read(xml_file)
+
+    def get_file_contents(self, xml_file=None, schema_path="", invalid=True):
+        # Reset everything since we're loading a new file
         self.reset_state()
         self.invalid = invalid
-        # Check that the file exists
-        if os.path.isfile(xml_file):
-            basename, extension = os.path.splitext(os.path.basename(xml_file))
-            # If the file type is not allowed, return nothing
-            if extension in self.ext or self.allow_all:
-                # Get the file location of
-                self.load_file_and_schema(xml_file, schema_path)
-                self.add_data_set()
-                # Try to load the file, but raise an error if unable to.
-                # Check the file matches the XML schema
+        if xml_file is None:
+            xml_file = self.f_open.name
+        # We don't sure f_open since lxml handles opnening/closing files
+        if not self.f_open.closed:
+            self.f_open.close()
+
+        basename, _ = os.path.splitext(os.path.basename(xml_file))
+
+        try:
+            # Raises FileContentsException
+            self.load_file_and_schema(xml_file, schema_path)
+            self.current_datainfo = DataInfo()
+            # Raises FileContentsException if file doesn't meet CanSAS schema
+            self.is_cansas(self.extension)
+            self.invalid = False # If we reach this point then file must be valid CanSAS
+
+            # Parse each SASentry
+            entry_list = self.xmlroot.xpath('/ns:SASroot/ns:SASentry', namespaces={
+                'ns': self.cansas_defaults.get("ns")
+            })
+            # Look for a SASentry
+            self.names.append("SASentry")
+            self.set_processing_instructions()
+
+            for entry in entry_list:
+                self.current_datainfo.filename = basename + self.extension
+                self.current_datainfo.meta_data["loader"] = "CanSAS XML 1D"
+                self.current_datainfo.meta_data[PREPROCESS] = self.processing_instructions
+                self._parse_entry(entry)
+                has_error_dx = self.current_dataset.dx is not None
+                has_error_dy = self.current_dataset.dy is not None
+                self.remove_empty_q_values(has_error_dx=has_error_dx,
+                    has_error_dy=has_error_dy)
+                self.send_to_output() # Combine datasets with DataInfo
+                self.current_datainfo = DataInfo() # Reset DataInfo
+        except FileContentsException as fc_exc:
+            # File doesn't meet schema - try loading with a less strict schema
+            base_name = xml_reader.__file__
+            base_name = base_name.replace("\\", "/")
+            base = base_name.split("/sas/")[0]
+            if self.cansas_version == "1.1":
+                invalid_schema = INVALID_SCHEMA_PATH_1_1.format(base, self.cansas_defaults.get("schema"))
+            else:
+                invalid_schema = INVALID_SCHEMA_PATH_1_0.format(base, self.cansas_defaults.get("schema"))
+            self.set_schema(invalid_schema)
+            if self.invalid:
                 try:
-                    self.is_cansas(extension)
-                    self.invalid = False
-                    # Get each SASentry from XML file and add it to a list.
-                    entry_list = self.xmlroot.xpath(
-                            '/ns:SASroot/ns:SASentry',
-                            namespaces={'ns': self.cansas_defaults.get("ns")})
-                    self.names.append("SASentry")
-
-                    # Get all preprocessing events and encoding
-                    self.set_processing_instructions()
-
-                    # Parse each <SASentry> item
-                    for entry in entry_list:
-                        # Create a new DataInfo object for every <SASentry>
-
-                        # Set the file name and then parse the entry.
-                        self.current_datainfo.filename = basename + extension
-                        self.current_datainfo.meta_data["loader"] = "CanSAS XML 1D"
-                        self.current_datainfo.meta_data[PREPROCESS] = \
-                            self.processing_instructions
-
-                        # Parse the XML SASentry
-                        self._parse_entry(entry)
-                        # Combine datasets with datainfo
-                        self.add_data_set()
-                except RuntimeError:
-                    # If the file does not match the schema, raise this error
+                    # Load data with less strict schema
+                    self.read(xml_file, invalid_schema, False)
+
+                    # File can still be read but doesn't match schema, so raise exception
+                    self.load_file_and_schema(xml_file) # Reload strict schema so we can find where error are in file
                     invalid_xml = self.find_invalid_xml()
-                    invalid_xml = INVALID_XML.format(basename + extension) + invalid_xml
-                    self.errors.add(invalid_xml)
-                    # Try again with an invalid CanSAS schema, that requires only a data set in each
-                    base_name = xml_reader.__file__
-                    base_name = base_name.replace("\\", "/")
-                    base = base_name.split("/sas/")[0]
-                    if self.cansas_version == "1.1":
-                        invalid_schema = INVALID_SCHEMA_PATH_1_1.format(base, self.cansas_defaults.get("schema"))
-                    else:
-                        invalid_schema = INVALID_SCHEMA_PATH_1_0.format(base, self.cansas_defaults.get("schema"))
-                    self.set_schema(invalid_schema)
-                    try:
-                        if self.invalid:
-                            if self.is_cansas():
-                                self.output = self.read(xml_file, invalid_schema, False)
-                            else:
-                                raise RuntimeError
-                        else:
-                            raise RuntimeError
-                    except RuntimeError:
-                        x = np.zeros(1)
-                        y = np.zeros(1)
-                        self.current_data1d = Data1D(x,y)
-                        self.current_data1d.errors = self.errors
-                        return [self.current_data1d]
-        else:
-            self.output.append("Not a valid file path.")
-        # Return a list of parsed entries that dataloader can manage
-        return self.output
+                    invalid_xml = INVALID_XML.format(basename + self.extension) + invalid_xml
+                    raise DataReaderException(invalid_xml) # Handled by base class
+                except FileContentsException as fc_exc:
+                    if not self.extension in self.ext: # If the file has no associated loader
+                        raise DefaultReaderException(msg)
+                    msg = "CanSAS Reader could not load the file {}".format(xml_file)
+                    if fc_exc.message is not None: # Propagate error messages from earlier
+                        msg = fc_exc.message
+                    raise FileContentsException(msg)
+                    pass
+            else:
+                raise fc_exc
+        except Exception as e: # Convert all other exceptions to FileContentsExceptions
+            raise FileContentsException(e.message)
+
+
+    def load_file_and_schema(self, xml_file, schema_path=""):
+        base_name = xml_reader.__file__
+        base_name = base_name.replace("\\", "/")
+        base = base_name.split("/sas/")[0]
+
+        # Try and parse the XML file
+        try:
+            self.set_xml_file(xml_file)
+        except etree.XMLSyntaxError: # File isn't valid XML so can't be loaded
+            msg = "Cansas cannot load {}.\n Invalid XML syntax".format(xml_file)
+            raise FileContentsException(msg)
+
+        self.cansas_version = self.xmlroot.get("version", "1.0")
+        self.cansas_defaults = CANSAS_NS.get(self.cansas_version, "1.0")
+
+        if schema_path == "":
+            schema_path = "{}/sas/sascalc/dataloader/readers/schema/{}".format(
+                base, self.cansas_defaults.get("schema").replace("\\", "/")
+            )
+        self.set_schema(schema_path)
+
+    def is_cansas(self, ext="xml"):
+        """
+        Checks to see if the XML file is a CanSAS file
+
+        :param ext: The file extension of the data file
+        :raises FileContentsException: Raised if XML file isn't valid CanSAS
+        """
+        if self.validate_xml(): # Check file is valid XML
+            name = "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation"
+            value = self.xmlroot.get(name)
+            # Check schema CanSAS version matches file CanSAS version
+            if CANSAS_NS.get(self.cansas_version).get("ns") == value.rsplit(" ")[0]:
+                return True
+        if ext == "svs":
+            return True # Why is this required?
+        # If we get to this point then file isn't valid CanSAS
+        raise FileContentsException("The file is not valid CanSAS")
 
     def _parse_entry(self, dom, recurse=False):
-        """
-        Parse a SASEntry - new recursive method for parsing the dom of
-            the CanSAS data format. This will allow multiple data files
-            and extra nodes to be read in simultaneously.
-
-        :param dom: dom object with a namespace base of names
-        """
-
         if not self._is_call_local() and not recurse:
             self.reset_state()
-            self.add_data_set()
+            self.data = []
+            self.current_datainfo = DataInfo()
             self.names.append("SASentry")
             self.parent_class = "SASentry"
-        self._check_for_empty_data()
-        self.base_ns = "{0}{1}{2}".format("{", \
-                            CANSAS_NS.get(self.cansas_version).get("ns"), "}")
-
-        # Go through each child in the parent element
+        # Create an empty dataset if no data has been passed to the reader
+        if self.current_dataset is None:
+            self.current_dataset = plottable_1D(np.empty(0), np.empty(0),
+                np.empty(0), np.empty(0))
+        self.base_ns = "{" + CANSAS_NS.get(self.cansas_version).get("ns") + "}"
+
+        # Loop through each child in the parent element
         for node in dom:
             attr = node.attrib
@@ -217 +233 @@
             if tagname == "fitting_plug_in" or tagname == "pr_inversion" or tagname == "invariant":
                 continue
-
             # Get where to store content
             self.names.append(tagname_original)
@@ -233 +248 @@
                         else:
                             self.current_dataset.shape = ()
-                # Recursion step to access data within the group
-                self._parse_entry(node, True)
+                # Recurse to access data within the group
+                self._parse_entry(node, recurse=True)
                 if tagname == "SASsample":
                     self.current_datainfo.sample.name = name
@@ -244 +259 @@
                     self.aperture.name = name
                     self.aperture.type = type
-                self.add_intermediate()
+                self._add_intermediate()
             else:
                 if isinstance(self.current_dataset, plottable_2D):
@@ -261 +276 @@
                     self.current_datainfo.notes.append(data_point)
 
-                # I and Q - 1D data
+                # I and Q points
                 elif tagname == 'I' and isinstance(self.current_dataset, plottable_1D):
                     unit_list = unit.split("|")
@@ -283 +298 @@
                     self.current_dataset.dx = np.append(self.current_dataset.dx, data_point)
                 elif tagname == 'dQw':
+                    if self.current_dataset.dqw is None: self.current_dataset.dqw = np.empty(0)
                     self.current_dataset.dxw = np.append(self.current_dataset.dxw, data_point)
                 elif tagname == 'dQl':
+                    if self.current_dataset.dxl is None: self.current_dataset.dxl = np.empty(0)
                     self.current_dataset.dxl = np.append(self.current_dataset.dxl, data_point)
                 elif tagname == 'Qmean':
@@ -356 +373 @@
                 elif tagname == 'name' and self.parent_class == 'SASinstrument':
                     self.current_datainfo.instrument = data_point
+
                 # Detector Information
                 elif tagname == 'name' and self.parent_class == 'SASdetector':
@@ -401 +419 @@
                     self.detector.orientation.z = data_point
                     self.detector.orientation_unit = unit
+
                 # Collimation and Aperture
                 elif tagname == 'length' and self.parent_class == 'SAScollimation':
@@ -434 +453 @@
                 elif tagname == 'term' and self.parent_class == 'SASprocess':
                     unit = attr.get("unit", "")
-                    dic = {}
-                    dic["name"] = name
-                    dic["value"] = data_point
-                    dic["unit"] = unit
+                    dic = { "name": name, "value": data_point, "unit": unit }
                     self.process.term.append(dic)
 
@@ -490 +506 @@
         if not self._is_call_local() and not recurse:
             self.frm = ""
-            self.add_data_set()
+            self.current_datainfo.errors = set()
+            for error in self.errors:
+                self.current_datainfo.errors.add(error)
+            self.errors.clear()
+            self.send_to_output()
             empty = None
             return self.output[0], empty
 
-
     def _is_call_local(self):
-        """
-
-        """
         if self.frm == "":
             inter = inspect.stack()
@@ -510 +526 @@
         return True
 
-    def is_cansas(self, ext="xml"):
-        """
-        Checks to see if the xml file is a CanSAS file
-
-        :param ext: The file extension of the data file
-        """
-        if self.validate_xml():
-            name = "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation"
-            value = self.xmlroot.get(name)
-            if CANSAS_NS.get(self.cansas_version).get("ns") == \
-                    value.rsplit(" ")[0]:
-                return True
-        if ext == "svs":
-            return True
-        raise RuntimeError
-
-    def load_file_and_schema(self, xml_file, schema_path=""):
-        """
-        Loads the file and associates a schema, if a schema is passed in or if one already exists
-
-        :param xml_file: The xml file path sent to Reader.read
-        :param schema_path: The path to a schema associated with the xml_file, or find one based on the file
-        """
-        base_name = xml_reader.__file__
-        base_name = base_name.replace("\\", "/")
-        base = base_name.split("/sas/")[0]
-
-        # Load in xml file and get the cansas version from the header
-        self.set_xml_file(xml_file)
-        self.cansas_version = self.xmlroot.get("version", "1.0")
-
-        # Generic values for the cansas file based on the version
-        self.cansas_defaults = CANSAS_NS.get(self.cansas_version, "1.0")
-        if schema_path == "":
-            schema_path = "{0}/sas/sascalc/dataloader/readers/schema/{1}".format \
-                (base, self.cansas_defaults.get("schema")).replace("\\", "/")
-
-        # Link a schema to the XML file.
-        self.set_schema(schema_path)
-
-    def add_data_set(self):
-        """
-        Adds the current_dataset to the list of outputs after preforming final processing on the data and then calls a
-        private method to generate a new data set.
-
-        :param key: NeXus group name for current tree level
-        """
-
-        if self.current_datainfo and self.current_dataset:
-            self._final_cleanup()
-        self.data = []
-        self.current_datainfo = DataInfo()
-
-    def _initialize_new_data_set(self, node=None):
-        """
-        A private class method to generate a new 1D data object.
-        Outside methods should call add_data_set() to be sure any existing data is stored properly.
-
-        :param node: XML node to determine if 1D or 2D data
-        """
-        x = np.array(0)
-        y = np.array(0)
-        for child in node:
-            if child.tag.replace(self.base_ns, "") == "Idata":
-                for i_child in child:
-                    if i_child.tag.replace(self.base_ns, "") == "Qx":
-                        self.current_dataset = plottable_2D()
-                        return
-        self.current_dataset = plottable_1D(x, y)
-
-    def add_intermediate(self):
+    def _add_intermediate(self):
         """
         This method stores any intermediate objects within the final data set after fully reading the set.
-
-        :param parent: The NXclass name for the h5py Group object that just finished being processed
-        """
-
+        """
         if self.parent_class == 'SASprocess':
             self.current_datainfo.process.append(self.process)
@@ -605 +548 @@
             self._check_for_empty_resolution()
             self.data.append(self.current_dataset)
-
-    def _final_cleanup(self):
-        """
-        Final cleanup of the Data1D object to be sure it has all the
-        appropriate information needed for perspectives
-        """
-
-        # Append errors to dataset and reset class errors
-        self.current_datainfo.errors = set()
-        for error in self.errors:
-            self.current_datainfo.errors.add(error)
-        self.errors.clear()
-
-        # Combine all plottables with datainfo and append each to output
-        # Type cast data arrays to float64 and find min/max as appropriate
-        for dataset in self.data:
-            if isinstance(dataset, plottable_1D):
-                if dataset.x is not None:
-                    dataset.x = np.delete(dataset.x, [0])
-                    dataset.x = dataset.x.astype(np.float64)
-                    dataset.xmin = np.min(dataset.x)
-                    dataset.xmax = np.max(dataset.x)
-                if dataset.y is not None:
-                    dataset.y = np.delete(dataset.y, [0])
-                    dataset.y = dataset.y.astype(np.float64)
-                    dataset.ymin = np.min(dataset.y)
-                    dataset.ymax = np.max(dataset.y)
-                if dataset.dx is not None:
-                    dataset.dx = np.delete(dataset.dx, [0])
-                    dataset.dx = dataset.dx.astype(np.float64)
-                if dataset.dxl is not None:
-                    dataset.dxl = np.delete(dataset.dxl, [0])
-                    dataset.dxl = dataset.dxl.astype(np.float64)
-                if dataset.dxw is not None:
-                    dataset.dxw = np.delete(dataset.dxw, [0])
-                    dataset.dxw = dataset.dxw.astype(np.float64)
-                if dataset.dy is not None:
-                    dataset.dy = np.delete(dataset.dy, [0])
-                    dataset.dy = dataset.dy.astype(np.float64)
-                np.trim_zeros(dataset.x)
-                np.trim_zeros(dataset.y)
-                np.trim_zeros(dataset.dy)
-            elif isinstance(dataset, plottable_2D):
-                dataset.data = dataset.data.astype(np.float64)
-                dataset.qx_data = dataset.qx_data.astype(np.float64)
-                dataset.xmin = np.min(dataset.qx_data)
-                dataset.xmax = np.max(dataset.qx_data)
-                dataset.qy_data = dataset.qy_data.astype(np.float64)
-                dataset.ymin = np.min(dataset.qy_data)
-                dataset.ymax = np.max(dataset.qy_data)
-                dataset.q_data = np.sqrt(dataset.qx_data * dataset.qx_data
-                                         + dataset.qy_data * dataset.qy_data)
-                if dataset.err_data is not None:
-                    dataset.err_data = dataset.err_data.astype(np.float64)
-                if dataset.dqx_data is not None:
-                    dataset.dqx_data = dataset.dqx_data.astype(np.float64)
-                if dataset.dqy_data is not None:
-                    dataset.dqy_data = dataset.dqy_data.astype(np.float64)
-                if dataset.mask is not None:
-                    dataset.mask = dataset.mask.astype(dtype=bool)
-
-                if len(dataset.shape) == 2:
-                    n_rows, n_cols = dataset.shape
-                    dataset.y_bins = dataset.qy_data[0::int(n_cols)]
-                    dataset.x_bins = dataset.qx_data[:int(n_cols)]
-                    dataset.data = dataset.data.flatten()
-                else:
-                    dataset.y_bins = []
-                    dataset.x_bins = []
-                    dataset.data = dataset.data.flatten()
-
-            final_dataset = combine_data(dataset, self.current_datainfo)
-            self.output.append(final_dataset)
-
-    def _create_unique_key(self, dictionary, name, numb=0):
-        """
-        Create a unique key value for any dictionary to prevent overwriting
-        Recurse until a unique key value is found.
-
-        :param dictionary: A dictionary with any number of entries
-        :param name: The index of the item to be added to dictionary
-        :param numb: The number to be appended to the name, starts at 0
-        """
-        if dictionary.get(name) is not None:
-            numb += 1
-            name = name.split("_")[0]
-            name += "_{0}".format(numb)
-            name = self._create_unique_key(dictionary, name, numb)
-        return name
 
     def _get_node_value(self, node, tagname):
@@ -801 +655 @@
         return node_value, value_unit
 
-    def _check_for_empty_data(self):
-        """
-        Creates an empty data set if no data is passed to the reader
-
-        :param data1d: presumably a Data1D object
-        """
-        if self.current_dataset is None:
-            x_vals = np.empty(0)
-            y_vals = np.empty(0)
-            dx_vals = np.empty(0)
-            dy_vals = np.empty(0)
-            dxl = np.empty(0)
-            dxw = np.empty(0)
-            self.current_dataset = plottable_1D(x_vals, y_vals, dx_vals, dy_vals)
-            self.current_dataset.dxl = dxl
-            self.current_dataset.dxw = dxw
-
     def _check_for_empty_resolution(self):
         """
-        A method to check all resolution data sets are the same size as I and Q
-        """
-        if isinstance(self.current_dataset, plottable_1D):
-            dql_exists = False
-            dqw_exists = False
-            dq_exists = False
-            di_exists = False
-            if self.current_dataset.dxl is not None:
-                dql_exists = True
-            if self.current_dataset.dxw is not None:
-                dqw_exists = True
-            if self.current_dataset.dx is not None:
-                dq_exists = True
-            if self.current_dataset.dy is not None:
-                di_exists = True
-            if dqw_exists and not dql_exists:
-                array_size = self.current_dataset.dxw.size - 1
-                self.current_dataset.dxl = np.append(self.current_dataset.dxl,
-                                                     np.zeros([array_size]))
-            elif dql_exists and not dqw_exists:
-                array_size = self.current_dataset.dxl.size - 1
-                self.current_dataset.dxw = np.append(self.current_dataset.dxw,
-                                                     np.zeros([array_size]))
-            elif not dql_exists and not dqw_exists and not dq_exists:
-                array_size = self.current_dataset.x.size - 1
-                self.current_dataset.dx = np.append(self.current_dataset.dx,
-                                                    np.zeros([array_size]))
-            if not di_exists:
-                array_size = self.current_dataset.y.size - 1
-                self.current_dataset.dy = np.append(self.current_dataset.dy,
-                                                    np.zeros([array_size]))
-        elif isinstance(self.current_dataset, plottable_2D):
-            dqx_exists = False
-            dqy_exists = False
-            di_exists = False
-            mask_exists = False
-            if self.current_dataset.dqx_data is not None:
-                dqx_exists = True
-            if self.current_dataset.dqy_data is not None:
-                dqy_exists = True
-            if self.current_dataset.err_data is not None:
-                di_exists = True
-            if self.current_dataset.mask is not None:
-                mask_exists = True
-            if not dqy_exists:
-                array_size = self.current_dataset.qy_data.size - 1
-                self.current_dataset.dqy_data = np.append(
-                    self.current_dataset.dqy_data, np.zeros([array_size]))
-            if not dqx_exists:
-                array_size = self.current_dataset.qx_data.size - 1
-                self.current_dataset.dqx_data = np.append(
-                    self.current_dataset.dqx_data, np.zeros([array_size]))
-            if not di_exists:
-                array_size = self.current_dataset.data.size - 1
-                self.current_dataset.err_data = np.append(
-                    self.current_dataset.err_data, np.zeros([array_size]))
-            if not mask_exists:
-                array_size = self.current_dataset.data.size - 1
-                self.current_dataset.mask = np.append(
-                    self.current_dataset.mask,
-                    np.ones([array_size] ,dtype=bool))
-
-    ####### All methods below are for writing CanSAS XML files #######
-
+        a method to check all resolution data sets are the same size as I and q
+        """
+        dql_exists = False
+        dqw_exists = False
+        dq_exists = False
+        di_exists = False
+        if self.current_dataset.dxl is not None:
+            dql_exists = True
+        if self.current_dataset.dxw is not None:
+            dqw_exists = True
+        if self.current_dataset.dx is not None:
+            dq_exists = True
+        if self.current_dataset.dy is not None:
+            di_exists = True
+        if dqw_exists and not dql_exists:
+            array_size = self.current_dataset.dxw.size - 1
+            self.current_dataset.dxl = np.append(self.current_dataset.dxl,
+                                                 np.zeros([array_size]))
+        elif dql_exists and not dqw_exists:
+            array_size = self.current_dataset.dxl.size - 1
+            self.current_dataset.dxw = np.append(self.current_dataset.dxw,
+                                                 np.zeros([array_size]))
+        elif not dql_exists and not dqw_exists and not dq_exists:
+            array_size = self.current_dataset.x.size - 1
+            self.current_dataset.dx = np.append(self.current_dataset.dx,
+                                                np.zeros([array_size]))
+        if not di_exists:
+            array_size = self.current_dataset.y.size - 1
+            self.current_dataset.dy = np.append(self.current_dataset.dy,
+                                                np.zeros([array_size]))
+
+    def _initialize_new_data_set(self, node=None):
+        if node is not None:
+            for child in node:
+                if child.tag.replace(self.base_ns, "") == "Idata":
+                    for i_child in child:
+                        if i_child.tag.replace(self.base_ns, "") == "Qx":
+                            self.current_dataset = plottable_2D()
+                            return
+        self.current_dataset = plottable_1D(np.array(0), np.array(0))
+
+    ## Writing Methods
     def write(self, filename, datainfo):
         """
@@ -1514 +1329 @@
             exec "storage.%s = entry.text.strip()" % variable
 
-
 # DO NOT REMOVE Called by outside packages:
 #    sas.sasgui.perspectives.invariant.invariant_state

src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py

@@ -13 +13 @@
     TransmissionSpectrum, Detector
 from sas.sascalc.dataloader.data_info import combine_data_info_with_plottable
-
-
-class Reader():
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException, DefaultReaderException
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+
+
+class Reader(FileReader):
     """
     A class for reading in CanSAS v2.0 data files. The existing iteration opens
@@ -40 +42 @@
     # Raw file contents to be processed
     raw_data = None
-    # Data info currently being read in
-    current_datainfo = None
-    # SASdata set currently being read in
-    current_dataset = None
     # List of plottable1D objects that should be linked to the current_datainfo
     data1d = None
@@ -56 +54 @@
     # Flag to bypass extension check
     allow_all = True
-    # List of files to return
-    output = None
-
-    def read(self, filename):
+
+    def get_file_contents(self):
         """
         This is the general read method that all SasView data_loaders must have.
@@ -68 +64 @@
         # Reinitialize when loading a new data file to reset all class variables
         self.reset_class_variables()
+
+        filename = self.f_open.name
+        self.f_open.close() # IO handled by h5py
+
         # Check that the file exists
         if os.path.isfile(filename):
@@ -75 +75 @@
             if extension in self.ext or self.allow_all:
                 # Load the data file
-                self.raw_data = h5py.File(filename, 'r')
+                try:
+                    self.raw_data = h5py.File(filename, 'r')
+                except Exception as e:
+                    if extension not in self.ext:
+                        msg = "CanSAS2.0 HDF5 Reader could not load file {}".format(basename + extension)
+                        raise DefaultReaderException(msg)
+                    raise FileContentsException(e.message)
                 # Read in all child elements of top level SASroot
                 self.read_children(self.raw_data, [])
@@ -427 +433 @@
         Data1D and Data2D objects
         """
-
         # Type cast data arrays to float64
         if len(self.current_datainfo.trans_spectrum) > 0:
@@ -451 +456 @@
         # Type cast data arrays to float64 and find min/max as appropriate
         for dataset in self.data2d:
-            dataset.data = dataset.data.astype(np.float64)
-            dataset.err_data = dataset.err_data.astype(np.float64)
-            if dataset.qx_data is not None:
-                dataset.xmin = np.min(dataset.qx_data)
-                dataset.xmax = np.max(dataset.qx_data)
-                dataset.qx_data = dataset.qx_data.astype(np.float64)
-            if dataset.dqx_data is not None:
-                dataset.dqx_data = dataset.dqx_data.astype(np.float64)
-            if dataset.qy_data is not None:
-                dataset.ymin = np.min(dataset.qy_data)
-                dataset.ymax = np.max(dataset.qy_data)
-                dataset.qy_data = dataset.qy_data.astype(np.float64)
-            if dataset.dqy_data is not None:
-                dataset.dqy_data = dataset.dqy_data.astype(np.float64)
-            if dataset.q_data is not None:
-                dataset.q_data = dataset.q_data.astype(np.float64)
             zeros = np.ones(dataset.data.size, dtype=bool)
             try:
@@ -489 +478 @@
                 dataset.x_bins = dataset.qx_data[:n_cols]
                 dataset.data = dataset.data.flatten()
-
-            final_dataset = combine_data_info_with_plottable(
-                dataset, self.current_datainfo)
-            self.output.append(final_dataset)
+            self.current_dataset = dataset
+            self.send_to_output()
 
         for dataset in self.data1d:
-            if dataset.x is not None:
-                dataset.x = dataset.x.astype(np.float64)
-                dataset.xmin = np.min(dataset.x)
-                dataset.xmax = np.max(dataset.x)
-            if dataset.y is not None:
-                dataset.y = dataset.y.astype(np.float64)
-                dataset.ymin = np.min(dataset.y)
-                dataset.ymax = np.max(dataset.y)
-            if dataset.dx is not None:
-                dataset.dx = dataset.dx.astype(np.float64)
-            if dataset.dxl is not None:
-                dataset.dxl = dataset.dxl.astype(np.float64)
-            if dataset.dxw is not None:
-                dataset.dxw = dataset.dxw.astype(np.float64)
-            if dataset.dy is not None:
-                dataset.dy = dataset.dy.astype(np.float64)
-            final_dataset = combine_data_info_with_plottable(
-                dataset, self.current_datainfo)
-            self.output.append(final_dataset)
+            self.current_dataset = dataset
+            self.send_to_output()
 
     def add_data_set(self, key=""):

src/sas/sascalc/dataloader/readers/danse_reader.py

@@ -5 +5 @@
 #This software was developed by the University of Tennessee as part of the
 #Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
-#project funded by the US National Science Foundation. 
+#project funded by the US National Science Foundation.
 #If you use DANSE applications to do scientific research that leads to
 #publication, we ask that you acknowledge the use of the software with the
@@ -14 +14 @@
 import math
 import os
-import sys
 import numpy as np
 import logging
-from sas.sascalc.dataloader.data_info import Data2D, Detector
+from sas.sascalc.dataloader.data_info import plottable_2D, DataInfo, Detector
 from sas.sascalc.dataloader.manipulations import reader2D_converter
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException, DataReaderException
 
 logger = logging.getLogger(__name__)
@@ -30 +31 @@
 
 
-class Reader:
+class Reader(FileReader):
     """
     Example data manipulation
@@ -40 +41 @@
     ## Extension
     ext  = ['.sans', '.SANS']
-        
-    def read(self, filename=None):
-        """
-        Open and read the data in a file
-        @param file: path of the file
-        """
-        
-        read_it = False
-        for item in self.ext:
-            if filename.lower().find(item) >= 0:
-                read_it = True
-                
-        if read_it:
+
+    def get_file_contents(self):
+        self.current_datainfo = DataInfo()
+        self.current_dataset = plottable_2D()
+        self.output = []
+
+        loaded_correctly = True
+        error_message = ""
+
+        # defaults
+        # wavelength in Angstrom
+        wavelength = 10.0
+        # Distance in meter
+        distance   = 11.0
+        # Pixel number of center in x
+        center_x   = 65
+        # Pixel number of center in y
+        center_y   = 65
+        # Pixel size [mm]
+        pixel      = 5.0
+        # Size in x, in pixels
+        size_x     = 128
+        # Size in y, in pixels
+        size_y     = 128
+        # Format version
+        fversion   = 1.0
+
+        self.current_datainfo.filename = os.path.basename(self.f_open.name)
+        detector = Detector()
+        self.current_datainfo.detector.append(detector)
+
+        self.current_dataset.data = np.zeros([size_x, size_y])
+        self.current_dataset.err_data = np.zeros([size_x, size_y])
+
+        read_on = True
+        data_start_line = 1
+        while read_on:
+            line = self.f_open.readline()
+            data_start_line += 1
+            if line.find("DATA:") >= 0:
+                read_on = False
+                break
+            toks = line.split(':')
             try:
-                datafile = open(filename, 'r')
-            except:
-                raise  RuntimeError,"danse_reader cannot open %s" % (filename)
-        
-            # defaults
-            # wavelength in Angstrom
-            wavelength = 10.0
-            # Distance in meter
-            distance   = 11.0
-            # Pixel number of center in x
-            center_x   = 65
-            # Pixel number of center in y
-            center_y   = 65
-            # Pixel size [mm]
-            pixel      = 5.0
-            # Size in x, in pixels
-            size_x     = 128
-            # Size in y, in pixels
-            size_y     = 128
-            # Format version
-            fversion   = 1.0
-            
-            output = Data2D()
-            output.filename = os.path.basename(filename)
-            detector = Detector()
-            output.detector.append(detector)
-            
-            output.data = np.zeros([size_x,size_y])
-            output.err_data = np.zeros([size_x, size_y])
-            
-            data_conv_q = None
-            data_conv_i = None
-            
-            if has_converter == True and output.Q_unit != '1/A':
-                data_conv_q = Converter('1/A')
-                # Test it
-                data_conv_q(1.0, output.Q_unit)
-                
-            if has_converter == True and output.I_unit != '1/cm':
-                data_conv_i = Converter('1/cm')
-                # Test it
-                data_conv_i(1.0, output.I_unit)
-        
-            read_on = True
-            while read_on:
-                line = datafile.readline()
-                if line.find("DATA:") >= 0:
-                    read_on = False
-                    break
-                toks = line.split(':')
                 if toks[0] == "FORMATVERSION":
                     fversion = float(toks[1])
-                if toks[0] == "WAVELENGTH":
+                elif toks[0] == "WAVELENGTH":
                     wavelength = float(toks[1])
                 elif toks[0] == "DISTANCE":
@@ -120 +101 @@
                 elif toks[0] == "SIZE_Y":
                     size_y = int(toks[1])
-            
-            # Read the data
-            data = []
-            error = []
-            if fversion == 1.0:
-                data_str = datafile.readline()
-                data = data_str.split(' ')
-            else:
-                read_on = True
-                while read_on:
-                    data_str = datafile.readline()
-                    if len(data_str) == 0:
-                        read_on = False
-                    else:
-                        toks = data_str.split()
-                        try:
-                            val = float(toks[0])
-                            err = float(toks[1])
-                            if data_conv_i is not None:
-                                val = data_conv_i(val, units=output._yunit)
-                                err = data_conv_i(err, units=output._yunit)
-                            data.append(val)
-                            error.append(err)
-                        except:
-                            logger.info("Skipping line:%s,%s" %(data_str,
-                                                                sys.exc_value))
-            
-            # Initialize
-            x_vals = []
-            y_vals = []
-            ymin = None
-            ymax = None
-            xmin = None
-            xmax = None
-            
-            # Qx and Qy vectors
-            theta = pixel / distance / 100.0
-            stepq = 4.0 * math.pi / wavelength * math.sin(theta / 2.0)
-            for i_x in range(size_x):
-                theta = (i_x - center_x + 1) * pixel / distance / 100.0
-                qx = 4.0 * math.pi / wavelength * math.sin(theta / 2.0)
-                
-                if has_converter == True and output.Q_unit != '1/A':
-                    qx = data_conv_q(qx, units=output.Q_unit)
-                
-                x_vals.append(qx)
-                if xmin is None or qx < xmin:
-                    xmin = qx
-                if xmax is None or qx > xmax:
-                    xmax = qx
-            
-            ymin = None
-            ymax = None
-            for i_y in range(size_y):
-                theta = (i_y - center_y + 1) * pixel / distance / 100.0
-                qy = 4.0 * math.pi / wavelength * math.sin(theta/2.0)
-                
-                if has_converter == True and output.Q_unit != '1/A':
-                    qy = data_conv_q(qy, units=output.Q_unit)
-                
-                y_vals.append(qy)
-                if ymin is None or qy < ymin:
-                    ymin = qy
-                if ymax is None or qy > ymax:
-                    ymax = qy
-            
-            # Store the data in the 2D array
-            i_x = 0
-            i_y = -1
-            
-            for i_pt in range(len(data)):
-                try:
-                    value = float(data[i_pt])
-                except:
-                    # For version 1.0, the data were still
-                    # stored as strings at this point.
-                    msg = "Skipping entry (v1.0):%s,%s" % (str(data[i_pt]),
-                                                           sys.exc_value)
-                    logger.info(msg)
-                
-                # Get bin number
-                if math.fmod(i_pt, size_x) == 0:
-                    i_x = 0
-                    i_y += 1
-                else:
-                    i_x += 1
-                    
-                output.data[i_y][i_x] = value
-                if fversion>1.0:
-                    output.err_data[i_y][i_x] = error[i_pt]
-                
-            # Store all data
-            # Store wavelength
-            if has_converter == True and output.source.wavelength_unit != 'A':
-                conv = Converter('A')
-                wavelength = conv(wavelength,
-                                  units=output.source.wavelength_unit)
-            output.source.wavelength = wavelength
-                
-            # Store distance
-            if has_converter == True and detector.distance_unit != 'm':
-                conv = Converter('m')
-                distance = conv(distance, units=detector.distance_unit)
-            detector.distance = distance
-            
-            # Store pixel size
-            if has_converter == True and detector.pixel_size_unit != 'mm':
-                conv = Converter('mm')
-                pixel = conv(pixel, units=detector.pixel_size_unit)
-            detector.pixel_size.x = pixel
-            detector.pixel_size.y = pixel
-
-            # Store beam center in distance units
-            detector.beam_center.x = center_x * pixel
-            detector.beam_center.y = center_y * pixel
-            
-            # Store limits of the image (2D array)
-            xmin = xmin - stepq / 2.0
-            xmax = xmax + stepq / 2.0
-            ymin = ymin - stepq /2.0
-            ymax = ymax + stepq / 2.0
-            
-            if has_converter == True and output.Q_unit != '1/A':
-                xmin = data_conv_q(xmin, units=output.Q_unit)
-                xmax = data_conv_q(xmax, units=output.Q_unit)
-                ymin = data_conv_q(ymin, units=output.Q_unit)
-                ymax = data_conv_q(ymax, units=output.Q_unit)
-            output.xmin = xmin
-            output.xmax = xmax
-            output.ymin = ymin
-            output.ymax = ymax
-            
-            # Store x and y axis bin centers
-            output.x_bins = x_vals
-            output.y_bins = y_vals
-           
-            # Units
-            if data_conv_q is not None:
-                output.xaxis("\\rm{Q_{x}}", output.Q_unit)
-                output.yaxis("\\rm{Q_{y}}", output.Q_unit)
-            else:
-                output.xaxis("\\rm{Q_{x}}", 'A^{-1}')
-                output.yaxis("\\rm{Q_{y}}", 'A^{-1}')
-                
-            if data_conv_i is not None:
-                output.zaxis("\\rm{Intensity}", output.I_unit)
-            else:
-                output.zaxis("\\rm{Intensity}", "cm^{-1}")
-           
-            if not fversion >= 1.0:
-                msg = "Danse_reader can't read this file %s" % filename
-                raise ValueError, msg
-            else:
-                logger.info("Danse_reader Reading %s \n" % filename)
-            
-            # Store loading process information
-            output.meta_data['loader'] = self.type_name
-            output = reader2D_converter(output)
-            return output
-        
-        return None
+            except ValueError as e:
+                error_message += "Unable to parse {}. Default value used.\n".format(toks[0])
+                loaded_correctly = False
+
+        # Read the data
+        data = []
+        error = []
+        if not fversion >= 1.0:
+            msg = "danse_reader can't read this file {}".format(self.f_open.name)
+            raise FileContentsException(msg)
+
+        for line_num, data_str in enumerate(self.f_open.readlines()):
+            toks = data_str.split()
+            try:
+                val = float(toks[0])
+                err = float(toks[1])
+                data.append(val)
+                error.append(err)
+            except ValueError as exc:
+                msg = "Unable to parse line {}: {}".format(line_num + data_start_line, data_str.strip())
+                raise FileContentsException(msg)
+
+        num_pts = size_x * size_y
+        if len(data) < num_pts:
+            msg = "Not enough data points provided. Expected {} but got {}".format(
+                size_x * size_y, len(data))
+            raise FileContentsException(msg)
+        elif len(data) > num_pts:
+            error_message += ("Too many data points provided. Expected {0} but"
+                " got {1}. Only the first {0} will be used.\n").format(num_pts, len(data))
+            loaded_correctly = False
+            data = data[:num_pts]
+            error = error[:num_pts]
+
+        # Qx and Qy vectors
+        theta = pixel / distance / 100.0
+        i_x = np.arange(size_x)
+        theta = (i_x - center_x + 1) * pixel / distance / 100.0
+        x_vals = 4.0 * np.pi / wavelength * np.sin(theta / 2.0)
+        xmin = x_vals.min()
+        xmax = x_vals.max()
+
+        i_y = np.arange(size_y)
+        theta = (i_y - center_y + 1) * pixel / distance / 100.0
+        y_vals = 4.0 * np.pi / wavelength * np.sin(theta / 2.0)
+        ymin = y_vals.min()
+        ymax = y_vals.max()
+
+        self.current_dataset.data = np.array(data, dtype=np.float64).reshape((size_y, size_x))
+        if fversion > 1.0:
+            self.current_dataset.err_data = np.array(error, dtype=np.float64).reshape((size_y, size_x))
+
+        # Store all data
+        # Store wavelength
+        if has_converter == True and self.current_datainfo.source.wavelength_unit != 'A':
+            conv = Converter('A')
+            wavelength = conv(wavelength,
+                              units=self.current_datainfo.source.wavelength_unit)
+        self.current_datainfo.source.wavelength = wavelength
+
+        # Store distance
+        if has_converter == True and detector.distance_unit != 'm':
+            conv = Converter('m')
+            distance = conv(distance, units=detector.distance_unit)
+        detector.distance = distance
+
+        # Store pixel size
+        if has_converter == True and detector.pixel_size_unit != 'mm':
+            conv = Converter('mm')
+            pixel = conv(pixel, units=detector.pixel_size_unit)
+        detector.pixel_size.x = pixel
+        detector.pixel_size.y = pixel
+
+        # Store beam center in distance units
+        detector.beam_center.x = center_x * pixel
+        detector.beam_center.y = center_y * pixel
+
+
+        self.current_dataset.xaxis("\\rm{Q_{x}}", 'A^{-1}')
+        self.current_dataset.yaxis("\\rm{Q_{y}}", 'A^{-1}')
+        self.current_dataset.zaxis("\\rm{Intensity}", "cm^{-1}")
+
+        self.current_dataset.x_bins = x_vals
+        self.current_dataset.y_bins = y_vals
+
+        # Reshape data
+        x_vals = np.tile(x_vals, (size_y, 1)).flatten()
+        y_vals = np.tile(y_vals, (size_x, 1)).T.flatten()
+        if self.current_dataset.err_data == np.all(np.array(None)) or np.any(self.current_dataset.err_data <= 0):
+            new_err_data = np.sqrt(np.abs(self.current_dataset.data))
+        else:
+            new_err_data = self.current_dataset.err_data.flatten()
+
+        self.current_dataset.err_data = new_err_data
+        self.current_dataset.qx_data = x_vals
+        self.current_dataset.qy_data = y_vals
+        self.current_dataset.q_data = np.sqrt(x_vals**2 + y_vals**2)
+        self.current_dataset.mask = np.ones(len(x_vals), dtype=bool)
+
+        # Store loading process information
+        self.current_datainfo.meta_data['loader'] = self.type_name
+
+        self.send_to_output()
+
+        if not loaded_correctly:
+            raise DataReaderException(error_message)

src/sas/sascalc/dataloader/readers/red2d_reader.py

@@ -5 +5 @@
 #This software was developed by the University of Tennessee as part of the
 #Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
-#project funded by the US National Science Foundation. 
+#project funded by the US National Science Foundation.
 #See the license text in license.txt
 #copyright 2008, University of Tennessee
 ######################################################################
-from __future__ import print_function
-
 import os
 import numpy as np
 import math
-from sas.sascalc.dataloader.data_info import Data2D, Detector
+from sas.sascalc.dataloader.data_info import plottable_2D, DataInfo, Detector
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException
 
 # Look for unit converter
@@ -22 +22 @@
 except:
     has_converter = False
-    
-    
+
+
 def check_point(x_point):
     """
@@ -33 +33 @@
     except:
         return 0
-      
-        
-class Reader:
+
+
+class Reader(FileReader):
     """ Simple data reader for Igor data files """
     ## File type
@@ -43 +43 @@
     ## Extension
     ext = ['.DAT', '.dat']
-    
+
     def write(self, filename, data):
         """
         Write to .dat
-        
+
         :param filename: file name to write
         :param data: data2D
@@ -53 +53 @@
         import time
         # Write the file
-        fd = open(filename, 'w')
-        t = time.localtime()
-        time_str = time.strftime("%H:%M on %b %d %y", t)
-        
-        header_str = "Data columns are Qx - Qy - I(Qx,Qy)\n\nASCII data"
-        header_str += " created at %s \n\n" % time_str
-        # simple 2D header
-        fd.write(header_str)
-        # write qx qy I values
-        for i in range(len(data.data)):
-            fd.write("%g  %g  %g\n" % (data.qx_data[i],
-                                        data.qy_data[i],
-                                       data.data[i]))
-        # close
-        fd.close()
-
-    def read(self, filename=None):
-        """ Read file """
-        if not os.path.isfile(filename):
-            raise ValueError, \
-            "Specified file %s is not a regular file" % filename
-
+        try:
+            fd = open(filename, 'w')
+            t = time.localtime()
+            time_str = time.strftime("%H:%M on %b %d %y", t)
+
+            header_str = "Data columns are Qx - Qy - I(Qx,Qy)\n\nASCII data"
+            header_str += " created at %s \n\n" % time_str
+            # simple 2D header
+            fd.write(header_str)
+            # write qx qy I values
+            for i in range(len(data.data)):
+                fd.write("%g  %g  %g\n" % (data.qx_data[i],
+                                            data.qy_data[i],
+                                           data.data[i]))
+        finally:
+            fd.close()
+
+    def get_file_contents(self):
         # Read file
-        f = open(filename, 'r')
-        buf = f.read()
-        f.close()
+        buf = self.f_open.read()
+        self.f_open.close()
         # Instantiate data object
-        output = Data2D()
-        output.filename = os.path.basename(filename)
-        detector = Detector()
-        if len(output.detector) > 0:
-            print(str(output.detector[0]))
-        output.detector.append(detector)
-                
+        self.current_dataset = plottable_2D()
+        self.current_datainfo = DataInfo()
+        self.current_datainfo.filename = os.path.basename(self.f_open.name)
+        self.current_datainfo.detector.append(Detector())
+
         # Get content
-        dataStarted = False
-        
+        data_started = False
+
         ## Defaults
         lines = buf.split('\n')
         x = []
         y = []
-        
+
         wavelength = None
         distance = None
         transmission = None
-        
+
         pixel_x = None
         pixel_y = None
-        
-        isInfo = False
-        isCenter = False
-
-        data_conv_q = None
-        data_conv_i = None
-        
-        # Set units: This is the unit assumed for Q and I in the data file.
-        if has_converter == True and output.Q_unit != '1/A':
-            data_conv_q = Converter('1/A')
-            # Test it
-            data_conv_q(1.0, output.Q_unit)
-            
-        if has_converter == True and output.I_unit != '1/cm':
-            data_conv_i = Converter('1/cm')
-            # Test it
-            data_conv_i(1.0, output.I_unit)
-        
-              
+
+        is_info = False
+        is_center = False
+
         # Remove the last lines before the for loop if the lines are empty
         # to calculate the exact number of data points
@@ -135 +113 @@
             ## Reading the header applies only to IGOR/NIST 2D q_map data files
             # Find setup info line
-            if isInfo:
-                isInfo = False
+            if is_info:
+                is_info = False
                 line_toks = line.split()
                 # Wavelength in Angstrom
@@ -143 +121 @@
                     # Units
                     if has_converter == True and \
-                    output.source.wavelength_unit != 'A':
+                    self.current_datainfo.source.wavelength_unit != 'A':
                         conv = Converter('A')
                         wavelength = conv(wavelength,
-                                          units=output.source.wavelength_unit)
+                                          units=self.current_datainfo.source.wavelength_unit)
                 except:
                     #Not required
@@ -154 +132 @@
                     distance = float(line_toks[3])
                     # Units
-                    if has_converter == True and detector.distance_unit != 'm':
+                    if has_converter == True and self.current_datainfo.detector[0].distance_unit != 'm':
                         conv = Converter('m')
-                        distance = conv(distance, units=detector.distance_unit)
+                        distance = conv(distance,
+                            units=self.current_datainfo.detector[0].distance_unit)
                 except:
                     #Not required
                     pass
-                
+
                 # Distance in meters
                 try:
@@ -167 +146 @@
                     #Not required
                     pass
-                                            
+
             if line.count("LAMBDA") > 0:
-                isInfo = True
-                
+                is_info = True
+
             # Find center info line
-            if isCenter:
-                isCenter = False
+            if is_center:
+                is_center = False
                 line_toks = line.split()
                 # Center in bin number
@@ -180 +159 @@
 
             if line.count("BCENT") > 0:
-                isCenter = True
+                is_center = True
             # Check version
             if line.count("Data columns") > 0:
@@ -187 +166 @@
             # Find data start
             if line.count("ASCII data") > 0:
-                dataStarted = True
+                data_started = True
                 continue
 
             ## Read and get data.
-            if dataStarted == True:
+            if data_started == True:
                 line_toks = line.split()
                 if len(line_toks) == 0:
                     #empty line
                     continue
-                # the number of columns must be stayed same 
+                # the number of columns must be stayed same
                 col_num = len(line_toks)
                 break
@@ -204 +183 @@
         # index for lines_array
         lines_index = np.arange(len(lines))
-        
+
         # get the data lines
         data_lines = lines_array[lines_index >= (line_num - 1)]
@@ -213 +192 @@
         # split all data to one big list w/" "separator
         data_list = data_list.split()
- 
+
         # Check if the size is consistent with data, otherwise
         #try the tab(\t) separator
@@ -233 +212 @@
             data_point = data_array.reshape(row_num, col_num).transpose()
         except:
-            msg = "red2d_reader: Can't read this file: Not a proper file format"
-            raise ValueError, msg
+            msg = "red2d_reader can't read this file: Incorrect number of data points provided."
+            raise FileContentsException(msg)
         ## Get the all data: Let's HARDcoding; Todo find better way
         # Defaults
@@ -257 +236 @@
         #if col_num > (6 + ver): mask[data_point[(6 + ver)] < 1] = False
         q_data = np.sqrt(qx_data*qx_data+qy_data*qy_data+qz_data*qz_data)
-           
-        # Extra protection(it is needed for some data files): 
+
+        # Extra protection(it is needed for some data files):
         # If all mask elements are False, put all True
         if not mask.any():
             mask[mask == False] = True
-  
+
         # Store limits of the image in q space
         xmin = np.min(qx_data)
@@ -269 +248 @@
         ymax = np.max(qy_data)
 
-        # units
-        if has_converter == True and output.Q_unit != '1/A':
-            xmin = data_conv_q(xmin, units=output.Q_unit)
-            xmax = data_conv_q(xmax, units=output.Q_unit)
-            ymin = data_conv_q(ymin, units=output.Q_unit)
-            ymax = data_conv_q(ymax, units=output.Q_unit)
-            
         ## calculate the range of the qx and qy_data
         x_size = math.fabs(xmax - xmin)
         y_size = math.fabs(ymax - ymin)
-        
+
         # calculate the number of pixels in the each axes
         npix_y = math.floor(math.sqrt(len(data)))
         npix_x = math.floor(len(data) / npix_y)
-        
+
         # calculate the size of bins
         xstep = x_size / (npix_x - 1)
         ystep = y_size / (npix_y - 1)
-        
+
         # store x and y axis bin centers in q space
         x_bins = np.arange(xmin, xmax + xstep, xstep)
         y_bins = np.arange(ymin, ymax + ystep, ystep)
-       
+
         # get the limits of q values
         xmin = xmin - xstep / 2
@@ -297 +269 @@
         ymin = ymin - ystep / 2
         ymax = ymax + ystep / 2
-        
+
         #Store data in outputs
         #TODO: Check the lengths
-        output.data = data
+        self.current_dataset.data = data
         if (err_data == 1).all():
-            output.err_data = np.sqrt(np.abs(data))
-            output.err_data[output.err_data == 0.0] = 1.0
+            self.current_dataset.err_data = np.sqrt(np.abs(data))
+            self.current_dataset.err_data[self.current_dataset.err_data == 0.0] = 1.0
         else:
-            output.err_data = err_data
-            
-        output.qx_data = qx_data
-        output.qy_data = qy_data
-        output.q_data = q_data
-        output.mask = mask
-        
-        output.x_bins = x_bins
-        output.y_bins = y_bins
-               
-        output.xmin = xmin
-        output.xmax = xmax
-        output.ymin = ymin
-        output.ymax = ymax
-        
-        output.source.wavelength = wavelength
-        
+            self.current_dataset.err_data = err_data
+
+        self.current_dataset.qx_data = qx_data
+        self.current_dataset.qy_data = qy_data
+        self.current_dataset.q_data = q_data
+        self.current_dataset.mask = mask
+
+        self.current_dataset.x_bins = x_bins
+        self.current_dataset.y_bins = y_bins
+
+        self.current_dataset.xmin = xmin
+        self.current_dataset.xmax = xmax
+        self.current_dataset.ymin = ymin
+        self.current_dataset.ymax = ymax
+
+        self.current_datainfo.source.wavelength = wavelength
+
         # Store pixel size in mm
-        detector.pixel_size.x = pixel_x
-        detector.pixel_size.y = pixel_y
-        
+        self.current_datainfo.detector[0].pixel_size.x = pixel_x
+        self.current_datainfo.detector[0].pixel_size.y = pixel_y
+
         # Store the sample to detector distance
-        detector.distance = distance
-        
+        self.current_datainfo.detector[0].distance = distance
+
         # optional data: if all of dq data == 0, do not pass to output
         if len(dqx_data) == len(qx_data) and dqx_data.any() != 0:
@@ -340 +312 @@
                     cos_th = qx_data / diag
                     sin_th = qy_data / diag
-                    output.dqx_data = np.sqrt((dqx_data * cos_th) * \
+                    self.current_dataset.dqx_data = np.sqrt((dqx_data * cos_th) * \
                                                  (dqx_data * cos_th) \
                                                  + (dqy_data * sin_th) * \
                                                   (dqy_data * sin_th))
-                    output.dqy_data = np.sqrt((dqx_data * sin_th) * \
+                    self.current_dataset.dqy_data = np.sqrt((dqx_data * sin_th) * \
                                                  (dqx_data * sin_th) \
                                                  + (dqy_data * cos_th) * \
                                                   (dqy_data * cos_th))
                 else:
-                    output.dqx_data = dqx_data
-                    output.dqy_data = dqy_data
+                    self.current_dataset.dqx_data = dqx_data
+                    self.current_dataset.dqy_data = dqy_data
 
         # Units of axes
-        if data_conv_q is not None:
-            output.xaxis("\\rm{Q_{x}}", output.Q_unit)
-            output.yaxis("\\rm{Q_{y}}", output.Q_unit)
-        else:
-            output.xaxis("\\rm{Q_{x}}", 'A^{-1}')
-            output.yaxis("\\rm{Q_{y}}", 'A^{-1}')
-        if data_conv_i is not None:
-            output.zaxis("\\rm{Intensity}", output.I_unit)
-        else:
-            output.zaxis("\\rm{Intensity}", "cm^{-1}")
-    
+        self.current_dataset.xaxis("\\rm{Q_{x}}", 'A^{-1}')
+        self.current_dataset.yaxis("\\rm{Q_{y}}", 'A^{-1}')
+        self.current_dataset.zaxis("\\rm{Intensity}", "cm^{-1}")
+
         # Store loading process information
-        output.meta_data['loader'] = self.type_name
-
-        return output
+        self.current_datainfo.meta_data['loader'] = self.type_name
+
+        self.send_to_output()

src/sas/sascalc/dataloader/readers/sesans_reader.py

@@ -8 +8 @@
 import numpy as np
 import os
-from sas.sascalc.dataloader.data_info import Data1D
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
+from sas.sascalc.dataloader.data_info import plottable_1D, DataInfo
+from sas.sascalc.dataloader.loader_exceptions import FileContentsException, DataReaderException
 
 # Check whether we have a converter available
@@ -18 +20 @@
 _ZERO = 1e-16
 
-
-class Reader:
+class Reader(FileReader):
     """
     Class to load sesans files (6 columns).
@@ -26 +27 @@
     type_name = "SESANS"
 
-    # Wildcards
+    ## Wildcards
     type = ["SESANS files (*.ses)|*.ses",
             "SESANS files (*..sesans)|*.sesans"]
@@ -35 +36 @@
     allow_all = True
 
-    def read(self, path):
-        """
-        Load data file
+    def get_file_contents(self):
+        self.current_datainfo = DataInfo()
+        self.current_dataset = plottable_1D(np.array([]), np.array([]))
+        self.current_datainfo.isSesans = True
+        self.output = []
 
-        :param path: file path
+        line = self.f_open.readline()
+        params = {}
+        while not line.startswith("BEGIN_DATA"):
+            terms = line.split()
+            if len(terms) >= 2:
+                params[terms[0]] = " ".join(terms[1:])
+            line = self.f_open.readline()
+        self.params = params
 
-        :return: SESANSData1D object, or None
+        if "FileFormatVersion" not in self.params:
+            raise FileContentsException("SES file missing FileFormatVersion")
+        if float(self.params["FileFormatVersion"]) >= 2.0:
+            raise FileContentsException("SASView only supports SES version 1")
 
-        :raise RuntimeError: when the file can't be opened
-        :raise ValueError: when the length of the data vectors are inconsistent
-        """
-        if os.path.isfile(path):
-            basename = os.path.basename(path)
-            _, extension = os.path.splitext(basename)
-            if not (self.allow_all or extension.lower() in self.ext):
-                raise RuntimeError(
-                    "{} has an unrecognized file extension".format(path))
+        if "SpinEchoLength_unit" not in self.params:
+            raise FileContentsException("SpinEchoLength has no units")
+        if "Wavelength_unit" not in self.params:
+            raise FileContentsException("Wavelength has no units")
+        if params["SpinEchoLength_unit"] != params["Wavelength_unit"]:
+            raise FileContentsException("The spin echo data has rudely used "
+                               "different units for the spin echo length "
+                               "and the wavelength.  While sasview could "
+                               "handle this instance, it is a violation "
+                               "of the file format and will not be "
+                               "handled by other software.")
+
+        headers = self.f_open.readline().split()
+
+        self._insist_header(headers, "SpinEchoLength")
+        self._insist_header(headers, "Depolarisation")
+        self._insist_header(headers, "Depolarisation_error")
+        self._insist_header(headers, "Wavelength")
+
+        data = np.loadtxt(self.f_open)
+
+        if data.shape[1] != len(headers):
+            raise FileContentsException(
+                "File has {} headers, but {} columns".format(
+                    len(headers),
+                    data.shape[1]))
+
+        if not data.size:
+            raise FileContentsException("{} is empty".format(path))
+        x = data[:, headers.index("SpinEchoLength")]
+        if "SpinEchoLength_error" in headers:
+            dx = data[:, headers.index("SpinEchoLength_error")]
         else:
-            raise RuntimeError("{} is not a file".format(path))
-        with open(path, 'r') as input_f:
-            line = input_f.readline()
-            params = {}
-            while not line.startswith("BEGIN_DATA"):
-                terms = line.split()
-                if len(terms) >= 2:
-                    params[terms[0]] = " ".join(terms[1:])
-                line = input_f.readline()
-            self.params = params
+            dx = x * 0.05
+        lam = data[:, headers.index("Wavelength")]
+        if "Wavelength_error" in headers:
+            dlam = data[:, headers.index("Wavelength_error")]
+        else:
+            dlam = lam * 0.05
+        y = data[:, headers.index("Depolarisation")]
+        dy = data[:, headers.index("Depolarisation_error")]
 
-            if "FileFormatVersion" not in self.params:
-                raise RuntimeError("SES file missing FileFormatVersion")
-            if float(self.params["FileFormatVersion"]) >= 2.0:
-                raise RuntimeError("SASView only supports SES version 1")
+        lam_unit = self._unit_fetch("Wavelength")
+        x, x_unit = self._unit_conversion(x, "A",
+                                          self._unit_fetch(
+                                              "SpinEchoLength"))
+        dx, dx_unit = self._unit_conversion(
+            dx, lam_unit,
+            self._unit_fetch("SpinEchoLength"))
+        dlam, dlam_unit = self._unit_conversion(
+            dlam, lam_unit,
+            self._unit_fetch("Wavelength"))
+        y_unit = self._unit_fetch("Depolarisation")
 
-            if "SpinEchoLength_unit" not in self.params:
-                raise RuntimeError("SpinEchoLength has no units")
-            if "Wavelength_unit" not in self.params:
-                raise RuntimeError("Wavelength has no units")
-            if params["SpinEchoLength_unit"] != params["Wavelength_unit"]:
-                raise RuntimeError("The spin echo data has rudely used "
-                                   "different units for the spin echo length "
-                                   "and the wavelength.  While sasview could "
-                                   "handle this instance, it is a violation "
-                                   "of the file format and will not be "
-                                   "handled by other software.")
+        self.current_dataset.x = x
+        self.current_dataset.y = y
+        self.current_dataset.lam = lam
+        self.current_dataset.dy = dy
+        self.current_dataset.dx = dx
+        self.current_dataset.dlam = dlam
+        self.current_datainfo.isSesans = True
 
-            headers = input_f.readline().split()
+        self.current_datainfo._yunit = y_unit
+        self.current_datainfo._xunit = x_unit
+        self.current_datainfo.source.wavelength_unit = lam_unit
+        self.current_datainfo.source.wavelength = lam
+        self.current_datainfo.filename = os.path.basename(self.f_open.name)
+        self.current_dataset.xaxis(r"\rm{z}", x_unit)
+        # Adjust label to ln P/(lam^2 t), remove lam column refs
+        self.current_dataset.yaxis(r"\rm{ln(P)/(t \lambda^2)}", y_unit)
+        # Store loading process information
+        self.current_datainfo.meta_data['loader'] = self.type_name
+        self.current_datainfo.sample.name = params["Sample"]
+        self.current_datainfo.sample.ID = params["DataFileTitle"]
+        self.current_datainfo.sample.thickness = self._unit_conversion(
+            float(params["Thickness"]), "cm",
+            self._unit_fetch("Thickness"))[0]
 
-            self._insist_header(headers, "SpinEchoLength")
-            self._insist_header(headers, "Depolarisation")
-            self._insist_header(headers, "Depolarisation_error")
-            self._insist_header(headers, "Wavelength")
+        self.current_datainfo.sample.zacceptance = (
+            float(params["Theta_zmax"]),
+            self._unit_fetch("Theta_zmax"))
 
-            data = np.loadtxt(input_f)
+        self.current_datainfo.sample.yacceptance = (
+            float(params["Theta_ymax"]),
+            self._unit_fetch("Theta_ymax"))
 
-            if data.shape[1] != len(headers):
-                raise RuntimeError(
-                    "File has {} headers, but {} columns".format(
-                        len(headers),
-                        data.shape[1]))
-
-            if not data.size:
-                raise RuntimeError("{} is empty".format(path))
-            x = data[:, headers.index("SpinEchoLength")]
-            if "SpinEchoLength_error" in headers:
-                dx = data[:, headers.index("SpinEchoLength_error")]
-            else:
-                dx = x * 0.05
-            lam = data[:, headers.index("Wavelength")]
-            if "Wavelength_error" in headers:
-                dlam = data[:, headers.index("Wavelength_error")]
-            else:
-                dlam = lam * 0.05
-            y = data[:, headers.index("Depolarisation")]
-            dy = data[:, headers.index("Depolarisation_error")]
-
-            lam_unit = self._unit_fetch("Wavelength")
-            x, x_unit = self._unit_conversion(x, "A",
-                                              self._unit_fetch(
-                                                  "SpinEchoLength"))
-            dx, dx_unit = self._unit_conversion(
-                dx, lam_unit,
-                self._unit_fetch("SpinEchoLength"))
-            dlam, dlam_unit = self._unit_conversion(
-                dlam, lam_unit,
-                self._unit_fetch("Wavelength"))
-            y_unit = self._unit_fetch("Depolarisation")
-
-            output = Data1D(x=x, y=y, lam=lam, dy=dy, dx=dx, dlam=dlam,
-                            isSesans=True)
-
-            output.y_unit = y_unit
-            output.x_unit = x_unit
-            output.source.wavelength_unit = lam_unit
-            output.source.wavelength = lam
-            self.filename = output.filename = basename
-            output.xaxis(r"\rm{z}", x_unit)
-            # Adjust label to ln P/(lam^2 t), remove lam column refs
-            output.yaxis(r"\rm{ln(P)/(t \lambda^2)}", y_unit)
-            # Store loading process information
-            output.meta_data['loader'] = self.type_name
-            output.sample.name = params["Sample"]
-            output.sample.ID = params["DataFileTitle"]
-            output.sample.thickness = self._unit_conversion(
-                float(params["Thickness"]), "cm",
-                self._unit_fetch("Thickness"))[0]
-
-            output.sample.zacceptance = (
-                float(params["Theta_zmax"]),
-                self._unit_fetch("Theta_zmax"))
-
-            output.sample.yacceptance = (
-                float(params["Theta_ymax"]),
-                self._unit_fetch("Theta_ymax"))
-            return output
+        self.send_to_output()
 
     @staticmethod
     def _insist_header(headers, name):
         if name not in headers:
-            raise RuntimeError(
+            raise FileContentsException(
                 "Missing {} column in spin echo data".format(name))
 

src/sas/sascalc/dataloader/readers/xml_reader.py

@@ -18 +18 @@
 from lxml import etree
 from lxml.builder import E
+from sas.sascalc.dataloader.file_reader_base_class import FileReader
 
 logger = logging.getLogger(__name__)
@@ -23 +24 @@
 PARSER = etree.ETCompatXMLParser(remove_comments=True, remove_pis=False)
 
-class XMLreader():
+class XMLreader(FileReader):
     """
     Generic XML read and write class. Mostly helper functions.
@@ -74 +75 @@
         except etree.XMLSyntaxError as xml_error:
             logger.info(xml_error)
+            raise xml_error
         except Exception:
             self.xml = None
@@ -91 +93 @@
         except etree.XMLSyntaxError as xml_error:
             logger.info(xml_error)
-        except Exception:
+            raise xml_error
+        except Exception as exc:
             self.xml = None
             self.xmldoc = None
             self.xmlroot = None
+            raise exc
 
     def set_schema(self, schema):
@@ -206 +210 @@
         Create a unique key value for any dictionary to prevent overwriting
         Recurses until a unique key value is found.
-        
+
         :param dictionary: A dictionary with any number of entries
         :param name: The index of the item to be added to dictionary
@@ -222 +226 @@
         Create an element tree for processing from an etree element
 
-        :param root: etree Element(s) 
+        :param root: etree Element(s)
         """
         return etree.ElementTree(root)

src/sas/sasgui/guiframe/local_perspectives/data_loader/data_loader.py

@@ -11 +11 @@
 
 from sas.sascalc.dataloader.loader import Loader
+from sas.sascalc.dataloader.loader_exceptions import NoKnownLoaderException
 from sas.sasgui.guiframe.plugin_base import PluginBase
 from sas.sasgui.guiframe.events import StatusEvent
@@ -41 +42 @@
 APPLICATION_WLIST = config.APPLICATION_WLIST
 
+
 class Plugin(PluginBase):
 
@@ -56 +58 @@
         """
         # menu for data files
-        menu_list = []
         data_file_hint = "load one or more data in the application"
         menu_list = [('&Load Data File(s)', data_file_hint, self.load_data)]
         gui_style = self.parent.get_style()
         style = gui_style & GUIFRAME.MULTIPLE_APPLICATIONS
-        style1 = gui_style & GUIFRAME.DATALOADER_ON
         if style == GUIFRAME.MULTIPLE_APPLICATIONS:
             # menu for data from folder
@@ -102 +102 @@
         self.get_data(file_list)
 
-
     def can_load_data(self):
         """
@@ -108 +107 @@
         """
         return True
-
 
     def _load_folder(self, event):
@@ -140 +138 @@
         """
         if error is not None or str(error).strip() != "":
-            dial = wx.MessageDialog(self.parent, str(error), 'Error Loading File',
+            dial = wx.MessageDialog(self.parent, str(error),
+                                    'Error Loading File',
                                     wx.OK | wx.ICON_EXCLAMATION)
             dial.ShowModal()
@@ -149 +148 @@
         """
         if os.path.isdir(path):
-            return [os.path.join(os.path.abspath(path), filename) for filename in os.listdir(path)]
+            return [os.path.join(os.path.abspath(path), filename) for filename
+                    in os.listdir(path)]
 
     def _process_data_and_errors(self, item, p_file, output, message):
@@ -178 +178 @@
         for p_file in path:
             basename = os.path.basename(p_file)
+            # Skip files that start with a period
+            if basename.startswith("."):
+                msg = "The folder included a potential hidden file - %s." \
+                      % basename
+                msg += " Do you wish to load this file as data?"
+                msg_box = wx.MessageDialog(None, msg, 'Warning',
+                                           wx.OK | wx.CANCEL)
+                if msg_box.ShowModal() == wx.ID_CANCEL:
+                    continue
             _, extension = os.path.splitext(basename)
             if extension.lower() in EXTENSIONS:
@@ -213 +222 @@
                 info="info")
 
-            except:
-                logger.error(sys.exc_value)
-
-                error_message = "The Data file you selected could not be loaded.\n"
-                error_message += "Make sure the content of your file"
-                error_message += " is properly formatted.\n"
-                error_message += "When contacting the SasView team, mention the"
-                error_message += " following:\n"
-                error_message += "Error: " + str(sys.exc_info()[1])
-                file_errors[basename] = [error_message]
-                self.load_update(output=output, message=error_message, info="warning")
+            except NoKnownLoaderException as e:
+                exception_occurred = True
+                logging.error(e.message)
+
+                error_message = "Loading data failed!\n" + e.message
+                self.load_update(output=None, message=e.message, info="warning")
+
+            except Exception as e:
+                exception_occurred = True
+                logger.error(e.message)
+
+                file_err = "The Data file you selected could not be "
+                file_err += "loaded.\nMake sure the content of your file"
+                file_err += " is properly formatted.\n"
+                file_err += "When contacting the SasView team, mention the"
+                file_err += " following:\n"
+                file_err += e.message
+                file_errors[basename] = [file_err]
 
         if len(file_errors) > 0:
@@ -233 +249 @@
                     error_message += message + "\n"
                 error_message += "\n"
-            self.load_update(output=output, message=error_message, info="error")
-
-        self.load_complete(output=output, message="Loading data complete!",
-            info="info")
+            if not exception_occurred: # Some data loaded but with errors
+                self.load_update(output=output, message=error_message, info="error")
+
+        if not exception_occurred: # Everything loaded as expected
+            self.load_complete(output=output, message="Loading data complete!",
+                               info="info")
+        else:
+            self.load_complete(output=None, message=error_message, info="error")
+
 
     def load_update(self, output=None, message="", info="warning"):
@@ -245 +266 @@
             wx.PostEvent(self.parent, StatusEvent(status=message, info=info,
                                                   type="progress"))
-    def load_complete(self, output, message="", error_message="", path=None,
-                      info="warning"):
-        """
-         post message to  status bar and return list of data
-        """
-        wx.PostEvent(self.parent, StatusEvent(status=message,
-                                              info=info,
+
+    def load_complete(self, output, message="", info="warning"):
+        """
+         post message to status bar and return list of data
+        """
+        wx.PostEvent(self.parent, StatusEvent(status=message, info=info,
                                               type="stop"))
-        # if error_message != "":
-        #    self.load_error(error_message)
-        self.parent.add_data(data_list=output)
+        if output is not None:
+            self.parent.add_data(data_list=output)

src/sas/sasgui/perspectives/file_converter/converter_panel.py

@@ -24 +24 @@
 from sas.sascalc.file_converter.otoko_loader import OTOKOLoader
 from sas.sascalc.file_converter.bsl_loader import BSLLoader
+from sas.sascalc.file_converter.ascii2d_loader import ASCII2DLoader
 from sas.sascalc.file_converter.nxcansas_writer import NXcanSASWriter
 from sas.sascalc.dataloader.data_info import Detector
@@ -35 +36 @@
     _STATICBOX_WIDTH = 410
     _BOX_WIDTH = 200
-    PANEL_SIZE = 480
+    PANEL_SIZE = 520
     FONT_VARIANT = 0
 else:
@@ -41 +42 @@
     _STATICBOX_WIDTH = 430
     _BOX_WIDTH = 200
-    PANEL_SIZE = 500
+    PANEL_SIZE = 540
     FONT_VARIANT = 1
 
@@ -352 +353 @@
             w.write(frame_data, output_path)
 
+    def convert_2d_data(self, dataset):
+        metadata = self.get_metadata()
+        for key, value in metadata.iteritems():
+            setattr(dataset[0], key, value)
+
+        w = NXcanSASWriter()
+        w.write(dataset, self.output.GetPath())
+
     def on_convert(self, event):
         """Called when the Convert button is clicked"""
@@ -367 +376 @@
                 qdata, iqdata = self.extract_otoko_data(self.q_input.GetPath())
                 self.convert_1d_data(qdata, iqdata)
+            elif self.data_type == 'ascii2d':
+                loader = ASCII2DLoader(self.iq_input.GetPath())
+                data = loader.load()
+                dataset = [data] # ASCII 2D only ever contains 1 frame
+                self.convert_2d_data(dataset)
             else: # self.data_type == 'bsl'
                 dataset = self.extract_bsl_data(self.iq_input.GetPath())
@@ -372 +386 @@
                     # Cancelled by user
                     return
-
-                metadata = self.get_metadata()
-                for key, value in metadata.iteritems():
-                    setattr(dataset[0], key, value)
-
-                w = NXcanSASWriter()
-                w.write(dataset, self.output.GetPath())
+                self.convert_2d_data(dataset)
+
         except Exception as ex:
             msg = str(ex)
@@ -399 +408 @@
     def validate_inputs(self):
         msg = "You must select a"
-        if self.q_input.GetPath() == '' and self.data_type != 'bsl':
+        if self.q_input.GetPath() == '' and self.data_type != 'bsl' \
+            and self.data_type != 'ascii2d':
             msg += " Q Axis input file."
         elif self.iq_input.GetPath() == '':
@@ -472 +482 @@
         dtype = event.GetEventObject().GetName()
         self.data_type = dtype
-        if dtype == 'bsl':
+        if dtype == 'bsl' or dtype == 'ascii2d':
             self.q_input.SetPath("")
             self.q_input.Disable()
@@ -500 +510 @@
 
         instructions = (
-        "Select linked single column 1D ASCII files containing the Q-axis and "
-        "Intensity-axis data, or 1D BSL/OTOKO files, or a 2D BSL/OTOKO file, "
-        "then choose where to save the converted file, and click Convert.\n"
-        "1D ASCII and BSL/OTOKO files can be converted to CanSAS (XML) or "
-        "NXcanSAS (HDF5) formats. 2D BSL/OTOKO files can only be converted to "
-        "the NXcanSAS format.\n"
-        "Metadata can be optionally added for the CanSAS XML format."
+        "If converting a 1D dataset, select linked single-column ASCII files "
+        "containing the Q-axis and intensity-axis data, or a 1D BSL/OTOKO file."
+        " If converting 2D data, select an ASCII file in the ISIS 2D file "
+        "format, or a 2D BSL/OTOKO file. Choose where to save the converted "
+        "file and click convert.\n"
+        "One dimensional ASCII and BSL/OTOKO files can be converted to CanSAS "
+        "(XML) or NXcanSAS (HDF5) formats. Two dimensional datasets can only be"
+        " converted to the NXcanSAS format.\n"
+        "Metadata can also be optionally added to the output file."
         )
 
@@ -526 +538 @@
             wx.ALIGN_CENTER_VERTICAL, 5)
         radio_sizer = wx.BoxSizer(wx.HORIZONTAL)
-        ascii_btn = wx.RadioButton(self, -1, "ASCII", name="ascii",
+        ascii_btn = wx.RadioButton(self, -1, "ASCII 1D", name="ascii",
             style=wx.RB_GROUP)
         ascii_btn.Bind(wx.EVT_RADIOBUTTON, self.datatype_changed)
         radio_sizer.Add(ascii_btn)
+        ascii2d_btn = wx.RadioButton(self, -1, "ASCII 2D", name="ascii2d")
+        ascii2d_btn.Bind(wx.EVT_RADIOBUTTON, self.datatype_changed)
+        radio_sizer.Add(ascii2d_btn)
         otoko_btn = wx.RadioButton(self, -1, "BSL 1D", name="otoko")
         otoko_btn.Bind(wx.EVT_RADIOBUTTON, self.datatype_changed)
         radio_sizer.Add(otoko_btn)
-        input_grid.Add(radio_sizer, (y,1), (1,1), wx.ALL, 5)
         bsl_btn = wx.RadioButton(self, -1, "BSL 2D", name="bsl")
         bsl_btn.Bind(wx.EVT_RADIOBUTTON, self.datatype_changed)
         radio_sizer.Add(bsl_btn)
+        input_grid.Add(radio_sizer, (y,1), (1,1), wx.ALL, 5)
         y += 1
 
@@ -549 +564 @@
         y += 1
 
-        iq_label = wx.StaticText(self, -1, "Intensity-Axis Data: ")
+        iq_label = wx.StaticText(self, -1, "Intensity Data: ")
         input_grid.Add(iq_label, (y,0), (1,1), wx.ALIGN_CENTER_VERTICAL, 5)
 
@@ -647 +662 @@
 
     def __init__(self, parent=None, title='File Converter', base=None,
-        manager=None, size=(PANEL_SIZE * 1.05, PANEL_SIZE / 1.1),
+        manager=None, size=(PANEL_SIZE * 0.96, PANEL_SIZE * 0.9),
         *args, **kwargs):
         kwargs['title'] = title

src/sas/sasgui/perspectives/file_converter/file_converter.py

@@ -25 +25 @@
         Returns a set of menu entries
         """
-        help_txt = "Convert single column ASCII data to CanSAS format"
+        help_txt = "Convert ASCII or BSL/OTOKO data to CanSAS or NXcanSAS formats"
         return [("File Converter", help_txt, self.on_file_converter)]
 

src/sas/sasgui/perspectives/file_converter/media/file_converter_help.rst

@@ -18 +18 @@
 *   Single-column ASCII data, with lines that end without any delimiter,
     or with a comma or semi-colon delimiter
+*   2D `ISIS ASCII formatted
+    <http://www.isis.stfc.ac.uk/instruments/loq/software/
+    colette-ascii-file-format-descriptions9808.pdf>`_ data
 *   `1D BSL/OTOKO format
     <http://www.diamond.ac.uk/Beamlines/Soft-Condensed-Matter/small-angle/
@@ -36 +39 @@
 
 1) Select the files containing your Q-axis and Intensity-axis data
-2) Choose whether the files are in ASCII, 1D BSL/OTOKO or 2D BSL/OTOKO format
+2) Choose whether the files are in ASCII 1D, ASCII 2D, 1D BSL/OTOKO or 2D BSL/OTOKO format
 3) Choose where you would like to save the converted file
 4) Optionally, input some metadata such as sample size, detector name, etc
@@ -47 +50 @@
 file, a dialog will appear asking which frames you would like converted. You
 may enter a start frame, end frame & increment, and all frames in that subset
-will be converted. For example, entering 0, 50 and 10 will convert frames 0, 
+will be converted. For example, entering 0, 50 and 10 will convert frames 0,
 10, 20, 30, 40 & 50.
 
@@ -56 +59 @@
 single file, so there is an option in the *Select Frame* dialog to output each
 frame to its own file. The single file option will produce one file with
-multiple `<SASdata>` elements. The multiple file option will output a separate 
-file with one `<SASdata>` element for each frame. The frame number will also be 
+multiple `<SASdata>` elements. The multiple file option will output a separate
+file with one `<SASdata>` element for each frame. The frame number will also be
 appended to the file name.
 
-The multiple file option is not available when exporting to NXcanSAS because 
+The multiple file option is not available when exporting to NXcanSAS because
 the HDF5 format is more efficient at handling large amounts of data.