Changeset 6d62b7f in sasview for src/sas/sascalc/dataloader

Timestamp:

Sep 17, 2017 9:52:57 PM (7 years ago)

Author:

butler

Branches:

master, ESS_GUI, ESS_GUI_Docs, ESS_GUI_batch_fitting, ESS_GUI_bumps_abstraction, ESS_GUI_iss1116, ESS_GUI_iss879, ESS_GUI_iss959, ESS_GUI_opencl, ESS_GUI_ordering, ESS_GUI_sync_sascalc, magnetic_scatt, release-4.2.2, ticket-1009, ticket-1094-headless, ticket-1242-2d-resolution, ticket-1243, ticket-1249, ticket885, unittest-saveload

Children:

13374be

Parents:

ae69c690 (diff), cfd27dd (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge remote-tracking branch 'origin/master' into 4_1_issues

Location:

src/sas/sascalc/dataloader

Files:

• file_reader_base_class.py (modified) (5 diffs)
• readers/cansas_reader.py (modified) (8 diffs)
• readers/cansas_reader_HDF5.py (modified) (2 diffs)
• readers/xml_reader.py (modified) (1 diff)

src/sas/sascalc/dataloader/file_reader_base_class.py

@@ -115 +115 @@
                 data.y = np.asarray([data.y[i] for i in ind]).astype(np.float64)
                 if data.dx is not None:
+                    if len(data.dx) == 0:
+                        data.dx = None
+                        continue
                     data.dx = np.asarray([data.dx[i] for i in ind]).astype(np.float64)
                 if data.dxl is not None:
@@ -121 +124 @@
                     data.dxw = np.asarray([data.dxw[i] for i in ind]).astype(np.float64)
                 if data.dy is not None:
+                    if len(data.dy) == 0:
+                        data.dy = None
+                        continue
                     data.dy = np.asarray([data.dy[i] for i in ind]).astype(np.float64)
                 if data.lam is not None:
@@ -185 +191 @@
         self.output = []
 
-    def remove_empty_q_values(self, has_error_dx=False, has_error_dy=False):
+    def remove_empty_q_values(self, has_error_dx=False, has_error_dy=False,
+                              has_error_dxl=False, has_error_dxw=False):
         """
         Remove any point where Q == 0
@@ -192 +199 @@
         self.current_dataset.x = self.current_dataset.x[x != 0]
         self.current_dataset.y = self.current_dataset.y[x != 0]
-        self.current_dataset.dy = self.current_dataset.dy[x != 0] if \
-            has_error_dy else np.zeros(len(self.current_dataset.y))
-        self.current_dataset.dx = self.current_dataset.dx[x != 0] if \
-            has_error_dx else np.zeros(len(self.current_dataset.x))
+        if has_error_dy:
+            self.current_dataset.dy = self.current_dataset.dy[x != 0]
+        if has_error_dx:
+            self.current_dataset.dx = self.current_dataset.dx[x != 0]
+        if has_error_dxl:
+            self.current_dataset.dxl = self.current_dataset.dxl[x != 0]
+        if has_error_dxw:
+            self.current_dataset.dxw = self.current_dataset.dxw[x != 0]
 
     def reset_data_list(self, no_lines=0):
@@ -204 +215 @@
         x = np.zeros(no_lines)
         y = np.zeros(no_lines)
+        dx = np.zeros(no_lines)
         dy = np.zeros(no_lines)
-        dx = np.zeros(no_lines)
         self.current_dataset = plottable_1D(x, y, dx, dy)
 

src/sas/sascalc/dataloader/readers/cansas_reader.py

@@ -130 +130 @@
                 self.current_datainfo.meta_data[PREPROCESS] = self.processing_instructions
                 self._parse_entry(entry)
-                has_error_dx = self.current_dataset.dx is not None
-                has_error_dy = self.current_dataset.dy is not None
-                self.remove_empty_q_values(has_error_dx=has_error_dx,
-                    has_error_dy=has_error_dy)
-                self.send_to_output() # Combine datasets with DataInfo
-                self.current_datainfo = DataInfo() # Reset DataInfo
+                self.data_cleanup()
         except FileContentsException as fc_exc:
             # File doesn't meet schema - try loading with a less strict schema
@@ -154 +149 @@
                     self.load_file_and_schema(xml_file) # Reload strict schema so we can find where error are in file
                     invalid_xml = self.find_invalid_xml()
-                    invalid_xml = INVALID_XML.format(basename + self.extension) + invalid_xml
-                    raise DataReaderException(invalid_xml) # Handled by base class
+                    if invalid_xml != "":
+                        invalid_xml = INVALID_XML.format(basename + self.extension) + invalid_xml
+                        raise DataReaderException(invalid_xml) # Handled by base class
                 except FileContentsException as fc_exc:
                     msg = "CanSAS Reader could not load the file {}".format(xml_file)
@@ -279 +275 @@
                 # I and Q points
                 elif tagname == 'I' and isinstance(self.current_dataset, plottable_1D):
-                    unit_list = unit.split("|")
-                    if len(unit_list) > 1:
-                        self.current_dataset.yaxis(unit_list[0].strip(),
-                                                   unit_list[1].strip())
-                    else:
-                        self.current_dataset.yaxis("Intensity", unit)
+                    self.current_dataset.yaxis("Intensity", unit)
                     self.current_dataset.y = np.append(self.current_dataset.y, data_point)
                 elif tagname == 'Idev' and isinstance(self.current_dataset, plottable_1D):
                     self.current_dataset.dy = np.append(self.current_dataset.dy, data_point)
                 elif tagname == 'Q':
-                    unit_list = unit.split("|")
-                    if len(unit_list) > 1:
-                        self.current_dataset.xaxis(unit_list[0].strip(),
-                                                   unit_list[1].strip())
-                    else:
-                        self.current_dataset.xaxis("Q", unit)
+                    self.current_dataset.xaxis("Q", unit)
                     self.current_dataset.x = np.append(self.current_dataset.x, data_point)
                 elif tagname == 'Qdev':
                     self.current_dataset.dx = np.append(self.current_dataset.dx, data_point)
                 elif tagname == 'dQw':
-                    if self.current_dataset.dxw is None:
-                        self.current_dataset.dxw = np.empty(0)
-                    self.current_dataset.dxw = np.append(self.current_dataset.dxw, data_point)
+                   self.current_dataset.dxw = np.append(self.current_dataset.dxw, data_point)
                 elif tagname == 'dQl':
-                    if self.current_dataset.dxl is None:
-                        self.current_dataset.dxl = np.empty(0)
                     self.current_dataset.dxl = np.append(self.current_dataset.dxl, data_point)
                 elif tagname == 'Qmean':
@@ -312 +294 @@
                 elif tagname == 'Sesans':
                     self.current_datainfo.isSesans = bool(data_point)
+                    self.current_dataset.xaxis(attr.get('x_axis'),
+                                                attr.get('x_unit'))
+                    self.current_dataset.yaxis(attr.get('y_axis'),
+                                                attr.get('y_unit'))
                 elif tagname == 'yacceptance':
                     self.current_datainfo.sample.yacceptance = (data_point, unit)
@@ -512 +498 @@
             for error in self.errors:
                 self.current_datainfo.errors.add(error)
-            self.errors.clear()
-            self.send_to_output()
+            self.data_cleanup()
+            self.sort_one_d_data()
+            self.sort_two_d_data()
+            self.reset_data_list()
             empty = None
             return self.output[0], empty
+
+    def data_cleanup(self):
+        """
+        Clean up the data sets and refresh everything
+        :return: None
+        """
+        has_error_dx = self.current_dataset.dx is not None
+        has_error_dxl = self.current_dataset.dxl is not None
+        has_error_dxw = self.current_dataset.dxw is not None
+        has_error_dy = self.current_dataset.dy is not None
+        self.remove_empty_q_values(has_error_dx=has_error_dx,
+                                   has_error_dxl=has_error_dxl,
+                                   has_error_dxw=has_error_dxw,
+                                   has_error_dy=has_error_dy)
+        self.send_to_output()  # Combine datasets with DataInfo
+        self.current_datainfo = DataInfo()  # Reset DataInfo
 
     def _is_call_local(self):
@@ -642 +646 @@
                     value_unit = local_unit
             except KeyError:
-                err_msg = "CanSAS reader: unexpected "
-                err_msg += "\"{0}\" unit [{1}]; "
-                err_msg = err_msg.format(tagname, local_unit)
-                err_msg += "expecting [{0}]".format(default_unit)
+                # Do not throw an error for loading Sesans data in cansas xml
+                # This is a temporary fix.
+                if local_unit != "A" and local_unit != 'pol':
+                    err_msg = "CanSAS reader: unexpected "
+                    err_msg += "\"{0}\" unit [{1}]; "
+                    err_msg = err_msg.format(tagname, local_unit)
+                    err_msg += "expecting [{0}]".format(default_unit)
                 value_unit = local_unit
             except:
@@ -675 +682 @@
             di_exists = True
         if dqw_exists and not dql_exists:
-            array_size = self.current_dataset.dxw.size - 1
-            self.current_dataset.dxl = np.append(self.current_dataset.dxl,
-                                                 np.zeros([array_size]))
+            array_size = self.current_dataset.dxw.size
+            self.current_dataset.dxl = np.zeros(array_size)
         elif dql_exists and not dqw_exists:
-            array_size = self.current_dataset.dxl.size - 1
-            self.current_dataset.dxw = np.append(self.current_dataset.dxw,
-                                                 np.zeros([array_size]))
+            array_size = self.current_dataset.dxl.size
+            self.current_dataset.dxw = np.zeros(array_size)
         elif not dql_exists and not dqw_exists and not dq_exists:
-            array_size = self.current_dataset.x.size - 1
+            array_size = self.current_dataset.x.size
             self.current_dataset.dx = np.append(self.current_dataset.dx,
                                                 np.zeros([array_size]))
         if not di_exists:
-            array_size = self.current_dataset.y.size - 1
+            array_size = self.current_dataset.y.size
             self.current_dataset.dy = np.append(self.current_dataset.dy,
                                                 np.zeros([array_size]))
@@ -857 +862 @@
             node.append(point)
             self.write_node(point, "Q", datainfo.x[i],
-                            {'unit': datainfo._xaxis + " | " + datainfo._xunit})
+                            {'unit': datainfo.x_unit})
             if len(datainfo.y) >= i:
                 self.write_node(point, "I", datainfo.y[i],
-                                {'unit': datainfo._yaxis + " | " + datainfo._yunit})
+                                {'unit': datainfo.y_unit})
             if datainfo.dy is not None and len(datainfo.dy) > i:
                 self.write_node(point, "Idev", datainfo.dy[i],
-                                {'unit': datainfo._yaxis + " | " + datainfo._yunit})
+                                {'unit': datainfo.y_unit})
             if datainfo.dx is not None and len(datainfo.dx) > i:
                 self.write_node(point, "Qdev", datainfo.dx[i],
-                                {'unit': datainfo._xaxis + " | " + datainfo._xunit})
+                                {'unit': datainfo.x_unit})
             if datainfo.dxw is not None and len(datainfo.dxw) > i:
                 self.write_node(point, "dQw", datainfo.dxw[i],
-                                {'unit': datainfo._xaxis + " | " + datainfo._xunit})
+                                {'unit': datainfo.x_unit})
             if datainfo.dxl is not None and len(datainfo.dxl) > i:
                 self.write_node(point, "dQl", datainfo.dxl[i],
-                                {'unit': datainfo._xaxis + " | " + datainfo._xunit})
+                                {'unit': datainfo.x_unit})
         if datainfo.isSesans:
-            sesans = self.create_element("Sesans")
+            sesans_attrib = {'x_axis': datainfo._xaxis,
+                             'y_axis': datainfo._yaxis,
+                             'x_unit': datainfo.x_unit,
+                             'y_unit': datainfo.y_unit}
+            sesans = self.create_element("Sesans", attrib=sesans_attrib)
             sesans.text = str(datainfo.isSesans)
-            node.append(sesans)
-            self.write_node(node, "yacceptance", datainfo.sample.yacceptance[0],
+            entry_node.append(sesans)
+            self.write_node(entry_node, "yacceptance", datainfo.sample.yacceptance[0],
                              {'unit': datainfo.sample.yacceptance[1]})
-            self.write_node(node, "zacceptance", datainfo.sample.zacceptance[0],
+            self.write_node(entry_node, "zacceptance", datainfo.sample.zacceptance[0],
                              {'unit': datainfo.sample.zacceptance[1]})
 

src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py

@@ -140 +140 @@
 
             if isinstance(value, h5py.Group):
+                # Set parent class before recursion
                 self.parent_class = class_name
                 parent_list.append(key)
@@ -150 +151 @@
                 # Recursion step to access data within the group
                 self.read_children(value, parent_list)
+                # Reset parent class when returning from recursive method
+                self.parent_class = class_name
                 self.add_intermediate()
                 parent_list.remove(key)

src/sas/sascalc/dataloader/readers/xml_reader.py

@@ -134 +134 @@
             first_error = schema.assertValid(self.xmldoc)
         except etree.DocumentInvalid as err:
+            # Suppress errors for <'any'> elements
+            if "##other" in str(err):
+                return first_error
             first_error = str(err)
         return first_error