Changeset 66000ae in sasview for src/sas/sascalc/dataloader

Timestamp:

Sep 25, 2017 3:01:30 PM (7 years ago)

Author:

Paul Kienzle <pkienzle@…>

Branches:

master, ESS_GUI, ESS_GUI_Docs, ESS_GUI_batch_fitting, ESS_GUI_bumps_abstraction, ESS_GUI_iss1116, ESS_GUI_iss879, ESS_GUI_iss959, ESS_GUI_opencl, ESS_GUI_ordering, ESS_GUI_sync_sascalc, magnetic_scatt, release-4.2.2, ticket-1009, ticket-1094-headless, ticket-1242-2d-resolution, ticket-1243, ticket-1249, ticket885, unittest-saveload

Children:

fca1f50, 83db1cc

Parents:

dba8557 (diff), 72d3f1e (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'ticket-869' into ticket-887-reorg

Location:

src/sas/sascalc/dataloader

Files:

• data_info.py (modified) (1 diff)
• file_reader_base_class.py (modified) (2 diffs)
• readers/abs_reader.py (modified) (2 diffs)
• readers/ascii_reader.py (modified) (1 diff)
• readers/cansas_reader.py (modified) (10 diffs)
• readers/__init__.py (modified) (1 diff)

src/sas/sascalc/dataloader/data_info.py

@@ -1176 +1176 @@
         final_dataset.yaxis(data._yaxis, data._yunit)
         final_dataset.zaxis(data._zaxis, data._zunit)
-        final_dataset.x_bins = data.x_bins
-        final_dataset.y_bins = data.y_bins
+        if len(data.data.shape) == 2:
+            n_rows, n_cols = data.data.shape
+            final_dataset.y_bins = data.qy_data[0::int(n_cols)]
+            final_dataset.x_bins = data.qx_data[:int(n_cols)]
     else:
         return_string = "Should Never Happen: _combine_data_info_with_plottable input is not a plottable1d or " + \

src/sas/sascalc/dataloader/file_reader_base_class.py

@@ -167 +167 @@
                     dataset.x_bins = dataset.qx_data[:int(n_cols)]
                 dataset.data = dataset.data.flatten()
+                if len(dataset.data) > 0:
+                    dataset.xmin = np.min(dataset.qx_data)
+                    dataset.xmax = np.max(dataset.qx_data)
+                    dataset.ymin = np.min(dataset.qy_data)
+                    dataset.ymax = np.max(dataset.qx_data)
 
     def format_unit(self, unit=None):
@@ -191 +196 @@
         self.output = []
 
-    def remove_empty_q_values(self, has_error_dx=False, has_error_dy=False,
-                              has_error_dxl=False, has_error_dxw=False):
+    def data_cleanup(self):
+        """
+        Clean up the data sets and refresh everything
+        :return: None
+        """
+        self.remove_empty_q_values()
+        self.send_to_output()  # Combine datasets with DataInfo
+        self.current_datainfo = DataInfo()  # Reset DataInfo
+
+    def remove_empty_q_values(self):
         """
         Remove any point where Q == 0
         """
-        x = self.current_dataset.x
-        self.current_dataset.x = self.current_dataset.x[x != 0]
-        self.current_dataset.y = self.current_dataset.y[x != 0]
-        if has_error_dy:
-            self.current_dataset.dy = self.current_dataset.dy[x != 0]
-        if has_error_dx:
-            self.current_dataset.dx = self.current_dataset.dx[x != 0]
-        if has_error_dxl:
-            self.current_dataset.dxl = self.current_dataset.dxl[x != 0]
-        if has_error_dxw:
-            self.current_dataset.dxw = self.current_dataset.dxw[x != 0]
+        if isinstance(self.current_dataset, plottable_1D):
+            # Booleans for resolutions
+            has_error_dx = self.current_dataset.dx is not None
+            has_error_dxl = self.current_dataset.dxl is not None
+            has_error_dxw = self.current_dataset.dxw is not None
+            has_error_dy = self.current_dataset.dy is not None
+            # Create arrays of zeros for non-existent resolutions
+            if has_error_dxw and not has_error_dxl:
+                array_size = self.current_dataset.dxw.size - 1
+                self.current_dataset.dxl = np.append(self.current_dataset.dxl,
+                                                    np.zeros([array_size]))
+                has_error_dxl = True
+            elif has_error_dxl and not has_error_dxw:
+                array_size = self.current_dataset.dxl.size - 1
+                self.current_dataset.dxw = np.append(self.current_dataset.dxw,
+                                                    np.zeros([array_size]))
+                has_error_dxw = True
+            elif not has_error_dxl and not has_error_dxw and not has_error_dx:
+                array_size = self.current_dataset.x.size - 1
+                self.current_dataset.dx = np.append(self.current_dataset.dx,
+                                                    np.zeros([array_size]))
+                has_error_dx = True
+            if not has_error_dy:
+                array_size = self.current_dataset.y.size - 1
+                self.current_dataset.dy = np.append(self.current_dataset.dy,
+                                                    np.zeros([array_size]))
+                has_error_dy = True
+
+            # Remove points where q = 0
+            x = self.current_dataset.x
+            self.current_dataset.x = self.current_dataset.x[x != 0]
+            self.current_dataset.y = self.current_dataset.y[x != 0]
+            if has_error_dy:
+                self.current_dataset.dy = self.current_dataset.dy[x != 0]
+            if has_error_dx:
+                self.current_dataset.dx = self.current_dataset.dx[x != 0]
+            if has_error_dxl:
+                self.current_dataset.dxl = self.current_dataset.dxl[x != 0]
+            if has_error_dxw:
+                self.current_dataset.dxw = self.current_dataset.dxw[x != 0]
+        elif isinstance(self.current_dataset, plottable_2D):
+            has_error_dqx = self.current_dataset.dqx_data is not None
+            has_error_dqy = self.current_dataset.dqy_data is not None
+            has_error_dy = self.current_dataset.err_data is not None
+            has_mask = self.current_dataset.mask is not None
+            x = self.current_dataset.qx_data
+            self.current_dataset.data = self.current_dataset.data[x != 0]
+            self.current_dataset.qx_data = self.current_dataset.qx_data[x != 0]
+            self.current_dataset.qy_data = self.current_dataset.qy_data[x != 0]
+            self.current_dataset.q_data = np.sqrt(
+                np.square(self.current_dataset.qx_data) + np.square(
+                    self.current_dataset.qy_data))
+            if has_error_dy:
+                self.current_dataset.err_data = self.current_dataset.err_data[x != 0]
+            if has_error_dqx:
+                self.current_dataset.dqx_data = self.current_dataset.dqx_data[x != 0]
+            if has_error_dqy:
+                self.current_dataset.dqy_data = self.current_dataset.dqy_data[x != 0]
+            if has_mask:
+                self.current_dataset.mask = self.current_dataset.mask[x != 0]
 
     def reset_data_list(self, no_lines=0):

src/sas/sascalc/dataloader/readers/abs_reader.py

@@ -109 +109 @@
                 # Sample thickness in mm
                 try:
-                    value = float(line_toks[5])
+                    value = float(line_toks[5][:-1])
                     if self.has_converter and \
                             self.current_datainfo.sample.thickness_unit != 'cm':
@@ -202 +202 @@
                 is_data_started = True
 
-        self.remove_empty_q_values(True, True)
+        self.remove_empty_q_values()
 
         # Sanity check

src/sas/sascalc/dataloader/readers/ascii_reader.py

@@ -156 +156 @@
             raise FileContentsException(msg)
 
-        self.remove_empty_q_values(has_error_dx, has_error_dy)
+        self.remove_empty_q_values()
         self.current_dataset.xaxis("\\rm{Q}", 'A^{-1}')
         self.current_dataset.yaxis("\\rm{Intensity}", "cm^{-1}")

src/sas/sascalc/dataloader/readers/cansas_reader.py

@@ -104 +104 @@
             xml_file = self.f_open.name
         # We don't sure f_open since lxml handles opnening/closing files
-        if not self.f_open.closed:
-            self.f_open.close()
-
-        basename, _ = os.path.splitext(os.path.basename(xml_file))
-
         try:
             # Raises FileContentsException
             self.load_file_and_schema(xml_file, schema_path)
-            self.current_datainfo = DataInfo()
-            # Raises FileContentsException if file doesn't meet CanSAS schema
+            # Parse each SASentry
+            entry_list = self.xmlroot.xpath('/ns:SASroot/ns:SASentry',
+                                            namespaces={
+                                                'ns': self.cansas_defaults.get(
+                                                    "ns")
+                                            })
             self.is_cansas(self.extension)
-            self.invalid = False # If we reach this point then file must be valid CanSAS
-
-            # Parse each SASentry
-            entry_list = self.xmlroot.xpath('/ns:SASroot/ns:SASentry', namespaces={
-                'ns': self.cansas_defaults.get("ns")
-            })
-            # Look for a SASentry
-            self.names.append("SASentry")
             self.set_processing_instructions()
-
             for entry in entry_list:
-                self.current_datainfo.filename = basename + self.extension
-                self.current_datainfo.meta_data["loader"] = "CanSAS XML 1D"
-                self.current_datainfo.meta_data[PREPROCESS] = self.processing_instructions
                 self._parse_entry(entry)
                 self.data_cleanup()
@@ -150 +137 @@
                     invalid_xml = self.find_invalid_xml()
                     if invalid_xml != "":
+                        basename, _ = os.path.splitext(
+                            os.path.basename(self.f_open.name))
                         invalid_xml = INVALID_XML.format(basename + self.extension) + invalid_xml
                         raise DataReaderException(invalid_xml) # Handled by base class
@@ -164 +153 @@
         except Exception as e: # Convert all other exceptions to FileContentsExceptions
             raise FileContentsException(e.message)
-
+        finally:
+            if not self.f_open.closed:
+                self.f_open.close()
 
     def load_file_and_schema(self, xml_file, schema_path=""):
@@ -209 +200 @@
         if not self._is_call_local() and not recurse:
             self.reset_state()
+        if not recurse:
+            self.current_datainfo = DataInfo()
+            # Raises FileContentsException if file doesn't meet CanSAS schema
+            self.invalid = False
+            # Look for a SASentry
             self.data = []
-            self.current_datainfo = DataInfo()
+            self.parent_class = "SASentry"
             self.names.append("SASentry")
-            self.parent_class = "SASentry"
+            self.current_datainfo.meta_data["loader"] = "CanSAS XML 1D"
+            self.current_datainfo.meta_data[
+                PREPROCESS] = self.processing_instructions
+        if self._is_call_local() and not recurse:
+            basename, _ = os.path.splitext(os.path.basename(self.f_open.name))
+            self.current_datainfo.filename = basename + self.extension
         # Create an empty dataset if no data has been passed to the reader
         if self.current_dataset is None:
-            self.current_dataset = plottable_1D(np.empty(0), np.empty(0),
-                np.empty(0), np.empty(0))
+            self._initialize_new_data_set(dom)
         self.base_ns = "{" + CANSAS_NS.get(self.cansas_version).get("ns") + "}"
 
@@ -228 +228 @@
             tagname_original = tagname
             # Skip this iteration when loading in save state information
-            if tagname == "fitting_plug_in" or tagname == "pr_inversion" or tagname == "invariant":
+            if tagname in ["fitting_plug_in", "pr_inversion", "invariant", "corfunc"]:
                 continue
             # Get where to store content
@@ -258 +258 @@
                 self._add_intermediate()
             else:
+                # TODO: Clean this up to make it faster (fewer if/elifs)
                 if isinstance(self.current_dataset, plottable_2D):
                     data_point = node.text
@@ -502 +503 @@
             self.sort_two_d_data()
             self.reset_data_list()
-            empty = None
-            return self.output[0], empty
-
-    def data_cleanup(self):
-        """
-        Clean up the data sets and refresh everything
-        :return: None
-        """
-        has_error_dx = self.current_dataset.dx is not None
-        has_error_dxl = self.current_dataset.dxl is not None
-        has_error_dxw = self.current_dataset.dxw is not None
-        has_error_dy = self.current_dataset.dy is not None
-        self.remove_empty_q_values(has_error_dx=has_error_dx,
-                                   has_error_dxl=has_error_dxl,
-                                   has_error_dxw=has_error_dxw,
-                                   has_error_dy=has_error_dy)
-        self.send_to_output()  # Combine datasets with DataInfo
-        self.current_datainfo = DataInfo()  # Reset DataInfo
+            return self.output[0], None
 
     def _is_call_local(self):
@@ -553 +537 @@
             self.aperture = Aperture()
         elif self.parent_class == 'SASdata':
-            self._check_for_empty_resolution()
             self.data.append(self.current_dataset)
 
@@ -609 +592 @@
         if 'unit' in attr and attr.get('unit') is not None:
             try:
-                local_unit = attr['unit']
+                unit = attr['unit']
+                unit_list = unit.split("|")
+                if len(unit_list) > 1:
+                    self.current_dataset.xaxis(unit_list[0].strip(),
+                                               unit_list[1].strip())
+                    local_unit = unit_list[1]
+                else:
+                    local_unit = unit
                 unitname = self.ns_list.current_level.get("unit", "")
                 if "SASdetector" in self.names:
@@ -665 +655 @@
         return node_value, value_unit
 
-    def _check_for_empty_resolution(self):
-        """
-        a method to check all resolution data sets are the same size as I and q
-        """
-        dql_exists = False
-        dqw_exists = False
-        dq_exists = False
-        di_exists = False
-        if self.current_dataset.dxl is not None:
-            dql_exists = True
-        if self.current_dataset.dxw is not None:
-            dqw_exists = True
-        if self.current_dataset.dx is not None:
-            dq_exists = True
-        if self.current_dataset.dy is not None:
-            di_exists = True
-        if dqw_exists and not dql_exists:
-            array_size = self.current_dataset.dxw.size
-            self.current_dataset.dxl = np.zeros(array_size)
-        elif dql_exists and not dqw_exists:
-            array_size = self.current_dataset.dxl.size
-            self.current_dataset.dxw = np.zeros(array_size)
-        elif not dql_exists and not dqw_exists and not dq_exists:
-            array_size = self.current_dataset.x.size
-            self.current_dataset.dx = np.append(self.current_dataset.dx,
-                                                np.zeros([array_size]))
-        if not di_exists:
-            array_size = self.current_dataset.y.size
-            self.current_dataset.dy = np.append(self.current_dataset.dy,
-                                                np.zeros([array_size]))
-
     def _initialize_new_data_set(self, node=None):
         if node is not None:

src/sas/sascalc/dataloader/readers/__init__.py

@@ -1 +1 @@
 # Method to associate extensions to default readers
 from associations import read_associations
-
-
-# Method to return the location of the XML settings file
-def get_data_path():
-    """
-        Return the location of the settings file for the data readers.
-    """
-    import os
-    return os.path.dirname(__file__)