Changeset a40e913 in sasview for src/sas/sascalc/dataloader

Timestamp:

Jan 11, 2018 8:29:18 AM (7 years ago)

Author:

GitHub <noreply@…>

Branches:

master, magnetic_scatt, release-4.2.2, ticket-1009, ticket-1094-headless, ticket-1242-2d-resolution, ticket-1243, ticket-1249, ticket885, unittest-saveload

Children:

8cc9048, 6eb02a5

Parents:

2bdf61a (diff), a58b5a0 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

git-author:

Paul Butler <butlerpd@…> (01/11/18 08:29:18)

git-committer:

GitHub <noreply@…> (01/11/18 08:29:18)

Message:

Merge pull request #135 from SasView?/ticket-1037

Ticket 1037: Remove nan values from 1D and 2D data objects

closes #1037

Location:

src/sas/sascalc/dataloader

Files:

• file_reader_base_class.py (modified) (6 diffs)
• readers/abs_reader.py (modified) (4 diffs)
• readers/associations.py (modified) (1 diff)

src/sas/sascalc/dataloader/file_reader_base_class.py

@@ -7 +7 @@
 import os
 import sys
-import re
+import math
 import logging
 from abc import abstractmethod
@@ -25 +25 @@
     def decode(s):
         return s.decode() if isinstance(s, bytes) else s
+
+# Data 1D fields for iterative purposes
+FIELDS_1D = ('x', 'y', 'dx', 'dy', 'dxl', 'dxw')
+# Data 2D fields for iterative purposes
+FIELDS_2D = ('data', 'qx_data', 'qy_data', 'q_data', 'err_data',
+                 'dqx_data', 'dqy_data', 'mask')
+
 
 class FileReader(object):
@@ -102 +109 @@
         self.current_dataset = None
         self.filepath = None
+        self.ind = None
         self.output = []
 
@@ -159 +167 @@
                 # Sort data by increasing x and remove 1st point
                 ind = np.lexsort((data.y, data.x))
-                data.x = np.asarray([data.x[i] for i in ind]).astype(np.float64)
-                data.y = np.asarray([data.y[i] for i in ind]).astype(np.float64)
+                data.x = self._reorder_1d_array(data.x, ind)
+                data.y = self._reorder_1d_array(data.y, ind)
                 if data.dx is not None:
                     if len(data.dx) == 0:
                         data.dx = None
                         continue
-                    data.dx = np.asarray([data.dx[i] for i in ind]).astype(np.float64)
+                    data.dx = self._reorder_1d_array(data.dx, ind)
                 if data.dxl is not None:
-                    data.dxl = np.asarray([data.dxl[i] for i in ind]).astype(np.float64)
+                    data.dxl = self._reorder_1d_array(data.dxl, ind)
                 if data.dxw is not None:
-                    data.dxw = np.asarray([data.dxw[i] for i in ind]).astype(np.float64)
+                    data.dxw = self._reorder_1d_array(data.dxw, ind)
                 if data.dy is not None:
                     if len(data.dy) == 0:
                         data.dy = None
                         continue
-                    data.dy = np.asarray([data.dy[i] for i in ind]).astype(np.float64)
+                    data.dy = self._reorder_1d_array(data.dy, ind)
                 if data.lam is not None:
-                    data.lam = np.asarray([data.lam[i] for i in ind]).astype(np.float64)
+                    data.lam = self._reorder_1d_array(data.lam, ind)
                 if data.dlam is not None:
-                    data.dlam = np.asarray([data.dlam[i] for i in ind]).astype(np.float64)
+                    data.dlam = self._reorder_1d_array(data.dlam, ind)
+                data = self._remove_nans_in_data(data)
                 if len(data.x) > 0:
                     data.xmin = np.min(data.x)
@@ -184 +193 @@
                     data.ymin = np.min(data.y)
                     data.ymax = np.max(data.y)
+
+    @staticmethod
+    def _reorder_1d_array(array, ind):
+        """
+        Reorders a 1D array based on the indices passed as ind
+        :param array: Array to be reordered
+        :param ind: Indices used to reorder array
+        :return: reordered array
+        """
+        array = np.asarray(array, dtype=np.float64)
+        return array[ind]
+
+    @staticmethod
+    def _remove_nans_in_data(data):
+        """
+        Remove data points where nan is loaded
+        :param data: 1D or 2D data object
+        :return: data with nan points removed
+        """
+        if isinstance(data, Data1D):
+            fields = FIELDS_1D
+        elif isinstance(data, Data2D):
+            fields = FIELDS_2D
+        else:
+            return data
+        # Make array of good points - all others will be removed
+        good = np.isfinite(getattr(data, fields[0]))
+        for name in fields[1:]:
+            array = getattr(data, name)
+            if array is not None:
+                # Update good points only if not already changed
+                good &= np.isfinite(array)
+        if not np.all(good):
+            for name in fields:
+                array = getattr(data, name)
+                if array is not None:
+                    setattr(data, name, array[good])
+        return data
 
     def sort_two_d_data(self):
@@ -214 +261 @@
                     dataset.x_bins = dataset.qx_data[:int(n_cols)]
                 dataset.data = dataset.data.flatten()
+                dataset = self._remove_nans_in_data(dataset)
                 if len(dataset.data) > 0:
                     dataset.xmin = np.min(dataset.qx_data)

src/sas/sascalc/dataloader/readers/abs_reader.py

@@ -29 +29 @@
     type_name = "IGOR 1D"
     # Wildcards
-    type = ["IGOR 1D files (*.abs)|*.abs"]
+    type = ["IGOR 1D files (*.abs)|*.abs", "IGOR 1D USANS files (*.cor)|*.cor"]
     # List of allowed extensions
-    ext = ['.abs']
+    ext = ['.abs', '.cor']
 
     def get_file_contents(self):
@@ -46 +46 @@
         self.current_datainfo = DataInfo()
         self.current_datainfo.filename = filepath
-        self.reset_data_list(len(lines))
         detector = Detector()
         data_line = 0
@@ -188 +187 @@
                     self.current_dataset.y[data_line] = _y
                     self.current_dataset.dy[data_line] = _dy
-                    self.current_dataset.dx[data_line] = _dx
+                    if _dx > 0:
+                        self.current_dataset.dx[data_line] = _dx
+                    else:
+                        if data_line == 0:
+                            self.current_dataset.dx = None
+                            self.current_dataset.dxl = np.zeros(len(lines))
+                            self.current_dataset.dxw = np.zeros(len(lines))
+                        self.current_dataset.dxl[data_line] = abs(_dx)
+                        self.current_dataset.dxw[data_line] = 0
                     data_line += 1
 
@@ -197 +204 @@
                     pass
 
+            # SANS Data:
             # The 6 columns are | Q (1/A) | I(Q) (1/cm) | std. dev.
             # I(Q) (1/cm) | sigmaQ | meanQ | ShadowFactor|
-            if line.count("The 6 columns") > 0:
+            # USANS Data:
+            # EMP LEVEL: <value> ; BKG LEVEL: <value>
+            if line.startswith("The 6 columns") or line.startswith("EMP LEVEL"):
                 is_data_started = True
 

src/sas/sascalc/dataloader/readers/associations.py

@@ -26 +26 @@
     ".dat": "red2d_reader",
     ".abs": "abs_reader",
+    ".cor": "abs_reader",
     ".sans": "danse_reader",
     ".pdh": "anton_paar_saxs_reader"