Changes in sasmodels/model_test.py [1a6cd57:e09d1e0] in sasmodels

File:

• sasmodels/model_test.py (modified) (7 diffs)

sasmodels/model_test.py

@@ -97 +97 @@
         is_py = callable(model_info.Iq)
 
+        # Some OpenCL drivers seem to be flaky, and are not producing the
+        # expected result.  Since we don't have known test values yet for
+        # all of our models, we are instead going to compare the results
+        # for the 'smoke test' (that is, evaluation at q=0.1 for the default
+        # parameters just to see that the model runs to completion) between
+        # the OpenCL and the DLL.  To do this, we define a 'stash' which is
+        # shared between OpenCL and DLL tests.  This is just a list.  If the
+        # list is empty (which it will be when DLL runs, if the DLL runs
+        # first), then the results are appended to the list.  If the list
+        # is not empty (which it will be when OpenCL runs second), the results
+        # are compared to the results stored in the first element of the list.
+        # This is a horrible stateful hack which only makes sense because the
+        # test suite is thrown away after being run once.
+        stash = []
+
         if is_py:  # kernel implemented in python
             test_name = "Model: %s, Kernel: python"%model_name
@@ -103 +118 @@
                                  test_method_name,
                                  platform="dll",  # so that
-                                 dtype="double")
+                                 dtype="double",
+                                 stash=stash)
             suite.addTest(test)
         else:   # kernel implemented in C
+
+            # test using dll if desired
+            if 'dll' in loaders or not core.HAVE_OPENCL:
+                test_name = "Model: %s, Kernel: dll"%model_name
+                test_method_name = "test_%s_dll" % model_info.id
+                test = ModelTestCase(test_name, model_info,
+                                     test_method_name,
+                                     platform="dll",
+                                     dtype="double",
+                                     stash=stash)
+                suite.addTest(test)
+
             # test using opencl if desired and available
             if 'opencl' in loaders and core.HAVE_OPENCL:
@@ -116 +144 @@
                 test = ModelTestCase(test_name, model_info,
                                      test_method_name,
-                                     platform="ocl", dtype=None)
+                                     platform="ocl", dtype=None,
+                                     stash=stash)
                 #print("defining", test_name)
-                suite.addTest(test)
-
-            # test using dll if desired
-            if 'dll' in loaders or not core.HAVE_OPENCL:
-                test_name = "Model: %s, Kernel: dll"%model_name
-                test_method_name = "test_%s_dll" % model_info.id
-                test = ModelTestCase(test_name, model_info,
-                                     test_method_name,
-                                     platform="dll",
-                                     dtype="double")
                 suite.addTest(test)
 
@@ -144 +163 @@
         """
         def __init__(self, test_name, model_info, test_method_name,
-                     platform, dtype):
-            # type: (str, ModelInfo, str, str, DType) -> None
+                     platform, dtype, stash):
+            # type: (str, ModelInfo, str, str, DType, List[Any]) -> None
             self.test_name = test_name
             self.info = model_info
             self.platform = platform
             self.dtype = dtype
+            self.stash = stash  # container for the results of the first run
 
             setattr(self, test_method_name, self.run_all)
@@ -174 +194 @@
                 ]
 
-            tests = self.info.tests
+            tests = smoke_tests + self.info.tests
             try:
                 model = build_model(self.info, dtype=self.dtype,
                                     platform=self.platform)
-                for test in smoke_tests + tests:
-                    self.run_one(model, test)
-
-                if not tests and self.platform == "dll":
-                    ## Uncomment the following to make forgetting the test
-                    ## values an error.  Only do so for the "dll" tests
-                    ## to reduce noise from both opencl and dll, and because
-                    ## python kernels use platform="dll".
-                    #raise Exception("No test cases provided")
-                    pass
+                results = [self.run_one(model, test) for test in tests]
+                if self.stash:
+                    for test, target, actual in zip(tests, self.stash[0], results):
+                        assert np.all(abs(target-actual)<2e-5*abs(actual)),\
+                            "expected %s but got %s for %s"%(target, actual, test[0])
+                else:
+                    self.stash.append(results)
+
+                # Check for missing tests.  Only do so for the "dll" tests
+                # to reduce noise from both opencl and dll, and because
+                # python kernels use platform="dll".
+                if self.platform == "dll":
+                    missing = []
+                    ## Uncomment the following to require test cases
+                    #missing = self._find_missing_tests()
+                    if missing:
+                        raise ValueError("Missing tests for "+", ".join(missing))
 
             except:
                 annotate_exception(self.test_name)
                 raise
+
+        def _find_missing_tests(self):
+            # type: () -> None
+            """make sure there are 1D, 2D, ER and VR tests as appropriate"""
+            model_has_VR = callable(self.info.VR)
+            model_has_ER = callable(self.info.ER)
+            model_has_1D = True
+            model_has_2D = any(p.type == 'orientation'
+                               for p in self.info.parameters.kernel_parameters)
+
+            # Lists of tests that have a result that is not None
+            single = [test for test in self.info.tests
+                      if not isinstance(test[2], list) and test[2] is not None]
+            tests_has_VR = any(test[1] == 'VR' for test in single)
+            tests_has_ER = any(test[1] == 'ER' for test in single)
+            tests_has_1D_single = any(isinstance(test[1], float) for test in single)
+            tests_has_2D_single = any(isinstance(test[1], tuple) for test in single)
+
+            multiple = [test for test in self.info.tests
+                        if isinstance(test[2], list)
+                            and not all(result is None for result in test[2])]
+            tests_has_1D_multiple = any(isinstance(test[1][0], float)
+                                        for test in multiple)
+            tests_has_2D_multiple = any(isinstance(test[1][0], tuple)
+                                        for test in multiple)
+
+            missing = []
+            if model_has_VR and not tests_has_VR:
+                missing.append("VR")
+            if model_has_ER and not tests_has_ER:
+                missing.append("ER")
+            if model_has_1D and not (tests_has_1D_single or tests_has_1D_multiple):
+                missing.append("1D")
+            if model_has_2D and not (tests_has_2D_single or tests_has_2D_multiple):
+                missing.append("2D")
+
+            return missing
 
         def run_one(self, model, test):
@@ -207 +271 @@
 
             if x[0] == 'ER':
-                actual = [call_ER(model.info, pars)]
+                actual = np.array([call_ER(model.info, pars)])
             elif x[0] == 'VR':
-                actual = [call_VR(model.info, pars)]
+                actual = np.array([call_VR(model.info, pars)])
             elif isinstance(x[0], tuple):
                 qx, qy = zip(*x)
@@ -238 +302 @@
                                     'f(%s); expected:%s; actual:%s'
                                     % (xi, yi, actual_yi))
+            return actual
 
     return ModelTestCase