Changeset 0011ecc in sasmodels

Timestamp:

Mar 8, 2017 9:38:54 AM (8 years ago)

Author:

GitHub <noreply@…>

Branches:

master, core_shell_microgels, costrafo411, magnetic_model, ticket-1257-vesicle-product, ticket_1156, ticket_1265_superball, ticket_822_more_unit_tests

Children:

5e2fd41, 3a45c2c

Parents:

9ae85f0 (diff), 598857b (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

git-author:

Jeff Krzywon <krzywon@…> (03/08/17 09:38:54)

git-committer:

GitHub <noreply@…> (03/08/17 09:38:54)

Message:

Merge pull request #35 from SasView?/ticket-814

Finds models that throw errors when running the test routine. Fixes #814

Location:

sasmodels

Files:

• compare.py (modified) (1 diff)
• compare_many.py (modified) (8 diffs)
• core.py (modified) (1 diff)
• model_test.py (modified) (8 diffs)
• modelinfo.py (modified) (2 diffs)

sasmodels/compare.py

@@ -321 +321 @@
     if '*' in name:
         name = name.split('*')[0]
+
+    # Suppress magnetism for python models (not yet implemented)
+    if callable(model_info.Iq):
+        pars.update(suppress_magnetism(pars))
 
     if name == 'barbell':

sasmodels/compare_many.py

@@ -106 +106 @@
     header = ('\n"Model","%s","Count","%d","Dimension","%s"'
               % (name, N, "2D" if is_2d else "1D"))
-    if not mono: header += ',"Cutoff",%g'%(cutoff,)
+    if not mono:
+        header += ',"Cutoff",%g'%(cutoff,)
     print(header)
 
@@ -161 +162 @@
     max_diff = [0]
     for k in range(N):
-        print("%s %d"%(name, k), file=sys.stderr)
+        print("Model %s %d"%(name, k+1), file=sys.stderr)
         seed = np.random.randint(1e6)
         pars_i = randomize_pars(model_info, pars, seed)
         constrain_pars(model_info, pars_i)
-        constrain_new_to_old(model_info, pars_i)
+        if 'sasview' in (base, comp):
+            constrain_new_to_old(model_info, pars_i)
         if mono:
             pars_i = suppress_pd(pars_i)
@@ -187 +189 @@
     Print the command usage string.
     """
-    print("usage: compare_many.py MODEL COUNT (1dNQ|2dNQ) (CUTOFF|mono) (single|double|quad)")
+    print("usage: compare_many.py MODEL COUNT (1dNQ|2dNQ) (CUTOFF|mono) (single|double|quad)",
+          file=sys.stderr)
 
 
@@ -204 +207 @@
     print("""\
 
-MODEL is the model name of the model or "all" for all the models
-in alphabetical order.
+MODEL is the model name of the model or one of the model types listed in
+sasmodels.core.list_models (all, py, c, double, single, opencl, 1d, 2d,
+nonmagnetic, magnetic).  Model types can be combined, such as 2d+single.
 
 COUNT is the number of randomly generated parameter sets to try. A value
@@ -220 +224 @@
 below the cutoff will be ignored.  Use "mono" for monodisperse models.  The
 choice of polydisperse parameters, and the number of points in the distribution
-is set in compare.py defaults for each model.
+is set in compare.py defaults for each model.  Polydispersity is given in the
+"demo" attribute of each model.
 
 PRECISION is the floating point precision to use for comparisons.  If two
-precisions are given, then compare one to the other, ignoring sasview.
+precisions are given, then compare one to the other.  Precision is one of
+fast, single, double for GPU or single!, double!, quad! for DLL.  If no
+precision is given, then use single and double! respectively.
 
 Available models:
@@ -233 +240 @@
     Main program.
     """
-    if len(argv) not in (5, 6):
+    if len(argv) not in (3, 4, 5, 6):
         print_help()
         return
 
-    model = argv[0]
-    if not (model in MODELS) and (model != "all"):
-        print('Bad model %s.  Use "all" or one of:'%model)
+    target = argv[0]
+    try:
+        model_list = [target] if target in MODELS else core.list_models(target)
+    except ValueError:
+        print('Bad model %s.  Use model type or one of:' % target, file=sys.stderr)
         print_models()
+        print('model types: all, py, c, double, single, opencl, 1d, 2d, nonmagnetic, magnetic')
         return
     try:
@@ -247 +257 @@
         assert argv[2][1] == 'd'
         Nq = int(argv[2][2:])
-        mono = argv[3] == 'mono'
+        mono = len(argv) <= 3 or argv[3] == 'mono'
         cutoff = float(argv[3]) if not mono else 0
-        base = argv[4]
-        comp = argv[5] if len(argv) > 5 else "sasview"
+        base = argv[4] if len(argv) > 4 else "single"
+        comp = argv[5] if len(argv) > 5 else "double!"
     except Exception:
         traceback.print_exc()
@@ -258 +268 @@
     data, index = make_data({'qmax':1.0, 'is2d':is2D, 'nq':Nq, 'res':0.,
                              'accuracy': 'Low', 'view':'log', 'zero': False})
-    model_list = [model] if model != "all" else MODELS
     for model in model_list:
         compare_instance(model, data, index, N=count, mono=mono,

sasmodels/core.py

@@ -69 +69 @@
         * magnetic: models with an sld
         * nommagnetic: models without an sld
-    """
-    if kind and kind not in KINDS:
+
+    For multiple conditions, combine with plus.  For example, *c+single+2d*
+    would return all oriented models implemented in C which can be computed
+    accurately with single precision arithmetic.
+    """
+    if kind and any(k not in KINDS for k in kind.split('+')):
         raise ValueError("kind not in " + ", ".join(KINDS))
     files = sorted(glob(joinpath(generate.MODEL_PATH, "[a-zA-Z]*.py")))
     available_models = [basename(f)[:-3] for f in files]
-    selected = [name for name in available_models if _matches(name, kind)]
+    if kind and '+' in kind:
+        all_kinds = kind.split('+')
+        condition = lambda name: all(_matches(name, k) for k in all_kinds)
+    else:
+        condition = lambda name: _matches(name, kind)
+    selected = [name for name in available_models if condition(name)]
 
     return selected

sasmodels/model_test.py

@@ -97 +97 @@
         is_py = callable(model_info.Iq)
 
+        # Some OpenCL drivers seem to be flaky, and are not producing the
+        # expected result.  Since we don't have known test values yet for
+        # all of our models, we are instead going to compare the results
+        # for the 'smoke test' (that is, evaluation at q=0.1 for the default
+        # parameters just to see that the model runs to completion) between
+        # the OpenCL and the DLL.  To do this, we define a 'stash' which is
+        # shared between OpenCL and DLL tests.  This is just a list.  If the
+        # list is empty (which it will be when DLL runs, if the DLL runs
+        # first), then the results are appended to the list.  If the list
+        # is not empty (which it will be when OpenCL runs second), the results
+        # are compared to the results stored in the first element of the list.
+        # This is a horrible stateful hack which only makes sense because the
+        # test suite is thrown away after being run once.
+        stash = []
+
         if is_py:  # kernel implemented in python
             test_name = "Model: %s, Kernel: python"%model_name
@@ -103 +118 @@
                                  test_method_name,
                                  platform="dll",  # so that
-                                 dtype="double")
+                                 dtype="double",
+                                 stash=stash)
             suite.addTest(test)
         else:   # kernel implemented in C
+
+            # test using dll if desired
+            if 'dll' in loaders or not core.HAVE_OPENCL:
+                test_name = "Model: %s, Kernel: dll"%model_name
+                test_method_name = "test_%s_dll" % model_info.id
+                test = ModelTestCase(test_name, model_info,
+                                     test_method_name,
+                                     platform="dll",
+                                     dtype="double",
+                                     stash=stash)
+                suite.addTest(test)
+
             # test using opencl if desired and available
             if 'opencl' in loaders and core.HAVE_OPENCL:
@@ -116 +144 @@
                 test = ModelTestCase(test_name, model_info,
                                      test_method_name,
-                                     platform="ocl", dtype=None)
+                                     platform="ocl", dtype=None,
+                                     stash=stash)
                 #print("defining", test_name)
-                suite.addTest(test)
-
-            # test using dll if desired
-            if 'dll' in loaders or not core.HAVE_OPENCL:
-                test_name = "Model: %s, Kernel: dll"%model_name
-                test_method_name = "test_%s_dll" % model_info.id
-                test = ModelTestCase(test_name, model_info,
-                                     test_method_name,
-                                     platform="dll",
-                                     dtype="double")
                 suite.addTest(test)
 
@@ -144 +163 @@
         """
         def __init__(self, test_name, model_info, test_method_name,
-                     platform, dtype):
-            # type: (str, ModelInfo, str, str, DType) -> None
+                     platform, dtype, stash):
+            # type: (str, ModelInfo, str, str, DType, List[Any]) -> None
             self.test_name = test_name
             self.info = model_info
             self.platform = platform
             self.dtype = dtype
+            self.stash = stash  # container for the results of the first run
 
             setattr(self, test_method_name, self.run_all)
@@ -167 +187 @@
                 #({}, (0.0, 0.0), None),
                 # test vector form
-                ({}, [0.1]*2, [None]*2),
+                ({}, [0.001, 0.01, 0.1], [None]*3),
                 ({}, [(0.1, 0.1)]*2, [None]*2),
                 # test that ER/VR will run if they exist
@@ -174 +194 @@
                 ]
 
-            tests = self.info.tests
+            tests = smoke_tests + self.info.tests
             try:
                 model = build_model(self.info, dtype=self.dtype,
                                     platform=self.platform)
-                for test in smoke_tests + tests:
-                    self.run_one(model, test)
-
-                if not tests and self.platform == "dll":
-                    ## Uncomment the following to make forgetting the test
-                    ## values an error.  Only do so for the "dll" tests
-                    ## to reduce noise from both opencl and dll, and because
-                    ## python kernels use platform="dll".
-                    #raise Exception("No test cases provided")
-                    pass
+                results = [self.run_one(model, test) for test in tests]
+                if self.stash:
+                    for test, target, actual in zip(tests, self.stash[0], results):
+                        assert np.all(abs(target-actual) < 5e-5*abs(actual)),\
+                            "GPU/CPU comparison expected %s but got %s for %s"%(target, actual, test[0])
+                else:
+                    self.stash.append(results)
+
+                # Check for missing tests.  Only do so for the "dll" tests
+                # to reduce noise from both opencl and dll, and because
+                # python kernels use platform="dll".
+                if self.platform == "dll":
+                    missing = []
+                    ## Uncomment the following to require test cases
+                    #missing = self._find_missing_tests()
+                    if missing:
+                        raise ValueError("Missing tests for "+", ".join(missing))
 
             except:
                 annotate_exception(self.test_name)
                 raise
+
+        def _find_missing_tests(self):
+            # type: () -> None
+            """make sure there are 1D, 2D, ER and VR tests as appropriate"""
+            model_has_VR = callable(self.info.VR)
+            model_has_ER = callable(self.info.ER)
+            model_has_1D = True
+            model_has_2D = any(p.type == 'orientation'
+                               for p in self.info.parameters.kernel_parameters)
+
+            # Lists of tests that have a result that is not None
+            single = [test for test in self.info.tests
+                      if not isinstance(test[2], list) and test[2] is not None]
+            tests_has_VR = any(test[1] == 'VR' for test in single)
+            tests_has_ER = any(test[1] == 'ER' for test in single)
+            tests_has_1D_single = any(isinstance(test[1], float) for test in single)
+            tests_has_2D_single = any(isinstance(test[1], tuple) for test in single)
+
+            multiple = [test for test in self.info.tests
+                        if isinstance(test[2], list)
+                            and not all(result is None for result in test[2])]
+            tests_has_1D_multiple = any(isinstance(test[1][0], float)
+                                        for test in multiple)
+            tests_has_2D_multiple = any(isinstance(test[1][0], tuple)
+                                        for test in multiple)
+
+            missing = []
+            if model_has_VR and not tests_has_VR:
+                missing.append("VR")
+            if model_has_ER and not tests_has_ER:
+                missing.append("ER")
+            if model_has_1D and not (tests_has_1D_single or tests_has_1D_multiple):
+                missing.append("1D")
+            if model_has_2D and not (tests_has_2D_single or tests_has_2D_multiple):
+                missing.append("2D")
+
+            return missing
 
         def run_one(self, model, test):
@@ -207 +271 @@
 
             if x[0] == 'ER':
-                actual = [call_ER(model.info, pars)]
+                actual = np.array([call_ER(model.info, pars)])
             elif x[0] == 'VR':
-                actual = [call_VR(model.info, pars)]
+                actual = np.array([call_VR(model.info, pars)])
             elif isinstance(x[0], tuple):
                 qx, qy = zip(*x)
@@ -238 +302 @@
                                     'f(%s); expected:%s; actual:%s'
                                     % (xi, yi, actual_yi))
+            return actual
 
     return ModelTestCase

sasmodels/modelinfo.py

@@ -730 +730 @@
     info.docs = kernel_module.__doc__
     info.category = getattr(kernel_module, 'category', None)
-    info.single = getattr(kernel_module, 'single', True)
-    info.opencl = getattr(kernel_module, 'opencl', True)
     info.structure_factor = getattr(kernel_module, 'structure_factor', False)
     info.profile_axes = getattr(kernel_module, 'profile_axes', ['x', 'y'])
@@ -745 +743 @@
     info.profile = getattr(kernel_module, 'profile', None) # type: ignore
     info.sesans = getattr(kernel_module, 'sesans', None) # type: ignore
+    # Default single and opencl to True for C models.  Python models have callable Iq.
+    info.opencl = getattr(kernel_module, 'opencl', not callable(info.Iq))
+    info.single = getattr(kernel_module, 'single', not callable(info.Iq))
 
     # multiplicity info