source: sasmodels/sasmodels/model_test.py @ a08c47f

core_shell_microgelsmagnetic_modelticket-1257-vesicle-productticket_1156ticket_1265_superballticket_822_more_unit_tests
Last change on this file since a08c47f was a08c47f, checked in by Adam Washington <adam.washington@…>, 5 months ago

Merge remote-tracking branch 'upstream/beta_approx' into test_args

  • Property mode set to 100755
File size: 22.3 KB
Line 
1# -*- coding: utf-8 -*-
2"""
3Run model unit tests.
4
5Usage::
6
7    python -m sasmodels.model_test [opencl|cuda|dll] model1 model2 ...
8
9    if model1 is 'all', then all except the remaining models will be tested
10
11Each model is tested using the default parameters at q=0.1, (qx, qy)=(0.1, 0.1),
12and Fq is called to make sure R_eff, volume and volume ratio are computed.
13The return values at these points are not considered.  The test is only to
14verify that the models run to completion, and do not produce inf or NaN.
15
16Tests are defined with the *tests* attribute in the model.py file.  *tests*
17is a list of individual tests to run, where each test consists of the
18parameter values for the test, the q-values and the expected results.  For
19the effective radius test and volume ratio tests, use the extended output
20form, which checks each output of kernel.Fq. For 1-D tests, either specify
21the q value or a list of q-values, and the corresponding I(q) value, or
22list of I(q) values.
23
24That is::
25
26    tests = [
27        [ {parameters}, q, I(q)],
28        [ {parameters}, [q], [I(q)] ],
29        [ {parameters}, [q1, q2, ...], [I(q1), I(q2), ...]],
30
31        [ {parameters}, (qx, qy), I(qx, Iqy)],
32        [ {parameters}, [(qx1, qy1), (qx2, qy2), ...],
33                        [I(qx1, qy1), I(qx2, qy2), ...]],
34
35        [ {parameters}, q, F(q), F^2(q), R_eff, V, V_r ],
36        ...
37    ]
38
39Parameters are *key:value* pairs, where key is one of the parameters of the
40model and value is the value to use for the test.  Any parameters not given
41in the parameter list will take on the default parameter value.
42
43Precision defaults to 5 digits (relative).
44"""
45from __future__ import print_function
46
47import argparse
48import sys
49import unittest
50import traceback
51
52try:
53    from StringIO import StringIO
54except ImportError:
55    # StringIO.StringIO renamed to io.StringIO in Python 3
56    # Note: io.StringIO exists in python 2, but using unicode instead of str
57    from io import StringIO
58
59import numpy as np  # type: ignore
60
61from . import core
62from .core import list_models, load_model_info, build_model
63from .direct_model import call_kernel, call_Fq
64from .exception import annotate_exception
65from .modelinfo import expand_pars
66from .kernelcl import use_opencl
67from .kernelcuda import use_cuda
68from . import product
69
70# pylint: disable=unused-import
71try:
72    from typing import List, Iterator, Callable
73except ImportError:
74    pass
75else:
76    from .modelinfo import ParameterTable, ParameterSet, TestCondition, ModelInfo
77    from .kernel import KernelModel
78# pylint: enable=unused-import
79
80def make_suite(loaders, models):
81    # type: (List[str], List[str]) -> unittest.TestSuite
82    """
83    Construct the pyunit test suite.
84
85    *loaders* is the list of kernel drivers to use (dll, opencl or cuda).
86    For python model the python driver is always used.
87
88    *models* is the list of models to test, or *["all"]* to test all models.
89    """
90    suite = unittest.TestSuite()
91
92    if models[0] in core.KINDS:
93        skip = models[1:]
94        models = list_models(models[0])
95    else:
96        skip = []
97    for model_name in models:
98        if model_name not in skip:
99            model_info = load_model_info(model_name)
100            _add_model_to_suite(loaders, suite, model_info)
101
102    return suite
103
104def _add_model_to_suite(loaders, suite, model_info):
105    ModelTestCase = _hide_model_case_from_nose()
106
107    #print('------')
108    #print('found tests in', model_name)
109    #print('------')
110
111    # if ispy then use the dll loader to call pykernel
112    # don't try to call cl kernel since it will not be
113    # available in some environmentes.
114    is_py = callable(model_info.Iq)
115
116    # Some OpenCL drivers seem to be flaky, and are not producing the
117    # expected result.  Since we don't have known test values yet for
118    # all of our models, we are instead going to compare the results
119    # for the 'smoke test' (that is, evaluation at q=0.1 for the default
120    # parameters just to see that the model runs to completion) between
121    # the OpenCL and the DLL.  To do this, we define a 'stash' which is
122    # shared between OpenCL and DLL tests.  This is just a list.  If the
123    # list is empty (which it will be when DLL runs, if the DLL runs
124    # first), then the results are appended to the list.  If the list
125    # is not empty (which it will be when OpenCL runs second), the results
126    # are compared to the results stored in the first element of the list.
127    # This is a horrible stateful hack which only makes sense because the
128    # test suite is thrown away after being run once.
129    stash = []
130
131    if is_py:  # kernel implemented in python
132        test_name = "%s-python"%model_info.name
133        test_method_name = "test_%s_python" % model_info.id
134        test = ModelTestCase(test_name, model_info,
135                                test_method_name,
136                                platform="dll",  # so that
137                                dtype="double",
138                                stash=stash)
139        suite.addTest(test)
140    else:   # kernel implemented in C
141
142        # test using dll if desired
143        if 'dll' in loaders or not use_opencl():
144            test_name = "%s-dll"%model_info.name
145            test_method_name = "test_%s_dll" % model_info.id
146            test = ModelTestCase(test_name, model_info,
147                                    test_method_name,
148                                    platform="dll",
149                                    dtype="double",
150                                    stash=stash)
151            suite.addTest(test)
152
153        # test using opencl if desired and available
154        if 'opencl' in loaders and use_opencl():
155            test_name = "%s-opencl"%model_info.name
156            test_method_name = "test_%s_opencl" % model_info.id
157            # Using dtype=None so that the models that are only
158            # correct for double precision are not tested using
159            # single precision.  The choice is determined by the
160            # presence of *single=False* in the model file.
161            test = ModelTestCase(test_name, model_info,
162                                    test_method_name,
163                                    platform="ocl", dtype=None,
164                                    stash=stash)
165            #print("defining", test_name)
166            suite.addTest(test)
167
168        # test using cuda if desired and available
169        if 'cuda' in loaders and use_cuda():
170            test_name = "%s-cuda"%model_name
171            test_method_name = "test_%s_cuda" % model_info.id
172            # Using dtype=None so that the models that are only
173            # correct for double precision are not tested using
174            # single precision.  The choice is determined by the
175            # presence of *single=False* in the model file.
176            test = ModelTestCase(test_name, model_info,
177                                    test_method_name,
178                                    platform="cuda", dtype=None,
179                                    stash=stash)
180            #print("defining", test_name)
181            suite.addTest(test)
182
183
184def _hide_model_case_from_nose():
185    # type: () -> type
186    class ModelTestCase(unittest.TestCase):
187        """
188        Test suit for a particular model with a particular kernel driver.
189
190        The test suite runs a simple smoke test to make sure the model
191        functions, then runs the list of tests at the bottom of the model
192        description file.
193        """
194        def __init__(self, test_name, model_info, test_method_name,
195                     platform, dtype, stash):
196            # type: (str, ModelInfo, str, str, DType, List[Any]) -> None
197            self.test_name = test_name
198            self.info = model_info
199            self.platform = platform
200            self.dtype = dtype
201            self.stash = stash  # container for the results of the first run
202
203            setattr(self, test_method_name, self.run_all)
204            unittest.TestCase.__init__(self, test_method_name)
205
206        def run_all(self):
207            # type: () -> None
208            """
209            Run all the tests in the test suite, including smoke tests.
210            """
211            smoke_tests = [
212                # test validity at reasonable values
213                ({}, 0.1, None),
214                ({}, (0.1, 0.1), None),
215                # test validity at q = 0
216                #({}, 0.0, None),
217                #({}, (0.0, 0.0), None),
218                # test vector form
219                ({}, [0.001, 0.01, 0.1], [None]*3),
220                ({}, [(0.1, 0.1)]*2, [None]*2),
221                # test that Fq will run, and return R_eff, V, V_r
222                ({}, 0.1, None, None, None, None, None),
223                ]
224            tests = smoke_tests
225            #tests = []
226            if self.info.tests is not None:
227                tests += self.info.tests
228            S_tests = [test for test in tests if '@S' in test[0]]
229            P_tests = [test for test in tests if '@S' not in test[0]]
230            try:
231                model = build_model(self.info, dtype=self.dtype,
232                                    platform=self.platform)
233                results = [self.run_one(model, test) for test in P_tests]
234                for test in S_tests:
235                    # pull the S model name out of the test defn
236                    pars = test[0].copy()
237                    s_name = pars.pop('@S')
238                    ps_test = [pars] + list(test[1:])
239                    # build the P@S model
240                    s_info = load_model_info(s_name)
241                    ps_info = product.make_product_info(self.info, s_info)
242                    ps_model = build_model(ps_info, dtype=self.dtype,
243                                           platform=self.platform)
244                    # run the tests
245                    results.append(self.run_one(ps_model, ps_test))
246
247                if self.stash:
248                    for test, target, actual in zip(tests, self.stash[0], results):
249                        assert np.all(abs(target-actual) < 5e-5*abs(actual)), \
250                            ("GPU/CPU comparison expected %s but got %s for %s"
251                             % (target, actual, test[0]))
252                else:
253                    self.stash.append(results)
254
255                # Check for missing tests.  Only do so for the "dll" tests
256                # to reduce noise from both opencl and cuda, and because
257                # python kernels use platform="dll".
258                if self.platform == "dll":
259                    missing = []
260                    ## Uncomment the following to require test cases
261                    #missing = self._find_missing_tests()
262                    if missing:
263                        raise ValueError("Missing tests for "+", ".join(missing))
264
265            except:
266                annotate_exception(self.test_name)
267                raise
268
269        def _find_missing_tests(self):
270            # type: () -> None
271            """make sure there are 1D and 2D tests as appropriate"""
272            model_has_1D = True
273            model_has_2D = any(p.type == 'orientation'
274                               for p in self.info.parameters.kernel_parameters)
275
276            # Lists of tests that have a result that is not None
277            single = [test for test in self.info.tests
278                      if not isinstance(test[2], list) and test[2] is not None]
279            tests_has_1D_single = any(isinstance(test[1], float) for test in single)
280            tests_has_2D_single = any(isinstance(test[1], tuple) for test in single)
281
282            multiple = [test for test in self.info.tests
283                        if isinstance(test[2], list)
284                        and not all(result is None for result in test[2])]
285            tests_has_1D_multiple = any(isinstance(test[1][0], float)
286                                        for test in multiple)
287            tests_has_2D_multiple = any(isinstance(test[1][0], tuple)
288                                        for test in multiple)
289
290            missing = []
291            if model_has_1D and not (tests_has_1D_single or tests_has_1D_multiple):
292                missing.append("1D")
293            if model_has_2D and not (tests_has_2D_single or tests_has_2D_multiple):
294                missing.append("2D")
295
296            return missing
297
298        def run_one(self, model, test):
299            # type: (KernelModel, TestCondition) -> None
300            """Run a single test case."""
301            user_pars, x, y = test[:3]
302            pars = expand_pars(self.info.parameters, user_pars)
303            invalid = invalid_pars(self.info.parameters, pars)
304            if invalid:
305                raise ValueError("Unknown parameters in test: " + ", ".join(invalid))
306
307            if not isinstance(y, list):
308                y = [y]
309            if not isinstance(x, list):
310                x = [x]
311
312            self.assertEqual(len(y), len(x))
313
314            if isinstance(x[0], tuple):
315                qx, qy = zip(*x)
316                q_vectors = [np.array(qx), np.array(qy)]
317            else:
318                q_vectors = [np.array(x)]
319
320            kernel = model.make_kernel(q_vectors)
321            if len(test) == 3:
322                actual = call_kernel(kernel, pars)
323                self._check_vectors(x, y, actual, 'I')
324                return actual
325            else:
326                y1 = y
327                y2 = test[3] if not isinstance(test[3], list) else [test[3]]
328                F1, F2, R_eff, volume, volume_ratio = call_Fq(kernel, pars)
329                if F1 is not None:  # F1 is none for models with Iq instead of Fq
330                    self._check_vectors(x, y1, F1, 'F')
331                self._check_vectors(x, y2, F2, 'F^2')
332                self._check_scalar(test[4], R_eff, 'R_eff')
333                self._check_scalar(test[5], volume, 'volume')
334                self._check_scalar(test[6], volume_ratio, 'form:shell ratio')
335                return F2
336
337        def _check_scalar(self, target, actual, name):
338            if target is None:
339                # smoke test --- make sure it runs and produces a value
340                self.assertTrue(not np.isnan(actual),
341                                'invalid %s: %s' % (name, actual))
342            elif np.isnan(target):
343                # make sure nans match
344                self.assertTrue(np.isnan(actual),
345                                '%s: expected:%s; actual:%s'
346                                % (name, target, actual))
347            else:
348                # is_near does not work for infinite values, so also test
349                # for exact values.
350                self.assertTrue(target == actual or is_near(target, actual, 5),
351                                '%s: expected:%s; actual:%s'
352                                % (name, target, actual))
353
354        def _check_vectors(self, x, target, actual, name='I'):
355            self.assertTrue(len(actual) > 0,
356                            '%s(...) expected return'%name)
357            if target is None:
358                return
359            self.assertEqual(len(target), len(actual),
360                             '%s(...) returned wrong length'%name)
361            for xi, yi, actual_yi in zip(x, target, actual):
362                if yi is None:
363                    # smoke test --- make sure it runs and produces a value
364                    self.assertTrue(not np.isnan(actual_yi),
365                                    'invalid %s(%s): %s' % (name, xi, actual_yi))
366                elif np.isnan(yi):
367                    # make sure nans match
368                    self.assertTrue(np.isnan(actual_yi),
369                                    '%s(%s): expected:%s; actual:%s'
370                                    % (name, xi, yi, actual_yi))
371                else:
372                    # is_near does not work for infinite values, so also test
373                    # for exact values.
374                    self.assertTrue(yi == actual_yi or is_near(yi, actual_yi, 5),
375                                    '%s(%s); expected:%s; actual:%s'
376                                    % (name, xi, yi, actual_yi))
377
378    return ModelTestCase
379
380def invalid_pars(partable, pars):
381    # type: (ParameterTable, Dict[str, float])
382    """
383    Return a list of parameter names that are not part of the model.
384    """
385    names = set(p.id for p in partable.call_parameters)
386    invalid = []
387    for par in sorted(pars.keys()):
388        # special handling of R_eff mode, which is not a usual parameter
389        if par == product.RADIUS_MODE_ID:
390            continue
391        parts = par.split('_pd')
392        if len(parts) > 1 and parts[1] not in ("", "_n", "nsigma", "type"):
393            invalid.append(par)
394            continue
395        if parts[0] not in names:
396            invalid.append(par)
397    return invalid
398
399
400def is_near(target, actual, digits=5):
401    # type: (float, float, int) -> bool
402    """
403    Returns true if *actual* is within *digits* significant digits of *target*.
404    """
405    import math
406    shift = 10**math.ceil(math.log10(abs(target)))
407    return abs(target-actual)/shift < 1.5*10**-digits
408
409# CRUFT: old interface; should be deprecated and removed
410def run_one(model_name):
411    # msg = "use check_model(model_info) rather than run_one(model_name)"
412    # warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
413    try:
414        model_info = load_model_info(model_name)
415    except Exception:
416        output = traceback.format_exc()
417        return output
418
419    success, output = check_model(model_info)
420    return output
421
422def check_model(model_info):
423    # type: (ModelInfo) -> str
424    """
425    Run the tests for a single model, capturing the output.
426
427    Returns success status and the output string.
428    """
429    # Note that running main() directly did not work from within the
430    # wxPython pycrust console.  Instead of the results appearing in the
431    # window they were printed to the underlying console.
432    from unittest.runner import TextTestResult, _WritelnDecorator
433
434    # Build a object to capture and print the test results
435    stream = _WritelnDecorator(StringIO())  # Add writeln() method to stream
436    verbosity = 2
437    descriptions = True
438    result = TextTestResult(stream, descriptions, verbosity)
439
440    # Build a test suite containing just the model
441    loaders = ['opencl' if use_opencl() else 'cuda' if use_cuda() else 'dll']
442    suite = unittest.TestSuite()
443    _add_model_to_suite(loaders, suite, model_info)
444
445    # Warn if there are no user defined tests.
446    # Note: the test suite constructed above only has one test in it, which
447    # runs through some smoke tests to make sure the model runs, then runs
448    # through the input-output pairs given in the model definition file.  To
449    # check if any such pairs are defined, therefore, we just need to check if
450    # they are in the first test of the test suite.  We do this with an
451    # iterator since we don't have direct access to the list of tests in the
452    # test suite.
453    # In Qt5 suite.run() will clear all tests in the suite after running
454    # with no way of retaining them for the test below, so let's check
455    # for user tests before running the suite.
456    for test in suite:
457        if not test.info.tests:
458            stream.writeln("Note: %s has no user defined tests."%model_info.name)
459        break
460    else:
461        stream.writeln("Note: no test suite created --- this should never happen")
462
463    # Run the test suite
464    suite.run(result)
465
466    # Print the failures and errors
467    for _, tb in result.errors:
468        stream.writeln(tb)
469    for _, tb in result.failures:
470        stream.writeln(tb)
471
472    output = stream.getvalue()
473    stream.close()
474    return result.wasSuccessful(), output
475
476
477def main():
478    # type: (*str) -> int
479    """
480    Run tests given is models.
481
482    Returns 0 if success or 1 if any tests fail.
483    """
484    try:
485        from xmlrunner import XMLTestRunner as TestRunner
486        test_args = {'output': 'logs'}
487    except ImportError:
488        from unittest import TextTestRunner as TestRunner
489        test_args = {}
490
491    parser = argparse.ArgumentParser(description="Test SasModels Models")
492    parser.add_argument("-v", "--verbose", action="store_const",
493                        default=1, const=2, help="Use verbose output")
494    parser.add_argument("engine", metavar="[engine]",
495                        help="Engines on which to run the test.  "
496                        "Valid values are opencl, dll, and opencl_and_dll. "
497                        "Defaults to opencl_and_dll if no value is given")
498    parser.add_argument("models", nargs="*",
499                        help='The names of the models to be tested.  '
500                        'If the first model is "all", then all except the '
501                        'remaining models will be tested.')
502    args, models = parser.parse_known_args()
503
504    if args.engine == "opencl":
505        if not use_opencl():
506            print("opencl is not available")
507            return 1
508        loaders = ['opencl']
509    elif args.engine == "dll":
510        loaders = ["dll"]
511    elif args.engine == "cuda":
512        if not use_cuda():
513            print("cuda is not available")
514            return 1
515        loaders = ['cuda']
516    else:
517        # Default to running both engines
518        loaders = ['dll']
519        if use_opencl():
520            loaders.append('opencl')
521        if use_cuda():
522            loaders.append('cuda')
523        args.models.insert(0, args.engine)
524
525    runner = TestRunner(verbosity=args.verbose, **test_args)
526    result = runner.run(make_suite(loaders, args.models))
527    return 1 if result.failures or result.errors else 0
528
529
530def model_tests():
531    # type: () -> Iterator[Callable[[], None]]
532    """
533    Test runner visible to nosetests.
534
535    Run "nosetests sasmodels" on the command line to invoke it.
536    """
537    loaders = ['dll']
538    if use_opencl():
539        loaders.append('opencl')
540    if use_cuda():
541        loaders.append('cuda')
542    tests = make_suite(loaders, ['all'])
543    def build_test(test):
544        # In order for nosetest to show the test name, wrap the test.run_all
545        # instance in function that takes the test name as a parameter which
546        # will be displayed when the test is run.  Do this as a function so
547        # that it properly captures the context for tests that captured and
548        # run later.  If done directly in the for loop, then the looping
549        # variable test will be shared amongst all the tests, and we will be
550        # repeatedly testing vesicle.
551
552        # Note: in sasview sas.sasgui.perspectives.fitting.gpu_options
553        # requires that the test.description field be set.
554        wrap = lambda: test.run_all()
555        wrap.description = test.test_name
556        return wrap
557        # The following would work with nosetests and pytest:
558        #     return lambda name: test.run_all(), test.test_name
559
560    for test in tests:
561        yield build_test(test)
562
563
564if __name__ == "__main__":
565    sys.exit(main())
Note: See TracBrowser for help on using the repository browser.