source: sasmodels/sasmodels/model_test.py @ f942f4d

ticket-1257-vesicle-productticket_1156ticket_822_more_unit_tests
Last change on this file since f942f4d was f942f4d, checked in by Paul Kienzle <pkienzle@…>, 7 months ago

Merge branch 'ticket_822_v5_unit_tests' of github.com:sasview/sasmodels into ticket_822_v5_unit_tests

  • Property mode set to 100755
File size: 23.2 KB
Line 
1# -*- coding: utf-8 -*-
2"""
3Run model unit tests.
4
5Usage::
6
7    python -m sasmodels.model_test [opencl|cuda|dll|all] model1 model2 ...
8
9If model1 is 'all', then all except the remaining models will be tested.
10Subgroups are also possible, such as 'py', 'single' or '1d'.  See
11:func:`core.list_models` for details.
12
13Each model is tested using the default parameters at q=0.1, (qx, qy)=(0.1, 0.1),
14and Fq is called to make sure R_eff, volume and volume ratio are computed.
15The return values at these points are not considered.  The test is only to
16verify that the models run to completion, and do not produce inf or NaN.
17
18Tests are defined with the *tests* attribute in the model.py file.  *tests*
19is a list of individual tests to run, where each test consists of the
20parameter values for the test, the q-values and the expected results.  For
21the effective radius test and volume ratio tests, use the extended output
22form, which checks each output of kernel.Fq. For 1-D tests, either specify
23the q value or a list of q-values, and the corresponding I(q) value, or
24list of I(q) values.
25
26That is::
27
28    tests = [
29        [ {parameters}, q, I(q)],
30        [ {parameters}, [q], [I(q)] ],
31        [ {parameters}, [q1, q2, ...], [I(q1), I(q2), ...]],
32
33        [ {parameters}, (qx, qy), I(qx, Iqy)],
34        [ {parameters}, [(qx1, qy1), (qx2, qy2), ...],
35                        [I(qx1, qy1), I(qx2, qy2), ...]],
36
37        [ {parameters}, q, F(q), F^2(q), R_eff, V, V_r ],
38        ...
39    ]
40
41Parameters are *key:value* pairs, where key is one of the parameters of the
42model and value is the value to use for the test.  Any parameters not given
43in the parameter list will take on the default parameter value.
44
45Precision defaults to 5 digits (relative).
46"""
47from __future__ import print_function
48
49import argparse
50import sys
51import unittest
52import traceback
53
54try:
55    from StringIO import StringIO
56except ImportError:
57    # StringIO.StringIO renamed to io.StringIO in Python 3
58    # Note: io.StringIO exists in python 2, but using unicode instead of str
59    from io import StringIO
60
61import numpy as np  # type: ignore
62
63from .core import list_models, load_model_info, build_model
64from .direct_model import call_kernel, call_Fq
65from .exception import annotate_exception
66from .modelinfo import expand_pars
67from .kernelcl import use_opencl
68from .kernelcuda import use_cuda
69from . import product
70
71# pylint: disable=unused-import
72try:
73    from typing import List, Iterator, Callable
74except ImportError:
75    pass
76else:
77    from .modelinfo import ParameterTable, ParameterSet, TestCondition, ModelInfo
78    from .kernel import KernelModel
79# pylint: enable=unused-import
80
81def make_suite(loaders, models):
82    # type: (List[str], List[str]) -> unittest.TestSuite
83    """
84    Construct the pyunit test suite.
85
86    *loaders* is the list of kernel drivers to use (dll, opencl or cuda).
87    For python model the python driver is always used.
88
89    *models* is the list of models to test, or *["all"]* to test all models.
90    """
91    suite = unittest.TestSuite()
92
93    try:
94        # See if the first model parses as a model group
95        group = list_models(models[0])
96        skip = models[1:]
97        models = group
98    except Exception:
99        skip = []
100    for model_name in models:
101        if model_name not in skip:
102            model_info = load_model_info(model_name)
103            _add_model_to_suite(loaders, suite, model_info)
104
105    return suite
106
107def _add_model_to_suite(loaders, suite, model_info):
108    ModelTestCase = _hide_model_case_from_nose()
109
110    #print('------')
111    #print('found tests in', model_name)
112    #print('------')
113
114    # if ispy then use the dll loader to call pykernel
115    # don't try to call cl kernel since it will not be
116    # available in some environmentes.
117    is_py = callable(model_info.Iq)
118
119    # Some OpenCL drivers seem to be flaky, and are not producing the
120    # expected result.  Since we don't have known test values yet for
121    # all of our models, we are instead going to compare the results
122    # for the 'smoke test' (that is, evaluation at q=0.1 for the default
123    # parameters just to see that the model runs to completion) between
124    # the OpenCL and the DLL.  To do this, we define a 'stash' which is
125    # shared between OpenCL and DLL tests.  This is just a list.  If the
126    # list is empty (which it will be when DLL runs, if the DLL runs
127    # first), then the results are appended to the list.  If the list
128    # is not empty (which it will be when OpenCL runs second), the results
129    # are compared to the results stored in the first element of the list.
130    # This is a horrible stateful hack which only makes sense because the
131    # test suite is thrown away after being run once.
132    stash = []
133
134    if is_py:  # kernel implemented in python
135        test_name = "%s-python"%model_info.name
136        test_method_name = "test_%s_python" % model_info.id
137        test = ModelTestCase(test_name, model_info,
138                             test_method_name,
139                             platform="dll",  # so that
140                             dtype="double",
141                             stash=stash)
142        suite.addTest(test)
143    else:   # kernel implemented in C
144
145        # test using dll if desired
146        if 'dll' in loaders or not use_opencl():
147            test_name = "%s-dll"%model_info.name
148            test_method_name = "test_%s_dll" % model_info.id
149            test = ModelTestCase(test_name, model_info,
150                                 test_method_name,
151                                 platform="dll",
152                                 dtype="double",
153                                 stash=stash)
154            suite.addTest(test)
155
156        # test using opencl if desired and available
157        if 'opencl' in loaders and use_opencl():
158            test_name = "%s-opencl"%model_info.name
159            test_method_name = "test_%s_opencl" % model_info.id
160            # Using dtype=None so that the models that are only
161            # correct for double precision are not tested using
162            # single precision.  The choice is determined by the
163            # presence of *single=False* in the model file.
164            test = ModelTestCase(test_name, model_info,
165                                 test_method_name,
166                                 platform="ocl", dtype=None,
167                                 stash=stash)
168            #print("defining", test_name)
169            suite.addTest(test)
170
171        # test using cuda if desired and available
172        if 'cuda' in loaders and use_cuda():
173            test_name = "%s-cuda" % model_info.id
174            test_method_name = "test_%s_cuda" % model_info.id
175            # Using dtype=None so that the models that are only
176            # correct for double precision are not tested using
177            # single precision.  The choice is determined by the
178            # presence of *single=False* in the model file.
179            test = ModelTestCase(test_name, model_info,
180                                 test_method_name,
181                                 platform="cuda", dtype=None,
182                                 stash=stash)
183            #print("defining", test_name)
184            suite.addTest(test)
185
186
187def _hide_model_case_from_nose():
188    # type: () -> type
189    class ModelTestCase(unittest.TestCase):
190        """
191        Test suit for a particular model with a particular kernel driver.
192
193        The test suite runs a simple smoke test to make sure the model
194        functions, then runs the list of tests at the bottom of the model
195        description file.
196        """
197        def __init__(self, test_name, model_info, test_method_name,
198                     platform, dtype, stash):
199            # type: (str, ModelInfo, str, str, DType, List[Any]) -> None
200            self.test_name = test_name
201            self.info = model_info
202            self.platform = platform
203            self.dtype = dtype
204            self.stash = stash  # container for the results of the first run
205
206            setattr(self, test_method_name, self.run_all)
207            unittest.TestCase.__init__(self, test_method_name)
208
209        def run_all(self):
210            # type: () -> None
211            """
212            Run all the tests in the test suite, including smoke tests.
213            """
214            smoke_tests = [
215                # test validity at reasonable values
216                ({}, 0.1, None),
217                ({}, (0.1, 0.1), None),
218                # test validity at q = 0
219                #({}, 0.0, None),
220                #({}, (0.0, 0.0), None),
221                # test vector form
222                ({}, [0.001, 0.01, 0.1], [None]*3),
223                ({}, [(0.1, 0.1)]*2, [None]*2),
224                # test that Fq will run, and return R_eff, V, V_r
225                ({}, 0.1, None, None, None, None, None),
226                ]
227            tests = smoke_tests
228            #tests = []
229            if self.info.tests is not None:
230                tests += self.info.tests
231            S_tests = [test for test in tests if '@S' in test[0]]
232            P_tests = [test for test in tests if '@S' not in test[0]]
233            try:
234                model = build_model(self.info, dtype=self.dtype,
235                                    platform=self.platform)
236                results = [self.run_one(model, test) for test in P_tests]
237                for test in S_tests:
238                    # pull the S model name out of the test defn
239                    pars = test[0].copy()
240                    s_name = pars.pop('@S')
241                    ps_test = [pars] + list(test[1:])
242                    #print("PS TEST PARAMS!!!",ps_test)
243                    # build the P@S model
244                    s_info = load_model_info(s_name)
245                    ps_info = product.make_product_info(self.info, s_info)
246                    ps_model = build_model(ps_info, dtype=self.dtype,
247                                           platform=self.platform)
248                    # run the tests
249                    self.info = ps_model.info
250                    #print("SELF.INFO PARAMS!!!",[p.id for p in self.info.parameters.call_parameters])
251                    #print("PS MODEL PARAMETERS:",[p.id for p in ps_model.info.parameters.call_parameters])
252                    results.append(self.run_one(ps_model, ps_test))
253
254                if self.stash:
255                    for test, target, actual in zip(tests, self.stash[0], results):
256                        assert np.all(abs(target-actual) < 5e-5*abs(actual)), \
257                            ("GPU/CPU comparison expected %s but got %s for %s"
258                             % (target, actual, test[0]))
259                else:
260                    self.stash.append(results)
261
262                # Check for missing tests.  Only do so for the "dll" tests
263                # to reduce noise from both opencl and cuda, and because
264                # python kernels use platform="dll".
265                if self.platform == "dll":
266                    missing = []
267                    ## Uncomment the following to require test cases
268                    #missing = self._find_missing_tests()
269                    if missing:
270                        raise ValueError("Missing tests for "+", ".join(missing))
271
272            except:
273                annotate_exception(self.test_name)
274                raise
275
276        def _find_missing_tests(self):
277            # type: () -> None
278            """make sure there are 1D and 2D tests as appropriate"""
279            model_has_1D = True
280            model_has_2D = any(p.type == 'orientation'
281                               for p in self.info.parameters.kernel_parameters)
282
283            # Lists of tests that have a result that is not None
284            single = [test for test in self.info.tests
285                      if not isinstance(test[2], list) and test[2] is not None]
286            tests_has_1D_single = any(isinstance(test[1], float) for test in single)
287            tests_has_2D_single = any(isinstance(test[1], tuple) for test in single)
288
289            multiple = [test for test in self.info.tests
290                        if isinstance(test[2], list)
291                        and not all(result is None for result in test[2])]
292            tests_has_1D_multiple = any(isinstance(test[1][0], float)
293                                        for test in multiple)
294            tests_has_2D_multiple = any(isinstance(test[1][0], tuple)
295                                        for test in multiple)
296
297            missing = []
298            if model_has_1D and not (tests_has_1D_single or tests_has_1D_multiple):
299                missing.append("1D")
300            if model_has_2D and not (tests_has_2D_single or tests_has_2D_multiple):
301                missing.append("2D")
302
303            return missing
304
305        def run_one(self, model, test):
306            # type: (KernelModel, TestCondition) -> None
307            """Run a single test case."""
308            user_pars, x, y = test[:3]
309            pars = expand_pars(self.info.parameters, user_pars)
310            invalid = invalid_pars(self.info.parameters, pars)
311            if invalid:
312                raise ValueError("Unknown parameters in test: " + ", ".join(invalid))
313
314            if not isinstance(y, list):
315                y = [y]
316            if not isinstance(x, list):
317                x = [x]
318
319            self.assertEqual(len(y), len(x))
320
321            if isinstance(x[0], tuple):
322                qx, qy = zip(*x)
323                q_vectors = [np.array(qx), np.array(qy)]
324            else:
325                q_vectors = [np.array(x)]
326
327            kernel = model.make_kernel(q_vectors)
328            if len(test) == 3:
329                actual = call_kernel(kernel, pars)
330                self._check_vectors(x, y, actual, 'I')
331                return actual
332            else:
333                y1 = y
334                y2 = test[3] if isinstance(test[3], list) else [test[3]]
335                F, Fsq, R_eff, volume, volume_ratio = call_Fq(kernel, pars)
336                if F is not None:  # F is none for models with Iq instead of Fq
337                    self._check_vectors(x, y1, F, 'F')
338                self._check_vectors(x, y2, Fsq, 'F^2')
339                self._check_scalar(test[4], R_eff, 'R_eff')
340                self._check_scalar(test[5], volume, 'volume')
341                self._check_scalar(test[6], volume_ratio, 'form:shell ratio')
342                return Fsq
343
344        def _check_scalar(self, target, actual, name):
345            if target is None:
346                # smoke test --- make sure it runs and produces a value
347                self.assertTrue(not np.isnan(actual),
348                                'invalid %s: %s' % (name, actual))
349            elif np.isnan(target):
350                # make sure nans match
351                self.assertTrue(np.isnan(actual),
352                                '%s: expected:%s; actual:%s'
353                                % (name, target, actual))
354            else:
355                # is_near does not work for infinite values, so also test
356                # for exact values.
357                self.assertTrue(target == actual or is_near(target, actual, 5),
358                                '%s: expected:%s; actual:%s'
359                                % (name, target, actual))
360
361        def _check_vectors(self, x, target, actual, name='I'):
362            self.assertTrue(len(actual) > 0,
363                            '%s(...) expected return'%name)
364            if target is None:
365                return
366            self.assertEqual(len(target), len(actual),
367                             '%s(...) returned wrong length'%name)
368            for xi, yi, actual_yi in zip(x, target, actual):
369                if yi is None:
370                    # smoke test --- make sure it runs and produces a value
371                    self.assertTrue(not np.isnan(actual_yi),
372                                    'invalid %s(%s): %s' % (name, xi, actual_yi))
373                elif np.isnan(yi):
374                    # make sure nans match
375                    self.assertTrue(np.isnan(actual_yi),
376                                    '%s(%s): expected:%s; actual:%s'
377                                    % (name, xi, yi, actual_yi))
378                else:
379                    # is_near does not work for infinite values, so also test
380                    # for exact values.
381                    self.assertTrue(yi == actual_yi or is_near(yi, actual_yi, 5),
382                                    '%s(%s); expected:%s; actual:%s'
383                                    % (name, xi, yi, actual_yi))
384
385    return ModelTestCase
386
387def invalid_pars(partable, pars):
388    # type: (ParameterTable, Dict[str, float])
389    """
390    Return a list of parameter names that are not part of the model.
391    """
392    names = set(p.id for p in partable.call_parameters)
393    invalid = []
394    for par in sorted(pars.keys()):
395        # special handling of R_eff mode, which is not a usual parameter
396        if par == product.RADIUS_MODE_ID:
397            continue
398        if par == product.RADIUS_TYPE_ID:
399            continue
400        if par == product.STRUCTURE_MODE_ID:
401            continue
402        parts = par.split('_pd')
403        if len(parts) > 1 and parts[1] not in ("", "_n", "nsigma", "type"):
404            invalid.append(par)
405            continue
406        if parts[0] not in names:
407            invalid.append(par)
408    return invalid
409
410
411def is_near(target, actual, digits=5):
412    # type: (float, float, int) -> bool
413    """
414    Returns true if *actual* is within *digits* significant digits of *target*.
415    """
416    import math
417    if target == 0.:
418        return actual == 0.
419    shift = 10**math.ceil(math.log10(np.abs(target)))
420    return np.abs(target-actual)/shift < 1.5*10**-digits
421
422# CRUFT: old interface; should be deprecated and removed
423def run_one(model_name):
424    # type: (str) -> str
425    """
426    [Deprecated] Run the tests associated with *model_name*.
427
428    Use the following instead::
429
430        succss, output = check_model(load_model_info(model_name))
431    """
432    # msg = "use check_model(model_info) rather than run_one(model_name)"
433    # warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
434    try:
435        model_info = load_model_info(model_name)
436    except Exception:
437        output = traceback.format_exc()
438        return output
439
440    _, output = check_model(model_info)
441    return output
442
443def check_model(model_info):
444    # type: (ModelInfo) -> Tuple[bool, str]
445    """
446    Run the tests for a single model, capturing the output.
447
448    Returns success status and the output string.
449    """
450    # Note that running main() directly did not work from within the
451    # wxPython pycrust console.  Instead of the results appearing in the
452    # window they were printed to the underlying console.
453    from unittest.runner import TextTestResult, _WritelnDecorator
454
455    # Build a object to capture and print the test results
456    stream = _WritelnDecorator(StringIO())  # Add writeln() method to stream
457    verbosity = 2
458    descriptions = True
459    result = TextTestResult(stream, descriptions, verbosity)
460
461    # Build a test suite containing just the model
462    loaders = ['opencl' if use_opencl() else 'cuda' if use_cuda() else 'dll']
463    suite = unittest.TestSuite()
464    _add_model_to_suite(loaders, suite, model_info)
465
466    # Warn if there are no user defined tests.
467    # Note: the test suite constructed above only has one test in it, which
468    # runs through some smoke tests to make sure the model runs, then runs
469    # through the input-output pairs given in the model definition file.  To
470    # check if any such pairs are defined, therefore, we just need to check if
471    # they are in the first test of the test suite.  We do this with an
472    # iterator since we don't have direct access to the list of tests in the
473    # test suite.
474    # In Qt5 suite.run() will clear all tests in the suite after running
475    # with no way of retaining them for the test below, so let's check
476    # for user tests before running the suite.
477    for test in suite:
478        if not test.info.tests:
479            stream.writeln("Note: %s has no user defined tests."%model_info.name)
480        break
481    else:
482        stream.writeln("Note: no test suite created --- this should never happen")
483
484    # Run the test suite
485    suite.run(result)
486
487    # Print the failures and errors
488    for _, tb in result.errors:
489        stream.writeln(tb)
490    for _, tb in result.failures:
491        stream.writeln(tb)
492
493    output = stream.getvalue()
494    stream.close()
495    return result.wasSuccessful(), output
496
497
498def model_tests():
499    # type: () -> Iterator[Callable[[], None]]
500    """
501    Test runner visible to nosetests.
502
503    Run "nosetests sasmodels" on the command line to invoke it.
504    """
505    loaders = ['dll']
506    if use_opencl():
507        loaders.append('opencl')
508    if use_cuda():
509        loaders.append('cuda')
510    tests = make_suite(loaders, ['all'])
511    def _build_test(test):
512        # In order for nosetest to show the test name, wrap the test.run_all
513        # instance in function that takes the test name as a parameter which
514        # will be displayed when the test is run.  Do this as a function so
515        # that it properly captures the context for tests that captured and
516        # run later.  If done directly in the for loop, then the looping
517        # variable test will be shared amongst all the tests, and we will be
518        # repeatedly testing vesicle.
519
520        # Note: in sasview sas.sasgui.perspectives.fitting.gpu_options
521        # requires that the test.description field be set.
522        wrap = lambda: test.run_all()
523        wrap.description = test.test_name
524        return wrap
525        # The following would work with nosetests and pytest:
526        #     return lambda name: test.run_all(), test.test_name
527
528    for test in tests:
529        yield _build_test(test)
530
531
532def main():
533    # type: (*str) -> int
534    """
535    Run tests given is models.
536
537    Returns 0 if success or 1 if any tests fail.
538    """
539    try:
540        from xmlrunner import XMLTestRunner as TestRunner
541        test_args = {'output': 'logs'}
542    except ImportError:
543        from unittest import TextTestRunner as TestRunner
544        test_args = {}
545
546    parser = argparse.ArgumentParser(description="Test SasModels Models")
547    parser.add_argument("-v", "--verbose", action="store_const",
548                        default=1, const=2, help="Use verbose output")
549    parser.add_argument("-e", "--engine", default="all",
550                        help="Engines on which to run the test.  "
551                        "Valid values are opencl, cuda, dll, and all. "
552                        "Defaults to all if no value is given")
553    parser.add_argument("models", nargs="*",
554                        help="The names of the models to be tested.  "
555                        "If the first model is 'all', then all but the listed "
556                        "models will be tested.  See core.list_models() for "
557                        "names of other groups, such as 'py' or 'single'.")
558    opts = parser.parse_args()
559
560    if opts.engine == "opencl":
561        if not use_opencl():
562            print("opencl is not available")
563            return 1
564        loaders = ['opencl']
565    elif opts.engine == "dll":
566        loaders = ["dll"]
567    elif opts.engine == "cuda":
568        if not use_cuda():
569            print("cuda is not available")
570            return 1
571        loaders = ['cuda']
572    elif opts.engine == "all":
573        loaders = ['dll']
574        if use_opencl():
575            loaders.append('opencl')
576        if use_cuda():
577            loaders.append('cuda')
578    else:
579        print("unknown engine " + opts.engine)
580        return 1
581
582    runner = TestRunner(verbosity=opts.verbose, **test_args)
583    result = runner.run(make_suite(loaders, opts.models))
584    return 1 if result.failures or result.errors else 0
585
586
587if __name__ == "__main__":
588    sys.exit(main())
Note: See TracBrowser for help on using the repository browser.