source: sasmodels/sasmodels/model_test.py @ b297ba9

core_shell_microgelsmagnetic_modelticket-1257-vesicle-productticket_1156ticket_1265_superballticket_822_more_unit_tests
Last change on this file since b297ba9 was b297ba9, checked in by Paul Kienzle <pkienzle@…>, 5 months ago

lint

  • Property mode set to 100755
File size: 22.7 KB
Line 
1# -*- coding: utf-8 -*-
2"""
3Run model unit tests.
4
5Usage::
6
7    python -m sasmodels.model_test [opencl|cuda|dll|all] model1 model2 ...
8
9If model1 is 'all', then all except the remaining models will be tested.
10Subgroups are also possible, such as 'py', 'single' or '1d'.  See
11:func:`core.list_models` for details.
12
13Each model is tested using the default parameters at q=0.1, (qx, qy)=(0.1, 0.1),
14and Fq is called to make sure R_eff, volume and volume ratio are computed.
15The return values at these points are not considered.  The test is only to
16verify that the models run to completion, and do not produce inf or NaN.
17
18Tests are defined with the *tests* attribute in the model.py file.  *tests*
19is a list of individual tests to run, where each test consists of the
20parameter values for the test, the q-values and the expected results.  For
21the effective radius test and volume ratio tests, use the extended output
22form, which checks each output of kernel.Fq. For 1-D tests, either specify
23the q value or a list of q-values, and the corresponding I(q) value, or
24list of I(q) values.
25
26That is::
27
28    tests = [
29        [ {parameters}, q, I(q)],
30        [ {parameters}, [q], [I(q)] ],
31        [ {parameters}, [q1, q2, ...], [I(q1), I(q2), ...]],
32
33        [ {parameters}, (qx, qy), I(qx, Iqy)],
34        [ {parameters}, [(qx1, qy1), (qx2, qy2), ...],
35                        [I(qx1, qy1), I(qx2, qy2), ...]],
36
37        [ {parameters}, q, F(q), F^2(q), R_eff, V, V_r ],
38        ...
39    ]
40
41Parameters are *key:value* pairs, where key is one of the parameters of the
42model and value is the value to use for the test.  Any parameters not given
43in the parameter list will take on the default parameter value.
44
45Precision defaults to 5 digits (relative).
46"""
47from __future__ import print_function
48
49import argparse
50import sys
51import unittest
52import traceback
53
54try:
55    from StringIO import StringIO
56except ImportError:
57    # StringIO.StringIO renamed to io.StringIO in Python 3
58    # Note: io.StringIO exists in python 2, but using unicode instead of str
59    from io import StringIO
60
61import numpy as np  # type: ignore
62
63from .core import list_models, load_model_info, build_model
64from .direct_model import call_kernel, call_Fq
65from .exception import annotate_exception
66from .modelinfo import expand_pars
67from .kernelcl import use_opencl
68from .kernelcuda import use_cuda
69from . import product
70
71# pylint: disable=unused-import
72try:
73    from typing import List, Iterator, Callable
74except ImportError:
75    pass
76else:
77    from .modelinfo import ParameterTable, ParameterSet, TestCondition, ModelInfo
78    from .kernel import KernelModel
79# pylint: enable=unused-import
80
81def make_suite(loaders, models):
82    # type: (List[str], List[str]) -> unittest.TestSuite
83    """
84    Construct the pyunit test suite.
85
86    *loaders* is the list of kernel drivers to use (dll, opencl or cuda).
87    For python model the python driver is always used.
88
89    *models* is the list of models to test, or *["all"]* to test all models.
90    """
91    suite = unittest.TestSuite()
92
93    try:
94        # See if the first model parses as a model group
95        group = list_models(models[0])
96        skip = models[1:]
97        models = group
98    except Exception:
99        skip = []
100    for model_name in models:
101        if model_name not in skip:
102            model_info = load_model_info(model_name)
103            _add_model_to_suite(loaders, suite, model_info)
104
105    return suite
106
107def _add_model_to_suite(loaders, suite, model_info):
108    ModelTestCase = _hide_model_case_from_nose()
109
110    #print('------')
111    #print('found tests in', model_name)
112    #print('------')
113
114    # if ispy then use the dll loader to call pykernel
115    # don't try to call cl kernel since it will not be
116    # available in some environmentes.
117    is_py = callable(model_info.Iq)
118
119    # Some OpenCL drivers seem to be flaky, and are not producing the
120    # expected result.  Since we don't have known test values yet for
121    # all of our models, we are instead going to compare the results
122    # for the 'smoke test' (that is, evaluation at q=0.1 for the default
123    # parameters just to see that the model runs to completion) between
124    # the OpenCL and the DLL.  To do this, we define a 'stash' which is
125    # shared between OpenCL and DLL tests.  This is just a list.  If the
126    # list is empty (which it will be when DLL runs, if the DLL runs
127    # first), then the results are appended to the list.  If the list
128    # is not empty (which it will be when OpenCL runs second), the results
129    # are compared to the results stored in the first element of the list.
130    # This is a horrible stateful hack which only makes sense because the
131    # test suite is thrown away after being run once.
132    stash = []
133
134    if is_py:  # kernel implemented in python
135        test_name = "%s-python"%model_info.name
136        test_method_name = "test_%s_python" % model_info.id
137        test = ModelTestCase(test_name, model_info,
138                             test_method_name,
139                             platform="dll",  # so that
140                             dtype="double",
141                             stash=stash)
142        suite.addTest(test)
143    else:   # kernel implemented in C
144
145        # test using dll if desired
146        if 'dll' in loaders or not use_opencl():
147            test_name = "%s-dll"%model_info.name
148            test_method_name = "test_%s_dll" % model_info.id
149            test = ModelTestCase(test_name, model_info,
150                                 test_method_name,
151                                 platform="dll",
152                                 dtype="double",
153                                 stash=stash)
154            suite.addTest(test)
155
156        # test using opencl if desired and available
157        if 'opencl' in loaders and use_opencl():
158            test_name = "%s-opencl"%model_info.name
159            test_method_name = "test_%s_opencl" % model_info.id
160            # Using dtype=None so that the models that are only
161            # correct for double precision are not tested using
162            # single precision.  The choice is determined by the
163            # presence of *single=False* in the model file.
164            test = ModelTestCase(test_name, model_info,
165                                 test_method_name,
166                                 platform="ocl", dtype=None,
167                                 stash=stash)
168            #print("defining", test_name)
169            suite.addTest(test)
170
171        # test using cuda if desired and available
172        if 'cuda' in loaders and use_cuda():
173            test_name = "%s-cuda" % model_info.id
174            test_method_name = "test_%s_cuda" % model_info.id
175            # Using dtype=None so that the models that are only
176            # correct for double precision are not tested using
177            # single precision.  The choice is determined by the
178            # presence of *single=False* in the model file.
179            test = ModelTestCase(test_name, model_info,
180                                 test_method_name,
181                                 platform="cuda", dtype=None,
182                                 stash=stash)
183            #print("defining", test_name)
184            suite.addTest(test)
185
186
187def _hide_model_case_from_nose():
188    # type: () -> type
189    class ModelTestCase(unittest.TestCase):
190        """
191        Test suit for a particular model with a particular kernel driver.
192
193        The test suite runs a simple smoke test to make sure the model
194        functions, then runs the list of tests at the bottom of the model
195        description file.
196        """
197        def __init__(self, test_name, model_info, test_method_name,
198                     platform, dtype, stash):
199            # type: (str, ModelInfo, str, str, DType, List[Any]) -> None
200            self.test_name = test_name
201            self.info = model_info
202            self.platform = platform
203            self.dtype = dtype
204            self.stash = stash  # container for the results of the first run
205
206            setattr(self, test_method_name, self.run_all)
207            unittest.TestCase.__init__(self, test_method_name)
208
209        def run_all(self):
210            # type: () -> None
211            """
212            Run all the tests in the test suite, including smoke tests.
213            """
214            smoke_tests = [
215                # test validity at reasonable values
216                ({}, 0.1, None),
217                ({}, (0.1, 0.1), None),
218                # test validity at q = 0
219                #({}, 0.0, None),
220                #({}, (0.0, 0.0), None),
221                # test vector form
222                ({}, [0.001, 0.01, 0.1], [None]*3),
223                ({}, [(0.1, 0.1)]*2, [None]*2),
224                # test that Fq will run, and return R_eff, V, V_r
225                ({}, 0.1, None, None, None, None, None),
226                ]
227            tests = smoke_tests
228            #tests = []
229            if self.info.tests is not None:
230                tests += self.info.tests
231            S_tests = [test for test in tests if '@S' in test[0]]
232            P_tests = [test for test in tests if '@S' not in test[0]]
233            try:
234                model = build_model(self.info, dtype=self.dtype,
235                                    platform=self.platform)
236                results = [self.run_one(model, test) for test in P_tests]
237                for test in S_tests:
238                    # pull the S model name out of the test defn
239                    pars = test[0].copy()
240                    s_name = pars.pop('@S')
241                    ps_test = [pars] + list(test[1:])
242                    # build the P@S model
243                    s_info = load_model_info(s_name)
244                    ps_info = product.make_product_info(self.info, s_info)
245                    ps_model = build_model(ps_info, dtype=self.dtype,
246                                           platform=self.platform)
247                    # run the tests
248                    results.append(self.run_one(ps_model, ps_test))
249
250                if self.stash:
251                    for test, target, actual in zip(tests, self.stash[0], results):
252                        assert np.all(abs(target-actual) < 5e-5*abs(actual)), \
253                            ("GPU/CPU comparison expected %s but got %s for %s"
254                             % (target, actual, test[0]))
255                else:
256                    self.stash.append(results)
257
258                # Check for missing tests.  Only do so for the "dll" tests
259                # to reduce noise from both opencl and cuda, and because
260                # python kernels use platform="dll".
261                if self.platform == "dll":
262                    missing = []
263                    ## Uncomment the following to require test cases
264                    #missing = self._find_missing_tests()
265                    if missing:
266                        raise ValueError("Missing tests for "+", ".join(missing))
267
268            except:
269                annotate_exception(self.test_name)
270                raise
271
272        def _find_missing_tests(self):
273            # type: () -> None
274            """make sure there are 1D and 2D tests as appropriate"""
275            model_has_1D = True
276            model_has_2D = any(p.type == 'orientation'
277                               for p in self.info.parameters.kernel_parameters)
278
279            # Lists of tests that have a result that is not None
280            single = [test for test in self.info.tests
281                      if not isinstance(test[2], list) and test[2] is not None]
282            tests_has_1D_single = any(isinstance(test[1], float) for test in single)
283            tests_has_2D_single = any(isinstance(test[1], tuple) for test in single)
284
285            multiple = [test for test in self.info.tests
286                        if isinstance(test[2], list)
287                        and not all(result is None for result in test[2])]
288            tests_has_1D_multiple = any(isinstance(test[1][0], float)
289                                        for test in multiple)
290            tests_has_2D_multiple = any(isinstance(test[1][0], tuple)
291                                        for test in multiple)
292
293            missing = []
294            if model_has_1D and not (tests_has_1D_single or tests_has_1D_multiple):
295                missing.append("1D")
296            if model_has_2D and not (tests_has_2D_single or tests_has_2D_multiple):
297                missing.append("2D")
298
299            return missing
300
301        def run_one(self, model, test):
302            # type: (KernelModel, TestCondition) -> None
303            """Run a single test case."""
304            user_pars, x, y = test[:3]
305            pars = expand_pars(self.info.parameters, user_pars)
306            invalid = invalid_pars(self.info.parameters, pars)
307            if invalid:
308                raise ValueError("Unknown parameters in test: " + ", ".join(invalid))
309
310            if not isinstance(y, list):
311                y = [y]
312            if not isinstance(x, list):
313                x = [x]
314
315            self.assertEqual(len(y), len(x))
316
317            if isinstance(x[0], tuple):
318                qx, qy = zip(*x)
319                q_vectors = [np.array(qx), np.array(qy)]
320            else:
321                q_vectors = [np.array(x)]
322
323            kernel = model.make_kernel(q_vectors)
324            if len(test) == 3:
325                actual = call_kernel(kernel, pars)
326                self._check_vectors(x, y, actual, 'I')
327                return actual
328            else:
329                y1 = y
330                y2 = test[3] if not isinstance(test[3], list) else [test[3]]
331                F1, F2, R_eff, volume, volume_ratio = call_Fq(kernel, pars)
332                if F1 is not None:  # F1 is none for models with Iq instead of Fq
333                    self._check_vectors(x, y1, F1, 'F')
334                self._check_vectors(x, y2, F2, 'F^2')
335                self._check_scalar(test[4], R_eff, 'R_eff')
336                self._check_scalar(test[5], volume, 'volume')
337                self._check_scalar(test[6], volume_ratio, 'form:shell ratio')
338                return F2
339
340        def _check_scalar(self, target, actual, name):
341            if target is None:
342                # smoke test --- make sure it runs and produces a value
343                self.assertTrue(not np.isnan(actual),
344                                'invalid %s: %s' % (name, actual))
345            elif np.isnan(target):
346                # make sure nans match
347                self.assertTrue(np.isnan(actual),
348                                '%s: expected:%s; actual:%s'
349                                % (name, target, actual))
350            else:
351                # is_near does not work for infinite values, so also test
352                # for exact values.
353                self.assertTrue(target == actual or is_near(target, actual, 5),
354                                '%s: expected:%s; actual:%s'
355                                % (name, target, actual))
356
357        def _check_vectors(self, x, target, actual, name='I'):
358            self.assertTrue(len(actual) > 0,
359                            '%s(...) expected return'%name)
360            if target is None:
361                return
362            self.assertEqual(len(target), len(actual),
363                             '%s(...) returned wrong length'%name)
364            for xi, yi, actual_yi in zip(x, target, actual):
365                if yi is None:
366                    # smoke test --- make sure it runs and produces a value
367                    self.assertTrue(not np.isnan(actual_yi),
368                                    'invalid %s(%s): %s' % (name, xi, actual_yi))
369                elif np.isnan(yi):
370                    # make sure nans match
371                    self.assertTrue(np.isnan(actual_yi),
372                                    '%s(%s): expected:%s; actual:%s'
373                                    % (name, xi, yi, actual_yi))
374                else:
375                    # is_near does not work for infinite values, so also test
376                    # for exact values.
377                    self.assertTrue(yi == actual_yi or is_near(yi, actual_yi, 5),
378                                    '%s(%s); expected:%s; actual:%s'
379                                    % (name, xi, yi, actual_yi))
380
381    return ModelTestCase
382
383def invalid_pars(partable, pars):
384    # type: (ParameterTable, Dict[str, float])
385    """
386    Return a list of parameter names that are not part of the model.
387    """
388    names = set(p.id for p in partable.call_parameters)
389    invalid = []
390    for par in sorted(pars.keys()):
391        # special handling of R_eff mode, which is not a usual parameter
392        if par == product.RADIUS_MODE_ID:
393            continue
394        parts = par.split('_pd')
395        if len(parts) > 1 and parts[1] not in ("", "_n", "nsigma", "type"):
396            invalid.append(par)
397            continue
398        if parts[0] not in names:
399            invalid.append(par)
400    return invalid
401
402
403def is_near(target, actual, digits=5):
404    # type: (float, float, int) -> bool
405    """
406    Returns true if *actual* is within *digits* significant digits of *target*.
407    """
408    import math
409    shift = 10**math.ceil(math.log10(abs(target)))
410    return abs(target-actual)/shift < 1.5*10**-digits
411
412# CRUFT: old interface; should be deprecated and removed
413def run_one(model_name):
414    # type: (str) -> str
415    """
416    [Deprecated] Run the tests associated with *model_name*.
417
418    Use the following instead::
419
420        succss, output = check_model(load_model_info(model_name))
421    """
422    # msg = "use check_model(model_info) rather than run_one(model_name)"
423    # warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
424    try:
425        model_info = load_model_info(model_name)
426    except Exception:
427        output = traceback.format_exc()
428        return output
429
430    _, output = check_model(model_info)
431    return output
432
433def check_model(model_info):
434    # type: (ModelInfo) -> Tuple[bool, str]
435    """
436    Run the tests for a single model, capturing the output.
437
438    Returns success status and the output string.
439    """
440    # Note that running main() directly did not work from within the
441    # wxPython pycrust console.  Instead of the results appearing in the
442    # window they were printed to the underlying console.
443    from unittest.runner import TextTestResult, _WritelnDecorator
444
445    # Build a object to capture and print the test results
446    stream = _WritelnDecorator(StringIO())  # Add writeln() method to stream
447    verbosity = 2
448    descriptions = True
449    result = TextTestResult(stream, descriptions, verbosity)
450
451    # Build a test suite containing just the model
452    loaders = ['opencl' if use_opencl() else 'cuda' if use_cuda() else 'dll']
453    suite = unittest.TestSuite()
454    _add_model_to_suite(loaders, suite, model_info)
455
456    # Warn if there are no user defined tests.
457    # Note: the test suite constructed above only has one test in it, which
458    # runs through some smoke tests to make sure the model runs, then runs
459    # through the input-output pairs given in the model definition file.  To
460    # check if any such pairs are defined, therefore, we just need to check if
461    # they are in the first test of the test suite.  We do this with an
462    # iterator since we don't have direct access to the list of tests in the
463    # test suite.
464    # In Qt5 suite.run() will clear all tests in the suite after running
465    # with no way of retaining them for the test below, so let's check
466    # for user tests before running the suite.
467    for test in suite:
468        if not test.info.tests:
469            stream.writeln("Note: %s has no user defined tests."%model_info.name)
470        break
471    else:
472        stream.writeln("Note: no test suite created --- this should never happen")
473
474    # Run the test suite
475    suite.run(result)
476
477    # Print the failures and errors
478    for _, tb in result.errors:
479        stream.writeln(tb)
480    for _, tb in result.failures:
481        stream.writeln(tb)
482
483    output = stream.getvalue()
484    stream.close()
485    return result.wasSuccessful(), output
486
487
488def model_tests():
489    # type: () -> Iterator[Callable[[], None]]
490    """
491    Test runner visible to nosetests.
492
493    Run "nosetests sasmodels" on the command line to invoke it.
494    """
495    loaders = ['dll']
496    if use_opencl():
497        loaders.append('opencl')
498    if use_cuda():
499        loaders.append('cuda')
500    tests = make_suite(loaders, ['all'])
501    def _build_test(test):
502        # In order for nosetest to show the test name, wrap the test.run_all
503        # instance in function that takes the test name as a parameter which
504        # will be displayed when the test is run.  Do this as a function so
505        # that it properly captures the context for tests that captured and
506        # run later.  If done directly in the for loop, then the looping
507        # variable test will be shared amongst all the tests, and we will be
508        # repeatedly testing vesicle.
509
510        # Note: in sasview sas.sasgui.perspectives.fitting.gpu_options
511        # requires that the test.description field be set.
512        wrap = lambda: test.run_all()
513        wrap.description = test.test_name
514        return wrap
515        # The following would work with nosetests and pytest:
516        #     return lambda name: test.run_all(), test.test_name
517
518    for test in tests:
519        yield _build_test(test)
520
521
522def main():
523    # type: (*str) -> int
524    """
525    Run tests given is models.
526
527    Returns 0 if success or 1 if any tests fail.
528    """
529    try:
530        from xmlrunner import XMLTestRunner as TestRunner
531        test_args = {'output': 'logs'}
532    except ImportError:
533        from unittest import TextTestRunner as TestRunner
534        test_args = {}
535
536    parser = argparse.ArgumentParser(description="Test SasModels Models")
537    parser.add_argument("-v", "--verbose", action="store_const",
538                        default=1, const=2, help="Use verbose output")
539    parser.add_argument("-e", "--engine", default="all",
540                        help="Engines on which to run the test.  "
541                        "Valid values are opencl, cuda, dll, and all. "
542                        "Defaults to all if no value is given")
543    parser.add_argument("models", nargs="*",
544                        help="The names of the models to be tested.  "
545                        "If the first model is 'all', then all but the listed "
546                        "models will be tested.  See core.list_models() for "
547                        "names of other groups, such as 'py' or 'single'.")
548    opts = parser.parse_args()
549
550    if opts.engine == "opencl":
551        if not use_opencl():
552            print("opencl is not available")
553            return 1
554        loaders = ['opencl']
555    elif opts.engine == "dll":
556        loaders = ["dll"]
557    elif opts.engine == "cuda":
558        if not use_cuda():
559            print("cuda is not available")
560            return 1
561        loaders = ['cuda']
562    elif opts.engine == "all":
563        loaders = ['dll']
564        if use_opencl():
565            loaders.append('opencl')
566        if use_cuda():
567            loaders.append('cuda')
568    else:
569        print("unknown engine " + opts.engine)
570        return 1
571
572    runner = TestRunner(verbosity=opts.verbose, **test_args)
573    result = runner.run(make_suite(loaders, opts.models))
574    return 1 if result.failures or result.errors else 0
575
576
577if __name__ == "__main__":
578    sys.exit(main())
Note: See TracBrowser for help on using the repository browser.