source: sasmodels/sasmodels/model_test.py @ 304c775

core_shell_microgelsmagnetic_modelticket-1257-vesicle-productticket_1156ticket_1265_superballticket_822_more_unit_tests
Last change on this file since 304c775 was 304c775, checked in by Paul Kienzle <pkienzle@…>, 6 years ago

provide method for testing Fq results. Refs #1202.

  • Property mode set to 100755
File size: 21.5 KB
Line 
1# -*- coding: utf-8 -*-
2"""
3Run model unit tests.
4
5Usage::
6
7    python -m sasmodels.model_test [opencl|cuda|dll] model1 model2 ...
8
9    if model1 is 'all', then all except the remaining models will be tested
10
11Each model is tested using the default parameters at q=0.1, (qx, qy)=(0.1, 0.1),
12and the ER and VR are computed.  The return values at these points are not
13considered.  The test is only to verify that the models run to completion,
14and do not produce inf or NaN.
15
16Tests are defined with the *tests* attribute in the model.py file.  *tests*
17is a list of individual tests to run, where each test consists of the
18parameter values for the test, the q-values and the expected results.  For
19the effective radius test, the q-value should be 'ER'.  For the VR test,
20the q-value should be 'VR'.  For 1-D tests, either specify the q value or
21a list of q-values, and the corresponding I(q) value, or list of I(q) values.
22
23That is::
24
25    tests = [
26        [ {parameters}, q, I(q)],
27        [ {parameters}, [q], [I(q)] ],
28        [ {parameters}, [q1, q2, ...], [I(q1), I(q2), ...]],
29
30        [ {parameters}, (qx, qy), I(qx, Iqy)],
31        [ {parameters}, [(qx1, qy1), (qx2, qy2), ...],
32                        [I(qx1, qy1), I(qx2, qy2), ...]],
33
34        [ {parameters}, 'ER', ER(pars) ],
35        [ {parameters}, 'VR', VR(pars) ],
36        ...
37    ]
38
39Parameters are *key:value* pairs, where key is one of the parameters of the
40model and value is the value to use for the test.  Any parameters not given
41in the parameter list will take on the default parameter value.
42
43Precision defaults to 5 digits (relative).
44"""
45from __future__ import print_function
46
47import sys
48import unittest
49
50try:
51    from StringIO import StringIO
52except ImportError:
53    # StringIO.StringIO renamed to io.StringIO in Python 3
54    # Note: io.StringIO exists in python 2, but using unicode instead of str
55    from io import StringIO
56
57import numpy as np  # type: ignore
58
59from . import core
60from .core import list_models, load_model_info, build_model
61from .direct_model import call_kernel, call_Fq
62from .exception import annotate_exception
63from .modelinfo import expand_pars
64from .kernelcl import use_opencl
65from .kernelcuda import use_cuda
66
67# pylint: disable=unused-import
68try:
69    from typing import List, Iterator, Callable
70except ImportError:
71    pass
72else:
73    from .modelinfo import ParameterTable, ParameterSet, TestCondition, ModelInfo
74    from .kernel import KernelModel
75# pylint: enable=unused-import
76
77
78def make_suite(loaders, models):
79    # type: (List[str], List[str]) -> unittest.TestSuite
80    """
81    Construct the pyunit test suite.
82
83    *loaders* is the list of kernel drivers to use (dll, opencl or cuda).
84    For python model the python driver is always used.
85
86    *models* is the list of models to test, or *["all"]* to test all models.
87    """
88    ModelTestCase = _hide_model_case_from_nose()
89    suite = unittest.TestSuite()
90
91    if models[0] in core.KINDS:
92        skip = models[1:]
93        models = list_models(models[0])
94    else:
95        skip = []
96    for model_name in models:
97        if model_name in skip:
98            continue
99        model_info = load_model_info(model_name)
100
101        #print('------')
102        #print('found tests in', model_name)
103        #print('------')
104
105        # if ispy then use the dll loader to call pykernel
106        # don't try to call cl kernel since it will not be
107        # available in some environmentes.
108        is_py = callable(model_info.Iq)
109
110        # Some OpenCL drivers seem to be flaky, and are not producing the
111        # expected result.  Since we don't have known test values yet for
112        # all of our models, we are instead going to compare the results
113        # for the 'smoke test' (that is, evaluation at q=0.1 for the default
114        # parameters just to see that the model runs to completion) between
115        # the OpenCL and the DLL.  To do this, we define a 'stash' which is
116        # shared between OpenCL and DLL tests.  This is just a list.  If the
117        # list is empty (which it will be when DLL runs, if the DLL runs
118        # first), then the results are appended to the list.  If the list
119        # is not empty (which it will be when OpenCL runs second), the results
120        # are compared to the results stored in the first element of the list.
121        # This is a horrible stateful hack which only makes sense because the
122        # test suite is thrown away after being run once.
123        stash = []
124
125        if is_py:  # kernel implemented in python
126            test_name = "%s-python"%model_name
127            test_method_name = "test_%s_python" % model_info.id
128            test = ModelTestCase(test_name, model_info,
129                                 test_method_name,
130                                 platform="dll",  # so that
131                                 dtype="double",
132                                 stash=stash)
133            suite.addTest(test)
134        else:   # kernel implemented in C
135
136            # test using dll if desired
137            if 'dll' in loaders:
138                test_name = "%s-dll"%model_name
139                test_method_name = "test_%s_dll" % model_info.id
140                test = ModelTestCase(test_name, model_info,
141                                     test_method_name,
142                                     platform="dll",
143                                     dtype="double",
144                                     stash=stash)
145                suite.addTest(test)
146
147            # test using opencl if desired and available
148            if 'opencl' in loaders and use_opencl():
149                test_name = "%s-opencl"%model_name
150                test_method_name = "test_%s_opencl" % model_info.id
151                # Using dtype=None so that the models that are only
152                # correct for double precision are not tested using
153                # single precision.  The choice is determined by the
154                # presence of *single=False* in the model file.
155                test = ModelTestCase(test_name, model_info,
156                                     test_method_name,
157                                     platform="ocl", dtype=None,
158                                     stash=stash)
159                #print("defining", test_name)
160                suite.addTest(test)
161
162            # test using cuda if desired and available
163            if 'cuda' in loaders and use_cuda():
164                test_name = "%s-cuda"%model_name
165                test_method_name = "test_%s_cuda" % model_info.id
166                # Using dtype=None so that the models that are only
167                # correct for double precision are not tested using
168                # single precision.  The choice is determined by the
169                # presence of *single=False* in the model file.
170                test = ModelTestCase(test_name, model_info,
171                                     test_method_name,
172                                     platform="cuda", dtype=None,
173                                     stash=stash)
174                #print("defining", test_name)
175                suite.addTest(test)
176
177    return suite
178
179def _hide_model_case_from_nose():
180    # type: () -> type
181    class ModelTestCase(unittest.TestCase):
182        """
183        Test suit for a particular model with a particular kernel driver.
184
185        The test suite runs a simple smoke test to make sure the model
186        functions, then runs the list of tests at the bottom of the model
187        description file.
188        """
189        def __init__(self, test_name, model_info, test_method_name,
190                     platform, dtype, stash):
191            # type: (str, ModelInfo, str, str, DType, List[Any]) -> None
192            self.test_name = test_name
193            self.info = model_info
194            self.platform = platform
195            self.dtype = dtype
196            self.stash = stash  # container for the results of the first run
197
198            setattr(self, test_method_name, self.run_all)
199            unittest.TestCase.__init__(self, test_method_name)
200
201        def run_all(self):
202            # type: () -> None
203            """
204            Run all the tests in the test suite, including smoke tests.
205            """
206            smoke_tests = [
207                # test validity at reasonable values
208                ({}, 0.1, None),
209                ({}, (0.1, 0.1), None),
210                # test validity at q = 0
211                #({}, 0.0, None),
212                #({}, (0.0, 0.0), None),
213                # test vector form
214                ({}, [0.001, 0.01, 0.1], [None]*3),
215                ({}, [(0.1, 0.1)]*2, [None]*2),
216                # test that ER/VR will run if they exist
217                ({}, 0.1, None, None, None, None, None),
218                ]
219            tests = smoke_tests
220            #tests = []
221            if self.info.tests is not None:
222                tests += self.info.tests
223            try:
224                model = build_model(self.info, dtype=self.dtype,
225                                    platform=self.platform)
226                results = [self.run_one(model, test) for test in tests]
227                if self.stash:
228                    for test, target, actual in zip(tests, self.stash[0], results):
229                        assert np.all(abs(target-actual) < 5e-5*abs(actual)), \
230                            ("GPU/CPU comparison expected %s but got %s for %s"
231                             % (target, actual, test[0]))
232                else:
233                    self.stash.append(results)
234
235                # Check for missing tests.  Only do so for the "dll" tests
236                # to reduce noise from both opencl and cuda, and because
237                # python kernels use platform="dll".
238                if self.platform == "dll":
239                    missing = []
240                    ## Uncomment the following to require test cases
241                    #missing = self._find_missing_tests()
242                    if missing:
243                        raise ValueError("Missing tests for "+", ".join(missing))
244
245            except:
246                annotate_exception(self.test_name)
247                raise
248
249        def _find_missing_tests(self):
250            # type: () -> None
251            """make sure there are 1D, 2D, ER and VR tests as appropriate"""
252            model_has_VR = callable(self.info.VR)
253            model_has_ER = callable(self.info.ER)
254            model_has_1D = True
255            model_has_2D = any(p.type == 'orientation'
256                               for p in self.info.parameters.kernel_parameters)
257
258            # Lists of tests that have a result that is not None
259            single = [test for test in self.info.tests
260                      if not isinstance(test[2], list) and test[2] is not None]
261            tests_has_VR = any(test[1] == 'VR' for test in single)
262            tests_has_ER = any(test[1] == 'ER' for test in single)
263            tests_has_1D_single = any(isinstance(test[1], float) for test in single)
264            tests_has_2D_single = any(isinstance(test[1], tuple) for test in single)
265
266            multiple = [test for test in self.info.tests
267                        if isinstance(test[2], list)
268                        and not all(result is None for result in test[2])]
269            tests_has_1D_multiple = any(isinstance(test[1][0], float)
270                                        for test in multiple)
271            tests_has_2D_multiple = any(isinstance(test[1][0], tuple)
272                                        for test in multiple)
273
274            missing = []
275            if model_has_VR and not tests_has_VR:
276                missing.append("VR")
277            if model_has_ER and not tests_has_ER:
278                missing.append("ER")
279            if model_has_1D and not (tests_has_1D_single or tests_has_1D_multiple):
280                missing.append("1D")
281            if model_has_2D and not (tests_has_2D_single or tests_has_2D_multiple):
282                missing.append("2D")
283
284            return missing
285
286        def run_one(self, model, test):
287            # type: (KernelModel, TestCondition) -> None
288            """Run a single test case."""
289            user_pars, x, y = test[:3]
290            pars = expand_pars(self.info.parameters, user_pars)
291            invalid = invalid_pars(self.info.parameters, pars)
292            if invalid:
293                raise ValueError("Unknown parameters in test: " + ", ".join(invalid))
294
295            if not isinstance(y, list):
296                y = [y]
297            if not isinstance(x, list):
298                x = [x]
299
300            self.assertEqual(len(y), len(x))
301
302            if isinstance(x[0], tuple):
303                qx, qy = zip(*x)
304                q_vectors = [np.array(qx), np.array(qy)]
305            else:
306                q_vectors = [np.array(x)]
307
308            kernel = model.make_kernel(q_vectors)
309            if len(test) == 3:
310                actual = call_kernel(kernel, pars)
311                self._check_vectors(x, y, actual, 'I')
312                return actual
313            else:
314                y1 = y
315                y2 = test[3] if not isinstance(test[3], list) else [test[3]]
316                F1, F2, R_eff, volume, volume_ratio = call_Fq(kernel, pars)
317                if F1 is not None:  # F1 is none for models with Iq instead of Fq
318                    self._check_vectors(x, y1, F1, 'F')
319                self._check_vectors(x, y2, F2, 'F^2')
320                self._check_scalar(test[4], R_eff, 'R_eff')
321                self._check_scalar(test[5], volume, 'volume')
322                self._check_scalar(test[6], volume_ratio, 'form:shell ratio')
323                return F2
324
325        def _check_scalar(self, target, actual, name):
326            if target is None:
327                # smoke test --- make sure it runs and produces a value
328                self.assertTrue(not np.isnan(actual),
329                                'invalid %s: %s' % (name, actual))
330            elif np.isnan(target):
331                # make sure nans match
332                self.assertTrue(np.isnan(actual),
333                                '%s: expected:%s; actual:%s'
334                                % (name, target, actual))
335            else:
336                # is_near does not work for infinite values, so also test
337                # for exact values.
338                self.assertTrue(target == actual or is_near(target, actual, 5),
339                                '%s: expected:%s; actual:%s'
340                                % (name, target, actual))
341
342        def _check_vectors(self, x, target, actual, name='I'):
343            self.assertTrue(len(actual) > 0,
344                            '%s(...) expected return'%name)
345            if target is None:
346                return
347            self.assertEqual(len(target), len(actual),
348                             '%s(...) returned wrong length'%name)
349            for xi, yi, actual_yi in zip(x, target, actual):
350                if yi is None:
351                    # smoke test --- make sure it runs and produces a value
352                    self.assertTrue(not np.isnan(actual_yi),
353                                    'invalid %s(%s): %s' % (name, xi, actual_yi))
354                elif np.isnan(yi):
355                    # make sure nans match
356                    self.assertTrue(np.isnan(actual_yi),
357                                    '%s(%s): expected:%s; actual:%s'
358                                    % (name, xi, yi, actual_yi))
359                else:
360                    # is_near does not work for infinite values, so also test
361                    # for exact values.
362                    self.assertTrue(yi == actual_yi or is_near(yi, actual_yi, 5),
363                                    '%s(%s); expected:%s; actual:%s'
364                                    % (name, xi, yi, actual_yi))
365
366    return ModelTestCase
367
368def invalid_pars(partable, pars):
369    # type: (ParameterTable, Dict[str, float])
370    """
371    Return a list of parameter names that are not part of the model.
372    """
373    names = set(p.id for p in partable.call_parameters)
374    invalid = []
375    for par in sorted(pars.keys()):
376        # special handling of R_eff mode, which is not a usual parameter
377        if par == 'radius_effective_type':
378            continue
379        parts = par.split('_pd')
380        if len(parts) > 1 and parts[1] not in ("", "_n", "nsigma", "type"):
381            invalid.append(par)
382            continue
383        if parts[0] not in names:
384            invalid.append(par)
385    return invalid
386
387
388def is_near(target, actual, digits=5):
389    # type: (float, float, int) -> bool
390    """
391    Returns true if *actual* is within *digits* significant digits of *target*.
392    """
393    import math
394    shift = 10**math.ceil(math.log10(abs(target)))
395    return abs(target-actual)/shift < 1.5*10**-digits
396
397def run_one(model):
398    # type: (str) -> str
399    """
400    Run the tests for a single model, printing the results to stdout.
401
402    *model* can by a python file, which is handy for checking user defined
403    plugin models.
404    """
405    # Note that running main() directly did not work from within the
406    # wxPython pycrust console.  Instead of the results appearing in the
407    # window they were printed to the underlying console.
408    from unittest.runner import TextTestResult, _WritelnDecorator
409
410    # Build a object to capture and print the test results
411    stream = _WritelnDecorator(StringIO())  # Add writeln() method to stream
412    verbosity = 2
413    descriptions = True
414    result = TextTestResult(stream, descriptions, verbosity)
415
416    # Build a test suite containing just the model
417    loader = 'opencl' if use_opencl() else 'cuda' if use_cuda() else 'dll'
418    models = [model]
419    try:
420        suite = make_suite([loader], models)
421    except Exception:
422        import traceback
423        stream.writeln(traceback.format_exc())
424        return
425
426    # Warn if there are no user defined tests.
427    # Note: the test suite constructed above only has one test in it, which
428    # runs through some smoke tests to make sure the model runs, then runs
429    # through the input-output pairs given in the model definition file.  To
430    # check if any such pairs are defined, therefore, we just need to check if
431    # they are in the first test of the test suite.  We do this with an
432    # iterator since we don't have direct access to the list of tests in the
433    # test suite.
434    # In Qt5 suite.run() will clear all tests in the suite after running
435    # with no way of retaining them for the test below, so let's check
436    # for user tests before running the suite.
437    for test in suite:
438        if not test.info.tests:
439            stream.writeln("Note: %s has no user defined tests."%model)
440        break
441    else:
442        stream.writeln("Note: no test suite created --- this should never happen")
443
444    # Run the test suite
445    suite.run(result)
446
447    # Print the failures and errors
448    for _, tb in result.errors:
449        stream.writeln(tb)
450    for _, tb in result.failures:
451        stream.writeln(tb)
452
453    output = stream.getvalue()
454    stream.close()
455    return output
456
457
458def main(*models):
459    # type: (*str) -> int
460    """
461    Run tests given is models.
462
463    Returns 0 if success or 1 if any tests fail.
464    """
465    try:
466        from xmlrunner import XMLTestRunner as TestRunner
467        test_args = {'output': 'logs'}
468    except ImportError:
469        from unittest import TextTestRunner as TestRunner
470        test_args = {}
471
472    if models and models[0] == '-v':
473        verbosity = 2
474        models = models[1:]
475    else:
476        verbosity = 1
477    if models and models[0] == 'opencl':
478        if not use_opencl():
479            print("opencl is not available")
480            return 1
481        loaders = ['opencl']
482        models = models[1:]
483    elif models and models[0] == 'cuda':
484        if not use_cuda():
485            print("cuda is not available")
486            return 1
487        loaders = ['cuda']
488        models = models[1:]
489    elif models and models[0] == 'dll':
490        # TODO: test if compiler is available?
491        loaders = ['dll']
492        models = models[1:]
493    else:
494        loaders = ['dll']
495        if use_opencl():
496            loaders.append('opencl')
497        if use_cuda():
498            loaders.append('cuda')
499    if not models:
500        print("""\
501usage:
502  python -m sasmodels.model_test [-v] [opencl|cuda|dll] model1 model2 ...
503
504If -v is included on the command line, then use verbose output.
505
506If no platform is specified, then models will be tested with dll, and
507if available, OpenCL and CUDA; the compute target is ignored for pure python models.
508
509If model1 is 'all', then all except the remaining models will be tested.
510
511""")
512
513        return 1
514
515    runner = TestRunner(verbosity=verbosity, **test_args)
516    result = runner.run(make_suite(loaders, models))
517    return 1 if result.failures or result.errors else 0
518
519
520def model_tests():
521    # type: () -> Iterator[Callable[[], None]]
522    """
523    Test runner visible to nosetests.
524
525    Run "nosetests sasmodels" on the command line to invoke it.
526    """
527    loaders = ['dll']
528    if use_opencl():
529        loaders.append('opencl')
530    if use_cuda():
531        loaders.append('cuda')
532    tests = make_suite(loaders, ['all'])
533    def build_test(test):
534        # In order for nosetest to show the test name, wrap the test.run_all
535        # instance in function that takes the test name as a parameter which
536        # will be displayed when the test is run.  Do this as a function so
537        # that it properly captures the context for tests that captured and
538        # run later.  If done directly in the for loop, then the looping
539        # variable test will be shared amongst all the tests, and we will be
540        # repeatedly testing vesicle.
541
542        # Note: in sasview sas.sasgui.perspectives.fitting.gpu_options
543        # requires that the test.description field be set.
544        wrap = lambda: test.run_all()
545        wrap.description = test.test_name
546        return wrap
547        # The following would work with nosetests and pytest:
548        #     return lambda name: test.run_all(), test.test_name
549
550    for test in tests:
551        yield build_test(test)
552
553
554if __name__ == "__main__":
555    sys.exit(main(*sys.argv[1:]))
Note: See TracBrowser for help on using the repository browser.