source: sasmodels/sasmodels/model_test.py @ 1a8b91c

Last change on this file since 1a8b91c was 9b7fac6, checked in by Piotr Rozyczko <rozyczko@…>, 6 years ago

Shuffle tests around to circumvent Qt5 unittest issue

  • Property mode set to 100644
File size: 18.9 KB
Line 
1# -*- coding: utf-8 -*-
2"""
3Run model unit tests.
4
5Usage::
6
7    python -m sasmodels.model_test [opencl|dll|opencl_and_dll] model1 model2 ...
8
9    if model1 is 'all', then all except the remaining models will be tested
10
11Each model is tested using the default parameters at q=0.1, (qx, qy)=(0.1, 0.1),
12and the ER and VR are computed.  The return values at these points are not
13considered.  The test is only to verify that the models run to completion,
14and do not produce inf or NaN.
15
16Tests are defined with the *tests* attribute in the model.py file.  *tests*
17is a list of individual tests to run, where each test consists of the
18parameter values for the test, the q-values and the expected results.  For
19the effective radius test, the q-value should be 'ER'.  For the VR test,
20the q-value should be 'VR'.  For 1-D tests, either specify the q value or
21a list of q-values, and the corresponding I(q) value, or list of I(q) values.
22
23That is::
24
25    tests = [
26        [ {parameters}, q, I(q)],
27        [ {parameters}, [q], [I(q)] ],
28        [ {parameters}, [q1, q2, ...], [I(q1), I(q2), ...]],
29
30        [ {parameters}, (qx, qy), I(qx, Iqy)],
31        [ {parameters}, [(qx1, qy1), (qx2, qy2), ...],
32                        [I(qx1, qy1), I(qx2, qy2), ...]],
33
34        [ {parameters}, 'ER', ER(pars) ],
35        [ {parameters}, 'VR', VR(pars) ],
36        ...
37    ]
38
39Parameters are *key:value* pairs, where key is one of the parameters of the
40model and value is the value to use for the test.  Any parameters not given
41in the parameter list will take on the default parameter value.
42
43Precision defaults to 5 digits (relative).
44"""
45from __future__ import print_function
46
47import sys
48import unittest
49
50try:
51    from StringIO import StringIO
52except ImportError:
53    # StringIO.StringIO renamed to io.StringIO in Python 3
54    # Note: io.StringIO exists in python 2, but using unicode instead of str
55    from io import StringIO
56
57import numpy as np  # type: ignore
58
59from . import core
60from .core import list_models, load_model_info, build_model
61from .direct_model import call_kernel, call_ER, call_VR
62from .exception import annotate_exception
63from .modelinfo import expand_pars
64from .kernelcl import use_opencl
65
66# pylint: disable=unused-import
67try:
68    from typing import List, Iterator, Callable
69except ImportError:
70    pass
71else:
72    from .modelinfo import ParameterTable, ParameterSet, TestCondition, ModelInfo
73    from .kernel import KernelModel
74# pylint: enable=unused-import
75
76
77def make_suite(loaders, models):
78    # type: (List[str], List[str]) -> unittest.TestSuite
79    """
80    Construct the pyunit test suite.
81
82    *loaders* is the list of kernel drivers to use, which is one of
83    *["dll", "opencl"]*, *["dll"]* or *["opencl"]*.  For python models,
84    the python driver is always used.
85
86    *models* is the list of models to test, or *["all"]* to test all models.
87    """
88    ModelTestCase = _hide_model_case_from_nose()
89    suite = unittest.TestSuite()
90
91    if models[0] in core.KINDS:
92        skip = models[1:]
93        models = list_models(models[0])
94    else:
95        skip = []
96    for model_name in models:
97        if model_name in skip:
98            continue
99        model_info = load_model_info(model_name)
100
101        #print('------')
102        #print('found tests in', model_name)
103        #print('------')
104
105        # if ispy then use the dll loader to call pykernel
106        # don't try to call cl kernel since it will not be
107        # available in some environmentes.
108        is_py = callable(model_info.Iq)
109
110        # Some OpenCL drivers seem to be flaky, and are not producing the
111        # expected result.  Since we don't have known test values yet for
112        # all of our models, we are instead going to compare the results
113        # for the 'smoke test' (that is, evaluation at q=0.1 for the default
114        # parameters just to see that the model runs to completion) between
115        # the OpenCL and the DLL.  To do this, we define a 'stash' which is
116        # shared between OpenCL and DLL tests.  This is just a list.  If the
117        # list is empty (which it will be when DLL runs, if the DLL runs
118        # first), then the results are appended to the list.  If the list
119        # is not empty (which it will be when OpenCL runs second), the results
120        # are compared to the results stored in the first element of the list.
121        # This is a horrible stateful hack which only makes sense because the
122        # test suite is thrown away after being run once.
123        stash = []
124
125        if is_py:  # kernel implemented in python
126            test_name = "%s-python"%model_name
127            test_method_name = "test_%s_python" % model_info.id
128            test = ModelTestCase(test_name, model_info,
129                                 test_method_name,
130                                 platform="dll",  # so that
131                                 dtype="double",
132                                 stash=stash)
133            suite.addTest(test)
134        else:   # kernel implemented in C
135
136            # test using dll if desired
137            if 'dll' in loaders or not use_opencl():
138                test_name = "%s-dll"%model_name
139                test_method_name = "test_%s_dll" % model_info.id
140                test = ModelTestCase(test_name, model_info,
141                                     test_method_name,
142                                     platform="dll",
143                                     dtype="double",
144                                     stash=stash)
145                suite.addTest(test)
146
147            # test using opencl if desired and available
148            if 'opencl' in loaders and use_opencl():
149                test_name = "%s-opencl"%model_name
150                test_method_name = "test_%s_opencl" % model_info.id
151                # Using dtype=None so that the models that are only
152                # correct for double precision are not tested using
153                # single precision.  The choice is determined by the
154                # presence of *single=False* in the model file.
155                test = ModelTestCase(test_name, model_info,
156                                     test_method_name,
157                                     platform="ocl", dtype=None,
158                                     stash=stash)
159                #print("defining", test_name)
160                suite.addTest(test)
161
162    return suite
163
164def _hide_model_case_from_nose():
165    # type: () -> type
166    class ModelTestCase(unittest.TestCase):
167        """
168        Test suit for a particular model with a particular kernel driver.
169
170        The test suite runs a simple smoke test to make sure the model
171        functions, then runs the list of tests at the bottom of the model
172        description file.
173        """
174        def __init__(self, test_name, model_info, test_method_name,
175                     platform, dtype, stash):
176            # type: (str, ModelInfo, str, str, DType, List[Any]) -> None
177            self.test_name = test_name
178            self.info = model_info
179            self.platform = platform
180            self.dtype = dtype
181            self.stash = stash  # container for the results of the first run
182
183            setattr(self, test_method_name, self.run_all)
184            unittest.TestCase.__init__(self, test_method_name)
185
186        def run_all(self):
187            # type: () -> None
188            """
189            Run all the tests in the test suite, including smoke tests.
190            """
191            smoke_tests = [
192                # test validity at reasonable values
193                ({}, 0.1, None),
194                ({}, (0.1, 0.1), None),
195                # test validity at q = 0
196                #({}, 0.0, None),
197                #({}, (0.0, 0.0), None),
198                # test vector form
199                ({}, [0.001, 0.01, 0.1], [None]*3),
200                ({}, [(0.1, 0.1)]*2, [None]*2),
201                # test that ER/VR will run if they exist
202                ({}, 'ER', None),
203                ({}, 'VR', None),
204                ]
205            tests = smoke_tests
206            #tests = []
207            if self.info.tests is not None:
208                tests += self.info.tests
209            try:
210                model = build_model(self.info, dtype=self.dtype,
211                                    platform=self.platform)
212                results = [self.run_one(model, test) for test in tests]
213                if self.stash:
214                    for test, target, actual in zip(tests, self.stash[0], results):
215                        assert np.all(abs(target-actual) < 5e-5*abs(actual)), \
216                            ("GPU/CPU comparison expected %s but got %s for %s"
217                             % (target, actual, test[0]))
218                else:
219                    self.stash.append(results)
220
221                # Check for missing tests.  Only do so for the "dll" tests
222                # to reduce noise from both opencl and dll, and because
223                # python kernels use platform="dll".
224                if self.platform == "dll":
225                    missing = []
226                    ## Uncomment the following to require test cases
227                    #missing = self._find_missing_tests()
228                    if missing:
229                        raise ValueError("Missing tests for "+", ".join(missing))
230
231            except:
232                annotate_exception(self.test_name)
233                raise
234
235        def _find_missing_tests(self):
236            # type: () -> None
237            """make sure there are 1D, 2D, ER and VR tests as appropriate"""
238            model_has_VR = callable(self.info.VR)
239            model_has_ER = callable(self.info.ER)
240            model_has_1D = True
241            model_has_2D = any(p.type == 'orientation'
242                               for p in self.info.parameters.kernel_parameters)
243
244            # Lists of tests that have a result that is not None
245            single = [test for test in self.info.tests
246                      if not isinstance(test[2], list) and test[2] is not None]
247            tests_has_VR = any(test[1] == 'VR' for test in single)
248            tests_has_ER = any(test[1] == 'ER' for test in single)
249            tests_has_1D_single = any(isinstance(test[1], float) for test in single)
250            tests_has_2D_single = any(isinstance(test[1], tuple) for test in single)
251
252            multiple = [test for test in self.info.tests
253                        if isinstance(test[2], list)
254                        and not all(result is None for result in test[2])]
255            tests_has_1D_multiple = any(isinstance(test[1][0], float)
256                                        for test in multiple)
257            tests_has_2D_multiple = any(isinstance(test[1][0], tuple)
258                                        for test in multiple)
259
260            missing = []
261            if model_has_VR and not tests_has_VR:
262                missing.append("VR")
263            if model_has_ER and not tests_has_ER:
264                missing.append("ER")
265            if model_has_1D and not (tests_has_1D_single or tests_has_1D_multiple):
266                missing.append("1D")
267            if model_has_2D and not (tests_has_2D_single or tests_has_2D_multiple):
268                missing.append("2D")
269
270            return missing
271
272        def run_one(self, model, test):
273            # type: (KernelModel, TestCondition) -> None
274            """Run a single test case."""
275            user_pars, x, y = test
276            pars = expand_pars(self.info.parameters, user_pars)
277            invalid = invalid_pars(self.info.parameters, pars)
278            if invalid:
279                raise ValueError("Unknown parameters in test: " + ", ".join(invalid))
280
281            if not isinstance(y, list):
282                y = [y]
283            if not isinstance(x, list):
284                x = [x]
285
286            self.assertEqual(len(y), len(x))
287
288            if x[0] == 'ER':
289                actual = np.array([call_ER(model.info, pars)])
290            elif x[0] == 'VR':
291                actual = np.array([call_VR(model.info, pars)])
292            elif isinstance(x[0], tuple):
293                qx, qy = zip(*x)
294                q_vectors = [np.array(qx), np.array(qy)]
295                kernel = model.make_kernel(q_vectors)
296                actual = call_kernel(kernel, pars)
297            else:
298                q_vectors = [np.array(x)]
299                kernel = model.make_kernel(q_vectors)
300                actual = call_kernel(kernel, pars)
301
302            self.assertTrue(len(actual) > 0)
303            self.assertEqual(len(y), len(actual))
304
305            for xi, yi, actual_yi in zip(x, y, actual):
306                if yi is None:
307                    # smoke test --- make sure it runs and produces a value
308                    self.assertTrue(not np.isnan(actual_yi),
309                                    'invalid f(%s): %s' % (xi, actual_yi))
310                elif np.isnan(yi):
311                    self.assertTrue(np.isnan(actual_yi),
312                                    'f(%s): expected:%s; actual:%s'
313                                    % (xi, yi, actual_yi))
314                else:
315                    # is_near does not work for infinite values, so also test
316                    # for exact values.  Note that this will not
317                    self.assertTrue(yi == actual_yi or is_near(yi, actual_yi, 5),
318                                    'f(%s); expected:%s; actual:%s'
319                                    % (xi, yi, actual_yi))
320            return actual
321
322    return ModelTestCase
323
324def invalid_pars(partable, pars):
325    # type: (ParameterTable, Dict[str, float])
326    """
327    Return a list of parameter names that are not part of the model.
328    """
329    names = set(p.id for p in partable.call_parameters)
330    invalid = []
331    for par in sorted(pars.keys()):
332        parts = par.split('_pd')
333        if len(parts) > 1 and parts[1] not in ("", "_n", "nsigma", "type"):
334            invalid.append(par)
335            continue
336        if parts[0] not in names:
337            invalid.append(par)
338    return invalid
339
340
341def is_near(target, actual, digits=5):
342    # type: (float, float, int) -> bool
343    """
344    Returns true if *actual* is within *digits* significant digits of *target*.
345    """
346    import math
347    shift = 10**math.ceil(math.log10(abs(target)))
348    return abs(target-actual)/shift < 1.5*10**-digits
349
350def run_one(model):
351    # type: (str) -> str
352    """
353    Run the tests for a single model, printing the results to stdout.
354
355    *model* can by a python file, which is handy for checking user defined
356    plugin models.
357    """
358    # Note that running main() directly did not work from within the
359    # wxPython pycrust console.  Instead of the results appearing in the
360    # window they were printed to the underlying console.
361    from unittest.runner import TextTestResult, _WritelnDecorator
362
363    # Build a object to capture and print the test results
364    stream = _WritelnDecorator(StringIO())  # Add writeln() method to stream
365    verbosity = 2
366    descriptions = True
367    result = TextTestResult(stream, descriptions, verbosity)
368
369    # Build a test suite containing just the model
370    loaders = ['opencl'] if use_opencl() else ['dll']
371    models = [model]
372    try:
373        suite = make_suite(loaders, models)
374    except Exception:
375        import traceback
376        stream.writeln(traceback.format_exc())
377        return
378
379    # Warn if there are no user defined tests.
380    # Note: the test suite constructed above only has one test in it, which
381    # runs through some smoke tests to make sure the model runs, then runs
382    # through the input-output pairs given in the model definition file.  To
383    # check if any such pairs are defined, therefore, we just need to check if
384    # they are in the first test of the test suite.  We do this with an
385    # iterator since we don't have direct access to the list of tests in the
386    # test suite.
387    # In Qt5 suite.run() will clear all tests in the suite after running
388    # with no way of retaining them for the test below, so let's check
389    # for user tests before running the suite.
390    for test in suite:
391        if not test.info.tests:
392            stream.writeln("Note: %s has no user defined tests."%model)
393        break
394    else:
395        stream.writeln("Note: no test suite created --- this should never happen")
396
397    # Run the test suite
398    suite.run(result)
399
400    # Print the failures and errors
401    for _, tb in result.errors:
402        stream.writeln(tb)
403    for _, tb in result.failures:
404        stream.writeln(tb)
405
406    output = stream.getvalue()
407    stream.close()
408    return output
409
410
411def main(*models):
412    # type: (*str) -> int
413    """
414    Run tests given is models.
415
416    Returns 0 if success or 1 if any tests fail.
417    """
418    try:
419        from xmlrunner import XMLTestRunner as TestRunner
420        test_args = {'output': 'logs'}
421    except ImportError:
422        from unittest import TextTestRunner as TestRunner
423        test_args = {}
424
425    if models and models[0] == '-v':
426        verbosity = 2
427        models = models[1:]
428    else:
429        verbosity = 1
430    if models and models[0] == 'opencl':
431        if not use_opencl():
432            print("opencl is not available")
433            return 1
434        loaders = ['opencl']
435        models = models[1:]
436    elif models and models[0] == 'dll':
437        # TODO: test if compiler is available?
438        loaders = ['dll']
439        models = models[1:]
440    elif models and models[0] == 'opencl_and_dll':
441        loaders = ['opencl', 'dll'] if use_opencl() else ['dll']
442        models = models[1:]
443    else:
444        loaders = ['opencl', 'dll'] if use_opencl() else ['dll']
445    if not models:
446        print("""\
447usage:
448  python -m sasmodels.model_test [-v] [opencl|dll] model1 model2 ...
449
450If -v is included on the command line, then use verbose output.
451
452If neither opencl nor dll is specified, then models will be tested with
453both OpenCL and dll; the compute target is ignored for pure python models.
454
455If model1 is 'all', then all except the remaining models will be tested.
456
457""")
458
459        return 1
460
461    runner = TestRunner(verbosity=verbosity, **test_args)
462    result = runner.run(make_suite(loaders, models))
463    return 1 if result.failures or result.errors else 0
464
465
466def model_tests():
467    # type: () -> Iterator[Callable[[], None]]
468    """
469    Test runner visible to nosetests.
470
471    Run "nosetests sasmodels" on the command line to invoke it.
472    """
473    loaders = ['opencl', 'dll'] if use_opencl() else ['dll']
474    tests = make_suite(loaders, ['all'])
475    def build_test(test):
476        # In order for nosetest to show the test name, wrap the test.run_all
477        # instance in function that takes the test name as a parameter which
478        # will be displayed when the test is run.  Do this as a function so
479        # that it properly captures the context for tests that captured and
480        # run later.  If done directly in the for loop, then the looping
481        # variable test will be shared amongst all the tests, and we will be
482        # repeatedly testing vesicle.
483
484        # Note: in sasview sas.sasgui.perspectives.fitting.gpu_options
485        # requires that the test.description field be set.
486        wrap = lambda: test.run_all()
487        wrap.description = test.test_name
488        return wrap
489        # The following would work with nosetests and pytest:
490        #     return lambda name: test.run_all(), test.test_name
491
492    for test in tests:
493        yield build_test(test)
494
495
496if __name__ == "__main__":
497    sys.exit(main(*sys.argv[1:]))
Note: See TracBrowser for help on using the repository browser.