source: sasmodels/compare_many.py @ 34d49af

core_shell_microgelscostrafo411magnetic_modelrelease_v0.94release_v0.95ticket-1257-vesicle-productticket_1156ticket_1265_superballticket_822_more_unit_tests
Last change on this file since 34d49af was 216a9e1, checked in by Paul Kienzle <pkienzle@…>, 10 years ago

batch generate random old/new comparisons across all models

  • Property mode set to 100755
File size: 4.1 KB
Line 
1#!/usr/bin/env python
2
3import sys
4
5import numpy as np
6
7from sasmodels.gpu import environment
8from compare import (MODELS, randomize_model, suppress_pd, eval_sasview,
9                     eval_opencl, eval_ctypes, make_data)
10
11def get_stats(target, model_eval, name, pars, data, index, dtype, cutoff):
12    value, _ = model_eval(name, pars, data, dtype='single', cutoff=cutoff)
13    resid = abs(value-target)[index]
14    relerr = resid/target[index]
15    srel = np.argsort(relerr)
16    p90 = int(len(relerr)*0.90)
17    p95 = int(len(relerr)*0.95)
18    maxrel = np.max(relerr)
19    rel95 = relerr[srel[p95]]
20    maxabs = np.max(resid[srel[p95:]])
21    maxval = np.max(value[srel[p95:]])
22    return maxrel,rel95,maxabs,maxval
23
24def print_column_headers(pars, parts):
25    stats = list('Max rel err|95% rel err|Max abs err above 90% rel|Max value above 90% rel'.split('|'))
26    groups = ['']
27    for p in parts:
28        groups.append(p)
29        groups.extend(['']*(len(stats)-1))
30    columns = ['Seed'] + stats*len(parts) +  list(sorted(pars.keys()))
31    print(','.join('"%s"'%c for c in groups))
32    print(','.join('"%s"'%c for c in columns))
33
34def compare_instance(model, data, index, N=1, mono=True, cutoff=1e-5):
35    name, pars = MODELS[model]()
36    header = '\n"Model","%s","Count","%d"'%(name, N)
37    if not mono: header += ',"Cutoff",%g'%(cutoff,)
38    print(header)
39    first = True
40    for _ in range(N):
41        pars, seed = randomize_model(name, pars)
42        if mono: suppress_pd(pars)
43
44        target, _ = eval_sasview(name, pars, data)
45
46        env = environment()
47        gpu_single = get_stats(target, eval_opencl, name, pars, data, index, 'single', cutoff)
48        if env.has_double:
49            gpu_double = get_stats(target, eval_opencl, name, pars, data, index, 'double', cutoff)
50        else:
51            gpu_double = [0]*len(gpu_single)
52        cpu_double = get_stats(target, eval_ctypes, name, pars, data, index, 'double', cutoff)
53
54        values = list(gpu_single) + list(gpu_double) + list(cpu_double) + [v for _,v in sorted(pars.items())]
55        if gpu_single[0] > 5e-5:
56            if first:
57                print_column_headers(pars,'GPU single|GPU double|CPU double'.split('|'))
58                first = False
59            print(("%d,"%seed)+','.join("%g"%v for v in values))
60
61def main():
62    try:
63        model = sys.argv[1]
64        assert (model in MODELS) or (model == "all")
65        count = int(sys.argv[2])
66        is2D = sys.argv[3].startswith('2d')
67        assert sys.argv[3][1] == 'd'
68        Nq = int(sys.argv[3][2:])
69        mono = sys.argv[4] == 'mono'
70        cutoff = float(sys.argv[4]) if not mono else 0
71    except:
72        import traceback; traceback.print_exc()
73        models = "\n    ".join("%-7s: %s"%(k,v.__name__.replace('_',' '))
74                               for k,v in sorted(MODELS.items()))
75        print("""\
76usage: compare_many.py MODEL COUNT (1dNQ|2dNQ) (CUTOFF|mono)
77
78MODEL is the model name of the model, which is one of:
79    %s
80or "all" for all the models in alphabetical order.
81
82COUNT is the number of randomly generated parameter sets to try. A value
83of "10000" is a reasonable check for monodisperse models, or "100" for
84polydisperse models.   For a quick check, use "100" and "5" respectively.
85
86NQ is the number of Q values to calculate.  If it starts with "1d", then
87it is a 1-dimensional problem, with log spaced Q points from 1e-3 to 1.0.
88If it starts with "2d" then it is a 2-dimensional problem, with linearly
89spaced points Q points from -1.0 to 1.0 in each dimension. The usual
90values are "1d100" for 1-D and "2d32" for 2-D.
91
92CUTOFF is the cutoff value to use for the polydisperse distribution. Weights
93below the cutoff will be ignored.  Use "mono" for monodisperse models.  The
94choice of polydisperse parameters, and the number of points in the distribution
95is set in compare.py defaults for each model.
96"""%(models,))
97        sys.exit(1)
98
99    data, index = make_data(qmax=1.0, is2D=is2D, Nq=Nq)
100    model_list = [model] if model != "all" else list(sorted(MODELS.keys()))
101    for model in model_list:
102        compare_instance(model, data, index, N=count, mono=mono, cutoff=cutoff)
103
104if __name__ == "__main__":
105    main()
Note: See TracBrowser for help on using the repository browser.