[216a9e1] | 1 | #!/usr/bin/env python |
---|
| 2 | |
---|
| 3 | import sys |
---|
| 4 | |
---|
| 5 | import numpy as np |
---|
| 6 | |
---|
| 7 | from sasmodels.gpu import environment |
---|
| 8 | from compare import (MODELS, randomize_model, suppress_pd, eval_sasview, |
---|
| 9 | eval_opencl, eval_ctypes, make_data) |
---|
| 10 | |
---|
| 11 | def get_stats(target, model_eval, name, pars, data, index, dtype, cutoff): |
---|
| 12 | value, _ = model_eval(name, pars, data, dtype='single', cutoff=cutoff) |
---|
| 13 | resid = abs(value-target)[index] |
---|
| 14 | relerr = resid/target[index] |
---|
| 15 | srel = np.argsort(relerr) |
---|
| 16 | p90 = int(len(relerr)*0.90) |
---|
| 17 | p95 = int(len(relerr)*0.95) |
---|
| 18 | maxrel = np.max(relerr) |
---|
| 19 | rel95 = relerr[srel[p95]] |
---|
| 20 | maxabs = np.max(resid[srel[p95:]]) |
---|
| 21 | maxval = np.max(value[srel[p95:]]) |
---|
| 22 | return maxrel,rel95,maxabs,maxval |
---|
| 23 | |
---|
| 24 | def print_column_headers(pars, parts): |
---|
| 25 | stats = list('Max rel err|95% rel err|Max abs err above 90% rel|Max value above 90% rel'.split('|')) |
---|
| 26 | groups = [''] |
---|
| 27 | for p in parts: |
---|
| 28 | groups.append(p) |
---|
| 29 | groups.extend(['']*(len(stats)-1)) |
---|
| 30 | columns = ['Seed'] + stats*len(parts) + list(sorted(pars.keys())) |
---|
| 31 | print(','.join('"%s"'%c for c in groups)) |
---|
| 32 | print(','.join('"%s"'%c for c in columns)) |
---|
| 33 | |
---|
| 34 | def compare_instance(model, data, index, N=1, mono=True, cutoff=1e-5): |
---|
| 35 | name, pars = MODELS[model]() |
---|
| 36 | header = '\n"Model","%s","Count","%d"'%(name, N) |
---|
| 37 | if not mono: header += ',"Cutoff",%g'%(cutoff,) |
---|
| 38 | print(header) |
---|
| 39 | first = True |
---|
| 40 | for _ in range(N): |
---|
| 41 | pars, seed = randomize_model(name, pars) |
---|
| 42 | if mono: suppress_pd(pars) |
---|
| 43 | |
---|
| 44 | target, _ = eval_sasview(name, pars, data) |
---|
| 45 | |
---|
| 46 | env = environment() |
---|
| 47 | gpu_single = get_stats(target, eval_opencl, name, pars, data, index, 'single', cutoff) |
---|
| 48 | if env.has_double: |
---|
| 49 | gpu_double = get_stats(target, eval_opencl, name, pars, data, index, 'double', cutoff) |
---|
| 50 | else: |
---|
| 51 | gpu_double = [0]*len(gpu_single) |
---|
| 52 | cpu_double = get_stats(target, eval_ctypes, name, pars, data, index, 'double', cutoff) |
---|
| 53 | |
---|
| 54 | values = list(gpu_single) + list(gpu_double) + list(cpu_double) + [v for _,v in sorted(pars.items())] |
---|
| 55 | if gpu_single[0] > 5e-5: |
---|
| 56 | if first: |
---|
| 57 | print_column_headers(pars,'GPU single|GPU double|CPU double'.split('|')) |
---|
| 58 | first = False |
---|
| 59 | print(("%d,"%seed)+','.join("%g"%v for v in values)) |
---|
| 60 | |
---|
| 61 | def main(): |
---|
| 62 | try: |
---|
| 63 | model = sys.argv[1] |
---|
| 64 | assert (model in MODELS) or (model == "all") |
---|
| 65 | count = int(sys.argv[2]) |
---|
| 66 | is2D = sys.argv[3].startswith('2d') |
---|
| 67 | assert sys.argv[3][1] == 'd' |
---|
| 68 | Nq = int(sys.argv[3][2:]) |
---|
| 69 | mono = sys.argv[4] == 'mono' |
---|
| 70 | cutoff = float(sys.argv[4]) if not mono else 0 |
---|
| 71 | except: |
---|
| 72 | import traceback; traceback.print_exc() |
---|
| 73 | models = "\n ".join("%-7s: %s"%(k,v.__name__.replace('_',' ')) |
---|
| 74 | for k,v in sorted(MODELS.items())) |
---|
| 75 | print("""\ |
---|
| 76 | usage: compare_many.py MODEL COUNT (1dNQ|2dNQ) (CUTOFF|mono) |
---|
| 77 | |
---|
| 78 | MODEL is the model name of the model, which is one of: |
---|
| 79 | %s |
---|
| 80 | or "all" for all the models in alphabetical order. |
---|
| 81 | |
---|
| 82 | COUNT is the number of randomly generated parameter sets to try. A value |
---|
| 83 | of "10000" is a reasonable check for monodisperse models, or "100" for |
---|
| 84 | polydisperse models. For a quick check, use "100" and "5" respectively. |
---|
| 85 | |
---|
| 86 | NQ is the number of Q values to calculate. If it starts with "1d", then |
---|
| 87 | it is a 1-dimensional problem, with log spaced Q points from 1e-3 to 1.0. |
---|
| 88 | If it starts with "2d" then it is a 2-dimensional problem, with linearly |
---|
| 89 | spaced points Q points from -1.0 to 1.0 in each dimension. The usual |
---|
| 90 | values are "1d100" for 1-D and "2d32" for 2-D. |
---|
| 91 | |
---|
| 92 | CUTOFF is the cutoff value to use for the polydisperse distribution. Weights |
---|
| 93 | below the cutoff will be ignored. Use "mono" for monodisperse models. The |
---|
| 94 | choice of polydisperse parameters, and the number of points in the distribution |
---|
| 95 | is set in compare.py defaults for each model. |
---|
| 96 | """%(models,)) |
---|
| 97 | sys.exit(1) |
---|
| 98 | |
---|
| 99 | data, index = make_data(qmax=1.0, is2D=is2D, Nq=Nq) |
---|
| 100 | model_list = [model] if model != "all" else list(sorted(MODELS.keys())) |
---|
| 101 | for model in model_list: |
---|
| 102 | compare_instance(model, data, index, N=count, mono=mono, cutoff=cutoff) |
---|
| 103 | |
---|
| 104 | if __name__ == "__main__": |
---|
| 105 | main() |
---|