source: sasmodels/sasmodels/kerneldll.py @ 7e16db7

core_shell_microgelscostrafo411magnetic_modelrelease_v0.94release_v0.95ticket-1257-vesicle-productticket_1156ticket_1265_superballticket_822_more_unit_tests
Last change on this file since 7e16db7 was 7e16db7, checked in by Paul Kienzle <pkienzle@…>, 6 years ago

Capture errors from compiler

  • Property mode set to 100644
File size: 13.1 KB
Line 
1r"""
2DLL driver for C kernels
3
4The global attribute *ALLOW_SINGLE_PRECISION_DLLS* should be set to *True* if
5you wish to allow single precision floating point evaluation for the compiled
6models, otherwise it defaults to *False*.
7
8The compiler command line is stored in the attribute *COMPILE*, with string
9substitutions for %(source)s and %(output)s indicating what to compile and
10where to store it.  The actual command is system dependent.
11
12On windows systems, you have a choice of compilers.  *MinGW* is the GNU
13compiler toolchain, available in packages such as anaconda and PythonXY,
14or available stand alone. This toolchain has had difficulties on some
15systems, and may or may not work for you.  In order to build DLLs, *gcc*
16must be on your path.  If the environment variable *SAS_OPENMP* is given
17then -fopenmp is added to the compiler flags.  This requires a version
18of MinGW compiled with OpenMP support.
19
20An alternative toolchain uses the Microsoft Visual C++ compiler, available
21free from microsoft:
22
23    `<http://www.microsoft.com/en-us/download/details.aspx?id=44266>`_
24
25Again, this requires that the compiler is available on your path.  This is
26done by running vcvarsall.bat in a windows terminal.  Install locations are
27system dependent, such as:
28
29    C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\vcvarsall.bat
30
31or maybe
32
33    C:\Users\yourname\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python\9.0\vcvarsall.bat
34
35And again, the environment variable *SAS_OPENMP* controls whether OpenMP is
36used to compile the C code.  This requires the Microsoft vcomp90.dll library,
37which doesn't seem to be included with the compiler, nor does there appear
38to be a public download location.  There may be one on your machine already
39in a location such as:
40
41    C:\Windows\winsxs\x86_microsoft.vc90.openmp*\vcomp90.dll
42
43If you copy this onto your path, such as the python directory or the install
44directory for this application, then OpenMP should be supported.
45"""
46from __future__ import print_function
47
48import sys
49import os
50from os.path import join as joinpath, split as splitpath, splitext
51import subprocess
52import tempfile
53import ctypes as ct
54from ctypes import c_void_p, c_int, c_longdouble, c_double, c_float
55import logging
56
57import numpy as np
58
59from . import generate
60from .kernelpy import PyInput, PyModel
61from .exception import annotate_exception
62
63if os.name == 'nt':
64    # Windows compiler; check if TinyCC is available
65    try:
66        import tinycc
67    except ImportError:
68        tinycc = None
69    # call vcvarsall.bat before compiling to set path, headers, libs, etc.
70    if "VCINSTALLDIR" in os.environ:
71        # MSVC compiler is available, so use it.  OpenMP requires a copy of
72        # vcomp90.dll on the path.  One may be found here:
73        #       C:/Windows/winsxs/x86_microsoft.vc90.openmp*/vcomp90.dll
74        # Copy this to the python directory and uncomment the OpenMP COMPILE
75        # TODO: remove intermediate OBJ file created in the directory
76        # TODO: maybe don't use randomized name for the c file
77        # TODO: maybe ask distutils to find MSVC
78        CC = "cl /nologo /Ox /MD /W3 /GS- /DNDEBUG".split()
79        if "SAS_OPENMP" in os.environ:
80            CC.append("/openmp")
81        LN = "/link /DLL /INCREMENTAL:NO /MANIFEST".split()
82        def compile_command(source, output):
83            return CC + ["/Tp%s"%source] + LN + ["/OUT:%s"%output]
84    elif tinycc:
85        # TinyCC compiler.
86        CC = [tinycc.find_tcc_path()] + "-shared -rdynamic -Wall".split()
87        def compile_command(source, output):
88            return CC + ["%s"%source, "-o", "%s"%output]
89    else:
90        # MinGW compiler.
91        CC = "gcc -shared -std=c99 -O2 -Wall".split()
92        if "SAS_OPENMP" in os.environ:
93            CC.append("-fopenmp")
94        def compile_command(source, output):
95            return CC + ["%s"%source, "-o", "%s"%output, "-lm"]
96else:
97    # Generic unix compile
98    # On mac users will need the X code command line tools installed
99    #COMPILE = "gcc-mp-4.7 -shared -fPIC -std=c99 -fopenmp -O2 -Wall %s -o %s -lm -lgomp"
100    CC = "gcc -shared -fPIC -std=c99 -O2 -Wall".split()
101    # add openmp support if not running on a mac
102    if sys.platform != "darwin":
103        CC.append("-fopenmp")
104    def compile_command(source, output):
105        return CC + ["%s"%source, "-o", "%s"%output, "-lm"]
106
107# Windows-specific solution
108if os.name == 'nt':
109    # Assume the default location of module DLLs is in .sasmodels/compiled_models.
110    DLL_PATH = os.path.join(os.path.expanduser("~"), ".sasmodels", "compiled_models")
111    if not os.path.exists(DLL_PATH):
112        os.makedirs(DLL_PATH)
113else:
114    # Set up the default path for compiled modules.
115    DLL_PATH = tempfile.gettempdir()
116
117ALLOW_SINGLE_PRECISION_DLLS = True
118
119def compile(source, output):
120    command = compile_command(source=source, output=output)
121    command_str = " ".join('"%s"'%p if ' ' in p else p for p in command)
122    logging.info(command_str)
123    try:
124        subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT)
125    except subprocess.CalledProcessError as exc:
126        raise RuntimeError("compile failed.\n%s\n%s"%(command_str, exc.output))
127    if not os.path.exists(output):
128        raise RuntimeError("compile failed.  File is in %r"%source)
129
130def dll_path(model_info, dtype="double"):
131    """
132    Path to the compiled model defined by *model_info*.
133    """
134    basename = splitext(splitpath(model_info['filename'])[1])[0]
135    if np.dtype(dtype) == generate.F32:
136        basename += "32"
137    elif np.dtype(dtype) == generate.F64:
138        basename += "64"
139    else:
140        basename += "128"
141
142    # Hack to find precompiled dlls
143    path = joinpath(generate.DATA_PATH, '..', 'compiled_models', basename+'.so')
144    if os.path.exists(path):
145        return path
146
147    return joinpath(DLL_PATH, basename+'.so')
148
149def make_dll(source, model_info, dtype="double"):
150    """
151    Load the compiled model defined by *kernel_module*.
152
153    Recompile if any files are newer than the model file.
154
155    *dtype* is a numpy floating point precision specifier indicating whether
156    the model should be single or double precision.  The default is double
157    precision.
158
159    The DLL is not loaded until the kernel is called so models can
160    be defined without using too many resources.
161
162    Set *sasmodels.kerneldll.DLL_PATH* to the compiled dll output path.
163    The default is the system temporary directory.
164
165    Set *sasmodels.ALLOW_SINGLE_PRECISION_DLLS* to True if single precision
166    models are allowed as DLLs.
167    """
168    if callable(model_info.get('Iq', None)):
169        return PyModel(model_info)
170   
171    dtype = np.dtype(dtype)
172    if dtype == generate.F16:
173        raise ValueError("16 bit floats not supported")
174    if dtype == generate.F32 and not ALLOW_SINGLE_PRECISION_DLLS:
175        dtype = generate.F64  # Force 64-bit dll
176
177    if dtype == generate.F32: # 32-bit dll
178        tempfile_prefix = 'sas_' + model_info['name'] + '32_'
179    elif dtype == generate.F64:
180        tempfile_prefix = 'sas_' + model_info['name'] + '64_'
181    else:
182        tempfile_prefix = 'sas_' + model_info['name'] + '128_'
183
184    dll = dll_path(model_info, dtype)
185
186    if not os.path.exists(dll):
187        need_recompile = True
188    elif getattr(sys, 'frozen', None) is not None:
189        # TODO: don't suppress time stamp
190        # Currently suppressing recompile when running in a frozen environment
191        need_recompile = False
192    else:
193        dll_time = os.path.getmtime(dll)
194        source_files = generate.model_sources(model_info) + [model_info['filename']]
195        newest_source = max(os.path.getmtime(f) for f in source_files)
196        need_recompile = dll_time < newest_source
197    if need_recompile:
198        source = generate.convert_type(source, dtype)
199        fd, filename = tempfile.mkstemp(suffix=".c", prefix=tempfile_prefix)
200        with os.fdopen(fd, "w") as file:
201            file.write(source)
202        compile(source=filename, output=dll)
203        # comment the following to keep the generated c file
204        # Note: if there is a syntax error then compile raises an error
205        # and the source file will not be deleted.
206        os.unlink(filename)
207        #print("saving compiled file in %r"%filename)
208    return dll
209
210
211def load_dll(source, model_info, dtype="double"):
212    """
213    Create and load a dll corresponding to the source, info pair returned
214    from :func:`sasmodels.generate.make` compiled for the target precision.
215
216    See :func:`make_dll` for details on controlling the dll path and the
217    allowed floating point precision.
218    """
219    filename = make_dll(source, model_info, dtype=dtype)
220    return DllModel(filename, model_info, dtype=dtype)
221
222
223IQ_ARGS = [c_void_p, c_void_p, c_int]
224IQXY_ARGS = [c_void_p, c_void_p, c_void_p, c_int]
225
226class DllModel(object):
227    """
228    ctypes wrapper for a single model.
229
230    *source* and *model_info* are the model source and interface as returned
231    from :func:`gen.make`.
232
233    *dtype* is the desired model precision.  Any numpy dtype for single
234    or double precision floats will do, such as 'f', 'float32' or 'single'
235    for single and 'd', 'float64' or 'double' for double.  Double precision
236    is an optional extension which may not be available on all devices.
237
238    Call :meth:`release` when done with the kernel.
239    """
240   
241    def __init__(self, dllpath, model_info, dtype=generate.F32):
242        self.info = model_info
243        self.dllpath = dllpath
244        self.dll = None
245        self.dtype = np.dtype(dtype)
246
247    def _load_dll(self):
248        Nfixed1d = len(self.info['partype']['fixed-1d'])
249        Nfixed2d = len(self.info['partype']['fixed-2d'])
250        Npd1d = len(self.info['partype']['pd-1d'])
251        Npd2d = len(self.info['partype']['pd-2d'])
252
253        #print("dll", self.dllpath)
254        try:
255            self.dll = ct.CDLL(self.dllpath)
256        except:
257            annotate_exception("while loading "+self.dllpath)
258            raise
259
260        fp = (c_float if self.dtype == generate.F32
261              else c_double if self.dtype == generate.F64
262              else c_longdouble)
263        pd_args_1d = [c_void_p, fp] + [c_int]*Npd1d if Npd1d else []
264        pd_args_2d = [c_void_p, fp] + [c_int]*Npd2d if Npd2d else []
265        self.Iq = self.dll[generate.kernel_name(self.info, False)]
266        self.Iq.argtypes = IQ_ARGS + pd_args_1d + [fp]*Nfixed1d
267
268        self.Iqxy = self.dll[generate.kernel_name(self.info, True)]
269        self.Iqxy.argtypes = IQXY_ARGS + pd_args_2d + [fp]*Nfixed2d
270       
271        self.release()
272
273    def __getstate__(self):
274        return self.info, self.dllpath
275
276    def __setstate__(self, state):
277        self.info, self.dllpath = state
278        self.dll = None
279
280    def make_kernel(self, q_vectors):
281        q_input = PyInput(q_vectors, self.dtype)
282        if self.dll is None: self._load_dll()
283        kernel = self.Iqxy if q_input.is_2d else self.Iq
284        return DllKernel(kernel, self.info, q_input)
285
286    def release(self):
287        """
288        Release any resources associated with the model.
289        """
290        if os.name == 'nt':
291            #dll = ct.cdll.LoadLibrary(self.dllpath)
292            dll = ct.CDLL(self.dllpath)
293            libHandle = dll._handle
294            #libHandle = ct.c_void_p(dll._handle)
295            del dll, self.dll
296            self.dll = None
297            ct.windll.kernel32.FreeLibrary(libHandle)
298        else:   
299            pass 
300
301
302class DllKernel(object):
303    """
304    Callable SAS kernel.
305
306    *kernel* is the c function to call.
307
308    *model_info* is the module information
309
310    *q_input* is the DllInput q vectors at which the kernel should be
311    evaluated.
312
313    The resulting call method takes the *pars*, a list of values for
314    the fixed parameters to the kernel, and *pd_pars*, a list of (value, weight)
315    vectors for the polydisperse parameters.  *cutoff* determines the
316    integration limits: any points with combined weight less than *cutoff*
317    will not be calculated.
318
319    Call :meth:`release` when done with the kernel instance.
320    """
321    def __init__(self, kernel, model_info, q_input):
322        self.info = model_info
323        self.q_input = q_input
324        self.kernel = kernel
325        self.res = np.empty(q_input.nq, q_input.dtype)
326        dim = '2d' if q_input.is_2d else '1d'
327        self.fixed_pars = model_info['partype']['fixed-' + dim]
328        self.pd_pars = model_info['partype']['pd-' + dim]
329
330        # In dll kernel, but not in opencl kernel
331        self.p_res = self.res.ctypes.data
332
333    def __call__(self, fixed_pars, pd_pars, cutoff):
334        real = (np.float32 if self.q_input.dtype == generate.F32
335                else np.float64 if self.q_input.dtype == generate.F64
336                else np.float128)
337
338        nq = c_int(self.q_input.nq)
339        if pd_pars:
340            cutoff = real(cutoff)
341            loops_N = [np.uint32(len(p[0])) for p in pd_pars]
342            loops = np.hstack(pd_pars)
343            loops = np.ascontiguousarray(loops.T, self.q_input.dtype).flatten()
344            p_loops = loops.ctypes.data
345            dispersed = [p_loops, cutoff] + loops_N
346        else:
347            dispersed = []
348        fixed = [real(p) for p in fixed_pars]
349        args = self.q_input.q_pointers + [self.p_res, nq] + dispersed + fixed
350        #print(pars)
351        self.kernel(*args)
352
353        return self.res
354
355    def release(self):
356        """
357        Release any resources associated with the kernel.
358        """
359        pass
Note: See TracBrowser for help on using the repository browser.