source: sasmodels/sasmodels/kerneldll.py @ aa343d6

core_shell_microgelscostrafo411magnetic_modelrelease_v0.94release_v0.95ticket-1257-vesicle-productticket_1156ticket_1265_superballticket_822_more_unit_tests
Last change on this file since aa343d6 was aa343d6, checked in by Paul Kienzle <pkienzle@…>, 6 years ago

use cc as compiler and shell=False on linux/mac

  • Property mode set to 100644
File size: 13.2 KB
Line 
1r"""
2DLL driver for C kernels
3
4The global attribute *ALLOW_SINGLE_PRECISION_DLLS* should be set to *True* if
5you wish to allow single precision floating point evaluation for the compiled
6models, otherwise it defaults to *False*.
7
8The compiler command line is stored in the attribute *COMPILE*, with string
9substitutions for %(source)s and %(output)s indicating what to compile and
10where to store it.  The actual command is system dependent.
11
12On windows systems, you have a choice of compilers.  *MinGW* is the GNU
13compiler toolchain, available in packages such as anaconda and PythonXY,
14or available stand alone. This toolchain has had difficulties on some
15systems, and may or may not work for you.  In order to build DLLs, *gcc*
16must be on your path.  If the environment variable *SAS_OPENMP* is given
17then -fopenmp is added to the compiler flags.  This requires a version
18of MinGW compiled with OpenMP support.
19
20An alternative toolchain uses the Microsoft Visual C++ compiler, available
21free from microsoft:
22
23    `<http://www.microsoft.com/en-us/download/details.aspx?id=44266>`_
24
25Again, this requires that the compiler is available on your path.  This is
26done by running vcvarsall.bat in a windows terminal.  Install locations are
27system dependent, such as:
28
29    C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\vcvarsall.bat
30
31or maybe
32
33    C:\Users\yourname\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python\9.0\vcvarsall.bat
34
35And again, the environment variable *SAS_OPENMP* controls whether OpenMP is
36used to compile the C code.  This requires the Microsoft vcomp90.dll library,
37which doesn't seem to be included with the compiler, nor does there appear
38to be a public download location.  There may be one on your machine already
39in a location such as:
40
41    C:\Windows\winsxs\x86_microsoft.vc90.openmp*\vcomp90.dll
42
43If you copy this onto your path, such as the python directory or the install
44directory for this application, then OpenMP should be supported.
45"""
46from __future__ import print_function
47
48import sys
49import os
50from os.path import join as joinpath, split as splitpath, splitext
51import subprocess
52import tempfile
53import ctypes as ct
54from ctypes import c_void_p, c_int, c_longdouble, c_double, c_float
55import logging
56
57import numpy as np
58
59from . import generate
60from .kernelpy import PyInput, PyModel
61from .exception import annotate_exception
62
63if os.name == 'nt':
64    # Windows compiler; check if TinyCC is available
65    try:
66        import tinycc
67    except ImportError:
68        tinycc = None
69    # call vcvarsall.bat before compiling to set path, headers, libs, etc.
70    if "VCINSTALLDIR" in os.environ:
71        # MSVC compiler is available, so use it.  OpenMP requires a copy of
72        # vcomp90.dll on the path.  One may be found here:
73        #       C:/Windows/winsxs/x86_microsoft.vc90.openmp*/vcomp90.dll
74        # Copy this to the python directory and uncomment the OpenMP COMPILE
75        # TODO: remove intermediate OBJ file created in the directory
76        # TODO: maybe don't use randomized name for the c file
77        # TODO: maybe ask distutils to find MSVC
78        CC = "cl /nologo /Ox /MD /W3 /GS- /DNDEBUG".split()
79        if "SAS_OPENMP" in os.environ:
80            CC.append("/openmp")
81        LN = "/link /DLL /INCREMENTAL:NO /MANIFEST".split()
82        def compile_command(source, output):
83            return CC + ["/Tp%s"%source] + LN + ["/OUT:%s"%output]
84    elif tinycc:
85        # TinyCC compiler.
86        CC = [tinycc.find_tcc_path()] + "-shared -rdynamic -Wall".split()
87        def compile_command(source, output):
88            return CC + ["%s"%source, "-o", "%s"%output]
89    else:
90        # MinGW compiler.
91        CC = "gcc -shared -std=c99 -O2 -Wall".split()
92        if "SAS_OPENMP" in os.environ:
93            CC.append("-fopenmp")
94        def compile_command(source, output):
95            return CC + ["%s"%source, "-o", "%s"%output, "-lm"]
96else:
97    # Generic unix compile
98    # On mac users will need the X code command line tools installed
99    #COMPILE = "gcc-mp-4.7 -shared -fPIC -std=c99 -fopenmp -O2 -Wall %s -o %s -lm -lgomp"
100    CC = "cc -shared -fPIC -std=c99 -O2 -Wall".split()
101    # add openmp support if not running on a mac
102    if sys.platform != "darwin":
103        CC.append("-fopenmp")
104    def compile_command(source, output):
105        return CC + ["%s"%source, "-o", "%s"%output, "-lm"]
106
107# Windows-specific solution
108if os.name == 'nt':
109    # Assume the default location of module DLLs is in .sasmodels/compiled_models.
110    DLL_PATH = os.path.join(os.path.expanduser("~"), ".sasmodels", "compiled_models")
111    if not os.path.exists(DLL_PATH):
112        os.makedirs(DLL_PATH)
113else:
114    # Set up the default path for compiled modules.
115    DLL_PATH = tempfile.gettempdir()
116
117ALLOW_SINGLE_PRECISION_DLLS = True
118
119def compile(source, output):
120    command = compile_command(source=source, output=output)
121    command_str = " ".join('"%s"'%p if ' ' in p else p for p in command)
122    logging.info(command_str)
123    shell = (os.name == 'nt')
124    try:
125        subprocess.check_output(command, shell=shell, stderr=subprocess.STDOUT)
126    except subprocess.CalledProcessError as exc:
127        raise RuntimeError("compile failed.\n%s\n%s"%(command_str, exc.output))
128    if not os.path.exists(output):
129        raise RuntimeError("compile failed.  File is in %r"%source)
130
131def dll_path(model_info, dtype="double"):
132    """
133    Path to the compiled model defined by *model_info*.
134    """
135    basename = splitext(splitpath(model_info['filename'])[1])[0]
136    if np.dtype(dtype) == generate.F32:
137        basename += "32"
138    elif np.dtype(dtype) == generate.F64:
139        basename += "64"
140    else:
141        basename += "128"
142
143    # Hack to find precompiled dlls
144    path = joinpath(generate.DATA_PATH, '..', 'compiled_models', basename+'.so')
145    if os.path.exists(path):
146        return path
147
148    return joinpath(DLL_PATH, basename+'.so')
149
150def make_dll(source, model_info, dtype="double"):
151    """
152    Load the compiled model defined by *kernel_module*.
153
154    Recompile if any files are newer than the model file.
155
156    *dtype* is a numpy floating point precision specifier indicating whether
157    the model should be single or double precision.  The default is double
158    precision.
159
160    The DLL is not loaded until the kernel is called so models can
161    be defined without using too many resources.
162
163    Set *sasmodels.kerneldll.DLL_PATH* to the compiled dll output path.
164    The default is the system temporary directory.
165
166    Set *sasmodels.ALLOW_SINGLE_PRECISION_DLLS* to True if single precision
167    models are allowed as DLLs.
168    """
169    if callable(model_info.get('Iq', None)):
170        return PyModel(model_info)
171   
172    dtype = np.dtype(dtype)
173    if dtype == generate.F16:
174        raise ValueError("16 bit floats not supported")
175    if dtype == generate.F32 and not ALLOW_SINGLE_PRECISION_DLLS:
176        dtype = generate.F64  # Force 64-bit dll
177
178    if dtype == generate.F32: # 32-bit dll
179        tempfile_prefix = 'sas_' + model_info['name'] + '32_'
180    elif dtype == generate.F64:
181        tempfile_prefix = 'sas_' + model_info['name'] + '64_'
182    else:
183        tempfile_prefix = 'sas_' + model_info['name'] + '128_'
184
185    dll = dll_path(model_info, dtype)
186
187    if not os.path.exists(dll):
188        need_recompile = True
189    elif getattr(sys, 'frozen', None) is not None:
190        # TODO: don't suppress time stamp
191        # Currently suppressing recompile when running in a frozen environment
192        need_recompile = False
193    else:
194        dll_time = os.path.getmtime(dll)
195        source_files = generate.model_sources(model_info) + [model_info['filename']]
196        newest_source = max(os.path.getmtime(f) for f in source_files)
197        need_recompile = dll_time < newest_source
198    if need_recompile:
199        source = generate.convert_type(source, dtype)
200        fd, filename = tempfile.mkstemp(suffix=".c", prefix=tempfile_prefix)
201        with os.fdopen(fd, "w") as file:
202            file.write(source)
203        compile(source=filename, output=dll)
204        # comment the following to keep the generated c file
205        # Note: if there is a syntax error then compile raises an error
206        # and the source file will not be deleted.
207        os.unlink(filename)
208        #print("saving compiled file in %r"%filename)
209    return dll
210
211
212def load_dll(source, model_info, dtype="double"):
213    """
214    Create and load a dll corresponding to the source, info pair returned
215    from :func:`sasmodels.generate.make` compiled for the target precision.
216
217    See :func:`make_dll` for details on controlling the dll path and the
218    allowed floating point precision.
219    """
220    filename = make_dll(source, model_info, dtype=dtype)
221    return DllModel(filename, model_info, dtype=dtype)
222
223
224IQ_ARGS = [c_void_p, c_void_p, c_int]
225IQXY_ARGS = [c_void_p, c_void_p, c_void_p, c_int]
226
227class DllModel(object):
228    """
229    ctypes wrapper for a single model.
230
231    *source* and *model_info* are the model source and interface as returned
232    from :func:`gen.make`.
233
234    *dtype* is the desired model precision.  Any numpy dtype for single
235    or double precision floats will do, such as 'f', 'float32' or 'single'
236    for single and 'd', 'float64' or 'double' for double.  Double precision
237    is an optional extension which may not be available on all devices.
238
239    Call :meth:`release` when done with the kernel.
240    """
241   
242    def __init__(self, dllpath, model_info, dtype=generate.F32):
243        self.info = model_info
244        self.dllpath = dllpath
245        self.dll = None
246        self.dtype = np.dtype(dtype)
247
248    def _load_dll(self):
249        Nfixed1d = len(self.info['partype']['fixed-1d'])
250        Nfixed2d = len(self.info['partype']['fixed-2d'])
251        Npd1d = len(self.info['partype']['pd-1d'])
252        Npd2d = len(self.info['partype']['pd-2d'])
253
254        #print("dll", self.dllpath)
255        try:
256            self.dll = ct.CDLL(self.dllpath)
257        except:
258            annotate_exception("while loading "+self.dllpath)
259            raise
260
261        fp = (c_float if self.dtype == generate.F32
262              else c_double if self.dtype == generate.F64
263              else c_longdouble)
264        pd_args_1d = [c_void_p, fp] + [c_int]*Npd1d if Npd1d else []
265        pd_args_2d = [c_void_p, fp] + [c_int]*Npd2d if Npd2d else []
266        self.Iq = self.dll[generate.kernel_name(self.info, False)]
267        self.Iq.argtypes = IQ_ARGS + pd_args_1d + [fp]*Nfixed1d
268
269        self.Iqxy = self.dll[generate.kernel_name(self.info, True)]
270        self.Iqxy.argtypes = IQXY_ARGS + pd_args_2d + [fp]*Nfixed2d
271       
272        self.release()
273
274    def __getstate__(self):
275        return self.info, self.dllpath
276
277    def __setstate__(self, state):
278        self.info, self.dllpath = state
279        self.dll = None
280
281    def make_kernel(self, q_vectors):
282        q_input = PyInput(q_vectors, self.dtype)
283        if self.dll is None: self._load_dll()
284        kernel = self.Iqxy if q_input.is_2d else self.Iq
285        return DllKernel(kernel, self.info, q_input)
286
287    def release(self):
288        """
289        Release any resources associated with the model.
290        """
291        if os.name == 'nt':
292            #dll = ct.cdll.LoadLibrary(self.dllpath)
293            dll = ct.CDLL(self.dllpath)
294            libHandle = dll._handle
295            #libHandle = ct.c_void_p(dll._handle)
296            del dll, self.dll
297            self.dll = None
298            ct.windll.kernel32.FreeLibrary(libHandle)
299        else:   
300            pass 
301
302
303class DllKernel(object):
304    """
305    Callable SAS kernel.
306
307    *kernel* is the c function to call.
308
309    *model_info* is the module information
310
311    *q_input* is the DllInput q vectors at which the kernel should be
312    evaluated.
313
314    The resulting call method takes the *pars*, a list of values for
315    the fixed parameters to the kernel, and *pd_pars*, a list of (value, weight)
316    vectors for the polydisperse parameters.  *cutoff* determines the
317    integration limits: any points with combined weight less than *cutoff*
318    will not be calculated.
319
320    Call :meth:`release` when done with the kernel instance.
321    """
322    def __init__(self, kernel, model_info, q_input):
323        self.info = model_info
324        self.q_input = q_input
325        self.kernel = kernel
326        self.res = np.empty(q_input.nq, q_input.dtype)
327        dim = '2d' if q_input.is_2d else '1d'
328        self.fixed_pars = model_info['partype']['fixed-' + dim]
329        self.pd_pars = model_info['partype']['pd-' + dim]
330
331        # In dll kernel, but not in opencl kernel
332        self.p_res = self.res.ctypes.data
333
334    def __call__(self, fixed_pars, pd_pars, cutoff):
335        real = (np.float32 if self.q_input.dtype == generate.F32
336                else np.float64 if self.q_input.dtype == generate.F64
337                else np.float128)
338
339        nq = c_int(self.q_input.nq)
340        if pd_pars:
341            cutoff = real(cutoff)
342            loops_N = [np.uint32(len(p[0])) for p in pd_pars]
343            loops = np.hstack(pd_pars)
344            loops = np.ascontiguousarray(loops.T, self.q_input.dtype).flatten()
345            p_loops = loops.ctypes.data
346            dispersed = [p_loops, cutoff] + loops_N
347        else:
348            dispersed = []
349        fixed = [real(p) for p in fixed_pars]
350        args = self.q_input.q_pointers + [self.p_res, nq] + dispersed + fixed
351        #print(pars)
352        self.kernel(*args)
353
354        return self.res
355
356    def release(self):
357        """
358        Release any resources associated with the kernel.
359        """
360        pass
Note: See TracBrowser for help on using the repository browser.