source: sasmodels/sasmodels/kerneldll.py @ aab3d87

core_shell_microgelscostrafo411magnetic_modelrelease_v0.94release_v0.95ticket-1257-vesicle-productticket_1156ticket_1265_superballticket_822_more_unit_tests
Last change on this file since aab3d87 was aab3d87, checked in by Paul Kienzle <pkienzle@…>, 8 years ago

tag model dlls with x86 or amd64 on windows to avoid future conflicts from a 64-bit release

  • Property mode set to 100644
File size: 13.3 KB
Line 
1r"""
2DLL driver for C kernels
3
4The global attribute *ALLOW_SINGLE_PRECISION_DLLS* should be set to *True* if
5you wish to allow single precision floating point evaluation for the compiled
6models, otherwise it defaults to *False*.
7
8The compiler command line is stored in the attribute *COMPILE*, with string
9substitutions for %(source)s and %(output)s indicating what to compile and
10where to store it.  The actual command is system dependent.
11
12On windows systems, you have a choice of compilers.  *MinGW* is the GNU
13compiler toolchain, available in packages such as anaconda and PythonXY,
14or available stand alone. This toolchain has had difficulties on some
15systems, and may or may not work for you.  In order to build DLLs, *gcc*
16must be on your path.  If the environment variable *SAS_OPENMP* is given
17then -fopenmp is added to the compiler flags.  This requires a version
18of MinGW compiled with OpenMP support.
19
20An alternative toolchain uses the Microsoft Visual C++ compiler, available
21free from microsoft:
22
23    `<http://www.microsoft.com/en-us/download/details.aspx?id=44266>`_
24
25Again, this requires that the compiler is available on your path.  This is
26done by running vcvarsall.bat in a windows terminal.  Install locations are
27system dependent, such as:
28
29    C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\vcvarsall.bat
30
31or maybe
32
33    C:\Users\yourname\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python\9.0\vcvarsall.bat
34
35And again, the environment variable *SAS_OPENMP* controls whether OpenMP is
36used to compile the C code.  This requires the Microsoft vcomp90.dll library,
37which doesn't seem to be included with the compiler, nor does there appear
38to be a public download location.  There may be one on your machine already
39in a location such as:
40
41    C:\Windows\winsxs\x86_microsoft.vc90.openmp*\vcomp90.dll
42
43If you copy this onto your path, such as the python directory or the install
44directory for this application, then OpenMP should be supported.
45"""
46from __future__ import print_function
47
48import sys
49import os
50from os.path import join as joinpath, split as splitpath, splitext
51import subprocess
52import tempfile
53import ctypes as ct
54from ctypes import c_void_p, c_int, c_longdouble, c_double, c_float
55import logging
56
57import numpy as np
58
59from . import generate
60from .kernelpy import PyInput, PyModel
61from .exception import annotate_exception
62
63if os.name == 'nt':
64    ARCH = "amd64" if sys.maxint > 2**32 else "x86"  # maxint=2**31-1 on 32 bit
65    # Windows compiler; check if TinyCC is available
66    try:
67        import tinycc
68    except ImportError:
69        tinycc = None
70    # call vcvarsall.bat before compiling to set path, headers, libs, etc.
71    if "VCINSTALLDIR" in os.environ:
72        # MSVC compiler is available, so use it.  OpenMP requires a copy of
73        # vcomp90.dll on the path.  One may be found here:
74        #       C:/Windows/winsxs/x86_microsoft.vc90.openmp*/vcomp90.dll
75        # Copy this to the python directory and uncomment the OpenMP COMPILE
76        # TODO: remove intermediate OBJ file created in the directory
77        # TODO: maybe don't use randomized name for the c file
78        # TODO: maybe ask distutils to find MSVC
79        CC = "cl /nologo /Ox /MD /W3 /GS- /DNDEBUG".split()
80        if "SAS_OPENMP" in os.environ:
81            CC.append("/openmp")
82        LN = "/link /DLL /INCREMENTAL:NO /MANIFEST".split()
83        def compile_command(source, output):
84            return CC + ["/Tp%s"%source] + LN + ["/OUT:%s"%output]
85    elif tinycc:
86        # TinyCC compiler.
87        CC = [tinycc.find_tcc_path()] + "-shared -rdynamic -Wall".split()
88        def compile_command(source, output):
89            return CC + ["%s"%source, "-o", "%s"%output]
90    else:
91        # MinGW compiler.
92        CC = "gcc -shared -std=c99 -O2 -Wall".split()
93        if "SAS_OPENMP" in os.environ:
94            CC.append("-fopenmp")
95        def compile_command(source, output):
96            return CC + ["%s"%source, "-o", "%s"%output, "-lm"]
97else:
98    ARCH = ""
99    # Generic unix compile
100    # On mac users will need the X code command line tools installed
101    #COMPILE = "gcc-mp-4.7 -shared -fPIC -std=c99 -fopenmp -O2 -Wall %s -o %s -lm -lgomp"
102    CC = "cc -shared -fPIC -std=c99 -O2 -Wall".split()
103    # add openmp support if not running on a mac
104    if sys.platform != "darwin":
105        CC.append("-fopenmp")
106    def compile_command(source, output):
107        return CC + ["%s"%source, "-o", "%s"%output, "-lm"]
108
109# Windows-specific solution
110if os.name == 'nt':
111    # Assume the default location of module DLLs is in .sasmodels/compiled_models.
112    DLL_PATH = os.path.join(os.path.expanduser("~"), ".sasmodels", "compiled_models")
113    if not os.path.exists(DLL_PATH):
114        os.makedirs(DLL_PATH)
115else:
116    # Set up the default path for compiled modules.
117    DLL_PATH = tempfile.gettempdir()
118
119ALLOW_SINGLE_PRECISION_DLLS = True
120
121def compile(source, output):
122    command = compile_command(source=source, output=output)
123    command_str = " ".join('"%s"'%p if ' ' in p else p for p in command)
124    logging.info(command_str)
125    shell = (os.name == 'nt')
126    try:
127        subprocess.check_output(command, shell=shell, stderr=subprocess.STDOUT)
128    except subprocess.CalledProcessError as exc:
129        raise RuntimeError("compile failed.\n%s\n%s"%(command_str, exc.output))
130    if not os.path.exists(output):
131        raise RuntimeError("compile failed.  File is in %r"%source)
132
133def dll_path(model_info, dtype="double"):
134    """
135    Path to the compiled model defined by *model_info*.
136    """
137    basename = splitext(splitpath(model_info['filename'])[1])[0]
138    if np.dtype(dtype) == generate.F32:
139        basename += "32"
140    elif np.dtype(dtype) == generate.F64:
141        basename += "64"
142    else:
143        basename += "128"
144
145    # Hack to find precompiled dlls
146    path = joinpath(generate.DATA_PATH, '..', 'compiled_models',
147                    basename + ARCH + '.so')
148    if os.path.exists(path):
149        return path
150
151    return joinpath(DLL_PATH, basename+'.so')
152
153def make_dll(source, model_info, dtype="double"):
154    """
155    Load the compiled model defined by *kernel_module*.
156
157    Recompile if any files are newer than the model file.
158
159    *dtype* is a numpy floating point precision specifier indicating whether
160    the model should be single or double precision.  The default is double
161    precision.
162
163    The DLL is not loaded until the kernel is called so models can
164    be defined without using too many resources.
165
166    Set *sasmodels.kerneldll.DLL_PATH* to the compiled dll output path.
167    The default is the system temporary directory.
168
169    Set *sasmodels.ALLOW_SINGLE_PRECISION_DLLS* to True if single precision
170    models are allowed as DLLs.
171    """
172    if callable(model_info.get('Iq', None)):
173        return PyModel(model_info)
174   
175    dtype = np.dtype(dtype)
176    if dtype == generate.F16:
177        raise ValueError("16 bit floats not supported")
178    if dtype == generate.F32 and not ALLOW_SINGLE_PRECISION_DLLS:
179        dtype = generate.F64  # Force 64-bit dll
180
181    # File name for generated C code
182    if dtype == generate.F32: # 32-bit dll
183        tempfile_prefix = 'sas_' + model_info['name'] + '32_'
184    elif dtype == generate.F64:
185        tempfile_prefix = 'sas_' + model_info['name'] + '64_'
186    else:
187        tempfile_prefix = 'sas_' + model_info['name'] + '128_'
188
189    dll = dll_path(model_info, dtype)
190
191    if not os.path.exists(dll):
192        need_recompile = True
193    elif getattr(sys, 'frozen', None) is not None:
194        # TODO: don't suppress time stamp
195        # Currently suppressing recompile when running in a frozen environment
196        need_recompile = False
197    else:
198        dll_time = os.path.getmtime(dll)
199        source_files = generate.model_sources(model_info) + [model_info['filename']]
200        newest_source = max(os.path.getmtime(f) for f in source_files)
201        need_recompile = dll_time < newest_source
202    if need_recompile:
203        source = generate.convert_type(source, dtype)
204        fd, filename = tempfile.mkstemp(suffix=".c", prefix=tempfile_prefix)
205        with os.fdopen(fd, "w") as file:
206            file.write(source)
207        compile(source=filename, output=dll)
208        # comment the following to keep the generated c file
209        # Note: if there is a syntax error then compile raises an error
210        # and the source file will not be deleted.
211        os.unlink(filename)
212        #print("saving compiled file in %r"%filename)
213    return dll
214
215
216def load_dll(source, model_info, dtype="double"):
217    """
218    Create and load a dll corresponding to the source, info pair returned
219    from :func:`sasmodels.generate.make` compiled for the target precision.
220
221    See :func:`make_dll` for details on controlling the dll path and the
222    allowed floating point precision.
223    """
224    filename = make_dll(source, model_info, dtype=dtype)
225    return DllModel(filename, model_info, dtype=dtype)
226
227
228IQ_ARGS = [c_void_p, c_void_p, c_int]
229IQXY_ARGS = [c_void_p, c_void_p, c_void_p, c_int]
230
231class DllModel(object):
232    """
233    ctypes wrapper for a single model.
234
235    *source* and *model_info* are the model source and interface as returned
236    from :func:`gen.make`.
237
238    *dtype* is the desired model precision.  Any numpy dtype for single
239    or double precision floats will do, such as 'f', 'float32' or 'single'
240    for single and 'd', 'float64' or 'double' for double.  Double precision
241    is an optional extension which may not be available on all devices.
242
243    Call :meth:`release` when done with the kernel.
244    """
245   
246    def __init__(self, dllpath, model_info, dtype=generate.F32):
247        self.info = model_info
248        self.dllpath = dllpath
249        self.dll = None
250        self.dtype = np.dtype(dtype)
251
252    def _load_dll(self):
253        Nfixed1d = len(self.info['partype']['fixed-1d'])
254        Nfixed2d = len(self.info['partype']['fixed-2d'])
255        Npd1d = len(self.info['partype']['pd-1d'])
256        Npd2d = len(self.info['partype']['pd-2d'])
257
258        #print("dll", self.dllpath)
259        try:
260            self.dll = ct.CDLL(self.dllpath)
261        except:
262            annotate_exception("while loading "+self.dllpath)
263            raise
264
265        fp = (c_float if self.dtype == generate.F32
266              else c_double if self.dtype == generate.F64
267              else c_longdouble)
268        pd_args_1d = [c_void_p, fp] + [c_int]*Npd1d if Npd1d else []
269        pd_args_2d = [c_void_p, fp] + [c_int]*Npd2d if Npd2d else []
270        self.Iq = self.dll[generate.kernel_name(self.info, False)]
271        self.Iq.argtypes = IQ_ARGS + pd_args_1d + [fp]*Nfixed1d
272
273        self.Iqxy = self.dll[generate.kernel_name(self.info, True)]
274        self.Iqxy.argtypes = IQXY_ARGS + pd_args_2d + [fp]*Nfixed2d
275       
276        self.release()
277
278    def __getstate__(self):
279        return self.info, self.dllpath
280
281    def __setstate__(self, state):
282        self.info, self.dllpath = state
283        self.dll = None
284
285    def make_kernel(self, q_vectors):
286        q_input = PyInput(q_vectors, self.dtype)
287        if self.dll is None: self._load_dll()
288        kernel = self.Iqxy if q_input.is_2d else self.Iq
289        return DllKernel(kernel, self.info, q_input)
290
291    def release(self):
292        """
293        Release any resources associated with the model.
294        """
295        if os.name == 'nt':
296            #dll = ct.cdll.LoadLibrary(self.dllpath)
297            dll = ct.CDLL(self.dllpath)
298            libHandle = dll._handle
299            #libHandle = ct.c_void_p(dll._handle)
300            del dll, self.dll
301            self.dll = None
302            ct.windll.kernel32.FreeLibrary(libHandle)
303        else:   
304            pass 
305
306
307class DllKernel(object):
308    """
309    Callable SAS kernel.
310
311    *kernel* is the c function to call.
312
313    *model_info* is the module information
314
315    *q_input* is the DllInput q vectors at which the kernel should be
316    evaluated.
317
318    The resulting call method takes the *pars*, a list of values for
319    the fixed parameters to the kernel, and *pd_pars*, a list of (value, weight)
320    vectors for the polydisperse parameters.  *cutoff* determines the
321    integration limits: any points with combined weight less than *cutoff*
322    will not be calculated.
323
324    Call :meth:`release` when done with the kernel instance.
325    """
326    def __init__(self, kernel, model_info, q_input):
327        self.info = model_info
328        self.q_input = q_input
329        self.kernel = kernel
330        self.res = np.empty(q_input.nq, q_input.dtype)
331        dim = '2d' if q_input.is_2d else '1d'
332        self.fixed_pars = model_info['partype']['fixed-' + dim]
333        self.pd_pars = model_info['partype']['pd-' + dim]
334
335        # In dll kernel, but not in opencl kernel
336        self.p_res = self.res.ctypes.data
337
338    def __call__(self, fixed_pars, pd_pars, cutoff):
339        real = (np.float32 if self.q_input.dtype == generate.F32
340                else np.float64 if self.q_input.dtype == generate.F64
341                else np.float128)
342
343        nq = c_int(self.q_input.nq)
344        if pd_pars:
345            cutoff = real(cutoff)
346            loops_N = [np.uint32(len(p[0])) for p in pd_pars]
347            loops = np.hstack(pd_pars)
348            loops = np.ascontiguousarray(loops.T, self.q_input.dtype).flatten()
349            p_loops = loops.ctypes.data
350            dispersed = [p_loops, cutoff] + loops_N
351        else:
352            dispersed = []
353        fixed = [real(p) for p in fixed_pars]
354        args = self.q_input.q_pointers + [self.p_res, nq] + dispersed + fixed
355        #print(pars)
356        self.kernel(*args)
357
358        return self.res
359
360    def release(self):
361        """
362        Release any resources associated with the kernel.
363        """
364        pass
Note: See TracBrowser for help on using the repository browser.