source: sasmodels/sasmodels/kerneldll.py @ 57b0148

Last change on this file since 57b0148 was 57b0148, checked in by wojciech, 5 years ago

Check-ups for 576

  • Property mode set to 100644
File size: 14.7 KB
Line 
1r"""
2DLL driver for C kernels
3
4The global attribute *ALLOW_SINGLE_PRECISION_DLLS* should be set to *True* if
5you wish to allow single precision floating point evaluation for the compiled
6models, otherwise it defaults to *False*.
7
8The compiler command line is stored in the attribute *COMPILE*, with string
9substitutions for %(source)s and %(output)s indicating what to compile and
10where to store it.  The actual command is system dependent.
11
12On windows systems, you have a choice of compilers.  *MinGW* is the GNU
13compiler toolchain, available in packages such as anaconda and PythonXY,
14or available stand alone. This toolchain has had difficulties on some
15systems, and may or may not work for you.  In order to build DLLs, *gcc*
16must be on your path.  If the environment variable *SAS_OPENMP* is given
17then -fopenmp is added to the compiler flags.  This requires a version
18of MinGW compiled with OpenMP support.
19
20An alternative toolchain uses the Microsoft Visual C++ compiler, available
21free from microsoft:
22
23    `<http://www.microsoft.com/en-us/download/details.aspx?id=44266>`_
24
25Again, this requires that the compiler is available on your path.  This is
26done by running vcvarsall.bat in a windows terminal.  Install locations are
27system dependent, such as:
28
29    C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\vcvarsall.bat
30
31or maybe
32
33    C:\Users\yourname\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python\9.0\vcvarsall.bat
34
35And again, the environment variable *SAS_OPENMP* controls whether OpenMP is
36used to compile the C code.  This requires the Microsoft vcomp90.dll library,
37which doesn't seem to be included with the compiler, nor does there appear
38to be a public download location.  There may be one on your machine already
39in a location such as:
40
41    C:\Windows\winsxs\x86_microsoft.vc90.openmp*\vcomp90.dll
42
43If you copy this onto your path, such as the python directory or the install
44directory for this application, then OpenMP should be supported.
45"""
46from __future__ import print_function
47
48import sys
49import os
50from os.path import join as joinpath, splitext
51import subprocess
52import tempfile
53import ctypes as ct  # type: ignore
54from ctypes import c_void_p, c_int32, c_longdouble, c_double, c_float  # type: ignore
55import logging
56
57import numpy as np  # type: ignore
58
59try:
60    import tinycc
61except ImportError:
62    tinycc = None
63
64from . import generate
65from .kernel import KernelModel, Kernel
66from .kernelpy import PyInput
67from .exception import annotate_exception
68from .generate import F16, F32, F64
69
70try:
71    from typing import Tuple, Callable, Any
72    from .modelinfo import ModelInfo
73    from .details import CallDetails
74except ImportError:
75    pass
76
77if "SAS_COMPILER" in os.environ:
78    compiler = os.environ["SAS_COMPILER"]
79elif os.name == 'nt':
80    # If vcvarsall.bat has been called, then VCINSTALLDIR is in the environment
81    # and we can use the MSVC compiler.  Otherwise, if tinycc is available
82    # the use it.  Otherwise, hope that mingw is available.
83    if "VCINSTALLDIR" in os.environ:
84        compiler = "msvc"
85    elif tinycc:
86        compiler = "tinycc"
87    else:
88        compiler = "mingw"
89else:
90    compiler = "unix"
91
92ARCH = "" if sys.maxint > 2**32 else "x86"  # maxint=2**31-1 on 32 bit
93if compiler == "unix":
94    # Generic unix compile
95    # On mac users will need the X code command line tools installed
96    #COMPILE = "gcc-mp-4.7 -shared -fPIC -std=c99 -fopenmp -O2 -Wall %s -o %s -lm -lgomp"
97    CC = "cc -shared -fPIC -std=c99 -O2 -Wall".split()
98    # add openmp support if not running on a mac
99    if sys.platform != "darwin":
100        CC.append("-fopenmp")
101    def compile_command(source, output):
102        """unix compiler command"""
103        return CC + [source, "-o", output, "-lm"]
104elif compiler == "msvc":
105    # Call vcvarsall.bat before compiling to set path, headers, libs, etc.
106    # MSVC compiler is available, so use it.  OpenMP requires a copy of
107    # vcomp90.dll on the path.  One may be found here:
108    #       C:/Windows/winsxs/x86_microsoft.vc90.openmp*/vcomp90.dll
109    # Copy this to the python directory and uncomment the OpenMP COMPILE
110    # TODO: remove intermediate OBJ file created in the directory
111    # TODO: maybe don't use randomized name for the c file
112    # TODO: maybe ask distutils to find MSVC
113    CC = "cl /nologo /Ox /MD /W3 /GS- /DNDEBUG".split()
114    if "SAS_OPENMP" in os.environ:
115        CC.append("/openmp")
116    LN = "/link /DLL /INCREMENTAL:NO /MANIFEST".split()
117    def compile_command(source, output):
118        """MSVC compiler command"""
119        return CC + ["/Tp%s"%source] + LN + ["/OUT:%s"%output]
120elif compiler == "tinycc":
121    # TinyCC compiler.
122    CC = [tinycc.TCC] + "-shared -rdynamic -Wall".split()
123    def compile_command(source, output):
124        """tinycc compiler command"""
125        return CC + [source, "-o", output]
126elif compiler == "mingw":
127    # MinGW compiler.
128    CC = "gcc -shared -std=c99 -O2 -Wall".split()
129    if "SAS_OPENMP" in os.environ:
130        CC.append("-fopenmp")
131    def compile_command(source, output):
132        """mingw compiler command"""
133        return CC + [source, "-o", output, "-lm"]
134
135# Windows-specific solution
136if os.name == 'nt':
137    # Assume the default location of module DLLs is in .sasmodels/compiled_models.
138    DLL_PATH = os.path.join(os.path.expanduser("~"), ".sasmodels", "compiled_models")
139    if not os.path.exists(DLL_PATH):
140        os.makedirs(DLL_PATH)
141else:
142    # Set up the default path for compiled modules.
143    DLL_PATH = tempfile.gettempdir()
144
145ALLOW_SINGLE_PRECISION_DLLS = True
146
147def compile(source, output):
148    # type: (str, str) -> None
149    """
150    Compile *source* producing *output*.
151
152    Raises RuntimeError if the compile failed or the output wasn't produced.
153    """
154    command = compile_command(source=source, output=output)
155    command_str = " ".join('"%s"'%p if ' ' in p else p for p in command)
156    logging.info(command_str)
157    try:
158        # need shell=True on windows to keep console box from popping up
159        shell = (os.name == 'nt')
160        subprocess.check_output(command, shell=shell, stderr=subprocess.STDOUT)
161    except subprocess.CalledProcessError as exc:
162        raise RuntimeError("compile failed.\n%s\n%s"%(command_str, exc.output))
163    if not os.path.exists(output):
164        raise RuntimeError("compile failed.  File is in %r"%source)
165
166def dll_name(model_info, dtype):
167    # type: (ModelInfo, np.dtype) ->  str
168    """
169    Name of the dll containing the model.  This is the base file name without
170    any path or extension, with a form such as 'sas_sphere32'.
171    """
172    bits = 8*dtype.itemsize
173    basename = "sas%d_%s"%(bits, model_info.id)
174    basename += ARCH + ".so"
175
176    # Hack to find precompiled dlls
177    path = joinpath(generate.DATA_PATH, '..', 'compiled_models', basename)
178    if os.path.exists(path):
179        return path
180
181    return joinpath(DLL_PATH, basename)
182
183
184def dll_path(model_info, dtype):
185    # type: (ModelInfo, np.dtype) -> str
186    """
187    Complete path to the dll for the model.  Note that the dll may not
188    exist yet if it hasn't been compiled.
189    """
190    return os.path.join(DLL_PATH, dll_name(model_info, dtype))
191
192
193def make_dll(source, model_info, dtype=F64):
194    # type: (str, ModelInfo, np.dtype) -> str
195    """
196    Returns the path to the compiled model defined by *kernel_module*.
197
198    If the model has not been compiled, or if the source file(s) are newer
199    than the dll, then *make_dll* will compile the model before returning.
200    This routine does not load the resulting dll.
201
202    *dtype* is a numpy floating point precision specifier indicating whether
203    the model should be single, double or long double precision.  The default
204    is double precision, *np.dtype('d')*.
205
206    Set *sasmodels.ALLOW_SINGLE_PRECISION_DLLS* to False if single precision
207    models are not allowed as DLLs.
208
209    Set *sasmodels.kerneldll.DLL_PATH* to the compiled dll output path.
210    The default is the system temporary directory.
211    """
212    if dtype == F16:
213        raise ValueError("16 bit floats not supported")
214    if dtype == F32 and not ALLOW_SINGLE_PRECISION_DLLS:
215        dtype = F64  # Force 64-bit dll
216    # Note: dtype may be F128 for long double precision
217
218    dll = dll_path(model_info, dtype)
219
220    if not os.path.exists(dll):
221        need_recompile = True
222    elif getattr(sys, 'frozen', None) is not None:
223        # TODO: don't suppress time stamp
224        # Currently suppressing recompile when running in a frozen environment
225        need_recompile = False
226    else:
227        dll_time = os.path.getmtime(dll)
228        newest_source = generate.dll_timestamp(model_info)
229        need_recompile = dll_time < newest_source
230    if need_recompile:
231        basename = splitext(os.path.basename(dll))[0] + "_"
232        system_fd, filename = tempfile.mkstemp(suffix=".c", prefix=basename)
233        source = generate.convert_type(source, dtype)
234        with os.fdopen(system_fd, "w") as file_handle:
235            file_handle.write(source)
236        compile(source=filename, output=dll)
237        # comment the following to keep the generated c file
238        # Note: if there is a syntax error then compile raises an error
239        # and the source file will not be deleted.
240        #os.unlink(filename)
241        print("saving compiled file in %r"%filename)
242    return dll
243
244
245def load_dll(source, model_info, dtype=F64):
246    # type: (str, ModelInfo, np.dtype) -> "DllModel"
247    """
248    Create and load a dll corresponding to the source, info pair returned
249    from :func:`sasmodels.generate.make` compiled for the target precision.
250
251    See :func:`make_dll` for details on controlling the dll path and the
252    allowed floating point precision.
253    """
254    filename = make_dll(source, model_info, dtype=dtype)
255    return DllModel(filename, model_info, dtype=dtype)
256
257
258class DllModel(KernelModel):
259    """
260    ctypes wrapper for a single model.
261
262    *source* and *model_info* are the model source and interface as returned
263    from :func:`gen.make`.
264
265    *dtype* is the desired model precision.  Any numpy dtype for single
266    or double precision floats will do, such as 'f', 'float32' or 'single'
267    for single and 'd', 'float64' or 'double' for double.  Double precision
268    is an optional extension which may not be available on all devices.
269
270    Call :meth:`release` when done with the kernel.
271    """
272    def __init__(self, dllpath, model_info, dtype=generate.F32):
273        # type: (str, ModelInfo, np.dtype) -> None
274        self.info = model_info
275        self.dllpath = dllpath
276        self._dll = None  # type: ct.CDLL
277        self._kernels = None # type: List[Callable, Callable]
278        self.dtype = np.dtype(dtype)
279
280    def _load_dll(self):
281        # type: () -> None
282        print("dll", self.dllpath)
283        try:
284            self._dll = ct.CDLL(self.dllpath)
285        except:
286            annotate_exception("while loading "+self.dllpath)
287            raise
288
289        float_type = (c_float if self.dtype == generate.F32
290                      else c_double if self.dtype == generate.F64
291                      else c_longdouble)
292
293        # int, int, int, int*, double*, double*, double*, double*, double
294        argtypes = [c_int32]*3 + [c_void_p]*4 + [float_type]
295        names = [generate.kernel_name(self.info, variant)
296                 for variant in ("Iq", "Iqxy", "Imagnetic")]
297        self._kernels = [self._dll[name] for name in names]
298        for k in self._kernels:
299            k.argtypes = argtypes
300
301    def __getstate__(self):
302        # type: () -> Tuple[ModelInfo, str]
303        return self.info, self.dllpath
304
305    def __setstate__(self, state):
306        # type: (Tuple[ModelInfo, str]) -> None
307        self.info, self.dllpath = state
308        self._dll = None
309
310    def make_kernel(self, q_vectors):
311        # type: (List[np.ndarray]) -> DllKernel
312        q_input = PyInput(q_vectors, self.dtype)
313        # Note: pickle not supported for DllKernel
314        if self._dll is None:
315            self._load_dll()
316        is_2d = len(q_vectors) == 2
317        kernel = self._kernels[1:3] if is_2d else [self._kernels[0]]*2
318        return DllKernel(kernel, self.info, q_input)
319
320    def release(self):
321        # type: () -> None
322        """
323        Release any resources associated with the model.
324        """
325        if os.name == 'nt':
326            #dll = ct.cdll.LoadLibrary(self.dllpath)
327            dll = ct.CDLL(self.dllpath)
328            dll_handle = dll._handle
329            #dll_handle = ct.c_void_p(dll._handle)
330            del dll, self._dll
331            self._dll = None
332            ct.windll.kernel32.FreeLibrary(dll_handle)
333        else:
334            pass
335
336
337class DllKernel(Kernel):
338    """
339    Callable SAS kernel.
340
341    *kernel* is the c function to call.
342
343    *model_info* is the module information
344
345    *q_input* is the DllInput q vectors at which the kernel should be
346    evaluated.
347
348    The resulting call method takes the *pars*, a list of values for
349    the fixed parameters to the kernel, and *pd_pars*, a list of (value, weight)
350    vectors for the polydisperse parameters.  *cutoff* determines the
351    integration limits: any points with combined weight less than *cutoff*
352    will not be calculated.
353
354    Call :meth:`release` when done with the kernel instance.
355    """
356    def __init__(self, kernel, model_info, q_input):
357        # type: (Callable[[], np.ndarray], ModelInfo, PyInput) -> None
358        self.kernel = kernel
359        self.info = model_info
360        self.q_input = q_input
361        self.dtype = q_input.dtype
362        self.dim = '2d' if q_input.is_2d else '1d'
363        self.result = np.empty(q_input.nq+1, q_input.dtype)
364        self.real = (np.float32 if self.q_input.dtype == generate.F32
365                     else np.float64 if self.q_input.dtype == generate.F64
366                     else np.float128)
367
368    def __call__(self, call_details, values, cutoff, magnetic):
369        # type: (CallDetails, np.ndarray, np.ndarray, float, bool) -> np.ndarray
370
371        kernel = self.kernel[1 if magnetic else 0]
372        args = [
373            self.q_input.nq, # nq
374            None, # pd_start
375            None, # pd_stop pd_stride[MAX_PD]
376            call_details.buffer.ctypes.data, # problem
377            values.ctypes.data,  #pars
378            self.q_input.q.ctypes.data, #q
379            self.result.ctypes.data,   # results
380            self.real(cutoff), # cutoff
381        ]
382        #print("Calling DLL")
383        #call_details.show(values)
384        step = 100
385        for start in range(0, call_details.num_eval, step):
386            stop = min(start + step, call_details.num_eval)
387            args[1:3] = [start, stop]
388            kernel(*args) # type: ignore
389
390        #print("returned",self.q_input.q, self.result)
391        pd_norm = self.result[self.q_input.nq]
392        scale = values[0]/(pd_norm if pd_norm!=0.0 else 1.0)
393        background = values[1]
394        #print("scale",scale,background)
395        return scale*self.result[:self.q_input.nq] + background
396
397    def release(self):
398        # type: () -> None
399        """
400        Release any resources associated with the kernel.
401        """
402        self.q_input.release()
Note: See TracBrowser for help on using the repository browser.