source: sasmodels/sasmodels/kerneldll.py @ 49d1f8b8

core_shell_microgelsmagnetic_modelticket-1257-vesicle-productticket_1156ticket_1265_superballticket_822_more_unit_tests
Last change on this file since 49d1f8b8 was 2d81cfe, checked in by Paul Kienzle <pkienzle@…>, 6 years ago

lint

  • Property mode set to 100644
File size: 15.3 KB
Line 
1r"""
2DLL driver for C kernels
3
4If the environment variable *SAS_OPENMP* is set, then sasmodels
5will attempt to compile with OpenMP flags so that the model can use all
6available kernels.  This may or may not be available on your compiler
7toolchain.  Depending on operating system and environment.
8
9Windows does not have provide a compiler with the operating system.
10Instead, we assume that TinyCC is installed and available.  This can
11be done with a simple pip command if it is not already available::
12
13    pip install tinycc
14
15If Microsoft Visual C++ is available (because VCINSTALLDIR is
16defined in the environment), then that will be used instead.
17Microsoft Visual C++ for Python is available from Microsoft:
18
19    `<http://www.microsoft.com/en-us/download/details.aspx?id=44266>`_
20
21If neither compiler is available, sasmodels will check for *MinGW*,
22the GNU compiler toolchain. This available in packages such as Anaconda
23and PythonXY, or available stand alone. This toolchain has had
24difficulties on some systems, and may or may not work for you.
25
26You can control which compiler to use by setting SAS_COMPILER in the
27environment:
28
29  - tinycc (Windows): use the TinyCC compiler shipped with SasView
30  - msvc (Windows): use the Microsoft Visual C++ compiler
31  - mingw (Windows): use the MinGW GNU cc compiler
32  - unix (Linux): use the system cc compiler.
33  - unix (Mac): use the clang compiler. You will need XCode installed, and
34    the XCode command line tools. Mac comes with OpenCL drivers, so generally
35    this will not be needed.
36
37Both *msvc* and *mingw* require that the compiler is available on your path.
38For *msvc*, this can done by running vcvarsall.bat in a windows terminal.
39Install locations are system dependent, such as:
40
41    C:\Program Files (x86)\Common Files\Microsoft\Visual
42    C++ for Python\9.0\vcvarsall.bat
43
44or maybe
45
46    C:\Users\yourname\AppData\Local\Programs\Common\Microsoft\Visual
47    C++ for Python\9.0\vcvarsall.bat
48
49OpenMP for *msvc* requires the Microsoft vcomp90.dll library, which doesn't
50seem to be included with the compiler, nor does there appear to be a public
51download location.  There may be one on your machine already in a location
52such as:
53
54    C:\Windows\winsxs\x86_microsoft.vc90.openmp*\vcomp90.dll
55
56If you copy this to somewhere on your path, such as the python directory or
57the install directory for this application, then OpenMP should be supported.
58
59For full control of the compiler, define a function
60*compile_command(source,output)* which takes the name of the source file
61and the name of the output file and returns a compile command that can be
62evaluated in the shell.  For even more control, replace the entire
63*compile(source,output)* function.
64
65The global attribute *ALLOW_SINGLE_PRECISION_DLLS* should be set to *False* if
66you wish to prevent single precision floating point evaluation for the compiled
67models, otherwise set it defaults to *True*.
68"""
69from __future__ import print_function
70
71import sys
72import os
73from os.path import join as joinpath, splitext
74import subprocess
75import tempfile
76import ctypes as ct  # type: ignore
77import _ctypes as _ct
78import logging
79
80import numpy as np  # type: ignore
81
82try:
83    import tinycc
84except ImportError:
85    tinycc = None
86
87from . import generate
88from .kernel import KernelModel, Kernel
89from .kernelpy import PyInput
90from .exception import annotate_exception
91from .generate import F16, F32, F64
92
93# pylint: disable=unused-import
94try:
95    from typing import Tuple, Callable, Any
96    from .modelinfo import ModelInfo
97    from .details import CallDetails
98except ImportError:
99    pass
100# pylint: enable=unused-import
101
102if "SAS_COMPILER" in os.environ:
103    COMPILER = os.environ["SAS_COMPILER"]
104elif os.name == 'nt':
105    if tinycc is not None:
106        COMPILER = "tinycc"
107    elif "VCINSTALLDIR" in os.environ:
108        # If vcvarsall.bat has been called, then VCINSTALLDIR is in the environment
109        # and we can use the MSVC compiler.  Otherwise, if tinycc is available
110        # the use it.  Otherwise, hope that mingw is available.
111        COMPILER = "msvc"
112    else:
113        COMPILER = "mingw"
114else:
115    COMPILER = "unix"
116
117ARCH = "" if ct.sizeof(ct.c_void_p) > 4 else "x86"  # 4 byte pointers on x86
118if COMPILER == "unix":
119    # Generic unix compile
120    # On mac users will need the X code command line tools installed
121    #COMPILE = "gcc-mp-4.7 -shared -fPIC -std=c99 -fopenmp -O2 -Wall %s -o %s -lm -lgomp"
122    CC = "cc -shared -fPIC -std=c99 -O2 -Wall".split()
123    # add openmp support if not running on a mac
124    if sys.platform != "darwin":
125        CC.append("-fopenmp")
126    def compile_command(source, output):
127        """unix compiler command"""
128        return CC + [source, "-o", output, "-lm"]
129elif COMPILER == "msvc":
130    # Call vcvarsall.bat before compiling to set path, headers, libs, etc.
131    # MSVC compiler is available, so use it.  OpenMP requires a copy of
132    # vcomp90.dll on the path.  One may be found here:
133    #       C:/Windows/winsxs/x86_microsoft.vc90.openmp*/vcomp90.dll
134    # Copy this to the python directory and uncomment the OpenMP COMPILE
135    # TODO: remove intermediate OBJ file created in the directory
136    # TODO: maybe don't use randomized name for the c file
137    # TODO: maybe ask distutils to find MSVC
138    CC = "cl /nologo /Ox /MD /W3 /GS- /DNDEBUG".split()
139    if "SAS_OPENMP" in os.environ:
140        CC.append("/openmp")
141    LN = "/link /DLL /INCREMENTAL:NO /MANIFEST".split()
142    def compile_command(source, output):
143        """MSVC compiler command"""
144        return CC + ["/Tp%s"%source] + LN + ["/OUT:%s"%output]
145elif COMPILER == "tinycc":
146    # TinyCC compiler.
147    CC = [tinycc.TCC] + "-shared -rdynamic -Wall".split()
148    def compile_command(source, output):
149        """tinycc compiler command"""
150        return CC + [source, "-o", output]
151elif COMPILER == "mingw":
152    # MinGW compiler.
153    CC = "gcc -shared -std=c99 -O2 -Wall".split()
154    if "SAS_OPENMP" in os.environ:
155        CC.append("-fopenmp")
156    def compile_command(source, output):
157        """mingw compiler command"""
158        return CC + [source, "-o", output, "-lm"]
159
160# Windows-specific solution
161if os.name == 'nt':
162    # Assume the default location of module DLLs is in .sasmodels/compiled_models.
163    DLL_PATH = os.path.join(os.path.expanduser("~"), ".sasmodels", "compiled_models")
164    if not os.path.exists(DLL_PATH):
165        os.makedirs(DLL_PATH)
166else:
167    # Set up the default path for compiled modules.
168    DLL_PATH = tempfile.gettempdir()
169
170ALLOW_SINGLE_PRECISION_DLLS = True
171
172def compile(source, output):
173    # type: (str, str) -> None
174    """
175    Compile *source* producing *output*.
176
177    Raises RuntimeError if the compile failed or the output wasn't produced.
178    """
179    command = compile_command(source=source, output=output)
180    command_str = " ".join('"%s"'%p if ' ' in p else p for p in command)
181    logging.info(command_str)
182    try:
183        # need shell=True on windows to keep console box from popping up
184        shell = (os.name == 'nt')
185        subprocess.check_output(command, shell=shell, stderr=subprocess.STDOUT)
186    except subprocess.CalledProcessError as exc:
187        raise RuntimeError("compile failed.\n%s\n%s"%(command_str, exc.output))
188    if not os.path.exists(output):
189        raise RuntimeError("compile failed.  File is in %r"%source)
190
191def dll_name(model_info, dtype):
192    # type: (ModelInfo, np.dtype) ->  str
193    """
194    Name of the dll containing the model.  This is the base file name without
195    any path or extension, with a form such as 'sas_sphere32'.
196    """
197    bits = 8*dtype.itemsize
198    basename = "sas%d_%s"%(bits, model_info.id)
199    basename += ARCH + ".so"
200
201    # Hack to find precompiled dlls
202    path = joinpath(generate.DATA_PATH, '..', 'compiled_models', basename)
203    if os.path.exists(path):
204        return path
205
206    return joinpath(DLL_PATH, basename)
207
208
209def dll_path(model_info, dtype):
210    # type: (ModelInfo, np.dtype) -> str
211    """
212    Complete path to the dll for the model.  Note that the dll may not
213    exist yet if it hasn't been compiled.
214    """
215    return os.path.join(DLL_PATH, dll_name(model_info, dtype))
216
217
218def make_dll(source, model_info, dtype=F64):
219    # type: (str, ModelInfo, np.dtype) -> str
220    """
221    Returns the path to the compiled model defined by *kernel_module*.
222
223    If the model has not been compiled, or if the source file(s) are newer
224    than the dll, then *make_dll* will compile the model before returning.
225    This routine does not load the resulting dll.
226
227    *dtype* is a numpy floating point precision specifier indicating whether
228    the model should be single, double or long double precision.  The default
229    is double precision, *np.dtype('d')*.
230
231    Set *sasmodels.ALLOW_SINGLE_PRECISION_DLLS* to False if single precision
232    models are not allowed as DLLs.
233
234    Set *sasmodels.kerneldll.DLL_PATH* to the compiled dll output path.
235    The default is the system temporary directory.
236    """
237    if dtype == F16:
238        raise ValueError("16 bit floats not supported")
239    if dtype == F32 and not ALLOW_SINGLE_PRECISION_DLLS:
240        dtype = F64  # Force 64-bit dll
241    # Note: dtype may be F128 for long double precision
242
243    dll = dll_path(model_info, dtype)
244
245    if not os.path.exists(dll):
246        need_recompile = True
247    else:
248        dll_time = os.path.getmtime(dll)
249        newest_source = generate.dll_timestamp(model_info)
250        need_recompile = dll_time < newest_source
251    if need_recompile:
252        basename = splitext(os.path.basename(dll))[0] + "_"
253        system_fd, filename = tempfile.mkstemp(suffix=".c", prefix=basename)
254        source = generate.convert_type(source, dtype)
255        with os.fdopen(system_fd, "w") as file_handle:
256            file_handle.write(source)
257        compile(source=filename, output=dll)
258        # comment the following to keep the generated c file
259        # Note: if there is a syntax error then compile raises an error
260        # and the source file will not be deleted.
261        os.unlink(filename)
262        #print("saving compiled file in %r"%filename)
263    return dll
264
265
266def load_dll(source, model_info, dtype=F64):
267    # type: (str, ModelInfo, np.dtype) -> "DllModel"
268    """
269    Create and load a dll corresponding to the source, info pair returned
270    from :func:`sasmodels.generate.make` compiled for the target precision.
271
272    See :func:`make_dll` for details on controlling the dll path and the
273    allowed floating point precision.
274    """
275    filename = make_dll(source, model_info, dtype=dtype)
276    return DllModel(filename, model_info, dtype=dtype)
277
278
279class DllModel(KernelModel):
280    """
281    ctypes wrapper for a single model.
282
283    *source* and *model_info* are the model source and interface as returned
284    from :func:`gen.make`.
285
286    *dtype* is the desired model precision.  Any numpy dtype for single
287    or double precision floats will do, such as 'f', 'float32' or 'single'
288    for single and 'd', 'float64' or 'double' for double.  Double precision
289    is an optional extension which may not be available on all devices.
290
291    Call :meth:`release` when done with the kernel.
292    """
293    def __init__(self, dllpath, model_info, dtype=generate.F32):
294        # type: (str, ModelInfo, np.dtype) -> None
295        self.info = model_info
296        self.dllpath = dllpath
297        self._dll = None  # type: ct.CDLL
298        self._kernels = None # type: List[Callable, Callable]
299        self.dtype = np.dtype(dtype)
300
301    def _load_dll(self):
302        # type: () -> None
303        try:
304            self._dll = ct.CDLL(self.dllpath)
305        except:
306            annotate_exception("while loading "+self.dllpath)
307            raise
308
309        float_type = (ct.c_float if self.dtype == generate.F32
310                      else ct.c_double if self.dtype == generate.F64
311                      else ct.c_longdouble)
312
313        # int, int, int, int*, double*, double*, double*, double*, double
314        argtypes = [ct.c_int32]*3 + [ct.c_void_p]*4 + [float_type]
315        names = [generate.kernel_name(self.info, variant)
316                 for variant in ("Iq", "Iqxy", "Imagnetic")]
317        self._kernels = [self._dll[name] for name in names]
318        for k in self._kernels:
319            k.argtypes = argtypes
320
321    def __getstate__(self):
322        # type: () -> Tuple[ModelInfo, str]
323        return self.info, self.dllpath
324
325    def __setstate__(self, state):
326        # type: (Tuple[ModelInfo, str]) -> None
327        self.info, self.dllpath = state
328        self._dll = None
329
330    def make_kernel(self, q_vectors):
331        # type: (List[np.ndarray]) -> DllKernel
332        q_input = PyInput(q_vectors, self.dtype)
333        # Note: pickle not supported for DllKernel
334        if self._dll is None:
335            self._load_dll()
336        is_2d = len(q_vectors) == 2
337        kernel = self._kernels[1:3] if is_2d else [self._kernels[0]]*2
338        return DllKernel(kernel, self.info, q_input)
339
340    def release(self):
341        # type: () -> None
342        """
343        Release any resources associated with the model.
344        """
345        dll_handle = self._dll._handle
346        if os.name == 'nt':
347            ct.windll.kernel32.FreeLibrary(dll_handle)
348        else:
349            _ct.dlclose(dll_handle)
350        del self._dll
351        self._dll = None
352
353class DllKernel(Kernel):
354    """
355    Callable SAS kernel.
356
357    *kernel* is the c function to call.
358
359    *model_info* is the module information
360
361    *q_input* is the DllInput q vectors at which the kernel should be
362    evaluated.
363
364    The resulting call method takes the *pars*, a list of values for
365    the fixed parameters to the kernel, and *pd_pars*, a list of (value, weight)
366    vectors for the polydisperse parameters.  *cutoff* determines the
367    integration limits: any points with combined weight less than *cutoff*
368    will not be calculated.
369
370    Call :meth:`release` when done with the kernel instance.
371    """
372    def __init__(self, kernel, model_info, q_input):
373        # type: (Callable[[], np.ndarray], ModelInfo, PyInput) -> None
374        self.kernel = kernel
375        self.info = model_info
376        self.q_input = q_input
377        self.dtype = q_input.dtype
378        self.dim = '2d' if q_input.is_2d else '1d'
379        self.result = np.empty(q_input.nq+1, q_input.dtype)
380        self.real = (np.float32 if self.q_input.dtype == generate.F32
381                     else np.float64 if self.q_input.dtype == generate.F64
382                     else np.float128)
383
384    def __call__(self, call_details, values, cutoff, magnetic):
385        # type: (CallDetails, np.ndarray, np.ndarray, float, bool) -> np.ndarray
386
387        kernel = self.kernel[1 if magnetic else 0]
388        args = [
389            self.q_input.nq, # nq
390            None, # pd_start
391            None, # pd_stop pd_stride[MAX_PD]
392            call_details.buffer.ctypes.data, # problem
393            values.ctypes.data,  #pars
394            self.q_input.q.ctypes.data, #q
395            self.result.ctypes.data,   # results
396            self.real(cutoff), # cutoff
397        ]
398        #print("Calling DLL")
399        #call_details.show(values)
400        step = 100
401        for start in range(0, call_details.num_eval, step):
402            stop = min(start + step, call_details.num_eval)
403            args[1:3] = [start, stop]
404            kernel(*args) # type: ignore
405
406        #print("returned",self.q_input.q, self.result)
407        pd_norm = self.result[self.q_input.nq]
408        scale = values[0]/(pd_norm if pd_norm != 0.0 else 1.0)
409        background = values[1]
410        #print("scale",scale,background)
411        return scale*self.result[:self.q_input.nq] + background
412
413    def release(self):
414        # type: () -> None
415        """
416        Release any resources associated with the kernel.
417        """
418        self.q_input.release()
Note: See TracBrowser for help on using the repository browser.