source: sasmodels/sasmodels/kerneldll.py @ 99658f6

core_shell_microgelsmagnetic_modelticket-1257-vesicle-productticket_1156ticket_1265_superballticket_822_more_unit_tests
Last change on this file since 99658f6 was e44432d, checked in by Paul Kienzle <pkienzle@…>, 5 years ago

support hollow models in structure factor calculations

  • Property mode set to 100644
File size: 15.7 KB
Line 
1r"""
2DLL driver for C kernels
3
4If the environment variable *SAS_OPENMP* is set, then sasmodels
5will attempt to compile with OpenMP flags so that the model can use all
6available kernels.  This may or may not be available on your compiler
7toolchain.  Depending on operating system and environment.
8
9Windows does not have provide a compiler with the operating system.
10Instead, we assume that TinyCC is installed and available.  This can
11be done with a simple pip command if it is not already available::
12
13    pip install tinycc
14
15If Microsoft Visual C++ is available (because VCINSTALLDIR is
16defined in the environment), then that will be used instead.
17Microsoft Visual C++ for Python is available from Microsoft:
18
19    `<http://www.microsoft.com/en-us/download/details.aspx?id=44266>`_
20
21If neither compiler is available, sasmodels will check for *MinGW*,
22the GNU compiler toolchain. This available in packages such as Anaconda
23and PythonXY, or available stand alone. This toolchain has had
24difficulties on some systems, and may or may not work for you.
25
26You can control which compiler to use by setting SAS_COMPILER in the
27environment:
28
29  - tinycc (Windows): use the TinyCC compiler shipped with SasView
30  - msvc (Windows): use the Microsoft Visual C++ compiler
31  - mingw (Windows): use the MinGW GNU cc compiler
32  - unix (Linux): use the system cc compiler.
33  - unix (Mac): use the clang compiler. You will need XCode installed, and
34    the XCode command line tools. Mac comes with OpenCL drivers, so generally
35    this will not be needed.
36
37Both *msvc* and *mingw* require that the compiler is available on your path.
38For *msvc*, this can done by running vcvarsall.bat in a windows terminal.
39Install locations are system dependent, such as:
40
41    C:\Program Files (x86)\Common Files\Microsoft\Visual
42    C++ for Python\9.0\vcvarsall.bat
43
44or maybe
45
46    C:\Users\yourname\AppData\Local\Programs\Common\Microsoft\Visual
47    C++ for Python\9.0\vcvarsall.bat
48
49OpenMP for *msvc* requires the Microsoft vcomp90.dll library, which doesn't
50seem to be included with the compiler, nor does there appear to be a public
51download location.  There may be one on your machine already in a location
52such as:
53
54    C:\Windows\winsxs\x86_microsoft.vc90.openmp*\vcomp90.dll
55
56If you copy this to somewhere on your path, such as the python directory or
57the install directory for this application, then OpenMP should be supported.
58
59For full control of the compiler, define a function
60*compile_command(source,output)* which takes the name of the source file
61and the name of the output file and returns a compile command that can be
62evaluated in the shell.  For even more control, replace the entire
63*compile(source,output)* function.
64
65The global attribute *ALLOW_SINGLE_PRECISION_DLLS* should be set to *False* if
66you wish to prevent single precision floating point evaluation for the compiled
67models, otherwise set it defaults to *True*.
68"""
69from __future__ import print_function
70
71import sys
72import os
73from os.path import join as joinpath, splitext
74import subprocess
75import tempfile
76import ctypes as ct  # type: ignore
77import _ctypes as _ct
78import logging
79
80import numpy as np  # type: ignore
81
82try:
83    import tinycc
84except ImportError:
85    tinycc = None
86
87from . import generate
88from .kernel import KernelModel, Kernel
89from .kernelpy import PyInput
90from .exception import annotate_exception
91from .generate import F16, F32, F64
92
93# pylint: disable=unused-import
94try:
95    from typing import Tuple, Callable, Any
96    from .modelinfo import ModelInfo
97    from .details import CallDetails
98except ImportError:
99    pass
100# pylint: enable=unused-import
101
102# Compiler output is a byte stream that needs to be decode in python 3
103decode = (lambda s: s) if sys.version_info[0] < 3 else (lambda s: s.decode('utf8'))
104
105if "SAS_DLL_PATH" in os.environ:
106    SAS_DLL_PATH = os.environ["SAS_DLL_PATH"]
107else:
108    # Assume the default location of module DLLs is in .sasmodels/compiled_models.
109    SAS_DLL_PATH = os.path.join(os.path.expanduser("~"), ".sasmodels", "compiled_models")
110
111if "SAS_COMPILER" in os.environ:
112    COMPILER = os.environ["SAS_COMPILER"]
113elif os.name == 'nt':
114    if tinycc is not None:
115        COMPILER = "tinycc"
116    elif "VCINSTALLDIR" in os.environ:
117        # If vcvarsall.bat has been called, then VCINSTALLDIR is in the environment
118        # and we can use the MSVC compiler.  Otherwise, if tinycc is available
119        # the use it.  Otherwise, hope that mingw is available.
120        COMPILER = "msvc"
121    else:
122        COMPILER = "mingw"
123else:
124    COMPILER = "unix"
125
126ARCH = "" if ct.sizeof(ct.c_void_p) > 4 else "x86"  # 4 byte pointers on x86
127if COMPILER == "unix":
128    # Generic unix compile
129    # On mac users will need the X code command line tools installed
130    #COMPILE = "gcc-mp-4.7 -shared -fPIC -std=c99 -fopenmp -O2 -Wall %s -o %s -lm -lgomp"
131    CC = "cc -shared -fPIC -std=c99 -O2 -Wall".split()
132    # add openmp support if not running on a mac
133    if sys.platform != "darwin":
134        # OpenMP seems to be broken on gcc 5.4.0 (ubuntu 16.04.9)
135        # Shut it off for all unix until we can investigate.
136        #CC.append("-fopenmp")
137        pass
138    def compile_command(source, output):
139        """unix compiler command"""
140        return CC + [source, "-o", output, "-lm"]
141elif COMPILER == "msvc":
142    # Call vcvarsall.bat before compiling to set path, headers, libs, etc.
143    # MSVC compiler is available, so use it.  OpenMP requires a copy of
144    # vcomp90.dll on the path.  One may be found here:
145    #       C:/Windows/winsxs/x86_microsoft.vc90.openmp*/vcomp90.dll
146    # Copy this to the python directory and uncomment the OpenMP COMPILE
147    # TODO: remove intermediate OBJ file created in the directory
148    # TODO: maybe don't use randomized name for the c file
149    # TODO: maybe ask distutils to find MSVC
150    CC = "cl /nologo /Ox /MD /W3 /GS- /DNDEBUG".split()
151    if "SAS_OPENMP" in os.environ:
152        CC.append("/openmp")
153    LN = "/link /DLL /INCREMENTAL:NO /MANIFEST".split()
154    def compile_command(source, output):
155        """MSVC compiler command"""
156        return CC + ["/Tp%s"%source] + LN + ["/OUT:%s"%output]
157elif COMPILER == "tinycc":
158    # TinyCC compiler.
159    CC = [tinycc.TCC] + "-shared -rdynamic -Wall".split()
160    def compile_command(source, output):
161        """tinycc compiler command"""
162        return CC + [source, "-o", output]
163elif COMPILER == "mingw":
164    # MinGW compiler.
165    CC = "gcc -shared -std=c99 -O2 -Wall".split()
166    if "SAS_OPENMP" in os.environ:
167        CC.append("-fopenmp")
168    def compile_command(source, output):
169        """mingw compiler command"""
170        return CC + [source, "-o", output, "-lm"]
171
172ALLOW_SINGLE_PRECISION_DLLS = True
173
174def compile(source, output):
175    # type: (str, str) -> None
176    """
177    Compile *source* producing *output*.
178
179    Raises RuntimeError if the compile failed or the output wasn't produced.
180    """
181    command = compile_command(source=source, output=output)
182    command_str = " ".join('"%s"'%p if ' ' in p else p for p in command)
183    logging.info(command_str)
184    try:
185        # need shell=True on windows to keep console box from popping up
186        shell = (os.name == 'nt')
187        subprocess.check_output(command, shell=shell, stderr=subprocess.STDOUT)
188    except subprocess.CalledProcessError as exc:
189        output = decode(exc.output)
190        raise RuntimeError("compile failed.\n%s\n%s"%(command_str, output))
191    if not os.path.exists(output):
192        raise RuntimeError("compile failed.  File is in %r"%source)
193
194def dll_name(model_info, dtype):
195    # type: (ModelInfo, np.dtype) ->  str
196    """
197    Name of the dll containing the model.  This is the base file name without
198    any path or extension, with a form such as 'sas_sphere32'.
199    """
200    bits = 8*dtype.itemsize
201    basename = "sas%d_%s"%(bits, model_info.id)
202    basename += ARCH + ".so"
203
204    # Hack to find precompiled dlls
205    path = joinpath(generate.DATA_PATH, '..', 'compiled_models', basename)
206    if os.path.exists(path):
207        return path
208
209    return joinpath(SAS_DLL_PATH, basename)
210
211
212def dll_path(model_info, dtype):
213    # type: (ModelInfo, np.dtype) -> str
214    """
215    Complete path to the dll for the model.  Note that the dll may not
216    exist yet if it hasn't been compiled.
217    """
218    return os.path.join(SAS_DLL_PATH, dll_name(model_info, dtype))
219
220
221def make_dll(source, model_info, dtype=F64):
222    # type: (str, ModelInfo, np.dtype) -> str
223    """
224    Returns the path to the compiled model defined by *kernel_module*.
225
226    If the model has not been compiled, or if the source file(s) are newer
227    than the dll, then *make_dll* will compile the model before returning.
228    This routine does not load the resulting dll.
229
230    *dtype* is a numpy floating point precision specifier indicating whether
231    the model should be single, double or long double precision.  The default
232    is double precision, *np.dtype('d')*.
233
234    Set *sasmodels.ALLOW_SINGLE_PRECISION_DLLS* to False if single precision
235    models are not allowed as DLLs.
236
237    Set *sasmodels.kerneldll.SAS_DLL_PATH* to the compiled dll output path.
238    Alternatively, set the environment variable *SAS_DLL_PATH*.
239    The default is in ~/.sasmodels/compiled_models.
240    """
241    if dtype == F16:
242        raise ValueError("16 bit floats not supported")
243    if dtype == F32 and not ALLOW_SINGLE_PRECISION_DLLS:
244        dtype = F64  # Force 64-bit dll
245    # Note: dtype may be F128 for long double precision
246
247    dll = dll_path(model_info, dtype)
248
249    if not os.path.exists(dll):
250        need_recompile = True
251    else:
252        dll_time = os.path.getmtime(dll)
253        newest_source = generate.dll_timestamp(model_info)
254        need_recompile = dll_time < newest_source
255    if need_recompile:
256        # Make sure the DLL path exists
257        if not os.path.exists(SAS_DLL_PATH):
258            os.makedirs(SAS_DLL_PATH)
259        basename = splitext(os.path.basename(dll))[0] + "_"
260        system_fd, filename = tempfile.mkstemp(suffix=".c", prefix=basename)
261        source = generate.convert_type(source, dtype)
262        with os.fdopen(system_fd, "w") as file_handle:
263            file_handle.write(source)
264        compile(source=filename, output=dll)
265        # comment the following to keep the generated c file
266        # Note: if there is a syntax error then compile raises an error
267        # and the source file will not be deleted.
268        os.unlink(filename)
269        #print("saving compiled file in %r"%filename)
270    return dll
271
272
273def load_dll(source, model_info, dtype=F64):
274    # type: (str, ModelInfo, np.dtype) -> "DllModel"
275    """
276    Create and load a dll corresponding to the source, info pair returned
277    from :func:`sasmodels.generate.make` compiled for the target precision.
278
279    See :func:`make_dll` for details on controlling the dll path and the
280    allowed floating point precision.
281    """
282    filename = make_dll(source, model_info, dtype=dtype)
283    return DllModel(filename, model_info, dtype=dtype)
284
285
286class DllModel(KernelModel):
287    """
288    ctypes wrapper for a single model.
289
290    *source* and *model_info* are the model source and interface as returned
291    from :func:`gen.make`.
292
293    *dtype* is the desired model precision.  Any numpy dtype for single
294    or double precision floats will do, such as 'f', 'float32' or 'single'
295    for single and 'd', 'float64' or 'double' for double.  Double precision
296    is an optional extension which may not be available on all devices.
297
298    Call :meth:`release` when done with the kernel.
299    """
300    def __init__(self, dllpath, model_info, dtype=generate.F32):
301        # type: (str, ModelInfo, np.dtype) -> None
302        self.info = model_info
303        self.dllpath = dllpath
304        self._dll = None  # type: ct.CDLL
305        self._kernels = None # type: List[Callable, Callable]
306        self.dtype = np.dtype(dtype)
307
308    def _load_dll(self):
309        # type: () -> None
310        try:
311            self._dll = ct.CDLL(self.dllpath)
312        except:
313            annotate_exception("while loading "+self.dllpath)
314            raise
315
316        float_type = (ct.c_float if self.dtype == generate.F32
317                      else ct.c_double if self.dtype == generate.F64
318                      else ct.c_longdouble)
319
320        # int, int, int, int*, double*, double*, double*, double*, double
321        argtypes = [ct.c_int32]*3 + [ct.c_void_p]*4 + [float_type, ct.c_int32]
322        names = [generate.kernel_name(self.info, variant)
323                 for variant in ("Iq", "Iqxy", "Imagnetic")]
324        self._kernels = [self._dll[name] for name in names]
325        for k in self._kernels:
326            k.argtypes = argtypes
327
328    def __getstate__(self):
329        # type: () -> Tuple[ModelInfo, str]
330        return self.info, self.dllpath
331
332    def __setstate__(self, state):
333        # type: (Tuple[ModelInfo, str]) -> None
334        self.info, self.dllpath = state
335        self._dll = None
336
337    def make_kernel(self, q_vectors):
338        # type: (List[np.ndarray]) -> DllKernel
339        q_input = PyInput(q_vectors, self.dtype)
340        # Note: pickle not supported for DllKernel
341        if self._dll is None:
342            self._load_dll()
343        is_2d = len(q_vectors) == 2
344        kernel = self._kernels[1:3] if is_2d else [self._kernels[0]]*2
345        return DllKernel(kernel, self.info, q_input)
346
347    def release(self):
348        # type: () -> None
349        """
350        Release any resources associated with the model.
351        """
352        dll_handle = self._dll._handle
353        if os.name == 'nt':
354            ct.windll.kernel32.FreeLibrary(dll_handle)
355        else:
356            _ct.dlclose(dll_handle)
357        del self._dll
358        self._dll = None
359
360class DllKernel(Kernel):
361    """
362    Callable SAS kernel.
363
364    *kernel* is the c function to call.
365
366    *model_info* is the module information
367
368    *q_input* is the DllInput q vectors at which the kernel should be
369    evaluated.
370
371    The resulting call method takes the *pars*, a list of values for
372    the fixed parameters to the kernel, and *pd_pars*, a list of (value, weight)
373    vectors for the polydisperse parameters.  *cutoff* determines the
374    integration limits: any points with combined weight less than *cutoff*
375    will not be calculated.
376
377    Call :meth:`release` when done with the kernel instance.
378    """
379    def __init__(self, kernel, model_info, q_input):
380        # type: (Callable[[], np.ndarray], ModelInfo, PyInput) -> None
381        #,model_info,q_input)
382        self.kernel = kernel
383        self.info = model_info
384        self.q_input = q_input
385        self.dtype = q_input.dtype
386        self.dim = '2d' if q_input.is_2d else '1d'
387        # leave room for f1/f2 results in case we need to compute beta for 1d models
388        nout = 2 if self.info.have_Fq else 1
389        # +4 for total weight, shell volume, effective radius, form volume
390        self.result = np.empty(q_input.nq*nout + 4, self.dtype)
391        self.real = (np.float32 if self.q_input.dtype == generate.F32
392                     else np.float64 if self.q_input.dtype == generate.F64
393                     else np.float128)
394
395    def _call_kernel(self, call_details, values, cutoff, magnetic, effective_radius_type):
396        # type: (CallDetails, np.ndarray, np.ndarray, float, bool, int) -> np.ndarray
397        kernel = self.kernel[1 if magnetic else 0]
398        args = [
399            self.q_input.nq, # nq
400            None, # pd_start
401            None, # pd_stop pd_stride[MAX_PD]
402            call_details.buffer.ctypes.data, # problem
403            values.ctypes.data,  # pars
404            self.q_input.q.ctypes.data, # q
405            self.result.ctypes.data,   # results
406            self.real(cutoff), # cutoff
407            effective_radius_type, # cutoff
408        ]
409        #print("Calling DLL")
410        #call_details.show(values)
411        step = 100
412        for start in range(0, call_details.num_eval, step):
413            stop = min(start + step, call_details.num_eval)
414            args[1:3] = [start, stop]
415            kernel(*args) # type: ignore
416
417    def release(self):
418        # type: () -> None
419        """
420        Release any resources associated with the kernel.
421        """
422        self.q_input.release()
Note: See TracBrowser for help on using the repository browser.