source: sasmodels/sasmodels/kerneldll.py @ 49d1f8b8

core_shell_microgelsmagnetic_modelticket-1257-vesicle-productticket_1156ticket_1265_superballticket_822_more_unit_tests
Last change on this file since 49d1f8b8 was 2d81cfe, checked in by Paul Kienzle <pkienzle@…>, 6 years ago

lint

  • Property mode set to 100644
File size: 15.3 KB
RevLine 
[92da231]1r"""
[eafc9fa]2DLL driver for C kernels
[750ffa5]3
[3764ec1]4If the environment variable *SAS_OPENMP* is set, then sasmodels
5will attempt to compile with OpenMP flags so that the model can use all
6available kernels.  This may or may not be available on your compiler
7toolchain.  Depending on operating system and environment.
[92da231]8
[3764ec1]9Windows does not have provide a compiler with the operating system.
10Instead, we assume that TinyCC is installed and available.  This can
11be done with a simple pip command if it is not already available::
[92da231]12
[3764ec1]13    pip install tinycc
[92da231]14
[3764ec1]15If Microsoft Visual C++ is available (because VCINSTALLDIR is
16defined in the environment), then that will be used instead.
17Microsoft Visual C++ for Python is available from Microsoft:
[92da231]18
[d138d43]19    `<http://www.microsoft.com/en-us/download/details.aspx?id=44266>`_
[92da231]20
[3764ec1]21If neither compiler is available, sasmodels will check for *MinGW*,
22the GNU compiler toolchain. This available in packages such as Anaconda
23and PythonXY, or available stand alone. This toolchain has had
24difficulties on some systems, and may or may not work for you.
25
26You can control which compiler to use by setting SAS_COMPILER in the
27environment:
28
29  - tinycc (Windows): use the TinyCC compiler shipped with SasView
30  - msvc (Windows): use the Microsoft Visual C++ compiler
31  - mingw (Windows): use the MinGW GNU cc compiler
32  - unix (Linux): use the system cc compiler.
33  - unix (Mac): use the clang compiler. You will need XCode installed, and
34    the XCode command line tools. Mac comes with OpenCL drivers, so generally
35    this will not be needed.
36
37Both *msvc* and *mingw* require that the compiler is available on your path.
38For *msvc*, this can done by running vcvarsall.bat in a windows terminal.
39Install locations are system dependent, such as:
[92da231]40
[2d81cfe]41    C:\Program Files (x86)\Common Files\Microsoft\Visual
42    C++ for Python\9.0\vcvarsall.bat
[92da231]43
44or maybe
45
[2d81cfe]46    C:\Users\yourname\AppData\Local\Programs\Common\Microsoft\Visual
47    C++ for Python\9.0\vcvarsall.bat
[92da231]48
[3764ec1]49OpenMP for *msvc* requires the Microsoft vcomp90.dll library, which doesn't
50seem to be included with the compiler, nor does there appear to be a public
51download location.  There may be one on your machine already in a location
52such as:
[92da231]53
54    C:\Windows\winsxs\x86_microsoft.vc90.openmp*\vcomp90.dll
55
[3764ec1]56If you copy this to somewhere on your path, such as the python directory or
57the install directory for this application, then OpenMP should be supported.
58
59For full control of the compiler, define a function
60*compile_command(source,output)* which takes the name of the source file
61and the name of the output file and returns a compile command that can be
62evaluated in the shell.  For even more control, replace the entire
63*compile(source,output)* function.
64
65The global attribute *ALLOW_SINGLE_PRECISION_DLLS* should be set to *False* if
66you wish to prevent single precision floating point evaluation for the compiled
67models, otherwise set it defaults to *True*.
[14de349]68"""
[eafc9fa]69from __future__ import print_function
[750ffa5]70
[5d4777d]71import sys
72import os
[40a87fa]73from os.path import join as joinpath, splitext
[7e16db7]74import subprocess
[df4dc86]75import tempfile
[7ae2b7f]76import ctypes as ct  # type: ignore
[886fa25]77import _ctypes as _ct
[821a9c6]78import logging
[14de349]79
[7ae2b7f]80import numpy as np  # type: ignore
[14de349]81
[739aad4]82try:
83    import tinycc
84except ImportError:
85    tinycc = None
86
[cb6ecf4]87from . import generate
[f619de7]88from .kernel import KernelModel, Kernel
89from .kernelpy import PyInput
[2c801fe]90from .exception import annotate_exception
[f619de7]91from .generate import F16, F32, F64
92
[2d81cfe]93# pylint: disable=unused-import
[f619de7]94try:
95    from typing import Tuple, Callable, Any
96    from .modelinfo import ModelInfo
97    from .details import CallDetails
98except ImportError:
99    pass
[2d81cfe]100# pylint: enable=unused-import
[14de349]101
[739aad4]102if "SAS_COMPILER" in os.environ:
[b2f1d2f]103    COMPILER = os.environ["SAS_COMPILER"]
[739aad4]104elif os.name == 'nt':
[b2f1d2f]105    if tinycc is not None:
106        COMPILER = "tinycc"
107    elif "VCINSTALLDIR" in os.environ:
108        # If vcvarsall.bat has been called, then VCINSTALLDIR is in the environment
109        # and we can use the MSVC compiler.  Otherwise, if tinycc is available
110        # the use it.  Otherwise, hope that mingw is available.
111        COMPILER = "msvc"
[fb69211]112    else:
[b2f1d2f]113        COMPILER = "mingw"
[5d4777d]114else:
[b2f1d2f]115    COMPILER = "unix"
[739aad4]116
[886fa25]117ARCH = "" if ct.sizeof(ct.c_void_p) > 4 else "x86"  # 4 byte pointers on x86
[b2f1d2f]118if COMPILER == "unix":
[5efe850]119    # Generic unix compile
120    # On mac users will need the X code command line tools installed
[7e16db7]121    #COMPILE = "gcc-mp-4.7 -shared -fPIC -std=c99 -fopenmp -O2 -Wall %s -o %s -lm -lgomp"
[aa343d6]122    CC = "cc -shared -fPIC -std=c99 -O2 -Wall".split()
[5efe850]123    # add openmp support if not running on a mac
[7e16db7]124    if sys.platform != "darwin":
125        CC.append("-fopenmp")
126    def compile_command(source, output):
[40a87fa]127        """unix compiler command"""
[15e74ad]128        return CC + [source, "-o", output, "-lm"]
[b2f1d2f]129elif COMPILER == "msvc":
[739aad4]130    # Call vcvarsall.bat before compiling to set path, headers, libs, etc.
131    # MSVC compiler is available, so use it.  OpenMP requires a copy of
132    # vcomp90.dll on the path.  One may be found here:
133    #       C:/Windows/winsxs/x86_microsoft.vc90.openmp*/vcomp90.dll
134    # Copy this to the python directory and uncomment the OpenMP COMPILE
135    # TODO: remove intermediate OBJ file created in the directory
136    # TODO: maybe don't use randomized name for the c file
137    # TODO: maybe ask distutils to find MSVC
138    CC = "cl /nologo /Ox /MD /W3 /GS- /DNDEBUG".split()
139    if "SAS_OPENMP" in os.environ:
140        CC.append("/openmp")
141    LN = "/link /DLL /INCREMENTAL:NO /MANIFEST".split()
142    def compile_command(source, output):
[40a87fa]143        """MSVC compiler command"""
[739aad4]144        return CC + ["/Tp%s"%source] + LN + ["/OUT:%s"%output]
[b2f1d2f]145elif COMPILER == "tinycc":
[739aad4]146    # TinyCC compiler.
147    CC = [tinycc.TCC] + "-shared -rdynamic -Wall".split()
148    def compile_command(source, output):
[40a87fa]149        """tinycc compiler command"""
[739aad4]150        return CC + [source, "-o", output]
[b2f1d2f]151elif COMPILER == "mingw":
[739aad4]152    # MinGW compiler.
153    CC = "gcc -shared -std=c99 -O2 -Wall".split()
154    if "SAS_OPENMP" in os.environ:
155        CC.append("-fopenmp")
156    def compile_command(source, output):
[40a87fa]157        """mingw compiler command"""
[739aad4]158        return CC + [source, "-o", output, "-lm"]
[df4dc86]159
[ff97458]160# Windows-specific solution
161if os.name == 'nt':
[3ecf034]162    # Assume the default location of module DLLs is in .sasmodels/compiled_models.
163    DLL_PATH = os.path.join(os.path.expanduser("~"), ".sasmodels", "compiled_models")
164    if not os.path.exists(DLL_PATH):
165        os.makedirs(DLL_PATH)
[ff97458]166else:
167    # Set up the default path for compiled modules.
168    DLL_PATH = tempfile.gettempdir()
[5d4777d]169
[5d316e9]170ALLOW_SINGLE_PRECISION_DLLS = True
[5d4777d]171
[7e16db7]172def compile(source, output):
[40a87fa]173    # type: (str, str) -> None
174    """
175    Compile *source* producing *output*.
176
177    Raises RuntimeError if the compile failed or the output wasn't produced.
178    """
[7e16db7]179    command = compile_command(source=source, output=output)
180    command_str = " ".join('"%s"'%p if ' ' in p else p for p in command)
181    logging.info(command_str)
182    try:
[15e74ad]183        # need shell=True on windows to keep console box from popping up
184        shell = (os.name == 'nt')
[aa343d6]185        subprocess.check_output(command, shell=shell, stderr=subprocess.STDOUT)
[7e16db7]186    except subprocess.CalledProcessError as exc:
187        raise RuntimeError("compile failed.\n%s\n%s"%(command_str, exc.output))
188    if not os.path.exists(output):
189        raise RuntimeError("compile failed.  File is in %r"%source)
[750ffa5]190
[151f3bc]191def dll_name(model_info, dtype):
[f619de7]192    # type: (ModelInfo, np.dtype) ->  str
[5d4777d]193    """
[151f3bc]194    Name of the dll containing the model.  This is the base file name without
195    any path or extension, with a form such as 'sas_sphere32'.
[5d4777d]196    """
[151f3bc]197    bits = 8*dtype.itemsize
[56b2687]198    basename = "sas%d_%s"%(bits, model_info.id)
[c7118f4]199    basename += ARCH + ".so"
[01b8659]200
201    # Hack to find precompiled dlls
[c7118f4]202    path = joinpath(generate.DATA_PATH, '..', 'compiled_models', basename)
[01b8659]203    if os.path.exists(path):
204        return path
205
[c7118f4]206    return joinpath(DLL_PATH, basename)
[5d4777d]207
[f619de7]208
[151f3bc]209def dll_path(model_info, dtype):
[f619de7]210    # type: (ModelInfo, np.dtype) -> str
[151f3bc]211    """
212    Complete path to the dll for the model.  Note that the dll may not
213    exist yet if it hasn't been compiled.
214    """
[a738209]215    return os.path.join(DLL_PATH, dll_name(model_info, dtype))
[5d4777d]216
[f619de7]217
218def make_dll(source, model_info, dtype=F64):
219    # type: (str, ModelInfo, np.dtype) -> str
[5d4777d]220    """
[f619de7]221    Returns the path to the compiled model defined by *kernel_module*.
[5d4777d]222
[f619de7]223    If the model has not been compiled, or if the source file(s) are newer
224    than the dll, then *make_dll* will compile the model before returning.
225    This routine does not load the resulting dll.
[5d4777d]226
[aa4946b]227    *dtype* is a numpy floating point precision specifier indicating whether
[f619de7]228    the model should be single, double or long double precision.  The default
229    is double precision, *np.dtype('d')*.
[5d4777d]230
[f619de7]231    Set *sasmodels.ALLOW_SINGLE_PRECISION_DLLS* to False if single precision
232    models are not allowed as DLLs.
[aa4946b]233
234    Set *sasmodels.kerneldll.DLL_PATH* to the compiled dll output path.
235    The default is the system temporary directory.
[5d4777d]236    """
[f619de7]237    if dtype == F16:
[5d316e9]238        raise ValueError("16 bit floats not supported")
[f619de7]239    if dtype == F32 and not ALLOW_SINGLE_PRECISION_DLLS:
240        dtype = F64  # Force 64-bit dll
241    # Note: dtype may be F128 for long double precision
[750ffa5]242
[17bbadd]243    dll = dll_path(model_info, dtype)
[e1454ab]244
245    if not os.path.exists(dll):
[5a91c6b]246        need_recompile = True
247    else:
248        dll_time = os.path.getmtime(dll)
[0dc34c3]249        newest_source = generate.dll_timestamp(model_info)
[5a91c6b]250        need_recompile = dll_time < newest_source
251    if need_recompile:
[40a87fa]252        basename = splitext(os.path.basename(dll))[0] + "_"
253        system_fd, filename = tempfile.mkstemp(suffix=".c", prefix=basename)
[f619de7]254        source = generate.convert_type(source, dtype)
[40a87fa]255        with os.fdopen(system_fd, "w") as file_handle:
256            file_handle.write(source)
[7e16db7]257        compile(source=filename, output=dll)
258        # comment the following to keep the generated c file
259        # Note: if there is a syntax error then compile raises an error
260        # and the source file will not be deleted.
261        os.unlink(filename)
262        #print("saving compiled file in %r"%filename)
[aa4946b]263    return dll
264
265
[f619de7]266def load_dll(source, model_info, dtype=F64):
267    # type: (str, ModelInfo, np.dtype) -> "DllModel"
[aa4946b]268    """
[823e620]269    Create and load a dll corresponding to the source, info pair returned
[aa4946b]270    from :func:`sasmodels.generate.make` compiled for the target precision.
271
272    See :func:`make_dll` for details on controlling the dll path and the
273    allowed floating point precision.
274    """
[17bbadd]275    filename = make_dll(source, model_info, dtype=dtype)
276    return DllModel(filename, model_info, dtype=dtype)
[5d4777d]277
[f619de7]278
279class DllModel(KernelModel):
[14de349]280    """
281    ctypes wrapper for a single model.
282
[17bbadd]283    *source* and *model_info* are the model source and interface as returned
[14de349]284    from :func:`gen.make`.
285
286    *dtype* is the desired model precision.  Any numpy dtype for single
287    or double precision floats will do, such as 'f', 'float32' or 'single'
288    for single and 'd', 'float64' or 'double' for double.  Double precision
289    is an optional extension which may not be available on all devices.
[ff7119b]290
291    Call :meth:`release` when done with the kernel.
[14de349]292    """
[17bbadd]293    def __init__(self, dllpath, model_info, dtype=generate.F32):
[f619de7]294        # type: (str, ModelInfo, np.dtype) -> None
[17bbadd]295        self.info = model_info
[ce27e21]296        self.dllpath = dllpath
[f619de7]297        self._dll = None  # type: ct.CDLL
[40a87fa]298        self._kernels = None # type: List[Callable, Callable]
[750ffa5]299        self.dtype = np.dtype(dtype)
[14de349]300
[ce27e21]301    def _load_dll(self):
[f619de7]302        # type: () -> None
[2c801fe]303        try:
[f619de7]304            self._dll = ct.CDLL(self.dllpath)
[4d76711]305        except:
306            annotate_exception("while loading "+self.dllpath)
[2c801fe]307            raise
[14de349]308
[886fa25]309        float_type = (ct.c_float if self.dtype == generate.F32
310                      else ct.c_double if self.dtype == generate.F64
311                      else ct.c_longdouble)
[ce27e21]312
[a738209]313        # int, int, int, int*, double*, double*, double*, double*, double
[886fa25]314        argtypes = [ct.c_int32]*3 + [ct.c_void_p]*4 + [float_type]
[9eb3632]315        names = [generate.kernel_name(self.info, variant)
316                 for variant in ("Iq", "Iqxy", "Imagnetic")]
317        self._kernels = [self._dll[name] for name in names]
318        for k in self._kernels:
319            k.argtypes = argtypes
[ce27e21]320
321    def __getstate__(self):
[f619de7]322        # type: () -> Tuple[ModelInfo, str]
[eafc9fa]323        return self.info, self.dllpath
[ce27e21]324
325    def __setstate__(self, state):
[f619de7]326        # type: (Tuple[ModelInfo, str]) -> None
[eafc9fa]327        self.info, self.dllpath = state
[f619de7]328        self._dll = None
[ce27e21]329
[48fbd50]330    def make_kernel(self, q_vectors):
[f619de7]331        # type: (List[np.ndarray]) -> DllKernel
[eafc9fa]332        q_input = PyInput(q_vectors, self.dtype)
[f619de7]333        # Note: pickle not supported for DllKernel
334        if self._dll is None:
335            self._load_dll()
[9eb3632]336        is_2d = len(q_vectors) == 2
337        kernel = self._kernels[1:3] if is_2d else [self._kernels[0]]*2
[3c56da87]338        return DllKernel(kernel, self.info, q_input)
[4d76711]339
[eafc9fa]340    def release(self):
[f619de7]341        # type: () -> None
[14de349]342        """
[eafc9fa]343        Release any resources associated with the model.
[14de349]344        """
[886fa25]345        dll_handle = self._dll._handle
[6ad0e87]346        if os.name == 'nt':
[40a87fa]347            ct.windll.kernel32.FreeLibrary(dll_handle)
348        else:
[886fa25]349            _ct.dlclose(dll_handle)
350        del self._dll
351        self._dll = None
[14de349]352
[f619de7]353class DllKernel(Kernel):
[ff7119b]354    """
355    Callable SAS kernel.
356
[b3f6bc3]357    *kernel* is the c function to call.
[ff7119b]358
[17bbadd]359    *model_info* is the module information
[ff7119b]360
[3c56da87]361    *q_input* is the DllInput q vectors at which the kernel should be
[ff7119b]362    evaluated.
363
364    The resulting call method takes the *pars*, a list of values for
[823e620]365    the fixed parameters to the kernel, and *pd_pars*, a list of (value, weight)
[ff7119b]366    vectors for the polydisperse parameters.  *cutoff* determines the
367    integration limits: any points with combined weight less than *cutoff*
368    will not be calculated.
369
370    Call :meth:`release` when done with the kernel instance.
371    """
[17bbadd]372    def __init__(self, kernel, model_info, q_input):
[f619de7]373        # type: (Callable[[], np.ndarray], ModelInfo, PyInput) -> None
[48fbd50]374        self.kernel = kernel
[17bbadd]375        self.info = model_info
[3c56da87]376        self.q_input = q_input
[39cc3be]377        self.dtype = q_input.dtype
[445d1c0]378        self.dim = '2d' if q_input.is_2d else '1d'
[a5b8477]379        self.result = np.empty(q_input.nq+1, q_input.dtype)
380        self.real = (np.float32 if self.q_input.dtype == generate.F32
381                     else np.float64 if self.q_input.dtype == generate.F64
382                     else np.float128)
[14de349]383
[32e3c9b]384    def __call__(self, call_details, values, cutoff, magnetic):
385        # type: (CallDetails, np.ndarray, np.ndarray, float, bool) -> np.ndarray
[48fbd50]386
[9eb3632]387        kernel = self.kernel[1 if magnetic else 0]
[303d8d6]388        args = [
389            self.q_input.nq, # nq
[9eb3632]390            None, # pd_start
391            None, # pd_stop pd_stride[MAX_PD]
[8d62008]392            call_details.buffer.ctypes.data, # problem
[303d8d6]393            values.ctypes.data,  #pars
[48fbd50]394            self.q_input.q.ctypes.data, #q
395            self.result.ctypes.data,   # results
[a5b8477]396            self.real(cutoff), # cutoff
[9eb3632]397        ]
[bde38b5]398        #print("Calling DLL")
399        #call_details.show(values)
[9eb3632]400        step = 100
[bde38b5]401        for start in range(0, call_details.num_eval, step):
402            stop = min(start + step, call_details.num_eval)
[9eb3632]403            args[1:3] = [start, stop]
404            kernel(*args) # type: ignore
405
406        #print("returned",self.q_input.q, self.result)
[14a15a3]407        pd_norm = self.result[self.q_input.nq]
[886fa25]408        scale = values[0]/(pd_norm if pd_norm != 0.0 else 1.0)
[9eb3632]409        background = values[1]
410        #print("scale",scale,background)
411        return scale*self.result[:self.q_input.nq] + background
[14de349]412
413    def release(self):
[f619de7]414        # type: () -> None
[eafc9fa]415        """
416        Release any resources associated with the kernel.
417        """
[f619de7]418        self.q_input.release()
Note: See TracBrowser for help on using the repository browser.