source: sasmodels/sasmodels/kerneldll.py @ 1a3559f

core_shell_microgelsmagnetic_modelticket-1257-vesicle-productticket_1156ticket_1265_superballticket_822_more_unit_tests
Last change on this file since 1a3559f was 1a3559f, checked in by Paul Kienzle <pkienzle@…>, 14 months ago

allow SAS_DLL_PATH to be set via environment variable

  • Property mode set to 100644
File size: 15.5 KB
Line 
1r"""
2DLL driver for C kernels
3
4If the environment variable *SAS_OPENMP* is set, then sasmodels
5will attempt to compile with OpenMP flags so that the model can use all
6available kernels.  This may or may not be available on your compiler
7toolchain.  Depending on operating system and environment.
8
9Windows does not have provide a compiler with the operating system.
10Instead, we assume that TinyCC is installed and available.  This can
11be done with a simple pip command if it is not already available::
12
13    pip install tinycc
14
15If Microsoft Visual C++ is available (because VCINSTALLDIR is
16defined in the environment), then that will be used instead.
17Microsoft Visual C++ for Python is available from Microsoft:
18
19    `<http://www.microsoft.com/en-us/download/details.aspx?id=44266>`_
20
21If neither compiler is available, sasmodels will check for *MinGW*,
22the GNU compiler toolchain. This available in packages such as Anaconda
23and PythonXY, or available stand alone. This toolchain has had
24difficulties on some systems, and may or may not work for you.
25
26You can control which compiler to use by setting SAS_COMPILER in the
27environment:
28
29  - tinycc (Windows): use the TinyCC compiler shipped with SasView
30  - msvc (Windows): use the Microsoft Visual C++ compiler
31  - mingw (Windows): use the MinGW GNU cc compiler
32  - unix (Linux): use the system cc compiler.
33  - unix (Mac): use the clang compiler. You will need XCode installed, and
34    the XCode command line tools. Mac comes with OpenCL drivers, so generally
35    this will not be needed.
36
37Both *msvc* and *mingw* require that the compiler is available on your path.
38For *msvc*, this can done by running vcvarsall.bat in a windows terminal.
39Install locations are system dependent, such as:
40
41    C:\Program Files (x86)\Common Files\Microsoft\Visual
42    C++ for Python\9.0\vcvarsall.bat
43
44or maybe
45
46    C:\Users\yourname\AppData\Local\Programs\Common\Microsoft\Visual
47    C++ for Python\9.0\vcvarsall.bat
48
49OpenMP for *msvc* requires the Microsoft vcomp90.dll library, which doesn't
50seem to be included with the compiler, nor does there appear to be a public
51download location.  There may be one on your machine already in a location
52such as:
53
54    C:\Windows\winsxs\x86_microsoft.vc90.openmp*\vcomp90.dll
55
56If you copy this to somewhere on your path, such as the python directory or
57the install directory for this application, then OpenMP should be supported.
58
59For full control of the compiler, define a function
60*compile_command(source,output)* which takes the name of the source file
61and the name of the output file and returns a compile command that can be
62evaluated in the shell.  For even more control, replace the entire
63*compile(source,output)* function.
64
65The global attribute *ALLOW_SINGLE_PRECISION_DLLS* should be set to *False* if
66you wish to prevent single precision floating point evaluation for the compiled
67models, otherwise set it defaults to *True*.
68"""
69from __future__ import print_function
70
71import sys
72import os
73from os.path import join as joinpath, splitext
74import subprocess
75import tempfile
76import ctypes as ct  # type: ignore
77import _ctypes as _ct
78import logging
79
80import numpy as np  # type: ignore
81
82try:
83    import tinycc
84except ImportError:
85    tinycc = None
86
87from . import generate
88from .kernel import KernelModel, Kernel
89from .kernelpy import PyInput
90from .exception import annotate_exception
91from .generate import F16, F32, F64
92
93# pylint: disable=unused-import
94try:
95    from typing import Tuple, Callable, Any
96    from .modelinfo import ModelInfo
97    from .details import CallDetails
98except ImportError:
99    pass
100# pylint: enable=unused-import
101
102if "SAS_DLL_PATH" in os.environ:
103    SAS_DLL_PATH = os.environ["SAS_DLL_PATH"]
104else:
105    # Assume the default location of module DLLs is in .sasmodels/compiled_models.
106    SAS_DLL_PATH = os.path.join(os.path.expanduser("~"), ".sasmodels", "compiled_models")
107
108if "SAS_COMPILER" in os.environ:
109    COMPILER = os.environ["SAS_COMPILER"]
110elif os.name == 'nt':
111    if tinycc is not None:
112        COMPILER = "tinycc"
113    elif "VCINSTALLDIR" in os.environ:
114        # If vcvarsall.bat has been called, then VCINSTALLDIR is in the environment
115        # and we can use the MSVC compiler.  Otherwise, if tinycc is available
116        # the use it.  Otherwise, hope that mingw is available.
117        COMPILER = "msvc"
118    else:
119        COMPILER = "mingw"
120else:
121    COMPILER = "unix"
122
123ARCH = "" if ct.sizeof(ct.c_void_p) > 4 else "x86"  # 4 byte pointers on x86
124if COMPILER == "unix":
125    # Generic unix compile
126    # On mac users will need the X code command line tools installed
127    #COMPILE = "gcc-mp-4.7 -shared -fPIC -std=c99 -fopenmp -O2 -Wall %s -o %s -lm -lgomp"
128    CC = "cc -shared -fPIC -std=c99 -O2 -Wall".split()
129    # add openmp support if not running on a mac
130    if sys.platform != "darwin":
131        # OpenMP seems to be broken on gcc 5.4.0 (ubuntu 16.04.9)
132        # Shut it off for all unix until we can investigate.
133        #CC.append("-fopenmp")
134        pass
135    def compile_command(source, output):
136        """unix compiler command"""
137        return CC + [source, "-o", output, "-lm"]
138elif COMPILER == "msvc":
139    # Call vcvarsall.bat before compiling to set path, headers, libs, etc.
140    # MSVC compiler is available, so use it.  OpenMP requires a copy of
141    # vcomp90.dll on the path.  One may be found here:
142    #       C:/Windows/winsxs/x86_microsoft.vc90.openmp*/vcomp90.dll
143    # Copy this to the python directory and uncomment the OpenMP COMPILE
144    # TODO: remove intermediate OBJ file created in the directory
145    # TODO: maybe don't use randomized name for the c file
146    # TODO: maybe ask distutils to find MSVC
147    CC = "cl /nologo /Ox /MD /W3 /GS- /DNDEBUG".split()
148    if "SAS_OPENMP" in os.environ:
149        CC.append("/openmp")
150    LN = "/link /DLL /INCREMENTAL:NO /MANIFEST".split()
151    def compile_command(source, output):
152        """MSVC compiler command"""
153        return CC + ["/Tp%s"%source] + LN + ["/OUT:%s"%output]
154elif COMPILER == "tinycc":
155    # TinyCC compiler.
156    CC = [tinycc.TCC] + "-shared -rdynamic -Wall".split()
157    def compile_command(source, output):
158        """tinycc compiler command"""
159        return CC + [source, "-o", output]
160elif COMPILER == "mingw":
161    # MinGW compiler.
162    CC = "gcc -shared -std=c99 -O2 -Wall".split()
163    if "SAS_OPENMP" in os.environ:
164        CC.append("-fopenmp")
165    def compile_command(source, output):
166        """mingw compiler command"""
167        return CC + [source, "-o", output, "-lm"]
168
169ALLOW_SINGLE_PRECISION_DLLS = True
170
171def compile(source, output):
172    # type: (str, str) -> None
173    """
174    Compile *source* producing *output*.
175
176    Raises RuntimeError if the compile failed or the output wasn't produced.
177    """
178    command = compile_command(source=source, output=output)
179    command_str = " ".join('"%s"'%p if ' ' in p else p for p in command)
180    logging.info(command_str)
181    try:
182        # need shell=True on windows to keep console box from popping up
183        shell = (os.name == 'nt')
184        subprocess.check_output(command, shell=shell, stderr=subprocess.STDOUT)
185    except subprocess.CalledProcessError as exc:
186        raise RuntimeError("compile failed.\n%s\n%s"%(command_str, exc.output))
187    if not os.path.exists(output):
188        raise RuntimeError("compile failed.  File is in %r"%source)
189
190def dll_name(model_info, dtype):
191    # type: (ModelInfo, np.dtype) ->  str
192    """
193    Name of the dll containing the model.  This is the base file name without
194    any path or extension, with a form such as 'sas_sphere32'.
195    """
196    bits = 8*dtype.itemsize
197    basename = "sas%d_%s"%(bits, model_info.id)
198    basename += ARCH + ".so"
199
200    # Hack to find precompiled dlls
201    path = joinpath(generate.DATA_PATH, '..', 'compiled_models', basename)
202    if os.path.exists(path):
203        return path
204
205    return joinpath(SAS_DLL_PATH, basename)
206
207
208def dll_path(model_info, dtype):
209    # type: (ModelInfo, np.dtype) -> str
210    """
211    Complete path to the dll for the model.  Note that the dll may not
212    exist yet if it hasn't been compiled.
213    """
214    return os.path.join(SAS_DLL_PATH, dll_name(model_info, dtype))
215
216
217def make_dll(source, model_info, dtype=F64):
218    # type: (str, ModelInfo, np.dtype) -> str
219    """
220    Returns the path to the compiled model defined by *kernel_module*.
221
222    If the model has not been compiled, or if the source file(s) are newer
223    than the dll, then *make_dll* will compile the model before returning.
224    This routine does not load the resulting dll.
225
226    *dtype* is a numpy floating point precision specifier indicating whether
227    the model should be single, double or long double precision.  The default
228    is double precision, *np.dtype('d')*.
229
230    Set *sasmodels.ALLOW_SINGLE_PRECISION_DLLS* to False if single precision
231    models are not allowed as DLLs.
232
233    Set *sasmodels.kerneldll.SAS_DLL_PATH* to the compiled dll output path.
234    Alternatively, set the environment variable *SAS_DLL_PATH*.
235    The default is in ~/.sasmodels/compiled_models.
236    """
237    if dtype == F16:
238        raise ValueError("16 bit floats not supported")
239    if dtype == F32 and not ALLOW_SINGLE_PRECISION_DLLS:
240        dtype = F64  # Force 64-bit dll
241    # Note: dtype may be F128 for long double precision
242
243    dll = dll_path(model_info, dtype)
244
245    if not os.path.exists(dll):
246        need_recompile = True
247    else:
248        dll_time = os.path.getmtime(dll)
249        newest_source = generate.dll_timestamp(model_info)
250        need_recompile = dll_time < newest_source
251    if need_recompile:
252        # Make sure the DLL path exists
253        if not os.path.exists(SAS_DLL_PATH):
254            os.makedirs(SAS_DLL_PATH)
255        basename = splitext(os.path.basename(dll))[0] + "_"
256        system_fd, filename = tempfile.mkstemp(suffix=".c", prefix=basename)
257        source = generate.convert_type(source, dtype)
258        with os.fdopen(system_fd, "w") as file_handle:
259            file_handle.write(source)
260        compile(source=filename, output=dll)
261        # comment the following to keep the generated c file
262        # Note: if there is a syntax error then compile raises an error
263        # and the source file will not be deleted.
264        os.unlink(filename)
265        #print("saving compiled file in %r"%filename)
266    return dll
267
268
269def load_dll(source, model_info, dtype=F64):
270    # type: (str, ModelInfo, np.dtype) -> "DllModel"
271    """
272    Create and load a dll corresponding to the source, info pair returned
273    from :func:`sasmodels.generate.make` compiled for the target precision.
274
275    See :func:`make_dll` for details on controlling the dll path and the
276    allowed floating point precision.
277    """
278    filename = make_dll(source, model_info, dtype=dtype)
279    return DllModel(filename, model_info, dtype=dtype)
280
281
282class DllModel(KernelModel):
283    """
284    ctypes wrapper for a single model.
285
286    *source* and *model_info* are the model source and interface as returned
287    from :func:`gen.make`.
288
289    *dtype* is the desired model precision.  Any numpy dtype for single
290    or double precision floats will do, such as 'f', 'float32' or 'single'
291    for single and 'd', 'float64' or 'double' for double.  Double precision
292    is an optional extension which may not be available on all devices.
293
294    Call :meth:`release` when done with the kernel.
295    """
296    def __init__(self, dllpath, model_info, dtype=generate.F32):
297        # type: (str, ModelInfo, np.dtype) -> None
298        self.info = model_info
299        self.dllpath = dllpath
300        self._dll = None  # type: ct.CDLL
301        self._kernels = None # type: List[Callable, Callable]
302        self.dtype = np.dtype(dtype)
303
304    def _load_dll(self):
305        # type: () -> None
306        try:
307            self._dll = ct.CDLL(self.dllpath)
308        except:
309            annotate_exception("while loading "+self.dllpath)
310            raise
311
312        float_type = (ct.c_float if self.dtype == generate.F32
313                      else ct.c_double if self.dtype == generate.F64
314                      else ct.c_longdouble)
315
316        # int, int, int, int*, double*, double*, double*, double*, double
317        argtypes = [ct.c_int32]*3 + [ct.c_void_p]*4 + [float_type]
318        names = [generate.kernel_name(self.info, variant)
319                 for variant in ("Iq", "Iqxy", "Imagnetic")]
320        self._kernels = [self._dll[name] for name in names]
321        for k in self._kernels:
322            k.argtypes = argtypes
323
324    def __getstate__(self):
325        # type: () -> Tuple[ModelInfo, str]
326        return self.info, self.dllpath
327
328    def __setstate__(self, state):
329        # type: (Tuple[ModelInfo, str]) -> None
330        self.info, self.dllpath = state
331        self._dll = None
332
333    def make_kernel(self, q_vectors):
334        # type: (List[np.ndarray]) -> DllKernel
335        q_input = PyInput(q_vectors, self.dtype)
336        # Note: pickle not supported for DllKernel
337        if self._dll is None:
338            self._load_dll()
339        is_2d = len(q_vectors) == 2
340        kernel = self._kernels[1:3] if is_2d else [self._kernels[0]]*2
341        return DllKernel(kernel, self.info, q_input)
342
343    def release(self):
344        # type: () -> None
345        """
346        Release any resources associated with the model.
347        """
348        dll_handle = self._dll._handle
349        if os.name == 'nt':
350            ct.windll.kernel32.FreeLibrary(dll_handle)
351        else:
352            _ct.dlclose(dll_handle)
353        del self._dll
354        self._dll = None
355
356class DllKernel(Kernel):
357    """
358    Callable SAS kernel.
359
360    *kernel* is the c function to call.
361
362    *model_info* is the module information
363
364    *q_input* is the DllInput q vectors at which the kernel should be
365    evaluated.
366
367    The resulting call method takes the *pars*, a list of values for
368    the fixed parameters to the kernel, and *pd_pars*, a list of (value, weight)
369    vectors for the polydisperse parameters.  *cutoff* determines the
370    integration limits: any points with combined weight less than *cutoff*
371    will not be calculated.
372
373    Call :meth:`release` when done with the kernel instance.
374    """
375    def __init__(self, kernel, model_info, q_input):
376        # type: (Callable[[], np.ndarray], ModelInfo, PyInput) -> None
377        self.kernel = kernel
378        self.info = model_info
379        self.q_input = q_input
380        self.dtype = q_input.dtype
381        self.dim = '2d' if q_input.is_2d else '1d'
382        self.result = np.empty(q_input.nq+1, q_input.dtype)
383        self.real = (np.float32 if self.q_input.dtype == generate.F32
384                     else np.float64 if self.q_input.dtype == generate.F64
385                     else np.float128)
386
387    def __call__(self, call_details, values, cutoff, magnetic):
388        # type: (CallDetails, np.ndarray, np.ndarray, float, bool) -> np.ndarray
389
390        kernel = self.kernel[1 if magnetic else 0]
391        args = [
392            self.q_input.nq, # nq
393            None, # pd_start
394            None, # pd_stop pd_stride[MAX_PD]
395            call_details.buffer.ctypes.data, # problem
396            values.ctypes.data,  #pars
397            self.q_input.q.ctypes.data, #q
398            self.result.ctypes.data,   # results
399            self.real(cutoff), # cutoff
400        ]
401        #print("Calling DLL")
402        #call_details.show(values)
403        step = 100
404        for start in range(0, call_details.num_eval, step):
405            stop = min(start + step, call_details.num_eval)
406            args[1:3] = [start, stop]
407            kernel(*args) # type: ignore
408
409        #print("returned",self.q_input.q, self.result)
410        pd_norm = self.result[self.q_input.nq]
411        scale = values[0]/(pd_norm if pd_norm != 0.0 else 1.0)
412        background = values[1]
413        #print("scale",scale,background)
414        return scale*self.result[:self.q_input.nq] + background
415
416    def release(self):
417        # type: () -> None
418        """
419        Release any resources associated with the kernel.
420        """
421        self.q_input.release()
Note: See TracBrowser for help on using the repository browser.