source: sasmodels/sasmodels/kerneldll.py @ 733a3e1

core_shell_microgelscostrafo411magnetic_modelrelease_v0.94release_v0.95ticket-1257-vesicle-productticket_1156ticket_1265_superballticket_822_more_unit_tests
Last change on this file since 733a3e1 was 3764ec1, checked in by Paul Kienzle <pkienzle@…>, 8 years ago

improve docs; support python 3.x

  • Property mode set to 100644
File size: 15.6 KB
Line 
1r"""
2DLL driver for C kernels
3
4If the environment variable *SAS_OPENMP* is set, then sasmodels
5will attempt to compile with OpenMP flags so that the model can use all
6available kernels.  This may or may not be available on your compiler
7toolchain.  Depending on operating system and environment.
8
9Windows does not have provide a compiler with the operating system.
10Instead, we assume that TinyCC is installed and available.  This can
11be done with a simple pip command if it is not already available::
12
13    pip install tinycc
14
15If Microsoft Visual C++ is available (because VCINSTALLDIR is
16defined in the environment), then that will be used instead.
17Microsoft Visual C++ for Python is available from Microsoft:
18
19    `<http://www.microsoft.com/en-us/download/details.aspx?id=44266>`_
20
21If neither compiler is available, sasmodels will check for *MinGW*,
22the GNU compiler toolchain. This available in packages such as Anaconda
23and PythonXY, or available stand alone. This toolchain has had
24difficulties on some systems, and may or may not work for you.
25
26You can control which compiler to use by setting SAS_COMPILER in the
27environment:
28
29  - tinycc (Windows): use the TinyCC compiler shipped with SasView
30  - msvc (Windows): use the Microsoft Visual C++ compiler
31  - mingw (Windows): use the MinGW GNU cc compiler
32  - unix (Linux): use the system cc compiler.
33  - unix (Mac): use the clang compiler. You will need XCode installed, and
34    the XCode command line tools. Mac comes with OpenCL drivers, so generally
35    this will not be needed.
36
37Both *msvc* and *mingw* require that the compiler is available on your path.
38For *msvc*, this can done by running vcvarsall.bat in a windows terminal.
39Install locations are system dependent, such as:
40
41    C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\vcvarsall.bat
42
43or maybe
44
45    C:\Users\yourname\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python\9.0\vcvarsall.bat
46
47OpenMP for *msvc* requires the Microsoft vcomp90.dll library, which doesn't
48seem to be included with the compiler, nor does there appear to be a public
49download location.  There may be one on your machine already in a location
50such as:
51
52    C:\Windows\winsxs\x86_microsoft.vc90.openmp*\vcomp90.dll
53
54If you copy this to somewhere on your path, such as the python directory or
55the install directory for this application, then OpenMP should be supported.
56
57For full control of the compiler, define a function
58*compile_command(source,output)* which takes the name of the source file
59and the name of the output file and returns a compile command that can be
60evaluated in the shell.  For even more control, replace the entire
61*compile(source,output)* function.
62
63The global attribute *ALLOW_SINGLE_PRECISION_DLLS* should be set to *False* if
64you wish to prevent single precision floating point evaluation for the compiled
65models, otherwise set it defaults to *True*.
66"""
67from __future__ import print_function
68
69import sys
70import os
71from os.path import join as joinpath, splitext
72import subprocess
73import tempfile
74import ctypes as ct  # type: ignore
75from ctypes import c_void_p, c_int32, c_longdouble, c_double, c_float  # type: ignore
76import logging
77
78import numpy as np  # type: ignore
79
80try:
81    import tinycc
82except ImportError:
83    tinycc = None
84
85from . import generate
86from .kernel import KernelModel, Kernel
87from .kernelpy import PyInput
88from .exception import annotate_exception
89from .generate import F16, F32, F64
90
91try:
92    from typing import Tuple, Callable, Any
93    from .modelinfo import ModelInfo
94    from .details import CallDetails
95except ImportError:
96    pass
97
98if "SAS_COMPILER" in os.environ:
99    compiler = os.environ["SAS_COMPILER"]
100elif os.name == 'nt':
101    # If vcvarsall.bat has been called, then VCINSTALLDIR is in the environment
102    # and we can use the MSVC compiler.  Otherwise, if tinycc is available
103    # the use it.  Otherwise, hope that mingw is available.
104    if "VCINSTALLDIR" in os.environ:
105        compiler = "msvc"
106    elif tinycc:
107        compiler = "tinycc"
108    else:
109        compiler = "mingw"
110else:
111    compiler = "unix"
112
113ARCH = "" if ct.sizeof(c_void_p) > 4 else "x86"  # 4 byte pointers on x86
114if compiler == "unix":
115    # Generic unix compile
116    # On mac users will need the X code command line tools installed
117    #COMPILE = "gcc-mp-4.7 -shared -fPIC -std=c99 -fopenmp -O2 -Wall %s -o %s -lm -lgomp"
118    CC = "cc -shared -fPIC -std=c99 -O2 -Wall".split()
119    # add openmp support if not running on a mac
120    if sys.platform != "darwin":
121        CC.append("-fopenmp")
122    def compile_command(source, output):
123        """unix compiler command"""
124        return CC + [source, "-o", output, "-lm"]
125elif compiler == "msvc":
126    # Call vcvarsall.bat before compiling to set path, headers, libs, etc.
127    # MSVC compiler is available, so use it.  OpenMP requires a copy of
128    # vcomp90.dll on the path.  One may be found here:
129    #       C:/Windows/winsxs/x86_microsoft.vc90.openmp*/vcomp90.dll
130    # Copy this to the python directory and uncomment the OpenMP COMPILE
131    # TODO: remove intermediate OBJ file created in the directory
132    # TODO: maybe don't use randomized name for the c file
133    # TODO: maybe ask distutils to find MSVC
134    CC = "cl /nologo /Ox /MD /W3 /GS- /DNDEBUG".split()
135    if "SAS_OPENMP" in os.environ:
136        CC.append("/openmp")
137    LN = "/link /DLL /INCREMENTAL:NO /MANIFEST".split()
138    def compile_command(source, output):
139        """MSVC compiler command"""
140        return CC + ["/Tp%s"%source] + LN + ["/OUT:%s"%output]
141elif compiler == "tinycc":
142    # TinyCC compiler.
143    CC = [tinycc.TCC] + "-shared -rdynamic -Wall".split()
144    def compile_command(source, output):
145        """tinycc compiler command"""
146        return CC + [source, "-o", output]
147elif compiler == "mingw":
148    # MinGW compiler.
149    CC = "gcc -shared -std=c99 -O2 -Wall".split()
150    if "SAS_OPENMP" in os.environ:
151        CC.append("-fopenmp")
152    def compile_command(source, output):
153        """mingw compiler command"""
154        return CC + [source, "-o", output, "-lm"]
155
156# Windows-specific solution
157if os.name == 'nt':
158    # Assume the default location of module DLLs is in .sasmodels/compiled_models.
159    DLL_PATH = os.path.join(os.path.expanduser("~"), ".sasmodels", "compiled_models")
160    if not os.path.exists(DLL_PATH):
161        os.makedirs(DLL_PATH)
162else:
163    # Set up the default path for compiled modules.
164    DLL_PATH = tempfile.gettempdir()
165
166ALLOW_SINGLE_PRECISION_DLLS = True
167
168def compile(source, output):
169    # type: (str, str) -> None
170    """
171    Compile *source* producing *output*.
172
173    Raises RuntimeError if the compile failed or the output wasn't produced.
174    """
175    command = compile_command(source=source, output=output)
176    command_str = " ".join('"%s"'%p if ' ' in p else p for p in command)
177    logging.info(command_str)
178    try:
179        # need shell=True on windows to keep console box from popping up
180        shell = (os.name == 'nt')
181        subprocess.check_output(command, shell=shell, stderr=subprocess.STDOUT)
182    except subprocess.CalledProcessError as exc:
183        raise RuntimeError("compile failed.\n%s\n%s"%(command_str, exc.output))
184    if not os.path.exists(output):
185        raise RuntimeError("compile failed.  File is in %r"%source)
186
187def dll_name(model_info, dtype):
188    # type: (ModelInfo, np.dtype) ->  str
189    """
190    Name of the dll containing the model.  This is the base file name without
191    any path or extension, with a form such as 'sas_sphere32'.
192    """
193    bits = 8*dtype.itemsize
194    basename = "sas%d_%s"%(bits, model_info.id)
195    basename += ARCH + ".so"
196
197    # Hack to find precompiled dlls
198    path = joinpath(generate.DATA_PATH, '..', 'compiled_models', basename)
199    if os.path.exists(path):
200        return path
201
202    return joinpath(DLL_PATH, basename)
203
204
205def dll_path(model_info, dtype):
206    # type: (ModelInfo, np.dtype) -> str
207    """
208    Complete path to the dll for the model.  Note that the dll may not
209    exist yet if it hasn't been compiled.
210    """
211    return os.path.join(DLL_PATH, dll_name(model_info, dtype))
212
213
214def make_dll(source, model_info, dtype=F64):
215    # type: (str, ModelInfo, np.dtype) -> str
216    """
217    Returns the path to the compiled model defined by *kernel_module*.
218
219    If the model has not been compiled, or if the source file(s) are newer
220    than the dll, then *make_dll* will compile the model before returning.
221    This routine does not load the resulting dll.
222
223    *dtype* is a numpy floating point precision specifier indicating whether
224    the model should be single, double or long double precision.  The default
225    is double precision, *np.dtype('d')*.
226
227    Set *sasmodels.ALLOW_SINGLE_PRECISION_DLLS* to False if single precision
228    models are not allowed as DLLs.
229
230    Set *sasmodels.kerneldll.DLL_PATH* to the compiled dll output path.
231    The default is the system temporary directory.
232    """
233    if dtype == F16:
234        raise ValueError("16 bit floats not supported")
235    if dtype == F32 and not ALLOW_SINGLE_PRECISION_DLLS:
236        dtype = F64  # Force 64-bit dll
237    # Note: dtype may be F128 for long double precision
238
239    dll = dll_path(model_info, dtype)
240
241    if not os.path.exists(dll):
242        need_recompile = True
243    elif getattr(sys, 'frozen', None) is not None:
244        # TODO: don't suppress time stamp
245        # Currently suppressing recompile when running in a frozen environment
246        need_recompile = False
247    else:
248        dll_time = os.path.getmtime(dll)
249        newest_source = generate.dll_timestamp(model_info)
250        need_recompile = dll_time < newest_source
251    if need_recompile:
252        basename = splitext(os.path.basename(dll))[0] + "_"
253        system_fd, filename = tempfile.mkstemp(suffix=".c", prefix=basename)
254        source = generate.convert_type(source, dtype)
255        with os.fdopen(system_fd, "w") as file_handle:
256            file_handle.write(source)
257        compile(source=filename, output=dll)
258        # comment the following to keep the generated c file
259        # Note: if there is a syntax error then compile raises an error
260        # and the source file will not be deleted.
261        os.unlink(filename)
262        #print("saving compiled file in %r"%filename)
263    return dll
264
265
266def load_dll(source, model_info, dtype=F64):
267    # type: (str, ModelInfo, np.dtype) -> "DllModel"
268    """
269    Create and load a dll corresponding to the source, info pair returned
270    from :func:`sasmodels.generate.make` compiled for the target precision.
271
272    See :func:`make_dll` for details on controlling the dll path and the
273    allowed floating point precision.
274    """
275    filename = make_dll(source, model_info, dtype=dtype)
276    return DllModel(filename, model_info, dtype=dtype)
277
278
279class DllModel(KernelModel):
280    """
281    ctypes wrapper for a single model.
282
283    *source* and *model_info* are the model source and interface as returned
284    from :func:`gen.make`.
285
286    *dtype* is the desired model precision.  Any numpy dtype for single
287    or double precision floats will do, such as 'f', 'float32' or 'single'
288    for single and 'd', 'float64' or 'double' for double.  Double precision
289    is an optional extension which may not be available on all devices.
290
291    Call :meth:`release` when done with the kernel.
292    """
293    def __init__(self, dllpath, model_info, dtype=generate.F32):
294        # type: (str, ModelInfo, np.dtype) -> None
295        self.info = model_info
296        self.dllpath = dllpath
297        self._dll = None  # type: ct.CDLL
298        self._kernels = None # type: List[Callable, Callable]
299        self.dtype = np.dtype(dtype)
300
301    def _load_dll(self):
302        # type: () -> None
303        #print("dll", self.dllpath)
304        try:
305            self._dll = ct.CDLL(self.dllpath)
306        except:
307            annotate_exception("while loading "+self.dllpath)
308            raise
309
310        float_type = (c_float if self.dtype == generate.F32
311                      else c_double if self.dtype == generate.F64
312                      else c_longdouble)
313
314        # int, int, int, int*, double*, double*, double*, double*, double
315        argtypes = [c_int32]*3 + [c_void_p]*4 + [float_type]
316        names = [generate.kernel_name(self.info, variant)
317                 for variant in ("Iq", "Iqxy", "Imagnetic")]
318        self._kernels = [self._dll[name] for name in names]
319        for k in self._kernels:
320            k.argtypes = argtypes
321
322    def __getstate__(self):
323        # type: () -> Tuple[ModelInfo, str]
324        return self.info, self.dllpath
325
326    def __setstate__(self, state):
327        # type: (Tuple[ModelInfo, str]) -> None
328        self.info, self.dllpath = state
329        self._dll = None
330
331    def make_kernel(self, q_vectors):
332        # type: (List[np.ndarray]) -> DllKernel
333        q_input = PyInput(q_vectors, self.dtype)
334        # Note: pickle not supported for DllKernel
335        if self._dll is None:
336            self._load_dll()
337        is_2d = len(q_vectors) == 2
338        kernel = self._kernels[1:3] if is_2d else [self._kernels[0]]*2
339        return DllKernel(kernel, self.info, q_input)
340
341    def release(self):
342        # type: () -> None
343        """
344        Release any resources associated with the model.
345        """
346        if os.name == 'nt':
347            #dll = ct.cdll.LoadLibrary(self.dllpath)
348            dll = ct.CDLL(self.dllpath)
349            dll_handle = dll._handle
350            #dll_handle = ct.c_void_p(dll._handle)
351            del dll, self._dll
352            self._dll = None
353            ct.windll.kernel32.FreeLibrary(dll_handle)
354        else:
355            pass
356
357
358class DllKernel(Kernel):
359    """
360    Callable SAS kernel.
361
362    *kernel* is the c function to call.
363
364    *model_info* is the module information
365
366    *q_input* is the DllInput q vectors at which the kernel should be
367    evaluated.
368
369    The resulting call method takes the *pars*, a list of values for
370    the fixed parameters to the kernel, and *pd_pars*, a list of (value, weight)
371    vectors for the polydisperse parameters.  *cutoff* determines the
372    integration limits: any points with combined weight less than *cutoff*
373    will not be calculated.
374
375    Call :meth:`release` when done with the kernel instance.
376    """
377    def __init__(self, kernel, model_info, q_input):
378        # type: (Callable[[], np.ndarray], ModelInfo, PyInput) -> None
379        self.kernel = kernel
380        self.info = model_info
381        self.q_input = q_input
382        self.dtype = q_input.dtype
383        self.dim = '2d' if q_input.is_2d else '1d'
384        self.result = np.empty(q_input.nq+1, q_input.dtype)
385        self.real = (np.float32 if self.q_input.dtype == generate.F32
386                     else np.float64 if self.q_input.dtype == generate.F64
387                     else np.float128)
388
389    def __call__(self, call_details, values, cutoff, magnetic):
390        # type: (CallDetails, np.ndarray, np.ndarray, float, bool) -> np.ndarray
391
392        kernel = self.kernel[1 if magnetic else 0]
393        args = [
394            self.q_input.nq, # nq
395            None, # pd_start
396            None, # pd_stop pd_stride[MAX_PD]
397            call_details.buffer.ctypes.data, # problem
398            values.ctypes.data,  #pars
399            self.q_input.q.ctypes.data, #q
400            self.result.ctypes.data,   # results
401            self.real(cutoff), # cutoff
402        ]
403        #print("Calling DLL")
404        #call_details.show(values)
405        step = 100
406        for start in range(0, call_details.num_eval, step):
407            stop = min(start + step, call_details.num_eval)
408            args[1:3] = [start, stop]
409            kernel(*args) # type: ignore
410
411        #print("returned",self.q_input.q, self.result)
412        pd_norm = self.result[self.q_input.nq]
413        scale = values[0]/(pd_norm if pd_norm!=0.0 else 1.0)
414        background = values[1]
415        #print("scale",scale,background)
416        return scale*self.result[:self.q_input.nq] + background
417
418    def release(self):
419        # type: () -> None
420        """
421        Release any resources associated with the kernel.
422        """
423        self.q_input.release()
Note: See TracBrowser for help on using the repository browser.