source: sasmodels/sasmodels/gpu.py @ f5b9a6b

core_shell_microgelscostrafo411magnetic_modelrelease_v0.94release_v0.95ticket-1257-vesicle-productticket_1156ticket_1265_superballticket_822_more_unit_tests
Last change on this file since f5b9a6b was f5b9a6b, checked in by pkienzle, 9 years ago

fiddle calculation of q vector length as sent to the gpu

  • Property mode set to 100644
File size: 12.5 KB
Line 
1"""
2GPU support through OpenCL
3
4There should be a single GPU environment running on the system.  This
5environment is constructed on the first call to :func:`env`, and the
6same environment is returned on each call.
7
8After retrieving the environment, the next step is to create the kernel.
9This is done with a call to :meth:`GpuEnvironment.make_kernel`, which
10returns the type of data used by the kernel.
11
12Next a :class:`GpuData` object should be created with the correct kind
13of data.  This data object can be used by multiple kernels, for example,
14if the target model is a weighted sum of multiple kernels.  The data
15should include any extra evaluation points required to compute the proper
16data smearing.  This need not match the square grid for 2D data if there
17is an index saying which q points are active.
18
19Together the GpuData, the program, and a device form a :class:`GpuKernel`.
20This kernel is used during fitting, receiving new sets of parameters and
21evaluating them.  The output value is stored in an output buffer on the
22devices, where it can be combined with other structure factors and form
23factors and have instrumental resolution effects applied.
24"""
25import numpy as np
26import pyopencl as cl
27from pyopencl import mem_flags as mf
28
29from . import gen
30
31F64_DEFS = """\
32#ifdef cl_khr_fp64
33#  pragma OPENCL EXTENSION cl_khr_fp64: enable
34#endif
35"""
36
37# The max loops number is limited by the amount of local memory available
38# on the device.  You don't want to make this value too big because it will
39# waste resources, nor too small because it may interfere with users trying
40# to do their polydispersity calculations.  A value of 1024 should be much
41# larger than necessary given that cost grows as npts^k where k is the number
42# of polydisperse parameters.
43MAX_LOOPS = 2048
44
45def load_model(kernel_module, dtype="single"):
46    """
47    Load the OpenCL model defined by *kernel_module*.
48
49    Access to the OpenCL device is delayed until the kernel is called
50    so models can be defined without using too many resources.
51    """
52    source, info = gen.make(kernel_module)
53    ## for debugging, save source to a .cl file, edit it, and reload as model
54    #open(info['name']+'.cl','w').write(source)
55    #source = open(info['name']+'.cl','r').read()
56    return GpuModel(source, info, dtype)
57
58ENV = None
59def environment():
60    """
61    Returns a singleton :class:`GpuEnvironment`.
62
63    This provides an OpenCL context and one queue per device.
64    """
65    global ENV
66    if ENV is None:
67        ENV = GpuEnvironment()
68    return ENV
69
70def has_double(device):
71    """
72    Return true if device supports double precision.
73    """
74    return "cl_khr_fp64" in device.extensions
75
76def get_warp(kernel, queue):
77    """
78    Return the size of an execution batch for *kernel* running on *queue*.
79    """
80    return kernel.get_work_group_info(cl.kernel_work_group_info.PREFERRED_WORK_GROUP_SIZE_MULTIPLE,
81                                      queue.device)
82
83def _stretch_input(vector, dtype, extra=1e-3, boundary=32):
84    """
85    Stretch an input vector to the correct boundary.
86
87    Performance on the kernels can drop by a factor of two or more if the
88    number of values to compute does not fall on a nice power of two
89    boundary.   The trailing additional vector elements are given a
90    value of *extra*, and so f(*extra*) will be computed for each of
91    them.  The returned array will thus be a subset of the computed array.
92
93    *boundary* should be a power of 2 which is at least 32 for good
94    performance on current platforms (as of Jan 2015).  It should
95    probably be the max of get_warp(kernel,queue) and
96    device.min_data_type_align_size//4.
97    """
98    remainder = vector.size%boundary
99    if remainder != 0:
100        size = vector.size + (boundary - remainder)
101        vector = np.hstack((vector, [extra]*(size-vector.size)))
102    return np.ascontiguousarray(vector, dtype=dtype)
103
104
105def compile_model(context, source, dtype):
106    """
107    Build a model to run on the gpu.
108
109    Returns the compiled program and its type.  The returned type will
110    be float32 even if the desired type is float64 if any of the
111    devices in the context do not support the cl_khr_fp64 extension.
112    """
113    dtype = np.dtype(dtype)
114    if dtype==gen.F64 and not all(has_double(d) for d in context.devices):
115        raise RuntimeError("Double precision not supported for devices")
116
117    header = F64_DEFS if dtype == gen.F64 else ""
118    if dtype == gen.F32:
119        source = gen.use_single(source)
120    # Note: USE_SINCOS makes the intel cpu slower under opencl
121    if context.devices[0].type == cl.device_type.GPU:
122        header += "#define USE_SINCOS\n"
123    program  = cl.Program(context, header+source).build()
124    return program
125
126
127def make_result(self, size):
128    self.res = np.empty(size, dtype=self.dtype)
129    self.res_b = cl.Buffer(self.program.context, mf.READ_WRITE, self.res.nbytes)
130    return self.res, self.res_b
131
132
133# for now, this returns one device in the context
134# TODO: create a context that contains all devices on all platforms
135class GpuEnvironment(object):
136    """
137    GPU context, with possibly many devices, and one queue per device.
138    """
139    def __init__(self):
140        self.context = cl.create_some_context()
141        self.queues = [cl.CommandQueue(self.context, d)
142                       for d in self.context.devices]
143        # Byte boundary for data alignment
144        #self.data_boundary = max(d.min_data_type_align_size
145        #                         for d in self.context.devices)
146        self.has_double = all(has_double(d) for d in self.context.devices)
147        self.compiled = {}
148
149    def compile_program(self, name, source, dtype):
150        if name not in self.compiled:
151            #print "compiling",name
152            self.compiled[name] = compile_model(self.context, source, dtype)
153        return self.compiled[name]
154
155    def release_program(self, name):
156        if name in self.compiled:
157            self.compiled[name].release()
158            del self.compiled[name]
159
160class GpuModel(object):
161    """
162    GPU wrapper for a single model.
163
164    *source* and *info* are the model source and interface as returned
165    from :func:`gen.make`.
166
167    *dtype* is the desired model precision.  Any numpy dtype for single
168    or double precision floats will do, such as 'f', 'float32' or 'single'
169    for single and 'd', 'float64' or 'double' for double.  Double precision
170    is an optional extension which may not be available on all devices.
171    """
172    def __init__(self, source, info, dtype=gen.F32):
173        self.info = info
174        self.source = source
175        self.dtype = dtype
176        self.program = None # delay program creation
177
178    def __getstate__(self):
179        state = self.__dict__.copy()
180        state['program'] = None
181        return state
182
183    def __setstate__(self, state):
184        self.__dict__ = state.copy()
185
186    def __call__(self, input):
187        if self.dtype != input.dtype:
188            raise TypeError("data and kernel have different types")
189        if self.program is None:
190            self.program = environment().compile_program(self.info['name'],self.source, self.dtype)
191        kernel_name = gen.kernel_name(self.info, input.is_2D)
192        kernel = getattr(self.program, kernel_name)
193        return GpuKernel(kernel, self.info, input)
194
195    def release(self):
196        if self.program is not None:
197            environment().release_program(self.info['name'])
198            self.program = None
199
200    def make_input(self, q_vectors):
201        """
202        Make q input vectors available to the model.
203
204        This only needs to be done once for all models that operate on the
205        same input.  So for example, if you are adding two different models
206        together to compare to a data set, then only one model needs to
207        needs to call make_input, so long as the models have the same dtype.
208        """
209        # Note: the weird interface, where make_input doesn't care which
210        # model calls it, allows us to ask the model to define the data
211        # and the caller does not need to know if it is opencl or ctypes.
212        # The returned data object is opaque.
213        return GpuInput(q_vectors, dtype=self.dtype)
214
215# TODO: check that we don't need a destructor for buffers which go out of scope
216class GpuInput(object):
217    """
218    Make q data available to the gpu.
219
220    *q_vectors* is a list of q vectors, which will be *[q]* for 1-D data,
221    and *[qx, qy]* for 2-D data.  Internally, the vectors will be reallocated
222    to get the best performance on OpenCL, which may involve shifting and
223    stretching the array to better match the memory architecture.  Additional
224    points will be evaluated with *q=1e-3*.
225
226    *dtype* is the data type for the q vectors. The data type should be
227    set to match that of the kernel, which is an attribute of
228    :class:`GpuProgram`.  Note that not all kernels support double
229    precision, so even if the program was created for double precision,
230    the *GpuProgram.dtype* may be single precision.
231
232    Call :meth:`release` when complete.  Even if not called directly, the
233    buffer will be released when the data object is freed.
234    """
235    def __init__(self, q_vectors, dtype=gen.F32):
236        env = environment()
237        self.nq = q_vectors[0].size
238        self.dtype = np.dtype(dtype)
239        self.is_2D = (len(q_vectors) == 2)
240        # TODO: stretch input based on get_warp()
241        # not doing it now since warp depends on kernel, which is not known
242        # at this point, so instead using 32, which is good on the set of
243        # architectures tested so far.
244        self.q_vectors = [_stretch_input(q, self.dtype, 32) for q in q_vectors]
245        self.q_buffers = [
246            cl.Buffer(env.context,  mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=q)
247            for q in self.q_vectors
248        ]
249        self.global_size = [self.q_vectors[0].size]
250
251    def release(self):
252        for b in self.q_buffers:
253            b.release()
254        self.q_buffers = []
255
256class GpuKernel(object):
257    """
258    Callable SAS kernel.
259
260    *kernel* is the GpuKernel object to call.
261
262    *info* is the module information
263
264    *input* is the DllInput q vectors at which the kernel should be
265    evaluated.
266
267    The resulting call method takes the *pars*, a list of values for
268    the fixed parameters to the kernel, and *pd_pars*, a list of (value,weight)
269    vectors for the polydisperse parameters.  *cutoff* determines the
270    integration limits: any points with combined weight less than *cutoff*
271    will not be calculated.
272
273    Call :meth:`release` when done with the kernel instance.
274    """
275    def __init__(self, kernel, info, input):
276        self.input = input
277        self.kernel = kernel
278        self.info = info
279        self.res = np.empty(input.nq, input.dtype)
280        dim = '2d' if input.is_2D else '1d'
281        self.fixed_pars = info['partype']['fixed-'+dim]
282        self.pd_pars = info['partype']['pd-'+dim]
283
284        # Inputs and outputs for each kernel call
285        # Note: res may be shorter than res_b if global_size != nq
286        env = environment()
287        self.loops_b = [cl.Buffer(env.context, mf.READ_WRITE,
288                                  2*MAX_LOOPS*input.dtype.itemsize)
289                        for _ in env.queues]
290        self.res_b = [cl.Buffer(env.context, mf.READ_WRITE,
291                                input.global_size[0]*input.dtype.itemsize)
292                      for _ in env.queues]
293
294
295    def __call__(self, pars, pd_pars, cutoff=1e-5):
296        real = np.float32 if self.input.dtype == gen.F32 else np.float64
297        fixed = [real(p) for p in pars]
298        cutoff = real(cutoff)
299        loops = np.hstack(pd_pars)
300        loops = np.ascontiguousarray(loops.T, self.input.dtype).flatten()
301        Nloops = [np.uint32(len(p[0])) for p in pd_pars]
302        #print "loops",Nloops, loops
303
304        #import sys; print >>sys.stderr,"opencl eval",pars
305        #print "opencl eval",pars
306        if len(loops) > 2*MAX_LOOPS:
307            raise ValueError("too many polydispersity points")
308        device_num = 0
309        res_bi = self.res_b[device_num]
310        queuei = environment().queues[device_num]
311        loops_bi = self.loops_b[device_num]
312        loops_l = cl.LocalMemory(len(loops.data))
313        cl.enqueue_copy(queuei, loops_bi, loops)
314        #ctx = environment().context
315        #loops_bi = cl.Buffer(ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=loops)
316        args = self.input.q_buffers + [res_bi,loops_bi,loops_l,cutoff] + fixed + Nloops
317        self.kernel(queuei, self.input.global_size, None, *args)
318        cl.enqueue_copy(queuei, self.res, res_bi)
319
320        return self.res
321
322    def release(self):
323        for b in self.loops_b:
324            b.release()
325        self.loops_b = []
326        for b in self.res_b:
327            b.release()
328        self.res_b = []
329
330    def __del__(self):
331        self.release()
Note: See TracBrowser for help on using the repository browser.