2 * Copyright (C) 2013 Red Hat
3 * Author: Rob Clark <robdclark@gmail.com>
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published by
7 * the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program. If not, see <http://www.gnu.org/licenses/>.
23 * Cmdstream submission:
26 /* make sure these don't conflict w/ MSM_SUBMIT_BO_x */
27 #define BO_VALID 0x8000
28 #define BO_LOCKED 0x4000
29 #define BO_PINNED 0x2000
31 static struct msm_gem_submit *submit_create(struct drm_device *dev,
32 struct msm_gpu *gpu, int nr)
34 struct msm_gem_submit *submit;
35 int sz = sizeof(*submit) + (nr * sizeof(submit->bos[0]));
37 submit = kmalloc(sz, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
42 /* initially, until copy_from_user() and bo lookup succeeds: */
46 INIT_LIST_HEAD(&submit->bo_list);
47 ww_acquire_init(&submit->ticket, &reservation_ww_class);
53 static int submit_lookup_objects(struct msm_gem_submit *submit,
54 struct drm_msm_gem_submit *args, struct drm_file *file)
59 spin_lock(&file->table_lock);
61 for (i = 0; i < args->nr_bos; i++) {
62 struct drm_msm_gem_submit_bo submit_bo;
63 struct drm_gem_object *obj;
64 struct msm_gem_object *msm_obj;
65 void __user *userptr =
66 u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));
68 ret = copy_from_user(&submit_bo, userptr, sizeof(submit_bo));
74 if (submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) {
75 DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
80 submit->bos[i].flags = submit_bo.flags;
81 /* in validate_objects() we figure out if this is true: */
82 submit->bos[i].iova = submit_bo.presumed;
84 /* normally use drm_gem_object_lookup(), but for bulk lookup
85 * all under single table_lock just hit object_idr directly:
87 obj = idr_find(&file->object_idr, submit_bo.handle);
89 DRM_ERROR("invalid handle %u at index %u\n", submit_bo.handle, i);
94 msm_obj = to_msm_bo(obj);
96 if (!list_empty(&msm_obj->submit_entry)) {
97 DRM_ERROR("handle %u at index %u already on submit list\n",
103 drm_gem_object_reference(obj);
105 submit->bos[i].obj = msm_obj;
107 list_add_tail(&msm_obj->submit_entry, &submit->bo_list);
112 spin_unlock(&file->table_lock);
117 static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i)
119 struct msm_gem_object *msm_obj = submit->bos[i].obj;
121 if (submit->bos[i].flags & BO_PINNED)
122 msm_gem_put_iova(&msm_obj->base, submit->gpu->id);
124 if (submit->bos[i].flags & BO_LOCKED)
125 ww_mutex_unlock(&msm_obj->resv->lock);
127 if (!(submit->bos[i].flags & BO_VALID))
128 submit->bos[i].iova = 0;
130 submit->bos[i].flags &= ~(BO_LOCKED | BO_PINNED);
133 /* This is where we make sure all the bo's are reserved and pin'd: */
134 static int submit_validate_objects(struct msm_gem_submit *submit)
136 int contended, slow_locked = -1, i, ret = 0;
139 submit->valid = true;
141 for (i = 0; i < submit->nr_bos; i++) {
142 struct msm_gem_object *msm_obj = submit->bos[i].obj;
145 if (slow_locked == i)
150 if (!(submit->bos[i].flags & BO_LOCKED)) {
151 ret = ww_mutex_lock_interruptible(&msm_obj->resv->lock,
155 submit->bos[i].flags |= BO_LOCKED;
159 /* if locking succeeded, pin bo: */
160 ret = msm_gem_get_iova_locked(&msm_obj->base,
161 submit->gpu->id, &iova);
163 /* this would break the logic in the fail path.. there is no
164 * reason for this to happen, but just to be on the safe side
165 * let's notice if this starts happening in the future:
167 WARN_ON(ret == -EDEADLK);
172 submit->bos[i].flags |= BO_PINNED;
174 if (iova == submit->bos[i].iova) {
175 submit->bos[i].flags |= BO_VALID;
177 submit->bos[i].iova = iova;
178 submit->bos[i].flags &= ~BO_VALID;
179 submit->valid = false;
183 ww_acquire_done(&submit->ticket);
189 submit_unlock_unpin_bo(submit, i);
192 submit_unlock_unpin_bo(submit, slow_locked);
194 if (ret == -EDEADLK) {
195 struct msm_gem_object *msm_obj = submit->bos[contended].obj;
196 /* we lost out in a seqno race, lock and retry.. */
197 ret = ww_mutex_lock_slow_interruptible(&msm_obj->resv->lock,
200 submit->bos[contended].flags |= BO_LOCKED;
201 slow_locked = contended;
209 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
210 struct msm_gem_object **obj, uint32_t *iova, bool *valid)
212 if (idx >= submit->nr_bos) {
213 DRM_ERROR("invalid buffer index: %u (out of %u)\n",
214 idx, submit->nr_bos);
219 *obj = submit->bos[idx].obj;
221 *iova = submit->bos[idx].iova;
223 *valid = !!(submit->bos[idx].flags & BO_VALID);
228 /* process the reloc's and patch up the cmdstream as needed: */
229 static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *obj,
230 uint32_t offset, uint32_t nr_relocs, uint64_t relocs)
232 uint32_t i, last_offset = 0;
237 DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset);
241 /* For now, just map the entire thing. Eventually we probably
242 * to do it page-by-page, w/ kmap() if not vmap()d..
244 ptr = msm_gem_vaddr_locked(&obj->base);
248 DBG("failed to map: %d", ret);
252 for (i = 0; i < nr_relocs; i++) {
253 struct drm_msm_gem_submit_reloc submit_reloc;
254 void __user *userptr =
255 u64_to_user_ptr(relocs + (i * sizeof(submit_reloc)));
259 ret = copy_from_user(&submit_reloc, userptr, sizeof(submit_reloc));
263 if (submit_reloc.submit_offset % 4) {
264 DRM_ERROR("non-aligned reloc offset: %u\n",
265 submit_reloc.submit_offset);
269 /* offset in dwords: */
270 off = submit_reloc.submit_offset / 4;
272 if ((off >= (obj->base.size / 4)) ||
273 (off < last_offset)) {
274 DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
278 ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);
285 iova += submit_reloc.reloc_offset;
287 if (submit_reloc.shift < 0)
288 iova >>= -submit_reloc.shift;
290 iova <<= submit_reloc.shift;
292 ptr[off] = iova | submit_reloc.or;
300 static void submit_cleanup(struct msm_gem_submit *submit, bool fail)
304 for (i = 0; i < submit->nr_bos; i++) {
305 struct msm_gem_object *msm_obj = submit->bos[i].obj;
306 submit_unlock_unpin_bo(submit, i);
307 list_del_init(&msm_obj->submit_entry);
308 drm_gem_object_unreference(&msm_obj->base);
311 ww_acquire_fini(&submit->ticket);
314 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
315 struct drm_file *file)
317 struct msm_drm_private *priv = dev->dev_private;
318 struct drm_msm_gem_submit *args = data;
319 struct msm_file_private *ctx = file->driver_priv;
320 struct msm_gem_submit *submit;
321 struct msm_gpu *gpu = priv->gpu;
328 /* for now, we just have 3d pipe.. eventually this would need to
329 * be more clever to dispatch to appropriate gpu module:
331 if (args->pipe != MSM_PIPE_3D0)
334 if (args->nr_cmds > MAX_CMDS)
337 submit = submit_create(dev, gpu, args->nr_bos);
341 mutex_lock(&dev->struct_mutex);
343 ret = submit_lookup_objects(submit, args, file);
347 ret = submit_validate_objects(submit);
351 for (i = 0; i < args->nr_cmds; i++) {
352 struct drm_msm_gem_submit_cmd submit_cmd;
353 void __user *userptr =
354 u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
355 struct msm_gem_object *msm_obj;
358 ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
364 /* validate input from userspace: */
365 switch (submit_cmd.type) {
366 case MSM_SUBMIT_CMD_BUF:
367 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
368 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
371 DRM_ERROR("invalid type: %08x\n", submit_cmd.type);
376 ret = submit_bo(submit, submit_cmd.submit_idx,
377 &msm_obj, &iova, NULL);
381 if (submit_cmd.size % 4) {
382 DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
388 if ((submit_cmd.size + submit_cmd.submit_offset) >=
389 msm_obj->base.size) {
390 DRM_ERROR("invalid cmdstream size: %u\n", submit_cmd.size);
395 submit->cmd[i].type = submit_cmd.type;
396 submit->cmd[i].size = submit_cmd.size / 4;
397 submit->cmd[i].iova = iova + submit_cmd.submit_offset;
398 submit->cmd[i].idx = submit_cmd.submit_idx;
403 ret = submit_reloc(submit, msm_obj, submit_cmd.submit_offset,
404 submit_cmd.nr_relocs, submit_cmd.relocs);
411 ret = msm_gpu_submit(gpu, submit, ctx);
413 args->fence = submit->fence;
416 submit_cleanup(submit, !!ret);
417 mutex_unlock(&dev->struct_mutex);