From 560d3ab0e564e644d585022340f313962828859d Mon Sep 17 00:00:00 2001 From: lingmingqiang Date: Fri, 16 Aug 2019 14:45:13 +0800 Subject: [PATCH] uacce: Break the limit of 2^(MAX_ORDER) page size DMA memory. driver inclusion category: bugfix bugzilla: NA CVE: NA In this patch, we try to reserve more DMA memory for usre space application.At first, we bring SS(share static region) slice, which includes a continuous physical address memory. And, a SS region of a Warpdrive queue can have multiple slices. Before mapping to user space VMA, the slices will be sorted in a physical increasing order and merged those whose physical addresses are continuous. After reserving the memory, several IOCAL system call will be done to get the slices' physical address information for user space. Signed-off-by: yumeng Reviewed-by: xuzaibo Signed-off-by: lingmingqiang Reviewed-by: Zhou Wang Reviewed-by: Yang Yingliang Signed-off-by: Yang Yingliang --- drivers/uacce/uacce.c | 425 +++++++++++++++++++++++++++---------- include/linux/uacce.h | 11 +- include/uapi/linux/uacce.h | 5 + 3 files changed, 332 insertions(+), 109 deletions(-) diff --git a/drivers/uacce/uacce.c b/drivers/uacce/uacce.c index 572151ba38ad..61ece3a0c4b0 100644 --- a/drivers/uacce/uacce.c +++ b/drivers/uacce/uacce.c @@ -2,6 +2,7 @@ /* Copyright (c) 2018 HiSilicon Limited. */ #include #include +#include #include #include #include @@ -290,6 +291,7 @@ static inline void uacce_iommu_unmap_qfr(struct uacce_queue *q, static int uacce_queue_map_qfr(struct uacce_queue *q, struct uacce_qfile_region *qfr) { + /* Only IOMMU mode does this map */ if (!(qfr->flags & UACCE_QFRF_MAP) || (qfr->flags & UACCE_QFRF_DMA)) return 0; @@ -350,7 +352,7 @@ static const struct vm_operations_struct uacce_shm_vm_ops = { static int uacce_qfr_alloc_pages(struct uacce_qfile_region *qfr) { - int gfp_mask = GFP_ATOMIC | __GFP_ZERO; + gfp_t gfp_mask = GFP_ATOMIC | __GFP_ZERO; int i, j; qfr->pages = kcalloc(qfr->nr_pages, sizeof(*qfr->pages), gfp_mask); @@ -398,8 +400,12 @@ static inline int uacce_queue_mmap_qfr(struct uacce_queue *q, ret = remap_pfn_range(vma, vma->vm_start + i * PAGE_SIZE, page_to_pfn(qfr->pages[i]), PAGE_SIZE, vma->vm_page_prot); - if (ret) + if (ret) { + dev_err(q->uacce->pdev, + "remap_pfm_range fail(nr_pgs=%d)!\n", + qfr->nr_pages); return ret; + } } #else @@ -410,16 +416,194 @@ static inline int uacce_queue_mmap_qfr(struct uacce_queue *q, return 0; } -static struct uacce_qfile_region *uacce_create_region(struct uacce_queue *q, +static void uacce_free_dma_buffers(struct uacce_queue *q) +{ + struct uacce_qfile_region *qfr = q->qfrs[UACCE_QFRT_SS]; + struct uacce *uacce = q->uacce; + int i = 0; + + if (!qfr->dma_list) + return; + while (i < qfr->dma_list[0].total_num) { + WARN_ON(!qfr->dma_list[i].size || !qfr->dma_list[i].dma); + dev_dbg(uacce->pdev, "free dma qfr %s (kaddr=%lx, dma=%llx)\n", + uacce_qfrt_str(qfr), + (unsigned long)qfr->dma_list[i].kaddr, + qfr->dma_list[i].dma); + dma_free_coherent(uacce->pdev, qfr->dma_list[i].size, + qfr->dma_list[i].kaddr, + qfr->dma_list[i].dma); + i++; + } + kfree(qfr->dma_list); + qfr->dma_list = NULL; +} + +static int get_sort_base(struct uacce_dma_slice *list, int low, int high, + struct uacce_dma_slice *tmp) +{ + tmp->kaddr = list[low].kaddr; + tmp->size = list[low].size; + tmp->dma = list[low].dma; + + if (low >= high) + return -EINVAL; + while (low < high) { + while (low < high && list[high].dma > tmp->dma) + high--; + list[low].kaddr = list[high].kaddr; + list[low].dma = list[high].dma; + list[low].size = list[high].size; + while (low < high && list[low].dma < tmp->dma) + low++; + list[high].kaddr = list[low].kaddr; + list[high].dma = list[low].dma; + list[high].size = list[low].size; + } + list[low].kaddr = tmp->kaddr; + list[low].dma = tmp->dma; + list[low].size = tmp->size; + + return low; +} + +static void uacce_sort_dma_buffers(struct uacce_dma_slice *list, int low, + int high, struct uacce_dma_slice *tmp) +{ + int pilot, top = 0; + int *idx_list; + + idx_list = kcalloc(list[0].total_num, sizeof(int), + GFP_KERNEL | __GFP_ZERO); + if (!idx_list) + return; + + pilot = get_sort_base(list, low, high, tmp); + if (pilot < 0) + return; + if (pilot > low + 1) { + idx_list[top++] = low; + idx_list[top++] = pilot - 1; + } + if (pilot < high - 1) { + idx_list[top++] = pilot + 1; + idx_list[top++] = high; + } + while (top > 0) { + high = idx_list[--top]; + low = idx_list[--top]; + pilot = get_sort_base(list, low, high, tmp); + if (pilot > low + 1) { + idx_list[top++] = low; + idx_list[top++] = pilot - 1; + } + if (pilot < high - 1) { + idx_list[top++] = pilot + 1; + idx_list[top++] = high; + } + } + + kfree(idx_list); +} + +static int uacce_alloc_dma_buffers(struct uacce_queue *q, + struct vm_area_struct *vma) +{ + struct uacce_qfile_region *qfr = q->qfrs[UACCE_QFRT_SS]; + unsigned long size = vma->vm_end - vma->vm_start; + unsigned long max_size = PAGE_SIZE << (MAX_ORDER - 1); + unsigned long start = vma->vm_start; + struct uacce *uacce = q->uacce; + struct uacce_dma_slice *slice; + int i, ss_num; + + /* Set maximum slice size is 128MB */ + if (max_size > UACCE_GRAN_NUM_MASK << UACCE_GRAN_SHIFT) + max_size = (UACCE_GRAN_NUM_MASK + 1) << (UACCE_GRAN_SHIFT - 1); + + ss_num = (size + max_size - 1) / max_size; + slice = kcalloc(ss_num + 1, sizeof(*slice), GFP_KERNEL | __GFP_ZERO); + if (!slice) + return -ENOMEM; + + qfr->dma_list = slice; + for (i = 0; i < ss_num; i++) { + if (start + max_size > vma->vm_end) + size = vma->vm_end - start; + else + size = max_size; + dev_dbg(uacce->pdev, "allocate dma %ld pages\n", + (size + PAGE_SIZE - 1) >> PAGE_SHIFT); + slice[i].kaddr = dma_alloc_coherent(uacce->pdev, (size + + PAGE_SIZE - 1) & PAGE_MASK, + &slice[i].dma, GFP_KERNEL); + if (!slice[i].kaddr) { + dev_err(uacce->pdev, "alloc dma slice(sz=%ld) fail!\n", + size); + slice[0].total_num = i; + return -ENOMEM; + } + slice[i].size = (size + PAGE_SIZE - 1) & PAGE_MASK; + slice[i].total_num = ss_num; + start += size; + } + + uacce_sort_dma_buffers(slice, 0, slice[0].total_num - 1, + &slice[ss_num]); + + return 0; +} + +static int uacce_mmap_dma_buffers(struct uacce_queue *q, + struct vm_area_struct *vma) +{ + struct uacce_qfile_region *qfr = q->qfrs[UACCE_QFRT_SS]; + struct uacce_dma_slice *slice = qfr->dma_list; + struct uacce *uacce = q->uacce; + unsigned long vm_pgoff; + int ret = 0; + int i = 0; + + /* + * dma_mmap_coherent() requires vm_pgoff as 0 + * restore vm_pfoff to initial value for mmap() + */ + vm_pgoff = vma->vm_pgoff; + vma->vm_pgoff = 0; + while (i < slice[0].total_num && slice[i].size) { + vma->vm_end = vma->vm_start + slice[i].size; + ret = dma_mmap_coherent(uacce->pdev, vma, slice[i].kaddr, + slice[i].dma, + slice[i].size); + if (ret) { + dev_err(uacce->pdev, + "mmap dma buf fail(dma=0x%llx,size=0x%x)!\n", + slice[i].dma, slice[i].size); + goto DMA_MMAP_FAIL; + } + + i++; + vma->vm_start = vma->vm_end; + } + + /* System unmap_region will clean the results, we need do nothing */ +DMA_MMAP_FAIL: + vma->vm_pgoff = vm_pgoff; + vma->vm_start = qfr->iova; + vma->vm_end = vma->vm_start + (qfr->nr_pages << PAGE_SHIFT); + + return ret; +} + +static struct +uacce_qfile_region *uacce_create_region(struct uacce_queue *q, struct vm_area_struct *vma, enum uacce_qfrt type, u32 flags) { struct uacce_qfile_region *qfr; struct uacce *uacce = q->uacce; - unsigned long vm_pgoff; int ret = -ENOMEM; - dev_dbg(uacce->pdev, "create qfr (type=%x, flags=%x)\n", type, flags); qfr = kzalloc(sizeof(*qfr), GFP_ATOMIC); if (!qfr) return ERR_PTR(-ENOMEM); @@ -428,6 +612,7 @@ static struct uacce_qfile_region *uacce_create_region(struct uacce_queue *q, qfr->flags = flags; qfr->iova = vma->vm_start; qfr->nr_pages = vma_pages(vma); + q->qfrs[type] = qfr; if (vma->vm_flags & VM_READ) qfr->prot |= IOMMU_READ; @@ -437,26 +622,24 @@ static struct uacce_qfile_region *uacce_create_region(struct uacce_queue *q, if (flags & UACCE_QFRF_SELFMT) { ret = uacce->ops->mmap(q, vma, qfr); - if (ret) + if (ret) { + dev_err(uacce->pdev, "uacce driver mmap fail!\n"); goto err_with_qfr; + } return qfr; } /* allocate memory */ if (flags & UACCE_QFRF_DMA) { - dev_dbg(uacce->pdev, "allocate dma %d pages\n", qfr->nr_pages); - qfr->kaddr = dma_alloc_coherent(uacce->pdev, qfr->nr_pages << - PAGE_SHIFT, &qfr->dma, - GFP_KERNEL); - if (!qfr->kaddr) { - ret = -ENOMEM; - goto err_with_qfr; - } + ret = uacce_alloc_dma_buffers(q, vma); + if (ret) + goto err_with_pages; } else { - dev_dbg(uacce->pdev, "allocate %d pages\n", qfr->nr_pages); ret = uacce_qfr_alloc_pages(qfr); - if (ret) + if (ret) { + dev_err(uacce->pdev, "alloc page fail!\n"); goto err_with_qfr; + } } /* map to device */ @@ -466,22 +649,10 @@ static struct uacce_qfile_region *uacce_create_region(struct uacce_queue *q, /* mmap to user space */ if (flags & UACCE_QFRF_MMAP) { - if (flags & UACCE_QFRF_DMA) { - - /* dma_mmap_coherent() requires vm_pgoff as 0 - * restore vm_pfoff to initial value for mmap() - */ - dev_dbg(uacce->pdev, "mmap dma qfr\n"); - vm_pgoff = vma->vm_pgoff; - vma->vm_pgoff = 0; - ret = dma_mmap_coherent(uacce->pdev, vma, qfr->kaddr, - qfr->dma, - qfr->nr_pages << PAGE_SHIFT); - vma->vm_pgoff = vm_pgoff; - } else { + if (flags & UACCE_QFRF_DMA) + ret = uacce_mmap_dma_buffers(q, vma); + else ret = uacce_queue_mmap_qfr(q, qfr, vma); - } - if (ret) goto err_with_mapped_qfr; } @@ -492,16 +663,20 @@ static struct uacce_qfile_region *uacce_create_region(struct uacce_queue *q, uacce_queue_unmap_qfr(q, qfr); err_with_pages: if (flags & UACCE_QFRF_DMA) - dma_free_coherent(uacce->pdev, qfr->nr_pages << PAGE_SHIFT, - qfr->kaddr, qfr->dma); + uacce_free_dma_buffers(q); else uacce_qfr_free_pages(qfr); err_with_qfr: kfree(qfr); - + q->qfrs[type] = NULL; return ERR_PTR(ret); } +static struct uacce_qfile_region noiommu_ss_default_qfr = { + .type = UACCE_QFRT_SS, + .flags = UACCE_QFRF_DMA, +}; + /* we assume you have uacce_queue_unmap_qfr(q, qfr) from all related queues */ static void uacce_destroy_region(struct uacce_queue *q, struct uacce_qfile_region *qfr) @@ -509,11 +684,7 @@ static void uacce_destroy_region(struct uacce_queue *q, struct uacce *uacce = q->uacce; if (qfr->flags & UACCE_QFRF_DMA) { - dev_dbg(uacce->pdev, "free dma qfr %s (kaddr=%pK, dma=%llx)\n", - uacce_qfrt_str(qfr), qfr->kaddr, - qfr->dma); - dma_free_coherent(uacce->pdev, qfr->nr_pages << PAGE_SHIFT, - qfr->kaddr, qfr->dma); + uacce_free_dma_buffers(q); } else if (qfr->pages) { if (qfr->flags & UACCE_QFRF_KMAP && qfr->kaddr) { dev_dbg(uacce->pdev, "vunmap qfr %s\n", @@ -524,41 +695,58 @@ static void uacce_destroy_region(struct uacce_queue *q, uacce_qfr_free_pages(qfr); } - kfree(qfr); + if (qfr != &noiommu_ss_default_qfr) + kfree(qfr); } -static long uacce_cmd_share_qfr(struct uacce_queue *tgt, int fd) +static long uacce_cmd_share_qfr(struct uacce_queue *src, int fd) { + struct device *dev = &src->uacce->dev; struct file *filep = fget(fd); - struct uacce_queue *src; + struct uacce_queue *tgt; int ret = -EINVAL; - if (!filep) + if (!filep) { + dev_err(dev, "filep is NULL!\n"); return ret; + } - if (filep->f_op != &uacce_fops) + if (filep->f_op != &uacce_fops) { + dev_err(dev, "file ops mismatch!\n"); goto out_with_fd; + } - src = filep->private_data; - if (!src) + tgt = filep->private_data; + if (!tgt) { + dev_err(dev, "target queue is not exist!\n"); goto out_with_fd; + } - /* no ssva is needed if the dev can do fault-from-dev */ - if (tgt->uacce->flags & UACCE_DEV_FAULT_FROM_DEV) + /* no SVA is needed if the dev can do fault-from-dev */ + if (tgt->uacce->flags & UACCE_DEV_FAULT_FROM_DEV) { + dev_err(dev, "No need to share in SVA device\n"); goto out_with_fd; + } dev_dbg(&src->uacce->dev, "share ss with %s\n", dev_name(&tgt->uacce->dev)); - if (!src->qfrs[UACCE_QFRT_SS] || tgt->qfrs[UACCE_QFRT_SS]) + if (!src->qfrs[UACCE_QFRT_SS] || tgt->qfrs[UACCE_QFRT_SS]) { + dev_err(dev, "src q's SS not exists or target q's SS exists!\n"); goto out_with_fd; + } ret = uacce_queue_map_qfr(tgt, src->qfrs[UACCE_QFRT_SS]); if (ret) goto out_with_fd; - tgt->qfrs[UACCE_QFRT_SS] = src->qfrs[UACCE_QFRT_SS]; - list_add(&tgt->list, &src->qfrs[UACCE_QFRT_SS]->qs); + /* In No-IOMMU mode, taget queue uses default SS qfr */ + if (src->qfrs[UACCE_QFRT_SS]->flags & UACCE_QFRF_DMA) { + tgt->qfrs[UACCE_QFRT_SS] = &noiommu_ss_default_qfr; + } else { + tgt->qfrs[UACCE_QFRT_SS] = src->qfrs[UACCE_QFRT_SS]; + list_add(&tgt->list, &src->qfrs[UACCE_QFRT_SS]->qs); + } ret = 0; out_with_fd: @@ -583,7 +771,7 @@ static int uacce_start_queue(struct uacce_queue *q) PAGE_KERNEL); if (!qfr->kaddr) { ret = -ENOMEM; - dev_dbg(dev, "fail to kmap %s qfr(%d pages)\n", + dev_err(dev, "fail to kmap %s qfr(%d pages)\n", uacce_qfrt_str(qfr), qfr->nr_pages); goto err_with_vmap; } @@ -595,8 +783,10 @@ static int uacce_start_queue(struct uacce_queue *q) } ret = q->uacce->ops->start_queue(q); - if (ret < 0) + if (ret < 0) { + dev_err(dev, "uacce fails to start queue!\n"); goto err_with_vmap; + } dev_dbg(&q->uacce->dev, "uacce queue state switch to STARTED\n"); q->state = UACCE_Q_STARTED; @@ -614,24 +804,38 @@ static int uacce_start_queue(struct uacce_queue *q) return ret; } -static long uacce_get_ss_dma(struct uacce_queue *q, unsigned long *arg) +static long uacce_get_ss_dma(struct uacce_queue *q, void __user *arg) { struct uacce *uacce = q->uacce; - unsigned long dma = 0; + struct uacce_dma_slice *slice; + unsigned long slice_idx = 0; + unsigned long dma, size; + long ret = -EFAULT; - if (!(uacce->flags & UACCE_DEV_NOIOMMU)) + if (!(uacce->flags & UACCE_DEV_NOIOMMU) || !q->qfrs[UACCE_QFRT_SS]) { + dev_err(&uacce->dev, "no ss dma region!\n"); return -EINVAL; + } - if (q->qfrs[UACCE_QFRT_SS]) { - dma = (unsigned long)(q->qfrs[UACCE_QFRT_SS]->dma); - dev_dbg(&uacce->dev, "%s(%lx).\n", __func__, dma); - } else { + slice = q->qfrs[UACCE_QFRT_SS]->dma_list; + if (copy_from_user(&slice_idx, arg, sizeof(unsigned long))) + return ret; + + if (slice[0].total_num - 1 < slice_idx) { + dev_err(&uacce->dev, "no ss slice idx %ld err!\n", slice_idx); return -EINVAL; } + dma = slice[slice_idx].dma; + size = slice[slice_idx].size; + if (!dma || !size) { + dev_err(&uacce->dev, "%ldth ss region no exist!\n", slice_idx); + return -ENODEV; + } + dma = dma | (size >> UACCE_GRAN_SHIFT); + if (copy_to_user(arg, &dma, sizeof(unsigned long))) + return ret; - *arg = dma; - - return 0; + return (long)(slice[0].total_num - 1 - slice_idx); } static long uacce_fops_unl_ioctl(struct file *filep, @@ -639,7 +843,6 @@ static long uacce_fops_unl_ioctl(struct file *filep, { struct uacce_queue *q; struct uacce *uacce; - unsigned long dma = 0; long ret = 0; uacce_qs_wlock(); @@ -658,13 +861,13 @@ static long uacce_fops_unl_ioctl(struct file *filep, switch (cmd) { case UACCE_CMD_SHARE_SVAS: - ret = uacce_cmd_share_qfr(q, arg); + ret = uacce_cmd_share_qfr(q, (int)arg); break; case UACCE_CMD_START: ret = uacce_start_queue(q); break; case UACCE_CMD_GET_SS_DMA: - ret = uacce_get_ss_dma(q, &dma); + ret = uacce_get_ss_dma(q, (void __user *)arg); break; case UACCE_CMD_PUT_Q: ret = uacce_put_queue(q); @@ -681,10 +884,6 @@ static long uacce_fops_unl_ioctl(struct file *filep, uacce_qs_wunlock(); - if (cmd == UACCE_CMD_GET_SS_DMA && !ret) - if (copy_to_user((void __user *)arg, &dma, sizeof(dma))) - ret = -EFAULT; - return ret; } @@ -709,25 +908,27 @@ static int uacce_dev_open_check(struct uacce *uacce) */ if (uacce->flags & (UACCE_DEV_PASID | UACCE_DEV_NOIOMMU)) return 0; - else { - if (!atomic_read(&uacce->ref)) - return 0; - dev_info(&uacce->dev, "this device can be openned only once\n"); - return -EBUSY; - } + if (!atomic_read(&uacce->ref)) + return 0; + + dev_info(&uacce->dev, "this device can be openned only once\n"); + return -EBUSY; } -/* To be fixed: only drain queue relatives */ static int uacce_queue_drain(struct uacce_queue *q) { - struct uacce *uacce = q->uacce; struct uacce_qfile_region *qfr; bool is_to_free_region; + struct uacce *uacce; + int state; int i; - if ((q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED) && - uacce->ops->stop_queue) + uacce = q->uacce; + + state = (q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED) ? 1 : + 0; + if (state && uacce->ops->stop_queue) uacce->ops->stop_queue(q); for (i = 0; i < UACCE_QFRT_MAX; i++) { @@ -737,7 +938,7 @@ static int uacce_queue_drain(struct uacce_queue *q) is_to_free_region = false; uacce_queue_unmap_qfr(q, qfr); - if (i == UACCE_QFRT_SS) { + if (i == UACCE_QFRT_SS && !(qfr->flags & UACCE_QFRF_DMA)) { list_del(&q->list); if (list_empty(&qfr->qs)) is_to_free_region = true; @@ -751,8 +952,7 @@ static int uacce_queue_drain(struct uacce_queue *q) if (uacce->flags & UACCE_DEV_SVA) iommu_sva_unbind_device(uacce->pdev, q->pasid); #endif - if ((q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED) && - uacce->ops->put_queue) + if (state && uacce->ops->put_queue) uacce->ops->put_queue(q); /* @@ -797,18 +997,25 @@ static int uacce_fops_open(struct inode *inode, struct file *filep) { struct uacce_queue *q; struct uacce *uacce; - int ret; int pasid = 0; + int ret; uacce = idr_find(&uacce_idr, iminor(inode)); - if (!uacce) + if (!uacce) { + pr_err("fail to find uacce device!\n"); return -ENODEV; + } - if (!uacce->ops->get_queue) + if (!uacce->ops->get_queue) { + dev_err(uacce->pdev, "uacce driver get_queue is NULL!\n"); return -EINVAL; + } - if (!try_module_get(uacce->pdev->driver->owner)) + if (!try_module_get(uacce->pdev->driver->owner)) { + dev_err(uacce->pdev, "uacce try to get module(%s) fail!\n", + uacce->pdev->driver->name); return -ENODEV; + } ret = uacce_dev_open_check(uacce); if (ret) goto err_open; @@ -816,8 +1023,10 @@ static int uacce_fops_open(struct inode *inode, struct file *filep) if (uacce->flags & UACCE_DEV_PASID) { ret = iommu_sva_bind_device(uacce->pdev, current->mm, &pasid, IOMMU_SVA_FEAT_IOPF, NULL); - if (ret) + if (ret) { + dev_err(uacce->pdev, "iommu SVA binds fail!\n"); goto err_open; + } } #endif uacce_qs_wlock(); @@ -827,7 +1036,6 @@ static int uacce_fops_open(struct inode *inode, struct file *filep) uacce_qs_wunlock(); goto err_unbind; } - q->pasid = pasid; q->uacce = uacce; q->mm = current->mm; @@ -886,8 +1094,8 @@ static enum uacce_qfrt uacce_get_region_type(struct uacce *uacce, struct vm_area_struct *vma) { enum uacce_qfrt type = UACCE_QFRT_MAX; - int i; size_t next_start = UACCE_QFR_NA; + int i; for (i = UACCE_QFRT_MAX - 1; i >= 0; i--) { if (vma->vm_pgoff >= uacce->qf_pg_start[i]) { @@ -906,21 +1114,26 @@ static enum uacce_qfrt uacce_get_region_type(struct uacce *uacce, case UACCE_QFRT_DKO: if ((uacce->flags & UACCE_DEV_PASID) || - (uacce->flags & UACCE_DEV_NOIOMMU)) + (uacce->flags & UACCE_DEV_NOIOMMU)) { + dev_err(&uacce->dev, "No DKO as device has PASID or no IOMMU!\n"); return UACCE_QFRT_INVALID; + } break; case UACCE_QFRT_DUS: break; case UACCE_QFRT_SS: + /* todo: this can be valid to protect the process space */ - if (uacce->flags & UACCE_DEV_FAULT_FROM_DEV) + if (uacce->flags & UACCE_DEV_FAULT_FROM_DEV) { + dev_err(&uacce->dev, "no SS in SVA mode!\n"); return UACCE_QFRT_INVALID; + } break; default: - dev_err(&uacce->dev, "uacce bug (%d)!\n", type); + dev_err(&uacce->dev, "uacce invalid type(%d)!\n", type); return UACCE_QFRT_INVALID; } @@ -933,15 +1146,13 @@ static enum uacce_qfrt uacce_get_region_type(struct uacce *uacce, } if (next_start == UACCE_QFR_NA) { - dev_err(&uacce->dev, "uacce config error. \ - make sure setting SS offset properly\n"); + dev_err(&uacce->dev, "uacce config error. make sure setting SS offset properly\n"); return UACCE_QFRT_INVALID; } if (vma_pages(vma) != next_start - uacce->qf_pg_start[type]) { - dev_err(&uacce->dev, "invalid mmap size " - "(%ld vs %ld pages) for region %s.\n", + dev_err(&uacce->dev, "invalid mmap size, (%ld vs %ld pages) for region %s.\n", vma_pages(vma), next_start - uacce->qf_pg_start[type], qfrt_str[type]); @@ -954,10 +1165,10 @@ static enum uacce_qfrt uacce_get_region_type(struct uacce *uacce, static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma) { + struct uacce_qfile_region *qfr; struct uacce_queue *q; struct uacce *uacce; enum uacce_qfrt type; - struct uacce_qfile_region *qfr; unsigned int flags = 0; int ret; @@ -982,8 +1193,10 @@ static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma) goto out_with_lock; } + /* this type of qfr has mapped already */ if (q->qfrs[type]) { - ret = -EBUSY; + dev_err(uacce->pdev, "%d type qfr is existing!\n", type); + ret = -EEXIST; goto out_with_lock; } @@ -1034,13 +1247,11 @@ static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma) ret = PTR_ERR(qfr); goto out_with_lock; } - q->qfrs[type] = qfr; - if (type == UACCE_QFRT_SS) { + if (type == UACCE_QFRT_SS && !(qfr->flags & UACCE_QFRF_DMA)) { INIT_LIST_HEAD(&qfr->qs); list_add(&q->list, &q->qfrs[type]->qs); } - uacce_qs_wunlock(); return 0; @@ -1191,6 +1402,7 @@ static ssize_t qfrs_offset_show(struct device *dev, return ret; } + static DEVICE_ATTR_RO(qfrs_offset); static ssize_t isolate_show(struct device *dev, @@ -1235,7 +1447,6 @@ static ssize_t isolate_strategy_store(struct device *dev, } static DEVICE_ATTR_RW(isolate_strategy); - static ssize_t dev_state_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -1334,8 +1545,8 @@ static int uacce_dev_match(struct device *dev, void *data) static bool uacce_iommu_has_sw_msi(struct iommu_group *group, phys_addr_t *base) { - struct list_head group_resv_regions; struct iommu_resv_region *region, *next; + struct list_head group_resv_regions; bool ret = false; INIT_LIST_HEAD(&group_resv_regions); @@ -1366,11 +1577,11 @@ static bool uacce_iommu_has_sw_msi(struct iommu_group *group, static int uacce_set_iommu_domain(struct uacce *uacce) { + struct device *dev = uacce->pdev; struct iommu_domain *domain; struct iommu_group *group; - struct device *dev = uacce->pdev; - bool resv_msi; phys_addr_t resv_msi_base = 0; + bool resv_msi; int ret; if (uacce->flags & UACCE_DEV_NOIOMMU) @@ -1442,8 +1653,8 @@ static int uacce_set_iommu_domain(struct uacce *uacce) static void uacce_unset_iommu_domain(struct uacce *uacce) { - struct iommu_domain *domain; struct device *dev = uacce->pdev; + struct iommu_domain *domain; if (uacce->flags & UACCE_DEV_NOIOMMU) return; diff --git a/include/linux/uacce.h b/include/linux/uacce.h index 657eff5e36ff..7b3f1f8618ea 100644 --- a/include/linux/uacce.h +++ b/include/linux/uacce.h @@ -30,6 +30,13 @@ struct uacce_err_isolate { atomic_t is_isolate; }; +struct uacce_dma_slice { + void *kaddr; /* kernel address for ss */ + dma_addr_t dma; /* dma address, if created by dma api */ + u32 size; /* Size of this dma slice */ + u32 total_num; /* Total slices in this dma list */ +}; + struct uacce_qfile_region { enum uacce_qfrt type; unsigned long iova; /* iova share between user and device space */ @@ -38,8 +45,8 @@ struct uacce_qfile_region { int prot; unsigned int flags; struct list_head qs; /* qs sharing the same region, for ss */ - void *kaddr; /* kernel addr, for dko */ - dma_addr_t dma; /* dma address, if created by dma api */ + void *kaddr; /* kernel address for dko */ + struct uacce_dma_slice *dma_list; }; /** diff --git a/include/uapi/linux/uacce.h b/include/uapi/linux/uacce.h index f1495b25b5c2..886e213e721b 100644 --- a/include/uapi/linux/uacce.h +++ b/include/uapi/linux/uacce.h @@ -53,4 +53,9 @@ enum uacce_qfrt { }; #define UACCE_QFRT_INVALID UACCE_QFRT_MAX +/* Pass DMA SS region slice size by granularity 64KB */ +#define UACCE_GRAN_SIZE 0x10000ull +#define UACCE_GRAN_SHIFT 16 +#define UACCE_GRAN_NUM_MASK 0xfffull + #endif -- GitLab