2115 lines
50 KiB
C
2115 lines
50 KiB
C
/*
|
|
* Copyright (c) 2012-2015, 2018 The Linux Foundation. All rights reserved.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License version 2 and
|
|
* only version 2 as published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
*/
|
|
#include <linux/slab.h>
|
|
#include <linux/completion.h>
|
|
#include <linux/pagemap.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/fs.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/module.h>
|
|
#include <linux/cdev.h>
|
|
#include <linux/list.h>
|
|
#include <linux/hash.h>
|
|
#include <linux/msm_ion.h>
|
|
#include <soc/qcom/smd.h>
|
|
#include <soc/qcom/subsystem_notif.h>
|
|
#include <linux/msm_iommu_domains.h>
|
|
#include <linux/scatterlist.h>
|
|
#include <linux/fs.h>
|
|
#include <linux/uaccess.h>
|
|
#include <linux/device.h>
|
|
#include <linux/of.h>
|
|
#include <linux/of_address.h>
|
|
#include <linux/of_platform.h>
|
|
#include <linux/dma-contiguous.h>
|
|
#include <linux/dma-buf.h>
|
|
#include <linux/iommu.h>
|
|
#include <linux/kref.h>
|
|
#include <linux/sort.h>
|
|
#include "adsprpc_compat.h"
|
|
#include "adsprpc_shared.h"
|
|
|
|
#ifndef ION_ADSPRPC_HEAP_ID
|
|
#define ION_ADSPRPC_HEAP_ID ION_AUDIO_HEAP_ID
|
|
#endif /*ION_ADSPRPC_HEAP_ID*/
|
|
|
|
#define RPC_TIMEOUT (5 * HZ)
|
|
#define RPC_HASH_BITS 5
|
|
#define RPC_HASH_SZ (1 << RPC_HASH_BITS)
|
|
#define BALIGN 32
|
|
#define NUM_CHANNELS 2
|
|
|
|
#define LOCK_MMAP(kernel)\
|
|
do {\
|
|
if (!kernel)\
|
|
down_read(¤t->mm->mmap_sem);\
|
|
} while (0)
|
|
|
|
#define UNLOCK_MMAP(kernel)\
|
|
do {\
|
|
if (!kernel)\
|
|
up_read(¤t->mm->mmap_sem);\
|
|
} while (0)
|
|
|
|
#define FASTRPC_CTX_MAGIC (0xbeeddeed)
|
|
#define FASTRPC_CTX_MAX (256)
|
|
#define FASTRPC_CTXID_MASK (0xFF0)
|
|
|
|
#define IS_CACHE_ALIGNED(x) (((x) & ((L1_CACHE_BYTES)-1)) == 0)
|
|
|
|
static inline uintptr_t buf_page_start(void *buf)
|
|
{
|
|
uintptr_t start = (uintptr_t) buf & PAGE_MASK;
|
|
return start;
|
|
}
|
|
|
|
static inline uintptr_t buf_page_offset(void *buf)
|
|
{
|
|
uintptr_t offset = (uintptr_t) buf & (PAGE_SIZE - 1);
|
|
return offset;
|
|
}
|
|
|
|
static inline int buf_num_pages(void *buf, size_t len)
|
|
{
|
|
uintptr_t start = buf_page_start(buf) >> PAGE_SHIFT;
|
|
uintptr_t end = (((uintptr_t) buf + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
|
|
int nPages = end - start + 1;
|
|
return nPages;
|
|
}
|
|
|
|
static inline uint32_t buf_page_size(uint32_t size)
|
|
{
|
|
uint32_t sz = (size + (PAGE_SIZE - 1)) & PAGE_MASK;
|
|
return sz > PAGE_SIZE ? sz : PAGE_SIZE;
|
|
}
|
|
|
|
static inline uint64_t ptr_to_uint64(void *ptr)
|
|
{
|
|
uint64_t addr = (uint64_t)((uintptr_t)ptr);
|
|
|
|
return addr;
|
|
}
|
|
|
|
static inline int buf_get_pages(void *addr, ssize_t sz, int nr_pages,
|
|
int access, struct smq_phy_page *pages,
|
|
int nr_elems, struct smq_phy_page *range)
|
|
{
|
|
struct vm_area_struct *vma, *vmaend;
|
|
uintptr_t start = buf_page_start(addr);
|
|
uintptr_t end = buf_page_start((void *)((uintptr_t)addr + sz - 1));
|
|
uint32_t len = nr_pages << PAGE_SHIFT;
|
|
unsigned long pfn, pfnend, paddr;
|
|
int n = -1, err = 0;
|
|
|
|
VERIFY(err, 0 != access_ok(access ? VERIFY_WRITE : VERIFY_READ,
|
|
(void __user *)start, len));
|
|
if (err)
|
|
goto bail;
|
|
VERIFY(err, 0 != (vma = find_vma(current->mm, start)));
|
|
if (err)
|
|
goto bail;
|
|
VERIFY(err, 0 != (vmaend = find_vma(current->mm, end)));
|
|
if (err)
|
|
goto bail;
|
|
n = 0;
|
|
if (follow_pfn(vma, start, &pfn))
|
|
goto bail;
|
|
if (follow_pfn(vmaend, end, &pfnend))
|
|
goto bail;
|
|
VERIFY(err, (pfn + nr_pages - 1) == pfnend);
|
|
if (err)
|
|
goto bail;
|
|
VERIFY(err, nr_elems > 0);
|
|
if (err)
|
|
goto bail;
|
|
VERIFY(err, __pfn_to_phys(pfnend) <= UINT_MAX);
|
|
if (err)
|
|
goto bail;
|
|
paddr = __pfn_to_phys(pfn);
|
|
if (range->size && (paddr < range->addr))
|
|
goto bail;
|
|
if (range->size && ((paddr - range->addr + len) > range->size))
|
|
goto bail;
|
|
pages->addr = paddr;
|
|
pages->size = len;
|
|
n++;
|
|
bail:
|
|
return n;
|
|
}
|
|
|
|
struct fastrpc_buf {
|
|
struct ion_handle *handle;
|
|
void *virt;
|
|
ion_phys_addr_t phys;
|
|
size_t size;
|
|
int used;
|
|
};
|
|
|
|
struct smq_context_list;
|
|
|
|
struct overlap {
|
|
uintptr_t start;
|
|
uintptr_t end;
|
|
int raix;
|
|
uintptr_t mstart;
|
|
uintptr_t mend;
|
|
uintptr_t offset;
|
|
};
|
|
|
|
|
|
struct smq_invoke_ctx {
|
|
struct hlist_node hn;
|
|
struct completion work;
|
|
int retval;
|
|
int pid;
|
|
int tgid;
|
|
remote_arg_t *pra;
|
|
remote_arg_t *rpra;
|
|
struct fastrpc_buf obuf;
|
|
struct fastrpc_buf *abufs;
|
|
struct fastrpc_device *dev;
|
|
struct fastrpc_apps *apps;
|
|
struct file_data *fdata;
|
|
int *fds;
|
|
struct ion_handle **handles;
|
|
int nbufs;
|
|
bool smmu;
|
|
uint32_t sc;
|
|
struct overlap *overs;
|
|
struct overlap **overps;
|
|
unsigned int magic;
|
|
uint64_t ctxid;
|
|
};
|
|
|
|
struct smq_context_list {
|
|
struct hlist_head pending;
|
|
struct hlist_head interrupted;
|
|
spinlock_t hlock;
|
|
};
|
|
|
|
struct fastrpc_smmu {
|
|
struct iommu_group *group;
|
|
struct iommu_domain *domain;
|
|
int domain_id;
|
|
bool enabled;
|
|
};
|
|
|
|
struct fastrpc_channel_context {
|
|
smd_channel_t *chan;
|
|
struct device *dev;
|
|
struct completion work;
|
|
struct fastrpc_smmu smmu;
|
|
struct kref kref;
|
|
struct notifier_block nb;
|
|
int ssrcount;
|
|
};
|
|
|
|
struct fastrpc_apps {
|
|
struct fastrpc_channel_context channel[NUM_CHANNELS];
|
|
struct smq_context_list clst;
|
|
struct ion_client *iclient;
|
|
struct cdev cdev;
|
|
struct class *class;
|
|
struct mutex smd_mutex;
|
|
struct smq_phy_page range;
|
|
dev_t dev_no;
|
|
int compat;
|
|
spinlock_t wrlock;
|
|
spinlock_t hlock;
|
|
struct hlist_head htbl[RPC_HASH_SZ];
|
|
spinlock_t ctxlock;
|
|
struct smq_invoke_ctx *ctxtable[FASTRPC_CTX_MAX];
|
|
};
|
|
|
|
struct fastrpc_mmap {
|
|
struct hlist_node hn;
|
|
struct ion_handle *handle;
|
|
void *virt;
|
|
ion_phys_addr_t phys;
|
|
uintptr_t *vaddrin;
|
|
uintptr_t vaddrout;
|
|
size_t size;
|
|
int refs;
|
|
};
|
|
|
|
struct file_data {
|
|
spinlock_t hlock;
|
|
struct hlist_head hlst;
|
|
uint32_t mode;
|
|
int cid;
|
|
int tgid;
|
|
int ssrcount;
|
|
};
|
|
|
|
struct fastrpc_device {
|
|
uint32_t tgid;
|
|
struct hlist_node hn;
|
|
struct fastrpc_buf buf;
|
|
};
|
|
|
|
struct fastrpc_channel_info {
|
|
char *name;
|
|
char *node;
|
|
char *group;
|
|
char *subsys;
|
|
int channel;
|
|
};
|
|
|
|
static struct fastrpc_apps gfa;
|
|
|
|
static const struct fastrpc_channel_info gcinfo[NUM_CHANNELS] = {
|
|
{
|
|
.name = "adsprpc-smd",
|
|
.node = "qcom,msm-audio-ion",
|
|
.group = "lpass_audio",
|
|
.subsys = "adsp",
|
|
.channel = SMD_APPS_QDSP,
|
|
},
|
|
{
|
|
.name = "mdsprpc-smd",
|
|
.subsys = "modem",
|
|
.channel = SMD_APPS_MODEM,
|
|
},
|
|
};
|
|
|
|
static int map_iommu_mem(struct ion_handle *handle, struct file_data *fdata,
|
|
ion_phys_addr_t *iova, unsigned long size)
|
|
{
|
|
struct fastrpc_apps *me = &gfa;
|
|
struct fastrpc_mmap *map = NULL, *mapmatch = NULL;
|
|
struct hlist_node *n;
|
|
unsigned long len = size;
|
|
int cid = fdata->cid;
|
|
int err = 0;
|
|
|
|
spin_lock(&fdata->hlock);
|
|
hlist_for_each_entry_safe(map, n, &fdata->hlst, hn) {
|
|
if (handle == map->handle) {
|
|
mapmatch = map;
|
|
break;
|
|
}
|
|
}
|
|
spin_unlock(&fdata->hlock);
|
|
|
|
if (mapmatch) {
|
|
*iova = mapmatch->phys;
|
|
return 0;
|
|
}
|
|
|
|
mutex_lock(&me->smd_mutex);
|
|
VERIFY(err, fdata->ssrcount == me->channel[cid].ssrcount);
|
|
if (!err)
|
|
VERIFY(err, 0 == ion_map_iommu(me->iclient, handle,
|
|
me->channel[cid].smmu.domain_id, 0,
|
|
SZ_4K, 0, iova, &len, 0, 0));
|
|
mutex_unlock(&me->smd_mutex);
|
|
return err;
|
|
}
|
|
|
|
static void unmap_iommu_mem(struct ion_handle *handle, struct file_data *fdata,
|
|
int cached)
|
|
{
|
|
struct fastrpc_apps *me = &gfa;
|
|
struct fastrpc_mmap *map = NULL, *mapmatch = NULL;
|
|
struct hlist_node *n;
|
|
int cid = fdata->cid;
|
|
|
|
if (cached) {
|
|
spin_lock(&fdata->hlock);
|
|
hlist_for_each_entry_safe(map, n, &fdata->hlst, hn) {
|
|
if (handle == map->handle) {
|
|
mapmatch = map;
|
|
break;
|
|
}
|
|
}
|
|
spin_unlock(&fdata->hlock);
|
|
}
|
|
|
|
if (!mapmatch) {
|
|
mutex_lock(&me->smd_mutex);
|
|
if (fdata->ssrcount == me->channel[cid].ssrcount)
|
|
ion_unmap_iommu(me->iclient, handle,
|
|
me->channel[cid].smmu.domain_id, 0);
|
|
mutex_unlock(&me->smd_mutex);
|
|
}
|
|
}
|
|
|
|
static void free_mem(struct fastrpc_buf *buf, struct file_data *fd)
|
|
{
|
|
struct fastrpc_apps *me = &gfa;
|
|
|
|
if (!IS_ERR_OR_NULL(buf->handle)) {
|
|
if (me->channel[fd->cid].smmu.enabled && buf->phys) {
|
|
unmap_iommu_mem(buf->handle, fd, 0);
|
|
buf->phys = 0;
|
|
}
|
|
if (!IS_ERR_OR_NULL(buf->virt)) {
|
|
ion_unmap_kernel(me->iclient, buf->handle);
|
|
buf->virt = NULL;
|
|
}
|
|
ion_free(me->iclient, buf->handle);
|
|
buf->handle = NULL;
|
|
}
|
|
}
|
|
|
|
static void free_map(struct fastrpc_mmap *map, struct file_data *fdata)
|
|
{
|
|
struct fastrpc_apps *me = &gfa;
|
|
int cid = fdata->cid;
|
|
|
|
if (!IS_ERR_OR_NULL(map->handle)) {
|
|
if (me->channel[cid].smmu.enabled && map->phys) {
|
|
unmap_iommu_mem(map->handle, fdata, 0);
|
|
map->phys = 0;
|
|
}
|
|
if (!IS_ERR_OR_NULL(map->virt)) {
|
|
ion_unmap_kernel(me->iclient, map->handle);
|
|
map->virt = NULL;
|
|
}
|
|
ion_free(me->iclient, map->handle);
|
|
}
|
|
map->handle = NULL;
|
|
}
|
|
|
|
static int alloc_mem(struct fastrpc_buf *buf, struct file_data *fdata)
|
|
{
|
|
struct fastrpc_apps *me = &gfa;
|
|
struct ion_client *clnt = gfa.iclient;
|
|
struct sg_table *sg;
|
|
int err = 0;
|
|
int cid = fdata->cid;
|
|
unsigned int heap;
|
|
buf->handle = NULL;
|
|
buf->virt = NULL;
|
|
buf->phys = 0;
|
|
heap = me->channel[cid].smmu.enabled ? ION_HEAP(ION_IOMMU_HEAP_ID) :
|
|
ION_HEAP(ION_ADSP_HEAP_ID);
|
|
buf->handle = ion_alloc(clnt, buf->size, SZ_4K, heap, ION_FLAG_CACHED);
|
|
VERIFY(err, 0 == IS_ERR_OR_NULL(buf->handle));
|
|
if (err)
|
|
goto bail;
|
|
buf->virt = ion_map_kernel(clnt, buf->handle);
|
|
VERIFY(err, 0 == IS_ERR_OR_NULL(buf->virt));
|
|
if (err)
|
|
goto bail;
|
|
if (me->channel[cid].smmu.enabled) {
|
|
VERIFY(err, 0 == map_iommu_mem(buf->handle, fdata,
|
|
&buf->phys, buf->size));
|
|
if (err)
|
|
goto bail;
|
|
} else {
|
|
VERIFY(err, 0 != (sg = ion_sg_table(clnt, buf->handle)));
|
|
if (err)
|
|
goto bail;
|
|
buf->phys = sg_dma_address(sg->sgl);
|
|
}
|
|
bail:
|
|
if (err && !IS_ERR_OR_NULL(buf->handle))
|
|
free_mem(buf, fdata);
|
|
return err;
|
|
}
|
|
|
|
static int context_restore_interrupted(struct fastrpc_apps *me,
|
|
struct fastrpc_ioctl_invoke_fd *invokefd,
|
|
struct file_data *fdata,
|
|
struct smq_invoke_ctx **po)
|
|
{
|
|
int err = 0;
|
|
struct smq_invoke_ctx *ctx = NULL, *ictx = NULL;
|
|
struct hlist_node *n;
|
|
struct fastrpc_ioctl_invoke *invoke = &invokefd->inv;
|
|
spin_lock(&me->clst.hlock);
|
|
hlist_for_each_entry_safe(ictx, n, &me->clst.interrupted, hn) {
|
|
if (ictx->pid == current->pid) {
|
|
if (invoke->sc != ictx->sc || ictx->fdata != fdata)
|
|
err = -1;
|
|
else {
|
|
ctx = ictx;
|
|
hlist_del(&ctx->hn);
|
|
hlist_add_head(&ctx->hn, &me->clst.pending);
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
spin_unlock(&me->clst.hlock);
|
|
if (ctx)
|
|
*po = ctx;
|
|
return err;
|
|
}
|
|
|
|
#define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
|
|
static int overlap_ptr_cmp(const void *a, const void *b)
|
|
{
|
|
struct overlap *pa = *((struct overlap **)a);
|
|
struct overlap *pb = *((struct overlap **)b);
|
|
/* sort with lowest starting buffer first */
|
|
int st = CMP(pa->start, pb->start);
|
|
/* sort with highest ending buffer first */
|
|
int ed = CMP(pb->end, pa->end);
|
|
return st == 0 ? ed : st;
|
|
}
|
|
|
|
static int context_build_overlap(struct smq_invoke_ctx *ctx)
|
|
{
|
|
int err = 0, i;
|
|
remote_arg_t *pra = ctx->pra;
|
|
int inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
|
|
int outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc);
|
|
int nbufs = inbufs + outbufs;
|
|
struct overlap max;
|
|
ctx->overs = kzalloc(sizeof(*ctx->overs) * (nbufs), GFP_KERNEL);
|
|
VERIFY(err, !IS_ERR_OR_NULL(ctx->overs));
|
|
if (err)
|
|
goto bail;
|
|
ctx->overps = kzalloc(sizeof(*ctx->overps) * (nbufs), GFP_KERNEL);
|
|
VERIFY(err, !IS_ERR_OR_NULL(ctx->overps));
|
|
if (err)
|
|
goto bail;
|
|
for (i = 0; i < nbufs; ++i) {
|
|
ctx->overs[i].start = (uintptr_t)pra[i].buf.pv;
|
|
ctx->overs[i].end = ctx->overs[i].start + pra[i].buf.len;
|
|
if (pra[i].buf.len) {
|
|
VERIFY(err, ctx->overs[i].end > ctx->overs[i].start);
|
|
if (err)
|
|
goto bail;
|
|
}
|
|
ctx->overs[i].raix = i;
|
|
ctx->overps[i] = &ctx->overs[i];
|
|
}
|
|
sort(ctx->overps, nbufs, sizeof(*ctx->overps), overlap_ptr_cmp, 0);
|
|
max.start = 0;
|
|
max.end = 0;
|
|
for (i = 0; i < nbufs; ++i) {
|
|
if (ctx->overps[i]->start < max.end) {
|
|
ctx->overps[i]->mstart = max.end;
|
|
ctx->overps[i]->mend = ctx->overps[i]->end;
|
|
ctx->overps[i]->offset = max.end -
|
|
ctx->overps[i]->start;
|
|
if (ctx->overps[i]->end > max.end) {
|
|
max.end = ctx->overps[i]->end;
|
|
} else {
|
|
ctx->overps[i]->mend = 0;
|
|
ctx->overps[i]->mstart = 0;
|
|
}
|
|
} else {
|
|
ctx->overps[i]->mend = ctx->overps[i]->end;
|
|
ctx->overps[i]->mstart = ctx->overps[i]->start;
|
|
ctx->overps[i]->offset = 0;
|
|
max = *ctx->overps[i];
|
|
}
|
|
}
|
|
bail:
|
|
return err;
|
|
}
|
|
|
|
|
|
static void context_free(struct smq_invoke_ctx *ctx, int remove);
|
|
|
|
static int context_alloc(struct fastrpc_apps *me, uint32_t kernel,
|
|
struct fastrpc_ioctl_invoke_fd *invokefd,
|
|
struct file_data *fdata,
|
|
struct smq_invoke_ctx **po)
|
|
{
|
|
int err = 0, bufs, ii, size = 0;
|
|
struct smq_invoke_ctx *ctx = NULL;
|
|
struct smq_context_list *clst = &me->clst;
|
|
|
|
struct fastrpc_ioctl_invoke *invoke = &invokefd->inv;
|
|
|
|
bufs = REMOTE_SCALARS_INBUFS(invoke->sc) +
|
|
REMOTE_SCALARS_OUTBUFS(invoke->sc);
|
|
if (bufs) {
|
|
size = bufs * sizeof(*ctx->pra);
|
|
if (invokefd->fds)
|
|
size = size + bufs * sizeof(*ctx->fds) +
|
|
bufs * sizeof(*ctx->handles);
|
|
}
|
|
|
|
VERIFY(err, NULL != (ctx = kzalloc(sizeof(*ctx) + size, GFP_KERNEL)));
|
|
if (err)
|
|
goto bail;
|
|
|
|
INIT_HLIST_NODE(&ctx->hn);
|
|
ctx->apps = me;
|
|
ctx->fdata = fdata;
|
|
ctx->pra = (remote_arg_t *)(&ctx[1]);
|
|
ctx->fds = invokefd->fds == NULL ? NULL : (int *)(&ctx->pra[bufs]);
|
|
ctx->handles = invokefd->fds == NULL ? NULL :
|
|
(struct ion_handle **)(&ctx->fds[bufs]);
|
|
if (!kernel) {
|
|
VERIFY(err, 0 == copy_from_user(ctx->pra, invoke->pra,
|
|
bufs * sizeof(*ctx->pra)));
|
|
if (err)
|
|
goto bail;
|
|
} else {
|
|
memmove(ctx->pra, invoke->pra, bufs * sizeof(*ctx->pra));
|
|
}
|
|
|
|
if (invokefd->fds) {
|
|
if (!kernel) {
|
|
VERIFY(err, 0 == copy_from_user(ctx->fds, invokefd->fds,
|
|
bufs * sizeof(*ctx->fds)));
|
|
if (err)
|
|
goto bail;
|
|
} else {
|
|
memmove(ctx->fds, invokefd->fds,
|
|
bufs * sizeof(*ctx->fds));
|
|
}
|
|
}
|
|
ctx->sc = invoke->sc;
|
|
if (bufs) {
|
|
VERIFY(err, 0 == context_build_overlap(ctx));
|
|
if (err)
|
|
goto bail;
|
|
}
|
|
ctx->retval = -1;
|
|
ctx->pid = current->pid;
|
|
ctx->tgid = current->tgid;
|
|
init_completion(&ctx->work);
|
|
ctx->magic = FASTRPC_CTX_MAGIC;
|
|
spin_lock(&clst->hlock);
|
|
hlist_add_head(&ctx->hn, &clst->pending);
|
|
spin_unlock(&clst->hlock);
|
|
|
|
spin_lock(&me->ctxlock);
|
|
for (ii = 0; ii < FASTRPC_CTX_MAX; ii++) {
|
|
if (!me->ctxtable[ii]) {
|
|
me->ctxtable[ii] = ctx;
|
|
ctx->ctxid = (ptr_to_uint64(ctx) & ~0xFFF)|(ii << 4);
|
|
break;
|
|
}
|
|
}
|
|
spin_unlock(&me->ctxlock);
|
|
VERIFY(err, ii < FASTRPC_CTX_MAX);
|
|
if (err) {
|
|
pr_err("adsprpc: out of context memory\n");
|
|
goto bail;
|
|
}
|
|
|
|
*po = ctx;
|
|
bail:
|
|
if (ctx && err)
|
|
context_free(ctx, 1);
|
|
return err;
|
|
}
|
|
|
|
static void context_save_interrupted(struct smq_invoke_ctx *ctx)
|
|
{
|
|
struct smq_context_list *clst = &ctx->apps->clst;
|
|
spin_lock(&clst->hlock);
|
|
hlist_del(&ctx->hn);
|
|
hlist_add_head(&ctx->hn, &clst->interrupted);
|
|
spin_unlock(&clst->hlock);
|
|
}
|
|
|
|
static void add_dev(struct fastrpc_apps *me, struct fastrpc_device *dev);
|
|
|
|
static void context_free(struct smq_invoke_ctx *ctx, int remove)
|
|
{
|
|
struct smq_context_list *clst = &ctx->apps->clst;
|
|
struct fastrpc_apps *apps = ctx->apps;
|
|
struct ion_client *clnt = apps->iclient;
|
|
int cid = ctx->fdata->cid;
|
|
int ssrcount = ctx->fdata->ssrcount;
|
|
struct fastrpc_smmu *smmu = &apps->channel[cid].smmu;
|
|
struct fastrpc_buf *b;
|
|
struct fastrpc_apps *me = &gfa;
|
|
int i, bufs;
|
|
if (ctx->smmu) {
|
|
bufs = REMOTE_SCALARS_INBUFS(ctx->sc) +
|
|
REMOTE_SCALARS_OUTBUFS(ctx->sc);
|
|
if (ctx->fds) {
|
|
for (i = 0; i < bufs; i++) {
|
|
if (IS_ERR_OR_NULL(ctx->handles[i]))
|
|
continue;
|
|
unmap_iommu_mem(ctx->handles[i], ctx->fdata, 1);
|
|
ion_free(clnt, ctx->handles[i]);
|
|
}
|
|
}
|
|
mutex_lock(&apps->smd_mutex);
|
|
if (ssrcount == apps->channel[cid].ssrcount)
|
|
iommu_detach_group(smmu->domain, smmu->group);
|
|
mutex_unlock(&apps->smd_mutex);
|
|
}
|
|
for (i = 0, b = ctx->abufs; i < ctx->nbufs; ++i, ++b)
|
|
free_mem(b, ctx->fdata);
|
|
|
|
kfree(ctx->abufs);
|
|
ctx->magic = 0;
|
|
ctx->ctxid = 0;
|
|
|
|
spin_lock(&me->ctxlock);
|
|
for (i = 0; i < FASTRPC_CTX_MAX; i++) {
|
|
if (me->ctxtable[i] == ctx) {
|
|
me->ctxtable[i] = NULL;
|
|
break;
|
|
}
|
|
}
|
|
spin_unlock(&me->ctxlock);
|
|
if (ctx->dev) {
|
|
add_dev(apps, ctx->dev);
|
|
if (ctx->obuf.handle != ctx->dev->buf.handle)
|
|
free_mem(&ctx->obuf, ctx->fdata);
|
|
}
|
|
if (remove) {
|
|
spin_lock(&clst->hlock);
|
|
hlist_del(&ctx->hn);
|
|
spin_unlock(&clst->hlock);
|
|
}
|
|
kfree(ctx->overps);
|
|
kfree(ctx->overs);
|
|
kfree(ctx);
|
|
}
|
|
|
|
static void context_notify_user(struct smq_invoke_ctx *ctx, int retval)
|
|
{
|
|
ctx->retval = retval;
|
|
complete(&ctx->work);
|
|
}
|
|
|
|
static void context_notify_all_users(struct smq_context_list *me, int cid)
|
|
{
|
|
struct smq_invoke_ctx *ictx = NULL;
|
|
struct hlist_node *n;
|
|
spin_lock(&me->hlock);
|
|
hlist_for_each_entry_safe(ictx, n, &me->pending, hn) {
|
|
if (ictx->fdata->cid == cid)
|
|
complete(&ictx->work);
|
|
}
|
|
hlist_for_each_entry_safe(ictx, n, &me->interrupted, hn) {
|
|
if (ictx->fdata->cid == cid)
|
|
complete(&ictx->work);
|
|
}
|
|
spin_unlock(&me->hlock);
|
|
|
|
}
|
|
|
|
static void context_list_ctor(struct smq_context_list *me)
|
|
{
|
|
INIT_HLIST_HEAD(&me->interrupted);
|
|
INIT_HLIST_HEAD(&me->pending);
|
|
spin_lock_init(&me->hlock);
|
|
}
|
|
|
|
static void context_list_dtor(struct fastrpc_apps *me,
|
|
struct smq_context_list *clst)
|
|
{
|
|
struct smq_invoke_ctx *ictx = NULL, *ctxfree;
|
|
struct hlist_node *n;
|
|
do {
|
|
ctxfree = NULL;
|
|
spin_lock(&clst->hlock);
|
|
hlist_for_each_entry_safe(ictx, n, &clst->interrupted, hn) {
|
|
hlist_del(&ictx->hn);
|
|
ctxfree = ictx;
|
|
break;
|
|
}
|
|
spin_unlock(&clst->hlock);
|
|
if (ctxfree)
|
|
context_free(ctxfree, 0);
|
|
} while (ctxfree);
|
|
do {
|
|
ctxfree = 0;
|
|
spin_lock(&clst->hlock);
|
|
hlist_for_each_entry_safe(ictx, n, &clst->pending, hn) {
|
|
hlist_del(&ictx->hn);
|
|
ctxfree = ictx;
|
|
break;
|
|
}
|
|
spin_unlock(&clst->hlock);
|
|
if (ctxfree)
|
|
context_free(ctxfree, 0);
|
|
} while (ctxfree);
|
|
}
|
|
|
|
static int get_page_list(uint32_t kernel, struct smq_invoke_ctx *ctx)
|
|
{
|
|
struct fastrpc_apps *me = &gfa;
|
|
struct smq_phy_page *pgstart, *pages;
|
|
struct smq_invoke_buf *list;
|
|
struct fastrpc_buf *ibuf = &ctx->dev->buf;
|
|
struct fastrpc_buf *obuf = &ctx->obuf;
|
|
remote_arg_t *pra = ctx->pra;
|
|
size_t rlen;
|
|
uint32_t sc = ctx->sc;
|
|
int cid = ctx->fdata->cid;
|
|
int i, err = 0;
|
|
int inbufs = REMOTE_SCALARS_INBUFS(sc);
|
|
int outbufs = REMOTE_SCALARS_OUTBUFS(sc);
|
|
|
|
LOCK_MMAP(kernel);
|
|
*obuf = *ibuf;
|
|
retry:
|
|
list = smq_invoke_buf_start((remote_arg_t *)obuf->virt, sc);
|
|
pgstart = smq_phy_page_start(sc, list);
|
|
pages = pgstart + 1;
|
|
rlen = obuf->size - ((uintptr_t)pages - (uintptr_t)obuf->virt);
|
|
if (rlen < 0) {
|
|
rlen = ((uintptr_t)pages - (uintptr_t)obuf->virt) - obuf->size;
|
|
obuf->size += buf_page_size(rlen);
|
|
VERIFY(err, 0 == alloc_mem(obuf, ctx->fdata));
|
|
if (err)
|
|
goto bail;
|
|
goto retry;
|
|
}
|
|
pgstart->addr = obuf->phys;
|
|
pgstart->size = obuf->size;
|
|
for (i = 0; i < inbufs + outbufs; ++i) {
|
|
void *buf;
|
|
int num;
|
|
size_t len;
|
|
|
|
list[i].num = 0;
|
|
list[i].pgidx = 0;
|
|
len = pra[i].buf.len;
|
|
VERIFY(err, len >= 0);
|
|
if (err)
|
|
goto bail;
|
|
if (!len)
|
|
continue;
|
|
buf = pra[i].buf.pv;
|
|
num = buf_num_pages(buf, len);
|
|
if (!kernel) {
|
|
if (me->channel[cid].smmu.enabled) {
|
|
VERIFY(err, 0 != access_ok(i >= inbufs ?
|
|
VERIFY_WRITE : VERIFY_READ,
|
|
(void __user *)buf, len));
|
|
if (err)
|
|
goto bail;
|
|
if (ctx->fds && (ctx->fds[i] >= 0))
|
|
list[i].num = 1;
|
|
} else {
|
|
list[i].num = buf_get_pages(buf, len, num,
|
|
i >= inbufs, pages,
|
|
rlen / sizeof(*pages), &me->range);
|
|
}
|
|
}
|
|
VERIFY(err, list[i].num >= 0);
|
|
if (err)
|
|
goto bail;
|
|
if (list[i].num) {
|
|
list[i].pgidx = pages - pgstart;
|
|
pages = pages + list[i].num;
|
|
} else if (rlen > sizeof(*pages)) {
|
|
list[i].pgidx = pages - pgstart;
|
|
pages = pages + 1;
|
|
} else {
|
|
if (obuf->handle != ibuf->handle)
|
|
free_mem(obuf, ctx->fdata);
|
|
obuf->size += buf_page_size(sizeof(*pages));
|
|
VERIFY(err, 0 == alloc_mem(obuf, ctx->fdata));
|
|
if (err)
|
|
goto bail;
|
|
goto retry;
|
|
}
|
|
rlen = obuf->size - ((uintptr_t)pages - (uintptr_t)obuf->virt);
|
|
}
|
|
obuf->used = obuf->size - rlen;
|
|
bail:
|
|
if (err && (obuf->handle != ibuf->handle))
|
|
free_mem(obuf, ctx->fdata);
|
|
UNLOCK_MMAP(kernel);
|
|
return err;
|
|
}
|
|
|
|
static inline int is_overlapped_outbuf(struct smq_invoke_ctx *ctx, int oix)
|
|
{
|
|
int inbufs, outbufs;
|
|
|
|
inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
|
|
outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc);
|
|
if (!ctx->overps[oix]->mstart)
|
|
return 1;
|
|
oix = oix + 1;
|
|
if ((oix < inbufs + outbufs) && !ctx->overps[oix]->mstart &&
|
|
ctx->overps[oix]->raix < inbufs)
|
|
return 1;
|
|
return 0;
|
|
}
|
|
|
|
static int clear_user_outbufs(struct smq_invoke_ctx *ctx)
|
|
{
|
|
remote_arg_t *pra = ctx->pra;
|
|
remote_arg_t *rpra = ctx->rpra;
|
|
uintptr_t ptr, end;
|
|
int oix, err = 0;
|
|
int inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
|
|
int outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc);
|
|
|
|
for (oix = 0; oix < inbufs + outbufs; ++oix) {
|
|
int i = ctx->overps[oix]->raix;
|
|
if ((i < inbufs) || (pra[i].buf.pv != rpra[i].buf.pv) ||
|
|
is_overlapped_outbuf(ctx, oix))
|
|
continue;
|
|
VERIFY(err, 0 == clear_user(rpra[i].buf.pv,
|
|
(rpra[i].buf.len < 8) ? rpra[i].buf.len : 8));
|
|
if (err)
|
|
goto bail;
|
|
ptr = buf_page_start(rpra[i].buf.pv) + PAGE_SIZE;
|
|
end = (uintptr_t)rpra[i].buf.pv + rpra[i].buf.len;
|
|
for (; ptr < end; ptr += PAGE_SIZE) {
|
|
VERIFY(err, 0 == clear_user((void *)ptr,
|
|
((end - ptr) < 8) ? end - ptr : 8));
|
|
if (err)
|
|
goto bail;
|
|
}
|
|
}
|
|
|
|
bail:
|
|
return err;
|
|
}
|
|
|
|
static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx,
|
|
remote_arg_t *upra)
|
|
{
|
|
struct fastrpc_apps *me = &gfa;
|
|
struct smq_invoke_buf *list;
|
|
struct fastrpc_buf *pbuf = &ctx->obuf, *obufs = NULL;
|
|
struct smq_phy_page *pages;
|
|
struct vm_area_struct *vma;
|
|
struct ion_handle **handles = ctx->handles;
|
|
void *args;
|
|
remote_arg_t *pra = ctx->pra;
|
|
remote_arg_t *rpra = ctx->rpra;
|
|
size_t rlen, used, size, copylen = 0;
|
|
uint32_t sc = ctx->sc, start;
|
|
int i, inh, bufs = 0, err = 0, oix;
|
|
int inbufs = REMOTE_SCALARS_INBUFS(sc);
|
|
int outbufs = REMOTE_SCALARS_OUTBUFS(sc);
|
|
int cid = ctx->fdata->cid;
|
|
int *fds = ctx->fds, idx, num;
|
|
ion_phys_addr_t iova;
|
|
|
|
list = smq_invoke_buf_start(rpra, sc);
|
|
pages = smq_phy_page_start(sc, list);
|
|
used = ALIGN(pbuf->used, BALIGN);
|
|
args = (void *)((char *)pbuf->virt + used);
|
|
rlen = pbuf->size - used;
|
|
|
|
/* map ion buffers */
|
|
for (i = 0; i < inbufs + outbufs; ++i) {
|
|
rpra[i].buf.len = pra[i].buf.len;
|
|
if (!pra[i].buf.len)
|
|
continue;
|
|
if (me->channel[cid].smmu.enabled &&
|
|
fds && (fds[i] >= 0)) {
|
|
unsigned long len;
|
|
start = buf_page_start(pra[i].buf.pv);
|
|
len = buf_page_size(pra[i].buf.len);
|
|
num = buf_num_pages(pra[i].buf.pv, pra[i].buf.len);
|
|
idx = list[i].pgidx;
|
|
handles[i] = ion_import_dma_buf(me->iclient, fds[i]);
|
|
VERIFY(err, 0 == IS_ERR_OR_NULL(handles[i]));
|
|
if (err)
|
|
goto bail;
|
|
VERIFY(err, 0 == map_iommu_mem(handles[i],
|
|
ctx->fdata, &iova, len));
|
|
if (err)
|
|
goto bail;
|
|
VERIFY(err, 0 != (vma = find_vma(current->mm, start)));
|
|
if (err)
|
|
goto bail;
|
|
rpra[i].buf.pv = pra[i].buf.pv;
|
|
pages[idx].addr = iova + (start - vma->vm_start);
|
|
pages[idx].size = num << PAGE_SHIFT;
|
|
continue;
|
|
} else if (list[i].num) {
|
|
rpra[i].buf.pv = pra[i].buf.pv;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
/* calculate len requreed for copying */
|
|
for (oix = 0; oix < inbufs + outbufs; ++oix) {
|
|
int i = ctx->overps[oix]->raix;
|
|
uintptr_t mstart, mend;
|
|
|
|
if (!pra[i].buf.len)
|
|
continue;
|
|
if (list[i].num)
|
|
continue;
|
|
if (ctx->overps[oix]->offset == 0)
|
|
copylen = ALIGN(copylen, BALIGN);
|
|
mstart = ctx->overps[oix]->mstart;
|
|
mend = ctx->overps[oix]->mend;
|
|
VERIFY(err, (mend - mstart) <= LONG_MAX);
|
|
if (err)
|
|
goto bail;
|
|
copylen += mend - mstart;
|
|
VERIFY(err, copylen >= 0);
|
|
if (err)
|
|
goto bail;
|
|
}
|
|
|
|
/* alocate new buffer */
|
|
if (copylen > rlen) {
|
|
struct fastrpc_buf *b;
|
|
pbuf->used = pbuf->size - rlen;
|
|
VERIFY(err, 0 != (b = krealloc(obufs,
|
|
(bufs + 1) * sizeof(*obufs), GFP_KERNEL)));
|
|
if (err)
|
|
goto bail;
|
|
obufs = b;
|
|
pbuf = obufs + bufs;
|
|
pbuf->size = buf_num_pages(0, copylen) * PAGE_SIZE;
|
|
VERIFY(err, 0 == alloc_mem(pbuf, ctx->fdata));
|
|
if (err)
|
|
goto bail;
|
|
bufs++;
|
|
args = pbuf->virt;
|
|
rlen = pbuf->size;
|
|
|
|
}
|
|
|
|
/* copy non ion buffers */
|
|
for (oix = 0; oix < inbufs + outbufs; ++oix) {
|
|
int i = ctx->overps[oix]->raix;
|
|
size_t mlen = ctx->overps[oix]->mend - ctx->overps[oix]->mstart;
|
|
if (!pra[i].buf.len)
|
|
continue;
|
|
if (list[i].num)
|
|
continue;
|
|
|
|
if (ctx->overps[oix]->offset == 0) {
|
|
rlen -= ALIGN((uintptr_t)args, BALIGN) -
|
|
(uintptr_t)args;
|
|
args = (void *)ALIGN((uintptr_t)args, BALIGN);
|
|
}
|
|
VERIFY(err, rlen >= mlen);
|
|
if (err)
|
|
goto bail;
|
|
list[i].num = 1;
|
|
rpra[i].buf.pv = args - ctx->overps[oix]->offset;
|
|
pages[list[i].pgidx].addr =
|
|
buf_page_start((void *)((uintptr_t)pbuf->phys -
|
|
ctx->overps[oix]->offset +
|
|
(pbuf->size - rlen)));
|
|
pages[list[i].pgidx].size = buf_num_pages(rpra[i].buf.pv,
|
|
rpra[i].buf.len) * PAGE_SIZE;
|
|
if (i < inbufs) {
|
|
if (!kernel) {
|
|
VERIFY(err, 0 == copy_from_user(rpra[i].buf.pv,
|
|
pra[i].buf.pv, pra[i].buf.len));
|
|
if (err)
|
|
goto bail;
|
|
} else {
|
|
memmove(rpra[i].buf.pv, pra[i].buf.pv,
|
|
pra[i].buf.len);
|
|
}
|
|
}
|
|
args = (void *)((uintptr_t)args + mlen);
|
|
rlen -= mlen;
|
|
}
|
|
|
|
if (!kernel) {
|
|
VERIFY(err, 0 == clear_user_outbufs(ctx));
|
|
if (err)
|
|
goto bail;
|
|
}
|
|
for (oix = 0; oix < inbufs + outbufs; ++oix) {
|
|
int i = ctx->overps[oix]->raix;
|
|
if (rpra[i].buf.len && ctx->overps[oix]->mstart)
|
|
dmac_flush_range(rpra[i].buf.pv,
|
|
(char *)rpra[i].buf.pv + rpra[i].buf.len);
|
|
}
|
|
pbuf->used = pbuf->size - rlen;
|
|
size = sizeof(*rpra) * REMOTE_SCALARS_INHANDLES(sc);
|
|
if (size) {
|
|
inh = inbufs + outbufs;
|
|
if (!kernel) {
|
|
VERIFY(err, 0 == copy_from_user(&rpra[inh], &upra[inh],
|
|
size));
|
|
if (err)
|
|
goto bail;
|
|
} else {
|
|
memmove(&rpra[inh], &upra[inh], size);
|
|
}
|
|
}
|
|
dmac_flush_range(rpra, (char *)rpra + used);
|
|
bail:
|
|
ctx->abufs = obufs;
|
|
ctx->nbufs = bufs;
|
|
return err;
|
|
}
|
|
|
|
static int put_args(uint32_t kernel, uint32_t sc, remote_arg_t *pra,
|
|
remote_arg_t *rpra, remote_arg_t *upra)
|
|
{
|
|
int i, inbufs, outbufs, outh, size;
|
|
int err = 0;
|
|
|
|
inbufs = REMOTE_SCALARS_INBUFS(sc);
|
|
outbufs = REMOTE_SCALARS_OUTBUFS(sc);
|
|
for (i = inbufs; i < inbufs + outbufs; ++i) {
|
|
if (rpra[i].buf.pv != pra[i].buf.pv) {
|
|
if (!kernel) {
|
|
VERIFY(err, 0 == copy_to_user(pra[i].buf.pv,
|
|
rpra[i].buf.pv, rpra[i].buf.len));
|
|
if (err)
|
|
goto bail;
|
|
} else {
|
|
memmove(pra[i].buf.pv, rpra[i].buf.pv,
|
|
rpra[i].buf.len);
|
|
}
|
|
}
|
|
}
|
|
size = sizeof(*rpra) * REMOTE_SCALARS_OUTHANDLES(sc);
|
|
if (size) {
|
|
outh = inbufs + outbufs + REMOTE_SCALARS_INHANDLES(sc);
|
|
if (!kernel) {
|
|
VERIFY(err, 0 == copy_to_user(&upra[outh], &rpra[outh],
|
|
size));
|
|
if (err)
|
|
goto bail;
|
|
} else {
|
|
memmove(&upra[outh], &rpra[outh], size);
|
|
}
|
|
}
|
|
bail:
|
|
return err;
|
|
}
|
|
|
|
static void inv_args_pre(uint32_t sc, remote_arg_t *rpra)
|
|
{
|
|
int i, inbufs, outbufs;
|
|
uintptr_t end;
|
|
|
|
inbufs = REMOTE_SCALARS_INBUFS(sc);
|
|
outbufs = REMOTE_SCALARS_OUTBUFS(sc);
|
|
for (i = inbufs; i < inbufs + outbufs; ++i) {
|
|
if (!rpra[i].buf.len)
|
|
continue;
|
|
if (buf_page_start(rpra) == buf_page_start(rpra[i].buf.pv))
|
|
continue;
|
|
if (!IS_CACHE_ALIGNED((uintptr_t)rpra[i].buf.pv))
|
|
dmac_flush_range(rpra[i].buf.pv,
|
|
(char *)rpra[i].buf.pv + 1);
|
|
end = (uintptr_t)rpra[i].buf.pv + rpra[i].buf.len;
|
|
if (!IS_CACHE_ALIGNED(end))
|
|
dmac_flush_range((char *)end,
|
|
(char *)end + 1);
|
|
}
|
|
}
|
|
|
|
static void inv_args(struct smq_invoke_ctx *ctx)
|
|
{
|
|
uint32_t sc = ctx->sc;
|
|
remote_arg_t *rpra = ctx->rpra;
|
|
int used = ctx->obuf.used;
|
|
int i, inbufs, outbufs;
|
|
int inv = 0;
|
|
|
|
inbufs = REMOTE_SCALARS_INBUFS(sc);
|
|
outbufs = REMOTE_SCALARS_OUTBUFS(sc);
|
|
for (i = inbufs; i < inbufs + outbufs; ++i) {
|
|
if (buf_page_start(rpra) == buf_page_start(rpra[i].buf.pv))
|
|
inv = 1;
|
|
else if (rpra[i].buf.len)
|
|
dmac_inv_range(rpra[i].buf.pv,
|
|
(char *)rpra[i].buf.pv + rpra[i].buf.len);
|
|
}
|
|
|
|
if (inv || REMOTE_SCALARS_OUTHANDLES(sc))
|
|
dmac_inv_range(rpra, (char *)rpra + used);
|
|
}
|
|
|
|
static int fastrpc_invoke_send(struct fastrpc_apps *me,
|
|
uint32_t kernel, uint32_t handle,
|
|
uint32_t sc, struct smq_invoke_ctx *ctx,
|
|
struct fastrpc_buf *buf)
|
|
{
|
|
struct smq_msg msg;
|
|
int err = 0, len;
|
|
msg.pid = current->tgid;
|
|
msg.tid = current->pid;
|
|
if (kernel)
|
|
msg.pid = 0;
|
|
msg.invoke.header.ctx = ctx->ctxid;
|
|
msg.invoke.header.handle = handle;
|
|
msg.invoke.header.sc = sc;
|
|
msg.invoke.page.addr = buf->phys;
|
|
msg.invoke.page.size = buf_page_size(buf->used);
|
|
spin_lock(&me->wrlock);
|
|
len = smd_write(me->channel[ctx->fdata->cid].chan, &msg, sizeof(msg));
|
|
spin_unlock(&me->wrlock);
|
|
VERIFY(err, len == sizeof(msg));
|
|
return err;
|
|
}
|
|
|
|
static void fastrpc_deinit(void)
|
|
{
|
|
struct fastrpc_apps *me = &gfa;
|
|
int i;
|
|
|
|
for (i = 0; i < NUM_CHANNELS; i++) {
|
|
if (me->channel[i].chan) {
|
|
(void)smd_close(me->channel[i].chan);
|
|
me->channel[i].chan = 0;
|
|
}
|
|
}
|
|
ion_client_destroy(me->iclient);
|
|
me->iclient = 0;
|
|
}
|
|
|
|
static void fastrpc_read_handler(int cid)
|
|
{
|
|
struct fastrpc_apps *me = &gfa;
|
|
struct smq_invoke_rsp rsp = {0};
|
|
int ret = 0, err = 0;
|
|
uint32_t index;
|
|
|
|
do {
|
|
ret = smd_read_from_cb(me->channel[cid].chan, &rsp,
|
|
sizeof(rsp));
|
|
if (ret != sizeof(rsp))
|
|
break;
|
|
index = (uint32_t)((rsp.ctx & FASTRPC_CTXID_MASK) >> 4);
|
|
VERIFY(err, index < FASTRPC_CTX_MAX);
|
|
if (err)
|
|
goto bail;
|
|
|
|
VERIFY(err, !IS_ERR_OR_NULL(me->ctxtable[index]));
|
|
if (err)
|
|
goto bail;
|
|
|
|
VERIFY(err, ((me->ctxtable[index]->ctxid == (rsp.ctx)) &&
|
|
me->ctxtable[index]->magic == FASTRPC_CTX_MAGIC));
|
|
if (err)
|
|
goto bail;
|
|
|
|
context_notify_user(me->ctxtable[index], rsp.retval);
|
|
} while (ret == sizeof(rsp));
|
|
|
|
bail:
|
|
if (err)
|
|
pr_err("adsprpc: invalid response or context\n");
|
|
}
|
|
|
|
static void smd_event_handler(void *priv, unsigned event)
|
|
{
|
|
struct fastrpc_apps *me = &gfa;
|
|
int cid = (int)(uintptr_t)priv;
|
|
|
|
switch (event) {
|
|
case SMD_EVENT_OPEN:
|
|
complete(&me->channel[cid].work);
|
|
break;
|
|
case SMD_EVENT_CLOSE:
|
|
context_notify_all_users(&me->clst, cid);
|
|
break;
|
|
case SMD_EVENT_DATA:
|
|
fastrpc_read_handler(cid);
|
|
break;
|
|
}
|
|
}
|
|
|
|
static int fastrpc_init(void)
|
|
{
|
|
int i, err = 0;
|
|
struct fastrpc_apps *me = &gfa;
|
|
struct device_node *node;
|
|
struct fastrpc_smmu *smmu;
|
|
bool enabled = 0;
|
|
|
|
spin_lock_init(&me->hlock);
|
|
spin_lock_init(&me->wrlock);
|
|
spin_lock_init(&me->ctxlock);
|
|
mutex_init(&me->smd_mutex);
|
|
context_list_ctor(&me->clst);
|
|
for (i = 0; i < RPC_HASH_SZ; ++i)
|
|
INIT_HLIST_HEAD(&me->htbl[i]);
|
|
me->iclient = msm_ion_client_create(DEVICE_NAME);
|
|
VERIFY(err, 0 == IS_ERR_OR_NULL(me->iclient));
|
|
if (err)
|
|
goto bail;
|
|
for (i = 0; i < NUM_CHANNELS; i++) {
|
|
init_completion(&me->channel[i].work);
|
|
if (!gcinfo[i].node)
|
|
continue;
|
|
smmu = &me->channel[i].smmu;
|
|
node = of_find_compatible_node(NULL, NULL, gcinfo[i].node);
|
|
if (node)
|
|
enabled = of_property_read_bool(node,
|
|
"qcom,smmu-enabled");
|
|
if (enabled)
|
|
smmu->group = iommu_group_find(gcinfo[i].group);
|
|
if (smmu->group)
|
|
smmu->domain = iommu_group_get_iommudata(smmu->group);
|
|
if (!IS_ERR_OR_NULL(smmu->domain)) {
|
|
smmu->domain_id = msm_find_domain_no(smmu->domain);
|
|
if (smmu->domain_id >= 0)
|
|
smmu->enabled = enabled;
|
|
}
|
|
}
|
|
return 0;
|
|
|
|
bail:
|
|
return err;
|
|
}
|
|
|
|
static void free_dev(struct fastrpc_device *dev, struct file_data *fdata)
|
|
{
|
|
if (dev) {
|
|
free_mem(&dev->buf, fdata);
|
|
kfree(dev);
|
|
module_put(THIS_MODULE);
|
|
}
|
|
}
|
|
|
|
static int alloc_dev(struct fastrpc_device **dev, struct file_data *fdata)
|
|
{
|
|
int err = 0;
|
|
struct fastrpc_device *fd = NULL;
|
|
|
|
VERIFY(err, 0 != try_module_get(THIS_MODULE));
|
|
if (err)
|
|
goto bail;
|
|
VERIFY(err, 0 != (fd = kzalloc(sizeof(*fd), GFP_KERNEL)));
|
|
if (err)
|
|
goto bail;
|
|
|
|
INIT_HLIST_NODE(&fd->hn);
|
|
|
|
fd->buf.size = PAGE_SIZE;
|
|
VERIFY(err, 0 == alloc_mem(&fd->buf, fdata));
|
|
if (err)
|
|
goto bail;
|
|
fd->tgid = current->tgid;
|
|
|
|
*dev = fd;
|
|
bail:
|
|
if (err)
|
|
free_dev(fd, fdata);
|
|
return err;
|
|
}
|
|
|
|
static int get_dev(struct fastrpc_apps *me, struct file_data *fdata,
|
|
struct fastrpc_device **rdev)
|
|
{
|
|
struct hlist_head *head;
|
|
struct fastrpc_device *dev = NULL, *devfree = NULL;
|
|
struct hlist_node *n;
|
|
uint32_t h = hash_32(current->tgid, RPC_HASH_BITS);
|
|
int err = 0;
|
|
|
|
spin_lock(&me->hlock);
|
|
head = &me->htbl[h];
|
|
hlist_for_each_entry_safe(dev, n, head, hn) {
|
|
if (dev->tgid == current->tgid) {
|
|
hlist_del(&dev->hn);
|
|
devfree = dev;
|
|
break;
|
|
}
|
|
}
|
|
spin_unlock(&me->hlock);
|
|
VERIFY(err, devfree != 0);
|
|
if (err)
|
|
goto bail;
|
|
*rdev = devfree;
|
|
bail:
|
|
if (err) {
|
|
free_dev(devfree, fdata);
|
|
err = alloc_dev(rdev, fdata);
|
|
}
|
|
return err;
|
|
}
|
|
|
|
static void add_dev(struct fastrpc_apps *me, struct fastrpc_device *dev)
|
|
{
|
|
struct hlist_head *head;
|
|
uint32_t h = hash_32(current->tgid, RPC_HASH_BITS);
|
|
|
|
spin_lock(&me->hlock);
|
|
head = &me->htbl[h];
|
|
hlist_add_head(&dev->hn, head);
|
|
spin_unlock(&me->hlock);
|
|
return;
|
|
}
|
|
|
|
static int fastrpc_release_current_dsp_process(struct file_data *fdata);
|
|
|
|
static int fastrpc_internal_invoke(struct fastrpc_apps *me, uint32_t mode,
|
|
uint32_t kernel,
|
|
struct fastrpc_ioctl_invoke_fd *invokefd,
|
|
struct file_data *fdata)
|
|
{
|
|
struct smq_invoke_ctx *ctx = NULL;
|
|
struct fastrpc_ioctl_invoke *invoke = &invokefd->inv;
|
|
int cid = fdata->cid;
|
|
int interrupted = 0;
|
|
int err = 0;
|
|
|
|
if (!kernel) {
|
|
VERIFY(err, 0 == context_restore_interrupted(me, invokefd,
|
|
fdata, &ctx));
|
|
if (err)
|
|
goto bail;
|
|
if (ctx)
|
|
goto wait;
|
|
}
|
|
|
|
VERIFY(err, 0 == context_alloc(me, kernel, invokefd, fdata, &ctx));
|
|
if (err)
|
|
goto bail;
|
|
|
|
if (me->channel[cid].smmu.enabled) {
|
|
mutex_lock(&me->smd_mutex);
|
|
VERIFY(err, fdata->ssrcount == me->channel[cid].ssrcount);
|
|
if (!err)
|
|
VERIFY(err, 0 == iommu_attach_group(
|
|
me->channel[cid].smmu.domain,
|
|
me->channel[cid].smmu.group));
|
|
mutex_unlock(&me->smd_mutex);
|
|
if (err)
|
|
goto bail;
|
|
ctx->smmu = 1;
|
|
}
|
|
if (REMOTE_SCALARS_LENGTH(ctx->sc)) {
|
|
VERIFY(err, 0 == get_dev(me, fdata, &ctx->dev));
|
|
if (err)
|
|
goto bail;
|
|
VERIFY(err, 0 == get_page_list(kernel, ctx));
|
|
if (err)
|
|
goto bail;
|
|
ctx->rpra = (remote_arg_t *)ctx->obuf.virt;
|
|
VERIFY(err, 0 == get_args(kernel, ctx, invoke->pra));
|
|
if (err)
|
|
goto bail;
|
|
}
|
|
|
|
inv_args_pre(ctx->sc, ctx->rpra);
|
|
if (FASTRPC_MODE_SERIAL == mode)
|
|
inv_args(ctx);
|
|
VERIFY(err, 0 == fastrpc_invoke_send(me, kernel, invoke->handle,
|
|
ctx->sc, ctx, &ctx->obuf));
|
|
if (err)
|
|
goto bail;
|
|
if (FASTRPC_MODE_PARALLEL == mode)
|
|
inv_args(ctx);
|
|
wait:
|
|
if (kernel)
|
|
wait_for_completion(&ctx->work);
|
|
else {
|
|
interrupted = wait_for_completion_interruptible(&ctx->work);
|
|
VERIFY(err, 0 == (err = interrupted));
|
|
if (err)
|
|
goto bail;
|
|
}
|
|
VERIFY(err, 0 == (err = ctx->retval));
|
|
if (err)
|
|
goto bail;
|
|
VERIFY(err, 0 == put_args(kernel, ctx->sc, ctx->pra, ctx->rpra,
|
|
invoke->pra));
|
|
if (err)
|
|
goto bail;
|
|
bail:
|
|
if (ctx && interrupted == -ERESTARTSYS)
|
|
context_save_interrupted(ctx);
|
|
else if (ctx)
|
|
context_free(ctx, 1);
|
|
if (fdata->ssrcount != me->channel[cid].ssrcount)
|
|
err = ECONNRESET;
|
|
return err;
|
|
}
|
|
|
|
static int map_buffer(struct fastrpc_apps *me, struct file_data *fdata,
|
|
int fd, char *buf, unsigned long len,
|
|
struct fastrpc_mmap **ppmap,
|
|
struct smq_phy_page **ppages, int *pnpages);
|
|
|
|
static int fastrpc_init_process(struct file_data *fdata,
|
|
struct fastrpc_ioctl_init *init)
|
|
{
|
|
int err = 0;
|
|
struct fastrpc_ioctl_invoke_fd ioctl;
|
|
struct smq_phy_page *pages = NULL;
|
|
struct fastrpc_mmap *map = NULL;
|
|
int npages = 0;
|
|
struct fastrpc_apps *me = &gfa;
|
|
if (init->flags == FASTRPC_INIT_ATTACH) {
|
|
remote_arg_t ra[1];
|
|
int tgid = current->tgid;
|
|
ra[0].buf.pv = &tgid;
|
|
ra[0].buf.len = sizeof(tgid);
|
|
ioctl.inv.handle = 1;
|
|
ioctl.inv.sc = REMOTE_SCALARS_MAKE(0, 1, 0);
|
|
ioctl.inv.pra = ra;
|
|
ioctl.fds = 0;
|
|
VERIFY(err, 0 == (err = fastrpc_internal_invoke(me,
|
|
FASTRPC_MODE_PARALLEL, 1, &ioctl, fdata)));
|
|
if (err)
|
|
goto bail;
|
|
} else if (init->flags == FASTRPC_INIT_CREATE) {
|
|
remote_arg_t ra[4];
|
|
int fds[4];
|
|
struct {
|
|
int pgid;
|
|
int namelen;
|
|
int filelen;
|
|
int pageslen;
|
|
} inbuf;
|
|
inbuf.pgid = current->tgid;
|
|
inbuf.namelen = strlen(current->comm);
|
|
inbuf.filelen = init->filelen;
|
|
if (!access_ok(0, (void const __user *)init->file,
|
|
init->filelen))
|
|
goto bail;
|
|
if (!access_ok(1, (void const __user *)init->mem,
|
|
init->memlen))
|
|
goto bail;
|
|
VERIFY(err, 0 == map_buffer(me, fdata, init->memfd,
|
|
(char *)init->mem, init->memlen,
|
|
&map, &pages, &npages));
|
|
if (err)
|
|
goto bail;
|
|
inbuf.pageslen = npages;
|
|
ra[0].buf.pv = &inbuf;
|
|
ra[0].buf.len = sizeof(inbuf);
|
|
fds[0] = 0;
|
|
|
|
ra[1].buf.pv = current->comm;
|
|
ra[1].buf.len = inbuf.namelen;
|
|
fds[1] = 0;
|
|
|
|
ra[2].buf.pv = (void *)init->file;
|
|
ra[2].buf.len = inbuf.filelen;
|
|
fds[2] = init->filefd;
|
|
|
|
ra[3].buf.pv = pages;
|
|
ra[3].buf.len = npages * sizeof(*pages);
|
|
fds[3] = 0;
|
|
|
|
ioctl.inv.handle = 1;
|
|
ioctl.inv.sc = REMOTE_SCALARS_MAKE(6, 4, 0);
|
|
ioctl.inv.pra = ra;
|
|
ioctl.fds = fds;
|
|
VERIFY(err, 0 == (err = fastrpc_internal_invoke(me,
|
|
FASTRPC_MODE_PARALLEL, 1, &ioctl, fdata)));
|
|
if (err)
|
|
goto bail;
|
|
spin_lock(&fdata->hlock);
|
|
map->vaddrout = 0;
|
|
hlist_add_head(&map->hn, &fdata->hlst);
|
|
spin_unlock(&fdata->hlock);
|
|
} else {
|
|
err = -ENOTTY;
|
|
}
|
|
bail:
|
|
kfree(pages);
|
|
if (err && map)
|
|
free_map(map, fdata);
|
|
return err;
|
|
}
|
|
|
|
static int fastrpc_release_current_dsp_process(struct file_data *fdata)
|
|
{
|
|
int err = 0;
|
|
struct fastrpc_apps *me = &gfa;
|
|
struct fastrpc_ioctl_invoke_fd ioctl;
|
|
remote_arg_t ra[1];
|
|
int tgid = 0;
|
|
|
|
tgid = fdata->tgid;
|
|
ra[0].buf.pv = &tgid;
|
|
ra[0].buf.len = sizeof(tgid);
|
|
ioctl.inv.handle = 1;
|
|
ioctl.inv.sc = REMOTE_SCALARS_MAKE(1, 1, 0);
|
|
ioctl.inv.pra = ra;
|
|
ioctl.fds = 0;
|
|
VERIFY(err, 0 == (err = fastrpc_internal_invoke(me,
|
|
FASTRPC_MODE_PARALLEL, 1, &ioctl, fdata)));
|
|
return err;
|
|
}
|
|
|
|
static int fastrpc_mmap_on_dsp(struct fastrpc_apps *me,
|
|
struct fastrpc_ioctl_mmap *mmap,
|
|
struct smq_phy_page *pages,
|
|
struct file_data *fdata, int num)
|
|
{
|
|
struct fastrpc_ioctl_invoke_fd ioctl;
|
|
remote_arg_t ra[3];
|
|
int err = 0;
|
|
struct {
|
|
int pid;
|
|
uint32_t flags;
|
|
uintptr_t vaddrin;
|
|
int num;
|
|
} inargs;
|
|
|
|
struct {
|
|
uintptr_t vaddrout;
|
|
} routargs;
|
|
inargs.pid = current->tgid;
|
|
inargs.vaddrin = (uintptr_t)mmap->vaddrin;
|
|
inargs.flags = mmap->flags;
|
|
inargs.num = me->compat ? num * sizeof(*pages) : num;
|
|
ra[0].buf.pv = &inargs;
|
|
ra[0].buf.len = sizeof(inargs);
|
|
|
|
ra[1].buf.pv = pages;
|
|
ra[1].buf.len = num * sizeof(*pages);
|
|
|
|
ra[2].buf.pv = &routargs;
|
|
ra[2].buf.len = sizeof(routargs);
|
|
|
|
ioctl.inv.handle = 1;
|
|
if (me->compat)
|
|
ioctl.inv.sc = REMOTE_SCALARS_MAKE(4, 2, 1);
|
|
else
|
|
ioctl.inv.sc = REMOTE_SCALARS_MAKE(2, 2, 1);
|
|
ioctl.inv.pra = ra;
|
|
ioctl.fds = 0;
|
|
VERIFY(err, 0 == (err = fastrpc_internal_invoke(me,
|
|
FASTRPC_MODE_PARALLEL, 1, &ioctl, fdata)));
|
|
mmap->vaddrout = (uintptr_t)routargs.vaddrout;
|
|
if (err)
|
|
goto bail;
|
|
bail:
|
|
return err;
|
|
}
|
|
|
|
static int fastrpc_munmap_on_dsp(struct fastrpc_apps *me,
|
|
struct fastrpc_ioctl_munmap *munmap,
|
|
struct file_data *fdata)
|
|
{
|
|
struct fastrpc_ioctl_invoke_fd ioctl;
|
|
remote_arg_t ra[1];
|
|
int err = 0;
|
|
struct {
|
|
int pid;
|
|
uintptr_t vaddrout;
|
|
size_t size;
|
|
} inargs;
|
|
|
|
inargs.pid = current->tgid;
|
|
inargs.size = munmap->size;
|
|
inargs.vaddrout = munmap->vaddrout;
|
|
ra[0].buf.pv = &inargs;
|
|
ra[0].buf.len = sizeof(inargs);
|
|
|
|
ioctl.inv.handle = 1;
|
|
if (me->compat)
|
|
ioctl.inv.sc = REMOTE_SCALARS_MAKE(5, 1, 0);
|
|
else
|
|
ioctl.inv.sc = REMOTE_SCALARS_MAKE(3, 1, 0);
|
|
ioctl.inv.pra = ra;
|
|
ioctl.fds = 0;
|
|
VERIFY(err, 0 == (err = fastrpc_internal_invoke(me,
|
|
FASTRPC_MODE_PARALLEL, 1, &ioctl, fdata)));
|
|
return err;
|
|
}
|
|
|
|
static int fastrpc_internal_munmap(struct fastrpc_apps *me,
|
|
struct file_data *fdata,
|
|
struct fastrpc_ioctl_munmap *munmap)
|
|
{
|
|
int err = 0;
|
|
struct fastrpc_mmap *map = 0, *mapfree = 0;
|
|
struct hlist_node *n;
|
|
|
|
spin_lock(&fdata->hlock);
|
|
hlist_for_each_entry_safe(map, n, &fdata->hlst, hn) {
|
|
if (map->vaddrout == munmap->vaddrout &&
|
|
map->size == munmap->size && --map->refs == 0) {
|
|
hlist_del(&map->hn);
|
|
mapfree = map;
|
|
break;
|
|
}
|
|
}
|
|
spin_unlock(&fdata->hlock);
|
|
if (mapfree) {
|
|
VERIFY(err, 0 == (err = fastrpc_munmap_on_dsp(me, munmap,
|
|
fdata)));
|
|
free_map(mapfree, fdata);
|
|
kfree(mapfree);
|
|
}
|
|
return err;
|
|
}
|
|
|
|
static int map_buffer(struct fastrpc_apps *me, struct file_data *fdata,
|
|
int fd, char *buf, unsigned long len,
|
|
struct fastrpc_mmap **ppmap,
|
|
struct smq_phy_page **ppages, int *pnpages)
|
|
{
|
|
struct ion_client *clnt = gfa.iclient;
|
|
struct ion_handle *handle = NULL;
|
|
struct fastrpc_mmap *map = NULL, *mapmatch = NULL;
|
|
struct smq_phy_page *pages = NULL;
|
|
struct hlist_node *n;
|
|
uintptr_t vaddrout = 0;
|
|
int num;
|
|
int err = 0;
|
|
|
|
handle = ion_import_dma_buf(clnt, fd);
|
|
VERIFY(err, 0 == IS_ERR_OR_NULL(handle));
|
|
if (err)
|
|
goto bail;
|
|
spin_lock(&fdata->hlock);
|
|
hlist_for_each_entry_safe(map, n, &fdata->hlst, hn) {
|
|
if (map->handle == handle) {
|
|
map->refs++;
|
|
mapmatch = map;
|
|
break;
|
|
}
|
|
}
|
|
spin_unlock(&fdata->hlock);
|
|
if (mapmatch) {
|
|
vaddrout = mapmatch->vaddrout;
|
|
return 0;
|
|
}
|
|
VERIFY(err, 0 != (map = kzalloc(sizeof(*map), GFP_KERNEL)));
|
|
if (err)
|
|
goto bail;
|
|
map->handle = handle;
|
|
handle = 0;
|
|
map->virt = ion_map_kernel(clnt, map->handle);
|
|
VERIFY(err, 0 == IS_ERR_OR_NULL(map->virt));
|
|
if (err)
|
|
goto bail;
|
|
num = buf_num_pages(buf, len);
|
|
VERIFY(err, 0 != (pages = kzalloc(num * sizeof(*pages), GFP_KERNEL)));
|
|
if (err)
|
|
goto bail;
|
|
|
|
if (me->channel[fdata->cid].smmu.enabled) {
|
|
VERIFY(err, 0 == map_iommu_mem(map->handle, fdata,
|
|
&map->phys, len));
|
|
if (err)
|
|
goto bail;
|
|
pages->addr = map->phys;
|
|
pages->size = len;
|
|
num = 1;
|
|
} else {
|
|
VERIFY(err, 0 < (num = buf_get_pages(buf, len, num, 1,
|
|
pages, num, &me->range)));
|
|
if (err)
|
|
goto bail;
|
|
}
|
|
map->refs = 1;
|
|
INIT_HLIST_NODE(&map->hn);
|
|
map->vaddrin = (uintptr_t *)buf;
|
|
map->vaddrout = vaddrout;
|
|
map->size = len;
|
|
if (ppages)
|
|
*ppages = pages;
|
|
pages = 0;
|
|
if (pnpages)
|
|
*pnpages = num;
|
|
if (ppmap)
|
|
*ppmap = map;
|
|
map = 0;
|
|
bail:
|
|
if (map)
|
|
free_map(map, fdata);
|
|
kfree(pages);
|
|
return err;
|
|
|
|
}
|
|
|
|
static int fastrpc_internal_mmap(struct fastrpc_apps *me,
|
|
struct file_data *fdata,
|
|
struct fastrpc_ioctl_mmap *mmap)
|
|
{
|
|
|
|
struct fastrpc_mmap *map = NULL;
|
|
struct smq_phy_page *pages = NULL;
|
|
int num = 0;
|
|
int err = 0;
|
|
VERIFY(err, 0 == map_buffer(me, fdata, mmap->fd, (char *)mmap->vaddrin,
|
|
mmap->size, &map, &pages, &num));
|
|
VERIFY(err, 0 == fastrpc_mmap_on_dsp(me, mmap, pages, fdata, num));
|
|
if (err)
|
|
goto bail;
|
|
map->vaddrout = mmap->vaddrout;
|
|
spin_lock(&fdata->hlock);
|
|
hlist_add_head(&map->hn, &fdata->hlst);
|
|
spin_unlock(&fdata->hlock);
|
|
bail:
|
|
if (err && map) {
|
|
free_map(map, fdata);
|
|
kfree(map);
|
|
}
|
|
kfree(pages);
|
|
return err;
|
|
}
|
|
|
|
static void cleanup_current_dev(struct file_data *fdata)
|
|
{
|
|
struct fastrpc_apps *me = &gfa;
|
|
uint32_t h = hash_32(current->tgid, RPC_HASH_BITS);
|
|
struct hlist_head *head;
|
|
struct hlist_node *n;
|
|
struct fastrpc_device *dev, *devfree;
|
|
|
|
rnext:
|
|
devfree = dev = NULL;
|
|
spin_lock(&me->hlock);
|
|
head = &me->htbl[h];
|
|
hlist_for_each_entry_safe(dev, n, head, hn) {
|
|
if (dev->tgid == current->tgid) {
|
|
hlist_del(&dev->hn);
|
|
devfree = dev;
|
|
break;
|
|
}
|
|
}
|
|
spin_unlock(&me->hlock);
|
|
if (devfree) {
|
|
free_dev(devfree, fdata);
|
|
goto rnext;
|
|
}
|
|
return;
|
|
}
|
|
|
|
static void fastrpc_channel_close(struct kref *kref)
|
|
{
|
|
struct fastrpc_apps *me = &gfa;
|
|
struct fastrpc_channel_context *ctx;
|
|
int cid;
|
|
|
|
ctx = container_of(kref, struct fastrpc_channel_context, kref);
|
|
smd_close(ctx->chan);
|
|
ctx->chan = 0;
|
|
mutex_unlock(&me->smd_mutex);
|
|
cid = ctx - &me->channel[0];
|
|
pr_info("'closed /dev/%s c %d %d'\n", gcinfo[cid].name,
|
|
MAJOR(me->dev_no), cid);
|
|
}
|
|
|
|
static int fastrpc_device_release(struct inode *inode, struct file *file)
|
|
{
|
|
struct file_data *fdata = (struct file_data *)file->private_data;
|
|
struct fastrpc_apps *me = &gfa;
|
|
struct smq_context_list *clst = &me->clst;
|
|
struct smq_invoke_ctx *ictx = NULL, *ctxfree;
|
|
struct hlist_node *n;
|
|
struct fastrpc_mmap *map = NULL;
|
|
int cid = MINOR(inode->i_rdev);
|
|
|
|
if (!fdata)
|
|
return 0;
|
|
|
|
(void)fastrpc_release_current_dsp_process(fdata);
|
|
do {
|
|
ctxfree = NULL;
|
|
spin_lock(&clst->hlock);
|
|
hlist_for_each_entry_safe(ictx, n, &clst->interrupted, hn) {
|
|
if ((ictx->tgid == current->tgid) &&
|
|
(ictx->fdata->cid == cid)) {
|
|
hlist_del(&ictx->hn);
|
|
ctxfree = ictx;
|
|
break;
|
|
}
|
|
}
|
|
spin_unlock(&clst->hlock);
|
|
if (ctxfree)
|
|
context_free(ctxfree, 0);
|
|
} while (ctxfree);
|
|
|
|
cleanup_current_dev(fdata);
|
|
file->private_data = 0;
|
|
hlist_for_each_entry_safe(map, n, &fdata->hlst, hn) {
|
|
hlist_del(&map->hn);
|
|
free_map(map, fdata);
|
|
kfree(map);
|
|
}
|
|
if (fdata->ssrcount == me->channel[cid].ssrcount)
|
|
kref_put_mutex(&me->channel[cid].kref,
|
|
fastrpc_channel_close, &me->smd_mutex);
|
|
kfree(fdata);
|
|
return 0;
|
|
}
|
|
|
|
static int fastrpc_device_open(struct inode *inode, struct file *filp)
|
|
{
|
|
int cid = MINOR(inode->i_rdev);
|
|
int err = 0, ssrcount;
|
|
struct fastrpc_apps *me = &gfa;
|
|
|
|
mutex_lock(&me->smd_mutex);
|
|
ssrcount = me->channel[cid].ssrcount;
|
|
if ((kref_get_unless_zero(&me->channel[cid].kref) == 0) ||
|
|
(me->channel[cid].chan == NULL)) {
|
|
VERIFY(err, 0 == smd_named_open_on_edge(
|
|
FASTRPC_SMD_GUID,
|
|
gcinfo[cid].channel,
|
|
&me->channel[cid].chan,
|
|
(void *)(uintptr_t)cid,
|
|
smd_event_handler));
|
|
if (err)
|
|
goto smd_bail;
|
|
VERIFY(err, 0 != wait_for_completion_timeout(
|
|
&me->channel[cid].work,
|
|
RPC_TIMEOUT));
|
|
if (err)
|
|
goto completion_bail;
|
|
kref_init(&me->channel[cid].kref);
|
|
pr_info("'opened /dev/%s c %d %d'\n", gcinfo[cid].name,
|
|
MAJOR(me->dev_no), cid);
|
|
}
|
|
mutex_unlock(&me->smd_mutex);
|
|
|
|
filp->private_data = NULL;
|
|
if (0 != try_module_get(THIS_MODULE)) {
|
|
struct file_data *fdata = NULL;
|
|
/* This call will cause a dev to be created
|
|
* which will addref this module
|
|
*/
|
|
VERIFY(err, 0 != (fdata = kzalloc(sizeof(*fdata), GFP_KERNEL)));
|
|
if (err)
|
|
goto bail;
|
|
|
|
spin_lock_init(&fdata->hlock);
|
|
INIT_HLIST_HEAD(&fdata->hlst);
|
|
fdata->cid = cid;
|
|
fdata->tgid = current->tgid;
|
|
fdata->ssrcount = ssrcount;
|
|
|
|
filp->private_data = fdata;
|
|
bail:
|
|
if (err) {
|
|
if (fdata) {
|
|
cleanup_current_dev(fdata);
|
|
kfree(fdata);
|
|
}
|
|
kref_put_mutex(&me->channel[cid].kref,
|
|
fastrpc_channel_close, &me->smd_mutex);
|
|
}
|
|
module_put(THIS_MODULE);
|
|
}
|
|
return err;
|
|
|
|
completion_bail:
|
|
smd_close(me->channel[cid].chan);
|
|
me->channel[cid].chan = NULL;
|
|
smd_bail:
|
|
mutex_unlock(&me->smd_mutex);
|
|
return err;
|
|
}
|
|
|
|
|
|
static long fastrpc_device_ioctl(struct file *file, unsigned int ioctl_num,
|
|
unsigned long ioctl_param)
|
|
{
|
|
struct fastrpc_apps *me = &gfa;
|
|
struct fastrpc_ioctl_invoke_fd invokefd;
|
|
struct fastrpc_ioctl_mmap mmap;
|
|
struct fastrpc_ioctl_munmap munmap;
|
|
struct fastrpc_ioctl_init init;
|
|
void *param = (char *)ioctl_param;
|
|
struct file_data *fdata = (struct file_data *)file->private_data;
|
|
int size = 0, err = 0;
|
|
|
|
switch (ioctl_num) {
|
|
case FASTRPC_IOCTL_INVOKE_FD:
|
|
case FASTRPC_IOCTL_INVOKE:
|
|
invokefd.fds = NULL;
|
|
size = (ioctl_num == FASTRPC_IOCTL_INVOKE) ?
|
|
sizeof(invokefd.inv) : sizeof(invokefd);
|
|
VERIFY(err, 0 == copy_from_user(&invokefd, param, size));
|
|
if (err)
|
|
goto bail;
|
|
VERIFY(err, 0 == (err = fastrpc_internal_invoke(me, fdata->mode,
|
|
0, &invokefd, fdata)));
|
|
if (err)
|
|
goto bail;
|
|
break;
|
|
case FASTRPC_IOCTL_MMAP:
|
|
VERIFY(err, 0 == copy_from_user(&mmap, param,
|
|
sizeof(mmap)));
|
|
if (err)
|
|
goto bail;
|
|
VERIFY(err, 0 == (err = fastrpc_internal_mmap(me, fdata,
|
|
&mmap)));
|
|
if (err)
|
|
goto bail;
|
|
VERIFY(err, 0 == copy_to_user(param, &mmap, sizeof(mmap)));
|
|
if (err)
|
|
goto bail;
|
|
break;
|
|
case FASTRPC_IOCTL_MUNMAP:
|
|
VERIFY(err, 0 == copy_from_user(&munmap, param,
|
|
sizeof(munmap)));
|
|
if (err)
|
|
goto bail;
|
|
VERIFY(err, 0 == (err = fastrpc_internal_munmap(me, fdata,
|
|
&munmap)));
|
|
if (err)
|
|
goto bail;
|
|
break;
|
|
case FASTRPC_IOCTL_SETMODE:
|
|
switch ((uint32_t)ioctl_param) {
|
|
case FASTRPC_MODE_PARALLEL:
|
|
case FASTRPC_MODE_SERIAL:
|
|
fdata->mode = (uint32_t)ioctl_param;
|
|
break;
|
|
default:
|
|
err = -ENOTTY;
|
|
break;
|
|
}
|
|
break;
|
|
case FASTRPC_IOCTL_INIT:
|
|
VERIFY(err, 0 == copy_from_user(&init, param,
|
|
sizeof(init)));
|
|
if (err)
|
|
goto bail;
|
|
VERIFY(err, init.filelen >= 0 && init.memlen >= 0);
|
|
if (err)
|
|
goto bail;
|
|
VERIFY(err, 0 == fastrpc_init_process(fdata, &init));
|
|
if (err)
|
|
goto bail;
|
|
break;
|
|
|
|
default:
|
|
err = -ENOTTY;
|
|
break;
|
|
}
|
|
bail:
|
|
return err;
|
|
}
|
|
|
|
static int fastrpc_restart_notifier_cb(struct notifier_block *nb,
|
|
unsigned long code,
|
|
void *data)
|
|
{
|
|
struct fastrpc_apps *me = &gfa;
|
|
struct fastrpc_channel_context *ctx;
|
|
int cid;
|
|
|
|
ctx = container_of(nb, struct fastrpc_channel_context, nb);
|
|
cid = ctx - &me->channel[0];
|
|
if (code == SUBSYS_BEFORE_SHUTDOWN) {
|
|
mutex_lock(&me->smd_mutex);
|
|
ctx->ssrcount++;
|
|
if (ctx->chan) {
|
|
smd_close(ctx->chan);
|
|
ctx->chan = NULL;
|
|
pr_info("'closed /dev/%s c %d %d'\n", gcinfo[cid].name,
|
|
MAJOR(me->dev_no), cid);
|
|
}
|
|
mutex_unlock(&me->smd_mutex);
|
|
context_notify_all_users(&me->clst, cid);
|
|
}
|
|
|
|
return NOTIFY_DONE;
|
|
}
|
|
|
|
static const struct file_operations fops = {
|
|
.open = fastrpc_device_open,
|
|
.release = fastrpc_device_release,
|
|
.unlocked_ioctl = fastrpc_device_ioctl,
|
|
.compat_ioctl = compat_fastrpc_device_ioctl,
|
|
};
|
|
|
|
static int __init fastrpc_device_init(void)
|
|
{
|
|
struct fastrpc_apps *me = &gfa;
|
|
struct device_node *ion_node, *node, *pnode;
|
|
struct platform_device *pdev;
|
|
const u32 *addr;
|
|
uint64_t size;
|
|
uint32_t val;
|
|
int i, err = 0;
|
|
|
|
memset(me, 0, sizeof(*me));
|
|
VERIFY(err, 0 == fastrpc_init());
|
|
if (err)
|
|
goto fastrpc_bail;
|
|
VERIFY(err, 0 == alloc_chrdev_region(&me->dev_no, 0, NUM_CHANNELS,
|
|
DEVICE_NAME));
|
|
if (err)
|
|
goto alloc_chrdev_bail;
|
|
cdev_init(&me->cdev, &fops);
|
|
me->cdev.owner = THIS_MODULE;
|
|
VERIFY(err, 0 == cdev_add(&me->cdev, MKDEV(MAJOR(me->dev_no), 0),
|
|
NUM_CHANNELS));
|
|
if (err)
|
|
goto cdev_init_bail;
|
|
me->class = class_create(THIS_MODULE, "fastrpc");
|
|
VERIFY(err, !IS_ERR(me->class));
|
|
if (err)
|
|
goto class_create_bail;
|
|
me->compat = (NULL == fops.compat_ioctl) ? 0 : 1;
|
|
ion_node = of_find_compatible_node(NULL, NULL, "qcom,msm-ion");
|
|
if (ion_node) {
|
|
for_each_available_child_of_node(ion_node, node) {
|
|
if (of_property_read_u32(node, "reg", &val))
|
|
continue;
|
|
if (val != ION_ADSP_HEAP_ID)
|
|
continue;
|
|
pdev = of_find_device_by_node(node);
|
|
if (!pdev)
|
|
break;
|
|
pnode = of_parse_phandle(node,
|
|
"linux,contiguous-region", 0);
|
|
if (!pnode)
|
|
break;
|
|
addr = of_get_address(pnode, 0, &size, NULL);
|
|
of_node_put(pnode);
|
|
if (!addr)
|
|
break;
|
|
me->range.addr = cma_get_base(&pdev->dev);
|
|
me->range.size = (size_t)size;
|
|
break;
|
|
}
|
|
}
|
|
for (i = 0; i < NUM_CHANNELS; i++) {
|
|
me->channel[i].dev = device_create(me->class, NULL,
|
|
MKDEV(MAJOR(me->dev_no), i),
|
|
NULL, gcinfo[i].name);
|
|
VERIFY(err, !IS_ERR(me->channel[i].dev));
|
|
if (err)
|
|
goto device_create_bail;
|
|
me->channel[i].ssrcount = 0;
|
|
me->channel[i].nb.notifier_call = fastrpc_restart_notifier_cb,
|
|
(void)subsys_notif_register_notifier(gcinfo[i].subsys,
|
|
&me->channel[i].nb);
|
|
}
|
|
|
|
return 0;
|
|
|
|
device_create_bail:
|
|
class_destroy(me->class);
|
|
class_create_bail:
|
|
cdev_del(&me->cdev);
|
|
cdev_init_bail:
|
|
unregister_chrdev_region(me->dev_no, NUM_CHANNELS);
|
|
alloc_chrdev_bail:
|
|
fastrpc_deinit();
|
|
fastrpc_bail:
|
|
return err;
|
|
}
|
|
|
|
static void __exit fastrpc_device_exit(void)
|
|
{
|
|
struct fastrpc_apps *me = &gfa;
|
|
int i;
|
|
|
|
context_list_dtor(me, &me->clst);
|
|
fastrpc_deinit();
|
|
for (i = 0; i < NUM_CHANNELS; i++) {
|
|
device_destroy(me->class, MKDEV(MAJOR(me->dev_no), i));
|
|
subsys_notif_unregister_notifier(gcinfo[i].subsys,
|
|
&me->channel[i].nb);
|
|
}
|
|
class_destroy(me->class);
|
|
cdev_del(&me->cdev);
|
|
unregister_chrdev_region(me->dev_no, NUM_CHANNELS);
|
|
}
|
|
|
|
late_initcall(fastrpc_device_init);
|
|
module_exit(fastrpc_device_exit);
|
|
|
|
MODULE_LICENSE("GPL v2");
|