提交 9ea44635 编写于 作者: L Linus Torvalds

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/dledford/rdma

Pull rdma updates from Doug Ledford:
 "Initial roundup of 4.6 merge window patches.

  This is the first of two pull requests.  It is the smaller request,
  but touches for more different things (this is everything but what is
  in or going into staging).  The pull request for the code in
  staging/rdma is on hold until after we decide what to do on the
  write/writev API issue and may be partially deferred until 4.7 as a
  result.

  Summary:

   - cxgb4 updates
   - nes updates
   - unification of iwarp portmapper code to core
   - add drain_cq API
   - various ib_core updates
   - minor ipoib updates
   - minor mlx4 updates
   - more significant mlx5 updates (including a minor merge conflict
     with net-next tree...merge is simple to resolve and Stephen's
     resolution was confirmed by Mellanox)
   - trivial net/9p rdma conversion
   - ocrdma RoCEv2 update
   - srpt updates"

* tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/dledford/rdma: (85 commits)
  iwpm: crash fix for large connections test
  iw_cxgb3: support for iWARP port mapping
  iw_cxgb4: remove port mapper related code
  iw_nes: remove port mapper related code
  iwcm: common code for port mapper
  net/9p: convert to new CQ API
  IB/mlx5: Add support for don't trap rules
  net/mlx5_core: Introduce forward to next priority action
  net/mlx5_core: Create anchor of last flow table
  iser: Accept arbitrary sg lists mapping if the device supports it
  mlx5: Add arbitrary sg list support
  IB/core: Add arbitrary sg_list support
  IB/mlx5: Expose correct max_fast_reg_page_list_len
  IB/mlx5: Make coding style more consistent
  IB/mlx5: Convert UMR CQ to new CQ API
  IB/ocrdma: Skip using unneeded intermediate variable
  IB/ocrdma: Skip using unneeded intermediate variable
  IB/ocrdma: Delete unnecessary variable initialisations in 11 functions
  IB/core: Documentation fix in the MAD header file
  IB/core: trivial prink cleanup.
  ...
...@@ -1043,8 +1043,8 @@ static void ib_cache_update(struct ib_device *device, ...@@ -1043,8 +1043,8 @@ static void ib_cache_update(struct ib_device *device,
ret = ib_query_port(device, port, tprops); ret = ib_query_port(device, port, tprops);
if (ret) { if (ret) {
printk(KERN_WARNING "ib_query_port failed (%d) for %s\n", pr_warn("ib_query_port failed (%d) for %s\n",
ret, device->name); ret, device->name);
goto err; goto err;
} }
...@@ -1067,8 +1067,8 @@ static void ib_cache_update(struct ib_device *device, ...@@ -1067,8 +1067,8 @@ static void ib_cache_update(struct ib_device *device,
for (i = 0; i < pkey_cache->table_len; ++i) { for (i = 0; i < pkey_cache->table_len; ++i) {
ret = ib_query_pkey(device, port, i, pkey_cache->table + i); ret = ib_query_pkey(device, port, i, pkey_cache->table + i);
if (ret) { if (ret) {
printk(KERN_WARNING "ib_query_pkey failed (%d) for %s (index %d)\n", pr_warn("ib_query_pkey failed (%d) for %s (index %d)\n",
ret, device->name, i); ret, device->name, i);
goto err; goto err;
} }
} }
...@@ -1078,8 +1078,8 @@ static void ib_cache_update(struct ib_device *device, ...@@ -1078,8 +1078,8 @@ static void ib_cache_update(struct ib_device *device,
ret = ib_query_gid(device, port, i, ret = ib_query_gid(device, port, i,
gid_cache->table + i, NULL); gid_cache->table + i, NULL);
if (ret) { if (ret) {
printk(KERN_WARNING "ib_query_gid failed (%d) for %s (index %d)\n", pr_warn("ib_query_gid failed (%d) for %s (index %d)\n",
ret, device->name, i); ret, device->name, i);
goto err; goto err;
} }
} }
...@@ -1161,8 +1161,7 @@ int ib_cache_setup_one(struct ib_device *device) ...@@ -1161,8 +1161,7 @@ int ib_cache_setup_one(struct ib_device *device)
GFP_KERNEL); GFP_KERNEL);
if (!device->cache.pkey_cache || if (!device->cache.pkey_cache ||
!device->cache.lmc_cache) { !device->cache.lmc_cache) {
printk(KERN_WARNING "Couldn't allocate cache " pr_warn("Couldn't allocate cache for %s\n", device->name);
"for %s\n", device->name);
return -ENOMEM; return -ENOMEM;
} }
......
...@@ -1206,6 +1206,10 @@ static int cma_save_req_info(const struct ib_cm_event *ib_event, ...@@ -1206,6 +1206,10 @@ static int cma_save_req_info(const struct ib_cm_event *ib_event,
req->has_gid = true; req->has_gid = true;
req->service_id = req_param->primary_path->service_id; req->service_id = req_param->primary_path->service_id;
req->pkey = be16_to_cpu(req_param->primary_path->pkey); req->pkey = be16_to_cpu(req_param->primary_path->pkey);
if (req->pkey != req_param->bth_pkey)
pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and primary path P_Key (0x%x)\n"
"RDMA CMA: in the future this may cause the request to be dropped\n",
req_param->bth_pkey, req->pkey);
break; break;
case IB_CM_SIDR_REQ_RECEIVED: case IB_CM_SIDR_REQ_RECEIVED:
req->device = sidr_param->listen_id->device; req->device = sidr_param->listen_id->device;
...@@ -1213,6 +1217,10 @@ static int cma_save_req_info(const struct ib_cm_event *ib_event, ...@@ -1213,6 +1217,10 @@ static int cma_save_req_info(const struct ib_cm_event *ib_event,
req->has_gid = false; req->has_gid = false;
req->service_id = sidr_param->service_id; req->service_id = sidr_param->service_id;
req->pkey = sidr_param->pkey; req->pkey = sidr_param->pkey;
if (req->pkey != sidr_param->bth_pkey)
pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and SIDR request payload P_Key (0x%x)\n"
"RDMA CMA: in the future this may cause the request to be dropped\n",
sidr_param->bth_pkey, req->pkey);
break; break;
default: default:
return -EINVAL; return -EINVAL;
...@@ -1713,7 +1721,7 @@ static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) ...@@ -1713,7 +1721,7 @@ static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE; event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
break; break;
default: default:
printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n", pr_err("RDMA CMA: unexpected IB CM event: %d\n",
ib_event->event); ib_event->event);
goto out; goto out;
} }
...@@ -2186,8 +2194,8 @@ static void cma_listen_on_dev(struct rdma_id_private *id_priv, ...@@ -2186,8 +2194,8 @@ static void cma_listen_on_dev(struct rdma_id_private *id_priv,
ret = rdma_listen(id, id_priv->backlog); ret = rdma_listen(id, id_priv->backlog);
if (ret) if (ret)
printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, " pr_warn("RDMA CMA: cma_listen_on_dev, error %d, listening on device %s\n",
"listening on device %s\n", ret, cma_dev->device->name); ret, cma_dev->device->name);
} }
static void cma_listen_on_all(struct rdma_id_private *id_priv) static void cma_listen_on_all(struct rdma_id_private *id_priv)
...@@ -3239,7 +3247,7 @@ static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, ...@@ -3239,7 +3247,7 @@ static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
event.status = 0; event.status = 0;
break; break;
default: default:
printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n", pr_err("RDMA CMA: unexpected IB CM event: %d\n",
ib_event->event); ib_event->event);
goto out; goto out;
} }
...@@ -4003,8 +4011,8 @@ static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id ...@@ -4003,8 +4011,8 @@ static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id
if ((dev_addr->bound_dev_if == ndev->ifindex) && if ((dev_addr->bound_dev_if == ndev->ifindex) &&
(net_eq(dev_net(ndev), dev_addr->net)) && (net_eq(dev_net(ndev), dev_addr->net)) &&
memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) { memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n", pr_info("RDMA CM addr change for ndev %s used by id %p\n",
ndev->name, &id_priv->id); ndev->name, &id_priv->id);
work = kzalloc(sizeof *work, GFP_KERNEL); work = kzalloc(sizeof *work, GFP_KERNEL);
if (!work) if (!work)
return -ENOMEM; return -ENOMEM;
...@@ -4287,7 +4295,7 @@ static int __init cma_init(void) ...@@ -4287,7 +4295,7 @@ static int __init cma_init(void)
goto err; goto err;
if (ibnl_add_client(RDMA_NL_RDMA_CM, RDMA_NL_RDMA_CM_NUM_OPS, cma_cb_table)) if (ibnl_add_client(RDMA_NL_RDMA_CM, RDMA_NL_RDMA_CM_NUM_OPS, cma_cb_table))
printk(KERN_WARNING "RDMA CMA: failed to add netlink callback\n"); pr_warn("RDMA CMA: failed to add netlink callback\n");
cma_configfs_init(); cma_configfs_init();
return 0; return 0;
......
...@@ -115,8 +115,8 @@ static int ib_device_check_mandatory(struct ib_device *device) ...@@ -115,8 +115,8 @@ static int ib_device_check_mandatory(struct ib_device *device)
for (i = 0; i < ARRAY_SIZE(mandatory_table); ++i) { for (i = 0; i < ARRAY_SIZE(mandatory_table); ++i) {
if (!*(void **) ((void *) device + mandatory_table[i].offset)) { if (!*(void **) ((void *) device + mandatory_table[i].offset)) {
printk(KERN_WARNING "Device %s is missing mandatory function %s\n", pr_warn("Device %s is missing mandatory function %s\n",
device->name, mandatory_table[i].name); device->name, mandatory_table[i].name);
return -EINVAL; return -EINVAL;
} }
} }
...@@ -255,8 +255,8 @@ static int add_client_context(struct ib_device *device, struct ib_client *client ...@@ -255,8 +255,8 @@ static int add_client_context(struct ib_device *device, struct ib_client *client
context = kmalloc(sizeof *context, GFP_KERNEL); context = kmalloc(sizeof *context, GFP_KERNEL);
if (!context) { if (!context) {
printk(KERN_WARNING "Couldn't allocate client context for %s/%s\n", pr_warn("Couldn't allocate client context for %s/%s\n",
device->name, client->name); device->name, client->name);
return -ENOMEM; return -ENOMEM;
} }
...@@ -343,29 +343,29 @@ int ib_register_device(struct ib_device *device, ...@@ -343,29 +343,29 @@ int ib_register_device(struct ib_device *device,
ret = read_port_immutable(device); ret = read_port_immutable(device);
if (ret) { if (ret) {
printk(KERN_WARNING "Couldn't create per port immutable data %s\n", pr_warn("Couldn't create per port immutable data %s\n",
device->name); device->name);
goto out; goto out;
} }
ret = ib_cache_setup_one(device); ret = ib_cache_setup_one(device);
if (ret) { if (ret) {
printk(KERN_WARNING "Couldn't set up InfiniBand P_Key/GID cache\n"); pr_warn("Couldn't set up InfiniBand P_Key/GID cache\n");
goto out; goto out;
} }
memset(&device->attrs, 0, sizeof(device->attrs)); memset(&device->attrs, 0, sizeof(device->attrs));
ret = device->query_device(device, &device->attrs, &uhw); ret = device->query_device(device, &device->attrs, &uhw);
if (ret) { if (ret) {
printk(KERN_WARNING "Couldn't query the device attributes\n"); pr_warn("Couldn't query the device attributes\n");
ib_cache_cleanup_one(device); ib_cache_cleanup_one(device);
goto out; goto out;
} }
ret = ib_device_register_sysfs(device, port_callback); ret = ib_device_register_sysfs(device, port_callback);
if (ret) { if (ret) {
printk(KERN_WARNING "Couldn't register device %s with driver model\n", pr_warn("Couldn't register device %s with driver model\n",
device->name); device->name);
ib_cache_cleanup_one(device); ib_cache_cleanup_one(device);
goto out; goto out;
} }
...@@ -566,8 +566,8 @@ void ib_set_client_data(struct ib_device *device, struct ib_client *client, ...@@ -566,8 +566,8 @@ void ib_set_client_data(struct ib_device *device, struct ib_client *client,
goto out; goto out;
} }
printk(KERN_WARNING "No client context found for %s/%s\n", pr_warn("No client context found for %s/%s\n",
device->name, client->name); device->name, client->name);
out: out:
spin_unlock_irqrestore(&device->client_data_lock, flags); spin_unlock_irqrestore(&device->client_data_lock, flags);
...@@ -960,13 +960,13 @@ static int __init ib_core_init(void) ...@@ -960,13 +960,13 @@ static int __init ib_core_init(void)
ret = class_register(&ib_class); ret = class_register(&ib_class);
if (ret) { if (ret) {
printk(KERN_WARNING "Couldn't create InfiniBand device class\n"); pr_warn("Couldn't create InfiniBand device class\n");
goto err_comp; goto err_comp;
} }
ret = ibnl_init(); ret = ibnl_init();
if (ret) { if (ret) {
printk(KERN_WARNING "Couldn't init IB netlink interface\n"); pr_warn("Couldn't init IB netlink interface\n");
goto err_sysfs; goto err_sysfs;
} }
......
...@@ -150,8 +150,8 @@ static void ib_fmr_batch_release(struct ib_fmr_pool *pool) ...@@ -150,8 +150,8 @@ static void ib_fmr_batch_release(struct ib_fmr_pool *pool)
#ifdef DEBUG #ifdef DEBUG
if (fmr->ref_count !=0) { if (fmr->ref_count !=0) {
printk(KERN_WARNING PFX "Unmapping FMR 0x%08x with ref count %d\n", pr_warn(PFX "Unmapping FMR 0x%08x with ref count %d\n",
fmr, fmr->ref_count); fmr, fmr->ref_count);
} }
#endif #endif
} }
...@@ -167,7 +167,7 @@ static void ib_fmr_batch_release(struct ib_fmr_pool *pool) ...@@ -167,7 +167,7 @@ static void ib_fmr_batch_release(struct ib_fmr_pool *pool)
ret = ib_unmap_fmr(&fmr_list); ret = ib_unmap_fmr(&fmr_list);
if (ret) if (ret)
printk(KERN_WARNING PFX "ib_unmap_fmr returned %d\n", ret); pr_warn(PFX "ib_unmap_fmr returned %d\n", ret);
spin_lock_irq(&pool->pool_lock); spin_lock_irq(&pool->pool_lock);
list_splice(&unmap_list, &pool->free_list); list_splice(&unmap_list, &pool->free_list);
...@@ -222,8 +222,7 @@ struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd *pd, ...@@ -222,8 +222,7 @@ struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd *pd,
device = pd->device; device = pd->device;
if (!device->alloc_fmr || !device->dealloc_fmr || if (!device->alloc_fmr || !device->dealloc_fmr ||
!device->map_phys_fmr || !device->unmap_fmr) { !device->map_phys_fmr || !device->unmap_fmr) {
printk(KERN_INFO PFX "Device %s does not support FMRs\n", pr_info(PFX "Device %s does not support FMRs\n", device->name);
device->name);
return ERR_PTR(-ENOSYS); return ERR_PTR(-ENOSYS);
} }
...@@ -233,13 +232,10 @@ struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd *pd, ...@@ -233,13 +232,10 @@ struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd *pd,
max_remaps = device->attrs.max_map_per_fmr; max_remaps = device->attrs.max_map_per_fmr;
pool = kmalloc(sizeof *pool, GFP_KERNEL); pool = kmalloc(sizeof *pool, GFP_KERNEL);
if (!pool) { if (!pool)
printk(KERN_WARNING PFX "couldn't allocate pool struct\n");
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
}
pool->cache_bucket = NULL; pool->cache_bucket = NULL;
pool->flush_function = params->flush_function; pool->flush_function = params->flush_function;
pool->flush_arg = params->flush_arg; pool->flush_arg = params->flush_arg;
...@@ -251,7 +247,7 @@ struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd *pd, ...@@ -251,7 +247,7 @@ struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd *pd,
kmalloc(IB_FMR_HASH_SIZE * sizeof *pool->cache_bucket, kmalloc(IB_FMR_HASH_SIZE * sizeof *pool->cache_bucket,
GFP_KERNEL); GFP_KERNEL);
if (!pool->cache_bucket) { if (!pool->cache_bucket) {
printk(KERN_WARNING PFX "Failed to allocate cache in pool\n"); pr_warn(PFX "Failed to allocate cache in pool\n");
ret = -ENOMEM; ret = -ENOMEM;
goto out_free_pool; goto out_free_pool;
} }
...@@ -275,7 +271,7 @@ struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd *pd, ...@@ -275,7 +271,7 @@ struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd *pd,
"ib_fmr(%s)", "ib_fmr(%s)",
device->name); device->name);
if (IS_ERR(pool->thread)) { if (IS_ERR(pool->thread)) {
printk(KERN_WARNING PFX "couldn't start cleanup thread\n"); pr_warn(PFX "couldn't start cleanup thread\n");
ret = PTR_ERR(pool->thread); ret = PTR_ERR(pool->thread);
goto out_free_pool; goto out_free_pool;
} }
...@@ -294,11 +290,8 @@ struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd *pd, ...@@ -294,11 +290,8 @@ struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd *pd,
for (i = 0; i < params->pool_size; ++i) { for (i = 0; i < params->pool_size; ++i) {
fmr = kmalloc(bytes_per_fmr, GFP_KERNEL); fmr = kmalloc(bytes_per_fmr, GFP_KERNEL);
if (!fmr) { if (!fmr)
printk(KERN_WARNING PFX "failed to allocate fmr "
"struct for FMR %d\n", i);
goto out_fail; goto out_fail;
}
fmr->pool = pool; fmr->pool = pool;
fmr->remap_count = 0; fmr->remap_count = 0;
...@@ -307,8 +300,8 @@ struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd *pd, ...@@ -307,8 +300,8 @@ struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd *pd,
fmr->fmr = ib_alloc_fmr(pd, params->access, &fmr_attr); fmr->fmr = ib_alloc_fmr(pd, params->access, &fmr_attr);
if (IS_ERR(fmr->fmr)) { if (IS_ERR(fmr->fmr)) {
printk(KERN_WARNING PFX "fmr_create failed " pr_warn(PFX "fmr_create failed for FMR %d\n",
"for FMR %d\n", i); i);
kfree(fmr); kfree(fmr);
goto out_fail; goto out_fail;
} }
...@@ -363,8 +356,8 @@ void ib_destroy_fmr_pool(struct ib_fmr_pool *pool) ...@@ -363,8 +356,8 @@ void ib_destroy_fmr_pool(struct ib_fmr_pool *pool)
} }
if (i < pool->pool_size) if (i < pool->pool_size)
printk(KERN_WARNING PFX "pool still has %d regions registered\n", pr_warn(PFX "pool still has %d regions registered\n",
pool->pool_size - i); pool->pool_size - i);
kfree(pool->cache_bucket); kfree(pool->cache_bucket);
kfree(pool); kfree(pool);
...@@ -463,7 +456,7 @@ struct ib_pool_fmr *ib_fmr_pool_map_phys(struct ib_fmr_pool *pool_handle, ...@@ -463,7 +456,7 @@ struct ib_pool_fmr *ib_fmr_pool_map_phys(struct ib_fmr_pool *pool_handle,
list_add(&fmr->list, &pool->free_list); list_add(&fmr->list, &pool->free_list);
spin_unlock_irqrestore(&pool->pool_lock, flags); spin_unlock_irqrestore(&pool->pool_lock, flags);
printk(KERN_WARNING PFX "fmr_map returns %d\n", result); pr_warn(PFX "fmr_map returns %d\n", result);
return ERR_PTR(result); return ERR_PTR(result);
} }
...@@ -517,8 +510,8 @@ int ib_fmr_pool_unmap(struct ib_pool_fmr *fmr) ...@@ -517,8 +510,8 @@ int ib_fmr_pool_unmap(struct ib_pool_fmr *fmr)
#ifdef DEBUG #ifdef DEBUG
if (fmr->ref_count < 0) if (fmr->ref_count < 0)
printk(KERN_WARNING PFX "FMR %p has ref count %d < 0\n", pr_warn(PFX "FMR %p has ref count %d < 0\n",
fmr, fmr->ref_count); fmr, fmr->ref_count);
#endif #endif
spin_unlock_irqrestore(&pool->pool_lock, flags); spin_unlock_irqrestore(&pool->pool_lock, flags);
......
...@@ -50,6 +50,8 @@ ...@@ -50,6 +50,8 @@
#include <rdma/iw_cm.h> #include <rdma/iw_cm.h>
#include <rdma/ib_addr.h> #include <rdma/ib_addr.h>
#include <rdma/iw_portmap.h>
#include <rdma/rdma_netlink.h>
#include "iwcm.h" #include "iwcm.h"
...@@ -57,6 +59,16 @@ MODULE_AUTHOR("Tom Tucker"); ...@@ -57,6 +59,16 @@ MODULE_AUTHOR("Tom Tucker");
MODULE_DESCRIPTION("iWARP CM"); MODULE_DESCRIPTION("iWARP CM");
MODULE_LICENSE("Dual BSD/GPL"); MODULE_LICENSE("Dual BSD/GPL");
static struct ibnl_client_cbs iwcm_nl_cb_table[] = {
[RDMA_NL_IWPM_REG_PID] = {.dump = iwpm_register_pid_cb},
[RDMA_NL_IWPM_ADD_MAPPING] = {.dump = iwpm_add_mapping_cb},
[RDMA_NL_IWPM_QUERY_MAPPING] = {.dump = iwpm_add_and_query_mapping_cb},
[RDMA_NL_IWPM_REMOTE_INFO] = {.dump = iwpm_remote_info_cb},
[RDMA_NL_IWPM_HANDLE_ERR] = {.dump = iwpm_mapping_error_cb},
[RDMA_NL_IWPM_MAPINFO] = {.dump = iwpm_mapping_info_cb},
[RDMA_NL_IWPM_MAPINFO_NUM] = {.dump = iwpm_ack_mapping_info_cb}
};
static struct workqueue_struct *iwcm_wq; static struct workqueue_struct *iwcm_wq;
struct iwcm_work { struct iwcm_work {
struct work_struct work; struct work_struct work;
...@@ -402,6 +414,11 @@ static void destroy_cm_id(struct iw_cm_id *cm_id) ...@@ -402,6 +414,11 @@ static void destroy_cm_id(struct iw_cm_id *cm_id)
} }
spin_unlock_irqrestore(&cm_id_priv->lock, flags); spin_unlock_irqrestore(&cm_id_priv->lock, flags);
if (cm_id->mapped) {
iwpm_remove_mapinfo(&cm_id->local_addr, &cm_id->m_local_addr);
iwpm_remove_mapping(&cm_id->local_addr, RDMA_NL_IWCM);
}
(void)iwcm_deref_id(cm_id_priv); (void)iwcm_deref_id(cm_id_priv);
} }
...@@ -426,6 +443,97 @@ void iw_destroy_cm_id(struct iw_cm_id *cm_id) ...@@ -426,6 +443,97 @@ void iw_destroy_cm_id(struct iw_cm_id *cm_id)
} }
EXPORT_SYMBOL(iw_destroy_cm_id); EXPORT_SYMBOL(iw_destroy_cm_id);
/**
* iw_cm_check_wildcard - If IP address is 0 then use original
* @pm_addr: sockaddr containing the ip to check for wildcard
* @cm_addr: sockaddr containing the actual IP address
* @cm_outaddr: sockaddr to set IP addr which leaving port
*
* Checks the pm_addr for wildcard and then sets cm_outaddr's
* IP to the actual (cm_addr).
*/
static void iw_cm_check_wildcard(struct sockaddr_storage *pm_addr,
struct sockaddr_storage *cm_addr,
struct sockaddr_storage *cm_outaddr)
{
if (pm_addr->ss_family == AF_INET) {
struct sockaddr_in *pm4_addr = (struct sockaddr_in *)pm_addr;
if (pm4_addr->sin_addr.s_addr == INADDR_ANY) {
struct sockaddr_in *cm4_addr =
(struct sockaddr_in *)cm_addr;
struct sockaddr_in *cm4_outaddr =
(struct sockaddr_in *)cm_outaddr;
cm4_outaddr->sin_addr = cm4_addr->sin_addr;
}
} else {
struct sockaddr_in6 *pm6_addr = (struct sockaddr_in6 *)pm_addr;
if (ipv6_addr_type(&pm6_addr->sin6_addr) == IPV6_ADDR_ANY) {
struct sockaddr_in6 *cm6_addr =
(struct sockaddr_in6 *)cm_addr;
struct sockaddr_in6 *cm6_outaddr =
(struct sockaddr_in6 *)cm_outaddr;
cm6_outaddr->sin6_addr = cm6_addr->sin6_addr;
}
}
}
/**
* iw_cm_map - Use portmapper to map the ports
* @cm_id: connection manager pointer
* @active: Indicates the active side when true
* returns nonzero for error only if iwpm_create_mapinfo() fails
*
* Tries to add a mapping for a port using the Portmapper. If
* successful in mapping the IP/Port it will check the remote
* mapped IP address for a wildcard IP address and replace the
* zero IP address with the remote_addr.
*/
static int iw_cm_map(struct iw_cm_id *cm_id, bool active)
{
struct iwpm_dev_data pm_reg_msg;
struct iwpm_sa_data pm_msg;
int status;
cm_id->m_local_addr = cm_id->local_addr;
cm_id->m_remote_addr = cm_id->remote_addr;
memcpy(pm_reg_msg.dev_name, cm_id->device->name,
sizeof(pm_reg_msg.dev_name));
memcpy(pm_reg_msg.if_name, cm_id->device->iwcm->ifname,
sizeof(pm_reg_msg.if_name));
if (iwpm_register_pid(&pm_reg_msg, RDMA_NL_IWCM) ||
!iwpm_valid_pid())
return 0;
cm_id->mapped = true;
pm_msg.loc_addr = cm_id->local_addr;
pm_msg.rem_addr = cm_id->remote_addr;
if (active)
status = iwpm_add_and_query_mapping(&pm_msg,
RDMA_NL_IWCM);
else
status = iwpm_add_mapping(&pm_msg, RDMA_NL_IWCM);
if (!status) {
cm_id->m_local_addr = pm_msg.mapped_loc_addr;
if (active) {
cm_id->m_remote_addr = pm_msg.mapped_rem_addr;
iw_cm_check_wildcard(&pm_msg.mapped_rem_addr,
&cm_id->remote_addr,
&cm_id->m_remote_addr);
}
}
return iwpm_create_mapinfo(&cm_id->local_addr,
&cm_id->m_local_addr,
RDMA_NL_IWCM);
}
/* /*
* CM_ID <-- LISTEN * CM_ID <-- LISTEN
* *
...@@ -452,7 +560,9 @@ int iw_cm_listen(struct iw_cm_id *cm_id, int backlog) ...@@ -452,7 +560,9 @@ int iw_cm_listen(struct iw_cm_id *cm_id, int backlog)
case IW_CM_STATE_IDLE: case IW_CM_STATE_IDLE:
cm_id_priv->state = IW_CM_STATE_LISTEN; cm_id_priv->state = IW_CM_STATE_LISTEN;
spin_unlock_irqrestore(&cm_id_priv->lock, flags); spin_unlock_irqrestore(&cm_id_priv->lock, flags);
ret = cm_id->device->iwcm->create_listen(cm_id, backlog); ret = iw_cm_map(cm_id, false);
if (!ret)
ret = cm_id->device->iwcm->create_listen(cm_id, backlog);
if (ret) if (ret)
cm_id_priv->state = IW_CM_STATE_IDLE; cm_id_priv->state = IW_CM_STATE_IDLE;
spin_lock_irqsave(&cm_id_priv->lock, flags); spin_lock_irqsave(&cm_id_priv->lock, flags);
...@@ -582,39 +692,37 @@ int iw_cm_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *iw_param) ...@@ -582,39 +692,37 @@ int iw_cm_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *iw_param)
spin_lock_irqsave(&cm_id_priv->lock, flags); spin_lock_irqsave(&cm_id_priv->lock, flags);
if (cm_id_priv->state != IW_CM_STATE_IDLE) { if (cm_id_priv->state != IW_CM_STATE_IDLE) {
spin_unlock_irqrestore(&cm_id_priv->lock, flags); ret = -EINVAL;
clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags); goto err;
wake_up_all(&cm_id_priv->connect_wait);
return -EINVAL;
} }
/* Get the ib_qp given the QPN */ /* Get the ib_qp given the QPN */
qp = cm_id->device->iwcm->get_qp(cm_id->device, iw_param->qpn); qp = cm_id->device->iwcm->get_qp(cm_id->device, iw_param->qpn);
if (!qp) { if (!qp) {
spin_unlock_irqrestore(&cm_id_priv->lock, flags); ret = -EINVAL;
clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags); goto err;
wake_up_all(&cm_id_priv->connect_wait);
return -EINVAL;
} }
cm_id->device->iwcm->add_ref(qp); cm_id->device->iwcm->add_ref(qp);
cm_id_priv->qp = qp; cm_id_priv->qp = qp;
cm_id_priv->state = IW_CM_STATE_CONN_SENT; cm_id_priv->state = IW_CM_STATE_CONN_SENT;
spin_unlock_irqrestore(&cm_id_priv->lock, flags); spin_unlock_irqrestore(&cm_id_priv->lock, flags);
ret = cm_id->device->iwcm->connect(cm_id, iw_param); ret = iw_cm_map(cm_id, true);
if (ret) { if (!ret)
spin_lock_irqsave(&cm_id_priv->lock, flags); ret = cm_id->device->iwcm->connect(cm_id, iw_param);
if (cm_id_priv->qp) { if (!ret)
cm_id->device->iwcm->rem_ref(qp); return 0; /* success */
cm_id_priv->qp = NULL;
}
spin_unlock_irqrestore(&cm_id_priv->lock, flags);
BUG_ON(cm_id_priv->state != IW_CM_STATE_CONN_SENT);
cm_id_priv->state = IW_CM_STATE_IDLE;
clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
wake_up_all(&cm_id_priv->connect_wait);
}
spin_lock_irqsave(&cm_id_priv->lock, flags);
if (cm_id_priv->qp) {
cm_id->device->iwcm->rem_ref(qp);
cm_id_priv->qp = NULL;
}
cm_id_priv->state = IW_CM_STATE_IDLE;
err:
spin_unlock_irqrestore(&cm_id_priv->lock, flags);
clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
wake_up_all(&cm_id_priv->connect_wait);
return ret; return ret;
} }
EXPORT_SYMBOL(iw_cm_connect); EXPORT_SYMBOL(iw_cm_connect);
...@@ -656,8 +764,23 @@ static void cm_conn_req_handler(struct iwcm_id_private *listen_id_priv, ...@@ -656,8 +764,23 @@ static void cm_conn_req_handler(struct iwcm_id_private *listen_id_priv,
goto out; goto out;
cm_id->provider_data = iw_event->provider_data; cm_id->provider_data = iw_event->provider_data;
cm_id->local_addr = iw_event->local_addr; cm_id->m_local_addr = iw_event->local_addr;
cm_id->remote_addr = iw_event->remote_addr; cm_id->m_remote_addr = iw_event->remote_addr;
cm_id->local_addr = listen_id_priv->id.local_addr;
ret = iwpm_get_remote_info(&listen_id_priv->id.m_local_addr,
&iw_event->remote_addr,
&cm_id->remote_addr,
RDMA_NL_IWCM);
if (ret) {
cm_id->remote_addr = iw_event->remote_addr;
} else {
iw_cm_check_wildcard(&listen_id_priv->id.m_local_addr,
&iw_event->local_addr,
&cm_id->local_addr);
iw_event->local_addr = cm_id->local_addr;
iw_event->remote_addr = cm_id->remote_addr;
}
cm_id_priv = container_of(cm_id, struct iwcm_id_private, id); cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);
cm_id_priv->state = IW_CM_STATE_CONN_RECV; cm_id_priv->state = IW_CM_STATE_CONN_RECV;
...@@ -753,8 +876,10 @@ static int cm_conn_rep_handler(struct iwcm_id_private *cm_id_priv, ...@@ -753,8 +876,10 @@ static int cm_conn_rep_handler(struct iwcm_id_private *cm_id_priv,
clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags); clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
BUG_ON(cm_id_priv->state != IW_CM_STATE_CONN_SENT); BUG_ON(cm_id_priv->state != IW_CM_STATE_CONN_SENT);
if (iw_event->status == 0) { if (iw_event->status == 0) {
cm_id_priv->id.local_addr = iw_event->local_addr; cm_id_priv->id.m_local_addr = iw_event->local_addr;
cm_id_priv->id.remote_addr = iw_event->remote_addr; cm_id_priv->id.m_remote_addr = iw_event->remote_addr;
iw_event->local_addr = cm_id_priv->id.local_addr;
iw_event->remote_addr = cm_id_priv->id.remote_addr;
cm_id_priv->state = IW_CM_STATE_ESTABLISHED; cm_id_priv->state = IW_CM_STATE_ESTABLISHED;
} else { } else {
/* REJECTED or RESET */ /* REJECTED or RESET */
...@@ -1044,6 +1169,17 @@ EXPORT_SYMBOL(iw_cm_init_qp_attr); ...@@ -1044,6 +1169,17 @@ EXPORT_SYMBOL(iw_cm_init_qp_attr);
static int __init iw_cm_init(void) static int __init iw_cm_init(void)
{ {
int ret;
ret = iwpm_init(RDMA_NL_IWCM);
if (ret)
pr_err("iw_cm: couldn't init iwpm\n");
ret = ibnl_add_client(RDMA_NL_IWCM, RDMA_NL_IWPM_NUM_OPS,
iwcm_nl_cb_table);
if (ret)
pr_err("iw_cm: couldn't register netlink callbacks\n");
iwcm_wq = create_singlethread_workqueue("iw_cm_wq"); iwcm_wq = create_singlethread_workqueue("iw_cm_wq");
if (!iwcm_wq) if (!iwcm_wq)
return -ENOMEM; return -ENOMEM;
...@@ -1063,6 +1199,8 @@ static void __exit iw_cm_cleanup(void) ...@@ -1063,6 +1199,8 @@ static void __exit iw_cm_cleanup(void)
{ {
unregister_net_sysctl_table(iwcm_ctl_table_hdr); unregister_net_sysctl_table(iwcm_ctl_table_hdr);
destroy_workqueue(iwcm_wq); destroy_workqueue(iwcm_wq);
ibnl_remove_client(RDMA_NL_IWCM);
iwpm_exit(RDMA_NL_IWCM);
} }
module_init(iw_cm_init); module_init(iw_cm_init);
......
...@@ -88,8 +88,8 @@ int iwpm_register_pid(struct iwpm_dev_data *pm_msg, u8 nl_client) ...@@ -88,8 +88,8 @@ int iwpm_register_pid(struct iwpm_dev_data *pm_msg, u8 nl_client)
ret = ibnl_put_attr(skb, nlh, sizeof(u32), &msg_seq, IWPM_NLA_REG_PID_SEQ); ret = ibnl_put_attr(skb, nlh, sizeof(u32), &msg_seq, IWPM_NLA_REG_PID_SEQ);
if (ret) if (ret)
goto pid_query_error; goto pid_query_error;
ret = ibnl_put_attr(skb, nlh, IWPM_IFNAME_SIZE, ret = ibnl_put_attr(skb, nlh, IFNAMSIZ,
pm_msg->if_name, IWPM_NLA_REG_IF_NAME); pm_msg->if_name, IWPM_NLA_REG_IF_NAME);
if (ret) if (ret)
goto pid_query_error; goto pid_query_error;
ret = ibnl_put_attr(skb, nlh, IWPM_DEVNAME_SIZE, ret = ibnl_put_attr(skb, nlh, IWPM_DEVNAME_SIZE,
...@@ -394,7 +394,7 @@ int iwpm_register_pid_cb(struct sk_buff *skb, struct netlink_callback *cb) ...@@ -394,7 +394,7 @@ int iwpm_register_pid_cb(struct sk_buff *skb, struct netlink_callback *cb)
/* always for found nlmsg_request */ /* always for found nlmsg_request */
kref_put(&nlmsg_request->kref, iwpm_free_nlmsg_request); kref_put(&nlmsg_request->kref, iwpm_free_nlmsg_request);
barrier(); barrier();
wake_up(&nlmsg_request->waitq); up(&nlmsg_request->sem);
return 0; return 0;
} }
EXPORT_SYMBOL(iwpm_register_pid_cb); EXPORT_SYMBOL(iwpm_register_pid_cb);
...@@ -463,7 +463,7 @@ int iwpm_add_mapping_cb(struct sk_buff *skb, struct netlink_callback *cb) ...@@ -463,7 +463,7 @@ int iwpm_add_mapping_cb(struct sk_buff *skb, struct netlink_callback *cb)
/* always for found request */ /* always for found request */
kref_put(&nlmsg_request->kref, iwpm_free_nlmsg_request); kref_put(&nlmsg_request->kref, iwpm_free_nlmsg_request);
barrier(); barrier();
wake_up(&nlmsg_request->waitq); up(&nlmsg_request->sem);
return 0; return 0;
} }
EXPORT_SYMBOL(iwpm_add_mapping_cb); EXPORT_SYMBOL(iwpm_add_mapping_cb);
...@@ -555,7 +555,7 @@ int iwpm_add_and_query_mapping_cb(struct sk_buff *skb, ...@@ -555,7 +555,7 @@ int iwpm_add_and_query_mapping_cb(struct sk_buff *skb,
/* always for found request */ /* always for found request */
kref_put(&nlmsg_request->kref, iwpm_free_nlmsg_request); kref_put(&nlmsg_request->kref, iwpm_free_nlmsg_request);
barrier(); barrier();
wake_up(&nlmsg_request->waitq); up(&nlmsg_request->sem);
return 0; return 0;
} }
EXPORT_SYMBOL(iwpm_add_and_query_mapping_cb); EXPORT_SYMBOL(iwpm_add_and_query_mapping_cb);
...@@ -749,7 +749,7 @@ int iwpm_mapping_error_cb(struct sk_buff *skb, struct netlink_callback *cb) ...@@ -749,7 +749,7 @@ int iwpm_mapping_error_cb(struct sk_buff *skb, struct netlink_callback *cb)
/* always for found request */ /* always for found request */
kref_put(&nlmsg_request->kref, iwpm_free_nlmsg_request); kref_put(&nlmsg_request->kref, iwpm_free_nlmsg_request);
barrier(); barrier();
wake_up(&nlmsg_request->waitq); up(&nlmsg_request->sem);
return 0; return 0;
} }
EXPORT_SYMBOL(iwpm_mapping_error_cb); EXPORT_SYMBOL(iwpm_mapping_error_cb);
...@@ -254,9 +254,9 @@ void iwpm_add_remote_info(struct iwpm_remote_info *rem_info) ...@@ -254,9 +254,9 @@ void iwpm_add_remote_info(struct iwpm_remote_info *rem_info)
} }
int iwpm_get_remote_info(struct sockaddr_storage *mapped_loc_addr, int iwpm_get_remote_info(struct sockaddr_storage *mapped_loc_addr,
struct sockaddr_storage *mapped_rem_addr, struct sockaddr_storage *mapped_rem_addr,
struct sockaddr_storage *remote_addr, struct sockaddr_storage *remote_addr,
u8 nl_client) u8 nl_client)
{ {
struct hlist_node *tmp_hlist_node; struct hlist_node *tmp_hlist_node;
struct hlist_head *hash_bucket_head; struct hlist_head *hash_bucket_head;
...@@ -322,6 +322,8 @@ struct iwpm_nlmsg_request *iwpm_get_nlmsg_request(__u32 nlmsg_seq, ...@@ -322,6 +322,8 @@ struct iwpm_nlmsg_request *iwpm_get_nlmsg_request(__u32 nlmsg_seq,
nlmsg_request->nl_client = nl_client; nlmsg_request->nl_client = nl_client;
nlmsg_request->request_done = 0; nlmsg_request->request_done = 0;
nlmsg_request->err_code = 0; nlmsg_request->err_code = 0;
sema_init(&nlmsg_request->sem, 1);
down(&nlmsg_request->sem);
return nlmsg_request; return nlmsg_request;
} }
...@@ -364,11 +366,9 @@ struct iwpm_nlmsg_request *iwpm_find_nlmsg_request(__u32 echo_seq) ...@@ -364,11 +366,9 @@ struct iwpm_nlmsg_request *iwpm_find_nlmsg_request(__u32 echo_seq)
int iwpm_wait_complete_req(struct iwpm_nlmsg_request *nlmsg_request) int iwpm_wait_complete_req(struct iwpm_nlmsg_request *nlmsg_request)
{ {
int ret; int ret;
init_waitqueue_head(&nlmsg_request->waitq);
ret = wait_event_timeout(nlmsg_request->waitq, ret = down_timeout(&nlmsg_request->sem, IWPM_NL_TIMEOUT);
(nlmsg_request->request_done != 0), IWPM_NL_TIMEOUT); if (ret) {
if (!ret) {
ret = -EINVAL; ret = -EINVAL;
pr_info("%s: Timeout %d sec for netlink request (seq = %u)\n", pr_info("%s: Timeout %d sec for netlink request (seq = %u)\n",
__func__, (IWPM_NL_TIMEOUT/HZ), nlmsg_request->nlmsg_seq); __func__, (IWPM_NL_TIMEOUT/HZ), nlmsg_request->nlmsg_seq);
......
...@@ -69,7 +69,7 @@ struct iwpm_nlmsg_request { ...@@ -69,7 +69,7 @@ struct iwpm_nlmsg_request {
u8 nl_client; u8 nl_client;
u8 request_done; u8 request_done;
u16 err_code; u16 err_code;
wait_queue_head_t waitq; struct semaphore sem;
struct kref kref; struct kref kref;
}; };
......
...@@ -44,7 +44,7 @@ static u64 value_read(int offset, int size, void *structure) ...@@ -44,7 +44,7 @@ static u64 value_read(int offset, int size, void *structure)
case 4: return be32_to_cpup((__be32 *) (structure + offset)); case 4: return be32_to_cpup((__be32 *) (structure + offset));
case 8: return be64_to_cpup((__be64 *) (structure + offset)); case 8: return be64_to_cpup((__be64 *) (structure + offset));
default: default:
printk(KERN_WARNING "Field size %d bits not handled\n", size * 8); pr_warn("Field size %d bits not handled\n", size * 8);
return 0; return 0;
} }
} }
...@@ -104,9 +104,8 @@ void ib_pack(const struct ib_field *desc, ...@@ -104,9 +104,8 @@ void ib_pack(const struct ib_field *desc,
} else { } else {
if (desc[i].offset_bits % 8 || if (desc[i].offset_bits % 8 ||
desc[i].size_bits % 8) { desc[i].size_bits % 8) {
printk(KERN_WARNING "Structure field %s of size %d " pr_warn("Structure field %s of size %d bits is not byte-aligned\n",
"bits is not byte-aligned\n", desc[i].field_name, desc[i].size_bits);
desc[i].field_name, desc[i].size_bits);
} }
if (desc[i].struct_size_bytes) if (desc[i].struct_size_bytes)
...@@ -132,7 +131,7 @@ static void value_write(int offset, int size, u64 val, void *structure) ...@@ -132,7 +131,7 @@ static void value_write(int offset, int size, u64 val, void *structure)
case 32: *(__be32 *) (structure + offset) = cpu_to_be32(val); break; case 32: *(__be32 *) (structure + offset) = cpu_to_be32(val); break;
case 64: *(__be64 *) (structure + offset) = cpu_to_be64(val); break; case 64: *(__be64 *) (structure + offset) = cpu_to_be64(val); break;
default: default:
printk(KERN_WARNING "Field size %d bits not handled\n", size * 8); pr_warn("Field size %d bits not handled\n", size * 8);
} }
} }
...@@ -188,9 +187,8 @@ void ib_unpack(const struct ib_field *desc, ...@@ -188,9 +187,8 @@ void ib_unpack(const struct ib_field *desc,
} else { } else {
if (desc[i].offset_bits % 8 || if (desc[i].offset_bits % 8 ||
desc[i].size_bits % 8) { desc[i].size_bits % 8) {
printk(KERN_WARNING "Structure field %s of size %d " pr_warn("Structure field %s of size %d bits is not byte-aligned\n",
"bits is not byte-aligned\n", desc[i].field_name, desc[i].size_bits);
desc[i].field_name, desc[i].size_bits);
} }
memcpy(structure + desc[i].struct_offset_bytes, memcpy(structure + desc[i].struct_offset_bytes,
......
...@@ -864,13 +864,12 @@ static void update_sm_ah(struct work_struct *work) ...@@ -864,13 +864,12 @@ static void update_sm_ah(struct work_struct *work)
struct ib_ah_attr ah_attr; struct ib_ah_attr ah_attr;
if (ib_query_port(port->agent->device, port->port_num, &port_attr)) { if (ib_query_port(port->agent->device, port->port_num, &port_attr)) {
printk(KERN_WARNING "Couldn't query port\n"); pr_warn("Couldn't query port\n");
return; return;
} }
new_ah = kmalloc(sizeof *new_ah, GFP_KERNEL); new_ah = kmalloc(sizeof *new_ah, GFP_KERNEL);
if (!new_ah) { if (!new_ah) {
printk(KERN_WARNING "Couldn't allocate new SM AH\n");
return; return;
} }
...@@ -880,7 +879,7 @@ static void update_sm_ah(struct work_struct *work) ...@@ -880,7 +879,7 @@ static void update_sm_ah(struct work_struct *work)
new_ah->pkey_index = 0; new_ah->pkey_index = 0;
if (ib_find_pkey(port->agent->device, port->port_num, if (ib_find_pkey(port->agent->device, port->port_num,
IB_DEFAULT_PKEY_FULL, &new_ah->pkey_index)) IB_DEFAULT_PKEY_FULL, &new_ah->pkey_index))
printk(KERN_ERR "Couldn't find index for default PKey\n"); pr_err("Couldn't find index for default PKey\n");
memset(&ah_attr, 0, sizeof ah_attr); memset(&ah_attr, 0, sizeof ah_attr);
ah_attr.dlid = port_attr.sm_lid; ah_attr.dlid = port_attr.sm_lid;
...@@ -889,7 +888,7 @@ static void update_sm_ah(struct work_struct *work) ...@@ -889,7 +888,7 @@ static void update_sm_ah(struct work_struct *work)
new_ah->ah = ib_create_ah(port->agent->qp->pd, &ah_attr); new_ah->ah = ib_create_ah(port->agent->qp->pd, &ah_attr);
if (IS_ERR(new_ah->ah)) { if (IS_ERR(new_ah->ah)) {
printk(KERN_WARNING "Couldn't create new SM AH\n"); pr_warn("Couldn't create new SM AH\n");
kfree(new_ah); kfree(new_ah);
return; return;
} }
...@@ -1221,7 +1220,7 @@ static void ib_sa_path_rec_callback(struct ib_sa_query *sa_query, ...@@ -1221,7 +1220,7 @@ static void ib_sa_path_rec_callback(struct ib_sa_query *sa_query,
rec.net = NULL; rec.net = NULL;
rec.ifindex = 0; rec.ifindex = 0;
rec.gid_type = IB_GID_TYPE_IB; rec.gid_type = IB_GID_TYPE_IB;
memset(rec.dmac, 0, ETH_ALEN); eth_zero_addr(rec.dmac);
query->callback(status, &rec, query->context); query->callback(status, &rec, query->context);
} else } else
query->callback(status, NULL, query->context); query->callback(status, NULL, query->context);
...@@ -1800,13 +1799,13 @@ static int __init ib_sa_init(void) ...@@ -1800,13 +1799,13 @@ static int __init ib_sa_init(void)
ret = ib_register_client(&sa_client); ret = ib_register_client(&sa_client);
if (ret) { if (ret) {
printk(KERN_ERR "Couldn't register ib_sa client\n"); pr_err("Couldn't register ib_sa client\n");
goto err1; goto err1;
} }
ret = mcast_init(); ret = mcast_init();
if (ret) { if (ret) {
printk(KERN_ERR "Couldn't initialize multicast handling\n"); pr_err("Couldn't initialize multicast handling\n");
goto err2; goto err2;
} }
......
...@@ -1234,7 +1234,7 @@ static int find_overflow_devnum(void) ...@@ -1234,7 +1234,7 @@ static int find_overflow_devnum(void)
ret = alloc_chrdev_region(&overflow_maj, 0, IB_UCM_MAX_DEVICES, ret = alloc_chrdev_region(&overflow_maj, 0, IB_UCM_MAX_DEVICES,
"infiniband_cm"); "infiniband_cm");
if (ret) { if (ret) {
printk(KERN_ERR "ucm: couldn't register dynamic device number\n"); pr_err("ucm: couldn't register dynamic device number\n");
return ret; return ret;
} }
} }
...@@ -1329,19 +1329,19 @@ static int __init ib_ucm_init(void) ...@@ -1329,19 +1329,19 @@ static int __init ib_ucm_init(void)
ret = register_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_MAX_DEVICES, ret = register_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_MAX_DEVICES,
"infiniband_cm"); "infiniband_cm");
if (ret) { if (ret) {
printk(KERN_ERR "ucm: couldn't register device number\n"); pr_err("ucm: couldn't register device number\n");
goto error1; goto error1;
} }
ret = class_create_file(&cm_class, &class_attr_abi_version.attr); ret = class_create_file(&cm_class, &class_attr_abi_version.attr);
if (ret) { if (ret) {
printk(KERN_ERR "ucm: couldn't create abi_version attribute\n"); pr_err("ucm: couldn't create abi_version attribute\n");
goto error2; goto error2;
} }
ret = ib_register_client(&ucm_client); ret = ib_register_client(&ucm_client);
if (ret) { if (ret) {
printk(KERN_ERR "ucm: couldn't register client\n"); pr_err("ucm: couldn't register client\n");
goto error3; goto error3;
} }
return 0; return 0;
......
...@@ -314,7 +314,7 @@ static void ucma_removal_event_handler(struct rdma_cm_id *cm_id) ...@@ -314,7 +314,7 @@ static void ucma_removal_event_handler(struct rdma_cm_id *cm_id)
} }
} }
if (!event_found) if (!event_found)
printk(KERN_ERR "ucma_removal_event_handler: warning: connect request event wasn't found\n"); pr_err("ucma_removal_event_handler: warning: connect request event wasn't found\n");
} }
static int ucma_event_handler(struct rdma_cm_id *cm_id, static int ucma_event_handler(struct rdma_cm_id *cm_id,
...@@ -1716,13 +1716,13 @@ static int __init ucma_init(void) ...@@ -1716,13 +1716,13 @@ static int __init ucma_init(void)
ret = device_create_file(ucma_misc.this_device, &dev_attr_abi_version); ret = device_create_file(ucma_misc.this_device, &dev_attr_abi_version);
if (ret) { if (ret) {
printk(KERN_ERR "rdma_ucm: couldn't create abi_version attr\n"); pr_err("rdma_ucm: couldn't create abi_version attr\n");
goto err1; goto err1;
} }
ucma_ctl_table_hdr = register_net_sysctl(&init_net, "net/rdma_ucm", ucma_ctl_table); ucma_ctl_table_hdr = register_net_sysctl(&init_net, "net/rdma_ucm", ucma_ctl_table);
if (!ucma_ctl_table_hdr) { if (!ucma_ctl_table_hdr) {
printk(KERN_ERR "rdma_ucm: couldn't register sysctl paths\n"); pr_err("rdma_ucm: couldn't register sysctl paths\n");
ret = -ENOMEM; ret = -ENOMEM;
goto err2; goto err2;
} }
......
...@@ -479,8 +479,8 @@ int ib_ud_header_unpack(void *buf, ...@@ -479,8 +479,8 @@ int ib_ud_header_unpack(void *buf,
buf += IB_LRH_BYTES; buf += IB_LRH_BYTES;
if (header->lrh.link_version != 0) { if (header->lrh.link_version != 0) {
printk(KERN_WARNING "Invalid LRH.link_version %d\n", pr_warn("Invalid LRH.link_version %d\n",
header->lrh.link_version); header->lrh.link_version);
return -EINVAL; return -EINVAL;
} }
...@@ -496,20 +496,20 @@ int ib_ud_header_unpack(void *buf, ...@@ -496,20 +496,20 @@ int ib_ud_header_unpack(void *buf,
buf += IB_GRH_BYTES; buf += IB_GRH_BYTES;
if (header->grh.ip_version != 6) { if (header->grh.ip_version != 6) {
printk(KERN_WARNING "Invalid GRH.ip_version %d\n", pr_warn("Invalid GRH.ip_version %d\n",
header->grh.ip_version); header->grh.ip_version);
return -EINVAL; return -EINVAL;
} }
if (header->grh.next_header != 0x1b) { if (header->grh.next_header != 0x1b) {
printk(KERN_WARNING "Invalid GRH.next_header 0x%02x\n", pr_warn("Invalid GRH.next_header 0x%02x\n",
header->grh.next_header); header->grh.next_header);
return -EINVAL; return -EINVAL;
} }
break; break;
default: default:
printk(KERN_WARNING "Invalid LRH.link_next_header %d\n", pr_warn("Invalid LRH.link_next_header %d\n",
header->lrh.link_next_header); header->lrh.link_next_header);
return -EINVAL; return -EINVAL;
} }
...@@ -525,14 +525,13 @@ int ib_ud_header_unpack(void *buf, ...@@ -525,14 +525,13 @@ int ib_ud_header_unpack(void *buf,
header->immediate_present = 1; header->immediate_present = 1;
break; break;
default: default:
printk(KERN_WARNING "Invalid BTH.opcode 0x%02x\n", pr_warn("Invalid BTH.opcode 0x%02x\n", header->bth.opcode);
header->bth.opcode);
return -EINVAL; return -EINVAL;
} }
if (header->bth.transport_header_version != 0) { if (header->bth.transport_header_version != 0) {
printk(KERN_WARNING "Invalid BTH.transport_header_version %d\n", pr_warn("Invalid BTH.transport_header_version %d\n",
header->bth.transport_header_version); header->bth.transport_header_version);
return -EINVAL; return -EINVAL;
} }
......
...@@ -1174,6 +1174,7 @@ ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file, ...@@ -1174,6 +1174,7 @@ ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file,
struct ib_uobject *uobj; struct ib_uobject *uobj;
struct ib_pd *pd; struct ib_pd *pd;
struct ib_mw *mw; struct ib_mw *mw;
struct ib_udata udata;
int ret; int ret;
if (out_len < sizeof(resp)) if (out_len < sizeof(resp))
...@@ -1195,7 +1196,12 @@ ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file, ...@@ -1195,7 +1196,12 @@ ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file,
goto err_free; goto err_free;
} }
mw = pd->device->alloc_mw(pd, cmd.mw_type); INIT_UDATA(&udata, buf + sizeof(cmd),
(unsigned long)cmd.response + sizeof(resp),
in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
out_len - sizeof(resp));
mw = pd->device->alloc_mw(pd, cmd.mw_type, &udata);
if (IS_ERR(mw)) { if (IS_ERR(mw)) {
ret = PTR_ERR(mw); ret = PTR_ERR(mw);
goto err_put; goto err_put;
...@@ -3086,6 +3092,14 @@ int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file, ...@@ -3086,6 +3092,14 @@ int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file,
!capable(CAP_NET_ADMIN)) || !capable(CAP_NET_RAW)) !capable(CAP_NET_ADMIN)) || !capable(CAP_NET_RAW))
return -EPERM; return -EPERM;
if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED)
return -EINVAL;
if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) &&
((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) ||
(cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT)))
return -EINVAL;
if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS) if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS)
return -EINVAL; return -EINVAL;
......
...@@ -683,12 +683,28 @@ struct ib_uverbs_event_file *ib_uverbs_lookup_comp_file(int fd) ...@@ -683,12 +683,28 @@ struct ib_uverbs_event_file *ib_uverbs_lookup_comp_file(int fd)
return ev_file; return ev_file;
} }
static int verify_command_mask(struct ib_device *ib_dev, __u32 command)
{
u64 mask;
if (command <= IB_USER_VERBS_CMD_OPEN_QP)
mask = ib_dev->uverbs_cmd_mask;
else
mask = ib_dev->uverbs_ex_cmd_mask;
if (mask & ((u64)1 << command))
return 0;
return -1;
}
static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf, static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,
size_t count, loff_t *pos) size_t count, loff_t *pos)
{ {
struct ib_uverbs_file *file = filp->private_data; struct ib_uverbs_file *file = filp->private_data;
struct ib_device *ib_dev; struct ib_device *ib_dev;
struct ib_uverbs_cmd_hdr hdr; struct ib_uverbs_cmd_hdr hdr;
__u32 command;
__u32 flags; __u32 flags;
int srcu_key; int srcu_key;
ssize_t ret; ssize_t ret;
...@@ -707,37 +723,34 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf, ...@@ -707,37 +723,34 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,
goto out; goto out;
} }
flags = (hdr.command & if (hdr.command & ~(__u32)(IB_USER_VERBS_CMD_FLAGS_MASK |
IB_USER_VERBS_CMD_FLAGS_MASK) >> IB_USER_VERBS_CMD_FLAGS_SHIFT; IB_USER_VERBS_CMD_COMMAND_MASK)) {
ret = -EINVAL;
goto out;
}
if (!flags) { command = hdr.command & IB_USER_VERBS_CMD_COMMAND_MASK;
__u32 command; if (verify_command_mask(ib_dev, command)) {
ret = -EOPNOTSUPP;
goto out;
}
if (hdr.command & ~(__u32)(IB_USER_VERBS_CMD_FLAGS_MASK | if (!file->ucontext &&
IB_USER_VERBS_CMD_COMMAND_MASK)) { command != IB_USER_VERBS_CMD_GET_CONTEXT) {
ret = -EINVAL; ret = -EINVAL;
goto out; goto out;
} }
command = hdr.command & IB_USER_VERBS_CMD_COMMAND_MASK; flags = (hdr.command &
IB_USER_VERBS_CMD_FLAGS_MASK) >> IB_USER_VERBS_CMD_FLAGS_SHIFT;
if (!flags) {
if (command >= ARRAY_SIZE(uverbs_cmd_table) || if (command >= ARRAY_SIZE(uverbs_cmd_table) ||
!uverbs_cmd_table[command]) { !uverbs_cmd_table[command]) {
ret = -EINVAL; ret = -EINVAL;
goto out; goto out;
} }
if (!file->ucontext &&
command != IB_USER_VERBS_CMD_GET_CONTEXT) {
ret = -EINVAL;
goto out;
}
if (!(ib_dev->uverbs_cmd_mask & (1ull << command))) {
ret = -ENOSYS;
goto out;
}
if (hdr.in_words * 4 != count) { if (hdr.in_words * 4 != count) {
ret = -EINVAL; ret = -EINVAL;
goto out; goto out;
...@@ -749,21 +762,11 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf, ...@@ -749,21 +762,11 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,
hdr.out_words * 4); hdr.out_words * 4);
} else if (flags == IB_USER_VERBS_CMD_FLAG_EXTENDED) { } else if (flags == IB_USER_VERBS_CMD_FLAG_EXTENDED) {
__u32 command;
struct ib_uverbs_ex_cmd_hdr ex_hdr; struct ib_uverbs_ex_cmd_hdr ex_hdr;
struct ib_udata ucore; struct ib_udata ucore;
struct ib_udata uhw; struct ib_udata uhw;
size_t written_count = count; size_t written_count = count;
if (hdr.command & ~(__u32)(IB_USER_VERBS_CMD_FLAGS_MASK |
IB_USER_VERBS_CMD_COMMAND_MASK)) {
ret = -EINVAL;
goto out;
}
command = hdr.command & IB_USER_VERBS_CMD_COMMAND_MASK;
if (command >= ARRAY_SIZE(uverbs_ex_cmd_table) || if (command >= ARRAY_SIZE(uverbs_ex_cmd_table) ||
!uverbs_ex_cmd_table[command]) { !uverbs_ex_cmd_table[command]) {
ret = -ENOSYS; ret = -ENOSYS;
...@@ -775,11 +778,6 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf, ...@@ -775,11 +778,6 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,
goto out; goto out;
} }
if (!(ib_dev->uverbs_ex_cmd_mask & (1ull << command))) {
ret = -ENOSYS;
goto out;
}
if (count < (sizeof(hdr) + sizeof(ex_hdr))) { if (count < (sizeof(hdr) + sizeof(ex_hdr))) {
ret = -EINVAL; ret = -EINVAL;
goto out; goto out;
...@@ -1058,7 +1056,7 @@ static int find_overflow_devnum(void) ...@@ -1058,7 +1056,7 @@ static int find_overflow_devnum(void)
ret = alloc_chrdev_region(&overflow_maj, 0, IB_UVERBS_MAX_DEVICES, ret = alloc_chrdev_region(&overflow_maj, 0, IB_UVERBS_MAX_DEVICES,
"infiniband_verbs"); "infiniband_verbs");
if (ret) { if (ret) {
printk(KERN_ERR "user_verbs: couldn't register dynamic device number\n"); pr_err("user_verbs: couldn't register dynamic device number\n");
return ret; return ret;
} }
} }
...@@ -1279,14 +1277,14 @@ static int __init ib_uverbs_init(void) ...@@ -1279,14 +1277,14 @@ static int __init ib_uverbs_init(void)
ret = register_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES, ret = register_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES,
"infiniband_verbs"); "infiniband_verbs");
if (ret) { if (ret) {
printk(KERN_ERR "user_verbs: couldn't register device number\n"); pr_err("user_verbs: couldn't register device number\n");
goto out; goto out;
} }
uverbs_class = class_create(THIS_MODULE, "infiniband_verbs"); uverbs_class = class_create(THIS_MODULE, "infiniband_verbs");
if (IS_ERR(uverbs_class)) { if (IS_ERR(uverbs_class)) {
ret = PTR_ERR(uverbs_class); ret = PTR_ERR(uverbs_class);
printk(KERN_ERR "user_verbs: couldn't create class infiniband_verbs\n"); pr_err("user_verbs: couldn't create class infiniband_verbs\n");
goto out_chrdev; goto out_chrdev;
} }
...@@ -1294,13 +1292,13 @@ static int __init ib_uverbs_init(void) ...@@ -1294,13 +1292,13 @@ static int __init ib_uverbs_init(void)
ret = class_create_file(uverbs_class, &class_attr_abi_version.attr); ret = class_create_file(uverbs_class, &class_attr_abi_version.attr);
if (ret) { if (ret) {
printk(KERN_ERR "user_verbs: couldn't create abi_version attribute\n"); pr_err("user_verbs: couldn't create abi_version attribute\n");
goto out_class; goto out_class;
} }
ret = ib_register_client(&uverbs_client); ret = ib_register_client(&uverbs_client);
if (ret) { if (ret) {
printk(KERN_ERR "user_verbs: couldn't register client\n"); pr_err("user_verbs: couldn't register client\n");
goto out_class; goto out_class;
} }
......
...@@ -1567,6 +1567,8 @@ EXPORT_SYMBOL(ib_check_mr_status); ...@@ -1567,6 +1567,8 @@ EXPORT_SYMBOL(ib_check_mr_status);
* - The last sg element is allowed to have length less than page_size. * - The last sg element is allowed to have length less than page_size.
* - If sg_nents total byte length exceeds the mr max_num_sge * page_size * - If sg_nents total byte length exceeds the mr max_num_sge * page_size
* then only max_num_sg entries will be mapped. * then only max_num_sg entries will be mapped.
* - If the MR was allocated with type IB_MR_TYPE_SG_GAPS_REG, non of these
* constraints holds and the page_size argument is ignored.
* *
* Returns the number of sg elements that were mapped to the memory region. * Returns the number of sg elements that were mapped to the memory region.
* *
...@@ -1657,3 +1659,167 @@ int ib_sg_to_pages(struct ib_mr *mr, ...@@ -1657,3 +1659,167 @@ int ib_sg_to_pages(struct ib_mr *mr,
return i; return i;
} }
EXPORT_SYMBOL(ib_sg_to_pages); EXPORT_SYMBOL(ib_sg_to_pages);
struct ib_drain_cqe {
struct ib_cqe cqe;
struct completion done;
};
static void ib_drain_qp_done(struct ib_cq *cq, struct ib_wc *wc)
{
struct ib_drain_cqe *cqe = container_of(wc->wr_cqe, struct ib_drain_cqe,
cqe);
complete(&cqe->done);
}
/*
* Post a WR and block until its completion is reaped for the SQ.
*/
static void __ib_drain_sq(struct ib_qp *qp)
{
struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
struct ib_drain_cqe sdrain;
struct ib_send_wr swr = {}, *bad_swr;
int ret;
if (qp->send_cq->poll_ctx == IB_POLL_DIRECT) {
WARN_ONCE(qp->send_cq->poll_ctx == IB_POLL_DIRECT,
"IB_POLL_DIRECT poll_ctx not supported for drain\n");
return;
}
swr.wr_cqe = &sdrain.cqe;
sdrain.cqe.done = ib_drain_qp_done;
init_completion(&sdrain.done);
ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
if (ret) {
WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
return;
}
ret = ib_post_send(qp, &swr, &bad_swr);
if (ret) {
WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
return;
}
wait_for_completion(&sdrain.done);
}
/*
* Post a WR and block until its completion is reaped for the RQ.
*/
static void __ib_drain_rq(struct ib_qp *qp)
{
struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
struct ib_drain_cqe rdrain;
struct ib_recv_wr rwr = {}, *bad_rwr;
int ret;
if (qp->recv_cq->poll_ctx == IB_POLL_DIRECT) {
WARN_ONCE(qp->recv_cq->poll_ctx == IB_POLL_DIRECT,
"IB_POLL_DIRECT poll_ctx not supported for drain\n");
return;
}
rwr.wr_cqe = &rdrain.cqe;
rdrain.cqe.done = ib_drain_qp_done;
init_completion(&rdrain.done);
ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
if (ret) {
WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
return;
}
ret = ib_post_recv(qp, &rwr, &bad_rwr);
if (ret) {
WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
return;
}
wait_for_completion(&rdrain.done);
}
/**
* ib_drain_sq() - Block until all SQ CQEs have been consumed by the
* application.
* @qp: queue pair to drain
*
* If the device has a provider-specific drain function, then
* call that. Otherwise call the generic drain function
* __ib_drain_sq().
*
* The caller must:
*
* ensure there is room in the CQ and SQ for the drain work request and
* completion.
*
* allocate the CQ using ib_alloc_cq() and the CQ poll context cannot be
* IB_POLL_DIRECT.
*
* ensure that there are no other contexts that are posting WRs concurrently.
* Otherwise the drain is not guaranteed.
*/
void ib_drain_sq(struct ib_qp *qp)
{
if (qp->device->drain_sq)
qp->device->drain_sq(qp);
else
__ib_drain_sq(qp);
}
EXPORT_SYMBOL(ib_drain_sq);
/**
* ib_drain_rq() - Block until all RQ CQEs have been consumed by the
* application.
* @qp: queue pair to drain
*
* If the device has a provider-specific drain function, then
* call that. Otherwise call the generic drain function
* __ib_drain_rq().
*
* The caller must:
*
* ensure there is room in the CQ and RQ for the drain work request and
* completion.
*
* allocate the CQ using ib_alloc_cq() and the CQ poll context cannot be
* IB_POLL_DIRECT.
*
* ensure that there are no other contexts that are posting WRs concurrently.
* Otherwise the drain is not guaranteed.
*/
void ib_drain_rq(struct ib_qp *qp)
{
if (qp->device->drain_rq)
qp->device->drain_rq(qp);
else
__ib_drain_rq(qp);
}
EXPORT_SYMBOL(ib_drain_rq);
/**
* ib_drain_qp() - Block until all CQEs have been consumed by the
* application on both the RQ and SQ.
* @qp: queue pair to drain
*
* The caller must:
*
* ensure there is room in the CQ(s), SQ, and RQ for drain work requests
* and completions.
*
* allocate the CQs using ib_alloc_cq() and the CQ poll context cannot be
* IB_POLL_DIRECT.
*
* ensure that there are no other contexts that are posting WRs concurrently.
* Otherwise the drain is not guaranteed.
*/
void ib_drain_qp(struct ib_qp *qp)
{
ib_drain_sq(qp);
ib_drain_rq(qp);
}
EXPORT_SYMBOL(ib_drain_qp);
...@@ -1877,7 +1877,7 @@ int iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) ...@@ -1877,7 +1877,7 @@ int iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
static int is_loopback_dst(struct iw_cm_id *cm_id) static int is_loopback_dst(struct iw_cm_id *cm_id)
{ {
struct net_device *dev; struct net_device *dev;
struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr; struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->m_remote_addr;
dev = ip_dev_find(&init_net, raddr->sin_addr.s_addr); dev = ip_dev_find(&init_net, raddr->sin_addr.s_addr);
if (!dev) if (!dev)
...@@ -1892,10 +1892,10 @@ int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) ...@@ -1892,10 +1892,10 @@ int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
struct iwch_ep *ep; struct iwch_ep *ep;
struct rtable *rt; struct rtable *rt;
int err = 0; int err = 0;
struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->local_addr; struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr; struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->m_remote_addr;
if (cm_id->remote_addr.ss_family != PF_INET) { if (cm_id->m_remote_addr.ss_family != PF_INET) {
err = -ENOSYS; err = -ENOSYS;
goto out; goto out;
} }
...@@ -1961,9 +1961,9 @@ int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) ...@@ -1961,9 +1961,9 @@ int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
state_set(&ep->com, CONNECTING); state_set(&ep->com, CONNECTING);
ep->tos = IPTOS_LOWDELAY; ep->tos = IPTOS_LOWDELAY;
memcpy(&ep->com.local_addr, &cm_id->local_addr, memcpy(&ep->com.local_addr, &cm_id->m_local_addr,
sizeof(ep->com.local_addr)); sizeof(ep->com.local_addr));
memcpy(&ep->com.remote_addr, &cm_id->remote_addr, memcpy(&ep->com.remote_addr, &cm_id->m_remote_addr,
sizeof(ep->com.remote_addr)); sizeof(ep->com.remote_addr));
/* send connect request to rnic */ /* send connect request to rnic */
...@@ -1992,7 +1992,7 @@ int iwch_create_listen(struct iw_cm_id *cm_id, int backlog) ...@@ -1992,7 +1992,7 @@ int iwch_create_listen(struct iw_cm_id *cm_id, int backlog)
might_sleep(); might_sleep();
if (cm_id->local_addr.ss_family != PF_INET) { if (cm_id->m_local_addr.ss_family != PF_INET) {
err = -ENOSYS; err = -ENOSYS;
goto fail1; goto fail1;
} }
...@@ -2008,7 +2008,7 @@ int iwch_create_listen(struct iw_cm_id *cm_id, int backlog) ...@@ -2008,7 +2008,7 @@ int iwch_create_listen(struct iw_cm_id *cm_id, int backlog)
cm_id->add_ref(cm_id); cm_id->add_ref(cm_id);
ep->com.cm_id = cm_id; ep->com.cm_id = cm_id;
ep->backlog = backlog; ep->backlog = backlog;
memcpy(&ep->com.local_addr, &cm_id->local_addr, memcpy(&ep->com.local_addr, &cm_id->m_local_addr,
sizeof(ep->com.local_addr)); sizeof(ep->com.local_addr));
/* /*
......
...@@ -657,7 +657,8 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, ...@@ -657,7 +657,8 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
return ERR_PTR(err); return ERR_PTR(err);
} }
static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd, enum ib_mw_type type) static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
struct ib_udata *udata)
{ {
struct iwch_dev *rhp; struct iwch_dev *rhp;
struct iwch_pd *php; struct iwch_pd *php;
......
此差异已折叠。
...@@ -815,8 +815,15 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc) ...@@ -815,8 +815,15 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc)
} }
} }
out: out:
if (wq) if (wq) {
if (unlikely(qhp->attr.state != C4IW_QP_STATE_RTS)) {
if (t4_sq_empty(wq))
complete(&qhp->sq_drained);
if (t4_rq_empty(wq))
complete(&qhp->rq_drained);
}
spin_unlock(&qhp->lock); spin_unlock(&qhp->lock);
}
return ret; return ret;
} }
......
...@@ -87,17 +87,6 @@ struct c4iw_debugfs_data { ...@@ -87,17 +87,6 @@ struct c4iw_debugfs_data {
int pos; int pos;
}; };
/* registered cxgb4 netlink callbacks */
static struct ibnl_client_cbs c4iw_nl_cb_table[] = {
[RDMA_NL_IWPM_REG_PID] = {.dump = iwpm_register_pid_cb},
[RDMA_NL_IWPM_ADD_MAPPING] = {.dump = iwpm_add_mapping_cb},
[RDMA_NL_IWPM_QUERY_MAPPING] = {.dump = iwpm_add_and_query_mapping_cb},
[RDMA_NL_IWPM_HANDLE_ERR] = {.dump = iwpm_mapping_error_cb},
[RDMA_NL_IWPM_REMOTE_INFO] = {.dump = iwpm_remote_info_cb},
[RDMA_NL_IWPM_MAPINFO] = {.dump = iwpm_mapping_info_cb},
[RDMA_NL_IWPM_MAPINFO_NUM] = {.dump = iwpm_ack_mapping_info_cb}
};
static int count_idrs(int id, void *p, void *data) static int count_idrs(int id, void *p, void *data)
{ {
int *countp = data; int *countp = data;
...@@ -242,13 +231,13 @@ static int dump_qp(int id, void *p, void *data) ...@@ -242,13 +231,13 @@ static int dump_qp(int id, void *p, void *data)
if (qp->ep) { if (qp->ep) {
if (qp->ep->com.local_addr.ss_family == AF_INET) { if (qp->ep->com.local_addr.ss_family == AF_INET) {
struct sockaddr_in *lsin = (struct sockaddr_in *) struct sockaddr_in *lsin = (struct sockaddr_in *)
&qp->ep->com.local_addr; &qp->ep->com.cm_id->local_addr;
struct sockaddr_in *rsin = (struct sockaddr_in *) struct sockaddr_in *rsin = (struct sockaddr_in *)
&qp->ep->com.remote_addr; &qp->ep->com.cm_id->remote_addr;
struct sockaddr_in *mapped_lsin = (struct sockaddr_in *) struct sockaddr_in *mapped_lsin = (struct sockaddr_in *)
&qp->ep->com.mapped_local_addr; &qp->ep->com.cm_id->m_local_addr;
struct sockaddr_in *mapped_rsin = (struct sockaddr_in *) struct sockaddr_in *mapped_rsin = (struct sockaddr_in *)
&qp->ep->com.mapped_remote_addr; &qp->ep->com.cm_id->m_remote_addr;
cc = snprintf(qpd->buf + qpd->pos, space, cc = snprintf(qpd->buf + qpd->pos, space,
"rc qp sq id %u rq id %u state %u " "rc qp sq id %u rq id %u state %u "
...@@ -264,15 +253,15 @@ static int dump_qp(int id, void *p, void *data) ...@@ -264,15 +253,15 @@ static int dump_qp(int id, void *p, void *data)
ntohs(mapped_rsin->sin_port)); ntohs(mapped_rsin->sin_port));
} else { } else {
struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *) struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *)
&qp->ep->com.local_addr; &qp->ep->com.cm_id->local_addr;
struct sockaddr_in6 *rsin6 = (struct sockaddr_in6 *) struct sockaddr_in6 *rsin6 = (struct sockaddr_in6 *)
&qp->ep->com.remote_addr; &qp->ep->com.cm_id->remote_addr;
struct sockaddr_in6 *mapped_lsin6 = struct sockaddr_in6 *mapped_lsin6 =
(struct sockaddr_in6 *) (struct sockaddr_in6 *)
&qp->ep->com.mapped_local_addr; &qp->ep->com.cm_id->m_local_addr;
struct sockaddr_in6 *mapped_rsin6 = struct sockaddr_in6 *mapped_rsin6 =
(struct sockaddr_in6 *) (struct sockaddr_in6 *)
&qp->ep->com.mapped_remote_addr; &qp->ep->com.cm_id->m_remote_addr;
cc = snprintf(qpd->buf + qpd->pos, space, cc = snprintf(qpd->buf + qpd->pos, space,
"rc qp sq id %u rq id %u state %u " "rc qp sq id %u rq id %u state %u "
...@@ -545,13 +534,13 @@ static int dump_ep(int id, void *p, void *data) ...@@ -545,13 +534,13 @@ static int dump_ep(int id, void *p, void *data)
if (ep->com.local_addr.ss_family == AF_INET) { if (ep->com.local_addr.ss_family == AF_INET) {
struct sockaddr_in *lsin = (struct sockaddr_in *) struct sockaddr_in *lsin = (struct sockaddr_in *)
&ep->com.local_addr; &ep->com.cm_id->local_addr;
struct sockaddr_in *rsin = (struct sockaddr_in *) struct sockaddr_in *rsin = (struct sockaddr_in *)
&ep->com.remote_addr; &ep->com.cm_id->remote_addr;
struct sockaddr_in *mapped_lsin = (struct sockaddr_in *) struct sockaddr_in *mapped_lsin = (struct sockaddr_in *)
&ep->com.mapped_local_addr; &ep->com.cm_id->m_local_addr;
struct sockaddr_in *mapped_rsin = (struct sockaddr_in *) struct sockaddr_in *mapped_rsin = (struct sockaddr_in *)
&ep->com.mapped_remote_addr; &ep->com.cm_id->m_remote_addr;
cc = snprintf(epd->buf + epd->pos, space, cc = snprintf(epd->buf + epd->pos, space,
"ep %p cm_id %p qp %p state %d flags 0x%lx " "ep %p cm_id %p qp %p state %d flags 0x%lx "
...@@ -569,13 +558,13 @@ static int dump_ep(int id, void *p, void *data) ...@@ -569,13 +558,13 @@ static int dump_ep(int id, void *p, void *data)
ntohs(mapped_rsin->sin_port)); ntohs(mapped_rsin->sin_port));
} else { } else {
struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *) struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *)
&ep->com.local_addr; &ep->com.cm_id->local_addr;
struct sockaddr_in6 *rsin6 = (struct sockaddr_in6 *) struct sockaddr_in6 *rsin6 = (struct sockaddr_in6 *)
&ep->com.remote_addr; &ep->com.cm_id->remote_addr;
struct sockaddr_in6 *mapped_lsin6 = (struct sockaddr_in6 *) struct sockaddr_in6 *mapped_lsin6 = (struct sockaddr_in6 *)
&ep->com.mapped_local_addr; &ep->com.cm_id->m_local_addr;
struct sockaddr_in6 *mapped_rsin6 = (struct sockaddr_in6 *) struct sockaddr_in6 *mapped_rsin6 = (struct sockaddr_in6 *)
&ep->com.mapped_remote_addr; &ep->com.cm_id->m_remote_addr;
cc = snprintf(epd->buf + epd->pos, space, cc = snprintf(epd->buf + epd->pos, space,
"ep %p cm_id %p qp %p state %d flags 0x%lx " "ep %p cm_id %p qp %p state %d flags 0x%lx "
...@@ -610,9 +599,9 @@ static int dump_listen_ep(int id, void *p, void *data) ...@@ -610,9 +599,9 @@ static int dump_listen_ep(int id, void *p, void *data)
if (ep->com.local_addr.ss_family == AF_INET) { if (ep->com.local_addr.ss_family == AF_INET) {
struct sockaddr_in *lsin = (struct sockaddr_in *) struct sockaddr_in *lsin = (struct sockaddr_in *)
&ep->com.local_addr; &ep->com.cm_id->local_addr;
struct sockaddr_in *mapped_lsin = (struct sockaddr_in *) struct sockaddr_in *mapped_lsin = (struct sockaddr_in *)
&ep->com.mapped_local_addr; &ep->com.cm_id->m_local_addr;
cc = snprintf(epd->buf + epd->pos, space, cc = snprintf(epd->buf + epd->pos, space,
"ep %p cm_id %p state %d flags 0x%lx stid %d " "ep %p cm_id %p state %d flags 0x%lx stid %d "
...@@ -623,9 +612,9 @@ static int dump_listen_ep(int id, void *p, void *data) ...@@ -623,9 +612,9 @@ static int dump_listen_ep(int id, void *p, void *data)
ntohs(mapped_lsin->sin_port)); ntohs(mapped_lsin->sin_port));
} else { } else {
struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *) struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *)
&ep->com.local_addr; &ep->com.cm_id->local_addr;
struct sockaddr_in6 *mapped_lsin6 = (struct sockaddr_in6 *) struct sockaddr_in6 *mapped_lsin6 = (struct sockaddr_in6 *)
&ep->com.mapped_local_addr; &ep->com.cm_id->m_local_addr;
cc = snprintf(epd->buf + epd->pos, space, cc = snprintf(epd->buf + epd->pos, space,
"ep %p cm_id %p state %d flags 0x%lx stid %d " "ep %p cm_id %p state %d flags 0x%lx stid %d "
...@@ -801,10 +790,9 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev) ...@@ -801,10 +790,9 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev)
rdev->lldi.vr->qp.size, rdev->lldi.vr->qp.size,
rdev->lldi.vr->cq.start, rdev->lldi.vr->cq.start,
rdev->lldi.vr->cq.size); rdev->lldi.vr->cq.size);
PDBG("udb len 0x%x udb base %p db_reg %p gts_reg %p " PDBG("udb %pR db_reg %p gts_reg %p "
"qpmask 0x%x cqmask 0x%x\n", "qpmask 0x%x cqmask 0x%x\n",
(unsigned)pci_resource_len(rdev->lldi.pdev, 2), &rdev->lldi.pdev->resource[2],
(void *)pci_resource_start(rdev->lldi.pdev, 2),
rdev->lldi.db_reg, rdev->lldi.gts_reg, rdev->lldi.db_reg, rdev->lldi.gts_reg,
rdev->qpmask, rdev->cqmask); rdev->qpmask, rdev->cqmask);
...@@ -1506,20 +1494,6 @@ static int __init c4iw_init_module(void) ...@@ -1506,20 +1494,6 @@ static int __init c4iw_init_module(void)
printk(KERN_WARNING MOD printk(KERN_WARNING MOD
"could not create debugfs entry, continuing\n"); "could not create debugfs entry, continuing\n");
if (ibnl_add_client(RDMA_NL_C4IW, RDMA_NL_IWPM_NUM_OPS,
c4iw_nl_cb_table))
pr_err("%s[%u]: Failed to add netlink callback\n"
, __func__, __LINE__);
err = iwpm_init(RDMA_NL_C4IW);
if (err) {
pr_err("port mapper initialization failed with %d\n", err);
ibnl_remove_client(RDMA_NL_C4IW);
c4iw_cm_term();
debugfs_remove_recursive(c4iw_debugfs_root);
return err;
}
cxgb4_register_uld(CXGB4_ULD_RDMA, &c4iw_uld_info); cxgb4_register_uld(CXGB4_ULD_RDMA, &c4iw_uld_info);
return 0; return 0;
...@@ -1537,8 +1511,6 @@ static void __exit c4iw_exit_module(void) ...@@ -1537,8 +1511,6 @@ static void __exit c4iw_exit_module(void)
} }
mutex_unlock(&dev_mutex); mutex_unlock(&dev_mutex);
cxgb4_unregister_uld(CXGB4_ULD_RDMA); cxgb4_unregister_uld(CXGB4_ULD_RDMA);
iwpm_exit(RDMA_NL_C4IW);
ibnl_remove_client(RDMA_NL_C4IW);
c4iw_cm_term(); c4iw_cm_term();
debugfs_remove_recursive(c4iw_debugfs_root); debugfs_remove_recursive(c4iw_debugfs_root);
} }
......
...@@ -476,6 +476,8 @@ struct c4iw_qp { ...@@ -476,6 +476,8 @@ struct c4iw_qp {
wait_queue_head_t wait; wait_queue_head_t wait;
struct timer_list timer; struct timer_list timer;
int sq_sig_all; int sq_sig_all;
struct completion rq_drained;
struct completion sq_drained;
}; };
static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp) static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp)
...@@ -753,7 +755,6 @@ enum c4iw_ep_flags { ...@@ -753,7 +755,6 @@ enum c4iw_ep_flags {
CLOSE_SENT = 3, CLOSE_SENT = 3,
TIMEOUT = 4, TIMEOUT = 4,
QP_REFERENCED = 5, QP_REFERENCED = 5,
RELEASE_MAPINFO = 6,
}; };
enum c4iw_ep_history { enum c4iw_ep_history {
...@@ -790,8 +791,6 @@ struct c4iw_ep_common { ...@@ -790,8 +791,6 @@ struct c4iw_ep_common {
struct mutex mutex; struct mutex mutex;
struct sockaddr_storage local_addr; struct sockaddr_storage local_addr;
struct sockaddr_storage remote_addr; struct sockaddr_storage remote_addr;
struct sockaddr_storage mapped_local_addr;
struct sockaddr_storage mapped_remote_addr;
struct c4iw_wr_wait wr_wait; struct c4iw_wr_wait wr_wait;
unsigned long flags; unsigned long flags;
unsigned long history; unsigned long history;
...@@ -843,45 +842,6 @@ struct c4iw_ep { ...@@ -843,45 +842,6 @@ struct c4iw_ep {
struct c4iw_ep_stats stats; struct c4iw_ep_stats stats;
}; };
static inline void print_addr(struct c4iw_ep_common *epc, const char *func,
const char *msg)
{
#define SINA(a) (&(((struct sockaddr_in *)(a))->sin_addr.s_addr))
#define SINP(a) ntohs(((struct sockaddr_in *)(a))->sin_port)
#define SIN6A(a) (&(((struct sockaddr_in6 *)(a))->sin6_addr))
#define SIN6P(a) ntohs(((struct sockaddr_in6 *)(a))->sin6_port)
if (c4iw_debug) {
switch (epc->local_addr.ss_family) {
case AF_INET:
PDBG("%s %s %pI4:%u/%u <-> %pI4:%u/%u\n",
func, msg, SINA(&epc->local_addr),
SINP(&epc->local_addr),
SINP(&epc->mapped_local_addr),
SINA(&epc->remote_addr),
SINP(&epc->remote_addr),
SINP(&epc->mapped_remote_addr));
break;
case AF_INET6:
PDBG("%s %s %pI6:%u/%u <-> %pI6:%u/%u\n",
func, msg, SIN6A(&epc->local_addr),
SIN6P(&epc->local_addr),
SIN6P(&epc->mapped_local_addr),
SIN6A(&epc->remote_addr),
SIN6P(&epc->remote_addr),
SIN6P(&epc->mapped_remote_addr));
break;
default:
break;
}
}
#undef SINA
#undef SINP
#undef SIN6A
#undef SIN6P
}
static inline struct c4iw_ep *to_ep(struct iw_cm_id *cm_id) static inline struct c4iw_ep *to_ep(struct iw_cm_id *cm_id)
{ {
return cm_id->provider_data; return cm_id->provider_data;
...@@ -961,7 +921,8 @@ int c4iw_map_mr_sg(struct ib_mr *ibmr, ...@@ -961,7 +921,8 @@ int c4iw_map_mr_sg(struct ib_mr *ibmr,
struct scatterlist *sg, struct scatterlist *sg,
int sg_nents); int sg_nents);
int c4iw_dealloc_mw(struct ib_mw *mw); int c4iw_dealloc_mw(struct ib_mw *mw);
struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type); struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
struct ib_udata *udata);
struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start,
u64 length, u64 virt, int acc, u64 length, u64 virt, int acc,
struct ib_udata *udata); struct ib_udata *udata);
...@@ -1016,6 +977,8 @@ extern int c4iw_wr_log; ...@@ -1016,6 +977,8 @@ extern int c4iw_wr_log;
extern int db_fc_threshold; extern int db_fc_threshold;
extern int db_coalescing_threshold; extern int db_coalescing_threshold;
extern int use_dsgl; extern int use_dsgl;
void c4iw_drain_rq(struct ib_qp *qp);
void c4iw_drain_sq(struct ib_qp *qp);
#endif #endif
...@@ -34,6 +34,7 @@ ...@@ -34,6 +34,7 @@
#include <linux/moduleparam.h> #include <linux/moduleparam.h>
#include <rdma/ib_umem.h> #include <rdma/ib_umem.h>
#include <linux/atomic.h> #include <linux/atomic.h>
#include <rdma/ib_user_verbs.h>
#include "iw_cxgb4.h" #include "iw_cxgb4.h"
...@@ -552,7 +553,8 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, ...@@ -552,7 +553,8 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
return ERR_PTR(err); return ERR_PTR(err);
} }
struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type) struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
struct ib_udata *udata)
{ {
struct c4iw_dev *rhp; struct c4iw_dev *rhp;
struct c4iw_pd *php; struct c4iw_pd *php;
...@@ -617,12 +619,14 @@ struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, ...@@ -617,12 +619,14 @@ struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd,
int ret = 0; int ret = 0;
int length = roundup(max_num_sg * sizeof(u64), 32); int length = roundup(max_num_sg * sizeof(u64), 32);
php = to_c4iw_pd(pd);
rhp = php->rhp;
if (mr_type != IB_MR_TYPE_MEM_REG || if (mr_type != IB_MR_TYPE_MEM_REG ||
max_num_sg > t4_max_fr_depth(use_dsgl)) max_num_sg > t4_max_fr_depth(&rhp->rdev.lldi.ulptx_memwrite_dsgl &&
use_dsgl))
return ERR_PTR(-EINVAL); return ERR_PTR(-EINVAL);
php = to_c4iw_pd(pd);
rhp = php->rhp;
mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
if (!mhp) { if (!mhp) {
ret = -ENOMEM; ret = -ENOMEM;
......
...@@ -339,7 +339,8 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro ...@@ -339,7 +339,8 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro
props->max_mr = c4iw_num_stags(&dev->rdev); props->max_mr = c4iw_num_stags(&dev->rdev);
props->max_pd = T4_MAX_NUM_PD; props->max_pd = T4_MAX_NUM_PD;
props->local_ca_ack_delay = 0; props->local_ca_ack_delay = 0;
props->max_fast_reg_page_list_len = t4_max_fr_depth(use_dsgl); props->max_fast_reg_page_list_len =
t4_max_fr_depth(dev->rdev.lldi.ulptx_memwrite_dsgl && use_dsgl);
return 0; return 0;
} }
...@@ -564,6 +565,8 @@ int c4iw_register_device(struct c4iw_dev *dev) ...@@ -564,6 +565,8 @@ int c4iw_register_device(struct c4iw_dev *dev)
dev->ibdev.get_protocol_stats = c4iw_get_mib; dev->ibdev.get_protocol_stats = c4iw_get_mib;
dev->ibdev.uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION; dev->ibdev.uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION;
dev->ibdev.get_port_immutable = c4iw_port_immutable; dev->ibdev.get_port_immutable = c4iw_port_immutable;
dev->ibdev.drain_sq = c4iw_drain_sq;
dev->ibdev.drain_rq = c4iw_drain_rq;
dev->ibdev.iwcm = kmalloc(sizeof(struct iw_cm_verbs), GFP_KERNEL); dev->ibdev.iwcm = kmalloc(sizeof(struct iw_cm_verbs), GFP_KERNEL);
if (!dev->ibdev.iwcm) if (!dev->ibdev.iwcm)
......
...@@ -606,7 +606,7 @@ static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe, ...@@ -606,7 +606,7 @@ static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe,
} }
static int build_memreg(struct t4_sq *sq, union t4_wr *wqe, static int build_memreg(struct t4_sq *sq, union t4_wr *wqe,
struct ib_reg_wr *wr, u8 *len16, u8 t5dev) struct ib_reg_wr *wr, u8 *len16, bool dsgl_supported)
{ {
struct c4iw_mr *mhp = to_c4iw_mr(wr->mr); struct c4iw_mr *mhp = to_c4iw_mr(wr->mr);
struct fw_ri_immd *imdp; struct fw_ri_immd *imdp;
...@@ -615,7 +615,7 @@ static int build_memreg(struct t4_sq *sq, union t4_wr *wqe, ...@@ -615,7 +615,7 @@ static int build_memreg(struct t4_sq *sq, union t4_wr *wqe,
int pbllen = roundup(mhp->mpl_len * sizeof(u64), 32); int pbllen = roundup(mhp->mpl_len * sizeof(u64), 32);
int rem; int rem;
if (mhp->mpl_len > t4_max_fr_depth(use_dsgl)) if (mhp->mpl_len > t4_max_fr_depth(dsgl_supported && use_dsgl))
return -EINVAL; return -EINVAL;
wqe->fr.qpbinde_to_dcacpu = 0; wqe->fr.qpbinde_to_dcacpu = 0;
...@@ -629,7 +629,7 @@ static int build_memreg(struct t4_sq *sq, union t4_wr *wqe, ...@@ -629,7 +629,7 @@ static int build_memreg(struct t4_sq *sq, union t4_wr *wqe,
wqe->fr.va_lo_fbo = cpu_to_be32(mhp->ibmr.iova & wqe->fr.va_lo_fbo = cpu_to_be32(mhp->ibmr.iova &
0xffffffff); 0xffffffff);
if (t5dev && use_dsgl && (pbllen > max_fr_immd)) { if (dsgl_supported && use_dsgl && (pbllen > max_fr_immd)) {
struct fw_ri_dsgl *sglp; struct fw_ri_dsgl *sglp;
for (i = 0; i < mhp->mpl_len; i++) for (i = 0; i < mhp->mpl_len; i++)
...@@ -808,9 +808,7 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, ...@@ -808,9 +808,7 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
fw_opcode = FW_RI_FR_NSMR_WR; fw_opcode = FW_RI_FR_NSMR_WR;
swsqe->opcode = FW_RI_FAST_REGISTER; swsqe->opcode = FW_RI_FAST_REGISTER;
err = build_memreg(&qhp->wq.sq, wqe, reg_wr(wr), &len16, err = build_memreg(&qhp->wq.sq, wqe, reg_wr(wr), &len16,
is_t5( qhp->rhp->rdev.lldi.ulptx_memwrite_dsgl);
qhp->rhp->rdev.lldi.adapter_type) ?
1 : 0);
break; break;
case IB_WR_LOCAL_INV: case IB_WR_LOCAL_INV:
if (wr->send_flags & IB_SEND_FENCE) if (wr->send_flags & IB_SEND_FENCE)
...@@ -1621,7 +1619,8 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, ...@@ -1621,7 +1619,8 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
unsigned int sqsize, rqsize; unsigned int sqsize, rqsize;
struct c4iw_ucontext *ucontext; struct c4iw_ucontext *ucontext;
int ret; int ret;
struct c4iw_mm_entry *mm1, *mm2, *mm3, *mm4, *mm5 = NULL; struct c4iw_mm_entry *sq_key_mm, *rq_key_mm = NULL, *sq_db_key_mm;
struct c4iw_mm_entry *rq_db_key_mm = NULL, *ma_sync_key_mm = NULL;
PDBG("%s ib_pd %p\n", __func__, pd); PDBG("%s ib_pd %p\n", __func__, pd);
...@@ -1697,6 +1696,8 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, ...@@ -1697,6 +1696,8 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
qhp->attr.max_ird = 0; qhp->attr.max_ird = 0;
qhp->sq_sig_all = attrs->sq_sig_type == IB_SIGNAL_ALL_WR; qhp->sq_sig_all = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;
spin_lock_init(&qhp->lock); spin_lock_init(&qhp->lock);
init_completion(&qhp->sq_drained);
init_completion(&qhp->rq_drained);
mutex_init(&qhp->mutex); mutex_init(&qhp->mutex);
init_waitqueue_head(&qhp->wait); init_waitqueue_head(&qhp->wait);
atomic_set(&qhp->refcnt, 1); atomic_set(&qhp->refcnt, 1);
...@@ -1706,29 +1707,30 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, ...@@ -1706,29 +1707,30 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
goto err2; goto err2;
if (udata) { if (udata) {
mm1 = kmalloc(sizeof *mm1, GFP_KERNEL); sq_key_mm = kmalloc(sizeof(*sq_key_mm), GFP_KERNEL);
if (!mm1) { if (!sq_key_mm) {
ret = -ENOMEM; ret = -ENOMEM;
goto err3; goto err3;
} }
mm2 = kmalloc(sizeof *mm2, GFP_KERNEL); rq_key_mm = kmalloc(sizeof(*rq_key_mm), GFP_KERNEL);
if (!mm2) { if (!rq_key_mm) {
ret = -ENOMEM; ret = -ENOMEM;
goto err4; goto err4;
} }
mm3 = kmalloc(sizeof *mm3, GFP_KERNEL); sq_db_key_mm = kmalloc(sizeof(*sq_db_key_mm), GFP_KERNEL);
if (!mm3) { if (!sq_db_key_mm) {
ret = -ENOMEM; ret = -ENOMEM;
goto err5; goto err5;
} }
mm4 = kmalloc(sizeof *mm4, GFP_KERNEL); rq_db_key_mm = kmalloc(sizeof(*rq_db_key_mm), GFP_KERNEL);
if (!mm4) { if (!rq_db_key_mm) {
ret = -ENOMEM; ret = -ENOMEM;
goto err6; goto err6;
} }
if (t4_sq_onchip(&qhp->wq.sq)) { if (t4_sq_onchip(&qhp->wq.sq)) {
mm5 = kmalloc(sizeof *mm5, GFP_KERNEL); ma_sync_key_mm = kmalloc(sizeof(*ma_sync_key_mm),
if (!mm5) { GFP_KERNEL);
if (!ma_sync_key_mm) {
ret = -ENOMEM; ret = -ENOMEM;
goto err7; goto err7;
} }
...@@ -1743,7 +1745,7 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, ...@@ -1743,7 +1745,7 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
uresp.rq_size = qhp->wq.rq.size; uresp.rq_size = qhp->wq.rq.size;
uresp.rq_memsize = qhp->wq.rq.memsize; uresp.rq_memsize = qhp->wq.rq.memsize;
spin_lock(&ucontext->mmap_lock); spin_lock(&ucontext->mmap_lock);
if (mm5) { if (ma_sync_key_mm) {
uresp.ma_sync_key = ucontext->key; uresp.ma_sync_key = ucontext->key;
ucontext->key += PAGE_SIZE; ucontext->key += PAGE_SIZE;
} else { } else {
...@@ -1761,28 +1763,29 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, ...@@ -1761,28 +1763,29 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
ret = ib_copy_to_udata(udata, &uresp, sizeof uresp); ret = ib_copy_to_udata(udata, &uresp, sizeof uresp);
if (ret) if (ret)
goto err8; goto err8;
mm1->key = uresp.sq_key; sq_key_mm->key = uresp.sq_key;
mm1->addr = qhp->wq.sq.phys_addr; sq_key_mm->addr = qhp->wq.sq.phys_addr;
mm1->len = PAGE_ALIGN(qhp->wq.sq.memsize); sq_key_mm->len = PAGE_ALIGN(qhp->wq.sq.memsize);
insert_mmap(ucontext, mm1); insert_mmap(ucontext, sq_key_mm);
mm2->key = uresp.rq_key; rq_key_mm->key = uresp.rq_key;
mm2->addr = virt_to_phys(qhp->wq.rq.queue); rq_key_mm->addr = virt_to_phys(qhp->wq.rq.queue);
mm2->len = PAGE_ALIGN(qhp->wq.rq.memsize); rq_key_mm->len = PAGE_ALIGN(qhp->wq.rq.memsize);
insert_mmap(ucontext, mm2); insert_mmap(ucontext, rq_key_mm);
mm3->key = uresp.sq_db_gts_key; sq_db_key_mm->key = uresp.sq_db_gts_key;
mm3->addr = (__force unsigned long)qhp->wq.sq.bar2_pa; sq_db_key_mm->addr = (u64)(unsigned long)qhp->wq.sq.bar2_pa;
mm3->len = PAGE_SIZE; sq_db_key_mm->len = PAGE_SIZE;
insert_mmap(ucontext, mm3); insert_mmap(ucontext, sq_db_key_mm);
mm4->key = uresp.rq_db_gts_key; rq_db_key_mm->key = uresp.rq_db_gts_key;
mm4->addr = (__force unsigned long)qhp->wq.rq.bar2_pa; rq_db_key_mm->addr = (u64)(unsigned long)qhp->wq.rq.bar2_pa;
mm4->len = PAGE_SIZE; rq_db_key_mm->len = PAGE_SIZE;
insert_mmap(ucontext, mm4); insert_mmap(ucontext, rq_db_key_mm);
if (mm5) { if (ma_sync_key_mm) {
mm5->key = uresp.ma_sync_key; ma_sync_key_mm->key = uresp.ma_sync_key;
mm5->addr = (pci_resource_start(rhp->rdev.lldi.pdev, 0) ma_sync_key_mm->addr =
+ PCIE_MA_SYNC_A) & PAGE_MASK; (pci_resource_start(rhp->rdev.lldi.pdev, 0) +
mm5->len = PAGE_SIZE; PCIE_MA_SYNC_A) & PAGE_MASK;
insert_mmap(ucontext, mm5); ma_sync_key_mm->len = PAGE_SIZE;
insert_mmap(ucontext, ma_sync_key_mm);
} }
} }
qhp->ibqp.qp_num = qhp->wq.sq.qid; qhp->ibqp.qp_num = qhp->wq.sq.qid;
...@@ -1795,15 +1798,15 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, ...@@ -1795,15 +1798,15 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
qhp->wq.rq.memsize, attrs->cap.max_recv_wr); qhp->wq.rq.memsize, attrs->cap.max_recv_wr);
return &qhp->ibqp; return &qhp->ibqp;
err8: err8:
kfree(mm5); kfree(ma_sync_key_mm);
err7: err7:
kfree(mm4); kfree(rq_db_key_mm);
err6: err6:
kfree(mm3); kfree(sq_db_key_mm);
err5: err5:
kfree(mm2); kfree(rq_key_mm);
err4: err4:
kfree(mm1); kfree(sq_key_mm);
err3: err3:
remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid); remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid);
err2: err2:
...@@ -1888,3 +1891,17 @@ int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, ...@@ -1888,3 +1891,17 @@ int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
init_attr->sq_sig_type = qhp->sq_sig_all ? IB_SIGNAL_ALL_WR : 0; init_attr->sq_sig_type = qhp->sq_sig_all ? IB_SIGNAL_ALL_WR : 0;
return 0; return 0;
} }
void c4iw_drain_sq(struct ib_qp *ibqp)
{
struct c4iw_qp *qp = to_c4iw_qp(ibqp);
wait_for_completion(&qp->sq_drained);
}
void c4iw_drain_rq(struct ib_qp *ibqp)
{
struct c4iw_qp *qp = to_c4iw_qp(ibqp);
wait_for_completion(&qp->rq_drained);
}
...@@ -310,7 +310,7 @@ static void aliasguid_query_handler(int status, ...@@ -310,7 +310,7 @@ static void aliasguid_query_handler(int status,
if (status) { if (status) {
pr_debug("(port: %d) failed: status = %d\n", pr_debug("(port: %d) failed: status = %d\n",
cb_ctx->port, status); cb_ctx->port, status);
rec->time_to_run = ktime_get_real_ns() + 1 * NSEC_PER_SEC; rec->time_to_run = ktime_get_boot_ns() + 1 * NSEC_PER_SEC;
goto out; goto out;
} }
...@@ -416,7 +416,7 @@ static void aliasguid_query_handler(int status, ...@@ -416,7 +416,7 @@ static void aliasguid_query_handler(int status,
be64_to_cpu((__force __be64)rec->guid_indexes), be64_to_cpu((__force __be64)rec->guid_indexes),
be64_to_cpu((__force __be64)applied_guid_indexes), be64_to_cpu((__force __be64)applied_guid_indexes),
be64_to_cpu((__force __be64)declined_guid_indexes)); be64_to_cpu((__force __be64)declined_guid_indexes));
rec->time_to_run = ktime_get_real_ns() + rec->time_to_run = ktime_get_boot_ns() +
resched_delay_sec * NSEC_PER_SEC; resched_delay_sec * NSEC_PER_SEC;
} else { } else {
rec->status = MLX4_GUID_INFO_STATUS_SET; rec->status = MLX4_GUID_INFO_STATUS_SET;
...@@ -708,7 +708,7 @@ static int get_low_record_time_index(struct mlx4_ib_dev *dev, u8 port, ...@@ -708,7 +708,7 @@ static int get_low_record_time_index(struct mlx4_ib_dev *dev, u8 port,
} }
} }
if (resched_delay_sec) { if (resched_delay_sec) {
u64 curr_time = ktime_get_real_ns(); u64 curr_time = ktime_get_boot_ns();
*resched_delay_sec = (low_record_time < curr_time) ? 0 : *resched_delay_sec = (low_record_time < curr_time) ? 0 :
div_u64((low_record_time - curr_time), NSEC_PER_SEC); div_u64((low_record_time - curr_time), NSEC_PER_SEC);
......
...@@ -1643,6 +1643,56 @@ static int mlx4_ib_tunnel_steer_add(struct ib_qp *qp, struct ib_flow_attr *flow_ ...@@ -1643,6 +1643,56 @@ static int mlx4_ib_tunnel_steer_add(struct ib_qp *qp, struct ib_flow_attr *flow_
return err; return err;
} }
static int mlx4_ib_add_dont_trap_rule(struct mlx4_dev *dev,
struct ib_flow_attr *flow_attr,
enum mlx4_net_trans_promisc_mode *type)
{
int err = 0;
if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_DMFS_UC_MC_SNIFFER) ||
(dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_STATIC) ||
(flow_attr->num_of_specs > 1) || (flow_attr->priority != 0)) {
return -EOPNOTSUPP;
}
if (flow_attr->num_of_specs == 0) {
type[0] = MLX4_FS_MC_SNIFFER;
type[1] = MLX4_FS_UC_SNIFFER;
} else {
union ib_flow_spec *ib_spec;
ib_spec = (union ib_flow_spec *)(flow_attr + 1);
if (ib_spec->type != IB_FLOW_SPEC_ETH)
return -EINVAL;
/* if all is zero than MC and UC */
if (is_zero_ether_addr(ib_spec->eth.mask.dst_mac)) {
type[0] = MLX4_FS_MC_SNIFFER;
type[1] = MLX4_FS_UC_SNIFFER;
} else {
u8 mac[ETH_ALEN] = {ib_spec->eth.mask.dst_mac[0] ^ 0x01,
ib_spec->eth.mask.dst_mac[1],
ib_spec->eth.mask.dst_mac[2],
ib_spec->eth.mask.dst_mac[3],
ib_spec->eth.mask.dst_mac[4],
ib_spec->eth.mask.dst_mac[5]};
/* Above xor was only on MC bit, non empty mask is valid
* only if this bit is set and rest are zero.
*/
if (!is_zero_ether_addr(&mac[0]))
return -EINVAL;
if (is_multicast_ether_addr(ib_spec->eth.val.dst_mac))
type[0] = MLX4_FS_MC_SNIFFER;
else
type[0] = MLX4_FS_UC_SNIFFER;
}
}
return err;
}
static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp, static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp,
struct ib_flow_attr *flow_attr, struct ib_flow_attr *flow_attr,
int domain) int domain)
...@@ -1653,6 +1703,10 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp, ...@@ -1653,6 +1703,10 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp,
struct mlx4_dev *dev = (to_mdev(qp->device))->dev; struct mlx4_dev *dev = (to_mdev(qp->device))->dev;
int is_bonded = mlx4_is_bonded(dev); int is_bonded = mlx4_is_bonded(dev);
if ((flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) &&
(flow_attr->type != IB_FLOW_ATTR_NORMAL))
return ERR_PTR(-EOPNOTSUPP);
memset(type, 0, sizeof(type)); memset(type, 0, sizeof(type));
mflow = kzalloc(sizeof(*mflow), GFP_KERNEL); mflow = kzalloc(sizeof(*mflow), GFP_KERNEL);
...@@ -1663,7 +1717,19 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp, ...@@ -1663,7 +1717,19 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp,
switch (flow_attr->type) { switch (flow_attr->type) {
case IB_FLOW_ATTR_NORMAL: case IB_FLOW_ATTR_NORMAL:
type[0] = MLX4_FS_REGULAR; /* If dont trap flag (continue match) is set, under specific
* condition traffic be replicated to given qp,
* without stealing it
*/
if (unlikely(flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP)) {
err = mlx4_ib_add_dont_trap_rule(dev,
flow_attr,
type);
if (err)
goto err_free;
} else {
type[0] = MLX4_FS_REGULAR;
}
break; break;
case IB_FLOW_ATTR_ALL_DEFAULT: case IB_FLOW_ATTR_ALL_DEFAULT:
...@@ -1675,8 +1741,8 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp, ...@@ -1675,8 +1741,8 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp,
break; break;
case IB_FLOW_ATTR_SNIFFER: case IB_FLOW_ATTR_SNIFFER:
type[0] = MLX4_FS_UC_SNIFFER; type[0] = MLX4_FS_MIRROR_RX_PORT;
type[1] = MLX4_FS_MC_SNIFFER; type[1] = MLX4_FS_MIRROR_SX_PORT;
break; break;
default: default:
......
...@@ -711,7 +711,8 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, ...@@ -711,7 +711,8 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 virt_addr, int access_flags, u64 virt_addr, int access_flags,
struct ib_udata *udata); struct ib_udata *udata);
int mlx4_ib_dereg_mr(struct ib_mr *mr); int mlx4_ib_dereg_mr(struct ib_mr *mr);
struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type); struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
struct ib_udata *udata);
int mlx4_ib_dealloc_mw(struct ib_mw *mw); int mlx4_ib_dealloc_mw(struct ib_mw *mw);
struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd, struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd,
enum ib_mr_type mr_type, enum ib_mr_type mr_type,
......
...@@ -32,6 +32,7 @@ ...@@ -32,6 +32,7 @@
*/ */
#include <linux/slab.h> #include <linux/slab.h>
#include <rdma/ib_user_verbs.h>
#include "mlx4_ib.h" #include "mlx4_ib.h"
...@@ -334,7 +335,8 @@ int mlx4_ib_dereg_mr(struct ib_mr *ibmr) ...@@ -334,7 +335,8 @@ int mlx4_ib_dereg_mr(struct ib_mr *ibmr)
return 0; return 0;
} }
struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type) struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
struct ib_udata *udata)
{ {
struct mlx4_ib_dev *dev = to_mdev(pd->device); struct mlx4_ib_dev *dev = to_mdev(pd->device);
struct mlx4_ib_mw *mw; struct mlx4_ib_mw *mw;
......
obj-$(CONFIG_MLX5_INFINIBAND) += mlx5_ib.o obj-$(CONFIG_MLX5_INFINIBAND) += mlx5_ib.o
mlx5_ib-y := main.o cq.o doorbell.o qp.o mem.o srq.o mr.o ah.o mad.o mlx5_ib-y := main.o cq.o doorbell.o qp.o mem.o srq.o mr.o ah.o mad.o gsi.o
mlx5_ib-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += odp.o mlx5_ib-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += odp.o
...@@ -207,7 +207,10 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe, ...@@ -207,7 +207,10 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
break; break;
case MLX5_CQE_RESP_SEND: case MLX5_CQE_RESP_SEND:
wc->opcode = IB_WC_RECV; wc->opcode = IB_WC_RECV;
wc->wc_flags = 0; wc->wc_flags = IB_WC_IP_CSUM_OK;
if (unlikely(!((cqe->hds_ip_ext & CQE_L3_OK) &&
(cqe->hds_ip_ext & CQE_L4_OK))))
wc->wc_flags = 0;
break; break;
case MLX5_CQE_RESP_SEND_IMM: case MLX5_CQE_RESP_SEND_IMM:
wc->opcode = IB_WC_RECV; wc->opcode = IB_WC_RECV;
...@@ -431,7 +434,7 @@ static int mlx5_poll_one(struct mlx5_ib_cq *cq, ...@@ -431,7 +434,7 @@ static int mlx5_poll_one(struct mlx5_ib_cq *cq,
struct mlx5_core_qp *mqp; struct mlx5_core_qp *mqp;
struct mlx5_ib_wq *wq; struct mlx5_ib_wq *wq;
struct mlx5_sig_err_cqe *sig_err_cqe; struct mlx5_sig_err_cqe *sig_err_cqe;
struct mlx5_core_mr *mmr; struct mlx5_core_mkey *mmkey;
struct mlx5_ib_mr *mr; struct mlx5_ib_mr *mr;
uint8_t opcode; uint8_t opcode;
uint32_t qpn; uint32_t qpn;
...@@ -536,17 +539,17 @@ static int mlx5_poll_one(struct mlx5_ib_cq *cq, ...@@ -536,17 +539,17 @@ static int mlx5_poll_one(struct mlx5_ib_cq *cq,
case MLX5_CQE_SIG_ERR: case MLX5_CQE_SIG_ERR:
sig_err_cqe = (struct mlx5_sig_err_cqe *)cqe64; sig_err_cqe = (struct mlx5_sig_err_cqe *)cqe64;
read_lock(&dev->mdev->priv.mr_table.lock); read_lock(&dev->mdev->priv.mkey_table.lock);
mmr = __mlx5_mr_lookup(dev->mdev, mmkey = __mlx5_mr_lookup(dev->mdev,
mlx5_base_mkey(be32_to_cpu(sig_err_cqe->mkey))); mlx5_base_mkey(be32_to_cpu(sig_err_cqe->mkey)));
if (unlikely(!mmr)) { if (unlikely(!mmkey)) {
read_unlock(&dev->mdev->priv.mr_table.lock); read_unlock(&dev->mdev->priv.mkey_table.lock);
mlx5_ib_warn(dev, "CQE@CQ %06x for unknown MR %6x\n", mlx5_ib_warn(dev, "CQE@CQ %06x for unknown MR %6x\n",
cq->mcq.cqn, be32_to_cpu(sig_err_cqe->mkey)); cq->mcq.cqn, be32_to_cpu(sig_err_cqe->mkey));
return -EINVAL; return -EINVAL;
} }
mr = to_mibmr(mmr); mr = to_mibmr(mmkey);
get_sig_err_item(sig_err_cqe, &mr->sig->err_item); get_sig_err_item(sig_err_cqe, &mr->sig->err_item);
mr->sig->sig_err_exists = true; mr->sig->sig_err_exists = true;
mr->sig->sigerr_count++; mr->sig->sigerr_count++;
...@@ -558,25 +561,51 @@ static int mlx5_poll_one(struct mlx5_ib_cq *cq, ...@@ -558,25 +561,51 @@ static int mlx5_poll_one(struct mlx5_ib_cq *cq,
mr->sig->err_item.expected, mr->sig->err_item.expected,
mr->sig->err_item.actual); mr->sig->err_item.actual);
read_unlock(&dev->mdev->priv.mr_table.lock); read_unlock(&dev->mdev->priv.mkey_table.lock);
goto repoll; goto repoll;
} }
return 0; return 0;
} }
static int poll_soft_wc(struct mlx5_ib_cq *cq, int num_entries,
struct ib_wc *wc)
{
struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
struct mlx5_ib_wc *soft_wc, *next;
int npolled = 0;
list_for_each_entry_safe(soft_wc, next, &cq->wc_list, list) {
if (npolled >= num_entries)
break;
mlx5_ib_dbg(dev, "polled software generated completion on CQ 0x%x\n",
cq->mcq.cqn);
wc[npolled++] = soft_wc->wc;
list_del(&soft_wc->list);
kfree(soft_wc);
}
return npolled;
}
int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
{ {
struct mlx5_ib_cq *cq = to_mcq(ibcq); struct mlx5_ib_cq *cq = to_mcq(ibcq);
struct mlx5_ib_qp *cur_qp = NULL; struct mlx5_ib_qp *cur_qp = NULL;
unsigned long flags; unsigned long flags;
int soft_polled = 0;
int npolled; int npolled;
int err = 0; int err = 0;
spin_lock_irqsave(&cq->lock, flags); spin_lock_irqsave(&cq->lock, flags);
for (npolled = 0; npolled < num_entries; npolled++) { if (unlikely(!list_empty(&cq->wc_list)))
err = mlx5_poll_one(cq, &cur_qp, wc + npolled); soft_polled = poll_soft_wc(cq, num_entries, wc);
for (npolled = 0; npolled < num_entries - soft_polled; npolled++) {
err = mlx5_poll_one(cq, &cur_qp, wc + soft_polled + npolled);
if (err) if (err)
break; break;
} }
...@@ -587,7 +616,7 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) ...@@ -587,7 +616,7 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
spin_unlock_irqrestore(&cq->lock, flags); spin_unlock_irqrestore(&cq->lock, flags);
if (err == 0 || err == -EAGAIN) if (err == 0 || err == -EAGAIN)
return npolled; return soft_polled + npolled;
else else
return err; return err;
} }
...@@ -595,16 +624,27 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) ...@@ -595,16 +624,27 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
{ {
struct mlx5_core_dev *mdev = to_mdev(ibcq->device)->mdev; struct mlx5_core_dev *mdev = to_mdev(ibcq->device)->mdev;
struct mlx5_ib_cq *cq = to_mcq(ibcq);
void __iomem *uar_page = mdev->priv.uuari.uars[0].map; void __iomem *uar_page = mdev->priv.uuari.uars[0].map;
unsigned long irq_flags;
int ret = 0;
spin_lock_irqsave(&cq->lock, irq_flags);
if (cq->notify_flags != IB_CQ_NEXT_COMP)
cq->notify_flags = flags & IB_CQ_SOLICITED_MASK;
mlx5_cq_arm(&to_mcq(ibcq)->mcq, if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && !list_empty(&cq->wc_list))
ret = 1;
spin_unlock_irqrestore(&cq->lock, irq_flags);
mlx5_cq_arm(&cq->mcq,
(flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT, MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT,
uar_page, uar_page,
MLX5_GET_DOORBELL_LOCK(&mdev->priv.cq_uar_lock), MLX5_GET_DOORBELL_LOCK(&mdev->priv.cq_uar_lock),
to_mcq(ibcq)->mcq.cons_index); to_mcq(ibcq)->mcq.cons_index);
return 0; return ret;
} }
static int alloc_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf, static int alloc_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf,
...@@ -757,6 +797,14 @@ static void destroy_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq) ...@@ -757,6 +797,14 @@ static void destroy_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq)
mlx5_db_free(dev->mdev, &cq->db); mlx5_db_free(dev->mdev, &cq->db);
} }
static void notify_soft_wc_handler(struct work_struct *work)
{
struct mlx5_ib_cq *cq = container_of(work, struct mlx5_ib_cq,
notify_work);
cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
}
struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
const struct ib_cq_init_attr *attr, const struct ib_cq_init_attr *attr,
struct ib_ucontext *context, struct ib_ucontext *context,
...@@ -807,6 +855,8 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, ...@@ -807,6 +855,8 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
&index, &inlen); &index, &inlen);
if (err) if (err)
goto err_create; goto err_create;
INIT_WORK(&cq->notify_work, notify_soft_wc_handler);
} }
cq->cqe_size = cqe_size; cq->cqe_size = cqe_size;
...@@ -832,6 +882,8 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, ...@@ -832,6 +882,8 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
cq->mcq.comp = mlx5_ib_cq_comp; cq->mcq.comp = mlx5_ib_cq_comp;
cq->mcq.event = mlx5_ib_cq_event; cq->mcq.event = mlx5_ib_cq_event;
INIT_LIST_HEAD(&cq->wc_list);
if (context) if (context)
if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) { if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
err = -EFAULT; err = -EFAULT;
...@@ -1219,3 +1271,27 @@ int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq) ...@@ -1219,3 +1271,27 @@ int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq)
cq = to_mcq(ibcq); cq = to_mcq(ibcq);
return cq->cqe_size; return cq->cqe_size;
} }
/* Called from atomic context */
int mlx5_ib_generate_wc(struct ib_cq *ibcq, struct ib_wc *wc)
{
struct mlx5_ib_wc *soft_wc;
struct mlx5_ib_cq *cq = to_mcq(ibcq);
unsigned long flags;
soft_wc = kmalloc(sizeof(*soft_wc), GFP_ATOMIC);
if (!soft_wc)
return -ENOMEM;
soft_wc->wc = *wc;
spin_lock_irqsave(&cq->lock, flags);
list_add_tail(&soft_wc->list, &cq->wc_list);
if (cq->notify_flags == IB_CQ_NEXT_COMP ||
wc->status != IB_WC_SUCCESS) {
cq->notify_flags = 0;
schedule_work(&cq->notify_work);
}
spin_unlock_irqrestore(&cq->lock, flags);
return 0;
}
/*
* Copyright (c) 2016, Mellanox Technologies. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "mlx5_ib.h"
struct mlx5_ib_gsi_wr {
struct ib_cqe cqe;
struct ib_wc wc;
int send_flags;
bool completed:1;
};
struct mlx5_ib_gsi_qp {
struct ib_qp ibqp;
struct ib_qp *rx_qp;
u8 port_num;
struct ib_qp_cap cap;
enum ib_sig_type sq_sig_type;
/* Serialize qp state modifications */
struct mutex mutex;
struct ib_cq *cq;
struct mlx5_ib_gsi_wr *outstanding_wrs;
u32 outstanding_pi, outstanding_ci;
int num_qps;
/* Protects access to the tx_qps. Post send operations synchronize
* with tx_qp creation in setup_qp(). Also protects the
* outstanding_wrs array and indices.
*/
spinlock_t lock;
struct ib_qp **tx_qps;
};
static struct mlx5_ib_gsi_qp *gsi_qp(struct ib_qp *qp)
{
return container_of(qp, struct mlx5_ib_gsi_qp, ibqp);
}
static bool mlx5_ib_deth_sqpn_cap(struct mlx5_ib_dev *dev)
{
return MLX5_CAP_GEN(dev->mdev, set_deth_sqpn);
}
static u32 next_outstanding(struct mlx5_ib_gsi_qp *gsi, u32 index)
{
return ++index % gsi->cap.max_send_wr;
}
#define for_each_outstanding_wr(gsi, index) \
for (index = gsi->outstanding_ci; index != gsi->outstanding_pi; \
index = next_outstanding(gsi, index))
/* Call with gsi->lock locked */
static void generate_completions(struct mlx5_ib_gsi_qp *gsi)
{
struct ib_cq *gsi_cq = gsi->ibqp.send_cq;
struct mlx5_ib_gsi_wr *wr;
u32 index;
for_each_outstanding_wr(gsi, index) {
wr = &gsi->outstanding_wrs[index];
if (!wr->completed)
break;
if (gsi->sq_sig_type == IB_SIGNAL_ALL_WR ||
wr->send_flags & IB_SEND_SIGNALED)
WARN_ON_ONCE(mlx5_ib_generate_wc(gsi_cq, &wr->wc));
wr->completed = false;
}
gsi->outstanding_ci = index;
}
static void handle_single_completion(struct ib_cq *cq, struct ib_wc *wc)
{
struct mlx5_ib_gsi_qp *gsi = cq->cq_context;
struct mlx5_ib_gsi_wr *wr =
container_of(wc->wr_cqe, struct mlx5_ib_gsi_wr, cqe);
u64 wr_id;
unsigned long flags;
spin_lock_irqsave(&gsi->lock, flags);
wr->completed = true;
wr_id = wr->wc.wr_id;
wr->wc = *wc;
wr->wc.wr_id = wr_id;
wr->wc.qp = &gsi->ibqp;
generate_completions(gsi);
spin_unlock_irqrestore(&gsi->lock, flags);
}
struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd,
struct ib_qp_init_attr *init_attr)
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
struct mlx5_ib_gsi_qp *gsi;
struct ib_qp_init_attr hw_init_attr = *init_attr;
const u8 port_num = init_attr->port_num;
const int num_pkeys = pd->device->attrs.max_pkeys;
const int num_qps = mlx5_ib_deth_sqpn_cap(dev) ? num_pkeys : 0;
int ret;
mlx5_ib_dbg(dev, "creating GSI QP\n");
if (port_num > ARRAY_SIZE(dev->devr.ports) || port_num < 1) {
mlx5_ib_warn(dev,
"invalid port number %d during GSI QP creation\n",
port_num);
return ERR_PTR(-EINVAL);
}
gsi = kzalloc(sizeof(*gsi), GFP_KERNEL);
if (!gsi)
return ERR_PTR(-ENOMEM);
gsi->tx_qps = kcalloc(num_qps, sizeof(*gsi->tx_qps), GFP_KERNEL);
if (!gsi->tx_qps) {
ret = -ENOMEM;
goto err_free;
}
gsi->outstanding_wrs = kcalloc(init_attr->cap.max_send_wr,
sizeof(*gsi->outstanding_wrs),
GFP_KERNEL);
if (!gsi->outstanding_wrs) {
ret = -ENOMEM;
goto err_free_tx;
}
mutex_init(&gsi->mutex);
mutex_lock(&dev->devr.mutex);
if (dev->devr.ports[port_num - 1].gsi) {
mlx5_ib_warn(dev, "GSI QP already exists on port %d\n",
port_num);
ret = -EBUSY;
goto err_free_wrs;
}
gsi->num_qps = num_qps;
spin_lock_init(&gsi->lock);
gsi->cap = init_attr->cap;
gsi->sq_sig_type = init_attr->sq_sig_type;
gsi->ibqp.qp_num = 1;
gsi->port_num = port_num;
gsi->cq = ib_alloc_cq(pd->device, gsi, init_attr->cap.max_send_wr, 0,
IB_POLL_SOFTIRQ);
if (IS_ERR(gsi->cq)) {
mlx5_ib_warn(dev, "unable to create send CQ for GSI QP. error %ld\n",
PTR_ERR(gsi->cq));
ret = PTR_ERR(gsi->cq);
goto err_free_wrs;
}
hw_init_attr.qp_type = MLX5_IB_QPT_HW_GSI;
hw_init_attr.send_cq = gsi->cq;
if (num_qps) {
hw_init_attr.cap.max_send_wr = 0;
hw_init_attr.cap.max_send_sge = 0;
hw_init_attr.cap.max_inline_data = 0;
}
gsi->rx_qp = ib_create_qp(pd, &hw_init_attr);
if (IS_ERR(gsi->rx_qp)) {
mlx5_ib_warn(dev, "unable to create hardware GSI QP. error %ld\n",
PTR_ERR(gsi->rx_qp));
ret = PTR_ERR(gsi->rx_qp);
goto err_destroy_cq;
}
dev->devr.ports[init_attr->port_num - 1].gsi = gsi;
mutex_unlock(&dev->devr.mutex);
return &gsi->ibqp;
err_destroy_cq:
ib_free_cq(gsi->cq);
err_free_wrs:
mutex_unlock(&dev->devr.mutex);
kfree(gsi->outstanding_wrs);
err_free_tx:
kfree(gsi->tx_qps);
err_free:
kfree(gsi);
return ERR_PTR(ret);
}
int mlx5_ib_gsi_destroy_qp(struct ib_qp *qp)
{
struct mlx5_ib_dev *dev = to_mdev(qp->device);
struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp);
const int port_num = gsi->port_num;
int qp_index;
int ret;
mlx5_ib_dbg(dev, "destroying GSI QP\n");
mutex_lock(&dev->devr.mutex);
ret = ib_destroy_qp(gsi->rx_qp);
if (ret) {
mlx5_ib_warn(dev, "unable to destroy hardware GSI QP. error %d\n",
ret);
mutex_unlock(&dev->devr.mutex);
return ret;
}
dev->devr.ports[port_num - 1].gsi = NULL;
mutex_unlock(&dev->devr.mutex);
gsi->rx_qp = NULL;
for (qp_index = 0; qp_index < gsi->num_qps; ++qp_index) {
if (!gsi->tx_qps[qp_index])
continue;
WARN_ON_ONCE(ib_destroy_qp(gsi->tx_qps[qp_index]));
gsi->tx_qps[qp_index] = NULL;
}
ib_free_cq(gsi->cq);
kfree(gsi->outstanding_wrs);
kfree(gsi->tx_qps);
kfree(gsi);
return 0;
}
static struct ib_qp *create_gsi_ud_qp(struct mlx5_ib_gsi_qp *gsi)
{
struct ib_pd *pd = gsi->rx_qp->pd;
struct ib_qp_init_attr init_attr = {
.event_handler = gsi->rx_qp->event_handler,
.qp_context = gsi->rx_qp->qp_context,
.send_cq = gsi->cq,
.recv_cq = gsi->rx_qp->recv_cq,
.cap = {
.max_send_wr = gsi->cap.max_send_wr,
.max_send_sge = gsi->cap.max_send_sge,
.max_inline_data = gsi->cap.max_inline_data,
},
.sq_sig_type = gsi->sq_sig_type,
.qp_type = IB_QPT_UD,
.create_flags = mlx5_ib_create_qp_sqpn_qp1(),
};
return ib_create_qp(pd, &init_attr);
}
static int modify_to_rts(struct mlx5_ib_gsi_qp *gsi, struct ib_qp *qp,
u16 qp_index)
{
struct mlx5_ib_dev *dev = to_mdev(qp->device);
struct ib_qp_attr attr;
int mask;
int ret;
mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_QKEY | IB_QP_PORT;
attr.qp_state = IB_QPS_INIT;
attr.pkey_index = qp_index;
attr.qkey = IB_QP1_QKEY;
attr.port_num = gsi->port_num;
ret = ib_modify_qp(qp, &attr, mask);
if (ret) {
mlx5_ib_err(dev, "could not change QP%d state to INIT: %d\n",
qp->qp_num, ret);
return ret;
}
attr.qp_state = IB_QPS_RTR;
ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
if (ret) {
mlx5_ib_err(dev, "could not change QP%d state to RTR: %d\n",
qp->qp_num, ret);
return ret;
}
attr.qp_state = IB_QPS_RTS;
attr.sq_psn = 0;
ret = ib_modify_qp(qp, &attr, IB_QP_STATE | IB_QP_SQ_PSN);
if (ret) {
mlx5_ib_err(dev, "could not change QP%d state to RTS: %d\n",
qp->qp_num, ret);
return ret;
}
return 0;
}
static void setup_qp(struct mlx5_ib_gsi_qp *gsi, u16 qp_index)
{
struct ib_device *device = gsi->rx_qp->device;
struct mlx5_ib_dev *dev = to_mdev(device);
struct ib_qp *qp;
unsigned long flags;
u16 pkey;
int ret;
ret = ib_query_pkey(device, gsi->port_num, qp_index, &pkey);
if (ret) {
mlx5_ib_warn(dev, "unable to read P_Key at port %d, index %d\n",
gsi->port_num, qp_index);
return;
}
if (!pkey) {
mlx5_ib_dbg(dev, "invalid P_Key at port %d, index %d. Skipping.\n",
gsi->port_num, qp_index);
return;
}
spin_lock_irqsave(&gsi->lock, flags);
qp = gsi->tx_qps[qp_index];
spin_unlock_irqrestore(&gsi->lock, flags);
if (qp) {
mlx5_ib_dbg(dev, "already existing GSI TX QP at port %d, index %d. Skipping\n",
gsi->port_num, qp_index);
return;
}
qp = create_gsi_ud_qp(gsi);
if (IS_ERR(qp)) {
mlx5_ib_warn(dev, "unable to create hardware UD QP for GSI: %ld\n",
PTR_ERR(qp));
return;
}
ret = modify_to_rts(gsi, qp, qp_index);
if (ret)
goto err_destroy_qp;
spin_lock_irqsave(&gsi->lock, flags);
WARN_ON_ONCE(gsi->tx_qps[qp_index]);
gsi->tx_qps[qp_index] = qp;
spin_unlock_irqrestore(&gsi->lock, flags);
return;
err_destroy_qp:
WARN_ON_ONCE(qp);
}
static void setup_qps(struct mlx5_ib_gsi_qp *gsi)
{
u16 qp_index;
for (qp_index = 0; qp_index < gsi->num_qps; ++qp_index)
setup_qp(gsi, qp_index);
}
int mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
int attr_mask)
{
struct mlx5_ib_dev *dev = to_mdev(qp->device);
struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp);
int ret;
mlx5_ib_dbg(dev, "modifying GSI QP to state %d\n", attr->qp_state);
mutex_lock(&gsi->mutex);
ret = ib_modify_qp(gsi->rx_qp, attr, attr_mask);
if (ret) {
mlx5_ib_warn(dev, "unable to modify GSI rx QP: %d\n", ret);
goto unlock;
}
if (to_mqp(gsi->rx_qp)->state == IB_QPS_RTS)
setup_qps(gsi);
unlock:
mutex_unlock(&gsi->mutex);
return ret;
}
int mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
int qp_attr_mask,
struct ib_qp_init_attr *qp_init_attr)
{
struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp);
int ret;
mutex_lock(&gsi->mutex);
ret = ib_query_qp(gsi->rx_qp, qp_attr, qp_attr_mask, qp_init_attr);
qp_init_attr->cap = gsi->cap;
mutex_unlock(&gsi->mutex);
return ret;
}
/* Call with gsi->lock locked */
static int mlx5_ib_add_outstanding_wr(struct mlx5_ib_gsi_qp *gsi,
struct ib_ud_wr *wr, struct ib_wc *wc)
{
struct mlx5_ib_dev *dev = to_mdev(gsi->rx_qp->device);
struct mlx5_ib_gsi_wr *gsi_wr;
if (gsi->outstanding_pi == gsi->outstanding_ci + gsi->cap.max_send_wr) {
mlx5_ib_warn(dev, "no available GSI work request.\n");
return -ENOMEM;
}
gsi_wr = &gsi->outstanding_wrs[gsi->outstanding_pi];
gsi->outstanding_pi = next_outstanding(gsi, gsi->outstanding_pi);
if (!wc) {
memset(&gsi_wr->wc, 0, sizeof(gsi_wr->wc));
gsi_wr->wc.pkey_index = wr->pkey_index;
gsi_wr->wc.wr_id = wr->wr.wr_id;
} else {
gsi_wr->wc = *wc;
gsi_wr->completed = true;
}
gsi_wr->cqe.done = &handle_single_completion;
wr->wr.wr_cqe = &gsi_wr->cqe;
return 0;
}
/* Call with gsi->lock locked */
static int mlx5_ib_gsi_silent_drop(struct mlx5_ib_gsi_qp *gsi,
struct ib_ud_wr *wr)
{
struct ib_wc wc = {
{ .wr_id = wr->wr.wr_id },
.status = IB_WC_SUCCESS,
.opcode = IB_WC_SEND,
.qp = &gsi->ibqp,
};
int ret;
ret = mlx5_ib_add_outstanding_wr(gsi, wr, &wc);
if (ret)
return ret;
generate_completions(gsi);
return 0;
}
/* Call with gsi->lock locked */
static struct ib_qp *get_tx_qp(struct mlx5_ib_gsi_qp *gsi, struct ib_ud_wr *wr)
{
struct mlx5_ib_dev *dev = to_mdev(gsi->rx_qp->device);
int qp_index = wr->pkey_index;
if (!mlx5_ib_deth_sqpn_cap(dev))
return gsi->rx_qp;
if (qp_index >= gsi->num_qps)
return NULL;
return gsi->tx_qps[qp_index];
}
int mlx5_ib_gsi_post_send(struct ib_qp *qp, struct ib_send_wr *wr,
struct ib_send_wr **bad_wr)
{
struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp);
struct ib_qp *tx_qp;
unsigned long flags;
int ret;
for (; wr; wr = wr->next) {
struct ib_ud_wr cur_wr = *ud_wr(wr);
cur_wr.wr.next = NULL;
spin_lock_irqsave(&gsi->lock, flags);
tx_qp = get_tx_qp(gsi, &cur_wr);
if (!tx_qp) {
ret = mlx5_ib_gsi_silent_drop(gsi, &cur_wr);
if (ret)
goto err;
spin_unlock_irqrestore(&gsi->lock, flags);
continue;
}
ret = mlx5_ib_add_outstanding_wr(gsi, &cur_wr, NULL);
if (ret)
goto err;
ret = ib_post_send(tx_qp, &cur_wr.wr, bad_wr);
if (ret) {
/* Undo the effect of adding the outstanding wr */
gsi->outstanding_pi = (gsi->outstanding_pi - 1) %
gsi->cap.max_send_wr;
goto err;
}
spin_unlock_irqrestore(&gsi->lock, flags);
}
return 0;
err:
spin_unlock_irqrestore(&gsi->lock, flags);
*bad_wr = wr;
return ret;
}
int mlx5_ib_gsi_post_recv(struct ib_qp *qp, struct ib_recv_wr *wr,
struct ib_recv_wr **bad_wr)
{
struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp);
return ib_post_recv(gsi->rx_qp, wr, bad_wr);
}
void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi)
{
if (!gsi)
return;
mutex_lock(&gsi->mutex);
setup_qps(gsi);
mutex_unlock(&gsi->mutex);
}
...@@ -31,8 +31,10 @@ ...@@ -31,8 +31,10 @@
*/ */
#include <linux/mlx5/cmd.h> #include <linux/mlx5/cmd.h>
#include <linux/mlx5/vport.h>
#include <rdma/ib_mad.h> #include <rdma/ib_mad.h>
#include <rdma/ib_smi.h> #include <rdma/ib_smi.h>
#include <rdma/ib_pma.h>
#include "mlx5_ib.h" #include "mlx5_ib.h"
enum { enum {
...@@ -57,20 +59,12 @@ int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey, int ignore_bkey, ...@@ -57,20 +59,12 @@ int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey, int ignore_bkey,
return mlx5_core_mad_ifc(dev->mdev, in_mad, response_mad, op_modifier, port); return mlx5_core_mad_ifc(dev->mdev, in_mad, response_mad, op_modifier, port);
} }
int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, static int process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_wc *in_wc, const struct ib_grh *in_grh,
const struct ib_mad_hdr *in, size_t in_mad_size, const struct ib_mad *in_mad, struct ib_mad *out_mad)
struct ib_mad_hdr *out, size_t *out_mad_size,
u16 *out_mad_pkey_index)
{ {
u16 slid; u16 slid;
int err; int err;
const struct ib_mad *in_mad = (const struct ib_mad *)in;
struct ib_mad *out_mad = (struct ib_mad *)out;
if (WARN_ON_ONCE(in_mad_size != sizeof(*in_mad) ||
*out_mad_size != sizeof(*out_mad)))
return IB_MAD_RESULT_FAILURE;
slid = in_wc ? in_wc->slid : be16_to_cpu(IB_LID_PERMISSIVE); slid = in_wc ? in_wc->slid : be16_to_cpu(IB_LID_PERMISSIVE);
...@@ -117,6 +111,156 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, ...@@ -117,6 +111,156 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
} }
static void pma_cnt_ext_assign(struct ib_pma_portcounters_ext *pma_cnt_ext,
void *out)
{
#define MLX5_SUM_CNT(p, cntr1, cntr2) \
(MLX5_GET64(query_vport_counter_out, p, cntr1) + \
MLX5_GET64(query_vport_counter_out, p, cntr2))
pma_cnt_ext->port_xmit_data =
cpu_to_be64(MLX5_SUM_CNT(out, transmitted_ib_unicast.octets,
transmitted_ib_multicast.octets) >> 2);
pma_cnt_ext->port_xmit_data =
cpu_to_be64(MLX5_SUM_CNT(out, received_ib_unicast.octets,
received_ib_multicast.octets) >> 2);
pma_cnt_ext->port_xmit_packets =
cpu_to_be64(MLX5_SUM_CNT(out, transmitted_ib_unicast.packets,
transmitted_ib_multicast.packets));
pma_cnt_ext->port_rcv_packets =
cpu_to_be64(MLX5_SUM_CNT(out, received_ib_unicast.packets,
received_ib_multicast.packets));
pma_cnt_ext->port_unicast_xmit_packets =
MLX5_GET64_BE(query_vport_counter_out,
out, transmitted_ib_unicast.packets);
pma_cnt_ext->port_unicast_rcv_packets =
MLX5_GET64_BE(query_vport_counter_out,
out, received_ib_unicast.packets);
pma_cnt_ext->port_multicast_xmit_packets =
MLX5_GET64_BE(query_vport_counter_out,
out, transmitted_ib_multicast.packets);
pma_cnt_ext->port_multicast_rcv_packets =
MLX5_GET64_BE(query_vport_counter_out,
out, received_ib_multicast.packets);
}
static void pma_cnt_assign(struct ib_pma_portcounters *pma_cnt,
void *out)
{
/* Traffic counters will be reported in
* their 64bit form via ib_pma_portcounters_ext by default.
*/
void *out_pma = MLX5_ADDR_OF(ppcnt_reg, out,
counter_set);
#define MLX5_ASSIGN_PMA_CNTR(counter_var, counter_name) { \
counter_var = MLX5_GET_BE(typeof(counter_var), \
ib_port_cntrs_grp_data_layout, \
out_pma, counter_name); \
}
MLX5_ASSIGN_PMA_CNTR(pma_cnt->symbol_error_counter,
symbol_error_counter);
MLX5_ASSIGN_PMA_CNTR(pma_cnt->link_error_recovery_counter,
link_error_recovery_counter);
MLX5_ASSIGN_PMA_CNTR(pma_cnt->link_downed_counter,
link_downed_counter);
MLX5_ASSIGN_PMA_CNTR(pma_cnt->port_rcv_errors,
port_rcv_errors);
MLX5_ASSIGN_PMA_CNTR(pma_cnt->port_rcv_remphys_errors,
port_rcv_remote_physical_errors);
MLX5_ASSIGN_PMA_CNTR(pma_cnt->port_rcv_switch_relay_errors,
port_rcv_switch_relay_errors);
MLX5_ASSIGN_PMA_CNTR(pma_cnt->port_xmit_discards,
port_xmit_discards);
MLX5_ASSIGN_PMA_CNTR(pma_cnt->port_xmit_constraint_errors,
port_xmit_constraint_errors);
MLX5_ASSIGN_PMA_CNTR(pma_cnt->port_rcv_constraint_errors,
port_rcv_constraint_errors);
MLX5_ASSIGN_PMA_CNTR(pma_cnt->link_overrun_errors,
link_overrun_errors);
MLX5_ASSIGN_PMA_CNTR(pma_cnt->vl15_dropped,
vl_15_dropped);
}
static int process_pma_cmd(struct ib_device *ibdev, u8 port_num,
const struct ib_mad *in_mad, struct ib_mad *out_mad)
{
struct mlx5_ib_dev *dev = to_mdev(ibdev);
int err;
void *out_cnt;
/* Decalring support of extended counters */
if (in_mad->mad_hdr.attr_id == IB_PMA_CLASS_PORT_INFO) {
struct ib_class_port_info cpi = {};
cpi.capability_mask = IB_PMA_CLASS_CAP_EXT_WIDTH;
memcpy((out_mad->data + 40), &cpi, sizeof(cpi));
return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
}
if (in_mad->mad_hdr.attr_id == IB_PMA_PORT_COUNTERS_EXT) {
struct ib_pma_portcounters_ext *pma_cnt_ext =
(struct ib_pma_portcounters_ext *)(out_mad->data + 40);
int sz = MLX5_ST_SZ_BYTES(query_vport_counter_out);
out_cnt = mlx5_vzalloc(sz);
if (!out_cnt)
return IB_MAD_RESULT_FAILURE;
err = mlx5_core_query_vport_counter(dev->mdev, 0,
port_num, out_cnt, sz);
if (!err)
pma_cnt_ext_assign(pma_cnt_ext, out_cnt);
} else {
struct ib_pma_portcounters *pma_cnt =
(struct ib_pma_portcounters *)(out_mad->data + 40);
int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
out_cnt = mlx5_vzalloc(sz);
if (!out_cnt)
return IB_MAD_RESULT_FAILURE;
err = mlx5_core_query_ib_ppcnt(dev->mdev, port_num,
out_cnt, sz);
if (!err)
pma_cnt_assign(pma_cnt, out_cnt);
}
kvfree(out_cnt);
if (err)
return IB_MAD_RESULT_FAILURE;
return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
}
int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
const struct ib_wc *in_wc, const struct ib_grh *in_grh,
const struct ib_mad_hdr *in, size_t in_mad_size,
struct ib_mad_hdr *out, size_t *out_mad_size,
u16 *out_mad_pkey_index)
{
struct mlx5_ib_dev *dev = to_mdev(ibdev);
struct mlx5_core_dev *mdev = dev->mdev;
const struct ib_mad *in_mad = (const struct ib_mad *)in;
struct ib_mad *out_mad = (struct ib_mad *)out;
if (WARN_ON_ONCE(in_mad_size != sizeof(*in_mad) ||
*out_mad_size != sizeof(*out_mad)))
return IB_MAD_RESULT_FAILURE;
memset(out_mad->data, 0, sizeof(out_mad->data));
if (MLX5_CAP_GEN(mdev, vport_counters) &&
in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT &&
in_mad->mad_hdr.method == IB_MGMT_METHOD_GET) {
return process_pma_cmd(ibdev, port_num, in_mad, out_mad);
} else {
return process_mad(ibdev, mad_flags, port_num, in_wc, in_grh,
in_mad, out_mad);
}
}
int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port) int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port)
{ {
struct ib_smp *in_mad = NULL; struct ib_smp *in_mad = NULL;
......
...@@ -487,6 +487,13 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, ...@@ -487,6 +487,13 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG; props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
if (MLX5_CAP_GEN(mdev, xrc)) if (MLX5_CAP_GEN(mdev, xrc))
props->device_cap_flags |= IB_DEVICE_XRC; props->device_cap_flags |= IB_DEVICE_XRC;
if (MLX5_CAP_GEN(mdev, imaicl)) {
props->device_cap_flags |= IB_DEVICE_MEM_WINDOW |
IB_DEVICE_MEM_WINDOW_TYPE_2B;
props->max_mw = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
/* We support 'Gappy' memory registration too */
props->device_cap_flags |= IB_DEVICE_SG_GAPS_REG;
}
props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
if (MLX5_CAP_GEN(mdev, sho)) { if (MLX5_CAP_GEN(mdev, sho)) {
props->device_cap_flags |= IB_DEVICE_SIGNATURE_HANDOVER; props->device_cap_flags |= IB_DEVICE_SIGNATURE_HANDOVER;
...@@ -504,6 +511,11 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, ...@@ -504,6 +511,11 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
(MLX5_CAP_ETH(dev->mdev, csum_cap))) (MLX5_CAP_ETH(dev->mdev, csum_cap)))
props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM; props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM;
if (MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
props->device_cap_flags |= IB_DEVICE_UD_TSO;
}
props->vendor_part_id = mdev->pdev->device; props->vendor_part_id = mdev->pdev->device;
props->hw_ver = mdev->pdev->revision; props->hw_ver = mdev->pdev->revision;
...@@ -529,7 +541,8 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, ...@@ -529,7 +541,8 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
props->local_ca_ack_delay = MLX5_CAP_GEN(mdev, local_ca_ack_delay); props->local_ca_ack_delay = MLX5_CAP_GEN(mdev, local_ca_ack_delay);
props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp;
props->max_srq_sge = max_rq_sg - 1; props->max_srq_sge = max_rq_sg - 1;
props->max_fast_reg_page_list_len = (unsigned int)-1; props->max_fast_reg_page_list_len =
1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size);
get_atomic_caps(dev, props); get_atomic_caps(dev, props);
props->masked_atomic_cap = IB_ATOMIC_NONE; props->masked_atomic_cap = IB_ATOMIC_NONE;
props->max_mcast_grp = 1 << MLX5_CAP_GEN(mdev, log_max_mcg); props->max_mcast_grp = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
...@@ -1369,11 +1382,20 @@ static int mlx5_ib_destroy_flow(struct ib_flow *flow_id) ...@@ -1369,11 +1382,20 @@ static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
return 0; return 0;
} }
static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)
{
priority *= 2;
if (!dont_trap)
priority++;
return priority;
}
#define MLX5_FS_MAX_TYPES 10 #define MLX5_FS_MAX_TYPES 10
#define MLX5_FS_MAX_ENTRIES 32000UL #define MLX5_FS_MAX_ENTRIES 32000UL
static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
struct ib_flow_attr *flow_attr) struct ib_flow_attr *flow_attr)
{ {
bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
struct mlx5_flow_namespace *ns = NULL; struct mlx5_flow_namespace *ns = NULL;
struct mlx5_ib_flow_prio *prio; struct mlx5_ib_flow_prio *prio;
struct mlx5_flow_table *ft; struct mlx5_flow_table *ft;
...@@ -1383,10 +1405,12 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, ...@@ -1383,10 +1405,12 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
int err = 0; int err = 0;
if (flow_attr->type == IB_FLOW_ATTR_NORMAL) { if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
if (flow_is_multicast_only(flow_attr)) if (flow_is_multicast_only(flow_attr) &&
!dont_trap)
priority = MLX5_IB_FLOW_MCAST_PRIO; priority = MLX5_IB_FLOW_MCAST_PRIO;
else else
priority = flow_attr->priority; priority = ib_prio_to_core_prio(flow_attr->priority,
dont_trap);
ns = mlx5_get_flow_namespace(dev->mdev, ns = mlx5_get_flow_namespace(dev->mdev,
MLX5_FLOW_NAMESPACE_BYPASS); MLX5_FLOW_NAMESPACE_BYPASS);
num_entries = MLX5_FS_MAX_ENTRIES; num_entries = MLX5_FS_MAX_ENTRIES;
...@@ -1434,6 +1458,7 @@ static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, ...@@ -1434,6 +1458,7 @@ static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
unsigned int spec_index; unsigned int spec_index;
u32 *match_c; u32 *match_c;
u32 *match_v; u32 *match_v;
u32 action;
int err = 0; int err = 0;
if (!is_valid_attr(flow_attr)) if (!is_valid_attr(flow_attr))
...@@ -1459,9 +1484,11 @@ static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, ...@@ -1459,9 +1484,11 @@ static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
/* Outer header support only */ /* Outer header support only */
match_criteria_enable = (!outer_header_zero(match_c)) << 0; match_criteria_enable = (!outer_header_zero(match_c)) << 0;
action = dst ? MLX5_FLOW_CONTEXT_ACTION_FWD_DEST :
MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
handler->rule = mlx5_add_flow_rule(ft, match_criteria_enable, handler->rule = mlx5_add_flow_rule(ft, match_criteria_enable,
match_c, match_v, match_c, match_v,
MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, action,
MLX5_FS_DEFAULT_FLOW_TAG, MLX5_FS_DEFAULT_FLOW_TAG,
dst); dst);
...@@ -1481,6 +1508,29 @@ static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, ...@@ -1481,6 +1508,29 @@ static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
return err ? ERR_PTR(err) : handler; return err ? ERR_PTR(err) : handler;
} }
static struct mlx5_ib_flow_handler *create_dont_trap_rule(struct mlx5_ib_dev *dev,
struct mlx5_ib_flow_prio *ft_prio,
struct ib_flow_attr *flow_attr,
struct mlx5_flow_destination *dst)
{
struct mlx5_ib_flow_handler *handler_dst = NULL;
struct mlx5_ib_flow_handler *handler = NULL;
handler = create_flow_rule(dev, ft_prio, flow_attr, NULL);
if (!IS_ERR(handler)) {
handler_dst = create_flow_rule(dev, ft_prio,
flow_attr, dst);
if (IS_ERR(handler_dst)) {
mlx5_del_flow_rule(handler->rule);
kfree(handler);
handler = handler_dst;
} else {
list_add(&handler_dst->list, &handler->list);
}
}
return handler;
}
enum { enum {
LEFTOVERS_MC, LEFTOVERS_MC,
LEFTOVERS_UC, LEFTOVERS_UC,
...@@ -1558,7 +1608,7 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, ...@@ -1558,7 +1608,7 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
if (domain != IB_FLOW_DOMAIN_USER || if (domain != IB_FLOW_DOMAIN_USER ||
flow_attr->port > MLX5_CAP_GEN(dev->mdev, num_ports) || flow_attr->port > MLX5_CAP_GEN(dev->mdev, num_ports) ||
flow_attr->flags) (flow_attr->flags & ~IB_FLOW_ATTR_FLAGS_DONT_TRAP))
return ERR_PTR(-EINVAL); return ERR_PTR(-EINVAL);
dst = kzalloc(sizeof(*dst), GFP_KERNEL); dst = kzalloc(sizeof(*dst), GFP_KERNEL);
...@@ -1577,8 +1627,13 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, ...@@ -1577,8 +1627,13 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
dst->tir_num = to_mqp(qp)->raw_packet_qp.rq.tirn; dst->tir_num = to_mqp(qp)->raw_packet_qp.rq.tirn;
if (flow_attr->type == IB_FLOW_ATTR_NORMAL) { if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
handler = create_flow_rule(dev, ft_prio, flow_attr, if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) {
dst); handler = create_dont_trap_rule(dev, ft_prio,
flow_attr, dst);
} else {
handler = create_flow_rule(dev, ft_prio, flow_attr,
dst);
}
} else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) { flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
handler = create_leftovers_rule(dev, ft_prio, flow_attr, handler = create_leftovers_rule(dev, ft_prio, flow_attr,
...@@ -1716,6 +1771,17 @@ static struct device_attribute *mlx5_class_attributes[] = { ...@@ -1716,6 +1771,17 @@ static struct device_attribute *mlx5_class_attributes[] = {
&dev_attr_reg_pages, &dev_attr_reg_pages,
}; };
static void pkey_change_handler(struct work_struct *work)
{
struct mlx5_ib_port_resources *ports =
container_of(work, struct mlx5_ib_port_resources,
pkey_change_work);
mutex_lock(&ports->devr->mutex);
mlx5_ib_gsi_pkey_change(ports->gsi);
mutex_unlock(&ports->devr->mutex);
}
static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context, static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
enum mlx5_dev_event event, unsigned long param) enum mlx5_dev_event event, unsigned long param)
{ {
...@@ -1752,6 +1818,8 @@ static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context, ...@@ -1752,6 +1818,8 @@ static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
case MLX5_DEV_EVENT_PKEY_CHANGE: case MLX5_DEV_EVENT_PKEY_CHANGE:
ibev.event = IB_EVENT_PKEY_CHANGE; ibev.event = IB_EVENT_PKEY_CHANGE;
port = (u8)param; port = (u8)param;
schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
break; break;
case MLX5_DEV_EVENT_GUID_CHANGE: case MLX5_DEV_EVENT_GUID_CHANGE:
...@@ -1838,7 +1906,7 @@ static void destroy_umrc_res(struct mlx5_ib_dev *dev) ...@@ -1838,7 +1906,7 @@ static void destroy_umrc_res(struct mlx5_ib_dev *dev)
mlx5_ib_warn(dev, "mr cache cleanup failed\n"); mlx5_ib_warn(dev, "mr cache cleanup failed\n");
mlx5_ib_destroy_qp(dev->umrc.qp); mlx5_ib_destroy_qp(dev->umrc.qp);
ib_destroy_cq(dev->umrc.cq); ib_free_cq(dev->umrc.cq);
ib_dealloc_pd(dev->umrc.pd); ib_dealloc_pd(dev->umrc.pd);
} }
...@@ -1853,7 +1921,6 @@ static int create_umr_res(struct mlx5_ib_dev *dev) ...@@ -1853,7 +1921,6 @@ static int create_umr_res(struct mlx5_ib_dev *dev)
struct ib_pd *pd; struct ib_pd *pd;
struct ib_cq *cq; struct ib_cq *cq;
struct ib_qp *qp; struct ib_qp *qp;
struct ib_cq_init_attr cq_attr = {};
int ret; int ret;
attr = kzalloc(sizeof(*attr), GFP_KERNEL); attr = kzalloc(sizeof(*attr), GFP_KERNEL);
...@@ -1870,15 +1937,12 @@ static int create_umr_res(struct mlx5_ib_dev *dev) ...@@ -1870,15 +1937,12 @@ static int create_umr_res(struct mlx5_ib_dev *dev)
goto error_0; goto error_0;
} }
cq_attr.cqe = 128; cq = ib_alloc_cq(&dev->ib_dev, NULL, 128, 0, IB_POLL_SOFTIRQ);
cq = ib_create_cq(&dev->ib_dev, mlx5_umr_cq_handler, NULL, NULL,
&cq_attr);
if (IS_ERR(cq)) { if (IS_ERR(cq)) {
mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n"); mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n");
ret = PTR_ERR(cq); ret = PTR_ERR(cq);
goto error_2; goto error_2;
} }
ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
init_attr->send_cq = cq; init_attr->send_cq = cq;
init_attr->recv_cq = cq; init_attr->recv_cq = cq;
...@@ -1945,7 +2009,7 @@ static int create_umr_res(struct mlx5_ib_dev *dev) ...@@ -1945,7 +2009,7 @@ static int create_umr_res(struct mlx5_ib_dev *dev)
mlx5_ib_destroy_qp(qp); mlx5_ib_destroy_qp(qp);
error_3: error_3:
ib_destroy_cq(cq); ib_free_cq(cq);
error_2: error_2:
ib_dealloc_pd(pd); ib_dealloc_pd(pd);
...@@ -1961,10 +2025,13 @@ static int create_dev_resources(struct mlx5_ib_resources *devr) ...@@ -1961,10 +2025,13 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
struct ib_srq_init_attr attr; struct ib_srq_init_attr attr;
struct mlx5_ib_dev *dev; struct mlx5_ib_dev *dev;
struct ib_cq_init_attr cq_attr = {.cqe = 1}; struct ib_cq_init_attr cq_attr = {.cqe = 1};
int port;
int ret = 0; int ret = 0;
dev = container_of(devr, struct mlx5_ib_dev, devr); dev = container_of(devr, struct mlx5_ib_dev, devr);
mutex_init(&devr->mutex);
devr->p0 = mlx5_ib_alloc_pd(&dev->ib_dev, NULL, NULL); devr->p0 = mlx5_ib_alloc_pd(&dev->ib_dev, NULL, NULL);
if (IS_ERR(devr->p0)) { if (IS_ERR(devr->p0)) {
ret = PTR_ERR(devr->p0); ret = PTR_ERR(devr->p0);
...@@ -2052,6 +2119,12 @@ static int create_dev_resources(struct mlx5_ib_resources *devr) ...@@ -2052,6 +2119,12 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
atomic_inc(&devr->p0->usecnt); atomic_inc(&devr->p0->usecnt);
atomic_set(&devr->s0->usecnt, 0); atomic_set(&devr->s0->usecnt, 0);
for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) {
INIT_WORK(&devr->ports[port].pkey_change_work,
pkey_change_handler);
devr->ports[port].devr = devr;
}
return 0; return 0;
error5: error5:
...@@ -2070,12 +2143,20 @@ static int create_dev_resources(struct mlx5_ib_resources *devr) ...@@ -2070,12 +2143,20 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
static void destroy_dev_resources(struct mlx5_ib_resources *devr) static void destroy_dev_resources(struct mlx5_ib_resources *devr)
{ {
struct mlx5_ib_dev *dev =
container_of(devr, struct mlx5_ib_dev, devr);
int port;
mlx5_ib_destroy_srq(devr->s1); mlx5_ib_destroy_srq(devr->s1);
mlx5_ib_destroy_srq(devr->s0); mlx5_ib_destroy_srq(devr->s0);
mlx5_ib_dealloc_xrcd(devr->x0); mlx5_ib_dealloc_xrcd(devr->x0);
mlx5_ib_dealloc_xrcd(devr->x1); mlx5_ib_dealloc_xrcd(devr->x1);
mlx5_ib_destroy_cq(devr->c0); mlx5_ib_destroy_cq(devr->c0);
mlx5_ib_dealloc_pd(devr->p0); mlx5_ib_dealloc_pd(devr->p0);
/* Make sure no change P_Key work items are still executing */
for (port = 0; port < dev->num_ports; ++port)
cancel_work_sync(&devr->ports[port].pkey_change_work);
} }
static u32 get_core_cap_flags(struct ib_device *ibdev) static u32 get_core_cap_flags(struct ib_device *ibdev)
...@@ -2198,6 +2279,7 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev) ...@@ -2198,6 +2279,7 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
(1ull << IB_USER_VERBS_CMD_ALLOC_PD) | (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
(1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
(1ull << IB_USER_VERBS_CMD_REG_MR) | (1ull << IB_USER_VERBS_CMD_REG_MR) |
(1ull << IB_USER_VERBS_CMD_REREG_MR) |
(1ull << IB_USER_VERBS_CMD_DEREG_MR) | (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
(1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
(1ull << IB_USER_VERBS_CMD_CREATE_CQ) | (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
...@@ -2258,6 +2340,7 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev) ...@@ -2258,6 +2340,7 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
dev->ib_dev.req_notify_cq = mlx5_ib_arm_cq; dev->ib_dev.req_notify_cq = mlx5_ib_arm_cq;
dev->ib_dev.get_dma_mr = mlx5_ib_get_dma_mr; dev->ib_dev.get_dma_mr = mlx5_ib_get_dma_mr;
dev->ib_dev.reg_user_mr = mlx5_ib_reg_user_mr; dev->ib_dev.reg_user_mr = mlx5_ib_reg_user_mr;
dev->ib_dev.rereg_user_mr = mlx5_ib_rereg_user_mr;
dev->ib_dev.dereg_mr = mlx5_ib_dereg_mr; dev->ib_dev.dereg_mr = mlx5_ib_dereg_mr;
dev->ib_dev.attach_mcast = mlx5_ib_mcg_attach; dev->ib_dev.attach_mcast = mlx5_ib_mcg_attach;
dev->ib_dev.detach_mcast = mlx5_ib_mcg_detach; dev->ib_dev.detach_mcast = mlx5_ib_mcg_detach;
...@@ -2269,6 +2352,14 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev) ...@@ -2269,6 +2352,14 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
mlx5_ib_internal_fill_odp_caps(dev); mlx5_ib_internal_fill_odp_caps(dev);
if (MLX5_CAP_GEN(mdev, imaicl)) {
dev->ib_dev.alloc_mw = mlx5_ib_alloc_mw;
dev->ib_dev.dealloc_mw = mlx5_ib_dealloc_mw;
dev->ib_dev.uverbs_cmd_mask |=
(1ull << IB_USER_VERBS_CMD_ALLOC_MW) |
(1ull << IB_USER_VERBS_CMD_DEALLOC_MW);
}
if (MLX5_CAP_GEN(mdev, xrc)) { if (MLX5_CAP_GEN(mdev, xrc)) {
dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd; dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd;
dev->ib_dev.dealloc_xrcd = mlx5_ib_dealloc_xrcd; dev->ib_dev.dealloc_xrcd = mlx5_ib_dealloc_xrcd;
......
...@@ -43,6 +43,7 @@ ...@@ -43,6 +43,7 @@
#include <linux/mlx5/srq.h> #include <linux/mlx5/srq.h>
#include <linux/types.h> #include <linux/types.h>
#include <linux/mlx5/transobj.h> #include <linux/mlx5/transobj.h>
#include <rdma/ib_user_verbs.h>
#define mlx5_ib_dbg(dev, format, arg...) \ #define mlx5_ib_dbg(dev, format, arg...) \
pr_debug("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \ pr_debug("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
...@@ -126,7 +127,7 @@ struct mlx5_ib_pd { ...@@ -126,7 +127,7 @@ struct mlx5_ib_pd {
}; };
#define MLX5_IB_FLOW_MCAST_PRIO (MLX5_BY_PASS_NUM_PRIOS - 1) #define MLX5_IB_FLOW_MCAST_PRIO (MLX5_BY_PASS_NUM_PRIOS - 1)
#define MLX5_IB_FLOW_LAST_PRIO (MLX5_IB_FLOW_MCAST_PRIO - 1) #define MLX5_IB_FLOW_LAST_PRIO (MLX5_BY_PASS_NUM_REGULAR_PRIOS - 1)
#if (MLX5_IB_FLOW_LAST_PRIO <= 0) #if (MLX5_IB_FLOW_LAST_PRIO <= 0)
#error "Invalid number of bypass priorities" #error "Invalid number of bypass priorities"
#endif #endif
...@@ -162,9 +163,31 @@ struct mlx5_ib_flow_db { ...@@ -162,9 +163,31 @@ struct mlx5_ib_flow_db {
#define MLX5_IB_SEND_UMR_UNREG IB_SEND_RESERVED_START #define MLX5_IB_SEND_UMR_UNREG IB_SEND_RESERVED_START
#define MLX5_IB_SEND_UMR_FAIL_IF_FREE (IB_SEND_RESERVED_START << 1) #define MLX5_IB_SEND_UMR_FAIL_IF_FREE (IB_SEND_RESERVED_START << 1)
#define MLX5_IB_SEND_UMR_UPDATE_MTT (IB_SEND_RESERVED_START << 2) #define MLX5_IB_SEND_UMR_UPDATE_MTT (IB_SEND_RESERVED_START << 2)
#define MLX5_IB_SEND_UMR_UPDATE_TRANSLATION (IB_SEND_RESERVED_START << 3)
#define MLX5_IB_SEND_UMR_UPDATE_PD (IB_SEND_RESERVED_START << 4)
#define MLX5_IB_SEND_UMR_UPDATE_ACCESS IB_SEND_RESERVED_END
#define MLX5_IB_QPT_REG_UMR IB_QPT_RESERVED1 #define MLX5_IB_QPT_REG_UMR IB_QPT_RESERVED1
/*
* IB_QPT_GSI creates the software wrapper around GSI, and MLX5_IB_QPT_HW_GSI
* creates the actual hardware QP.
*/
#define MLX5_IB_QPT_HW_GSI IB_QPT_RESERVED2
#define MLX5_IB_WR_UMR IB_WR_RESERVED1 #define MLX5_IB_WR_UMR IB_WR_RESERVED1
/* Private QP creation flags to be passed in ib_qp_init_attr.create_flags.
*
* These flags are intended for internal use by the mlx5_ib driver, and they
* rely on the range reserved for that use in the ib_qp_create_flags enum.
*/
/* Create a UD QP whose source QP number is 1 */
static inline enum ib_qp_create_flags mlx5_ib_create_qp_sqpn_qp1(void)
{
return IB_QP_CREATE_RESERVED_START;
}
struct wr_list { struct wr_list {
u16 opcode; u16 opcode;
u16 next; u16 next;
...@@ -325,11 +348,14 @@ struct mlx5_ib_cq_buf { ...@@ -325,11 +348,14 @@ struct mlx5_ib_cq_buf {
}; };
enum mlx5_ib_qp_flags { enum mlx5_ib_qp_flags {
MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK = 1 << 0, MLX5_IB_QP_LSO = IB_QP_CREATE_IPOIB_UD_LSO,
MLX5_IB_QP_SIGNATURE_HANDLING = 1 << 1, MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK = IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
MLX5_IB_QP_CROSS_CHANNEL = 1 << 2, MLX5_IB_QP_CROSS_CHANNEL = IB_QP_CREATE_CROSS_CHANNEL,
MLX5_IB_QP_MANAGED_SEND = 1 << 3, MLX5_IB_QP_MANAGED_SEND = IB_QP_CREATE_MANAGED_SEND,
MLX5_IB_QP_MANAGED_RECV = 1 << 4, MLX5_IB_QP_MANAGED_RECV = IB_QP_CREATE_MANAGED_RECV,
MLX5_IB_QP_SIGNATURE_HANDLING = 1 << 5,
/* QP uses 1 as its source QP number */
MLX5_IB_QP_SQPN_QP1 = 1 << 6,
}; };
struct mlx5_umr_wr { struct mlx5_umr_wr {
...@@ -373,6 +399,14 @@ struct mlx5_ib_cq { ...@@ -373,6 +399,14 @@ struct mlx5_ib_cq {
struct ib_umem *resize_umem; struct ib_umem *resize_umem;
int cqe_size; int cqe_size;
u32 create_flags; u32 create_flags;
struct list_head wc_list;
enum ib_cq_notify_flags notify_flags;
struct work_struct notify_work;
};
struct mlx5_ib_wc {
struct ib_wc wc;
struct list_head list;
}; };
struct mlx5_ib_srq { struct mlx5_ib_srq {
...@@ -413,7 +447,8 @@ struct mlx5_ib_mr { ...@@ -413,7 +447,8 @@ struct mlx5_ib_mr {
int ndescs; int ndescs;
int max_descs; int max_descs;
int desc_size; int desc_size;
struct mlx5_core_mr mmr; int access_mode;
struct mlx5_core_mkey mmkey;
struct ib_umem *umem; struct ib_umem *umem;
struct mlx5_shared_mr_info *smr_info; struct mlx5_shared_mr_info *smr_info;
struct list_head list; struct list_head list;
...@@ -425,19 +460,20 @@ struct mlx5_ib_mr { ...@@ -425,19 +460,20 @@ struct mlx5_ib_mr {
struct mlx5_core_sig_ctx *sig; struct mlx5_core_sig_ctx *sig;
int live; int live;
void *descs_alloc; void *descs_alloc;
int access_flags; /* Needed for rereg MR */
};
struct mlx5_ib_mw {
struct ib_mw ibmw;
struct mlx5_core_mkey mmkey;
}; };
struct mlx5_ib_umr_context { struct mlx5_ib_umr_context {
struct ib_cqe cqe;
enum ib_wc_status status; enum ib_wc_status status;
struct completion done; struct completion done;
}; };
static inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context)
{
context->status = -1;
init_completion(&context->done);
}
struct umr_common { struct umr_common {
struct ib_pd *pd; struct ib_pd *pd;
struct ib_cq *cq; struct ib_cq *cq;
...@@ -487,6 +523,14 @@ struct mlx5_mr_cache { ...@@ -487,6 +523,14 @@ struct mlx5_mr_cache {
unsigned long last_add; unsigned long last_add;
}; };
struct mlx5_ib_gsi_qp;
struct mlx5_ib_port_resources {
struct mlx5_ib_resources *devr;
struct mlx5_ib_gsi_qp *gsi;
struct work_struct pkey_change_work;
};
struct mlx5_ib_resources { struct mlx5_ib_resources {
struct ib_cq *c0; struct ib_cq *c0;
struct ib_xrcd *x0; struct ib_xrcd *x0;
...@@ -494,6 +538,9 @@ struct mlx5_ib_resources { ...@@ -494,6 +538,9 @@ struct mlx5_ib_resources {
struct ib_pd *p0; struct ib_pd *p0;
struct ib_srq *s0; struct ib_srq *s0;
struct ib_srq *s1; struct ib_srq *s1;
struct mlx5_ib_port_resources ports[2];
/* Protects changes to the port resources */
struct mutex mutex;
}; };
struct mlx5_roce { struct mlx5_roce {
...@@ -558,9 +605,9 @@ static inline struct mlx5_ib_qp *to_mibqp(struct mlx5_core_qp *mqp) ...@@ -558,9 +605,9 @@ static inline struct mlx5_ib_qp *to_mibqp(struct mlx5_core_qp *mqp)
return container_of(mqp, struct mlx5_ib_qp_base, mqp)->container_mibqp; return container_of(mqp, struct mlx5_ib_qp_base, mqp)->container_mibqp;
} }
static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mr *mmr) static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mkey *mmkey)
{ {
return container_of(mmr, struct mlx5_ib_mr, mmr); return container_of(mmkey, struct mlx5_ib_mr, mmkey);
} }
static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd) static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd)
...@@ -588,6 +635,11 @@ static inline struct mlx5_ib_mr *to_mmr(struct ib_mr *ibmr) ...@@ -588,6 +635,11 @@ static inline struct mlx5_ib_mr *to_mmr(struct ib_mr *ibmr)
return container_of(ibmr, struct mlx5_ib_mr, ibmr); return container_of(ibmr, struct mlx5_ib_mr, ibmr);
} }
static inline struct mlx5_ib_mw *to_mmw(struct ib_mw *ibmw)
{
return container_of(ibmw, struct mlx5_ib_mw, ibmw);
}
struct mlx5_ib_ah { struct mlx5_ib_ah {
struct ib_ah ibah; struct ib_ah ibah;
struct mlx5_av av; struct mlx5_av av;
...@@ -648,8 +700,14 @@ struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc); ...@@ -648,8 +700,14 @@ struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc);
struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 virt_addr, int access_flags, u64 virt_addr, int access_flags,
struct ib_udata *udata); struct ib_udata *udata);
struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
struct ib_udata *udata);
int mlx5_ib_dealloc_mw(struct ib_mw *mw);
int mlx5_ib_update_mtt(struct mlx5_ib_mr *mr, u64 start_page_index, int mlx5_ib_update_mtt(struct mlx5_ib_mr *mr, u64 start_page_index,
int npages, int zap); int npages, int zap);
int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
u64 length, u64 virt_addr, int access_flags,
struct ib_pd *pd, struct ib_udata *udata);
int mlx5_ib_dereg_mr(struct ib_mr *ibmr); int mlx5_ib_dereg_mr(struct ib_mr *ibmr);
struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
enum ib_mr_type mr_type, enum ib_mr_type mr_type,
...@@ -700,7 +758,6 @@ int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq); ...@@ -700,7 +758,6 @@ int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq);
int mlx5_mr_cache_init(struct mlx5_ib_dev *dev); int mlx5_mr_cache_init(struct mlx5_ib_dev *dev);
int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev); int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev);
int mlx5_mr_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift); int mlx5_mr_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift);
void mlx5_umr_cq_handler(struct ib_cq *cq, void *cq_context);
int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask, int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
struct ib_mr_status *mr_status); struct ib_mr_status *mr_status);
...@@ -739,6 +796,23 @@ static inline void mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp *qp) {} ...@@ -739,6 +796,23 @@ static inline void mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp *qp) {}
__be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev, u8 port_num, __be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev, u8 port_num,
int index); int index);
/* GSI QP helper functions */
struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd,
struct ib_qp_init_attr *init_attr);
int mlx5_ib_gsi_destroy_qp(struct ib_qp *qp);
int mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
int attr_mask);
int mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
int qp_attr_mask,
struct ib_qp_init_attr *qp_init_attr);
int mlx5_ib_gsi_post_send(struct ib_qp *qp, struct ib_send_wr *wr,
struct ib_send_wr **bad_wr);
int mlx5_ib_gsi_post_recv(struct ib_qp *qp, struct ib_recv_wr *wr,
struct ib_recv_wr **bad_wr);
void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi);
int mlx5_ib_generate_wc(struct ib_cq *ibcq, struct ib_wc *wc);
static inline void init_query_mad(struct ib_smp *mad) static inline void init_query_mad(struct ib_smp *mad)
{ {
mad->base_version = 1; mad->base_version = 1;
...@@ -758,7 +832,7 @@ static inline u8 convert_access(int acc) ...@@ -758,7 +832,7 @@ static inline u8 convert_access(int acc)
static inline int is_qp1(enum ib_qp_type qp_type) static inline int is_qp1(enum ib_qp_type qp_type)
{ {
return qp_type == IB_QPT_GSI; return qp_type == MLX5_IB_QPT_HW_GSI;
} }
#define MLX5_MAX_UMR_SHIFT 16 #define MLX5_MAX_UMR_SHIFT 16
......
此差异已折叠。
...@@ -142,13 +142,13 @@ static struct mlx5_ib_mr *mlx5_ib_odp_find_mr_lkey(struct mlx5_ib_dev *dev, ...@@ -142,13 +142,13 @@ static struct mlx5_ib_mr *mlx5_ib_odp_find_mr_lkey(struct mlx5_ib_dev *dev,
u32 key) u32 key)
{ {
u32 base_key = mlx5_base_mkey(key); u32 base_key = mlx5_base_mkey(key);
struct mlx5_core_mr *mmr = __mlx5_mr_lookup(dev->mdev, base_key); struct mlx5_core_mkey *mmkey = __mlx5_mr_lookup(dev->mdev, base_key);
struct mlx5_ib_mr *mr = container_of(mmr, struct mlx5_ib_mr, mmr); struct mlx5_ib_mr *mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);
if (!mmr || mmr->key != key || !mr->live) if (!mmkey || mmkey->key != key || !mr->live)
return NULL; return NULL;
return container_of(mmr, struct mlx5_ib_mr, mmr); return container_of(mmkey, struct mlx5_ib_mr, mmkey);
} }
static void mlx5_ib_page_fault_resume(struct mlx5_ib_qp *qp, static void mlx5_ib_page_fault_resume(struct mlx5_ib_qp *qp,
...@@ -232,7 +232,7 @@ static int pagefault_single_data_segment(struct mlx5_ib_qp *qp, ...@@ -232,7 +232,7 @@ static int pagefault_single_data_segment(struct mlx5_ib_qp *qp,
io_virt += pfault->mpfault.bytes_committed; io_virt += pfault->mpfault.bytes_committed;
bcnt -= pfault->mpfault.bytes_committed; bcnt -= pfault->mpfault.bytes_committed;
start_idx = (io_virt - (mr->mmr.iova & PAGE_MASK)) >> PAGE_SHIFT; start_idx = (io_virt - (mr->mmkey.iova & PAGE_MASK)) >> PAGE_SHIFT;
if (mr->umem->writable) if (mr->umem->writable)
access_mask |= ODP_WRITE_ALLOWED_BIT; access_mask |= ODP_WRITE_ALLOWED_BIT;
......
此差异已折叠。
...@@ -152,6 +152,13 @@ struct mlx5_ib_create_qp_resp { ...@@ -152,6 +152,13 @@ struct mlx5_ib_create_qp_resp {
__u32 uuar_index; __u32 uuar_index;
}; };
struct mlx5_ib_alloc_mw {
__u32 comp_mask;
__u8 num_klms;
__u8 reserved1;
__u16 reserved2;
};
static inline int get_qp_user_index(struct mlx5_ib_ucontext *ucontext, static inline int get_qp_user_index(struct mlx5_ib_ucontext *ucontext,
struct mlx5_ib_create_qp *ucmd, struct mlx5_ib_create_qp *ucmd,
int inlen, int inlen,
......
...@@ -2,7 +2,6 @@ config INFINIBAND_NES ...@@ -2,7 +2,6 @@ config INFINIBAND_NES
tristate "NetEffect RNIC Driver" tristate "NetEffect RNIC Driver"
depends on PCI && INET && INFINIBAND depends on PCI && INET && INFINIBAND
select LIBCRC32C select LIBCRC32C
select INET_LRO
---help--- ---help---
This is the RDMA Network Interface Card (RNIC) driver for This is the RDMA Network Interface Card (RNIC) driver for
NetEffect Ethernet Cluster Server Adapters. NetEffect Ethernet Cluster Server Adapters.
......
...@@ -111,17 +111,6 @@ static struct pci_device_id nes_pci_table[] = { ...@@ -111,17 +111,6 @@ static struct pci_device_id nes_pci_table[] = {
MODULE_DEVICE_TABLE(pci, nes_pci_table); MODULE_DEVICE_TABLE(pci, nes_pci_table);
/* registered nes netlink callbacks */
static struct ibnl_client_cbs nes_nl_cb_table[] = {
[RDMA_NL_IWPM_REG_PID] = {.dump = iwpm_register_pid_cb},
[RDMA_NL_IWPM_ADD_MAPPING] = {.dump = iwpm_add_mapping_cb},
[RDMA_NL_IWPM_QUERY_MAPPING] = {.dump = iwpm_add_and_query_mapping_cb},
[RDMA_NL_IWPM_REMOTE_INFO] = {.dump = iwpm_remote_info_cb},
[RDMA_NL_IWPM_HANDLE_ERR] = {.dump = iwpm_mapping_error_cb},
[RDMA_NL_IWPM_MAPINFO] = {.dump = iwpm_mapping_info_cb},
[RDMA_NL_IWPM_MAPINFO_NUM] = {.dump = iwpm_ack_mapping_info_cb}
};
static int nes_inetaddr_event(struct notifier_block *, unsigned long, void *); static int nes_inetaddr_event(struct notifier_block *, unsigned long, void *);
static int nes_net_event(struct notifier_block *, unsigned long, void *); static int nes_net_event(struct notifier_block *, unsigned long, void *);
static int nes_notifiers_registered; static int nes_notifiers_registered;
...@@ -682,17 +671,6 @@ static int nes_probe(struct pci_dev *pcidev, const struct pci_device_id *ent) ...@@ -682,17 +671,6 @@ static int nes_probe(struct pci_dev *pcidev, const struct pci_device_id *ent)
} }
nes_notifiers_registered++; nes_notifiers_registered++;
if (ibnl_add_client(RDMA_NL_NES, RDMA_NL_IWPM_NUM_OPS, nes_nl_cb_table))
printk(KERN_ERR PFX "%s[%u]: Failed to add netlink callback\n",
__func__, __LINE__);
ret = iwpm_init(RDMA_NL_NES);
if (ret) {
printk(KERN_ERR PFX "%s: port mapper initialization failed\n",
pci_name(pcidev));
goto bail7;
}
INIT_DELAYED_WORK(&nesdev->work, nes_recheck_link_status); INIT_DELAYED_WORK(&nesdev->work, nes_recheck_link_status);
/* Initialize network devices */ /* Initialize network devices */
...@@ -731,7 +709,6 @@ static int nes_probe(struct pci_dev *pcidev, const struct pci_device_id *ent) ...@@ -731,7 +709,6 @@ static int nes_probe(struct pci_dev *pcidev, const struct pci_device_id *ent)
nes_debug(NES_DBG_INIT, "netdev_count=%d, nesadapter->netdev_count=%d\n", nes_debug(NES_DBG_INIT, "netdev_count=%d, nesadapter->netdev_count=%d\n",
nesdev->netdev_count, nesdev->nesadapter->netdev_count); nesdev->netdev_count, nesdev->nesadapter->netdev_count);
ibnl_remove_client(RDMA_NL_NES);
nes_notifiers_registered--; nes_notifiers_registered--;
if (nes_notifiers_registered == 0) { if (nes_notifiers_registered == 0) {
...@@ -795,8 +772,6 @@ static void nes_remove(struct pci_dev *pcidev) ...@@ -795,8 +772,6 @@ static void nes_remove(struct pci_dev *pcidev)
nesdev->nesadapter->netdev_count--; nesdev->nesadapter->netdev_count--;
} }
} }
ibnl_remove_client(RDMA_NL_NES);
iwpm_exit(RDMA_NL_NES);
nes_notifiers_registered--; nes_notifiers_registered--;
if (nes_notifiers_registered == 0) { if (nes_notifiers_registered == 0) {
......
此差异已折叠。
...@@ -293,8 +293,8 @@ struct nes_cm_listener { ...@@ -293,8 +293,8 @@ struct nes_cm_listener {
struct list_head list; struct list_head list;
struct nes_cm_core *cm_core; struct nes_cm_core *cm_core;
u8 loc_mac[ETH_ALEN]; u8 loc_mac[ETH_ALEN];
nes_addr_t loc_addr, mapped_loc_addr; nes_addr_t loc_addr;
u16 loc_port, mapped_loc_port; u16 loc_port;
struct iw_cm_id *cm_id; struct iw_cm_id *cm_id;
enum nes_cm_conn_type conn_type; enum nes_cm_conn_type conn_type;
atomic_t ref_count; atomic_t ref_count;
...@@ -309,9 +309,7 @@ struct nes_cm_listener { ...@@ -309,9 +309,7 @@ struct nes_cm_listener {
/* per connection node and node state information */ /* per connection node and node state information */
struct nes_cm_node { struct nes_cm_node {
nes_addr_t loc_addr, rem_addr; nes_addr_t loc_addr, rem_addr;
nes_addr_t mapped_loc_addr, mapped_rem_addr;
u16 loc_port, rem_port; u16 loc_port, rem_port;
u16 mapped_loc_port, mapped_rem_port;
u8 loc_mac[ETH_ALEN]; u8 loc_mac[ETH_ALEN];
u8 rem_mac[ETH_ALEN]; u8 rem_mac[ETH_ALEN];
...@@ -368,11 +366,6 @@ struct nes_cm_info { ...@@ -368,11 +366,6 @@ struct nes_cm_info {
u16 rem_port; u16 rem_port;
nes_addr_t loc_addr; nes_addr_t loc_addr;
nes_addr_t rem_addr; nes_addr_t rem_addr;
u16 mapped_loc_port;
u16 mapped_rem_port;
nes_addr_t mapped_loc_addr;
nes_addr_t mapped_rem_addr;
enum nes_cm_conn_type conn_type; enum nes_cm_conn_type conn_type;
int backlog; int backlog;
}; };
......
...@@ -35,18 +35,11 @@ ...@@ -35,18 +35,11 @@
#include <linux/moduleparam.h> #include <linux/moduleparam.h>
#include <linux/netdevice.h> #include <linux/netdevice.h>
#include <linux/etherdevice.h> #include <linux/etherdevice.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/if_vlan.h> #include <linux/if_vlan.h>
#include <linux/inet_lro.h>
#include <linux/slab.h> #include <linux/slab.h>
#include "nes.h" #include "nes.h"
static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
module_param(nes_lro_max_aggr, uint, 0444);
MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
static int wide_ppm_offset; static int wide_ppm_offset;
module_param(wide_ppm_offset, int, 0644); module_param(wide_ppm_offset, int, 0644);
MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm"); MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
...@@ -1642,25 +1635,6 @@ static void nes_rq_wqes_timeout(unsigned long parm) ...@@ -1642,25 +1635,6 @@ static void nes_rq_wqes_timeout(unsigned long parm)
} }
static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
void **tcph, u64 *hdr_flags, void *priv)
{
unsigned int ip_len;
struct iphdr *iph;
skb_reset_network_header(skb);
iph = ip_hdr(skb);
if (iph->protocol != IPPROTO_TCP)
return -1;
ip_len = ip_hdrlen(skb);
skb_set_transport_header(skb, ip_len);
*tcph = tcp_hdr(skb);
*hdr_flags = LRO_IPV4 | LRO_TCP;
*iphdr = iph;
return 0;
}
/** /**
* nes_init_nic_qp * nes_init_nic_qp
*/ */
...@@ -1895,14 +1869,6 @@ int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev) ...@@ -1895,14 +1869,6 @@ int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
return -ENOMEM; return -ENOMEM;
} }
nesvnic->lro_mgr.max_aggr = nes_lro_max_aggr;
nesvnic->lro_mgr.max_desc = NES_MAX_LRO_DESCRIPTORS;
nesvnic->lro_mgr.lro_arr = nesvnic->lro_desc;
nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
nesvnic->lro_mgr.features = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
nesvnic->lro_mgr.dev = netdev;
nesvnic->lro_mgr.ip_summed = CHECKSUM_UNNECESSARY;
nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
return 0; return 0;
} }
...@@ -2809,13 +2775,10 @@ void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq) ...@@ -2809,13 +2775,10 @@ void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
u16 pkt_type; u16 pkt_type;
u16 rqes_processed = 0; u16 rqes_processed = 0;
u8 sq_cqes = 0; u8 sq_cqes = 0;
u8 nes_use_lro = 0;
head = cq->cq_head; head = cq->cq_head;
cq_size = cq->cq_size; cq_size = cq->cq_size;
cq->cqes_pending = 1; cq->cqes_pending = 1;
if (nesvnic->netdev->features & NETIF_F_LRO)
nes_use_lro = 1;
do { do {
if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) & if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
NES_NIC_CQE_VALID) { NES_NIC_CQE_VALID) {
...@@ -2950,10 +2913,7 @@ void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq) ...@@ -2950,10 +2913,7 @@ void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
__vlan_hwaccel_put_tag(rx_skb, htons(ETH_P_8021Q), vlan_tag); __vlan_hwaccel_put_tag(rx_skb, htons(ETH_P_8021Q), vlan_tag);
} }
if (nes_use_lro) napi_gro_receive(&nesvnic->napi, rx_skb);
lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
else
netif_receive_skb(rx_skb);
skip_rx_indicate0: skip_rx_indicate0:
; ;
...@@ -2984,8 +2944,6 @@ void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq) ...@@ -2984,8 +2944,6 @@ void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
} while (1); } while (1);
if (nes_use_lro)
lro_flush_all(&nesvnic->lro_mgr);
if (sq_cqes) { if (sq_cqes) {
barrier(); barrier();
/* restart the queue if it had been stopped */ /* restart the queue if it had been stopped */
......
...@@ -33,8 +33,6 @@ ...@@ -33,8 +33,6 @@
#ifndef __NES_HW_H #ifndef __NES_HW_H
#define __NES_HW_H #define __NES_HW_H
#include <linux/inet_lro.h>
#define NES_PHY_TYPE_CX4 1 #define NES_PHY_TYPE_CX4 1
#define NES_PHY_TYPE_1G 2 #define NES_PHY_TYPE_1G 2
#define NES_PHY_TYPE_ARGUS 4 #define NES_PHY_TYPE_ARGUS 4
...@@ -1049,8 +1047,6 @@ struct nes_hw_tune_timer { ...@@ -1049,8 +1047,6 @@ struct nes_hw_tune_timer {
#define NES_TIMER_ENABLE_LIMIT 4 #define NES_TIMER_ENABLE_LIMIT 4
#define NES_MAX_LINK_INTERRUPTS 128 #define NES_MAX_LINK_INTERRUPTS 128
#define NES_MAX_LINK_CHECK 200 #define NES_MAX_LINK_CHECK 200
#define NES_MAX_LRO_DESCRIPTORS 32
#define NES_LRO_MAX_AGGR 64
struct nes_adapter { struct nes_adapter {
u64 fw_ver; u64 fw_ver;
...@@ -1263,9 +1259,6 @@ struct nes_vnic { ...@@ -1263,9 +1259,6 @@ struct nes_vnic {
u8 next_qp_nic_index; u8 next_qp_nic_index;
u8 of_device_registered; u8 of_device_registered;
u8 rdma_enabled; u8 rdma_enabled;
u32 lro_max_aggr;
struct net_lro_mgr lro_mgr;
struct net_lro_desc lro_desc[NES_MAX_LRO_DESCRIPTORS];
struct timer_list event_timer; struct timer_list event_timer;
enum ib_event_type delayed_event; enum ib_event_type delayed_event;
enum ib_event_type last_dispatched_event; enum ib_event_type last_dispatched_event;
......
此差异已折叠。
...@@ -56,7 +56,8 @@ static int nes_dereg_mr(struct ib_mr *ib_mr); ...@@ -56,7 +56,8 @@ static int nes_dereg_mr(struct ib_mr *ib_mr);
/** /**
* nes_alloc_mw * nes_alloc_mw
*/ */
static struct ib_mw *nes_alloc_mw(struct ib_pd *ibpd, enum ib_mw_type type) static struct ib_mw *nes_alloc_mw(struct ib_pd *ibpd, enum ib_mw_type type,
struct ib_udata *udata)
{ {
struct nes_pd *nespd = to_nespd(ibpd); struct nes_pd *nespd = to_nespd(ibpd);
struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
...@@ -3768,6 +3769,8 @@ struct nes_ib_device *nes_init_ofa_device(struct net_device *netdev) ...@@ -3768,6 +3769,8 @@ struct nes_ib_device *nes_init_ofa_device(struct net_device *netdev)
nesibdev->ibdev.iwcm->create_listen = nes_create_listen; nesibdev->ibdev.iwcm->create_listen = nes_create_listen;
nesibdev->ibdev.iwcm->destroy_listen = nes_destroy_listen; nesibdev->ibdev.iwcm->destroy_listen = nes_destroy_listen;
nesibdev->ibdev.get_port_immutable = nes_port_immutable; nesibdev->ibdev.get_port_immutable = nes_port_immutable;
memcpy(nesibdev->ibdev.iwcm->ifname, netdev->name,
sizeof(nesibdev->ibdev.iwcm->ifname));
return nesibdev; return nesibdev;
} }
......
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册