Skip to content

K
Kernel

openeuler / Kernel
大约 2 年 前同步成功

通知 8
Star 0
Fork 0
K
Kernel
You need to sign in or sign up before continuing.
提交 6470d2e7 编写于 作者: M Maxim Mikityanskiy 提交者： Jakub Kicinski
浏览文件
操作

net/mlx5e: xsk: Use KSM for unaligned XSK 

UMR MTTs used in striding RQ have certain alignment requirements. While
it's guaranteed to work when UMR pages are aligned to the UMR page size,
in practice it works then UMR pages are aligned to 8 bytes. However,
it's still not enough flexibility for the unaligned mode of XSK. This
patch leverages KSM to map UMR pages without alignment requirements,
when unaligned XSK is active. The downside is that KSM entries are twice
as big as MTTs, which limits the maximum WQE size, so regular RQs and
aligned XSK continue using MTTs.
Signed-off-by: NMaxim Mikityanskiy <maximmi@nvidia.com>
Reviewed-by: NTariq Toukan <tariqt@nvidia.com>
Signed-off-by: NSaeed Mahameed <saeedm@nvidia.com>
Signed-off-by: NJakub Kicinski <kuba@kernel.org>
上级 c4418f34
显示空白变更内容
内联 并排
Showing with 191 addition and 100 deletion
drivers/net/ethernet/mellanox/mlx5/core/en.h
浏览文件 @ 6470d2e7
...@@ -112,8 +112,10 @@ struct page_pool; ...@@ -112,8 +112,10 @@ struct page_pool;
#define MLX5_ALIGN_MTTS(mtts) (ALIGN(mtts, 8)) #define MLX5_ALIGN_MTTS(mtts) (ALIGN(mtts, 8))
#define MLX5_ALIGNED_MTTS_OCTW(mtts) ((mtts) / 2) #define MLX5_ALIGNED_MTTS_OCTW(mtts) ((mtts) / 2)
#define MLX5_MTT_OCTW(mtts) (MLX5_ALIGNED_MTTS_OCTW(MLX5_ALIGN_MTTS(mtts))) #define MLX5_MTT_OCTW(mtts) (MLX5_ALIGNED_MTTS_OCTW(MLX5_ALIGN_MTTS(mtts)))
#define MLX5_KSM_OCTW(ksms) (ksms)
#define MLX5E_MAX_RQ_NUM_MTTS \ #define MLX5E_MAX_RQ_NUM_MTTS \
(ALIGN_DOWN(U16_MAX, 4) * 2) /* So that MLX5_MTT_OCTW(num_mtts) fits into u16 */ (ALIGN_DOWN(U16_MAX, 4) * 2) /* So that MLX5_MTT_OCTW(num_mtts) fits into u16 */
#define MLX5E_MAX_RQ_NUM_KSMS (U16_MAX - 1) /* So that num_ksms fits into u16. */
#define MLX5E_ORDER2_MAX_PACKET_MTU (order_base_2(10 * 1024)) #define MLX5E_ORDER2_MAX_PACKET_MTU (order_base_2(10 * 1024))
#define MLX5E_MIN_SKB_FRAG_SZ (MLX5_SKB_FRAG_SZ(MLX5_RX_HEADROOM)) #define MLX5E_MIN_SKB_FRAG_SZ (MLX5_SKB_FRAG_SZ(MLX5_RX_HEADROOM))
...@@ -265,6 +267,7 @@ struct mlx5e_umr_wqe { ...@@ -265,6 +267,7 @@ struct mlx5e_umr_wqe {
union { union {
DECLARE_FLEX_ARRAY(struct mlx5_mtt, inline_mtts); DECLARE_FLEX_ARRAY(struct mlx5_mtt, inline_mtts);
DECLARE_FLEX_ARRAY(struct mlx5_klm, inline_klms); DECLARE_FLEX_ARRAY(struct mlx5_klm, inline_klms);
DECLARE_FLEX_ARRAY(struct mlx5_ksm, inline_ksms);
}; };
}; };
...@@ -708,6 +711,7 @@ struct mlx5e_rq { ...@@ -708,6 +711,7 @@ struct mlx5e_rq {
u8 pages_per_wqe; u8 pages_per_wqe;
u8 umr_wqebbs; u8 umr_wqebbs;
u8 mtts_per_wqe; u8 mtts_per_wqe;
u8 unaligned;
struct mlx5e_shampo_hd *shampo; struct mlx5e_shampo_hd *shampo;
} mpwqe; } mpwqe;
}; };
...@@ -1007,7 +1011,8 @@ struct mlx5e_profile { ...@@ -1007,7 +1011,8 @@ struct mlx5e_profile {
void mlx5e_build_ptys2ethtool_map(void); void mlx5e_build_ptys2ethtool_map(void);
bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift); bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift,
bool unaligned);
void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close); void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close);
void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats); void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats);
......
drivers/net/ethernet/mellanox/mlx5/core/en/params.c
浏览文件 @ 6470d2e7
...@@ -27,15 +27,16 @@ u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xs ...@@ -27,15 +27,16 @@ u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xs
return max(req_page_shift, min_page_shift); return max(req_page_shift, min_page_shift);
} }
u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift) u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned)
{ {
u8 umr_entry_size = unaligned ? sizeof(struct mlx5_ksm) : sizeof(struct mlx5_mtt);
u8 max_pages_per_wqe, max_log_mpwqe_size; u8 max_pages_per_wqe, max_log_mpwqe_size;
u16 max_wqe_size; u16 max_wqe_size;
/* Keep in sync with MLX5_MPWRQ_MAX_PAGES_PER_WQE. */ /* Keep in sync with MLX5_MPWRQ_MAX_PAGES_PER_WQE. */
max_wqe_size = mlx5e_get_max_sq_aligned_wqebbs(mdev) * MLX5_SEND_WQE_BB; max_wqe_size = mlx5e_get_max_sq_aligned_wqebbs(mdev) * MLX5_SEND_WQE_BB;
max_pages_per_wqe = ALIGN_DOWN(max_wqe_size - sizeof(struct mlx5e_umr_wqe), max_pages_per_wqe = ALIGN_DOWN(max_wqe_size - sizeof(struct mlx5e_umr_wqe),
MLX5_UMR_MTT_ALIGNMENT) / sizeof(struct mlx5_mtt); MLX5_UMR_MTT_ALIGNMENT) / umr_entry_size;
max_log_mpwqe_size = ilog2(max_pages_per_wqe) + page_shift; max_log_mpwqe_size = ilog2(max_pages_per_wqe) + page_shift;
WARN_ON_ONCE(max_log_mpwqe_size < MLX5E_ORDER2_MAX_PACKET_MTU); WARN_ON_ONCE(max_log_mpwqe_size < MLX5E_ORDER2_MAX_PACKET_MTU);
...@@ -43,9 +44,9 @@ u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift) ...@@ -43,9 +44,9 @@ u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift)
return min_t(u8, max_log_mpwqe_size, MLX5_MPWRQ_MAX_LOG_WQE_SZ); return min_t(u8, max_log_mpwqe_size, MLX5_MPWRQ_MAX_LOG_WQE_SZ);
} }
u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift) u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned)
{ {
u8 log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift); u8 log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned);
u8 pages_per_wqe; u8 pages_per_wqe;
pages_per_wqe = log_wqe_sz > page_shift ? (1 << (log_wqe_sz - page_shift)) : 1; pages_per_wqe = log_wqe_sz > page_shift ? (1 << (log_wqe_sz - page_shift)) : 1;
...@@ -58,45 +59,58 @@ u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift) ...@@ -58,45 +59,58 @@ u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift)
return pages_per_wqe; return pages_per_wqe;
} }
u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift) u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned)
{ {
u8 pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, page_shift); u8 umr_entry_size = unaligned ? sizeof(struct mlx5_ksm) : sizeof(struct mlx5_mtt);
u8 pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, page_shift, unaligned);
u16 umr_wqe_sz; u16 umr_wqe_sz;
umr_wqe_sz = sizeof(struct mlx5e_umr_wqe) + umr_wqe_sz = sizeof(struct mlx5e_umr_wqe) +
ALIGN(pages_per_wqe * sizeof(struct mlx5_mtt), MLX5_UMR_MTT_ALIGNMENT); ALIGN(pages_per_wqe * umr_entry_size, MLX5_UMR_MTT_ALIGNMENT);
WARN_ON_ONCE(DIV_ROUND_UP(umr_wqe_sz, MLX5_SEND_WQE_DS) > MLX5_WQE_CTRL_DS_MASK); WARN_ON_ONCE(DIV_ROUND_UP(umr_wqe_sz, MLX5_SEND_WQE_DS) > MLX5_WQE_CTRL_DS_MASK);
return umr_wqe_sz; return umr_wqe_sz;
} }
u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift) u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned)
{ {
return DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(mdev, page_shift), MLX5_SEND_WQE_BB); return DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(mdev, page_shift, unaligned),
MLX5_SEND_WQE_BB);
} }
u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift) u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned)
{ {
/* Add another page as a buffer between WQEs. This page will absorb /* Add another page as a buffer between WQEs. This page will absorb
* write overflow by the hardware, when receiving packets larger than * write overflow by the hardware, when receiving packets larger than
* MTU. These oversize packets are dropped by the driver at a later * MTU. These oversize packets are dropped by the driver at a later
* stage. * stage.
*/ */
return MLX5_ALIGN_MTTS(mlx5e_mpwrq_pages_per_wqe(mdev, page_shift) + 1); return MLX5_ALIGN_MTTS(mlx5e_mpwrq_pages_per_wqe(mdev, page_shift, unaligned) + 1);
} }
static u8 mlx5e_mpwrq_max_log_rq_size(struct mlx5_core_dev *mdev, u8 page_shift) u32 mlx5e_mpwrq_max_num_entries(struct mlx5_core_dev *mdev, bool unaligned)
{ {
u8 mtts_per_wqe = mlx5e_mpwrq_mtts_per_wqe(mdev, page_shift); if (unaligned)
return min(MLX5E_MAX_RQ_NUM_KSMS,
1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size));
return ilog2(MLX5E_MAX_RQ_NUM_MTTS / mtts_per_wqe); return MLX5E_MAX_RQ_NUM_MTTS;
} }
u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift) static u8 mlx5e_mpwrq_max_log_rq_size(struct mlx5_core_dev *mdev, u8 page_shift,
bool unaligned)
{ {
return mlx5e_mpwrq_max_log_rq_size(mdev, page_shift) + u8 mtts_per_wqe = mlx5e_mpwrq_mtts_per_wqe(mdev, page_shift, unaligned);
mlx5e_mpwrq_log_wqe_sz(mdev, page_shift) - u32 max_entries = mlx5e_mpwrq_max_num_entries(mdev, unaligned);
return ilog2(max_entries / mtts_per_wqe);
}
u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned)
{
return mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, unaligned) +
mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned) -
MLX5E_ORDER2_MAX_PACKET_MTU; MLX5E_ORDER2_MAX_PACKET_MTU;
} }
...@@ -158,8 +172,10 @@ static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev, ...@@ -158,8 +172,10 @@ static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev,
{ {
u32 linear_stride_sz = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true); u32 linear_stride_sz = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
bool unaligned = xsk ? xsk->unaligned : false;
return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift) - order_base_2(linear_stride_sz); return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned) -
order_base_2(linear_stride_sz);
} }
bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
...@@ -184,9 +200,10 @@ bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, ...@@ -184,9 +200,10 @@ bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
static bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev, static bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
u8 log_stride_sz, u8 log_num_strides, u8 log_stride_sz, u8 log_num_strides,
u8 page_shift) u8 page_shift, bool unaligned)
{ {
if (log_stride_sz + log_num_strides != mlx5e_mpwrq_log_wqe_sz(mdev, page_shift)) if (log_stride_sz + log_num_strides !=
mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned))
return false; return false;
if (log_stride_sz < MLX5_MPWQE_LOG_STRIDE_SZ_BASE || if (log_stride_sz < MLX5_MPWQE_LOG_STRIDE_SZ_BASE ||
...@@ -207,6 +224,7 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, ...@@ -207,6 +224,7 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
struct mlx5e_xsk_param *xsk) struct mlx5e_xsk_param *xsk)
{ {
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
bool unaligned = xsk ? xsk->unaligned : false;
u8 log_num_strides; u8 log_num_strides;
u8 log_stride_sz; u8 log_stride_sz;
u8 log_wqe_sz; u8 log_wqe_sz;
...@@ -215,7 +233,7 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, ...@@ -215,7 +233,7 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
return false; return false;
log_stride_sz = order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true)); log_stride_sz = order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true));
log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift); log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned);
if (log_wqe_sz < log_stride_sz) if (log_wqe_sz < log_stride_sz)
return false; return false;
...@@ -223,7 +241,8 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, ...@@ -223,7 +241,8 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
log_num_strides = log_wqe_sz - log_stride_sz; log_num_strides = log_wqe_sz - log_stride_sz;
return mlx5e_verify_rx_mpwqe_strides(mdev, log_stride_sz, return mlx5e_verify_rx_mpwqe_strides(mdev, log_stride_sz,
log_num_strides, page_shift); log_num_strides, page_shift,
unaligned);
} }
u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev, u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev,
...@@ -231,10 +250,11 @@ u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev, ...@@ -231,10 +250,11 @@ u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev,
struct mlx5e_xsk_param *xsk) struct mlx5e_xsk_param *xsk)
{ {
u8 log_pkts_per_wqe, page_shift, max_log_rq_size; u8 log_pkts_per_wqe, page_shift, max_log_rq_size;
bool unaligned = xsk ? xsk->unaligned : false;
log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, xsk); log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, xsk);
page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
max_log_rq_size = mlx5e_mpwrq_max_log_rq_size(mdev, page_shift); max_log_rq_size = mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, unaligned);
/* Numbers are unsigned, don't subtract to avoid underflow. */ /* Numbers are unsigned, don't subtract to avoid underflow. */
if (params->log_rq_mtu_frames < if (params->log_rq_mtu_frames <
...@@ -289,8 +309,9 @@ u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev, ...@@ -289,8 +309,9 @@ u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
struct mlx5e_xsk_param *xsk) struct mlx5e_xsk_param *xsk)
{ {
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
bool unaligned = xsk ? xsk->unaligned : false;
return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift) - return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned) -
mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
} }
...@@ -441,7 +462,7 @@ int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params ...@@ -441,7 +462,7 @@ int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params
{ {
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, NULL); u8 page_shift = mlx5e_mpwrq_page_shift(mdev, NULL);
if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift)) if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, false))
return -EOPNOTSUPP; return -EOPNOTSUPP;
if (params->xdp_prog && !mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL)) if (params->xdp_prog && !mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL))
...@@ -454,9 +475,10 @@ int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *pa ...@@ -454,9 +475,10 @@ int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *pa
struct mlx5e_xsk_param *xsk) struct mlx5e_xsk_param *xsk)
{ {
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
bool unaligned = xsk ? xsk->unaligned : false;
u16 max_mtu_pkts; u16 max_mtu_pkts;
if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift)) if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, xsk->unaligned))
return -EOPNOTSUPP; return -EOPNOTSUPP;
if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))
...@@ -466,7 +488,7 @@ int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *pa ...@@ -466,7 +488,7 @@ int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *pa
* needed number of WQEs exceeds the maximum. * needed number of WQEs exceeds the maximum.
*/ */
max_mtu_pkts = min_t(u8, MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE, max_mtu_pkts = min_t(u8, MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE,
mlx5e_mpwrq_max_log_rq_pkts(mdev, page_shift)); mlx5e_mpwrq_max_log_rq_pkts(mdev, page_shift, unaligned));
if (params->log_rq_mtu_frames > max_mtu_pkts) { if (params->log_rq_mtu_frames > max_mtu_pkts) {
mlx5_core_err(mdev, "Current RQ length %d is too big for XSK with given frame size %u\n", mlx5_core_err(mdev, "Current RQ length %d is too big for XSK with given frame size %u\n",
1 << params->log_rq_mtu_frames, xsk->chunk_size); 1 << params->log_rq_mtu_frames, xsk->chunk_size);
...@@ -724,13 +746,15 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, ...@@ -724,13 +746,15 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev,
u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
bool unaligned = xsk ? xsk->unaligned : false;
if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size, if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size,
log_wqe_num_of_strides, log_wqe_num_of_strides,
page_shift)) { page_shift, unaligned)) {
mlx5_core_err(mdev, mlx5_core_err(mdev,
"Bad RX MPWQE params: log_stride_size %u, log_num_strides %u\n", "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u, unaligned %d\n",
log_wqe_stride_size, log_wqe_num_of_strides); log_wqe_stride_size, log_wqe_num_of_strides,
unaligned);
return -EINVAL; return -EINVAL;
} }
...@@ -850,13 +874,6 @@ static void mlx5e_build_ico_cq_param(struct mlx5_core_dev *mdev, ...@@ -850,13 +874,6 @@ static void mlx5e_build_ico_cq_param(struct mlx5_core_dev *mdev,
param->cq_period_mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; param->cq_period_mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
} }
static u8 mlx5e_get_rq_log_wq_sz(void *rqc)
{
void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
return MLX5_GET(wq, wq, log_wq_sz);
}
/* This function calculates the maximum number of headers entries that are needed /* This function calculates the maximum number of headers entries that are needed
* per WQE, the formula is based on the size of the reservations and the * per WQE, the formula is based on the size of the reservations and the
* restriction we have about max packets for reservation that is equal to max * restriction we have about max packets for reservation that is equal to max
...@@ -917,6 +934,19 @@ static u32 mlx5e_shampo_icosq_sz(struct mlx5_core_dev *mdev, ...@@ -917,6 +934,19 @@ static u32 mlx5e_shampo_icosq_sz(struct mlx5_core_dev *mdev,
return wqebbs; return wqebbs;
} }
static u32 mlx5e_mpwrq_total_umr_wqebbs(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk)
{
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
bool unaligned = xsk ? xsk->unaligned : false;
u8 umr_wqebbs;
umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, unaligned);
return umr_wqebbs * (1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));
}
static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev,
struct mlx5e_params *params, struct mlx5e_params *params,
struct mlx5e_rq_param *rqp) struct mlx5e_rq_param *rqp)
...@@ -928,8 +958,7 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, ...@@ -928,8 +958,7 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev,
return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
/* UMR WQEs for the regular RQ. */ /* UMR WQEs for the regular RQ. */
wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, PAGE_SHIFT) * wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, NULL);
(1 << mlx5e_get_rq_log_wq_sz(rqp->rqc));
/* If XDP program is attached, XSK may be turned on at any time without /* If XDP program is attached, XSK may be turned on at any time without
* restarting the channel. ICOSQ must be big enough to fit UMR WQEs of * restarting the channel. ICOSQ must be big enough to fit UMR WQEs of
...@@ -950,16 +979,17 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, ...@@ -950,16 +979,17 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev,
/* The headroom doesn't affect the calculation. */ /* The headroom doesn't affect the calculation. */
struct mlx5e_xsk_param xsk = { struct mlx5e_xsk_param xsk = {
.chunk_size = 1 << frame_shift, .chunk_size = 1 << frame_shift,
.unaligned = false,
}; };
u32 xsk_wqebbs;
u8 page_shift;
page_shift = mlx5e_mpwrq_page_shift(mdev, &xsk); /* XSK aligned mode. */
xsk_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift) * max_xsk_wqebbs = max(max_xsk_wqebbs,
(1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, &xsk)); mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
if (xsk_wqebbs > max_xsk_wqebbs) /* XSK unaligned mode, frame size is a power of two. */
max_xsk_wqebbs = xsk_wqebbs; xsk.unaligned = true;
max_xsk_wqebbs = max(max_xsk_wqebbs,
mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
} }
wqebbs += max_xsk_wqebbs; wqebbs += max_xsk_wqebbs;
......
drivers/net/ethernet/mellanox/mlx5/core/en/params.h
浏览文件 @ 6470d2e7
...@@ -9,6 +9,7 @@ ...@@ -9,6 +9,7 @@
struct mlx5e_xsk_param { struct mlx5e_xsk_param {
u16 headroom; u16 headroom;
u16 chunk_size; u16 chunk_size;
bool unaligned;
}; };
struct mlx5e_cq_param { struct mlx5e_cq_param {
...@@ -87,12 +88,13 @@ static inline bool mlx5e_qid_validate(const struct mlx5e_profile *profile, ...@@ -87,12 +88,13 @@ static inline bool mlx5e_qid_validate(const struct mlx5e_profile *profile,
/* Striding RQ dynamic parameters */ /* Striding RQ dynamic parameters */
u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk); u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk);
u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift); u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned);
u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift); u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned);
u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift); u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned);
u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift); u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned);
u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift); u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned);
u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift); u32 mlx5e_mpwrq_max_num_entries(struct mlx5_core_dev *mdev, bool unaligned);
u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned);
/* Parameter calculations */ /* Parameter calculations */
......
drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
浏览文件 @ 6470d2e7
...@@ -72,6 +72,7 @@ void mlx5e_build_xsk_param(struct xsk_buff_pool *pool, struct mlx5e_xsk_param *x ...@@ -72,6 +72,7 @@ void mlx5e_build_xsk_param(struct xsk_buff_pool *pool, struct mlx5e_xsk_param *x
{ {
xsk->headroom = xsk_pool_get_headroom(pool); xsk->headroom = xsk_pool_get_headroom(pool);
xsk->chunk_size = xsk_pool_get_chunk_size(pool); xsk->chunk_size = xsk_pool_get_chunk_size(pool);
xsk->unaligned = pool->unaligned;
} }
static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
......
drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
浏览文件 @ 6470d2e7
...@@ -7,8 +7,6 @@ ...@@ -7,8 +7,6 @@
#include "en.h" #include "en.h"
#include <net/xdp_sock_drv.h> #include <net/xdp_sock_drv.h>
#define MLX5E_MTT_PTAG_MASK 0xfffffffffffffff8ULL
/* RX data path */ /* RX data path */
struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
...@@ -23,7 +21,6 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq, ...@@ -23,7 +21,6 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
static inline int mlx5e_xsk_page_alloc_pool(struct mlx5e_rq *rq, static inline int mlx5e_xsk_page_alloc_pool(struct mlx5e_rq *rq,
struct mlx5e_dma_info *dma_info) struct mlx5e_dma_info *dma_info)
{ {
retry:
dma_info->xsk = xsk_buff_alloc(rq->xsk_pool); dma_info->xsk = xsk_buff_alloc(rq->xsk_pool);
if (!dma_info->xsk) if (!dma_info->xsk)
return -ENOMEM; return -ENOMEM;
...@@ -35,17 +32,6 @@ static inline int mlx5e_xsk_page_alloc_pool(struct mlx5e_rq *rq, ...@@ -35,17 +32,6 @@ static inline int mlx5e_xsk_page_alloc_pool(struct mlx5e_rq *rq,
*/ */
dma_info->addr = xsk_buff_xdp_get_frame_dma(dma_info->xsk); dma_info->addr = xsk_buff_xdp_get_frame_dma(dma_info->xsk);
/* MTT page mapping has alignment requirements. If they are not
* satisfied, leak the descriptor so that it won't come again, and try
* to allocate a new one.
*/
if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
if (unlikely(dma_info->addr & ~MLX5E_MTT_PTAG_MASK)) {
xsk_buff_discard(dma_info->xsk);
goto retry;
}
}
return 0; return 0;
} }
......
drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
浏览文件 @ 6470d2e7
...@@ -314,7 +314,7 @@ void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv, ...@@ -314,7 +314,7 @@ void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv,
/* Limitation for regular RQ. XSK RQ may clamp the queue length in /* Limitation for regular RQ. XSK RQ may clamp the queue length in
* mlx5e_mpwqe_get_log_rq_size. * mlx5e_mpwqe_get_log_rq_size.
*/ */
u8 max_log_mpwrq_pkts = mlx5e_mpwrq_max_log_rq_pkts(priv->mdev, PAGE_SHIFT); u8 max_log_mpwrq_pkts = mlx5e_mpwrq_max_log_rq_pkts(priv->mdev, PAGE_SHIFT, false);
param->rx_max_pending = 1 << min_t(u8, MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE, param->rx_max_pending = 1 << min_t(u8, MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE,
max_log_mpwrq_pkts); max_log_mpwrq_pkts);
......
drivers/net/ethernet/mellanox/mlx5/core/en_main.c
浏览文件 @ 6470d2e7
...@@ -68,7 +68,8 @@ ...@@ -68,7 +68,8 @@
#include "qos.h" #include "qos.h"
#include "en/trap.h" #include "en/trap.h"
bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift) bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift,
bool unaligned)
{ {
u16 umr_wqebbs, max_wqebbs; u16 umr_wqebbs, max_wqebbs;
bool striding_rq_umr; bool striding_rq_umr;
...@@ -78,7 +79,7 @@ bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_ ...@@ -78,7 +79,7 @@ bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_
if (!striding_rq_umr) if (!striding_rq_umr)
return false; return false;
umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift); umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, unaligned);
max_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); max_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev);
/* Sanity check; should never happen, because mlx5e_mpwrq_umr_wqebbs is /* Sanity check; should never happen, because mlx5e_mpwrq_umr_wqebbs is
* calculated from mlx5e_get_max_sq_aligned_wqebbs. * calculated from mlx5e_get_max_sq_aligned_wqebbs.
...@@ -208,9 +209,11 @@ static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, ...@@ -208,9 +209,11 @@ static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq,
{ {
struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
u16 octowords;
u8 ds_cnt; u8 ds_cnt;
ds_cnt = DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(rq->mdev, rq->mpwqe.page_shift), ds_cnt = DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(rq->mdev, rq->mpwqe.page_shift,
rq->mpwqe.unaligned),
MLX5_SEND_WQE_DS); MLX5_SEND_WQE_DS);
cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
...@@ -218,8 +221,9 @@ static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, ...@@ -218,8 +221,9 @@ static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq,
cseg->umr_mkey = rq->mpwqe.umr_mkey_be; cseg->umr_mkey = rq->mpwqe.umr_mkey_be;
ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE; ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE;
ucseg->xlt_octowords = octowords = rq->mpwqe.unaligned ? MLX5_KSM_OCTW(rq->mpwqe.pages_per_wqe) :
cpu_to_be16(MLX5_MTT_OCTW(rq->mpwqe.pages_per_wqe)); MLX5_MTT_OCTW(rq->mpwqe.pages_per_wqe);
ucseg->xlt_octowords = cpu_to_be16(octowords);
ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
} }
...@@ -279,39 +283,51 @@ static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node) ...@@ -279,39 +283,51 @@ static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node)
return 0; return 0;
} }
static int mlx5e_create_umr_mtt_mkey(struct mlx5_core_dev *mdev, static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev,
u64 npages, u8 page_shift, u32 *umr_mkey, u32 npages, u8 page_shift, u32 *umr_mkey,
dma_addr_t filler_addr) dma_addr_t filler_addr, bool unaligned)
{ {
struct mlx5_mtt *mtt; struct mlx5_mtt *mtt;
struct mlx5_ksm *ksm;
u32 octwords;
int inlen; int inlen;
void *mkc; void *mkc;
u32 *in; u32 *in;
int err; int err;
int i; int i;
inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + sizeof(*mtt) * npages; if (unaligned && !MLX5_CAP_GEN(mdev, fixed_buffer_size)) {
mlx5_core_warn(mdev, "Unaligned AF_XDP requires fixed_buffer_size capability\n");
return -EINVAL;
}
inlen = MLX5_FLEXIBLE_INLEN(mdev, MLX5_ST_SZ_BYTES(create_mkey_in),
unaligned ? sizeof(*ksm) : sizeof(*mtt),
npages);
if (inlen < 0)
return inlen;
in = kvzalloc(inlen, GFP_KERNEL); in = kvzalloc(inlen, GFP_KERNEL);
if (!in) if (!in)
return -ENOMEM; return -ENOMEM;
octwords = unaligned ? MLX5_KSM_OCTW(npages) : MLX5_MTT_OCTW(npages);
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
MLX5_SET(mkc, mkc, free, 1); MLX5_SET(mkc, mkc, free, 1);
MLX5_SET(mkc, mkc, umr_en, 1); MLX5_SET(mkc, mkc, umr_en, 1);
MLX5_SET(mkc, mkc, lw, 1); MLX5_SET(mkc, mkc, lw, 1);
MLX5_SET(mkc, mkc, lr, 1); MLX5_SET(mkc, mkc, lr, 1);
MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); MLX5_SET(mkc, mkc, access_mode_1_0,
unaligned ? MLX5_MKC_ACCESS_MODE_KSM : MLX5_MKC_ACCESS_MODE_MTT);
mlx5e_mkey_set_relaxed_ordering(mdev, mkc); mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
MLX5_SET(mkc, mkc, qpn, 0xffffff); MLX5_SET(mkc, mkc, qpn, 0xffffff);
MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn);
MLX5_SET64(mkc, mkc, len, npages << page_shift); MLX5_SET64(mkc, mkc, len, npages << page_shift);
MLX5_SET(mkc, mkc, translations_octword_size, MLX5_SET(mkc, mkc, translations_octword_size, octwords);
MLX5_MTT_OCTW(npages));
MLX5_SET(mkc, mkc, log_page_size, page_shift); MLX5_SET(mkc, mkc, log_page_size, page_shift);
MLX5_SET(create_mkey_in, in, translations_octword_actual_size, MLX5_SET(create_mkey_in, in, translations_octword_actual_size, octwords);
MLX5_MTT_OCTW(npages));
/* Initialize the mkey with all MTTs pointing to a default /* Initialize the mkey with all MTTs pointing to a default
* page (filler_addr). When the channels are activated, UMR * page (filler_addr). When the channels are activated, UMR
...@@ -319,9 +335,20 @@ static int mlx5e_create_umr_mtt_mkey(struct mlx5_core_dev *mdev, ...@@ -319,9 +335,20 @@ static int mlx5e_create_umr_mtt_mkey(struct mlx5_core_dev *mdev,
* the RQ's pool, while the gaps (wqe_overflow) remain mapped * the RQ's pool, while the gaps (wqe_overflow) remain mapped
* to the default page. * to the default page.
*/ */
if (unaligned) {
ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
for (i = 0; i < npages; i++)
ksm[i] = (struct mlx5_ksm) {
.key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
.va = cpu_to_be64(filler_addr),
};
} else {
mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
for (i = 0 ; i < npages ; i++) for (i = 0; i < npages; i++)
mtt[i].ptag = cpu_to_be64(filler_addr); mtt[i] = (struct mlx5_mtt) {
.ptag = cpu_to_be64(filler_addr),
};
}
err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
...@@ -364,12 +391,24 @@ static int mlx5e_create_umr_klm_mkey(struct mlx5_core_dev *mdev, ...@@ -364,12 +391,24 @@ static int mlx5e_create_umr_klm_mkey(struct mlx5_core_dev *mdev,
static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq) static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq)
{ {
u64 num_mtts = mlx5_wq_ll_get_size(&rq->mpwqe.wq) * rq->mpwqe.mtts_per_wqe; u32 wq_size = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
u32 num_entries, max_num_entries;
u32 umr_mkey; u32 umr_mkey;
int err; int err;
err = mlx5e_create_umr_mtt_mkey(mdev, num_mtts, rq->mpwqe.page_shift, max_num_entries = mlx5e_mpwrq_max_num_entries(mdev, rq->mpwqe.unaligned);
&umr_mkey, rq->wqe_overflow.addr);
/* Shouldn't overflow, the result is at most MLX5E_MAX_RQ_NUM_MTTS. */
if (WARN_ON_ONCE(check_mul_overflow(wq_size, (u32)rq->mpwqe.mtts_per_wqe,
&num_entries) ||
num_entries > max_num_entries))
mlx5_core_err(mdev, "%s: multiplication overflow: %u * %u > %u\n",
__func__, wq_size, rq->mpwqe.mtts_per_wqe,
max_num_entries);
err = mlx5e_create_umr_mkey(mdev, num_entries, rq->mpwqe.page_shift,
&umr_mkey, rq->wqe_overflow.addr,
rq->mpwqe.unaligned);
rq->mpwqe.umr_mkey_be = cpu_to_be32(umr_mkey); rq->mpwqe.umr_mkey_be = cpu_to_be32(umr_mkey);
return err; return err;
} }
...@@ -597,12 +636,16 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, ...@@ -597,12 +636,16 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
rq->mpwqe.unaligned = xsk ? xsk->unaligned : false;
rq->mpwqe.pages_per_wqe = rq->mpwqe.pages_per_wqe =
mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift); mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift,
rq->mpwqe.unaligned);
rq->mpwqe.umr_wqebbs = rq->mpwqe.umr_wqebbs =
mlx5e_mpwrq_umr_wqebbs(mdev, rq->mpwqe.page_shift); mlx5e_mpwrq_umr_wqebbs(mdev, rq->mpwqe.page_shift,
rq->mpwqe.unaligned);
rq->mpwqe.mtts_per_wqe = rq->mpwqe.mtts_per_wqe =
mlx5e_mpwrq_mtts_per_wqe(mdev, rq->mpwqe.page_shift); mlx5e_mpwrq_mtts_per_wqe(mdev, rq->mpwqe.page_shift,
rq->mpwqe.unaligned);
pool_size = rq->mpwqe.pages_per_wqe << pool_size = rq->mpwqe.pages_per_wqe <<
mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk); mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk);
...@@ -4932,7 +4975,7 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev) ...@@ -4932,7 +4975,7 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev)
if (!!MLX5_CAP_ETH(mdev, lro_cap) && if (!!MLX5_CAP_ETH(mdev, lro_cap) &&
!MLX5_CAP_ETH(mdev, tunnel_lro_vxlan) && !MLX5_CAP_ETH(mdev, tunnel_lro_vxlan) &&
!MLX5_CAP_ETH(mdev, tunnel_lro_gre) && !MLX5_CAP_ETH(mdev, tunnel_lro_gre) &&
mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT)) mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT, false))
netdev->vlan_features |= NETIF_F_LRO; netdev->vlan_features |= NETIF_F_LRO;
netdev->hw_features = netdev->vlan_features; netdev->hw_features = netdev->vlan_features;
......
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
浏览文件 @ 6470d2e7
...@@ -673,6 +673,7 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) ...@@ -673,6 +673,7 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
struct mlx5e_icosq *sq = rq->icosq; struct mlx5e_icosq *sq = rq->icosq;
struct mlx5_wq_cyc *wq = &sq->wq; struct mlx5_wq_cyc *wq = &sq->wq;
struct mlx5e_umr_wqe *umr_wqe; struct mlx5e_umr_wqe *umr_wqe;
u32 offset; /* 17-bit value with MTT. */
u16 pi; u16 pi;
int err; int err;
int i; int i;
...@@ -694,13 +695,27 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) ...@@ -694,13 +695,27 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
pi = mlx5e_icosq_get_next_pi(sq, rq->mpwqe.umr_wqebbs); pi = mlx5e_icosq_get_next_pi(sq, rq->mpwqe.umr_wqebbs);
umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi); umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi);
memcpy(umr_wqe, &rq->mpwqe.umr_wqe, offsetof(struct mlx5e_umr_wqe, inline_mtts)); memcpy(umr_wqe, &rq->mpwqe.umr_wqe, sizeof(struct mlx5e_umr_wqe));
if (unlikely(rq->mpwqe.unaligned)) {
for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, dma_info++) { for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, dma_info++) {
err = mlx5e_page_alloc(rq, dma_info); err = mlx5e_page_alloc(rq, dma_info);
if (unlikely(err)) if (unlikely(err))
goto err_unmap; goto err_unmap;
umr_wqe->inline_mtts[i].ptag = cpu_to_be64(dma_info->addr | MLX5_EN_WR); umr_wqe->inline_ksms[i] = (struct mlx5_ksm) {
.key = rq->mkey_be,
.va = cpu_to_be64(dma_info->addr),
};
}
} else {
for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, dma_info++) {
err = mlx5e_page_alloc(rq, dma_info);
if (unlikely(err))
goto err_unmap;
umr_wqe->inline_mtts[i] = (struct mlx5_mtt) {
.ptag = cpu_to_be64(dma_info->addr | MLX5_EN_WR),
};
}
} }
bitmap_zero(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe); bitmap_zero(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe);
...@@ -709,8 +724,11 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) ...@@ -709,8 +724,11 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
umr_wqe->ctrl.opmod_idx_opcode = umr_wqe->ctrl.opmod_idx_opcode =
cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) | cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
MLX5_OPCODE_UMR); MLX5_OPCODE_UMR);
umr_wqe->uctrl.xlt_offset =
cpu_to_be16(MLX5_ALIGNED_MTTS_OCTW(ix * rq->mpwqe.mtts_per_wqe)); offset = ix * rq->mpwqe.mtts_per_wqe;
if (!rq->mpwqe.unaligned)
offset = MLX5_ALIGNED_MTTS_OCTW(offset);
umr_wqe->uctrl.xlt_offset = cpu_to_be16(offset);
sq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) { sq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) {
.wqe_type = MLX5E_ICOSQ_WQE_UMR_RX, .wqe_type = MLX5E_ICOSQ_WQE_UMR_RX,
......
include/linux/mlx5/qp.h
浏览文件 @ 6470d2e7
...@@ -478,6 +478,12 @@ struct mlx5_klm { ...@@ -478,6 +478,12 @@ struct mlx5_klm {
__be64 va; __be64 va;
}; };
struct mlx5_ksm {
__be32 reserved;
__be32 key;
__be64 va;
};
struct mlx5_stride_block_entry { struct mlx5_stride_block_entry {
__be16 stride; __be16 stride;
__be16 bcount; __be16 bcount;
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或