// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2019-2020, Mellanox Technologies inc. All rights reserved. */

#ifdef HAVE_XSK_ZERO_COPY_SUPPORT
#ifdef HAVE_XDP_SOCK_DRV_H
#include <net/xdp_sock_drv.h>
#else
#include <net/xdp_sock.h>
#endif
#include "pool.h"
#include "setup.h"
#include "en/params.h"

static int mlx5e_xsk_map_pool(struct mlx5_core_dev *mdev,
#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
			      struct xsk_buff_pool *pool)
#else
			      struct xdp_umem *umem)
#endif
{
	struct device *dev = mlx5_core_dma_dev(mdev);

#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
	return xsk_pool_dma_map(pool, dev, DMA_ATTR_SKIP_CPU_SYNC);
#elif defined(HAVE_XSK_BUFF_ALLOC)
	return xsk_buff_dma_map(umem, dev, 0);
#else
       u32 i;

       for (i = 0; i < umem->npgs; i++) {
              dma_addr_t dma = dma_map_page(dev, umem->pgs[i], 0, PAGE_SIZE,
                                         DMA_BIDIRECTIONAL);

              if (unlikely(dma_mapping_error(dev, dma)))
                     goto err_unmap;
              umem->pages[i].dma = dma;
       }

       return 0;

err_unmap:
       while (i--) {
              dma_unmap_page(dev, umem->pages[i].dma, PAGE_SIZE,
                            DMA_BIDIRECTIONAL);
              umem->pages[i].dma = 0;
       }

       return -ENOMEM;
#endif /* HAVE_NETDEV_BPF_XSK_BUFF_POOL */
}

static void mlx5e_xsk_unmap_pool(struct mlx5e_priv *priv,
#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
			         struct xsk_buff_pool *pool)
#else
			         struct xdp_umem *umem)
#endif
{
#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
	return xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC);
#elif defined(HAVE_XSK_BUFF_ALLOC)
	return xsk_buff_dma_unmap(umem, 0);
#else
       struct device *dev = priv->mdev->device;
       u32 i;

       for (i = 0; i < umem->npgs; i++) {
              dma_unmap_page(dev, umem->pages[i].dma, PAGE_SIZE,
                            DMA_BIDIRECTIONAL);
              umem->pages[i].dma = 0;
       }
#endif
}

static int mlx5e_xsk_get_pools(struct mlx5e_xsk *xsk)
{
#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
	if (!xsk->pools) {
		xsk->pools = kcalloc(MLX5E_MAX_NUM_CHANNELS,
				     sizeof(*xsk->pools), GFP_KERNEL);
		if (unlikely(!xsk->pools))
#else
	if (!xsk->umems) {
		xsk->umems = kcalloc(MLX5E_MAX_NUM_CHANNELS,
				     sizeof(*xsk->umems), GFP_KERNEL);
		if (unlikely(!xsk->umems))
#endif
			return -ENOMEM;
	}

	xsk->refcnt++;
	xsk->ever_used = true;

	return 0;
}

static void mlx5e_xsk_put_pools(struct mlx5e_xsk *xsk)
{
	if (!--xsk->refcnt) {
#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
		kfree(xsk->pools);
		xsk->pools = NULL;
#else
		kfree(xsk->umems);
		xsk->umems = NULL;
#endif
	}
}

static int mlx5e_xsk_add_pool(struct mlx5e_xsk *xsk,
#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
			         struct xsk_buff_pool *pool,
#else
			         struct xdp_umem *pool,
#endif
				 u16 ix)
{
	int err;

	err = mlx5e_xsk_get_pools(xsk);
	if (unlikely(err))
		return err;

#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
	xsk->pools[ix] = pool;
#else
	xsk->umems[ix] = pool;
#endif
	return 0;
}

static void mlx5e_xsk_remove_pool(struct mlx5e_xsk *xsk, u16 ix)
{
#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
	xsk->pools[ix] = NULL;
#else
	xsk->umems[ix] = NULL;
#endif

	mlx5e_xsk_put_pools(xsk);
}

#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
static bool mlx5e_xsk_is_pool_sane(struct xsk_buff_pool *pool)
#else
static bool mlx5e_xsk_is_pool_sane(struct xdp_umem *umem)
#endif
{
#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
	return xsk_pool_get_headroom(pool) <= 0xffff &&
		xsk_pool_get_chunk_size(pool) <= 0xffff;
#elif defined(HAVE_XSK_BUFF_ALLOC)
	return xsk_umem_get_headroom(umem) <= 0xffff &&
		xsk_umem_get_chunk_size(umem) <= 0xffff;
#else
	return umem->headroom <= 0xffff &&
		umem->chunk_size_nohr <= 0xffff;
#endif
}

#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
void mlx5e_build_xsk_param(struct xsk_buff_pool *pool, struct mlx5e_xsk_param *xsk)
#else
void mlx5e_build_xsk_param(struct xdp_umem *umem, struct mlx5e_xsk_param *xsk)
#endif
{
#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
	xsk->headroom = xsk_pool_get_headroom(pool);
	xsk->chunk_size = xsk_pool_get_chunk_size(pool);
	xsk->unaligned = pool->unaligned;
#elif defined(HAVE_XSK_BUFF_ALLOC)
	xsk->headroom = xsk_umem_get_headroom(umem);
	xsk->chunk_size = xsk_umem_get_chunk_size(umem);
#ifdef HAVE_XDP_UMEM_FLAGS
	xsk->unaligned = umem->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG;
#else
	xsk->unaligned = false;
#endif
#else
	xsk->headroom = umem->headroom;
	xsk->chunk_size = umem->chunk_size_nohr + umem->headroom;
#ifdef HAVE_XDP_UMEM_FLAGS
	xsk->unaligned = umem->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG;
#else
	xsk->unaligned = false;
#endif
#endif
}

static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
				   struct xsk_buff_pool *pool,
#else
				   struct xdp_umem *pool,
#endif
				   u16 ix)
{
	struct mlx5e_params *params = &priv->channels.params;
	struct mlx5e_xsk_param xsk;
	struct mlx5e_channel *c;
	int err;

	if (unlikely(mlx5e_xsk_get_pool(&priv->channels.params, &priv->xsk, ix)))
		return -EBUSY;

	if (unlikely(!mlx5e_xsk_is_pool_sane(pool)))
		return -EINVAL;

	err = mlx5e_xsk_map_pool(mlx5_sd_ch_ix_get_dev(priv->mdev, ix), pool);
	if (unlikely(err))
		return err;

	err = mlx5e_xsk_add_pool(&priv->xsk, pool, ix);
	if (unlikely(err))
		goto err_unmap_pool;

	mlx5e_build_xsk_param(pool, &xsk);

	if (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
	    mlx5e_mpwrq_umr_mode(priv->mdev, &xsk) == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) {
		const char *recommendation = is_power_of_2(xsk.chunk_size) ?
			"Upgrade firmware" : "Disable striding RQ";

		mlx5_core_warn(priv->mdev, "Expected slowdown with XSK frame size %u. %s for better performance.\n",
			       xsk.chunk_size, recommendation);
	}

	if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
		/* XSK objects will be created on open. */
		goto validate_closed;
	}

	if (!params->xdp_prog) {
		/* XSK objects will be created when an XDP program is set,
		 * and the channels are reopened.
		 */
		goto validate_closed;
	}

	c = priv->channels.c[ix];

	err = mlx5e_open_xsk(priv, params, &xsk, pool, c);
	if (unlikely(err))
		goto err_remove_pool;

	mlx5e_activate_xsk(c);
	mlx5e_trigger_napi_icosq(c);

	/* Don't wait for WQEs, because the newer xdpsock sample doesn't provide
	 * any Fill Ring entries at the setup stage.
	 */

	mlx5e_rx_res_xsk_update(priv->rx_res, &priv->channels, ix, true);

	mlx5e_deactivate_rq(&c->rq);
	mlx5e_flush_rq(&c->rq, MLX5_RQC_STATE_RDY);

	return 0;

err_remove_pool:
	mlx5e_xsk_remove_pool(&priv->xsk, ix);

err_unmap_pool:
	mlx5e_xsk_unmap_pool(priv, pool);

	return err;

validate_closed:
	/* Check the configuration in advance, rather than fail at a later stage
	 * (in mlx5e_xdp_set or on open) and end up with no channels.
	 */
	if (!mlx5e_validate_xsk_param(params, &xsk, priv->mdev)) {
		err = -EINVAL;
		goto err_remove_pool;
	}

	return 0;
}

static int mlx5e_xsk_disable_locked(struct mlx5e_priv *priv, u16 ix)
{
#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
	struct xsk_buff_pool *pool = mlx5e_xsk_get_pool(&priv->channels.params,
#else
	struct xdp_umem *pool = mlx5e_xsk_get_pool(&priv->channels.params,
#endif
						   &priv->xsk, ix);
	struct mlx5e_channel *c;

	if (unlikely(!pool))
		return -EINVAL;

	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
		goto remove_pool;

	/* XSK RQ and SQ are only created if XDP program is set. */
	if (!priv->channels.params.xdp_prog)
		goto remove_pool;

	c = priv->channels.c[ix];

	mlx5e_activate_rq(&c->rq);
	mlx5e_trigger_napi_icosq(c);
	mlx5e_wait_for_min_rx_wqes(&c->rq, MLX5E_RQ_WQES_TIMEOUT);

	mlx5e_rx_res_xsk_update(priv->rx_res, &priv->channels, ix, false);

	mlx5e_deactivate_xsk(c);
	mlx5e_close_xsk(c);

remove_pool:
	mlx5e_xsk_remove_pool(&priv->xsk, ix);
	mlx5e_xsk_unmap_pool(priv, pool);

	return 0;
}

static int mlx5e_xsk_enable_pool(struct mlx5e_priv *priv,
#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
				   struct xsk_buff_pool *pool,
#else
				   struct xdp_umem *pool,
#endif
				   u16 ix)
{
	int err;

	mutex_lock(&priv->state_lock);
	err = mlx5e_xsk_enable_locked(priv, pool, ix);
	mutex_unlock(&priv->state_lock);

	return err;
}

static int mlx5e_xsk_disable_pool(struct mlx5e_priv *priv, u16 ix)
{
	int err;

	mutex_lock(&priv->state_lock);
	err = mlx5e_xsk_disable_locked(priv, ix);
	mutex_unlock(&priv->state_lock);

	return err;
}

int mlx5e_xsk_setup_pool(struct net_device *dev,
#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
			 struct xsk_buff_pool *pool,
#else
			 struct xdp_umem *pool,
#endif
			 u16 qid)
{
	struct mlx5e_priv *priv = netdev_priv(dev);
	struct mlx5e_params *params = &priv->channels.params;

	if (unlikely(qid >= params->num_channels))
		return -EINVAL;

	return pool ? mlx5e_xsk_enable_pool(priv, pool, qid) :
		      mlx5e_xsk_disable_pool(priv, qid);
}

#ifndef HAVE_XSK_BUFF_ALLOC
int mlx5e_xsk_resize_reuseq(struct xdp_umem *umem, u32 nentries)
{
       struct xdp_umem_fq_reuse *reuseq;

       reuseq = xsk_reuseq_prepare(nentries);
       if (unlikely(!reuseq))
              return -ENOMEM;
       xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq));

       return 0;
}
#endif
#endif /* HAVE_XSK_ZERO_COPY_SUPPORT */
