/*
 * Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES, ALL RIGHTS RESERVED.
 *
 * This software product is a proprietary product of NVIDIA CORPORATION &
 * AFFILIATES (the "Company") and all right, title, and interest in and to the
 * software product, including all associated intellectual property rights, are
 * and shall remain exclusively with the Company.
 *
 * This software product is governed by the End User License Agreement
 * provided with the software product.
 *
 */

#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <unistd.h>

#include "dpdk_live_shared.h"
#include "input.h"
#include "regex_dev.h"
#include "rules_file_utils.h"
#include "run_mode.h"
#include "rxpb_log.h"
#include "utils.h"

#include <doca_buf.h>
#include <doca_buf_inventory.h>
#include <doca_ctx.h>
#include <doca_dev.h>
#include <doca_error.h>
#include <doca_mmap.h>
#include <doca_regex.h>
#include <doca_regex_mempool.h>

#define MAX_MATCHES_PER_JOB 254
#define MAX_BF2_DESCRIPTORS 1024
/* Numer of doca_bufs must be > descriptors so job can be prepped when descriptor queue is full. */
#define TOTAL_DOCA_BUFS	    (MAX_BF2_DESCRIPTORS + 1)
#define MAX_PORTS	    2

struct per_core_globals {
	union {
		struct {
			struct doca_regex_mempool *job_meta;
			struct doca_buf_inventory *buf_inv[MAX_PORTS];
			struct doca_mmap *mmap[MAX_PORTS];
			struct doca_workq *workq;
			uint64_t last_idle_time;
			uint64_t enqueued;
			uint64_t enqueued_db;
			uint64_t dequeued;
			uint64_t job_id;
			uint64_t tx_busy_time;
			uint64_t nb_inflight_jobs;
			int req_offset;
		};
		unsigned char cache_align[CACHE_LINE_SIZE * 2];
	};
};

struct job_metadata {
	uint64_t id;
	uint64_t send_time;
	char *data;
	struct rte_mbuf *mbuf;
	uint16_t tx_port;
	struct doca_buf *doca_buf;
	struct doca_regex_search_result result;
};

struct doca_dev *doca_dev;
struct doca_regex *doca_regex;
static struct per_core_globals *core_vars;
/* Job format specific arrays. */
static uint16_t **input_subset_ids;
static uint64_t *input_job_ids;
static uint32_t input_total_jobs;
static exp_matches_t *input_exp_matches;
static uint32_t batch_sz;
static bool lat_mode;
static bool verbose;

static bool
parse_pci_addr_to_bdf(char const *pci_addr, struct doca_pci_bdf *bdf)
{
	uint32_t tmpu;
	char tmps[4];

	if (pci_addr == NULL || strlen(pci_addr) != 7 || pci_addr[2] != ':' || pci_addr[5] != '.')
		return false;

	tmps[0] = pci_addr[0];
	tmps[1] = pci_addr[1];
	tmps[2] = '\0';
	tmpu = strtoul(tmps, NULL, 16);
	if ((tmpu & 0xFFFFFF00) != 0)
		return false;
	bdf->bus = tmpu;

	tmps[0] = pci_addr[3];
	tmps[1] = pci_addr[4];
	tmps[2] = '\0';
	tmpu = strtoul(tmps, NULL, 16);
	if ((tmpu & 0xFFFFFFE0) != 0)
		return false;
	bdf->device = tmpu;

	tmps[0] = pci_addr[6];
	tmps[1] = '\0';
	tmpu = strtoul(tmps, NULL, 16);
	if ((tmpu & 0xFFFFFFF8) != 0)
		return false;
	bdf->function = tmpu;

	return true;
}

static struct doca_dev *
get_device_from_pci(char const *const pci_address)
{
	struct doca_devinfo *interesting_dev_info = NULL;
	struct doca_pci_bdf desired_bdf = {.raw = 0};
	struct doca_pci_bdf device_bdf = {.raw = 0};
	struct doca_devinfo **devlist = NULL;
	struct doca_dev *device = NULL;
	uint32_t dev_count = 0;
	uint32_t ii;

	if (!parse_pci_addr_to_bdf(pci_address, &desired_bdf)) {
		RXPB_LOG_ERR("Invalid PCI address: \"%s\"", pci_address);
		return NULL;
	}

	if (doca_devinfo_list_create(&devlist, &dev_count) != DOCA_SUCCESS)
		return NULL;

	for (ii = 0; ii < dev_count; ++ii) {
		if ((doca_devinfo_get_pci_addr(devlist[ii], &device_bdf) == DOCA_SUCCESS) &&
		    (desired_bdf.raw == device_bdf.raw)) {
			interesting_dev_info = devlist[ii];
			break;
		}
	}

	if (interesting_dev_info != NULL && doca_dev_open(interesting_dev_info, &device) != DOCA_SUCCESS)
		device = NULL; /* ensure this is NULL on failure */

	doca_devinfo_list_destroy(devlist);

	return device;
}

static int
regex_dev_doca_init(rb_conf *run_conf)
{
	const uint32_t num_cores = run_conf->cores;
	rxp_stats_t *stats;
	uint64_t rules_len;
	uint32_t i, j;
	char *rules;
	int ret;
	doca_error_t doca_ret;
	doca_dev = NULL;
	doca_regex = NULL;
	core_vars = NULL;
	verbose = false;
	batch_sz = run_conf->input_batches;

	if (!run_conf->regex_pcie) {
		RXPB_LOG_ERR("No Regex PCIe addr detected in DPDK opts.");
		return -EINVAL;
	}

	/* Get doca device from input pcie address. */
	doca_dev = get_device_from_pci(run_conf->regex_pcie);
	if (!doca_dev) {
		RXPB_LOG_ERR("No device matching PCI address: %s found.", run_conf->regex_pcie);
		return -EINVAL;
	}

	/* Create a doca regex instance. */
	doca_ret = doca_regex_create(&doca_regex);
	if (doca_ret != DOCA_SUCCESS) {
		RXPB_LOG_ERR("Failed to create a doca regex instance.");
		ret = -ENOMEM;
		goto err_close_doca_dev;
	}

	/* Attach doca_dev to regex */
	doca_ret = doca_ctx_dev_add(doca_regex_as_ctx(doca_regex), doca_dev);
	if (doca_ret != DOCA_SUCCESS) {
		RXPB_LOG_ERR("Failed to attach doca_dev to regex.");
		ret = -EINVAL;
		goto err_destroy_doca_regex;
	}

	doca_ret = doca_regex_set_huge_job_emulation_overlap_size(doca_regex, run_conf->sliding_window);
	if (doca_ret != DOCA_SUCCESS) {
		RXPB_LOG_ERR("Failed to set doca regex overlap size.");
		ret = -EINVAL;
		goto err_destroy_doca_regex;
	}

	/* Load in rules file. */
	ret = util_load_file_to_buffer(run_conf->compiled_rules_file, &rules, &rules_len, 0);
	if (ret) {
		RXPB_LOG_ERR("Failed to read in rules file.");
		goto err_destroy_doca_regex;
	}

	doca_ret = doca_regex_set_hardware_compiled_rules(doca_regex, (uint8_t const *)rules, rules_len);
	rte_free(rules);
	if (doca_ret != DOCA_SUCCESS) {
		RXPB_LOG_ERR("Failed program regex rules.");
		ret = -EINVAL;
		goto err_destroy_doca_regex;
	}

	doca_ret = doca_regex_set_workq_matches_memory_pool_size(doca_regex, MAX_MATCHES_PER_JOB * batch_sz);
	if (doca_ret != DOCA_SUCCESS) {
		RXPB_LOG_ERR("Failed create set doca mempool.");
		goto err_destroy_doca_regex;
	}

	/* Maintain global variables operating on each queue (per lcore). */
	core_vars = rte_zmalloc(NULL, sizeof(struct per_core_globals) * num_cores, 64);
	if (!core_vars) {
		RXPB_LOG_ERR("Mem fail on per core variable setup.");
		ret = -ENOMEM;
		goto err_destroy_doca_regex;
	}

	/* Allocate global data for each core. */
	for (i = 0; i < num_cores; i++) {
		core_vars[i].job_meta = doca_regex_mempool_create(sizeof(struct job_metadata), TOTAL_DOCA_BUFS);
		if (!core_vars[i].job_meta) {
			RXPB_LOG_ERR("Doca job metadata mempool failure.");
			ret = -ENOMEM;
			goto err_free_core_vars;
		}

		if (doca_buf_inventory_create(NULL, TOTAL_DOCA_BUFS, DOCA_BUF_EXTENSION_NONE,
					      &core_vars[i].buf_inv[0]) != DOCA_SUCCESS) {
			RXPB_LOG_ERR("Unable to create buffer inventory");
			ret = -ENOMEM;
			goto err_free_core_vars;
		}

		if (doca_buf_inventory_start(core_vars[i].buf_inv[0]) != DOCA_SUCCESS) {
			RXPB_LOG_ERR("Unable to start buffer inventory");
			ret = -ENOMEM;
			goto err_free_core_vars;
		}

		if (doca_workq_create(MAX_BF2_DESCRIPTORS, &core_vars[i].workq) != DOCA_SUCCESS) {
			RXPB_LOG_ERR("Unable to create workq");
			ret = -ENOMEM;
			goto err_free_core_vars;
		}

		/* Allocate 2nd doca_buf and job if 2 ports are in use. */
		if (run_conf->port2) {
			if (doca_buf_inventory_create(NULL, TOTAL_DOCA_BUFS, DOCA_BUF_EXTENSION_NONE,
						      &core_vars[i].buf_inv[1]) != DOCA_SUCCESS) {
				RXPB_LOG_ERR("Unable to create buffer inventory");
				ret = -ENOMEM;
				goto err_free_core_vars;
			}

			if (doca_buf_inventory_start(core_vars[i].buf_inv[1]) != DOCA_SUCCESS) {
				RXPB_LOG_ERR("Unable to start buffer inventory");
				ret = -ENOMEM;
				goto err_free_core_vars;
			}
		}
	}
	/* Create an mmap per core based on input data. */
	if (run_conf->input_data) {
		for (i = 0; i < num_cores; i++) {
			if (run_conf->remote_mmap_desc == NULL) {
				if (doca_mmap_create(NULL, &core_vars[i].mmap[0]) != DOCA_SUCCESS) {
					RXPB_LOG_ERR("Unable to create memory map");
					ret = -ENOMEM;
					goto err_free_core_vars;
				}

				if (doca_mmap_start(core_vars[i].mmap[0]) != DOCA_SUCCESS) {
					RXPB_LOG_ERR("Unable to start memory map");
					ret = -ENOMEM;
					goto err_free_core_vars;
				}

				if (doca_mmap_dev_add(core_vars[i].mmap[0], doca_dev) != DOCA_SUCCESS) {
					RXPB_LOG_ERR("Unable to add device to memory map");
					ret = -ENOTSUP;
					goto err_free_core_vars;
				}

				if (doca_mmap_populate(core_vars[i].mmap[0], run_conf->input_data,
						       run_conf->input_data_len, sysconf(_SC_PAGESIZE), NULL,
						       NULL) != DOCA_SUCCESS) {
					RXPB_LOG_ERR("Unable to add memory region to memory map");
					ret = -EINVAL;
					goto err_free_core_vars;
				}
			} else {
				if (doca_mmap_create_from_export(NULL, run_conf->remote_mmap_desc,
								 run_conf->remote_mmap_desc_len, doca_dev,
								 &core_vars[i].mmap[0]) != DOCA_SUCCESS) {
					RXPB_LOG_ERR("Unable to create mmap from export");
					ret = -EINVAL;
					goto err_free_core_vars;
				}
			}
		}
	} else {
		/* Running in live mode so need to get an mkey for each rx buffer. */
		const int port_count = run_conf->port2 ? 2 : 1;
		uint32_t buf_len = 0;
		void *buf_addr;
		int p;

		for (i = 0; i < num_cores; i++) {
			for (p = 0; p < port_count; p++) {
				ret = input_get_rx_buffer(run_conf, i, p, &buf_addr, &buf_len);
				if (ret) {
					RXPB_LOG_ERR("Failed to get address of rx buf on port %d.", p);
					ret = -EINVAL;
					goto err_free_core_vars;
				}

				if (doca_mmap_create(NULL, &core_vars[i].mmap[p]) != DOCA_SUCCESS) {
					RXPB_LOG_ERR("Unable to create memory map");
					ret = -ENOMEM;
					goto err_free_core_vars;
				}

				if (doca_mmap_start(core_vars[i].mmap[p]) != DOCA_SUCCESS) {
					RXPB_LOG_ERR("Unable to start memory map");
					ret = -ENOMEM;
					goto err_free_core_vars;
				}

				if (doca_mmap_dev_add(core_vars[i].mmap[p], doca_dev) != DOCA_SUCCESS) {
					RXPB_LOG_ERR("Unable to add device to memory map");
					ret = -ENOTSUP;
					goto err_free_core_vars;
				}

				if (doca_mmap_populate(core_vars[i].mmap[p], buf_addr, buf_len, sysconf(_SC_PAGESIZE),
						       NULL, NULL) != DOCA_SUCCESS) {
					RXPB_LOG_ERR("Unable to add memory region to memory map");
					ret = -EINVAL;
					goto err_free_core_vars;
				}
			}
		}
	}
	/* Start doca RegEx */
	doca_ret = doca_ctx_start(doca_regex_as_ctx(doca_regex));
	if (doca_ret != DOCA_SUCCESS) {
		RXPB_LOG_ERR("Failed to start doca regex.");
		goto err_destroy_doca_regex;
	}

	/** Attach workqs */
	for (i = 0; i < num_cores; i++) {
		if (doca_ctx_workq_add(doca_regex_as_ctx(doca_regex), core_vars[i].workq) != DOCA_SUCCESS) {
			RXPB_LOG_ERR("Unable to attach workq");
			ret = -ENOMEM;
			goto err_free_core_vars;
		}
	}

	verbose = run_conf->verbose;
	if (verbose && run_conf->input_mode == INPUT_REMOTE_MMAP) {
		RXPB_LOG_WARN_REC(run_conf, "Unable to provide verbose matches in remote mmap mode");
		verbose = false;
	}

	if (verbose) {
		ret = regex_dev_open_match_file(run_conf);
		if (ret) {
			RXPB_LOG_ERR("Failed to open verbose file.");
			ret = -ENOTSUP;
			goto err_free_core_vars;
		}
	}

	/* Init min latency stats to large value. */
	for (i = 0; i < num_cores; i++) {
		stats = (rxp_stats_t *)(run_conf->stats->regex_stats[i].custom);
		stats->min_lat = UINT64_MAX;
	}

	/* Grab a copy of job format specific arrays. */
	input_subset_ids = run_conf->input_subset_ids;
	input_job_ids = run_conf->input_job_ids;
	input_total_jobs = run_conf->input_len_cnt;
	input_exp_matches = run_conf->input_exp_matches;
	lat_mode = run_conf->latency_mode;

	return 0;

err_close_verbose_file:
	if (verbose)
		regex_dev_close_match_file(run_conf);
err_free_core_vars:
	for (i = 0; i < num_cores; i++) {
		if (core_vars[i].workq != NULL) {
			doca_ctx_workq_rm(doca_regex_as_ctx(doca_regex), core_vars[i].workq);
			doca_workq_destroy(core_vars[i].workq);
			core_vars[i].workq = NULL;
		}
		doca_regex_mempool_destroy(core_vars[i].job_meta);
		core_vars[i].job_meta = NULL;
		for (j = 0; j < MAX_PORTS; j++) {
			if (core_vars[i].buf_inv[j] != NULL) {
				doca_buf_inventory_stop(core_vars[i].buf_inv[j]);
				doca_buf_inventory_destroy(core_vars[i].buf_inv[j]);
				core_vars[i].buf_inv[j] = NULL;
			}
			if (core_vars[i].mmap[j]) {
				doca_mmap_dev_rm(core_vars[i].mmap[j], doca_dev);
				doca_mmap_stop(core_vars[i].mmap[j]);
				doca_mmap_destroy(core_vars[i].mmap[j]);
				core_vars[i].mmap[j] = NULL;
			}
		}
	}
	rte_free(core_vars);
	core_vars = NULL;
err_destroy_doca_regex:
	doca_ctx_stop(doca_regex_as_ctx(doca_regex));
	doca_regex_destroy(doca_regex);
	doca_regex = NULL;
err_close_doca_dev:
	doca_dev_close(doca_dev);
	doca_dev = NULL;

	return ret;
}

static inline int
regex_dev_doca_get_array_offset(uint64_t job_id)
{
	/* Job ids start at 1 while array starts at 0 so need to decrement before wrap. */
	return (job_id - 1) % input_total_jobs;
}

static void
regex_dev_doca_exp_matches(struct doca_regex_search_result *resp, uint64_t job_id, rxp_stats_t *rxp_stats)
{
	const uint32_t num_matches = resp->detected_matches;
	exp_match_t actual_match[num_matches];
	struct doca_regex_match *match;
	exp_matches_t actual_matches;
	rxp_exp_match_stats_t *stats;
	exp_matches_t *exp_matches;
	int i = 0;

	if (num_matches) {
		for (match = resp->matches; match != NULL;) {
			actual_match[i].rule_id = match->rule_id;
			actual_match[i].start_ptr = match->match_start;
			actual_match[i].length = match->length;
			match = match->next;
			i++;
		}
	}

	actual_matches.num_matches = num_matches;
	actual_matches.matches = &actual_match[0];

	stats = resp->status_flags == DOCA_REGEX_STATUS_SEARCH_FAILED ? &rxp_stats->max_exp : &rxp_stats->exp;
	exp_matches = &input_exp_matches[regex_dev_doca_get_array_offset(job_id)];

	regex_dev_verify_exp_matches(exp_matches, &actual_matches, stats);
}

static void
extract_results(int qid, regex_stats_t *stats, struct doca_event *event, dpdk_egress_t *dpdk_tx)
{
	rxp_stats_t *rxp_stats = (rxp_stats_t *)stats->custom;
	struct doca_regex_match *match, *cur;
	struct doca_regex_search_result *resp;
	struct job_metadata *job_meta;
	uint64_t time;

	stats->rx_valid++;
	resp = (struct doca_regex_search_result *)event->result.ptr;
	job_meta = (struct job_metadata *)event->user_data.ptr;
	/* Get time difference between now and when job was sent. */
	time = rte_get_timer_cycles();
	time -= job_meta->send_time;
	rxp_stats->tot_lat += time;

	if (time < rxp_stats->min_lat)
		rxp_stats->min_lat = time;
	if (time > rxp_stats->max_lat)
		rxp_stats->max_lat = time;

	/* Do expect matches checking on all results irrespective of flags or num_matches. */
	if (input_exp_matches)
		regex_dev_doca_exp_matches(resp, job_meta->id, rxp_stats);
	if (!resp->detected_matches) {
		doca_regex_mempool_put_obj(core_vars[qid].job_meta, job_meta);
		if (dpdk_tx)
			dpdk_live_add_to_tx(dpdk_tx, job_meta->tx_port, job_meta->mbuf);
		doca_buf_refcount_rm(job_meta->doca_buf, NULL);
		return;
	}

	if (resp->status_flags == DOCA_REGEX_STATUS_SEARCH_FAILED) {
		/* Doca does not give reason for fail so just mark as invalid. */
		rxp_stats->rx_invalid++;
	} else {
		stats->rx_buf_match_cnt++;
		stats->rx_total_match += resp->detected_matches;
	}

	for (match = resp->matches; match != NULL;) {
		cur = match;
		match = match->next;
		if (verbose && resp->status_flags != DOCA_REGEX_STATUS_SEARCH_FAILED) {
			uint64_t job_id = job_meta->id;
			char *match_data;
			/* May have to convert the incrementing rule id to user input ID. */
			if (input_job_ids)
				job_id = input_job_ids[regex_dev_doca_get_array_offset(job_id)];
			match_data = job_meta->data + cur->match_start;
			regex_dev_write_to_match_file(qid, job_id, cur->rule_id, cur->match_start, cur->length,
						      match_data);
		}
		doca_regex_mempool_put_obj(resp->matches_mempool, cur);
	}

	resp->matches = NULL;
	if (dpdk_tx)
		dpdk_live_add_to_tx(dpdk_tx, job_meta->tx_port, job_meta->mbuf);
	doca_regex_mempool_put_obj(core_vars[qid].job_meta, job_meta);
	doca_buf_refcount_rm(job_meta->doca_buf, NULL);
}

static int
regex_dev_doca_do_dequeue(int qid, struct per_core_globals *core, regex_stats_t *stats, dpdk_egress_t *dpdk_tx,
			  int wait_on_dequeue)
{
	rxp_stats_t *rxp_stats = (rxp_stats_t *)stats->custom;
	int total_dequeued = 0;
	struct doca_event event;

	for (;;) {
		/* Do not do a dequeue if we are live and have no buffer room to TX. */
		if (dpdk_tx) {
			int port1_cnt = dpdk_tx->port_cnt[PRIM_PORT_IDX];
			int port2_cnt = dpdk_tx->port_cnt[SEC_PORT_IDX];
			/* Don't do a pull if can't process the max size */
			if (port1_cnt + batch_sz > TX_RING_SIZE || port2_cnt + batch_sz > TX_RING_SIZE)
				break;
		}

		doca_error_t const res =
			doca_workq_progress_retrieve(core_vars[qid].workq, &event, DOCA_WORKQ_RETRIEVE_FLAGS_NONE);

		if (res == DOCA_SUCCESS) {
			/* If last time is non 0 then idle timer was started. */
			if (core->last_idle_time) {
				rxp_stats->rx_idle += rte_get_timer_cycles() - core->last_idle_time;
				core->last_idle_time = 0;
			}
			extract_results(qid, stats, &event, dpdk_tx);
			++total_dequeued;
			++core->dequeued;
		} else if (res == DOCA_ERROR_AGAIN) {
			if (core->last_idle_time == 0 && core->enqueued > 0)
				core->last_idle_time = rte_get_timer_cycles();
			if (total_dequeued >= wait_on_dequeue)
				break;
		} else {
			RXPB_LOG_ERR("Job dequeue error.");
			return -EINVAL;
		}
	}

	core->nb_inflight_jobs -= total_dequeued;

	return 0;
}

static int
regex_dev_doca_do_enqueue(int qid, struct job_metadata *job_meta, struct doca_buf *doca_buf, regex_stats_t *stats,
			  dpdk_egress_t *dpdk_tx, bool push_batch)
{
	rxp_stats_t *rxp_stats = (rxp_stats_t *)stats->custom;
	struct per_core_globals *core = &core_vars[qid];
	struct doca_regex_job_search job_request = {
		.base =
			{
				.type = DOCA_REGEX_JOB_SEARCH,
				.flags = DOCA_JOB_FLAGS_NONE,
				.ctx = doca_regex_as_ctx(doca_regex),
				.user_data = {.ptr = job_meta},
			},
		.rule_group_ids = {1, 0, 0, 0},
		.buffer = doca_buf,
		.result = &(job_meta->result),
		.allow_batching = !push_batch,
	};
	bool did_enqueue = false;
	doca_error_t res;
	int ret;

	if (input_subset_ids) {
		const int job_offset = regex_dev_doca_get_array_offset(core->job_id);
		int i;

		for (i = 0; i < MAX_SUBSET_IDS; i++)
			job_request.rule_group_ids[i] = input_subset_ids[job_offset][i];
	}

	ret = 0;
	for (; did_enqueue == false;) {
		res = doca_workq_submit(core_vars[qid].workq, (struct doca_job const *)(&job_request));
		if (res == DOCA_SUCCESS) {
			++(core_vars[qid].enqueued);
			if (push_batch)
				core_vars[qid].enqueued_db = core_vars[qid].enqueued;
			/* Update time in metadata for sent job. */
			job_meta->send_time = rte_get_timer_cycles();
			if (core_vars[qid].tx_busy_time != 0) {
				rxp_stats->tx_busy += job_meta->send_time - core_vars[qid].tx_busy_time;
				core_vars[qid].tx_busy_time = 0;
			}
			did_enqueue = true;
			++(core->nb_inflight_jobs);
		} else if (res != DOCA_ERROR_NO_MEMORY) {
			RXPB_LOG_ERR("Job enqueue error: %s", doca_get_error_string(res));
			ret = -EINVAL;
			break;
		} else if (res == DOCA_ERROR_NO_MEMORY && core_vars[qid].tx_busy_time == 0)
			core_vars[qid].tx_busy_time = rte_get_timer_cycles();

		/* Trigger dequeue when a batch has been sent or the descriptor queues are full. */
		if (push_batch || res == DOCA_ERROR_NO_MEMORY) {
			ret = regex_dev_doca_do_dequeue(qid, core, stats, dpdk_tx,
							(push_batch && lat_mode) ? core->nb_inflight_jobs : 0);
			if (ret < 0)
				break;
		}
	}

	return ret;
}

static int
regex_dev_doca_search(int qid, char *buf, int buf_len, bool push_batch, regex_stats_t *stats)
{
	struct per_core_globals *core = &core_vars[qid];
	struct job_metadata *job_meta;
	struct doca_buf *doca_buf;
	void *mbuf_data;

	/* Get a doca_buf from inventory with mkey from the associated mmap. */
	if (doca_buf_inventory_buf_by_addr(core->buf_inv[0], core->mmap[0], buf, buf_len, &doca_buf) != DOCA_SUCCESS) {
		RXPB_LOG_ERR("Failed to get doca_buf for job.");
		return -ENOMEM;
	}
	doca_buf_get_data(doca_buf, &mbuf_data);
	doca_buf_set_data(doca_buf, mbuf_data, buf_len);

	job_meta = doca_regex_mempool_get_obj(core->job_meta);
	if (!job_meta) {
		RXPB_LOG_ERR("Failed to get metadata for job.");
		return -ENOMEM;
	}

	job_meta->id = ++(core->job_id);
	job_meta->data = buf;
	job_meta->doca_buf = doca_buf;

	return regex_dev_doca_do_enqueue(qid, job_meta, doca_buf, stats, NULL, push_batch);
}

static int
regex_dev_doca_search_live(int qid, struct rte_mbuf *mbuf, int pay_off, uint16_t rx_port, uint16_t tx_port,
			   dpdk_egress_t *dpdk_tx, regex_stats_t *stats)
{
	struct per_core_globals *core = &core_vars[qid];
	struct job_metadata *job_meta;
	struct doca_buf *doca_buf;
	void *mbuf_data;

	job_meta = doca_regex_mempool_get_obj(core->job_meta);
	if (!job_meta) {
		RXPB_LOG_ERR("Failed to get metadata for job.");
		return -ENOMEM;
	}

	job_meta->id = ++(core->job_id);
	job_meta->data = (char *)mbuf->buf_addr + mbuf->data_off + pay_off;
	job_meta->mbuf = mbuf;
	job_meta->tx_port = tx_port;

	/* Get a doca_buf from inventory with mkey from the associated mmap. */
	if (doca_buf_inventory_buf_by_addr(core->buf_inv[rx_port], core->mmap[rx_port], job_meta->data,
					   mbuf->data_len - pay_off, &doca_buf) != DOCA_SUCCESS) {
		RXPB_LOG_ERR("Failed to get doca_buf for job.");
		return -ENOMEM;
	}
	doca_buf_get_data(doca_buf, &mbuf_data);
	doca_buf_set_data(doca_buf, mbuf_data, mbuf->data_len - pay_off);

	job_meta->doca_buf = doca_buf;

	return regex_dev_doca_do_enqueue(qid, job_meta, doca_buf, stats, dpdk_tx, /* push_batch*/ false);
}

static void
regex_dev_doca_force_batch_push(int qid, uint16_t rx_port, dpdk_egress_t *dpdk_tx, regex_stats_t *stats)
{
}

static void
regex_dev_doca_force_batch_pull(int qid, dpdk_egress_t *dpdk_tx, regex_stats_t *stats)
{
	/* Async dequeue is only needed if not in latency mode so set 'wait on' value to 0. */
	if (regex_dev_doca_do_dequeue(qid, &core_vars[qid], stats, dpdk_tx, 0))
		RXPB_LOG_ERR("Dequeue failed.");
}

/* Ensure all ops in flight are received before exiting. */
static void
regex_dev_doca_post_search(int qid, regex_stats_t *stats)
{
	uint64_t start, diff;
	struct doca_event event;

	/*
	 * If run was interrupted there may be jobs 'enqueued' but not actually sent (no doorbell).
	 * Let's only wait on those job we know have been sent via doorbell.
	 * There may be jobs in flight that are missed here but an interrupt is unexpected behaviour.
	 */
	if (force_quit)
		core_vars[qid].enqueued = core_vars[qid].enqueued_db;

	start = rte_rdtsc();
	while (core_vars[qid].enqueued > core_vars[qid].dequeued) {
		doca_error_t const res =
			doca_workq_progress_retrieve(core_vars[qid].workq, &event, DOCA_WORKQ_RETRIEVE_FLAGS_NONE);

		if (res == DOCA_SUCCESS) {
			extract_results(qid, stats, &event, NULL);
			++(core_vars[qid].dequeued);
		}

		/* Prevent infinite loops. */
		diff = rte_rdtsc() - start;
		if (diff > MAX_POST_SEARCH_DEQUEUE_CYCLES) {
			RXPB_LOG_ALERT("Post-processing appears to be in an infinite loop. Breaking...");
			break;
		}
	}
}

static void
regex_dev_doca_clean(rb_conf *run_conf)
{
	const int num_cores = run_conf->cores;
	int i, p;

	for (i = 0; i < num_cores; i++) {
		if (core_vars[i].workq != NULL) {
			doca_ctx_workq_rm(doca_regex_as_ctx(doca_regex), core_vars[i].workq);
			doca_workq_destroy(core_vars[i].workq);
			core_vars[i].workq = NULL;
		}
	}

	if (doca_regex) {
		doca_ctx_stop(doca_regex_as_ctx(doca_regex));
		doca_regex_destroy(doca_regex);
		doca_regex = NULL;
	}

	if (verbose)
		regex_dev_close_match_file(run_conf);

	for (i = 0; i < num_cores; i++) {
		doca_regex_mempool_destroy(core_vars[i].job_meta);
		core_vars[i].job_meta = NULL;
		for (p = 0; p < MAX_PORTS; p++) {
			if (core_vars[i].buf_inv[p] != NULL) {
				doca_buf_inventory_stop(core_vars[i].buf_inv[p]);
				doca_buf_inventory_destroy(core_vars[i].buf_inv[p]);
				core_vars[i].buf_inv[p] = NULL;
			}
			if (core_vars[i].mmap[p]) {
				doca_mmap_dev_rm(core_vars[i].mmap[p], doca_dev);
				doca_mmap_stop(core_vars[i].mmap[p]);
				doca_mmap_destroy(core_vars[i].mmap[p]);
				core_vars[i].mmap[p] = NULL;
			}
		}
	}

	rte_free(core_vars);
	core_vars = NULL;

	if (doca_dev) {
		doca_dev_close(doca_dev);
		doca_dev = NULL;
	}
}

static int
regex_dev_doca_compile(rb_conf *run_conf)
{
	return rules_file_compile_for_rxp(run_conf);
}

int
regex_dev_doca_regex_reg(regex_func_t *funcs)
{
	funcs->init_regex_dev = regex_dev_doca_init;
	funcs->search_regex = regex_dev_doca_search;
	funcs->search_regex_live = regex_dev_doca_search_live;
	funcs->force_batch_push = regex_dev_doca_force_batch_push;
	funcs->force_batch_pull = regex_dev_doca_force_batch_pull;
	funcs->post_search_regex = regex_dev_doca_post_search;
	funcs->clean_regex_dev = regex_dev_doca_clean;
	funcs->compile_regex_rules = regex_dev_doca_compile;

	return 0;
}
