/*
 * A GEM style device manager for PCIe based OpenCL accelerators.
 *
 * Copyright (C) 2021 Xilinx, Inc. All rights reserved.
 *
 * Authors: Chien-Wei Lan <chienwei@xilinx.com>
 *
 * This software is licensed under the terms of the GNU General Public
 * License version 2, as published by the Free Software Foundation, and
 * may be copied, distributed, and modified under those terms.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#include "../xocl_drv.h"
#include "kds_client.h"
#include "xrt_ert.h"

//#define	SCHED_VERBOSE	1

/* XRT ERT timer macros */
/* A low frequence timer for ERT to check if command timeout */
#define ERT_TICKS_PER_SEC	2
#define ERT_TIMER		(HZ / ERT_TICKS_PER_SEC) /* in jiffies */
#define ERT_EXEC_DEFAULT_TTL	(5UL * ERT_TICKS_PER_SEC)

#ifdef SCHED_VERBOSE
#define	ERTUSER_ERR(ert_user, fmt, arg...)	\
	xocl_err(ert_user->dev, fmt "", ##arg)
#define	ERTUSER_WARN(ert_user, fmt, arg...)	\
	xocl_warn(ert_user->dev, fmt "", ##arg)
#define	ERTUSER_INFO(ert_user, fmt, arg...)	\
	xocl_info(ert_user->dev, fmt "", ##arg)
#define	ERTUSER_DBG(ert_user, fmt, arg...)	\
	xocl_info(ert_user->dev, fmt "", ##arg)

#else
#define	ERTUSER_ERR(ert_user, fmt, arg...)	\
	xocl_err(ert_user->dev, fmt "", ##arg)
#define	ERTUSER_WARN(ert_user, fmt, arg...)	\
	xocl_warn(ert_user->dev, fmt "", ##arg)
#define	ERTUSER_INFO(ert_user, fmt, arg...)	\
	xocl_info(ert_user->dev, fmt "", ##arg)
#define	ERTUSER_DBG(ert_user, fmt, arg...)
#endif

#define sched_debug_packet(packet, size)				\
({									\
	int i;								\
	u32 *data = (u32 *)packet;					\
	for (i = 0; i < size; ++i)					    \
		DRM_INFO("packet(0x%p) execbuf[%d] = 0x%x\n", data, i, data[i]); \
})

struct ert_user_event {
	struct list_head	ev_entry;
	void			*client;
	struct completion	cmp;
};


struct ert_user_queue {
	struct list_head	head;
	uint32_t		num;
};


struct ert_cu_stat {
	u64		usage;
	u32		inflight;
};


struct xocl_ert_user {
	struct device		*dev;
	struct platform_device	*pdev;
	bool			polling_mode;
	struct mutex		lock;
	struct kds_ert		ert;

	/* Configure dynamically */
	unsigned int		num_slots;
	unsigned int		slot_size;
	bool			is_configured;
	bool			ctrl_busy;
	struct xocl_ert_sched_privdata ert_cfg_priv;

	struct ert_user_queue	pq;
	struct ert_user_queue	pq_ctrl;

	spinlock_t		pq_lock;
	/*
	 * Pending Q is used in thread that is submitting CU cmds.
	 * Other Qs are used in thread that is completing them.
	 * In order to prevent false sharing, they need to be in different
	 * cache lines. Hence we add a "padding" in between (assuming 128-byte
	 * is big enough for most CPU architectures).
	 */
	u64			padding[16];
	/* run queue */
	struct ert_user_queue	rq;
	struct ert_user_queue	rq_ctrl;


	struct semaphore	sem;
	/* submitted queue */
	struct ert_user_queue	sq;

	struct ert_user_queue	cq;

	u32			stop;
	bool			bad_state;

	struct mutex		ev_lock;
	struct list_head	events;

	struct timer_list	timer;
	atomic_t		tick;

	struct task_struct	*thread;

	uint32_t		ert_dmsg;
	uint32_t		echo;
	uint32_t		intr;

	/* TODO: Before we have partition queue, we need
	 * record CU statistics in this place.
	 */
	struct ert_cu_stat	cu_stat[MAX_CUS];
	uint32_t		num_cus;

	struct completion	comp;

	struct ert_queue	*queue;
	/* ert validate result cache*/
	struct ert_validate_cmd ert_valid;
	struct ert_access_valid_cmd ert_access_valid;
};

static void ert_submit_exit_cmd(struct xocl_ert_user *ert_user);
static inline void ert_user_cmd_submit(struct xocl_ert_user *ert_user, struct xrt_ert_command *ecmd);
static void free_ert_user_event(struct ert_user_event *event);

static ssize_t clock_timestamp_show(struct device *dev,
			   struct device_attribute *attr, char *buf)
{
	struct xocl_ert_user *ert_user = platform_get_drvdata(to_platform_device(dev));

	return sprintf(buf, "%u\n", ert_user->ert_valid.timestamp);
}

static DEVICE_ATTR_RO(clock_timestamp);

static ssize_t snap_shot_show(struct device *dev,
			   struct device_attribute *attr, char *buf)
{
	struct xocl_ert_user *ert_user = platform_get_drvdata(to_platform_device(dev));

	return sprintf(buf, "pq:%d pq_ctrl:%d,  rq:%d, rq_ctrl:%d, sq:%d cq:%d\n", ert_user->pq.num, ert_user->pq_ctrl.num,
			ert_user->rq.num, ert_user->rq_ctrl.num, ert_user->sq.num, ert_user->cq.num);
}

static DEVICE_ATTR_RO(snap_shot);

static ssize_t ert_dmsg_store(struct device *dev,
	struct device_attribute *da, const char *buf, size_t count)
{
	struct xocl_ert_user *ert_user = platform_get_drvdata(to_platform_device(dev));
	u32 val;

	mutex_lock(&ert_user->lock);
	if (kstrtou32(buf, 10, &val) == -EINVAL || val > 2) {
		xocl_err(&to_platform_device(dev)->dev,
			"usage: echo 0 or 1 > ert_dmsg");
		return -EINVAL;
	}

	ert_user->ert_dmsg = val;

	mutex_unlock(&ert_user->lock);
	return count;
}
static DEVICE_ATTR_WO(ert_dmsg);

static ssize_t ert_echo_store(struct device *dev,
	struct device_attribute *da, const char *buf, size_t count)
{
	struct xocl_ert_user *ert_user = platform_get_drvdata(to_platform_device(dev));
	u32 val;

	mutex_lock(&ert_user->lock);
	if (kstrtou32(buf, 10, &val) == -EINVAL || val > 2) {
		xocl_err(&to_platform_device(dev)->dev,
			"usage: echo 0 or 1 > ert_echo");
		return -EINVAL;
	}

	ert_user->echo = val;

	mutex_unlock(&ert_user->lock);
	return count;
}
static DEVICE_ATTR_WO(ert_echo);

static ssize_t ert_intr_store(struct device *dev,
	struct device_attribute *da, const char *buf, size_t count)
{
	struct xocl_ert_user *ert_user = platform_get_drvdata(to_platform_device(dev));
	u32 val;

	mutex_lock(&ert_user->lock);
	if (kstrtou32(buf, 10, &val) == -EINVAL || val > 2) {
		xocl_err(&to_platform_device(dev)->dev,
			"usage: echo 0 or 1 > ert_intr");
		return -EINVAL;
	}

	ert_user->intr = val;

	mutex_unlock(&ert_user->lock);
	return count;
}
static DEVICE_ATTR_WO(ert_intr);

static ssize_t mb_sleep_store(struct device *dev,
	struct device_attribute *da, const char *buf, size_t count)
{
	struct platform_device *pdev = to_platform_device(dev);
	struct xocl_ert_user *ert_user = platform_get_drvdata(pdev);
	xdev_handle_t xdev = xocl_get_xdev(pdev);
	u32 go_sleep;

	if (kstrtou32(buf, 10, &go_sleep) == -EINVAL || go_sleep > 2) {
		xocl_err(&to_platform_device(dev)->dev,
			"usage: echo 0 or 1 > mb_sleep");
		return -EINVAL;
	}

	if (go_sleep) {
		xocl_gpio_cfg(xdev, MB_WAKEUP_CLR);
		ert_submit_exit_cmd(ert_user);
		xocl_gpio_cfg(xdev, MB_SLEEP);
	} else
		xocl_gpio_cfg(xdev, MB_WAKEUP);

	return count;
}

static ssize_t mb_sleep_show(struct device *dev,
			   struct device_attribute *attr, char *buf)
{
	struct platform_device *pdev = to_platform_device(dev);
	xdev_handle_t xdev = xocl_get_xdev(pdev);

	return sprintf(buf, "%d", xocl_gpio_cfg(xdev, MB_STATUS));
}

static DEVICE_ATTR_RW(mb_sleep);

static ssize_t cq_read_cnt_show(struct device *dev,
			   struct device_attribute *attr, char *buf)
{
	struct xocl_ert_user *ert_user = platform_get_drvdata(to_platform_device(dev));

	return sprintf(buf, "%d\n", ert_user->ert_valid.cq_read_single);
}

static DEVICE_ATTR_RO(cq_read_cnt);

static ssize_t cq_write_cnt_show(struct device *dev,
			   struct device_attribute *attr, char *buf)
{
	struct xocl_ert_user *ert_user = platform_get_drvdata(to_platform_device(dev));

	return sprintf(buf, "%d\n", ert_user->ert_valid.cq_write_single);
}

static DEVICE_ATTR_RO(cq_write_cnt);

static ssize_t cu_read_cnt_show(struct device *dev,
			   struct device_attribute *attr, char *buf)
{
	struct xocl_ert_user *ert_user = platform_get_drvdata(to_platform_device(dev));

	return sprintf(buf, "%d\n", ert_user->ert_valid.cu_read_single);
}

static DEVICE_ATTR_RO(cu_read_cnt);

static ssize_t cu_write_cnt_show(struct device *dev,
			   struct device_attribute *attr, char *buf)
{
	struct xocl_ert_user *ert_user = platform_get_drvdata(to_platform_device(dev));

	return sprintf(buf, "%d\n", ert_user->ert_valid.cu_write_single);
}

static DEVICE_ATTR_RO(cu_write_cnt);

static ssize_t stat_show(struct device *dev,
			 struct device_attribute *attr, char *buf)
{
	struct xocl_ert_user *ert_user = platform_get_drvdata(to_platform_device(dev));
	ssize_t sz = 0;
	char *fmt = "%lld %d\n";
	int i;

	/* formatted CU statistics one CU per line */
	for (i = 0; i < ert_user->num_cus; i++) {
		sz += scnprintf(buf+sz, PAGE_SIZE - sz, fmt,
				ert_user->cu_stat[i].usage,
				ert_user->cu_stat[i].inflight);
	}

	return sz;
}
static DEVICE_ATTR_RO(stat);

static ssize_t data_integrity_show(struct device *dev,
			   struct device_attribute *attr, char *buf)
{
	struct xocl_ert_user *ert_user = platform_get_drvdata(to_platform_device(dev));
	uint32_t ret = 1;

	ret &= ert_user->ert_access_valid.h2h_access;
	ret &= ert_user->ert_access_valid.d2h_access;
	ret &= ert_user->ert_access_valid.h2d_access;
	ret &= ert_user->ert_access_valid.d2d_access;
	ret &= ert_user->ert_access_valid.d2cu_access;
	ret &= ert_user->ert_access_valid.wr_test;

	ERTUSER_INFO(ert_user, "h2h(%d), h2d(%d), d2d(%d), d2h(%d), d2cu(%d), wr_test(%d)\n"
			, ert_user->ert_access_valid.h2h_access
			, ert_user->ert_access_valid.h2d_access
			, ert_user->ert_access_valid.d2d_access
			, ert_user->ert_access_valid.d2h_access
			, ert_user->ert_access_valid.d2cu_access
			, ert_user->ert_access_valid.wr_test);

	return sprintf(buf, "%d\n", ret);
}

static DEVICE_ATTR_RO(data_integrity);

static struct attribute *ert_user_attrs[] = {
	&dev_attr_clock_timestamp.attr,
	&dev_attr_ert_dmsg.attr,
	&dev_attr_snap_shot.attr,
	&dev_attr_ert_echo.attr,
	&dev_attr_ert_intr.attr,
	&dev_attr_mb_sleep.attr,
	&dev_attr_cq_read_cnt.attr,
	&dev_attr_cq_write_cnt.attr,
	&dev_attr_cu_read_cnt.attr,
	&dev_attr_cu_write_cnt.attr,
	&dev_attr_stat.attr,
	&dev_attr_data_integrity.attr,
	NULL,
};

static struct attribute_group ert_user_attr_group = {
	.attrs = ert_user_attrs,
};

static inline void
ert_queue_poll(struct xocl_ert_user *ert_user)
{
	struct ert_queue *queue = ert_user->queue;

	if (!queue)
		return;

	queue->func->poll(queue->handle);
}

static inline int
ert_queue_submit(struct xocl_ert_user *ert_user, struct xrt_ert_command *ecmd)
{
	struct ert_queue *queue = ert_user->queue;

	if (!queue)
		return -ENODEV;

	return queue->func->submit(ecmd, queue->handle);
}

static inline void
ert_queue_intc_config(struct xocl_ert_user *ert_user, bool enable)
{
	struct ert_queue *queue = ert_user->queue;

	if (!queue) {
		ERTUSER_ERR(ert_user, "Couldn't find queue %s\n", __func__);
		return;
	}

	queue->func->intc_config(enable, queue->handle);
}

static inline int
ert_config_queue(struct xocl_ert_user *ert_user, uint32_t slot_size, bool polling)
{
	struct ert_queue *queue = ert_user->queue;

	if (!queue)
		return -ENODEV;

	return queue->func->queue_config(slot_size, polling, ert_user, queue->handle);
}

static inline uint32_t
ert_queue_max_slot_num(struct xocl_ert_user *ert_user)
{
	struct ert_queue *queue = ert_user->queue;

	if (!queue)
		return 0;

	return queue->func->max_slot_num(queue->handle);
}

static inline void
ert_queue_abort(struct xocl_ert_user *ert_user, void *client)
{
	struct ert_queue *queue = ert_user->queue;

	if (!queue)
		return;

	queue->func->abort(client, queue->handle);
}

static int ert_user_bulletin(struct platform_device *pdev, struct ert_cu_bulletin *brd)
{
	struct xocl_ert_user *ert_user = platform_get_drvdata(pdev);
	xdev_handle_t xdev = xocl_get_xdev(pdev);
	int ret = 0;

	if (!brd)
		return -EINVAL;

	brd->sta.configured = ert_user->is_configured;
	brd->cap.cu_intr = (xocl_gpio_cfg(xdev, MB_STATUS) != -ENODEV) ? 1 : 0;

	return ret;
}

static void ert_user_cmd_complete(struct xrt_ert_command *ecmd, void *core)
{
	struct xocl_ert_user *ert_user = ((struct xocl_ert_user *)(core));

	list_add_tail(&ecmd->list, &ert_user->cq.head);
	--ert_user->sq.num;
	++ert_user->cq.num;
}

static void ert_user_exit_cmd_complete(struct xrt_ert_command *ecmd, void *core)
{
	struct xocl_ert_user *ert_user = ((struct xocl_ert_user *)(core));

	complete(&ert_user->comp);

	list_add_tail(&ecmd->list, &ert_user->cq.head);
	--ert_user->sq.num;
	++ert_user->cq.num;
}

static void ert_user_cmd_notify(void *core)
{
	struct xocl_ert_user *ert_user = ((struct xocl_ert_user *)(core));

	up(&ert_user->sem);
}

static void
ert_user_free_payload(void *payload)
{
	kfree(payload);
}


static struct ert_packet *ert_user_alloc_epkt(enum ert_cmd_opcode opcode, u32 packet_size)
{
	struct ert_packet *epkt = kzalloc(packet_size, GFP_KERNEL);

	if (!epkt)
		return NULL;

	epkt->opcode = opcode;
	epkt->state = ERT_CMD_STATE_NEW;

	return epkt;
}

static struct xrt_ert_command *ert_user_alloc_ecmd(void *payload, u32 payload_size_in_bytes, u32 response_size)
{
	struct xrt_ert_command *ecmd = kzalloc(sizeof(struct xrt_ert_command)+response_size, GFP_KERNEL);

	if (!ecmd)
		return NULL;

	ecmd->payload = (void *)payload;
	ecmd->payload_size = payload_size_in_bytes/sizeof(u32);
	ecmd->cb.complete = ert_user_cmd_complete;

	ecmd->cb.notify = ert_user_cmd_notify;
	ecmd->response_size = response_size;

	return ecmd;
}

static void ert_submit_exit_cmd(struct xocl_ert_user *ert_user)
{
	u32 response_size = 0, payload_size = ert_user->slot_size;
	struct ert_packet *epkt = ert_user_alloc_epkt(ERT_EXIT, payload_size);
	struct xrt_ert_command *ecmd = NULL;

	if (!epkt)
		return;

	ERTUSER_WARN(ert_user, "%s\n", __func__);

	ecmd = ert_user_alloc_ecmd(epkt, payload_size, response_size);

	if (!ecmd) {
		kfree(epkt);
		return;
	}

	ecmd->cb.complete = ert_user_exit_cmd_complete;
	ecmd->cb.free_payload = ert_user_free_payload;

	ert_user_cmd_submit(ert_user, ecmd);
}


static void ert_submit_data_cmd(struct xocl_ert_user *ert_user)
{
	u32 response_size = ert_user->slot_size, payload_size = ert_user->slot_size;
	struct ert_packet *epkt = ert_user_alloc_epkt(ERT_ACCESS_TEST, payload_size);
	struct xrt_ert_command *ecmd = NULL;

	if (!epkt)
		return;

	ecmd = ert_user_alloc_ecmd(epkt, payload_size, response_size);

	if (!ecmd) {
		kfree(epkt);
		return;
	}

	ecmd->cb.free_payload = ert_user_free_payload;

	ert_user_cmd_submit(ert_user, ecmd);
}

static void ert_submit_data_cmds(struct xocl_ert_user *ert_user)
{
	int i = 1;

	for (; i < ert_user->num_slots; ++i)
		ert_submit_data_cmd(ert_user);
}


static int ert_user_enable(struct platform_device *pdev, bool enable)
{
	struct xocl_ert_user *ert_user = platform_get_drvdata(pdev);
	xdev_handle_t xdev = xocl_get_xdev(pdev);
	int ret = 0;

	if (enable) {
		xocl_gpio_cfg(xdev, MB_WAKEUP);

		ert_queue_intc_config(ert_user, true);
		xocl_gpio_cfg(xdev, INTR_TO_ERT);
	} else {
		xocl_gpio_cfg(xdev, MB_WAKEUP_CLR);
		ert_submit_exit_cmd(ert_user);
		xocl_gpio_cfg(xdev, MB_SLEEP);

		wait_for_completion(&ert_user->comp);

		ert_queue_intc_config(ert_user, false);
		xocl_gpio_cfg(xdev, INTR_TO_CU);
	}

	return ret;
}

static void ert_user_init_queue(struct platform_device *pdev, void *core)
{
	struct xocl_ert_user *ert_user = platform_get_drvdata(pdev);
	struct ert_queue *queue = (struct ert_queue *)core;

	ert_user->queue = queue;
	ert_user->num_slots = ert_queue_max_slot_num(ert_user);
}

static struct xocl_ert_user_funcs ert_user_ops = {
	.bulletin = ert_user_bulletin,
	.enable = ert_user_enable,
	.init_queue = ert_user_init_queue,
};

static void ert_user_submit(struct kds_ert *ert, struct kds_command *xcmd);


/*
 * cmd_opcode() - Command opcode
 *
 * @cmd: Command object
 * Return: Opcode of command
 */
static inline u32
epkt_cmd_opcode(struct ert_packet *epkt)
{
	return epkt->opcode;
}

static inline u32
cmd_opcode(struct xrt_ert_command *ecmd)
{
	struct ert_packet *epkt = (struct ert_packet *)ecmd->payload;

	return epkt_cmd_opcode(epkt);
}

static void ert_free_cmd(struct xrt_ert_command *ecmd)
{
	if (ecmd->cb.free_payload)
		ecmd->cb.free_payload(ecmd->payload);

	kfree(ecmd);
}

#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0)
static void ert_timer(unsigned long data)
{
	struct xocl_ert_user *ert_user = (struct xocl_ert_user *)data;
#else
static void ert_timer(struct timer_list *t)
{
	struct xocl_ert_user *ert_user = from_timer(ert_user, t, timer);
#endif

	atomic_inc(&ert_user->tick);

	mod_timer(&ert_user->timer, jiffies + ERT_TIMER);
}

static inline bool ert_ctrl_slot_cmd(struct xrt_ert_command *ecmd)
{
	bool ret;

	switch (cmd_opcode(ecmd)) {
	case ERT_EXIT:
	case ERT_CONFIGURE:
	case ERT_SK_CONFIG:
	case ERT_CU_STAT:
	case ERT_CLK_CALIB:
	case ERT_MB_VALIDATE:
	case ERT_ACCESS_TEST_C:
		ret = true;
		break;
	default:
		ret = false;
	}

	return ret;
}

static inline struct ert_user_event *
first_event_client_or_null(struct xocl_ert_user *ert_user)
{
	struct ert_user_event *curr = NULL;

	if (list_empty(&ert_user->events))
		return NULL;

	mutex_lock(&ert_user->ev_lock);
	if (list_empty(&ert_user->events))
		goto done;

	curr = list_first_entry(&ert_user->events, struct ert_user_event, ev_entry);

done:
	mutex_unlock(&ert_user->ev_lock);
	return curr;
}

static int ert_cfg_cmd(struct xocl_ert_user *ert_user, struct xrt_ert_command *ecmd)
{
	xdev_handle_t xdev = xocl_get_xdev(ert_user->pdev);
	uint32_t *cdma = xocl_rom_cdma_addr(xdev);
	unsigned int dsa = ert_user->ert_cfg_priv.dsa;
	unsigned int major = ert_user->ert_cfg_priv.major;
	struct ert_configure_cmd *cfg = (struct ert_configure_cmd *)ecmd->xcmd->execbuf;
	bool dataflow_enabled = cfg->dataflow;
	unsigned int ert_num_slots = 0, slot_size = 0, max_slot_num;
	uint64_t cq_range = ert_user->queue->size;

	if (cmd_opcode(ecmd) != ERT_CONFIGURE)
		return -EINVAL;

	if (major > 3) {
		ERTUSER_ERR(ert_user, "Unknown ERT major version\n");
		return -EINVAL;
	}

	ERTUSER_DBG(ert_user, "dsa52 = %d", dsa);

	// Mark command as control command to force slot 0 execution
	/*  1. cfg->slot_size need to be 32-bit aligned
	 *  2. the slot num max: 128
	 */
	ERTUSER_DBG(ert_user, "configuring scheduler cq_size(%lld)\n", cq_range);
	if (cq_range == 0 || cfg->slot_size == 0) {
		ERTUSER_ERR(ert_user, "should not have zeroed value of cq_size=%lld, slot_size=%d",
		    cq_range, cfg->slot_size);
		return -EINVAL;
	} else if (!IS_ALIGNED(cfg->slot_size, 4)) {
		ERTUSER_ERR(ert_user, "slot_size should be 4 bytes aligned, slot_size=%d",
		   cfg->slot_size);
		return -EINVAL;
	}

	slot_size = cfg->slot_size;

	max_slot_num = ert_queue_max_slot_num(ert_user);
	if (slot_size < (cq_range / max_slot_num))
		slot_size = cq_range / max_slot_num;

	ert_num_slots = cq_range / slot_size;

	if (!dataflow_enabled && cfg->cu_dma && ert_num_slots > 32) {
		// Max slot size is 32 because of cudma bug
		ERTUSER_INFO(ert_user, "Limitting CQ size to 32 due to ERT CUDMA bug\n");
		slot_size = cq_range / 32;
	}

	cfg->slot_size = slot_size;

	if (dataflow_enabled)
		ERTUSER_INFO(ert_user, "configuring embedded scheduler dataflow mode\n");

	if (XDEV(xdev)->priv.flags & XOCL_DSAFLAG_CUDMA_OFF)
		cfg->cu_dma = 0;

	cfg->dsa52 = dsa;
	cfg->cdma = cdma ? 1 : 0;
	cfg->dmsg = ert_user->ert_dmsg;
	cfg->echo = ert_user->echo;
	cfg->intr = ert_user->intr;

	// The KDS side of of the scheduler is now configured.  If ERT is
	// enabled, then the configure command will be started asynchronously
	// on ERT.  The shceduler is not marked configured until ERT has
	// completed (exec_finish_cmd); this prevents other processes from
	// submitting commands to same xclbin.  However we must also stop
	// other processes from submitting configure command on this same
	// xclbin while ERT asynchronous configure is running.
	//exec->configure_active = true;

	ERTUSER_INFO(ert_user, "scheduler config ert(%d), dataflow(%d), cudma(%d), cuisr(%d)\n"
		, cfg->ert
		, cfg->dataflow
		, cfg->cu_dma ? 1 : 0
		, cfg->cu_isr ? 1 : 0);

	return 0;
}

static int
ert_cfg_host(struct xocl_ert_user *ert_user, struct xrt_ert_command *ecmd)
{
	int ret = 0;
	struct ert_configure_cmd *cfg = (struct ert_configure_cmd *)ecmd->xcmd->execbuf;

	BUG_ON(cmd_opcode(ecmd) != ERT_CONFIGURE);

	ret = ert_config_queue(ert_user, cfg->slot_size, cfg->polling);
	if (ret)
		return ret;

	ert_user->slot_size = cfg->slot_size;

	ert_user->num_slots = ert_user->queue->size / cfg->slot_size;

	ert_user->polling_mode = cfg->polling;
	/* if polling, disable intc, vice versa */
	ert_queue_intc_config(ert_user, !ert_user->polling_mode);

	memset(ert_user->cu_stat, 0, cfg->num_cus * sizeof(struct ert_cu_stat));

	ert_user->num_cus = cfg->num_cus;

	ERTUSER_INFO(ert_user, "scheduler config ert completed, polling_mode(%d), slots(%d)\n"
		 , ert_user->polling_mode
		 , ert_user->num_slots);

	return 0;
}

static inline bool
ert_special_cmd(struct xrt_ert_command *ecmd)
{
	bool ret = ert_ctrl_slot_cmd(ecmd);

	switch (cmd_opcode(ecmd)) {
	case ERT_ACCESS_TEST:
	case ERT_SK_START:
		ret = true;
		break;
	default:
		break;
	}

	return ret;
}

static inline void
ert_post_process(struct xocl_ert_user *ert_user, struct xrt_ert_command *ecmd)
{
	struct ert_access_valid_cmd *cmd = NULL;
	uint32_t offset = 0;

	if (likely(!ert_special_cmd(ecmd)))
		return;

	ERTUSER_DBG(ert_user, "%s %d", __func__, cmd_opcode(ecmd));
	switch (cmd_opcode(ecmd)) {
	case ERT_CONFIGURE:
		ert_user->is_configured = true;
		break;
	case ERT_MB_VALIDATE:
	case ERT_CLK_CALIB:
		memcpy(&ert_user->ert_valid, ecmd->response, ecmd->response_size);
		break;
	case ERT_ACCESS_TEST:
	case ERT_ACCESS_TEST_C:
		cmd = (struct ert_access_valid_cmd *)ecmd->response;
		for (offset = sizeof(struct ert_access_valid_cmd); offset < ert_user->slot_size; offset++) {
			u32 val = ecmd->response[offset/sizeof(u32)];
			if (val != DEVICE_RW_PATTERN) {
				cmd->d2h_access = 0;
				ERTUSER_ERR(ert_user, "Device -> Host data integrity failed  offset 0x%x val %x\n", offset, val);
				break;
			}
		}
		ert_user->ert_access_valid.h2h_access &= cmd->h2h_access;
		ert_user->ert_access_valid.h2d_access &= cmd->h2d_access;
		ert_user->ert_access_valid.d2d_access &= cmd->d2d_access;
		ert_user->ert_access_valid.d2h_access &= cmd->d2h_access;
		ert_user->ert_access_valid.d2cu_access &= cmd->d2cu_access;
		ert_user->ert_access_valid.wr_test &= cmd->wr_test;
		break;
	case ERT_CU_STAT:
	case ERT_SK_START:
		memcpy(ecmd->xcmd->u_execbuf, ecmd->response, ecmd->response_size);
		break;
	default:
		break;
	}
}

static void
ert_init_access_test(struct xocl_ert_user *ert_user)
{
	ert_user->ert_access_valid.h2h_access = 1;
	ert_user->ert_access_valid.d2h_access = 1;
	ert_user->ert_access_valid.h2d_access = 1;
	ert_user->ert_access_valid.d2d_access = 1;
	ert_user->ert_access_valid.d2cu_access = 1;
	ert_user->ert_access_valid.wr_test  = 1;
}

static inline bool
ert_pre_process(struct xocl_ert_user *ert_user, struct xrt_ert_command *ecmd)
{
	bool bad_cmd = false;

	switch (cmd_opcode(ecmd)) {
	case ERT_EXEC_WRITE:
	case ERT_START_KEY_VAL:
	case ERT_START_CU:
	case ERT_SK_START:
		BUG_ON(ert_user->ctrl_busy);
#if KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE
	#if defined(CONFIG_SUSE_KERNEL)
		#if SLE_VERSION(SUSE_VERSION, SUSE_PATCHLEVEL, SUSE_AUXRELEASE) >= SLE_VERSION(15, 2, 0)
			__attribute__ ((__fallthrough__));
		#else
			__attribute__ ((fallthrough));
		#endif
	#elif defined(RHEL_RELEASE_CODE)
		#if RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8, 4)
			__attribute__ ((__fallthrough__));
		#else
			__attribute__ ((fallthrough));
		#endif
	#else
		__attribute__ ((fallthrough));
	#endif
#else
	__attribute__ ((__fallthrough__));
#endif
	case ERT_CLK_CALIB:
	case ERT_SK_CONFIG:
	case ERT_CU_STAT:
	case ERT_MB_VALIDATE:
	case ERT_ACCESS_TEST:
		BUG_ON(!ert_user->is_configured);
		bad_cmd = false;
		break;
	case ERT_ACCESS_TEST_C:
		BUG_ON(!ert_user->is_configured);
		ert_init_access_test(ert_user);
		ert_submit_data_cmds(ert_user);
		bad_cmd = false;
		break;			
	case ERT_CONFIGURE:
		if (ert_cfg_cmd(ert_user, ecmd))
			bad_cmd = true;
		break;
	case ERT_EXIT:
		bad_cmd = false;
		break;
	default:
		bad_cmd = true;
	}

	return bad_cmd;
}

/**
 * process_ert_cq() - Process cmd witch is completed
 * @ert_user: Target XRT CU
 */
static inline void process_ert_cq(struct xocl_ert_user *ert_user)
{
	struct kds_command *xcmd;
	struct xrt_ert_command *ecmd;

	if (!ert_user->cq.num)
		return;

	ERTUSER_DBG(ert_user, "-> %s\n", __func__);

	while (ert_user->cq.num) {
		ecmd = list_first_entry(&ert_user->cq.head, struct xrt_ert_command, list);
		list_del(&ecmd->list);
		xcmd = ecmd->xcmd;
		if (ert_ctrl_slot_cmd(ecmd))
			ert_user->ctrl_busy = false;

		ert_post_process(ert_user, ecmd);
		if (xcmd) {
			if ((cmd_opcode(ecmd) == ERT_START_CU
			|| cmd_opcode(ecmd) == ERT_EXEC_WRITE
			|| cmd_opcode(ecmd) == ERT_START_KEY_VAL) &&
				ecmd->complete_entry.hdr.cstate == KDS_COMPLETED) {
				ert_user->cu_stat[xcmd->cu_idx].inflight--;
				ert_user->cu_stat[xcmd->cu_idx].usage++;
			}
			set_xcmd_timestamp(xcmd, ecmd->complete_entry.hdr.cstate);
			xcmd->cb.notify_host(xcmd, ecmd->complete_entry.hdr.cstate);
			xcmd->cb.free(xcmd);
		}
		ert_free_cmd(ecmd);
		--ert_user->cq.num;
	}

	ERTUSER_DBG(ert_user, "<- %s\n", __func__);
}

/**
 * process_ert_sq() - Process cmd witch is submitted
 * @ert_user: Target XRT CU
 */

static inline
void process_ert_sq(struct xocl_ert_user *ert_user, struct ert_user_event *ev_client)
{
	if (!ert_user->sq.num)
		return;

	ert_queue_poll(ert_user);

	if (unlikely(ev_client))
		ert_queue_abort(ert_user, ev_client->client);
}

/**
 * process_ert_rq() - Process run queue
 * @ert_user: Target XRT ERT
 * @rq: Target running queue
 *
 * Return: return 0 if run queue is empty or no available slot
 *	   Otherwise, return 1
 */
static inline
int process_ert_rq(struct xocl_ert_user *ert_user, struct ert_user_queue *rq, struct ert_user_event *ev_client)
{
	struct xrt_ert_command *ecmd, *next;
	struct ert_packet *epkt = NULL;
	bool bad_cmd = false;

	if (!rq->num)
		return 0;

	ERTUSER_DBG(ert_user, "%s =>\n", __func__);

	list_for_each_entry_safe(ecmd, next, &rq->head, list) {
		struct kds_command *xcmd = ecmd->xcmd;

		if (unlikely(ert_user->bad_state || (xcmd && ev_client && ev_client->client == xcmd->client))) {
			ERTUSER_ERR(ert_user, "%s abort\n", __func__);
			ecmd->complete_entry.hdr.cstate = KDS_ABORT;
			bad_cmd = true;
		}

		if (ert_pre_process(ert_user, ecmd)) {
			ERTUSER_ERR(ert_user, "%s bad cmd, opcode: %d\n", __func__, cmd_opcode(ecmd));
			ecmd->complete_entry.hdr.cstate = KDS_ABORT;
			bad_cmd = true;
		}

		if (unlikely(bad_cmd)) {
			list_move_tail(&ecmd->list, &ert_user->cq.head);
			--rq->num;
			++ert_user->cq.num;
			continue;
		}

		/* Command is good, try to submit it */
		if (ert_ctrl_slot_cmd(ecmd)) {
			if (ert_user->ctrl_busy) {
				ERTUSER_DBG(ert_user, "ctrl slot is busy\n");
				return 0;
			}
			if (cmd_opcode(ecmd) != ERT_CU_STAT)
				ert_user->ctrl_busy = true;
		}

		if (cmd_opcode(ecmd) == ERT_CONFIGURE) {
			if (ert_cfg_host(ert_user, ecmd)) {
				ert_user->ctrl_busy = false;
				ERTUSER_ERR(ert_user, "%s unable to config queue\n", __func__);
				return 0;
			}
		}

		epkt = (struct ert_packet *)ecmd->payload;
		ERTUSER_DBG(ert_user, "%s op_code %d ecmd->handle %d\n", __func__, cmd_opcode(ecmd), ecmd->handle);

		//sched_debug_packet(epkt, epkt->count+sizeof(epkt->header)/sizeof(u32));

		/* Hardware could be pretty fast, add to sq before touch the CQ_status or cmd queue*/
		list_del(&ecmd->list);
		--rq->num;
		/* Even we don't move it to sq, we still like to track how many command we submitted to the queue entity */
		++ert_user->sq.num;
		if (ert_queue_submit(ert_user, ecmd)) {
			list_add(&ecmd->list, &rq->head);
			++rq->num;
			--ert_user->sq.num;
			return 0;
		}

		if (xcmd) {
			if ((cmd_opcode(ecmd) == ERT_START_CU
			  || cmd_opcode(ecmd) == ERT_EXEC_WRITE
			  || cmd_opcode(ecmd) == ERT_START_KEY_VAL))
				ert_user->cu_stat[ecmd->xcmd->cu_idx].inflight++;

			set_xcmd_timestamp(ecmd->xcmd, KDS_RUNNING);
		}
	}
	ERTUSER_DBG(ert_user, "%s <=\n", __func__);
	return 1;
}

/**
 * process_ert_pq() - Process pending queue
 * @ert_user: Target XRT ERT
 * @pq: Target pending queue
 * @rq: Target running queue
 *
 * Move all of the pending queue commands to the tail of run queue
 * and re-initialized pending queue
 */
static inline void process_ert_pq(struct xocl_ert_user *ert_user, struct ert_user_queue *pq, struct ert_user_queue *rq)
{
	unsigned long flags;

	/* Get pending queue command number without lock.
	 * The idea is to reduce the possibility of conflict on lock.
	 * Need to check pending command number again after lock.
	 */
	if (!pq->num)
		return;

	spin_lock_irqsave(&ert_user->pq_lock, flags);
	if (pq->num) {
		list_splice_tail_init(&pq->head, &rq->head);
		rq->num += pq->num;
		pq->num = 0;
	}
	spin_unlock_irqrestore(&ert_user->pq_lock, flags);
}


static inline void
ert_user_cmd_submit(struct xocl_ert_user *ert_user, struct xrt_ert_command *ecmd)
{
	unsigned long flags;
	bool first_command = false;

	spin_lock_irqsave(&ert_user->pq_lock, flags);
	switch (cmd_opcode(ecmd)) {
	case ERT_EXEC_WRITE:
	case ERT_START_KEY_VAL:	
	case ERT_START_CU:
	case ERT_ACCESS_TEST:
	case ERT_SK_START:
		list_add_tail(&ecmd->list, &ert_user->pq.head);
		++ert_user->pq.num;
		break;
	case ERT_CLK_CALIB:
	case ERT_SK_CONFIG:
	case ERT_CU_STAT:
	case ERT_MB_VALIDATE:
	case ERT_ACCESS_TEST_C:
	case ERT_CONFIGURE:
	case ERT_EXIT:
	default:
		list_add_tail(&ecmd->list, &ert_user->pq_ctrl.head);
		++ert_user->pq_ctrl.num;
		break;
	}
	first_command = ((ert_user->pq.num + ert_user->pq_ctrl.num) == 1);
	spin_unlock_irqrestore(&ert_user->pq_lock, flags);
	/* Add command to pending queue
	 * wakeup service thread if it is the first command
	 */
	if (first_command)
		up(&ert_user->sem);	
}

static void ert_user_submit(struct kds_ert *kds_ert, struct kds_command *xcmd)
{
	uint32_t response_size = 0, cu_idx = 0;
	struct xocl_ert_user *ert_user = container_of(kds_ert, struct xocl_ert_user, ert);
	struct ert_packet *epkt = (struct ert_packet *)xcmd->execbuf;
	struct xrt_ert_command *ecmd = NULL;


	switch (epkt_cmd_opcode((struct ert_packet *)xcmd->execbuf)) {
	case ERT_CU_STAT:
		response_size = ert_user->slot_size;
		break;
	case ERT_ACCESS_TEST_C:
	case ERT_ACCESS_TEST:
		response_size = ert_user->slot_size;
		break;
	case ERT_SK_START:
		response_size = 2 * sizeof(u32);
		break;
	case ERT_MB_VALIDATE:
	case ERT_CLK_CALIB:
		response_size = sizeof(struct ert_validate_cmd);
		break;
	case ERT_EXEC_WRITE:
	case ERT_START_KEY_VAL:
	case ERT_START_CU:
		cu_idx = xcmd->cu_idx;
		break;
	default:
		break;
	};

	ecmd = ert_user_alloc_ecmd(xcmd->execbuf, (epkt->count+1)*sizeof(uint32_t), response_size);

	if (!ecmd)
		return;

	ecmd->xcmd = xcmd;
	ecmd->cu_idx = cu_idx;
	ecmd->client = xcmd->client;

	ert_user_cmd_submit(ert_user, ecmd);

	ERTUSER_DBG(ert_user, "<-%s\n", __func__);
}


static inline bool ert_user_thread_sleep_condition(struct xocl_ert_user *ert_user)
{
	bool ret = false;
	bool polling_sleep = false, intr_sleep = false, no_event = false
	, no_completed_cmd = false, no_submmited_cmd = false
	, cant_submit = false, cant_submit_start = false, cant_submit_ctrl = false
	, no_need_to_fetch_new_cmd = false, no_need_to_fetch_start_cmd = false, no_need_to_fetch_ctrl_cmd = false;


	/* When ert_thread should go to sleep to save CPU usage
	 * 1. There is no event to be processed
	 * 2. We don't have to process command when
	 *    a. We can't submit cmd if we don't have cmd in running queue or submitted queue is full
	 *    b. There is no cmd in pending queue or we still have cmds in running queue
	 *    c. There is no cmd in completed queue
	 * 3. We are not in polling mode and there is no cmd in submitted queue
	 */

	no_completed_cmd = !ert_user->cq.num;

	cant_submit_start = (!ert_user->rq.num) || (ert_user->sq.num == (ert_user->num_slots-1));
	cant_submit_ctrl = (!ert_user->rq_ctrl.num) || (ert_user->sq.num == 1);
	cant_submit = cant_submit_start && cant_submit_ctrl;

	no_need_to_fetch_start_cmd = ert_user->rq.num != 0 || !ert_user->pq.num;
	no_need_to_fetch_ctrl_cmd = ert_user->rq_ctrl.num != 0 || !ert_user->pq_ctrl.num;
	no_need_to_fetch_new_cmd = no_need_to_fetch_ctrl_cmd && no_need_to_fetch_start_cmd;

	no_submmited_cmd = !ert_user->sq.num;

	polling_sleep = no_completed_cmd && no_need_to_fetch_new_cmd && no_submmited_cmd;
	intr_sleep = no_completed_cmd && no_need_to_fetch_new_cmd && cant_submit;

	no_event = first_event_client_or_null(ert_user) == NULL;


	ret = no_event && ((ert_user->polling_mode && polling_sleep) || (!ert_user->polling_mode && intr_sleep));

	return ret;
}

static void xocl_ert_user_remove_event(struct xocl_ert_user *ert_user, struct ert_user_event *client)
{
	struct ert_user_event *next = NULL, *curr = NULL;

	mutex_lock(&ert_user->ev_lock);
	if (list_empty(&ert_user->events))
		goto done;

	list_for_each_entry_safe(curr, next, &ert_user->events, ev_entry) {
		if (client != curr)
			continue;

		list_del(&curr->ev_entry);
		break;
	}

done:
	mutex_unlock(&ert_user->ev_lock);
}

int ert_user_thread(void *data)
{
	struct xocl_ert_user *ert_user = (struct xocl_ert_user *)data;
	int ret = 0;
	struct ert_user_event *ev_client = NULL;

	mod_timer(&ert_user->timer, jiffies + ERT_TIMER);

	while (!ert_user->stop) {

		ev_client = first_event_client_or_null(ert_user);

		/* Make sure to submit as many commands as possible.
		 * This is why we call continue here. This is important to make
		 * CU busy, especially CU has hardware queue.
		 */
		if (process_ert_rq(ert_user, &ert_user->rq_ctrl, ev_client))
			continue;

		if (process_ert_rq(ert_user, &ert_user->rq, ev_client))
			continue;
		/* process completed queue before submitted queue, for
		 * two reasons:
		 * - The last submitted command may be still running
		 * - while handling completed queue, running command might done
		 * - process_ert_sq_polling will check CU status, which is thru slow bus
		 */

		process_ert_sq(ert_user, ev_client);

		process_ert_cq(ert_user);

		if (unlikely(ev_client)) {
			xocl_ert_user_remove_event(ert_user, ev_client);
			complete(&ev_client->cmp);
		}

		/* If any event occured, we should drain all the related commands ASAP
		 * It only goes to sleep if there is no event
		 */
		if (ert_user_thread_sleep_condition(ert_user)) {
			if (down_interruptible(&ert_user->sem))
				ret = -ERESTARTSYS;
		}

		process_ert_pq(ert_user, &ert_user->pq, &ert_user->rq);
		process_ert_pq(ert_user, &ert_user->pq_ctrl, &ert_user->rq_ctrl);
	}
	del_timer_sync(&ert_user->timer);

	if (!ert_user->bad_state)
		ret = -EBUSY;

	return ret;
}

static struct ert_user_event*
alloc_ert_user_event(struct kds_client *client)
{
	struct ert_user_event *event = kzalloc(sizeof(struct ert_user_event), GFP_KERNEL);

	if (!event)
		return NULL;

	event->client = client;

	init_completion(&event->cmp);
	INIT_LIST_HEAD(&event->ev_entry);

	return event;
}

static void
free_ert_user_event(struct ert_user_event *event)
{
	kfree(event);
}

/**
 * xocl_ert_abort() - Sent an abort event to ERT thread
 * @ert: Target XRT ERT
 * @client: The client tries to abort commands
 *
 * This is used to ask ERT thread to abort all commands from the client.
 */
static void xocl_ert_user_abort(struct kds_ert *ert, struct kds_client *client, int cu_idx)
{
	struct ert_user_event *event = NULL, *curr = NULL;
	struct xocl_ert_user *ert_user = container_of(ert, struct xocl_ert_user, ert);

	mutex_lock(&ert_user->ev_lock);
	if (list_empty(&ert_user->events))
		goto add_event;

	/* avoid re-add the same client */
	list_for_each_entry(curr, &ert_user->events, ev_entry) {
		if (client == curr->client)
			goto done;
	}

add_event:
	event = alloc_ert_user_event(client);
	if (!event)
		goto done;

	client->ev_type = EV_ABORT;
	list_add_tail(&event->ev_entry, &ert_user->events);
	/* The process thread may asleep, we should wake it up if
	 * abort event takes place
	 */

	up(&ert_user->sem);
done:
	mutex_unlock(&ert_user->ev_lock);
}

/**
 * xocl_ert_abort() - Get done flag of abort
 * @ert: Target XRT ERT
 *
 * Use this to wait for abort event done
 */
static bool xocl_ert_user_abort_done(struct kds_ert *ert, struct kds_client *client, int cu_idx)
{
	struct ert_user_event *next = NULL, *curr = NULL;
	struct xocl_ert_user *ert_user = container_of(ert, struct xocl_ert_user, ert);

	mutex_lock(&ert_user->ev_lock);
	if (list_empty(&ert_user->events))
		goto done;

	list_for_each_entry_safe(curr, next, &ert_user->events, ev_entry) {
		if (client != curr->client)
			continue;

		list_del(&curr->ev_entry);
		free_ert_user_event(curr);
		break;
	}

done:
	mutex_unlock(&ert_user->ev_lock);

	return ert_user->bad_state;
}

/**
 * xocl_ert_abort_sync() - Sent an abort event to ERT thread
 * @ert: Target XRT ERT
 * @client: The client tries to abort commands
 *
 * This is used to ask ERT thread to abort all commands from the client.
 */
static bool xocl_ert_user_abort_sync(struct kds_ert *ert, struct kds_client *client, int cu_idx)
{
	struct xocl_ert_user *ert_user = container_of(ert, struct xocl_ert_user, ert);
	struct ert_user_event *event = NULL, *curr = NULL;

	mutex_lock(&ert_user->ev_lock);
	if (list_empty(&ert_user->events))
		goto add_event;

	/* avoid re-add the same client */
	list_for_each_entry(curr, &ert_user->events, ev_entry) {
		if (client == curr->client) {
			mutex_unlock(&ert_user->ev_lock);
			return ert_user->bad_state;
		}
	}

add_event:
	event = alloc_ert_user_event(client);
	if (!event) {
		mutex_unlock(&ert_user->ev_lock);
		return ert_user->bad_state;
	}
	client->ev_type = EV_ABORT;
	list_add_tail(&event->ev_entry, &ert_user->events);
	/* The process thread may asleep, we should wake it up if
	 * abort event takes place
	 */
	up(&ert_user->sem);

	mutex_unlock(&ert_user->ev_lock);

	wait_for_completion(&event->cmp);

	free_ert_user_event(event);

	return  ert_user->bad_state;
}


static int ert_user_remove(struct platform_device *pdev)
{
	struct xocl_ert_user *ert_user;
	void *hdl;

	ert_user = platform_get_drvdata(pdev);
	if (!ert_user) {
		xocl_err(&pdev->dev, "driver data is NULL");
		return -EINVAL;
	}

	ert_queue_intc_config(ert_user, false);

	sysfs_remove_group(&pdev->dev.kobj, &ert_user_attr_group);

	xocl_drvinst_release(ert_user, &hdl);

	ert_user->stop = 1;
	up(&ert_user->sem);
	(void) kthread_stop(ert_user->thread);

	platform_set_drvdata(pdev, NULL);

	xocl_drvinst_free(hdl);

	return 0;
}

static int ert_user_probe(struct platform_device *pdev)
{
	struct xocl_ert_user *ert_user;
	int err = 0;
	xdev_handle_t xdev = xocl_get_xdev(pdev);
	struct xocl_ert_sched_privdata *priv = NULL;
	bool ert = xocl_ert_on(xdev);

	/* If XOCL_DSAFLAG_MB_SCHE_OFF is set, we should not probe ert */
	if (!ert) {
		xocl_warn(&pdev->dev, "Disable ERT flag overwrite, don't probe ert_user");
		return -ENODEV;
	}

	ert_user = xocl_drvinst_alloc(&pdev->dev, sizeof(struct xocl_ert_user));
	if (!ert_user)
		return -ENOMEM;

	ert_user->dev = &pdev->dev;
	ert_user->pdev = pdev;
	/* Initialize pending queue and lock */
	INIT_LIST_HEAD(&ert_user->pq.head);
	INIT_LIST_HEAD(&ert_user->pq_ctrl.head);
	spin_lock_init(&ert_user->pq_lock);
	/* Initialize run queue */
	INIT_LIST_HEAD(&ert_user->rq.head);
	INIT_LIST_HEAD(&ert_user->rq_ctrl.head);

	/* Initialize completed queue */
	INIT_LIST_HEAD(&ert_user->cq.head);

	mutex_init(&ert_user->ev_lock);
	INIT_LIST_HEAD(&ert_user->events);

	sema_init(&ert_user->sem, 0);

#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0)
	setup_timer(&ert_user->timer, ert_timer, (unsigned long)ert_user);
#else
	timer_setup(&ert_user->timer, ert_timer, 0);
#endif
	atomic_set(&ert_user->tick, 0);

	init_completion(&ert_user->comp);

	ert_user->thread = kthread_run(ert_user_thread, ert_user, "ert_thread");

	platform_set_drvdata(pdev, ert_user);
	mutex_init(&ert_user->lock);

	if (XOCL_GET_SUBDEV_PRIV(&pdev->dev)) {
		priv = XOCL_GET_SUBDEV_PRIV(&pdev->dev);
		memcpy(&ert_user->ert_cfg_priv, priv, sizeof(*priv));
	} else {
		xocl_err(&pdev->dev, "did not get private data");
	}


	err = sysfs_create_group(&pdev->dev.kobj, &ert_user_attr_group);
	if (err) {
		xocl_err(&pdev->dev, "create ert_user sysfs attrs failed: %d", err);
		goto done;
	}

	ert_user->polling_mode = false;

	ert_user->ert.submit = ert_user_submit;
	ert_user->ert.abort = xocl_ert_user_abort;
	ert_user->ert.abort_done = xocl_ert_user_abort_done;
	ert_user->ert.abort_sync = xocl_ert_user_abort_sync;
	xocl_kds_init_ert(xdev, &ert_user->ert);

done:
	if (err) {
		ert_user_remove(pdev);
		return err;
	}
	return 0;
}

struct xocl_drv_private ert_user_priv = {
	.ops = &ert_user_ops,
	.dev = -1,
};

struct platform_device_id ert_user_id_table[] = {
	{ XOCL_DEVNAME(XOCL_ERT_USER), (kernel_ulong_t)&ert_user_priv },
	{ },
};

static struct platform_driver	ert_user_driver = {
	.probe		= ert_user_probe,
	.remove		= ert_user_remove,
	.driver		= {
		.name = XOCL_DEVNAME(XOCL_ERT_USER),
	},
	.id_table = ert_user_id_table,
};

int __init xocl_init_ert_user(void)
{
	return platform_driver_register(&ert_user_driver);
}

void xocl_fini_ert_user(void)
{
	platform_driver_unregister(&ert_user_driver);
}
