/* SPDX-License-Identifier: GPL-2.0 OR Apache-2.0 */
/*
 * A GEM style (optionally CMA backed) device manager for ZynQ based
 * OpenCL accelerators.
 *
 * Copyright (C) 2016-2022 Xilinx, Inc. All rights reserved.
 *
 * Authors:
 *    Sonal Santan <sonal.santan@xilinx.com>
 *    Umang Parekh <umang.parekh@xilinx.com>
 *    Min Ma       <min.ma@xilinx.com>
 *    Jan Stephan  <j.stephan@hzdr.de>
 *
 * This file is dual-licensed; you may select either the GNU General Public
 * License version 2 or Apache License, Version 2.0.
 */

#include <linux/delay.h>
#include <linux/dma-buf.h>
#include <linux/module.h>
#include <linux/fpga/fpga-mgr.h>
#include <linux/iommu.h>
#include <linux/pagemap.h>
#include <linux/io.h>
#include <linux/kernel.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
#include <linux/poll.h>
#include <linux/spinlock.h>
#include "zocl_drv.h"
#include "zocl_sk.h"
#include "zocl_aie.h"
#include "zocl_bo.h"
#include "sched_exec.h"
#include "zocl_xclbin.h"
#include "zocl_error.h"

#define ZOCL_DRIVER_NAME        "zocl"
#define ZOCL_DRIVER_DESC        "Zynq BO manager"

static char driver_date[9];

/* This should be the same as DRM_FILE_PAGE_OFFSET_START in drm_gem.c */
#if defined(CONFIG_ARM64)
#define ZOCL_FILE_PAGE_OFFSET   0x00100000
#else
#define ZOCL_FILE_PAGE_OFFSET   0x00010000
#endif

#ifndef VM_RESERVED
#define VM_RESERVED (VM_DONTEXPAND | VM_DONTDUMP)
#endif


int enable_xgq_ert = 1;
module_param(enable_xgq_ert, int, (S_IRUGO|S_IWUSR));
MODULE_PARM_DESC(enable_xgq_ert, "0 = legacy ERT mode, 1 = XGQ ERT mode (default)");

extern struct platform_driver zocl_ctrl_ert_driver;

static const struct vm_operations_struct reg_physical_vm_ops = {
#ifdef CONFIG_HAVE_IOREMAP_PROT
	.access = generic_access_phys,
#endif
};


#if KERNEL_VERSION(5, 3, 0) <= LINUX_VERSION_CODE
static int
match_name(struct device *dev, const void *data)
#else
static int
match_name(struct device *dev, void *data)
#endif
{
	const char *name = data;
	/*
	 * check if given name is substring inside dev.
	 * the dev_name is like: 20300030000.ert_hw
	 */
	return strstr(dev_name(dev), name) != NULL;
}

/**
 * zocl_pr_slot_init : PR Slot specific initialization
 *
 * @zdev: zocl device struct
 * @pdev: platform device struct
 *
 * Returns 0 if initialization successfully.
 * Returns -EINVAL if failed.
 */

static int zocl_pr_slot_init(struct drm_zocl_dev *zdev,
		                       struct platform_device *pdev)
{
	struct drm_zocl_slot *zocl_slot = NULL;
	int ret = 0;
	int i = 0;

	zdev->num_pr_slot = 0;
	/* TODO : Need to update this function based on the device tree */
	if (ZOCL_PLATFORM_ARM64) {
		u64 pr_num;
		if (!of_property_read_u64(pdev->dev.of_node,
					  "xlnx,pr-num-support", &pr_num))
			zdev->num_pr_slot = (int)pr_num;
	} else {
		/* Work around for CR-1119382 issue.
		 * ZOCL driver is crashing if it accessing the device tree node */
		if (false) {
			u32 pr_num;
			if (!of_property_read_u32(pdev->dev.of_node,
					  "xlnx,pr-num-support", &pr_num))
				zdev->num_pr_slot = (int)pr_num;
		}
	}

	/* If there is no information available for number of slot available
	 * for this device then consider it for a single slot device for
	 * backward compartability.
	 */
	if (zdev->num_pr_slot == 0)
		zdev->num_pr_slot = 1;

	for (i = 0; i < zdev->num_pr_slot; i++) {
		zocl_slot = (struct drm_zocl_slot *)
			kzalloc(sizeof(struct drm_zocl_slot), GFP_KERNEL);
		if (!zocl_slot)
			return -ENOMEM;

		/* Initial xclbin */
		ret = zocl_xclbin_init(zocl_slot);
		if (ret)
			return ret;

		mutex_init(&zocl_slot->slot_xclbin_lock);
 
		if (ZOCL_PLATFORM_ARM64) {
			zocl_slot->pr_isolation_freeze = 0x0;
			zocl_slot->pr_isolation_unfreeze = 0x3;
			if (of_property_read_u64(pdev->dev.of_node,
					"xlnx,pr-isolation-addr",
					&zocl_slot->pr_isolation_addr))
				zocl_slot->pr_isolation_addr = 0;
			if (of_property_read_bool(pdev->dev.of_node,
						  "xlnx,pr-decoupler")) {
				zocl_slot->pr_isolation_freeze = 0x1;
				zocl_slot->pr_isolation_unfreeze = 0x0;
			}
		} else {
			/* Work around for CR-1119382 issue.
			 * ZOCL driver is crashing if it accessing the device tree node */
			if (false) {
				u32 prop_addr = 0;

				if (of_property_read_u32(pdev->dev.of_node,
							 "xlnx,pr-isolation-addr",
							 &prop_addr))
					zocl_slot->pr_isolation_addr = 0;
				else
					zocl_slot->pr_isolation_addr = prop_addr;
			}
		}

		DRM_INFO("PR[%d] Isolation addr 0x%llx", i,
			 zocl_slot->pr_isolation_addr);

		zocl_slot->partial_overlay_id = -1;

		zdev->pr_slot[i] = zocl_slot;
	}

	zdev->full_overlay_id  = -1;

	return 0;
}

/**
 * zocl_pr_slot_fini : PR Slot specific Cleanup
 *
 * @zdev: zocl device struct
 *
 */
static void zocl_pr_slot_fini(struct drm_zocl_dev *zdev)
{
	struct drm_zocl_slot *zocl_slot = NULL;
	int i = 0;

	for (i = 0; i < zdev->num_pr_slot; i++) {
		zocl_slot = zdev->pr_slot[i];
		if (zocl_slot) {
			zocl_free_sections(zocl_slot);
			mutex_destroy(&zocl_slot->slot_xclbin_lock);
			zocl_xclbin_fini(zdev, zocl_slot);
			kfree(zocl_slot);
			zdev->pr_slot[i] = NULL;
		}
	}
}

/**
 * Initialize the aparture and allocate memory for it.
 *
 * @param	zdev: device structure
 *
 * @return	0 on success, Error code on failure.
 */
static int zocl_aperture_init(struct drm_zocl_dev *zdev)
{
	struct addr_aperture *apts = NULL;
	int i = 0;

	zdev->apertures = kcalloc(MAX_APT_NUM, sizeof(struct addr_aperture),
				 GFP_KERNEL);
	if (!zdev->apertures) {
		DRM_ERROR("Out of memory for Aperture\n");
		return -ENOMEM;
	}

	apts = zdev->apertures;
	/* Consider this magic number as uninitialized aperture identity */
	for (i = 0; i < MAX_APT_NUM; ++i)
		apts[i].addr = EMPTY_APT_VALUE;

	zdev->num_apts = 0;

	return 0;
}

/**
 * Cleanup the aparture
 *
 * @param	zdev: device structure
 *
 */
static void zocl_aperture_fini(struct drm_zocl_dev *zdev)
{
	/* Free aperture memory */
	if (zdev->apertures)
		vfree(zdev->apertures);

	zdev->apertures = NULL;
}


/**
 * get_reserved_mem_region - Get reserved memory region
 *
 * @dev: device struct
 * @res: resource struct
 *
 * Returns 0 is get reserved memory resion successfully.
 * Returns -EINVAL if not found.
 */
static int get_reserved_mem_region(struct device *dev, struct resource *res)
{
	struct device_node *np = NULL;
	int ret;

	np = of_parse_phandle(dev->of_node, "memory-region", 0);
	if (!np)
		return -EINVAL;

	ret = of_address_to_resource(np, 0, res);
	if (ret)
		return -EINVAL;

	return 0;
}

/**
 * Find platform device by name
 *
 * @param	name: device name
 *
 * Returns a platform device. Returns NULL if not found.
 */
struct platform_device *zocl_find_pdev(char *name)
{
	struct device *dev;
	struct platform_device *pdev;

	dev = bus_find_device(&platform_bus_type, NULL, (void *)name,
	    match_name);
	if (!dev)
		return NULL;

	pdev = container_of(dev, struct platform_device, dev);
	return pdev;
}

/**
 * Set scheduler CU index in aperture
 *
 * @param	zdev:	zocl device struct
 * @param	apt_idx: aperture index in the IP_LAYOUT ordering
 * @param	cu_idx: CU index in the scheduler ordering
 *
 */
void update_cu_idx_in_apt(struct drm_zocl_dev *zdev, int apt_idx, int cu_idx)
{
	struct addr_aperture *apts = zdev->apertures;

	/* Actually, we should consider lock this.
	 * So far, let do this without lock. Since the scheduler would only
	 * update this when xclbin was changed.
	 */
	apts[apt_idx].cu_idx = cu_idx;
}

/**
 * Get the index of the geiven phys address,
 *		   if it is the start of an aperture
 *
 * @param	zdev: zocl device struct
 * @param	addr: physical address of the aperture
 *
 * Returns the index if aperture was found, -EINVAL if not found.
 *
 */
int get_apt_index_by_addr(struct drm_zocl_dev *zdev, phys_addr_t addr)
{
	struct addr_aperture *apts = zdev->apertures;
	int i;

	/* Haven't consider search efficiency yet. */
	for (i = 0; i < zdev->num_apts; ++i)
		if (apts[i].addr == addr)
			break;

	return (i == zdev->num_apts) ? -EINVAL : i;
}

/**
 * Get the index of the geiven phys address,
 *		   if it is the start of an aperture
 *
 * @param	zdev:	zocl device struct
 * @param	cu_idx: CU index
 *
 * Returns the index if aperture was found, -EINVAL if not found.
 *
 */
int get_apt_index_by_cu_idx(struct drm_zocl_dev *zdev, int cu_idx)
{
	struct addr_aperture *apts = zdev->apertures;
	int i;

	WARN_ON(cu_idx >= MAX_CU_NUM);
	if (cu_idx >= MAX_CU_NUM)
		return -EINVAL;

	/* Haven't consider search efficiency yet. */
	for (i = 0; i < zdev->num_apts; ++i)
		if (apts[i].cu_idx == cu_idx)
			break;

	return (i == zdev->num_apts) ? -EINVAL : i;
}

/**
 * Create a new CU subdevice. And try to attach to the driver. This will force
 * cu probe to call.
 *
 * @param	zdev: zocl Device Instance
 * @param	info: CU related information
 *
 * @return	0 on success, Error code on failure.
 */
int subdev_create_cu(struct device *dev, struct xrt_cu_info *info, struct platform_device **pdevp)
{
	struct platform_device *pldev;
	struct resource res = {};
	int ret;

	pldev = platform_device_alloc("CU", PLATFORM_DEVID_AUTO);
	if (!pldev) {
		DRM_ERROR("Failed to alloc device CU\n");
		return -ENOMEM;
	}

	/*
	 * Only on U30 and some Versal platforms, the res_start is not zero.
	 * But the situations are different,
	 *
	 *     On U30, CUs are assigned to lower 4GB address space.
	 * Host doesn't know the CU base address, which is 0x80000000,
	 * but zocl get it from device tree and store it in res_start.
	 *
	 *     On Versal, CUs are assigned to above 4GB address space.
	 * Host knows the CU base address from xclbin.
	 * But on some specific shells, zocl will get base address in
	 * device tree. "Or" operation still works in this case.
	 */
	res.start = info->addr | zocl_get_zdev()->res_start;
	res.end = res.start + info->size - 1;
	res.flags = IORESOURCE_MEM;
	ret = platform_device_add_resources(pldev, &res, 1);
	if (ret) {
		DRM_ERROR("Failed to add resource\n");
		goto err;
	}

	ret = platform_device_add_data(pldev, info, sizeof(*info));
	if (ret) {
		DRM_ERROR("Failed to add data\n");
		goto err;
	}

	pldev->dev.parent = dev;

	ret = platform_device_add(pldev);
	if (ret) {
		DRM_ERROR("Failed to add device\n");
		goto err;
	}

	/*
	 * force probe to avoid dependence issue. if probing
	 * failed, it could be the driver is not registered.
	 */
	ret = device_attach(&pldev->dev);
	if (ret != 1) {
		ret = -EINVAL;
		DRM_ERROR("Failed to probe device\n");
		goto err1;
	}
	*pdevp = pldev;

	return 0;
err1:
	platform_device_del(pldev);
err:
	platform_device_put(pldev);
	return ret;
}

/* This function destroy and remove the platform-level devices
 * for all the CUs.
 *
 * @param	zdev: zocl Device Instance
 *
 */
void subdev_destroy_cu(struct drm_zocl_dev *zdev)
{
	int i;

	for (i = 0; i < MAX_CU_NUM; ++i) {
		if (!zdev->cu_pldev[i])
			continue;

		/* Remove the platform-level device */
		platform_device_del(zdev->cu_pldev[i]);
		/* Destroy the platform device */
		platform_device_put(zdev->cu_pldev[i]);
		zdev->cu_pldev[i] = NULL;
	}
}

/**
 * Create a new SCU subdevice. And try to attach to the driver. This will force
 * cu probe to call.
 *
 * @param	zdev: zocl Device Instance
 * @param	info: SCU related information
 *
 * @return	0 on success, Error code on failure.
 */
int subdev_create_scu(struct device *dev, struct xrt_cu_info *info, struct platform_device **pdevp)
{
	struct platform_device *pldev;
	int ret;

	pldev = platform_device_alloc("SCU", PLATFORM_DEVID_AUTO);
	if (!pldev) {
		DRM_ERROR("Failed to alloc device SCU\n");
		return -ENOMEM;
	}

	ret = platform_device_add_data(pldev, info, sizeof(*info));
	if (ret) {
		DRM_ERROR("Failed to add data\n");
		goto err;
	}

	pldev->dev.parent = dev;

	ret = platform_device_add(pldev);
	if (ret) {
		DRM_ERROR("Failed to add device\n");
		goto err;
	}

	/*
	 * force probe to avoid dependence issue. if probing
	 * failed, it could be the driver is not registered.
	 */
	ret = device_attach(&pldev->dev);
	if (ret != 1) {
		ret = -EINVAL;
		DRM_ERROR("Failed to probe device\n");
		goto err1;
	}
	*pdevp = pldev;

	return 0;
err1:
	platform_device_del(pldev);
err:
	platform_device_put(pldev);
	return ret;
}

/**
 * zocl_gem_create_object - Create drm_zocl_bo object instead of DRM CMA object.
 *
 * @dev: DRM device struct
 * @size: This size was not use, just to match function prototype.
 *
 */
struct drm_gem_object *
zocl_gem_create_object(struct drm_device *dev, size_t size)
{
	struct drm_zocl_bo *bo = kzalloc(sizeof(struct drm_zocl_bo), GFP_KERNEL);
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0)
	bo->gem_base.funcs = &zocl_gem_object_funcs;
#endif
	return (&bo->gem_base);
}

/* This callback function release GEM buffer objects and free memory associated
 * with it. This function is also responsable for free up the memory for BOs.
 *
 * @param	obj:	GEM buffer object
 *
 */
void zocl_free_bo(struct drm_gem_object *obj)
{
	struct drm_zocl_bo *zocl_obj;
	struct drm_zocl_dev *zdev;
	int npages;

	if (IS_ERR(obj) || !obj)
		return;

	DRM_DEBUG("Freeing BO\n");
	zocl_obj = to_zocl_bo(obj);
	zdev = obj->dev->dev_private;

	if (!zdev->domain) {
		zocl_describe(zocl_obj);
		if (zocl_obj->flags & ZOCL_BO_FLAGS_USERPTR)
			zocl_free_userptr_bo(obj);
		else if (zocl_obj->flags & ZOCL_BO_FLAGS_HOST_BO)
			zocl_free_host_bo(obj);
		else if (zocl_obj->flags & ZOCL_BO_FLAGS_CMA) {
			/* Update memory usage statistics */
			zocl_update_mem_stat(zdev, obj->size, -1,
			    zocl_obj->mem_index);
			/* free resources associated with a CMA GEM object */
			drm_gem_cma_free_object(obj);

		} else {
			if (zocl_obj->mm_node) {
				mutex_lock(&zdev->mm_lock);
				drm_mm_remove_node(zocl_obj->mm_node);
				mutex_unlock(&zdev->mm_lock);
				kfree(zocl_obj->mm_node);
				if (zocl_obj->vmapping) {
					memunmap(zocl_obj->vmapping);
					zocl_obj->vmapping = NULL;
				}
				zocl_update_mem_stat(zdev, obj->size, -1,
				    zocl_obj->mem_index);
			}
			/* release GEM buffer object resources */
			drm_gem_object_release(obj);
			kfree(zocl_obj);
		}

		return;
	}

	npages = obj->size >> PAGE_SHIFT;
	/* release GEM buffer object resources */
	drm_gem_object_release(obj);

	if (zocl_obj->vmapping)
		vunmap(zocl_obj->vmapping);
	zocl_obj->vmapping = NULL;

	zocl_iommu_unmap_bo(obj->dev, zocl_obj);
	if (zocl_obj->pages) {
		if (zocl_bo_userptr(zocl_obj)) {
#if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE
			release_pages(zocl_obj->pages, npages);
#else
			release_pages(zocl_obj->pages, npages, 0);
#endif
			kvfree(zocl_obj->pages);
		} else {
			drm_gem_put_pages(obj, zocl_obj->pages, false, false);

			/* Update memory usage statistics */
			zocl_update_mem_stat(zdev, obj->size, -1,
			    zocl_obj->mem_index);
		}
	}
	if (zocl_obj->sgt)
		sg_free_table(zocl_obj->sgt);
	zocl_obj->sgt = NULL;
	zocl_obj->pages = NULL;
	kfree(zocl_obj);
}

/* This function memory map for GEM objects.
 *
 * @param	flip:	file data structure
 * @param	vma:	struct to a virtual memory area
 *
 * @return	0 on success, Error code on failure.
 */
static int
zocl_gem_mmap(struct file *filp, struct vm_area_struct *vma)
{
	struct drm_gem_cma_object *cma_obj = NULL;
	struct drm_gem_object *gem_obj;
	struct drm_zocl_bo *bo;
	dma_addr_t paddr;
	pgprot_t prot;
	int rc;

	/**
	 * drm_gem_mmap may modify the vma prot as non-cacheable.
	 * We need to preserve this field and resume it in case
	 * the BO is cacheable.
	 */
	prot = vma->vm_page_prot;

	rc = drm_gem_mmap(filp, vma);

	if (rc)
		return rc;

	/**
	 * Clear the VM_PFNMAP flag that was set by drm_gem_mmap(),
	 * and set the vm_pgoff (used as a fake buffer offset by DRM)
	 * to 0 as we want to map the whole buffer.
	 */
	vma->vm_flags &= ~VM_PFNMAP;
	vma->vm_pgoff = 0;

	gem_obj = vma->vm_private_data;
	bo = to_zocl_bo(gem_obj);

	if (bo->flags & ZOCL_BO_FLAGS_CACHEABLE)
		/**
		 * Resume the protection field from mmap(). Most likely
		 * it will be cacheable. If there is a case that mmap()
		 * protection field explicitly tells us not to map with
		 * cache enabled, we should comply with it and overwrite
		 * the cacheable BO property.
		 */
		vma->vm_page_prot = prot;

	if (bo->flags & ZOCL_BO_FLAGS_CMA) {
		cma_obj = to_drm_gem_cma_obj(gem_obj);
		paddr = cma_obj->paddr;
	} else
		paddr = bo->mm_node->start;

	if ((!(bo->flags & ZOCL_BO_FLAGS_CMA)) ||
	    (bo->flags & ZOCL_BO_FLAGS_CMA &&
	    bo->flags & ZOCL_BO_FLAGS_CACHEABLE)) {
		/* Map PL-DDR and cacheable CMA */
		rc = remap_pfn_range(vma, vma->vm_start,
		    paddr >> PAGE_SHIFT, vma->vm_end - vma->vm_start,
		    vma->vm_page_prot);
	} else {
		/* Map non-cacheable CMA */
		rc = dma_mmap_wc(cma_obj->base.dev->dev, vma, cma_obj->vaddr,
		    paddr, vma->vm_end - vma->vm_start);
	}

	if (rc)
		drm_gem_vm_close(vma);

	return rc;
}

/* This function map two types of kernel address to user space.
 * The first type is pysical registers of a hardware IP, like CUs.
 * The second type is GEM buffer.
 *
 * @param	flip:	file data structure
 * @param	vma:	struct to a virtual memory area
 *
 * @return	0 on success, Error code on failure.
 */
static int zocl_mmap(struct file *filp, struct vm_area_struct *vma)
{
	struct drm_file     *priv = filp->private_data;
	struct drm_device   *dev = priv->minor->dev;
	struct drm_zocl_dev *zdev = dev->dev_private;
	struct addr_aperture *apts = zdev->apertures;
	struct drm_zocl_bo  *bo = NULL;
	unsigned long        vsize;
	phys_addr_t          phy_addr;
	int apt_idx;
	int rc;

	/* A GEM buffer object has a fake mmap offset start from page offset
	 * DRM_FILE_PAGE_OFFSET_START. See drm_gem_init().
	 * ZOCL_FILE_PAGE_OFFSET should equal to DRM_FILE_PAGE_OFFSET_START.
	 * ZOCL_FILE_PAGE_OFFSET is 4GB for 64 bits system.
	 */
	if (likely(vma->vm_pgoff >= ZOCL_FILE_PAGE_OFFSET)) {
		if (!zdev->domain)
			return zocl_gem_mmap(filp, vma);

		/* Map user's pages into his VM */
		rc = drm_gem_mmap(filp, vma);
		if (rc)
			return rc;
		/* vma->vm_private_data is set by drm_gem_mmap */
		bo = to_zocl_bo(vma->vm_private_data);

		bo->uaddr = vma->vm_start;
		/* Map user's VA into IOMMU */
		rc = zocl_iommu_map_bo(dev, bo);
		if (rc)
			return rc;
		vma->vm_flags &= ~VM_PFNMAP;
		vma->vm_flags |= VM_MIXEDMAP;
		/* Reset the fake offset used to identify the BO */
		vma->vm_pgoff = 0;
		return 0;
	}


	/* Only allow user to map register ranges in apertures list.
	 * Could not map from the middle of an aperture.
	 */
	apt_idx = vma->vm_pgoff;
	if (apt_idx < 0 || apt_idx >= zdev->num_apts) {
		DRM_ERROR("The offset is not in the apertures list\n");
		return -EINVAL;
	}
	phy_addr = apts[apt_idx].addr;
	vma->vm_pgoff = phy_addr >> PAGE_SHIFT;

	vsize = vma->vm_end - vma->vm_start;
	if (vsize > zdev->apertures[apt_idx].size)
		return -EINVAL;

	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
	vma->vm_flags |= VM_IO;
	vma->vm_flags |= VM_RESERVED;

	vma->vm_ops = &reg_physical_vm_ops;
	rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
				vsize, vma->vm_page_prot);

	return rc;
}

/**
 * Registering callback for fault handler.
 *
 * @param	vmf:	vm page fault instance
 *
 * @return	VM_FAULT_NOPAGE on success, Error code on failure.
 */
static vm_fault_t zocl_bo_fault(struct vm_fault *vmf)
{
	struct vm_area_struct *vma = vmf->vma;
	struct drm_gem_object *obj = vma->vm_private_data;
	struct drm_zocl_bo *bo = to_zocl_bo(obj);
	struct drm_zocl_dev *zdev = obj->dev->dev_private;
	struct page *page;
	pgoff_t offset;
	int err;

	if (!zdev->domain)
		return 0;

	if (!bo->pages)
		return VM_FAULT_SIGBUS;

	offset = ((unsigned long)vmf->address - vma->vm_start) >> PAGE_SHIFT;
	page = bo->pages[offset];

	err = vm_insert_page(vma, (unsigned long)vmf->address, page);
	switch (err) {
	case -EAGAIN:
	case 0:
	case -ERESTARTSYS:
	case -EINTR:
	case -EBUSY:
		return VM_FAULT_NOPAGE;
	case -ENOMEM:
		return VM_FAULT_OOM;
	}
	return VM_FAULT_SIGBUS;
}

/**
 * Driver callback when a new &struct drm_file is opened.
 * This function will create a new client for this device
 *
 * @param	dev:	DRM device structure
 * @param	flip:	DRM file private data
 *
 * @return	0 on success, Error code on failure.
 */
static int zocl_client_open(struct drm_device *dev, struct drm_file *filp)
{
	return zocl_create_client(dev->dev, &filp->driver_priv);
}

/**
 * Driver callback when a new &struct drm_file is closed.
 * This function will cleanup driver-private data structures
 * allocated in @open and destroy the client.
 *
 * @param	dev:	DRM device structure
 * @param	flip:	DRM file private data
 *
 * @return	0 on success, Error code on failure.
 */
static void zocl_client_release(struct drm_device *dev, struct drm_file *filp)
{
	return zocl_destroy_client(filp->driver_priv);
}

/**
 * Register a poll callback function for this driver
 *
 * @param	flip:	file data structure
 * @param	wait:	poll table
 *
 * @return	POLLIN on success, 0 on failure.
 */
static unsigned int zocl_poll(struct file *filp, poll_table *wait)
{
	return zocl_poll_client(filp, wait);
}

/**
 * Initialize iommu domain for this device
 *
 * @param	zdev: zocl Device Instance
 * @param	pdev: Platform Device Instance
 *
 * @return	0 on success, Error code on failure.
 */
static int zocl_iommu_init(struct drm_zocl_dev *zdev,
		struct platform_device *pdev)
{
	struct iommu_domain_geometry *geometry;
	u64 start, end;
	int ret;

	zdev->domain = iommu_domain_alloc(&platform_bus_type);
	if (!zdev->domain)
		return -ENOMEM;

	ret = iommu_attach_device(zdev->domain, &pdev->dev);
	if (ret) {
		DRM_INFO("IOMMU attach device failed. ret(%d)\n", ret);
		iommu_domain_free(zdev->domain);
		zdev->domain = NULL;
		return ret;
	}

	geometry = &zdev->domain->geometry;
	start = geometry->aperture_start;
	end = geometry->aperture_end;

	DRM_INFO("IOMMU aperture initialized (%#llx-%#llx)\n",
				start, end);

	return 0;
}

const struct vm_operations_struct zocl_bo_vm_ops = {
	.fault = zocl_bo_fault,
	.open  = drm_gem_vm_open,
	.close = drm_gem_vm_close,
};

static const struct drm_ioctl_desc zocl_ioctls[] = {
	DRM_IOCTL_DEF_DRV(ZOCL_CREATE_BO, zocl_create_bo_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_USERPTR_BO, zocl_userptr_bo_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_GET_HOST_BO, zocl_get_hbo_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_MAP_BO, zocl_map_bo_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_SYNC_BO, zocl_sync_bo_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_INFO_BO, zocl_info_bo_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_PWRITE_BO, zocl_pwrite_bo_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_PREAD_BO, zocl_pread_bo_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_EXECBUF, zocl_execbuf_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_READ_AXLF, zocl_read_axlf_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_SK_GETCMD, zocl_sk_getcmd_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_SK_CREATE, zocl_sk_create_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_SK_REPORT, zocl_sk_report_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_INFO_CU, zocl_info_cu_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_CTX, zocl_ctx_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_ERROR_INJECT, zocl_error_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_AIE_FD, zocl_aie_fd_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_AIE_RESET, zocl_aie_reset_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_AIE_GETCMD, zocl_aie_getcmd_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_AIE_PUTCMD, zocl_aie_putcmd_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
	DRM_IOCTL_DEF_DRV(ZOCL_AIE_FREQSCALE, zocl_aie_freqscale_ioctl,
			DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
};

static const struct file_operations zocl_driver_fops = {
	.owner          = THIS_MODULE,
	.open           = drm_open,
	.mmap           = zocl_mmap,
	.poll           = zocl_poll,
	.read           = drm_read,
	.unlocked_ioctl = drm_ioctl,
	.release        = drm_release,
};

static struct drm_driver zocl_driver = {
#if KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE
	.driver_features           = DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER,
#else
	.driver_features           = DRIVER_GEM | DRIVER_RENDER,
#endif
	.open                      = zocl_client_open,
	.postclose                 = zocl_client_release,
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0)
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)
	.gem_free_object_unlocked  = zocl_free_bo,
#else
	.gem_free_object           = zocl_free_bo,
#endif

	.gem_vm_ops                = &zocl_bo_vm_ops,
	.gem_prime_get_sg_table    = drm_gem_cma_prime_get_sg_table,
	.gem_prime_vmap            = drm_gem_cma_prime_vmap,
	.gem_prime_vunmap          = drm_gem_cma_prime_vunmap,
	.gem_prime_export          = drm_gem_prime_export,
#endif

	.gem_create_object         = zocl_gem_create_object,
	.prime_handle_to_fd        = drm_gem_prime_handle_to_fd,
	.prime_fd_to_handle        = drm_gem_prime_fd_to_handle,
	.gem_prime_import          = zocl_gem_import,
	.gem_prime_import_sg_table = drm_gem_cma_prime_import_sg_table,
	.gem_prime_mmap            = drm_gem_prime_mmap,
	.ioctls                    = zocl_ioctls,
	.num_ioctls                = ARRAY_SIZE(zocl_ioctls),
	.fops                      = &zocl_driver_fops,
	.name                      = ZOCL_DRIVER_NAME,
	.desc                      = ZOCL_DRIVER_DESC,
	.date                      = driver_date,
};

#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0)
const struct drm_gem_object_funcs zocl_gem_object_funcs = {
	.free = zocl_free_bo,
	.vm_ops = &zocl_bo_vm_ops,
	.get_sg_table = drm_gem_cma_get_sg_table,
	.vmap = drm_gem_cma_vmap,
	.export = drm_gem_prime_export,
};

const struct drm_gem_object_funcs zocl_cma_default_funcs = {
	.free = zocl_free_bo,
	.get_sg_table = drm_gem_cma_get_sg_table,
	.vm_ops = &zocl_bo_vm_ops,
};
#endif
static const struct zdev_data zdev_data_mpsoc = {
	.fpga_driver_name = "pcap",
};

static const struct zdev_data zdev_data_versal = {
	.fpga_driver_name = "versal_fpga"
};

static const struct of_device_id zocl_drm_of_match[] = {
	{ .compatible = "xlnx,zocl", .data = &zdev_data_mpsoc},
	{ .compatible = "xlnx,zoclsvm", .data = &zdev_data_mpsoc},
	{ .compatible = "xlnx,zocl-ert", .data = &zdev_data_mpsoc},
	{ .compatible = "xlnx,zocl-versal", .data = &zdev_data_versal},
	{ /* end of table */ },
};
MODULE_DEVICE_TABLE(of, zocl_drm_of_match);

/*
 *
 * Initialization of Xilinx openCL DRM platform device.
 *
 * @param        pdev: Platform Device Instance
 *
 * @return       0 on success, Error code on failure.
 *
 */
static int zocl_drm_platform_probe(struct platform_device *pdev)
{
	const struct of_device_id *id;
	struct drm_device *drm;
	struct drm_zocl_dev	*zdev;
	struct platform_device *subdev;
	struct resource res_mem;
	struct resource *res;
	struct device_node *fnode;
	int index;
	int irq;
	int ret;
	int year, mon, day;

	id = of_match_node(zocl_drm_of_match, pdev->dev.of_node);
	DRM_INFO("Probing for %s\n", id->compatible);

	/* Create zocl device and initial */
	zdev = devm_kzalloc(&pdev->dev, sizeof(*zdev), GFP_KERNEL);
	if (!zdev)
		return -ENOMEM;

	zdev->zdev_data_info = id->data;
	INIT_LIST_HEAD(&zdev->ctx_list);

	/* Record and get IRQ number */
	for (index = 0; index < MAX_CU_NUM; index++) {
		irq = platform_get_irq(pdev, index);
		if (irq < 0)
			break;
		DRM_DEBUG("CU(%d) IRQ %d\n", index, irq);
		zdev->irq[index] = irq;
	}
	zdev->cu_num = index;

	/* set to 0xFFFFFFFF(32bit) or 0xFFFFFFFFFFFFFFFF(64bit) */
	zdev->host_mem = (phys_addr_t) -1;
	zdev->host_mem_len = 0;
	/* Work around for CR-1119382 issue.
	 * ZOCL driver is crashing if it accessing the device tree node */
	if (ZOCL_PLATFORM_ARM64) {
		/* If reserved memory region are not found, just keep going */
		ret = get_reserved_mem_region(&pdev->dev, &res_mem);
		if (!ret) {
			DRM_INFO("Reserved memory for host at 0x%lx, size 0x%lx\n",
				 (unsigned long)res_mem.start,
				 (unsigned long)resource_size(&res_mem));
			zdev->host_mem = res_mem.start;
			zdev->host_mem_len = resource_size(&res_mem);
		}
	}
	mutex_init(&zdev->mm_lock);
	INIT_LIST_HEAD(&zdev->zm_list_head);

	subdev = zocl_find_pdev("ert_hw");
	if (subdev) {
		DRM_INFO("ert_hw found: 0x%llx\n", (uint64_t)(uintptr_t)subdev);
		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
		if (res) {
			zdev->res_start = res->start;
		}

		zdev->res_start = res->start;
		zdev->ert = (struct zocl_ert_dev *)platform_get_drvdata(subdev);
		//ert_hw is present only for PCIe + PS devices (ex: U30,VCK5000
		//Dont enable new kds for those devices
	}

	subdev = zocl_find_pdev("reset_ps");
	if (subdev) {
		DRM_INFO("reset_ps found: 0x%llx\n", (uint64_t)(uintptr_t)subdev);
		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
		if (!res) {
			DRM_ERROR("The base address of reset_ps is not found or 0\n");
			return -EINVAL;
		}

		zdev->watchdog = platform_get_drvdata(subdev);
	}

	/* Work around for CR-1119382 issue.
	 * ZOCL driver is crashing if it accessing the device tree node */
	if (ZOCL_PLATFORM_ARM64) {
		/* For Non PR platform, there is not need to have FPGA manager
		 * For PR platform, the FPGA manager is required. No good way to
		 * determin if it is a PR platform at probe.
		 */
		fnode = of_find_node_by_name(NULL,
				     zdev->zdev_data_info->fpga_driver_name);
		if (fnode) {
			zdev->fpga_mgr = of_fpga_mgr_get(fnode);
			if (IS_ERR(zdev->fpga_mgr))
				zdev->fpga_mgr = NULL;
			DRM_INFO("FPGA programming device %s founded.\n",
				 zdev->zdev_data_info->fpga_driver_name);
			of_node_put(fnode);
		}
	}

	/* Initialize Aperture */
	ret = zocl_aperture_init(zdev);
	if (ret)
		goto err_apt;

	/* Initialzie Slot */
	ret = zocl_pr_slot_init(zdev, pdev);
	if (ret)
		goto err_drm;

	/* Initialzie IOMMU */
	if (iommu_present(&platform_bus_type)) {
		/*
		 * Note: we ignore the return value of zocl_iommu_init().
		 * In the case of failing to initialize iommu, zocl
		 * driver will keep working with iommu disabled.
		 */
		(void) zocl_iommu_init(zdev, pdev);
	}

	platform_set_drvdata(pdev, zdev);

	/* Work around for CR-1119382 issue.
	 * ZOCL driver is crashing if it accessing the device tree node */
	if (ZOCL_PLATFORM_ARM64) {
		sscanf(XRT_DRIVER_VERSION, "%d.%d.%d",
		       &zocl_driver.major,
		       &zocl_driver.minor,
		       &zocl_driver.patchlevel);
		sscanf(XRT_DATE, "%d-%d-%d ", &year, &mon, &day);
		//e.g HASH_DATE ==> Wed, 4 Nov 2020 08:46:44 -0800
		//e.g XRT_DATE ==> 2020-11-04
		snprintf(driver_date, sizeof(driver_date),
			 "%d%02d%02d", year, mon, day);
	}
	/* Create and register DRM device */
	drm = drm_dev_alloc(&zocl_driver, &pdev->dev);
	if (IS_ERR(drm)) {
		ret = PTR_ERR(drm);
		goto err_drm;
	}

	ret = drm_dev_register(drm, 0);
	if (ret)
		goto err_sysfs;

	/* During attach, we don't request dma channel */
	zdev->zdev_dma_chan = NULL;

	/* doen with zdev initialization */
	drm->dev_private = zdev;
	zdev->ddev       = drm;

	ret = zocl_init_error(zdev);
	if (ret)
		goto err_sysfs;

	mutex_init(&zdev->aie_lock);

	/* Initial sysfs */
	rwlock_init(&zdev->attr_rwlock);
	ret = zocl_init_sysfs(drm->dev);
	if (ret)
		goto err_err;

	/* Now initial kds */
	ret = zocl_init_sched(zdev);
	if (ret)
		goto err_sched;

	return 0;

/* error out in exact reverse order of init */
err_sched:
	zocl_fini_sysfs(drm->dev);
err_err:
	mutex_destroy(&zdev->aie_lock);
	zocl_fini_error(zdev);
err_sysfs:
	ZOCL_DRM_DEV_PUT(drm);
err_drm:
	zocl_pr_slot_fini(zdev);
err_apt:
	zocl_aperture_fini(zdev);
	return ret;
}

/*
 *
 * Exit Xilinx openCL DRM platform device.
 *
 * @param        pdev: Platform Device Instance
 *
 * @return       0 on success, Error code on failure.
 *
 */
static int zocl_drm_platform_remove(struct platform_device *pdev)
{
	struct drm_zocl_dev *zdev = platform_get_drvdata(pdev);
	struct drm_device *drm = zdev->ddev;

	/* Cleanup of iommu domain, if exists */
	if (zdev->domain) {
		iommu_detach_device(zdev->domain, drm->dev);
		iommu_domain_free(zdev->domain);
	}

	/* If dma channel has been requested, make sure it is released */
	if (zdev->zdev_dma_chan) {
		dma_release_channel(zdev->zdev_dma_chan);
		zdev->zdev_dma_chan = NULL;
	}

	if (zdev->fpga_mgr)
		fpga_mgr_put(zdev->fpga_mgr);

	zocl_clear_mem(zdev);
	mutex_destroy(&zdev->mm_lock);
	zocl_pr_slot_fini(zdev);
	zocl_destroy_aie(zdev);
	mutex_destroy(&zdev->aie_lock);
	zocl_fini_sysfs(drm->dev);
	zocl_fini_error(zdev);

	zocl_fini_sched(zdev);

	if (zdev->apertures)
		kfree(zdev->apertures);

	drm_dev_unregister(drm);
	ZOCL_DRM_DEV_PUT(drm);

	return 0;
}

static struct platform_driver zocl_drm_private_driver = {
	.probe			= zocl_drm_platform_probe,
	.remove			= zocl_drm_platform_remove,
	.driver			= {
		.name	        = "zocl-drm",
		.of_match_table	= zocl_drm_of_match,
	},
};

static struct platform_driver *drivers[] = {
	&zocl_watchdog_driver,
	&zocl_ospi_versal_driver,
	&cu_driver,
	&scu_driver,
	&zocl_drm_private_driver,
	&zocl_csr_intc_driver,
	&zocl_xgq_intc_driver,
	&zocl_cu_xgq_driver,
	&zocl_ctrl_ert_driver,
	&zocl_rpu_channel_driver,
};

static int __init zocl_init(void)
{
	int ret = 0, total = ARRAY_SIZE(drivers), i;

	/* HACK: fix ert driver. */
	if (!enable_xgq_ert) {
		for (i = 0; i < total && ret >= 0; i++) {
			if (drivers[i] == &zocl_ctrl_ert_driver) {
				drivers[i] = &zocl_ert_driver;
				break;
			}
		}
	}

	for (i = 0; i < total && ret >= 0; i++)
		ret = platform_driver_register(drivers[i]);
	if (ret >= 0)
		return 0;

	/* Failed to register some drivers, undo. */
	while (--i >= 0)
		platform_driver_unregister(drivers[i]);
	return ret;
}
module_init(zocl_init);

static void __exit zocl_exit(void)
{
	int i = ARRAY_SIZE(drivers);

	while (--i >= 0)
		platform_driver_unregister(drivers[i]);
}
module_exit(zocl_exit);

MODULE_VERSION(XRT_DRIVER_VERSION);

MODULE_DESCRIPTION(ZOCL_DRIVER_DESC);
MODULE_AUTHOR("Sonal Santan <sonal.santan@xilinx.com>");
MODULE_LICENSE("GPL");
