gpu_scheduler.c

/*
 * Copyright 2015 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 *
 */
#include <linux/kthread.h>
#include <linux/wait.h>
#include <linux/sched.h>
#include <drm/drmP.h>
#include "gpu_scheduler.h"

/* Initialize a given run queue struct */
static void init_rq(struct amd_run_queue *rq)
{
	INIT_LIST_HEAD(&rq->head.list);
	rq->head.belongto_rq = rq;
	mutex_init(&rq->lock);
	atomic_set(&rq->nr_entity, 0);
	rq->current_entity = &rq->head;
}

/* Note: caller must hold the lock or in a atomic context */
static void rq_remove_entity(struct amd_run_queue *rq,
			     struct amd_sched_entity *entity)
{
	if (rq->current_entity == entity)
		rq->current_entity = list_entry(entity->list.prev,
						typeof(*entity), list);
	list_del_init(&entity->list);
	atomic_dec(&rq->nr_entity);
}

static void rq_add_entity(struct amd_run_queue *rq,
			  struct amd_sched_entity *entity)
{
	list_add_tail(&entity->list, &rq->head.list);
	atomic_inc(&rq->nr_entity);
}

/**
 * Select next entity from a specified run queue with round robin policy.
 * It could return the same entity as current one if current is the only
 * available one in the queue. Return NULL if nothing available.
 */
static struct amd_sched_entity *rq_select_entity(struct amd_run_queue *rq)
{
	struct amd_sched_entity *p = rq->current_entity;
	int i = atomic_read(&rq->nr_entity) + 1; /*real count + dummy head*/
	while (i) {
		p = list_entry(p->list.next, typeof(*p), list);
		if (!rq->check_entity_status(p)) {
			rq->current_entity = p;
			break;
		}
		i--;
	}
	return i ? p : NULL;
}

static bool context_entity_is_waiting(struct amd_context_entity *entity)
{
	/* TODO: sync obj for multi-ring synchronization */
	return false;
}

static int gpu_entity_check_status(struct amd_sched_entity *entity)
{
	struct amd_context_entity *tmp = NULL;

	if (entity == &entity->belongto_rq->head)
		return -1;

	tmp = container_of(entity, typeof(*tmp), generic_entity);
	if (kfifo_is_empty(&tmp->job_queue) ||
	    context_entity_is_waiting(tmp))
		return -1;

	return 0;
}

/**
 * Note: This function should only been called inside scheduler main
 * function for thread safety, there is no other protection here.
 * return ture if scheduler has something ready to run.
 *
 * For active_hw_rq, there is only one producer(scheduler thread) and
 * one consumer(ISR). It should be safe to use this function in scheduler
 * main thread to decide whether to continue emit more IBs.
*/
static bool is_scheduler_ready(struct amd_gpu_scheduler *sched)
{
	return !kfifo_is_full(&sched->active_hw_rq);
}

/**
 * Select next entity from the kernel run queue, if not available,
 * return null.
*/
static struct amd_context_entity *kernel_rq_select_context(
	struct amd_gpu_scheduler *sched)
{
	struct amd_sched_entity *sched_entity = NULL;
	struct amd_context_entity *tmp = NULL;
	struct amd_run_queue *rq = &sched->kernel_rq;

	mutex_lock(&rq->lock);
	sched_entity = rq_select_entity(rq);
	if (sched_entity)
		tmp = container_of(sched_entity,
				   typeof(*tmp),
				   generic_entity);
	mutex_unlock(&rq->lock);
	return tmp;
}

/**
 * Select next entity containing real IB submissions
*/
static struct amd_context_entity *select_context(
	struct amd_gpu_scheduler *sched)
{
	struct amd_context_entity *wake_entity = NULL;
	struct amd_context_entity *tmp;
	struct amd_run_queue *rq;

	if (!is_scheduler_ready(sched))
		return NULL;

	/* Kernel run queue has higher priority than normal run queue*/
	tmp = kernel_rq_select_context(sched);
	if (tmp != NULL)
		goto exit;

	WARN_ON(offsetof(struct amd_context_entity, generic_entity) != 0);

	rq = &sched->sched_rq;
	mutex_lock(&rq->lock);
	tmp = container_of(rq_select_entity(rq),
			   typeof(*tmp), generic_entity);
	mutex_unlock(&rq->lock);
exit:
	if (sched->current_entity && (sched->current_entity != tmp))
		wake_entity = sched->current_entity;
	sched->current_entity = tmp;
	if (wake_entity)
		wake_up(&wake_entity->wait_queue);
	return tmp;
}

/**
 * Init a context entity used by scheduler when submit to HW ring.
 *
 * @sched	The pointer to the scheduler
 * @entity	The pointer to a valid amd_context_entity
 * @parent	The parent entity of this amd_context_entity
 * @rq		The run queue this entity belongs
 * @context_id	The context id for this entity
 * @jobs	The max number of jobs in the job queue
 *
 * return 0 if succeed. negative error code on failure
*/
int amd_context_entity_init(struct amd_gpu_scheduler *sched,
			    struct amd_context_entity *entity,
			    struct amd_sched_entity *parent,
			    struct amd_run_queue *rq,
			    uint32_t context_id,
			    uint32_t jobs)
{
	uint64_t seq_ring = 0;

	if (!(sched && entity && rq))
		return -EINVAL;

	memset(entity, 0, sizeof(struct amd_context_entity));
	seq_ring = ((uint64_t)sched->ring_id) << 60;
	spin_lock_init(&entity->lock);
	entity->generic_entity.belongto_rq = rq;
	entity->generic_entity.parent = parent;
	entity->scheduler = sched;
	init_waitqueue_head(&entity->wait_queue);
	init_waitqueue_head(&entity->wait_emit);
	if(kfifo_alloc(&entity->job_queue,
		       jobs * sizeof(void *),
		       GFP_KERNEL))
		return -EINVAL;

	spin_lock_init(&entity->queue_lock);
	entity->tgid = (context_id == AMD_KERNEL_CONTEXT_ID) ?
		AMD_KERNEL_PROCESS_ID : current->tgid;
	entity->context_id = context_id;
	atomic64_set(&entity->last_emitted_v_seq, seq_ring);
	atomic64_set(&entity->last_queued_v_seq, seq_ring);
	atomic64_set(&entity->last_signaled_v_seq, seq_ring);

	/* Add the entity to the run queue */
	mutex_lock(&rq->lock);
	rq_add_entity(rq, &entity->generic_entity);
	mutex_unlock(&rq->lock);
	return 0;
}

/**
 * Query if entity is initialized
 *
 * @sched       Pointer to scheduler instance
 * @entity	The pointer to a valid scheduler entity
 *
 * return true if entity is initialized, false otherwise
*/
static bool is_context_entity_initialized(struct amd_gpu_scheduler *sched,
					  struct amd_context_entity *entity)
{
	return entity->scheduler == sched &&
		entity->generic_entity.belongto_rq != NULL;
}

static bool is_context_entity_idle(struct amd_gpu_scheduler *sched,
				   struct amd_context_entity *entity)
{
	/**
	 * Idle means no pending IBs, and the entity is not
	 * currently being used.
	*/
	barrier();
	if ((sched->current_entity != entity) &&
	    kfifo_is_empty(&entity->job_queue))
		return true;

	return false;
}

/**
 * Destroy a context entity
 *
 * @sched       Pointer to scheduler instance
 * @entity	The pointer to a valid scheduler entity
 *
 * return 0 if succeed. negative error code on failure
 */
int amd_context_entity_fini(struct amd_gpu_scheduler *sched,
			    struct amd_context_entity *entity)
{
	int r = 0;
	struct amd_run_queue *rq = entity->generic_entity.belongto_rq;

	if (!is_context_entity_initialized(sched, entity))
		return 0;

	/**
	 * The client will not queue more IBs during this fini, consume existing
	 * queued IBs
	*/
	r = wait_event_timeout(
		entity->wait_queue,
		is_context_entity_idle(sched, entity),
		msecs_to_jiffies(AMD_GPU_WAIT_IDLE_TIMEOUT_IN_MS)
		) ?  0 : -1;

	if (r) {
		if (entity->is_pending)
			DRM_INFO("Entity %u is in waiting state during fini,\
				all pending ibs will be canceled.\n",
				 entity->context_id);
	}

	mutex_lock(&rq->lock);
	rq_remove_entity(rq, &entity->generic_entity);
	mutex_unlock(&rq->lock);
	kfifo_free(&entity->job_queue);
	return r;
}

/**
 * Submit a normal job to the job queue
 *
 * @sched	The pointer to the scheduler
 * @c_entity    The pointer to amd_context_entity
 * @job		The pointer to job required to submit
 * return 0 if succeed. -1 if failed.
 *        -2 indicate queue is full for this client, client should wait untill
 *	     scheduler consum some queued command.
 *	  -1 other fail.
*/
int amd_sched_push_job(struct amd_gpu_scheduler *sched,
		       struct amd_context_entity *c_entity,
		       void *job)
{
	while (kfifo_in_spinlocked(&c_entity->job_queue, &job, sizeof(void *),
				   &c_entity->queue_lock) != sizeof(void *)) {
		/**
		 * Current context used up all its IB slots
		 * wait here, or need to check whether GPU is hung
		*/
		schedule();
	}

	wake_up_interruptible(&sched->wait_queue);
	return 0;
}

/**
 * Check the virtual sequence number for specified context
 *
 * @seq		The virtual sequence number to check
 * @c_entity	The pointer to a valid amd_context_entity
 *
 * return 0 if signaled, -1 else.
*/
int amd_sched_check_ts(struct amd_context_entity *c_entity, uint64_t seq)
{
	return (seq <= atomic64_read(&c_entity->last_signaled_v_seq)) ? 0 : -1;
}

/**
 * Wait for a virtual sequence number to be signaled or timeout
 *
 * @c_entity	The pointer to a valid context entity
 * @seq         The virtual sequence number to wait
 * @intr	Interruptible or not
 * @timeout	Timeout in ms, wait infinitely if <0
 * @emit        wait for emit or signal
 *
 * return =0 signaled ,  <0 failed
*/
static int amd_sched_wait(struct amd_context_entity *c_entity,
			  uint64_t seq,
			  bool intr,
			  long timeout,
			  bool emit)
{
	atomic64_t *v_seq = emit ? &c_entity->last_emitted_v_seq :
		&c_entity->last_signaled_v_seq;
	wait_queue_head_t *wait_queue = emit ? &c_entity->wait_emit :
		&c_entity->wait_queue;

	if (intr && (timeout < 0)) {
		wait_event_interruptible(
			*wait_queue,
			seq <= atomic64_read(v_seq));
		return 0;
	} else if (intr && (timeout >= 0)) {
		wait_event_interruptible_timeout(
			*wait_queue,
			seq <= atomic64_read(v_seq),
			msecs_to_jiffies(timeout));
		return (seq <= atomic64_read(v_seq)) ?
			0 : -1;
	} else if (!intr && (timeout < 0)) {
		wait_event(
			*wait_queue,
			seq <= atomic64_read(v_seq));
		return 0;
	} else if (!intr && (timeout >= 0)) {
		wait_event_timeout(
			*wait_queue,
			seq <= atomic64_read(v_seq),
			msecs_to_jiffies(timeout));
		return (seq <= atomic64_read(v_seq)) ?
			0 : -1;
	}
	return 0;
}

int amd_sched_wait_signal(struct amd_context_entity *c_entity,
			  uint64_t seq,
			  bool intr,
			  long timeout)
{
	return amd_sched_wait(c_entity, seq, intr, timeout, false);
}

int amd_sched_wait_emit(struct amd_context_entity *c_entity,
			uint64_t seq,
			bool intr,
			long timeout)
{
	return amd_sched_wait(c_entity, seq, intr, timeout, true);
}

static int amd_sched_main(void *param)
{
	int r;
	void *job;
	struct sched_param sparam = {.sched_priority = 1};
	struct amd_context_entity *c_entity = NULL;
	struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param;

	sched_setscheduler(current, SCHED_FIFO, &sparam);

	while (!kthread_should_stop()) {
		wait_event_interruptible(sched->wait_queue,
					 is_scheduler_ready(sched) &&
					 (c_entity = select_context(sched)));
		r = kfifo_out(&c_entity->job_queue, &job, sizeof(void *));
		if (r != sizeof(void *))
			continue;
		r = sched->ops->prepare_job(sched, c_entity, job);
		if (!r)
			WARN_ON(kfifo_in_spinlocked(
					&sched->active_hw_rq,
					&job,
					sizeof(void *),
					&sched->queue_lock) != sizeof(void *));
		mutex_lock(&sched->sched_lock);
		sched->ops->run_job(sched, c_entity, job);
		mutex_unlock(&sched->sched_lock);
	}
	return 0;
}

uint64_t amd_sched_get_handled_seq(struct amd_gpu_scheduler *sched)
{
	return sched->last_handled_seq;
}

/**
 * ISR to handle EOP inetrrupts
 *
 * @sched: gpu scheduler
 *
*/
void amd_sched_isr(struct amd_gpu_scheduler *sched)
{
	int r;
	void *job;
	r = kfifo_out_spinlocked(&sched->active_hw_rq,
				 &job, sizeof(void *),
				 &sched->queue_lock);

	if (r != sizeof(void *))
		job = NULL;

	sched->ops->process_job(sched, job);
	sched->last_handled_seq++;
	wake_up_interruptible(&sched->wait_queue);
}

/**
 * Create a gpu scheduler
 *
 * @device	The device context for this scheduler
 * @ops		The backend operations for this scheduler.
 * @id	        The scheduler is per ring, here is ring id.
 * @granularity	The minumum ms unit the scheduler will scheduled.
 * @preemption  Indicate whether this ring support preemption, 0 is no.
 *
 * return the pointer to scheduler for success, otherwise return NULL
*/
struct amd_gpu_scheduler *amd_sched_create(void *device,
					   struct amd_sched_backend_ops *ops,
					   unsigned ring,
					   unsigned granularity,
					   unsigned preemption,
					   unsigned hw_submission)
{
	struct amd_gpu_scheduler *sched;
	char name[20] = "gpu_sched[0]";

	sched = kzalloc(sizeof(struct amd_gpu_scheduler), GFP_KERNEL);
	if (!sched)
		return NULL;

	sched->device = device;
	sched->ops = ops;
	sched->granularity = granularity;
	sched->ring_id = ring;
	sched->preemption = preemption;
	sched->last_handled_seq = 0;

	snprintf(name, sizeof(name), "gpu_sched[%d]", ring);
	mutex_init(&sched->sched_lock);
	spin_lock_init(&sched->queue_lock);
	init_rq(&sched->sched_rq);
	sched->sched_rq.check_entity_status = gpu_entity_check_status;

	init_rq(&sched->kernel_rq);
	sched->kernel_rq.check_entity_status = gpu_entity_check_status;

	init_waitqueue_head(&sched->wait_queue);
	if(kfifo_alloc(&sched->active_hw_rq,
		       hw_submission * sizeof(void *),
		       GFP_KERNEL)) {
		kfree(sched);
		return NULL;
	}

	/* Each scheduler will run on a seperate kernel thread */
	sched->thread = kthread_create(amd_sched_main, sched, name);
	if (sched->thread) {
		wake_up_process(sched->thread);
		return sched;
	}

	DRM_ERROR("Failed to create scheduler for id %d.\n", ring);
	kfifo_free(&sched->active_hw_rq);
	kfree(sched);
	return NULL;
}

/**
 * Destroy a gpu scheduler
 *
 * @sched	The pointer to the scheduler
 *
 * return 0 if succeed. -1 if failed.
 */
int amd_sched_destroy(struct amd_gpu_scheduler *sched)
{
	kthread_stop(sched->thread);
	kfifo_free(&sched->active_hw_rq);
	kfree(sched);
	return  0;
}