kfd_priv.h 40.8 KB
Newer Older
O
Oded Gabbay 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
/*
 * Copyright 2014 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

#ifndef KFD_PRIV_H_INCLUDED
#define KFD_PRIV_H_INCLUDED

#include <linux/hashtable.h>
#include <linux/mmu_notifier.h>
#include <linux/mutex.h>
#include <linux/types.h>
#include <linux/atomic.h>
#include <linux/workqueue.h>
#include <linux/spinlock.h>
33
#include <linux/kfd_ioctl.h>
34
#include <linux/idr.h>
35
#include <linux/kfifo.h>
36
#include <linux/seq_file.h>
37
#include <linux/kref.h>
38
#include <linux/sysfs.h>
39
#include <linux/device_cgroup.h>
40 41 42
#include <drm/drm_file.h>
#include <drm/drm_drv.h>
#include <drm/drm_device.h>
43
#include <drm/drm_ioctl.h>
O
Oded Gabbay 已提交
44
#include <kgd_kfd_interface.h>
45
#include <linux/swap.h>
O
Oded Gabbay 已提交
46

47
#include "amd_shared.h"
48
#include "amdgpu.h"
49

L
Laura Abbott 已提交
50 51
#define KFD_MAX_RING_ENTRY_SIZE	8

52 53
#define KFD_SYSFS_FILE_MODE 0444

54 55 56 57 58 59 60 61 62 63 64
/* GPU ID hash width in bits */
#define KFD_GPU_ID_HASH_WIDTH 16

/* Use upper bits of mmap offset to store KFD driver specific information.
 * BITS[63:62] - Encode MMAP type
 * BITS[61:46] - Encode gpu_id. To identify to which GPU the offset belongs to
 * BITS[45:0]  - MMAP offset value
 *
 * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these
 *  defines are w.r.t to PAGE_SIZE
 */
65
#define KFD_MMAP_TYPE_SHIFT	62
66 67 68 69
#define KFD_MMAP_TYPE_MASK	(0x3ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_TYPE_DOORBELL	(0x3ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_TYPE_EVENTS	(0x2ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_TYPE_RESERVED_MEM	(0x1ULL << KFD_MMAP_TYPE_SHIFT)
O
Oak Zeng 已提交
70
#define KFD_MMAP_TYPE_MMIO	(0x0ULL << KFD_MMAP_TYPE_SHIFT)
71

72
#define KFD_MMAP_GPU_ID_SHIFT 46
73 74 75 76
#define KFD_MMAP_GPU_ID_MASK (((1ULL << KFD_GPU_ID_HASH_WIDTH) - 1) \
				<< KFD_MMAP_GPU_ID_SHIFT)
#define KFD_MMAP_GPU_ID(gpu_id) ((((uint64_t)gpu_id) << KFD_MMAP_GPU_ID_SHIFT)\
				& KFD_MMAP_GPU_ID_MASK)
77
#define KFD_MMAP_GET_GPU_ID(offset)    ((offset & KFD_MMAP_GPU_ID_MASK) \
78 79
				>> KFD_MMAP_GPU_ID_SHIFT)

B
Ben Goz 已提交
80 81
/*
 * When working with cp scheduler we should assign the HIQ manually or via
82
 * the amdgpu driver to a fixed hqd slot, here are the fixed HIQ hqd slot
B
Ben Goz 已提交
83 84 85 86 87 88 89
 * definitions for Kaveri. In Kaveri only the first ME queues participates
 * in the cp scheduling taking that in mind we set the HIQ slot in the
 * second ME.
 */
#define KFD_CIK_HIQ_PIPE 4
#define KFD_CIK_HIQ_QUEUE 0

90 91 92 93
/* Macro for allocating structures */
#define kfd_alloc_struct(ptr_to_struct)	\
	((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL))

94
#define KFD_MAX_NUM_OF_PROCESSES 512
95
#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
96

F
Felix Kuehling 已提交
97 98 99 100
/*
 * Size of the per-process TBA+TMA buffer: 2 pages
 *
 * The first page is the TBA used for the CWSR ISA code. The second
101
 * page is used as TMA for user-mode trap handler setup in daisy-chain mode.
F
Felix Kuehling 已提交
102 103 104 105
 */
#define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2)
#define KFD_CWSR_TMA_OFFSET PAGE_SIZE

106 107 108 109 110 111
#define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE		\
	(KFD_MAX_NUM_OF_PROCESSES *			\
			KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)

#define KFD_KERNEL_QUEUE_SIZE 2048

112 113
#define KFD_UNMAP_LATENCY_MS	(4000)

114 115 116 117 118 119 120 121 122 123
/*
 * 512 = 0x200
 * The doorbell index distance between SDMA RLC (2*i) and (2*i+1) in the
 * same SDMA engine on SOC15, which has 8-byte doorbells for SDMA.
 * 512 8-byte doorbell distance (i.e. one page away) ensures that SDMA RLC
 * (2*i+1) doorbells (in terms of the lower 12 bit address) lie exactly in
 * the OFFSET and SIZE set in registers like BIF_SDMA0_DOORBELL_RANGE.
 */
#define KFD_QUEUE_DOORBELL_MIRROR_OFFSET 512

124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
/**
 * enum kfd_ioctl_flags - KFD ioctl flags
 * Various flags that can be set in &amdkfd_ioctl_desc.flags to control how
 * userspace can use a given ioctl.
 */
enum kfd_ioctl_flags {
	/*
	 * @KFD_IOC_FLAG_CHECKPOINT_RESTORE:
	 * Certain KFD ioctls such as AMDKFD_IOC_CRIU_OP can potentially
	 * perform privileged operations and load arbitrary data into MQDs and
	 * eventually HQD registers when the queue is mapped by HWS. In order to
	 * prevent this we should perform additional security checks.
	 *
	 * This is equivalent to callers with the CHECKPOINT_RESTORE capability.
	 *
	 * Note: Since earlier versions of docker do not support CHECKPOINT_RESTORE,
	 * we also allow ioctls with SYS_ADMIN capability.
	 */
	KFD_IOC_FLAG_CHECKPOINT_RESTORE = BIT(0),
};
144
/*
145 146
 * Kernel module parameter to specify maximum number of supported queues per
 * device
147
 */
148
extern int max_num_of_queues_per_device;
149

B
Ben Goz 已提交
150

151 152 153
/* Kernel module parameter to specify the scheduling policy */
extern int sched_policy;

154 155 156 157 158 159
/*
 * Kernel module parameter to specify the maximum process
 * number per HW scheduler
 */
extern int hws_max_conc_proc;

F
Felix Kuehling 已提交
160 161
extern int cwsr_enable;

162 163 164 165 166 167
/*
 * Kernel module parameter to specify whether to send sigterm to HSA process on
 * unhandled exception
 */
extern int send_sigterm;

168 169 170 171 172 173
/*
 * This kernel module is used to simulate large bar machine on non-large bar
 * enabled machines.
 */
extern int debug_largebar;

174 175 176 177 178 179
/*
 * Ignore CRAT table during KFD initialization, can be used to work around
 * broken CRAT tables on some AMD systems
 */
extern int ignore_crat;

180
/* Set sh_mem_config.retry_disable on GFX v9 */
181
extern int amdgpu_noretry;
182

183
/* Halt if HWS hang is detected */
184 185
extern int halt_if_hws_hang;

186
/* Whether MEC FW support GWS barriers */
187 188
extern bool hws_gws_support;

189
/* Queue preemption timeout in ms */
190 191
extern int queue_preemption_timeout_ms;

192 193 194 195 196
/*
 * Don't evict process queues on vm fault
 */
extern int amdgpu_no_queue_eviction_on_vm_fault;

197
/* Enable eviction debug messages */
198 199
extern bool debug_evictions;

B
Ben Goz 已提交
200 201 202 203 204
enum cache_policy {
	cache_policy_coherent,
	cache_policy_noncoherent
};

205 206
#define KFD_GC_VERSION(dev) ((dev)->adev->ip_versions[GC_HWIP][0])
#define KFD_IS_SOC15(dev)   ((KFD_GC_VERSION(dev)) >= (IP_VERSION(9, 0, 1)))
207

208 209
struct kfd_event_interrupt_class {
	bool (*interrupt_isr)(struct kfd_dev *dev,
210 211
			const uint32_t *ih_ring_entry, uint32_t *patched_ihre,
			bool *patched_flag);
212
	void (*interrupt_wq)(struct kfd_dev *dev,
213
			const uint32_t *ih_ring_entry);
214 215
};

O
Oded Gabbay 已提交
216
struct kfd_device_info {
217
	uint32_t gfx_target_version;
218
	const struct kfd_event_interrupt_class *event_interrupt_class;
O
Oded Gabbay 已提交
219
	unsigned int max_pasid_bits;
220
	unsigned int max_no_of_hqd;
221
	unsigned int doorbell_size;
O
Oded Gabbay 已提交
222
	size_t ih_ring_entry_size;
223
	uint8_t num_of_watch_points;
224
	uint16_t mqd_size_aligned;
F
Felix Kuehling 已提交
225
	bool supports_cwsr;
226
	bool needs_iommu_device;
227
	bool needs_pci_atomics;
228
	uint32_t no_atomic_fw_version;
229
	unsigned int num_sdma_queues_per_engine;
O
Oded Gabbay 已提交
230 231
};

232 233 234
unsigned int kfd_get_num_sdma_engines(struct kfd_dev *kdev);
unsigned int kfd_get_num_xgmi_sdma_engines(struct kfd_dev *kdev);

235 236 237 238 239
struct kfd_mem_obj {
	uint32_t range_start;
	uint32_t range_end;
	uint64_t gpu_addr;
	uint32_t *cpu_ptr;
240
	void *gtt_mem;
241 242
};

243 244 245 246 247 248
struct kfd_vmid_info {
	uint32_t first_vmid_kfd;
	uint32_t last_vmid_kfd;
	uint32_t vmid_num_kfd;
};

O
Oded Gabbay 已提交
249
struct kfd_dev {
250
	struct amdgpu_device *adev;
O
Oded Gabbay 已提交
251

252
	struct kfd_device_info device_info;
O
Oded Gabbay 已提交
253
	struct pci_dev *pdev;
254
	struct drm_device *ddev;
O
Oded Gabbay 已提交
255 256 257

	unsigned int id;		/* topology stub index */

258 259 260 261
	phys_addr_t doorbell_base;	/* Start of actual doorbells used by
					 * KFD. It is aligned for mapping
					 * into user mode
					 */
262 263 264 265
	size_t doorbell_base_dw_offset;	/* Offset from the start of the PCI
					 * doorbell BAR to the first KFD
					 * doorbell in dwords. GFX reserves
					 * the segment before this offset.
266 267 268 269 270
					 */
	u32 __iomem *doorbell_kernel_ptr; /* This is a pointer for a doorbells
					   * page used by kernel queue
					   */

O
Oded Gabbay 已提交
271
	struct kgd2kfd_shared_resources shared_resources;
272
	struct kfd_vmid_info vm_info;
O
Oded Gabbay 已提交
273

274 275
	const struct kfd2kgd_calls *kfd2kgd;
	struct mutex doorbell_mutex;
J
Joe Perches 已提交
276 277
	DECLARE_BITMAP(doorbell_available_index,
			KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
278

279 280 281 282 283 284 285 286
	void *gtt_mem;
	uint64_t gtt_start_gpu_addr;
	void *gtt_start_cpu_ptr;
	void *gtt_sa_bitmap;
	struct mutex gtt_sa_lock;
	unsigned int gtt_sa_chunk_size;
	unsigned int gtt_sa_num_of_chunks;

287
	/* Interrupts */
288
	struct kfifo ih_fifo;
289
	struct workqueue_struct *ih_wq;
290 291 292
	struct work_struct interrupt_work;
	spinlock_t interrupt_lock;

B
Ben Goz 已提交
293 294
	/* QCM Device instance */
	struct device_queue_manager *dqm;
O
Oded Gabbay 已提交
295

B
Ben Goz 已提交
296
	bool init_complete;
297 298 299 300 301
	/*
	 * Interrupts of interest to KFD are copied
	 * from the HW ring into a SW ring.
	 */
	bool interrupts_active;
302 303

	/* Debug manager */
K
Kent Russell 已提交
304
	struct kfd_dbgmgr *dbgmgr;
F
Felix Kuehling 已提交
305

306 307
	/* Firmware versions */
	uint16_t mec_fw_version;
308
	uint16_t mec2_fw_version;
309 310
	uint16_t sdma_fw_version;

311 312 313
	/* Maximum process number mapped to HW scheduler */
	unsigned int max_proc_per_quantum;

F
Felix Kuehling 已提交
314 315 316 317
	/* CWSR */
	bool cwsr_enabled;
	const void *cwsr_isa;
	unsigned int cwsr_isa_size;
318 319 320

	/* xGMI */
	uint64_t hive_id;
321

322
	bool pci_atomic_requested;
323

324 325 326
	/* Use IOMMU v2 flag */
	bool use_iommu_v2;

327 328
	/* SRAM ECC flag */
	atomic_t sram_ecc_flag;
329 330 331

	/* Compute Profile ref. count */
	atomic_t compute_profile;
332

333
	/* Global GWS resource shared between processes */
334
	void *gws;
A
Amber Lin 已提交
335 336 337 338

	/* Clients watching SMI events */
	struct list_head smi_clients;
	spinlock_t smi_lock;
339 340

	uint32_t reset_seq_num;
341 342 343

	struct ida doorbell_ida;
	unsigned int max_doorbell_slices;
344 345

	int noretry;
346 347 348

	/* HMM page migration MEMORY_DEVICE_PRIVATE mapping */
	struct dev_pagemap pgmap;
O
Oded Gabbay 已提交
349 350
};

351 352 353 354 355 356
enum kfd_mempool {
	KFD_MEMPOOL_SYSTEM_CACHEABLE = 1,
	KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2,
	KFD_MEMPOOL_FRAMEBUFFER = 3,
};

O
Oded Gabbay 已提交
357 358 359 360 361
/* Character device interface */
int kfd_chardev_init(void);
void kfd_chardev_exit(void);
struct device *kfd_chardev(void);

B
Ben Goz 已提交
362
/**
363
 * enum kfd_unmap_queues_filter - Enum for queue filters.
B
Ben Goz 已提交
364
 *
365
 * @KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE: Preempts single queue.
B
Ben Goz 已提交
366
 *
367
 * @KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES: Preempts all queues in the
B
Ben Goz 已提交
368 369
 *						running queues list.
 *
370
 * @KFD_UNMAP_QUEUES_FILTER_BY_PASID: Preempts queues that belongs to
B
Ben Goz 已提交
371 372 373
 *						specific process.
 *
 */
374 375 376 377 378
enum kfd_unmap_queues_filter {
	KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE,
	KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
	KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
	KFD_UNMAP_QUEUES_FILTER_BY_PASID
B
Ben Goz 已提交
379
};
380

B
Ben Goz 已提交
381
/**
382
 * enum kfd_queue_type - Enum for various queue types.
B
Ben Goz 已提交
383 384 385
 *
 * @KFD_QUEUE_TYPE_COMPUTE: Regular user mode queue type.
 *
386
 * @KFD_QUEUE_TYPE_SDMA: SDMA user mode queue type.
B
Ben Goz 已提交
387 388 389 390
 *
 * @KFD_QUEUE_TYPE_HIQ: HIQ queue type.
 *
 * @KFD_QUEUE_TYPE_DIQ: DIQ queue type.
391 392
 *
 * @KFD_QUEUE_TYPE_SDMA_XGMI: Special SDMA queue for XGMI interface.
B
Ben Goz 已提交
393 394 395 396 397
 */
enum kfd_queue_type  {
	KFD_QUEUE_TYPE_COMPUTE,
	KFD_QUEUE_TYPE_SDMA,
	KFD_QUEUE_TYPE_HIQ,
398 399
	KFD_QUEUE_TYPE_DIQ,
	KFD_QUEUE_TYPE_SDMA_XGMI
B
Ben Goz 已提交
400 401
};

B
Ben Goz 已提交
402 403 404 405 406
enum kfd_queue_format {
	KFD_QUEUE_FORMAT_PM4,
	KFD_QUEUE_FORMAT_AQL
};

407 408 409 410 411
enum KFD_QUEUE_PRIORITY {
	KFD_QUEUE_PRIORITY_MINIMUM = 0,
	KFD_QUEUE_PRIORITY_MAXIMUM = 15
};

B
Ben Goz 已提交
412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437
/**
 * struct queue_properties
 *
 * @type: The queue type.
 *
 * @queue_id: Queue identifier.
 *
 * @queue_address: Queue ring buffer address.
 *
 * @queue_size: Queue ring buffer size.
 *
 * @priority: Defines the queue priority relative to other queues in the
 * process.
 * This is just an indication and HW scheduling may override the priority as
 * necessary while keeping the relative prioritization.
 * the priority granularity is from 0 to f which f is the highest priority.
 * currently all queues are initialized with the highest priority.
 *
 * @queue_percent: This field is partially implemented and currently a zero in
 * this field defines that the queue is non active.
 *
 * @read_ptr: User space address which points to the number of dwords the
 * cp read from the ring buffer. This field updates automatically by the H/W.
 *
 * @write_ptr: Defines the number of dwords written to the ring buffer.
 *
438 439 440
 * @doorbell_ptr: Notifies the H/W of new packet written to the queue ring
 * buffer. This field should be similar to write_ptr and the user should
 * update this field after updating the write_ptr.
B
Ben Goz 已提交
441 442 443
 *
 * @doorbell_off: The doorbell offset in the doorbell pci-bar.
 *
444 445
 * @is_interop: Defines if this is a interop queue. Interop queue means that
 * the queue can access both graphics and compute resources.
B
Ben Goz 已提交
446
 *
447 448 449 450 451
 * @is_evicted: Defines if the queue is evicted. Only active queues
 * are evicted, rendering them inactive.
 *
 * @is_active: Defines if the queue is active or not. @is_active and
 * @is_evicted are protected by the DQM lock.
B
Ben Goz 已提交
452
 *
453 454 455 456
 * @is_gws: Defines if the queue has been updated to be GWS-capable or not.
 * @is_gws should be protected by the DQM lock, since changing it can yield the
 * possibility of updating DQM state on number of GWS queues.
 *
B
Ben Goz 已提交
457 458 459 460 461 462 463
 * @vmid: If the scheduling mode is no cp scheduling the field defines the vmid
 * of the queue.
 *
 * This structure represents the queue properties for each queue no matter if
 * it's user mode or kernel mode queue.
 *
 */
464

B
Ben Goz 已提交
465 466
struct queue_properties {
	enum kfd_queue_type type;
B
Ben Goz 已提交
467
	enum kfd_queue_format format;
B
Ben Goz 已提交
468 469 470 471 472 473 474
	unsigned int queue_id;
	uint64_t queue_address;
	uint64_t  queue_size;
	uint32_t priority;
	uint32_t queue_percent;
	uint32_t *read_ptr;
	uint32_t *write_ptr;
475
	void __iomem *doorbell_ptr;
B
Ben Goz 已提交
476 477
	uint32_t doorbell_off;
	bool is_interop;
478
	bool is_evicted;
B
Ben Goz 已提交
479
	bool is_active;
480
	bool is_gws;
B
Ben Goz 已提交
481 482
	/* Not relevant for user mode queues in cp scheduling */
	unsigned int vmid;
B
Ben Goz 已提交
483 484 485 486
	/* Relevant only for sdma queues*/
	uint32_t sdma_engine_id;
	uint32_t sdma_queue_id;
	uint32_t sdma_vm_addr;
487 488 489 490 491
	/* Relevant only for VI */
	uint64_t eop_ring_buffer_address;
	uint32_t eop_ring_buffer_size;
	uint64_t ctx_save_restore_area_address;
	uint32_t ctx_save_restore_area_size;
F
Felix Kuehling 已提交
492 493 494
	uint32_t ctl_stack_size;
	uint64_t tba_addr;
	uint64_t tma_addr;
B
Ben Goz 已提交
495 496
};

497 498 499 500 501
#define QUEUE_IS_ACTIVE(q) ((q).queue_size > 0 &&	\
			    (q).queue_address != 0 &&	\
			    (q).queue_percent > 0 &&	\
			    !(q).is_evicted)

502 503 504 505 506 507 508 509 510 511 512 513 514
enum mqd_update_flag {
	UPDATE_FLAG_CU_MASK = 0,
};

struct mqd_update_info {
	union {
		struct {
			uint32_t count; /* Must be a multiple of 32 */
			uint32_t *ptr;
		} cu_mask;
	};
	enum mqd_update_flag update_flag;
};
515

B
Ben Goz 已提交
516 517 518 519 520
/**
 * struct queue
 *
 * @list: Queue linked list.
 *
521
 * @mqd: The queue MQD (memory queue descriptor).
B
Ben Goz 已提交
522 523 524 525 526 527 528 529
 *
 * @mqd_mem_obj: The MQD local gpu memory object.
 *
 * @gart_mqd_addr: The MQD gart mc address.
 *
 * @properties: The queue properties.
 *
 * @mec: Used only in no cp scheduling mode and identifies to micro engine id
530
 *	 that the queue should be executed on.
B
Ben Goz 已提交
531
 *
532 533
 * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe
 *	  id.
B
Ben Goz 已提交
534 535 536 537 538 539 540
 *
 * @queue: Used only in no cp scheduliong mode and identifies the queue's slot.
 *
 * @process: The kfd process that created this queue.
 *
 * @device: The kfd device that created this queue.
 *
541 542 543
 * @gws: Pointing to gws kgd_mem if this is a gws control queue; NULL
 * otherwise.
 *
B
Ben Goz 已提交
544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559
 * This structure represents user mode compute queues.
 * It contains all the necessary data to handle such queues.
 *
 */

struct queue {
	struct list_head list;
	void *mqd;
	struct kfd_mem_obj *mqd_mem_obj;
	uint64_t gart_mqd_addr;
	struct queue_properties properties;

	uint32_t mec;
	uint32_t pipe;
	uint32_t queue;

B
Ben Goz 已提交
560
	unsigned int sdma_id;
561
	unsigned int doorbell_id;
B
Ben Goz 已提交
562

B
Ben Goz 已提交
563 564
	struct kfd_process	*process;
	struct kfd_dev		*device;
565
	void *gws;
566 567 568

	/* procfs */
	struct kobject kobj;
B
Ben Goz 已提交
569 570
};

B
Ben Goz 已提交
571
enum KFD_MQD_TYPE {
572
	KFD_MQD_TYPE_HIQ = 0,		/* for hiq */
573 574
	KFD_MQD_TYPE_CP,		/* for cp queues and diq */
	KFD_MQD_TYPE_SDMA,		/* for sdma queues */
575
	KFD_MQD_TYPE_DIQ,		/* for diq */
B
Ben Goz 已提交
576 577 578
	KFD_MQD_TYPE_MAX
};

579 580 581 582 583 584
enum KFD_PIPE_PRIORITY {
	KFD_PIPE_PRIORITY_CS_LOW = 0,
	KFD_PIPE_PRIORITY_CS_MEDIUM,
	KFD_PIPE_PRIORITY_CS_HIGH
};

B
Ben Goz 已提交
585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612
struct scheduling_resources {
	unsigned int vmid_mask;
	enum kfd_queue_type type;
	uint64_t queue_mask;
	uint64_t gws_mask;
	uint32_t oac_mask;
	uint32_t gds_heap_base;
	uint32_t gds_heap_size;
};

struct process_queue_manager {
	/* data */
	struct kfd_process	*process;
	struct list_head	queues;
	unsigned long		*queue_slot_bitmap;
};

struct qcm_process_device {
	/* The Device Queue Manager that owns this data */
	struct device_queue_manager *dqm;
	struct process_queue_manager *pqm;
	/* Queues list */
	struct list_head queues_list;
	struct list_head priv_queue_list;

	unsigned int queue_count;
	unsigned int vmid;
	bool is_debug;
613
	unsigned int evicted; /* eviction counter, 0=active */
614 615 616 617 618 619

	/* This flag tells if we should reset all wavefronts on
	 * process termination
	 */
	bool reset_wavefronts;

620 621 622 623 624 625 626 627
	/* This flag tells us if this process has a GWS-capable
	 * queue that will be mapped into the runlist. It's
	 * possible to request a GWS BO, but not have the queue
	 * currently mapped, and this changes how the MAP_PROCESS
	 * PM4 packet is configured.
	 */
	bool mapped_gws_queue;

628
	/* All the memory management data should be here too */
B
Ben Goz 已提交
629
	uint64_t gds_context_area;
630
	/* Contains page table flags such as AMDGPU_PTE_VALID since gfx9 */
631
	uint64_t page_table_base;
B
Ben Goz 已提交
632 633 634 635 636 637 638
	uint32_t sh_mem_config;
	uint32_t sh_mem_bases;
	uint32_t sh_mem_ape1_base;
	uint32_t sh_mem_ape1_limit;
	uint32_t gds_size;
	uint32_t num_gws;
	uint32_t num_oac;
639
	uint32_t sh_hidden_private_base;
F
Felix Kuehling 已提交
640 641

	/* CWSR memory */
642
	struct kgd_mem *cwsr_mem;
F
Felix Kuehling 已提交
643
	void *cwsr_kaddr;
644
	uint64_t cwsr_base;
F
Felix Kuehling 已提交
645 646
	uint64_t tba_addr;
	uint64_t tma_addr;
647 648

	/* IB memory */
649
	struct kgd_mem *ib_mem;
650
	uint64_t ib_base;
651
	void *ib_kaddr;
652 653 654

	/* doorbell resources per process per device */
	unsigned long *doorbell_bitmap;
B
Ben Goz 已提交
655 656
};

657 658 659 660 661 662 663 664 665
/* KFD Memory Eviction */

/* Approx. wait time before attempting to restore evicted BOs */
#define PROCESS_RESTORE_TIME_MS 100
/* Approx. back off time if restore fails due to lack of memory */
#define PROCESS_BACK_OFF_TIME_MS 100
/* Approx. time before evicting the process again */
#define PROCESS_ACTIVE_TIME_MS 10

666 667 668 669 670 671 672 673
/* 8 byte handle containing GPU ID in the most significant 4 bytes and
 * idr_handle in the least significant 4 bytes
 */
#define MAKE_HANDLE(gpu_id, idr_handle) \
	(((uint64_t)(gpu_id) << 32) + idr_handle)
#define GET_GPU_ID(handle) (handle >> 32)
#define GET_IDR_HANDLE(handle) (handle & 0xFFFFFFFF)

674 675 676 677 678 679
enum kfd_pdd_bound {
	PDD_UNBOUND = 0,
	PDD_BOUND,
	PDD_BOUND_SUSPENDED,
};

680
#define MAX_SYSFS_FILENAME_LEN 15
681 682 683 684 685 686 687

/*
 * SDMA counter runs at 100MHz frequency.
 * We display SDMA activity in microsecond granularity in sysfs.
 * As a result, the divisor is 100.
 */
#define SDMA_ACTIVITY_DIVISOR  100
688

689 690 691 692 693
/* Data that is per-process-per device. */
struct kfd_process_device {
	/* The device that owns this data. */
	struct kfd_dev *dev;

694 695
	/* The process that owns this kfd_process_device. */
	struct kfd_process *process;
696

697 698 699
	/* per-process-per device QCM data structure */
	struct qcm_process_device qpd;

700 701 702 703 704 705 706 707
	/*Apertures*/
	uint64_t lds_base;
	uint64_t lds_limit;
	uint64_t gpuvm_base;
	uint64_t gpuvm_limit;
	uint64_t scratch_base;
	uint64_t scratch_limit;

708
	/* VM context for GPUVM allocations */
709
	struct file *drm_file;
710
	void *drm_priv;
711

712 713 714
	/* GPUVM allocations storage */
	struct idr alloc_idr;

715 716 717 718
	/* Flag used to tell the pdd has dequeued from the dqm.
	 * This is used to prevent dev->dqm->ops.process_termination() from
	 * being called twice when it is already called in IOMMU callback
	 * function.
719
	 */
720
	bool already_dequeued;
721
	bool runtime_inuse;
722 723 724

	/* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */
	enum kfd_pdd_bound bound;
725 726 727 728

	/* VRAM usage */
	uint64_t vram_usage;
	struct attribute attr_vram;
729 730 731 732 733 734
	char vram_filename[MAX_SYSFS_FILENAME_LEN];

	/* SDMA activity tracking */
	uint64_t sdma_past_activity_counter;
	struct attribute attr_sdma;
	char sdma_filename[MAX_SYSFS_FILENAME_LEN];
735 736 737 738 739 740 741

	/* Eviction activity tracking */
	uint64_t last_evict_timestamp;
	atomic64_t evict_duration_counter;
	struct attribute attr_evict;

	struct kobject *kobj_stats;
742
	unsigned int doorbell_index;
743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767

	/*
	 * @cu_occupancy: Reports occupancy of Compute Units (CU) of a process
	 * that is associated with device encoded by "this" struct instance. The
	 * value reflects CU usage by all of the waves launched by this process
	 * on this device. A very important property of occupancy parameter is
	 * that its value is a snapshot of current use.
	 *
	 * Following is to be noted regarding how this parameter is reported:
	 *
	 *  The number of waves that a CU can launch is limited by couple of
	 *  parameters. These are encoded by struct amdgpu_cu_info instance
	 *  that is part of every device definition. For GFX9 devices this
	 *  translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves
	 *  do not use scratch memory and 32 waves (max_scratch_slots_per_cu)
	 *  when they do use scratch memory. This could change for future
	 *  devices and therefore this example should be considered as a guide.
	 *
	 *  All CU's of a device are available for the process. This may not be true
	 *  under certain conditions - e.g. CU masking.
	 *
	 *  Finally number of CU's that are occupied by a process is affected by both
	 *  number of CU's a device has along with number of other competing processes
	 */
	struct attribute attr_cu_occupancy;
768 769 770 771 772 773 774 775 776

	/* sysfs counters for GPU retry fault and page migration tracking */
	struct kobject *kobj_counters;
	struct attribute attr_faults;
	struct attribute attr_page_in;
	struct attribute attr_page_out;
	uint64_t faults;
	uint64_t page_in;
	uint64_t page_out;
777 778 779 780 781 782
	/*
	 * If this process has been checkpointed before, then the user
	 * application will use the original gpu_id on the
	 * checkpointed node to refer to this device.
	 */
	uint32_t user_gpu_id;
783 784
};

785 786
#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)

P
Philip Yang 已提交
787 788 789 790
struct svm_range_list {
	struct mutex			lock;
	struct rb_root_cached		objects;
	struct list_head		list;
P
Philip Yang 已提交
791 792
	struct work_struct		deferred_list_work;
	struct list_head		deferred_range_list;
793
	struct list_head                criu_svm_metadata_list;
P
Philip Yang 已提交
794
	spinlock_t			deferred_list_lock;
795
	atomic_t			evicted_ranges;
796
	atomic_t			drain_pagefaults;
797
	struct delayed_work		restore_work;
798
	DECLARE_BITMAP(bitmap_supported, MAX_GPU_INSTANCE);
799
	struct task_struct 		*faulting_task;
P
Philip Yang 已提交
800 801
};

O
Oded Gabbay 已提交
802 803
/* Process data */
struct kfd_process {
804 805 806 807 808 809
	/*
	 * kfd_process are stored in an mm_struct*->kfd_process*
	 * hash table (kfd_processes in kfd_process.c)
	 */
	struct hlist_node kfd_processes;

810 811 812 813 814 815
	/*
	 * Opaque pointer to mm_struct. We don't hold a reference to
	 * it so it should never be dereferenced from here. This is
	 * only used for looking up processes by their mm.
	 */
	void *mm;
816

817 818 819
	struct kref ref;
	struct work_struct release_work;

820 821 822 823 824 825
	struct mutex mutex;

	/*
	 * In any process, the thread that started main() is the lead
	 * thread and outlives the rest.
	 * It is here because amd_iommu_bind_pasid wants a task_struct.
F
Felix Kuehling 已提交
826 827
	 * It can also be used for safely getting a reference to the
	 * mm_struct of the process.
828 829 830 831 832 833
	 */
	struct task_struct *lead_thread;

	/* We want to receive a notification when the mm_struct is destroyed */
	struct mmu_notifier mmu_notifier;

834
	u32 pasid;
835 836

	/*
837
	 * Array of kfd_process_device pointers,
838 839
	 * one for each device the process is using.
	 */
840 841
	struct kfd_process_device *pdds[MAX_GPU_INSTANCE];
	uint32_t n_pdds;
842

843 844
	struct process_queue_manager pqm;

845 846
	/*Is the user space process 32 bit?*/
	bool is_32bit_user_mode;
847 848 849

	/* Event-related data */
	struct mutex event_mutex;
850 851
	/* Event ID allocator and lookup */
	struct idr event_idr;
852
	/* Event page */
853
	u64 signal_handle;
854
	struct kfd_signal_page *signal_page;
855
	size_t signal_mapped_size;
856
	size_t signal_event_count;
857
	bool signal_event_limit_reached;
858 859 860 861 862 863 864 865

	/* Information used for memory eviction */
	void *kgd_process_info;
	/* Eviction fence that is attached to all the BOs of this process. The
	 * fence will be triggered during eviction and new one will be created
	 * during restore
	 */
	struct dma_fence *ef;
866 867 868 869 870 871 872 873 874 875

	/* Work items for evicting and restoring BOs */
	struct delayed_work eviction_work;
	struct delayed_work restore_work;
	/* seqno of the last scheduled eviction */
	unsigned int last_eviction_seqno;
	/* Approx. the last timestamp (in jiffies) when the process was
	 * restored after an eviction
	 */
	unsigned long last_restore_timestamp;
876 877 878

	/* Kobj for our procfs */
	struct kobject *kobj;
879
	struct kobject *kobj_queues;
880
	struct attribute attr_pasid;
P
Philip Yang 已提交
881

P
Philip Yang 已提交
882 883
	/* shared virtual memory registered by this process */
	struct svm_range_list svms;
884 885

	bool xnack_enabled;
886 887

	atomic_t poison;
888 889
	/* Queues are in paused stated because we are in the process of doing a CRIU checkpoint */
	bool queues_paused;
O
Oded Gabbay 已提交
890 891
};

892 893 894 895
#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
extern DECLARE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
extern struct srcu_struct kfd_processes_srcu;

896
/**
897 898 899 900 901
 * typedef amdkfd_ioctl_t - typedef for ioctl function pointer.
 *
 * @filep: pointer to file structure.
 * @p: amdkfd process pointer.
 * @data: pointer to arg that was copied from user.
902
 *
903
 * Return: returns ioctl completion code.
904 905 906 907 908 909 910 911 912 913 914
 */
typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p,
				void *data);

struct amdkfd_ioctl_desc {
	unsigned int cmd;
	int flags;
	amdkfd_ioctl_t *func;
	unsigned int cmd_drv;
	const char *name;
};
915
bool kfd_dev_is_large_bar(struct kfd_dev *dev);
916

917
int kfd_process_create_wq(void);
918
void kfd_process_destroy_wq(void);
F
Felix Kuehling 已提交
919
struct kfd_process *kfd_create_process(struct file *filep);
920
struct kfd_process *kfd_get_process(const struct task_struct *);
921
struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid);
922
struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
923 924

int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id);
925
int kfd_process_gpuid_from_adev(struct kfd_process *p,
926 927
			       struct amdgpu_device *adev, uint32_t *gpuid,
			       uint32_t *gpuidx);
928 929 930 931 932 933 934 935 936
static inline int kfd_process_gpuid_from_gpuidx(struct kfd_process *p,
				uint32_t gpuidx, uint32_t *gpuid) {
	return gpuidx < p->n_pdds ? p->pdds[gpuidx]->dev->id : -EINVAL;
}
static inline struct kfd_process_device *kfd_process_device_from_gpuidx(
				struct kfd_process *p, uint32_t gpuidx) {
	return gpuidx < p->n_pdds ? p->pdds[gpuidx] : NULL;
}

937
void kfd_unref_process(struct kfd_process *p);
938 939
int kfd_process_evict_queues(struct kfd_process *p);
int kfd_process_restore_queues(struct kfd_process *p);
940 941
void kfd_suspend_all_processes(void);
int kfd_resume_all_processes(void);
942

943 944 945 946 947
struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *process,
							 uint32_t gpu_id);

int kfd_process_get_user_gpu_id(struct kfd_process *p, uint32_t actual_gpu_id);

948 949
int kfd_process_device_init_vm(struct kfd_process_device *pdd,
			       struct file *drm_file);
950
struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
951
						struct kfd_process *p);
952
struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
953 954 955
							struct kfd_process *p);
struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
							struct kfd_process *p);
956

957 958
bool kfd_process_xnack_mode(struct kfd_process *p, bool supported);

959
int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
F
Felix Kuehling 已提交
960 961
			  struct vm_area_struct *vma);

962 963 964 965 966 967 968
/* KFD process API for creating and translating handles */
int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
					void *mem);
void *kfd_process_device_translate_handle(struct kfd_process_device *p,
					int handle);
void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
					int handle);
969
struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid);
970

971 972 973 974 975
/* PASIDs */
int kfd_pasid_init(void);
void kfd_pasid_exit(void);
bool kfd_set_pasid_limit(unsigned int new_limit);
unsigned int kfd_get_pasid_limit(void);
976 977
u32 kfd_pasid_alloc(void);
void kfd_pasid_free(u32 pasid);
978 979

/* Doorbells */
980
size_t kfd_doorbell_process_slice(struct kfd_dev *kfd);
981 982
int kfd_doorbell_init(struct kfd_dev *kfd);
void kfd_doorbell_fini(struct kfd_dev *kfd);
983 984
int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
		      struct vm_area_struct *vma);
985
void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
986 987 988
					unsigned int *doorbell_off);
void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr);
u32 read_kernel_doorbell(u32 __iomem *db);
989
void write_kernel_doorbell(void __iomem *db, u32 value);
990
void write_kernel_doorbell64(void __iomem *db, u64 value);
991
unsigned int kfd_get_doorbell_dw_offset_in_bar(struct kfd_dev *kfd,
992
					struct kfd_process_device *pdd,
993
					unsigned int doorbell_id);
994 995 996 997 998
phys_addr_t kfd_get_process_doorbells(struct kfd_process_device *pdd);
int kfd_alloc_process_doorbells(struct kfd_dev *kfd,
				unsigned int *doorbell_index);
void kfd_free_process_doorbells(struct kfd_dev *kfd,
				unsigned int doorbell_index);
999 1000 1001 1002 1003 1004 1005
/* GTT Sub-Allocator */

int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
			struct kfd_mem_obj **mem_obj);

int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj);

O
Oded Gabbay 已提交
1006 1007
extern struct device *kfd_device;

1008 1009 1010
/* KFD's procfs */
void kfd_procfs_init(void);
void kfd_procfs_shutdown(void);
1011 1012
int kfd_procfs_add_queue(struct queue *q);
void kfd_procfs_del_queue(struct queue *q);
1013

1014 1015 1016 1017 1018
/* Topology */
int kfd_topology_init(void);
void kfd_topology_shutdown(void);
int kfd_topology_add_device(struct kfd_dev *gpu);
int kfd_topology_remove_device(struct kfd_dev *gpu);
1019 1020
struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
						uint32_t proximity_domain);
1021
struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id);
1022 1023
struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
1024
struct kfd_dev *kfd_device_by_adev(const struct amdgpu_device *adev);
1025
int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev);
1026
int kfd_numa_node_to_apic_id(int numa_node_id);
1027
void kfd_double_confirm_iommu_support(struct kfd_dev *gpu);
1028

O
Oded Gabbay 已提交
1029
/* Interrupts */
1030 1031 1032
int kfd_interrupt_init(struct kfd_dev *dev);
void kfd_interrupt_exit(struct kfd_dev *dev);
bool enqueue_ih_ring_entry(struct kfd_dev *kfd,	const void *ih_ring_entry);
1033 1034 1035
bool interrupt_is_wanted(struct kfd_dev *dev,
				const uint32_t *ih_ring_entry,
				uint32_t *patched_ihre, bool *flag);
O
Oded Gabbay 已提交
1036

1037 1038 1039
/* amdkfd Apertures */
int kfd_init_apertures(struct kfd_process *process);

1040 1041 1042 1043
void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
				  uint64_t tba_addr,
				  uint64_t tma_addr);

1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059
/* CRIU */
/*
 * Need to increment KFD_CRIU_PRIV_VERSION each time a change is made to any of the CRIU private
 * structures:
 * kfd_criu_process_priv_data
 * kfd_criu_device_priv_data
 * kfd_criu_bo_priv_data
 * kfd_criu_queue_priv_data
 * kfd_criu_event_priv_data
 * kfd_criu_svm_range_priv_data
 */

#define KFD_CRIU_PRIV_VERSION 1

struct kfd_criu_process_priv_data {
	uint32_t version;
1060
	uint32_t xnack_mode;
1061 1062 1063 1064 1065 1066 1067 1068
};

struct kfd_criu_device_priv_data {
	/* For future use */
	uint64_t reserved;
};

struct kfd_criu_bo_priv_data {
1069 1070 1071
	uint64_t user_addr;
	uint32_t idr_handle;
	uint32_t mapped_gpuids[MAX_GPU_INSTANCE];
1072 1073
};

1074 1075 1076 1077 1078 1079 1080 1081 1082 1083
/*
 * The first 4 bytes of kfd_criu_queue_priv_data, kfd_criu_event_priv_data,
 * kfd_criu_svm_range_priv_data is the object type
 */
enum kfd_criu_object_type {
	KFD_CRIU_OBJECT_TYPE_QUEUE,
	KFD_CRIU_OBJECT_TYPE_EVENT,
	KFD_CRIU_OBJECT_TYPE_SVM_RANGE,
};

1084 1085
struct kfd_criu_svm_range_priv_data {
	uint32_t object_type;
1086 1087 1088 1089
	uint64_t start_addr;
	uint64_t size;
	/* Variable length array of attributes */
	struct kfd_ioctl_svm_attribute attrs[0];
1090 1091 1092 1093
};

struct kfd_criu_queue_priv_data {
	uint32_t object_type;
1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
	uint64_t q_address;
	uint64_t q_size;
	uint64_t read_ptr_addr;
	uint64_t write_ptr_addr;
	uint64_t doorbell_off;
	uint64_t eop_ring_buffer_address;
	uint64_t ctx_save_restore_area_address;
	uint32_t gpu_id;
	uint32_t type;
	uint32_t format;
	uint32_t q_id;
	uint32_t priority;
	uint32_t q_percent;
	uint32_t doorbell_id;
	uint32_t is_gws;
	uint32_t sdma_id;
	uint32_t eop_ring_buffer_size;
	uint32_t ctx_save_restore_area_size;
	uint32_t ctl_stack_size;
	uint32_t mqd_size;
1114 1115 1116 1117
};

struct kfd_criu_event_priv_data {
	uint32_t object_type;
1118 1119 1120 1121 1122 1123 1124 1125 1126 1127
	uint64_t user_handle;
	uint32_t event_id;
	uint32_t auto_reset;
	uint32_t type;
	uint32_t signaled;

	union {
		struct kfd_hsa_memory_exception_data memory_exception_data;
		struct kfd_hsa_hw_exception_data hw_exception_data;
	};
1128 1129
};

1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141
int kfd_process_get_queue_info(struct kfd_process *p,
			       uint32_t *num_queues,
			       uint64_t *priv_data_sizes);

int kfd_criu_checkpoint_queues(struct kfd_process *p,
			 uint8_t __user *user_priv_data,
			 uint64_t *priv_data_offset);

int kfd_criu_restore_queue(struct kfd_process *p,
			   uint8_t __user *user_priv_data,
			   uint64_t *priv_data_offset,
			   uint64_t max_priv_data_size);
1142 1143 1144 1145 1146 1147 1148 1149 1150 1151

int kfd_criu_checkpoint_events(struct kfd_process *p,
			 uint8_t __user *user_priv_data,
			 uint64_t *priv_data_offset);

int kfd_criu_restore_event(struct file *devkfd,
			   struct kfd_process *p,
			   uint8_t __user *user_priv_data,
			   uint64_t *priv_data_offset,
			   uint64_t max_priv_data_size);
1152 1153
/* CRIU - End */

B
Ben Goz 已提交
1154
/* Queue Context Management */
1155
int init_queue(struct queue **q, const struct queue_properties *properties);
B
Ben Goz 已提交
1156
void uninit_queue(struct queue *q);
1157
void print_queue_properties(struct queue_properties *q);
B
Ben Goz 已提交
1158 1159
void print_queue(struct queue *q);

1160 1161
struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
1162 1163
struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
1164 1165
struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
1166 1167
struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
1168 1169
struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
1170 1171
struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
1172 1173
struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
void device_queue_manager_uninit(struct device_queue_manager *dqm);
B
Ben Goz 已提交
1174 1175
struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
					enum kfd_queue_type type);
1176
void kernel_queue_uninit(struct kernel_queue *kq, bool hanging);
1177
int kfd_dqm_evict_pasid(struct device_queue_manager *dqm, u32 pasid);
B
Ben Goz 已提交
1178

1179 1180 1181 1182 1183 1184 1185
/* Process Queue Manager */
struct process_queue_node {
	struct queue *q;
	struct kernel_queue *kq;
	struct list_head process_queue_list;
};

1186 1187
void kfd_process_dequeue_from_device(struct kfd_process_device *pdd);
void kfd_process_dequeue_from_all_devices(struct kfd_process *p);
1188 1189 1190 1191 1192 1193
int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p);
void pqm_uninit(struct process_queue_manager *pqm);
int pqm_create_queue(struct process_queue_manager *pqm,
			    struct kfd_dev *dev,
			    struct file *f,
			    struct queue_properties *properties,
1194
			    unsigned int *qid,
1195
			    const struct kfd_criu_queue_priv_data *q_data,
1196
			    const void *restore_mqd,
1197
			    const void *restore_ctl_stack,
1198
			    uint32_t *p_doorbell_offset_in_process);
1199
int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
1200
int pqm_update_queue_properties(struct process_queue_manager *pqm, unsigned int qid,
1201
			struct queue_properties *p);
1202 1203
int pqm_update_mqd(struct process_queue_manager *pqm, unsigned int qid,
			struct mqd_update_info *minfo);
1204 1205
int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
			void *gws);
1206 1207
struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
						unsigned int qid);
1208 1209
struct queue *pqm_get_user_queue(struct process_queue_manager *pqm,
						unsigned int qid);
1210 1211 1212 1213 1214
int pqm_get_wave_state(struct process_queue_manager *pqm,
		       unsigned int qid,
		       void __user *ctl_stack,
		       u32 *ctl_stack_used_size,
		       u32 *save_area_used_size);
1215

1216 1217
int amdkfd_fence_wait_timeout(uint64_t *fence_addr,
			      uint64_t fence_value,
1218
			      unsigned int timeout_ms);
1219

1220 1221
int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm,
				  unsigned int qid,
1222 1223
				  u32 *mqd_size,
				  u32 *ctl_stack_size);
B
Ben Goz 已提交
1224 1225
/* Packet Manager */

1226 1227
#define KFD_FENCE_COMPLETED (100)
#define KFD_FENCE_INIT   (10)
B
Ben Goz 已提交
1228

B
Ben Goz 已提交
1229 1230 1231 1232 1233 1234
struct packet_manager {
	struct device_queue_manager *dqm;
	struct kernel_queue *priv_queue;
	struct mutex lock;
	bool allocated;
	struct kfd_mem_obj *ib_buffer_obj;
1235
	unsigned int ib_size_bytes;
1236
	bool is_over_subscription;
1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256

	const struct packet_manager_funcs *pmf;
};

struct packet_manager_funcs {
	/* Support ASIC-specific packet formats for PM4 packets */
	int (*map_process)(struct packet_manager *pm, uint32_t *buffer,
			struct qcm_process_device *qpd);
	int (*runlist)(struct packet_manager *pm, uint32_t *buffer,
			uint64_t ib, size_t ib_size_in_dwords, bool chain);
	int (*set_resources)(struct packet_manager *pm, uint32_t *buffer,
			struct scheduling_resources *res);
	int (*map_queues)(struct packet_manager *pm, uint32_t *buffer,
			struct queue *q, bool is_static);
	int (*unmap_queues)(struct packet_manager *pm, uint32_t *buffer,
			enum kfd_queue_type type,
			enum kfd_unmap_queues_filter mode,
			uint32_t filter_param, bool reset,
			unsigned int sdma_engine);
	int (*query_status)(struct packet_manager *pm, uint32_t *buffer,
1257
			uint64_t fence_address,	uint64_t fence_value);
1258 1259 1260 1261 1262 1263 1264 1265 1266 1267
	int (*release_mem)(uint64_t gpu_addr, uint32_t *buffer);

	/* Packet sizes */
	int map_process_size;
	int runlist_size;
	int set_resources_size;
	int map_queues_size;
	int unmap_queues_size;
	int query_status_size;
	int release_mem_size;
B
Ben Goz 已提交
1268 1269
};

1270
extern const struct packet_manager_funcs kfd_vi_pm_funcs;
1271
extern const struct packet_manager_funcs kfd_v9_pm_funcs;
1272
extern const struct packet_manager_funcs kfd_aldebaran_pm_funcs;
1273

1274
int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
1275
void pm_uninit(struct packet_manager *pm, bool hanging);
1276 1277 1278 1279
int pm_send_set_resources(struct packet_manager *pm,
				struct scheduling_resources *res);
int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues);
int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
1280
				uint64_t fence_value);
1281 1282

int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
1283
			enum kfd_unmap_queues_filter mode,
1284 1285 1286
			uint32_t filter_param, bool reset,
			unsigned int sdma_engine);

B
Ben Goz 已提交
1287 1288
void pm_release_ib(struct packet_manager *pm);

1289 1290 1291
/* Following PM funcs can be shared among VI and AI */
unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size);

1292 1293
uint64_t kfd_get_number_elems(struct kfd_dev *kfd);

1294 1295
/* Events */
extern const struct kfd_event_interrupt_class event_interrupt_class_cik;
1296 1297
extern const struct kfd_event_interrupt_class event_interrupt_class_v9;

1298
extern const struct kfd_device_global_init_class device_global_init_class_cik;
1299 1300 1301 1302 1303

void kfd_event_init_process(struct kfd_process *p);
void kfd_event_free_process(struct kfd_process *p);
int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma);
int kfd_wait_on_events(struct kfd_process *p,
1304
		       uint32_t num_events, void __user *data,
1305
		       bool all, uint32_t user_timeout_ms,
1306
		       uint32_t *wait_result);
1307
void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
1308
				uint32_t valid_id_bits);
1309
void kfd_signal_iommu_event(struct kfd_dev *dev,
1310 1311 1312
			    u32 pasid, unsigned long address,
			    bool is_write_requested, bool is_execute_requested);
void kfd_signal_hw_exception_event(u32 pasid);
1313 1314
int kfd_set_event(struct kfd_process *p, uint32_t event_id);
int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
1315 1316
int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset);

1317 1318 1319 1320
int kfd_event_create(struct file *devkfd, struct kfd_process *p,
		     uint32_t event_type, bool auto_reset, uint32_t node_id,
		     uint32_t *event_id, uint32_t *event_trigger_data,
		     uint64_t *event_page_offset, uint32_t *event_slot_index);
1321 1322

int kfd_get_num_events(struct kfd_process *p);
1323 1324
int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);

1325
void kfd_signal_vm_fault_event(struct kfd_dev *dev, u32 pasid,
S
shaoyunl 已提交
1326 1327
				struct kfd_vm_fault_info *info);

1328 1329
void kfd_signal_reset_event(struct kfd_dev *dev);

1330 1331
void kfd_signal_poison_consumed_event(struct kfd_dev *dev, u32 pasid);

1332
void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type);
1333

1334 1335
int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p);

1336 1337
bool kfd_is_locked(void);

1338 1339 1340 1341
/* Compute profile */
void kfd_inc_compute_active(struct kfd_dev *dev);
void kfd_dec_compute_active(struct kfd_dev *dev);

1342 1343 1344 1345
/* Cgroup Support */
/* Check with device cgroup if @kfd device is accessible */
static inline int kfd_devcgroup_check_permission(struct kfd_dev *kfd)
{
1346
#if defined(CONFIG_CGROUP_DEVICE) || defined(CONFIG_CGROUP_BPF)
1347 1348
	struct drm_device *ddev = kfd->ddev;

1349
	return devcgroup_check_permission(DEVCG_DEV_CHAR, DRM_MAJOR,
1350 1351 1352 1353 1354 1355 1356
					  ddev->render->index,
					  DEVCG_ACC_WRITE | DEVCG_ACC_READ);
#else
	return 0;
#endif
}

1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368
/* Debugfs */
#if defined(CONFIG_DEBUG_FS)

void kfd_debugfs_init(void);
void kfd_debugfs_fini(void);
int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data);
int pqm_debugfs_mqds(struct seq_file *m, void *data);
int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data);
int dqm_debugfs_hqds(struct seq_file *m, void *data);
int kfd_debugfs_rls_by_device(struct seq_file *m, void *data);
int pm_debugfs_runlist(struct seq_file *m, void *data);

1369 1370
int kfd_debugfs_hang_hws(struct kfd_dev *dev);
int pm_debugfs_hang_hws(struct packet_manager *pm);
1371
int dqm_debugfs_hang_hws(struct device_queue_manager *dqm);
1372

1373 1374 1375 1376 1377 1378 1379
#else

static inline void kfd_debugfs_init(void) {}
static inline void kfd_debugfs_fini(void) {}

#endif

O
Oded Gabbay 已提交
1380
#endif