kfd_priv.h 38.1 KB
Newer Older
O
Oded Gabbay 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
/*
 * Copyright 2014 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

#ifndef KFD_PRIV_H_INCLUDED
#define KFD_PRIV_H_INCLUDED

#include <linux/hashtable.h>
#include <linux/mmu_notifier.h>
#include <linux/mutex.h>
#include <linux/types.h>
#include <linux/atomic.h>
#include <linux/workqueue.h>
#include <linux/spinlock.h>
33
#include <linux/kfd_ioctl.h>
34
#include <linux/idr.h>
35
#include <linux/kfifo.h>
36
#include <linux/seq_file.h>
37
#include <linux/kref.h>
38
#include <linux/sysfs.h>
39
#include <linux/device_cgroup.h>
40 41 42
#include <drm/drm_file.h>
#include <drm/drm_drv.h>
#include <drm/drm_device.h>
43
#include <drm/drm_ioctl.h>
O
Oded Gabbay 已提交
44
#include <kgd_kfd_interface.h>
45
#include <linux/swap.h>
O
Oded Gabbay 已提交
46

47
#include "amd_shared.h"
48
#include "amdgpu.h"
49

L
Laura Abbott 已提交
50 51
#define KFD_MAX_RING_ENTRY_SIZE	8

52 53
#define KFD_SYSFS_FILE_MODE 0444

54 55 56 57 58 59 60 61 62 63 64
/* GPU ID hash width in bits */
#define KFD_GPU_ID_HASH_WIDTH 16

/* Use upper bits of mmap offset to store KFD driver specific information.
 * BITS[63:62] - Encode MMAP type
 * BITS[61:46] - Encode gpu_id. To identify to which GPU the offset belongs to
 * BITS[45:0]  - MMAP offset value
 *
 * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these
 *  defines are w.r.t to PAGE_SIZE
 */
65
#define KFD_MMAP_TYPE_SHIFT	62
66 67 68 69
#define KFD_MMAP_TYPE_MASK	(0x3ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_TYPE_DOORBELL	(0x3ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_TYPE_EVENTS	(0x2ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_TYPE_RESERVED_MEM	(0x1ULL << KFD_MMAP_TYPE_SHIFT)
O
Oak Zeng 已提交
70
#define KFD_MMAP_TYPE_MMIO	(0x0ULL << KFD_MMAP_TYPE_SHIFT)
71

72
#define KFD_MMAP_GPU_ID_SHIFT 46
73 74 75 76
#define KFD_MMAP_GPU_ID_MASK (((1ULL << KFD_GPU_ID_HASH_WIDTH) - 1) \
				<< KFD_MMAP_GPU_ID_SHIFT)
#define KFD_MMAP_GPU_ID(gpu_id) ((((uint64_t)gpu_id) << KFD_MMAP_GPU_ID_SHIFT)\
				& KFD_MMAP_GPU_ID_MASK)
77
#define KFD_MMAP_GET_GPU_ID(offset)    ((offset & KFD_MMAP_GPU_ID_MASK) \
78 79
				>> KFD_MMAP_GPU_ID_SHIFT)

B
Ben Goz 已提交
80 81
/*
 * When working with cp scheduler we should assign the HIQ manually or via
82
 * the amdgpu driver to a fixed hqd slot, here are the fixed HIQ hqd slot
B
Ben Goz 已提交
83 84 85 86 87 88 89
 * definitions for Kaveri. In Kaveri only the first ME queues participates
 * in the cp scheduling taking that in mind we set the HIQ slot in the
 * second ME.
 */
#define KFD_CIK_HIQ_PIPE 4
#define KFD_CIK_HIQ_QUEUE 0

90 91 92 93
/* Macro for allocating structures */
#define kfd_alloc_struct(ptr_to_struct)	\
	((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL))

94
#define KFD_MAX_NUM_OF_PROCESSES 512
95
#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
96

F
Felix Kuehling 已提交
97 98 99 100
/*
 * Size of the per-process TBA+TMA buffer: 2 pages
 *
 * The first page is the TBA used for the CWSR ISA code. The second
101
 * page is used as TMA for user-mode trap handler setup in daisy-chain mode.
F
Felix Kuehling 已提交
102 103 104 105
 */
#define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2)
#define KFD_CWSR_TMA_OFFSET PAGE_SIZE

106 107 108 109 110 111
#define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE		\
	(KFD_MAX_NUM_OF_PROCESSES *			\
			KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)

#define KFD_KERNEL_QUEUE_SIZE 2048

112 113
#define KFD_UNMAP_LATENCY_MS	(4000)

114 115 116 117 118 119 120 121 122 123
/*
 * 512 = 0x200
 * The doorbell index distance between SDMA RLC (2*i) and (2*i+1) in the
 * same SDMA engine on SOC15, which has 8-byte doorbells for SDMA.
 * 512 8-byte doorbell distance (i.e. one page away) ensures that SDMA RLC
 * (2*i+1) doorbells (in terms of the lower 12 bit address) lie exactly in
 * the OFFSET and SIZE set in registers like BIF_SDMA0_DOORBELL_RANGE.
 */
#define KFD_QUEUE_DOORBELL_MIRROR_OFFSET 512

124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
/**
 * enum kfd_ioctl_flags - KFD ioctl flags
 * Various flags that can be set in &amdkfd_ioctl_desc.flags to control how
 * userspace can use a given ioctl.
 */
enum kfd_ioctl_flags {
	/*
	 * @KFD_IOC_FLAG_CHECKPOINT_RESTORE:
	 * Certain KFD ioctls such as AMDKFD_IOC_CRIU_OP can potentially
	 * perform privileged operations and load arbitrary data into MQDs and
	 * eventually HQD registers when the queue is mapped by HWS. In order to
	 * prevent this we should perform additional security checks.
	 *
	 * This is equivalent to callers with the CHECKPOINT_RESTORE capability.
	 *
	 * Note: Since earlier versions of docker do not support CHECKPOINT_RESTORE,
	 * we also allow ioctls with SYS_ADMIN capability.
	 */
	KFD_IOC_FLAG_CHECKPOINT_RESTORE = BIT(0),
};
144
/*
145 146
 * Kernel module parameter to specify maximum number of supported queues per
 * device
147
 */
148
extern int max_num_of_queues_per_device;
149

B
Ben Goz 已提交
150

151 152 153
/* Kernel module parameter to specify the scheduling policy */
extern int sched_policy;

154 155 156 157 158 159
/*
 * Kernel module parameter to specify the maximum process
 * number per HW scheduler
 */
extern int hws_max_conc_proc;

F
Felix Kuehling 已提交
160 161
extern int cwsr_enable;

162 163 164 165 166 167
/*
 * Kernel module parameter to specify whether to send sigterm to HSA process on
 * unhandled exception
 */
extern int send_sigterm;

168 169 170 171 172 173
/*
 * This kernel module is used to simulate large bar machine on non-large bar
 * enabled machines.
 */
extern int debug_largebar;

174 175 176 177 178 179
/*
 * Ignore CRAT table during KFD initialization, can be used to work around
 * broken CRAT tables on some AMD systems
 */
extern int ignore_crat;

180
/* Set sh_mem_config.retry_disable on GFX v9 */
181
extern int amdgpu_noretry;
182

183
/* Halt if HWS hang is detected */
184 185
extern int halt_if_hws_hang;

186
/* Whether MEC FW support GWS barriers */
187 188
extern bool hws_gws_support;

189
/* Queue preemption timeout in ms */
190 191
extern int queue_preemption_timeout_ms;

192 193 194 195 196
/*
 * Don't evict process queues on vm fault
 */
extern int amdgpu_no_queue_eviction_on_vm_fault;

197
/* Enable eviction debug messages */
198 199
extern bool debug_evictions;

B
Ben Goz 已提交
200 201 202 203 204
enum cache_policy {
	cache_policy_coherent,
	cache_policy_noncoherent
};

205 206
#define KFD_GC_VERSION(dev) ((dev)->adev->ip_versions[GC_HWIP][0])
#define KFD_IS_SOC15(dev)   ((KFD_GC_VERSION(dev)) >= (IP_VERSION(9, 0, 1)))
207

208 209
struct kfd_event_interrupt_class {
	bool (*interrupt_isr)(struct kfd_dev *dev,
210 211
			const uint32_t *ih_ring_entry, uint32_t *patched_ihre,
			bool *patched_flag);
212
	void (*interrupt_wq)(struct kfd_dev *dev,
213
			const uint32_t *ih_ring_entry);
214 215
};

O
Oded Gabbay 已提交
216
struct kfd_device_info {
217
	uint32_t gfx_target_version;
218
	const struct kfd_event_interrupt_class *event_interrupt_class;
O
Oded Gabbay 已提交
219
	unsigned int max_pasid_bits;
220
	unsigned int max_no_of_hqd;
221
	unsigned int doorbell_size;
O
Oded Gabbay 已提交
222
	size_t ih_ring_entry_size;
223
	uint8_t num_of_watch_points;
224
	uint16_t mqd_size_aligned;
F
Felix Kuehling 已提交
225
	bool supports_cwsr;
226
	bool needs_iommu_device;
227
	bool needs_pci_atomics;
228
	uint32_t no_atomic_fw_version;
229
	unsigned int num_sdma_queues_per_engine;
O
Oded Gabbay 已提交
230 231
};

232 233 234
unsigned int kfd_get_num_sdma_engines(struct kfd_dev *kdev);
unsigned int kfd_get_num_xgmi_sdma_engines(struct kfd_dev *kdev);

235 236 237 238 239
struct kfd_mem_obj {
	uint32_t range_start;
	uint32_t range_end;
	uint64_t gpu_addr;
	uint32_t *cpu_ptr;
240
	void *gtt_mem;
241 242
};

243 244 245 246 247 248
struct kfd_vmid_info {
	uint32_t first_vmid_kfd;
	uint32_t last_vmid_kfd;
	uint32_t vmid_num_kfd;
};

O
Oded Gabbay 已提交
249
struct kfd_dev {
250
	struct amdgpu_device *adev;
O
Oded Gabbay 已提交
251

252
	struct kfd_device_info device_info;
O
Oded Gabbay 已提交
253
	struct pci_dev *pdev;
254
	struct drm_device *ddev;
O
Oded Gabbay 已提交
255 256 257

	unsigned int id;		/* topology stub index */

258 259 260 261
	phys_addr_t doorbell_base;	/* Start of actual doorbells used by
					 * KFD. It is aligned for mapping
					 * into user mode
					 */
262 263 264 265
	size_t doorbell_base_dw_offset;	/* Offset from the start of the PCI
					 * doorbell BAR to the first KFD
					 * doorbell in dwords. GFX reserves
					 * the segment before this offset.
266 267 268 269 270
					 */
	u32 __iomem *doorbell_kernel_ptr; /* This is a pointer for a doorbells
					   * page used by kernel queue
					   */

O
Oded Gabbay 已提交
271
	struct kgd2kfd_shared_resources shared_resources;
272
	struct kfd_vmid_info vm_info;
O
Oded Gabbay 已提交
273

274 275
	const struct kfd2kgd_calls *kfd2kgd;
	struct mutex doorbell_mutex;
J
Joe Perches 已提交
276 277
	DECLARE_BITMAP(doorbell_available_index,
			KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
278

279 280 281 282 283 284 285 286
	void *gtt_mem;
	uint64_t gtt_start_gpu_addr;
	void *gtt_start_cpu_ptr;
	void *gtt_sa_bitmap;
	struct mutex gtt_sa_lock;
	unsigned int gtt_sa_chunk_size;
	unsigned int gtt_sa_num_of_chunks;

287
	/* Interrupts */
288
	struct kfifo ih_fifo;
289
	struct workqueue_struct *ih_wq;
290 291 292
	struct work_struct interrupt_work;
	spinlock_t interrupt_lock;

B
Ben Goz 已提交
293 294
	/* QCM Device instance */
	struct device_queue_manager *dqm;
O
Oded Gabbay 已提交
295

B
Ben Goz 已提交
296
	bool init_complete;
297 298 299 300 301
	/*
	 * Interrupts of interest to KFD are copied
	 * from the HW ring into a SW ring.
	 */
	bool interrupts_active;
302 303

	/* Debug manager */
K
Kent Russell 已提交
304
	struct kfd_dbgmgr *dbgmgr;
F
Felix Kuehling 已提交
305

306 307
	/* Firmware versions */
	uint16_t mec_fw_version;
308
	uint16_t mec2_fw_version;
309 310
	uint16_t sdma_fw_version;

311 312 313
	/* Maximum process number mapped to HW scheduler */
	unsigned int max_proc_per_quantum;

F
Felix Kuehling 已提交
314 315 316 317
	/* CWSR */
	bool cwsr_enabled;
	const void *cwsr_isa;
	unsigned int cwsr_isa_size;
318 319 320

	/* xGMI */
	uint64_t hive_id;
321

322
	bool pci_atomic_requested;
323

324 325 326
	/* Use IOMMU v2 flag */
	bool use_iommu_v2;

327 328
	/* SRAM ECC flag */
	atomic_t sram_ecc_flag;
329 330 331

	/* Compute Profile ref. count */
	atomic_t compute_profile;
332

333
	/* Global GWS resource shared between processes */
334
	void *gws;
A
Amber Lin 已提交
335 336 337 338

	/* Clients watching SMI events */
	struct list_head smi_clients;
	spinlock_t smi_lock;
339 340

	uint32_t reset_seq_num;
341 342 343

	struct ida doorbell_ida;
	unsigned int max_doorbell_slices;
344 345

	int noretry;
346 347 348

	/* HMM page migration MEMORY_DEVICE_PRIVATE mapping */
	struct dev_pagemap pgmap;
O
Oded Gabbay 已提交
349 350
};

351 352 353 354 355 356
enum kfd_mempool {
	KFD_MEMPOOL_SYSTEM_CACHEABLE = 1,
	KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2,
	KFD_MEMPOOL_FRAMEBUFFER = 3,
};

O
Oded Gabbay 已提交
357 358 359 360 361
/* Character device interface */
int kfd_chardev_init(void);
void kfd_chardev_exit(void);
struct device *kfd_chardev(void);

B
Ben Goz 已提交
362
/**
363
 * enum kfd_unmap_queues_filter - Enum for queue filters.
B
Ben Goz 已提交
364
 *
365
 * @KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE: Preempts single queue.
B
Ben Goz 已提交
366
 *
367
 * @KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES: Preempts all queues in the
B
Ben Goz 已提交
368 369
 *						running queues list.
 *
370
 * @KFD_UNMAP_QUEUES_FILTER_BY_PASID: Preempts queues that belongs to
B
Ben Goz 已提交
371 372 373
 *						specific process.
 *
 */
374 375 376 377 378
enum kfd_unmap_queues_filter {
	KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE,
	KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
	KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
	KFD_UNMAP_QUEUES_FILTER_BY_PASID
B
Ben Goz 已提交
379
};
380

B
Ben Goz 已提交
381
/**
382
 * enum kfd_queue_type - Enum for various queue types.
B
Ben Goz 已提交
383 384 385
 *
 * @KFD_QUEUE_TYPE_COMPUTE: Regular user mode queue type.
 *
386
 * @KFD_QUEUE_TYPE_SDMA: SDMA user mode queue type.
B
Ben Goz 已提交
387 388 389 390
 *
 * @KFD_QUEUE_TYPE_HIQ: HIQ queue type.
 *
 * @KFD_QUEUE_TYPE_DIQ: DIQ queue type.
391 392
 *
 * @KFD_QUEUE_TYPE_SDMA_XGMI: Special SDMA queue for XGMI interface.
B
Ben Goz 已提交
393 394 395 396 397
 */
enum kfd_queue_type  {
	KFD_QUEUE_TYPE_COMPUTE,
	KFD_QUEUE_TYPE_SDMA,
	KFD_QUEUE_TYPE_HIQ,
398 399
	KFD_QUEUE_TYPE_DIQ,
	KFD_QUEUE_TYPE_SDMA_XGMI
B
Ben Goz 已提交
400 401
};

B
Ben Goz 已提交
402 403 404 405 406
enum kfd_queue_format {
	KFD_QUEUE_FORMAT_PM4,
	KFD_QUEUE_FORMAT_AQL
};

407 408 409 410 411
enum KFD_QUEUE_PRIORITY {
	KFD_QUEUE_PRIORITY_MINIMUM = 0,
	KFD_QUEUE_PRIORITY_MAXIMUM = 15
};

B
Ben Goz 已提交
412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437
/**
 * struct queue_properties
 *
 * @type: The queue type.
 *
 * @queue_id: Queue identifier.
 *
 * @queue_address: Queue ring buffer address.
 *
 * @queue_size: Queue ring buffer size.
 *
 * @priority: Defines the queue priority relative to other queues in the
 * process.
 * This is just an indication and HW scheduling may override the priority as
 * necessary while keeping the relative prioritization.
 * the priority granularity is from 0 to f which f is the highest priority.
 * currently all queues are initialized with the highest priority.
 *
 * @queue_percent: This field is partially implemented and currently a zero in
 * this field defines that the queue is non active.
 *
 * @read_ptr: User space address which points to the number of dwords the
 * cp read from the ring buffer. This field updates automatically by the H/W.
 *
 * @write_ptr: Defines the number of dwords written to the ring buffer.
 *
438 439 440
 * @doorbell_ptr: Notifies the H/W of new packet written to the queue ring
 * buffer. This field should be similar to write_ptr and the user should
 * update this field after updating the write_ptr.
B
Ben Goz 已提交
441 442 443
 *
 * @doorbell_off: The doorbell offset in the doorbell pci-bar.
 *
444 445
 * @is_interop: Defines if this is a interop queue. Interop queue means that
 * the queue can access both graphics and compute resources.
B
Ben Goz 已提交
446
 *
447 448 449 450 451
 * @is_evicted: Defines if the queue is evicted. Only active queues
 * are evicted, rendering them inactive.
 *
 * @is_active: Defines if the queue is active or not. @is_active and
 * @is_evicted are protected by the DQM lock.
B
Ben Goz 已提交
452
 *
453 454 455 456
 * @is_gws: Defines if the queue has been updated to be GWS-capable or not.
 * @is_gws should be protected by the DQM lock, since changing it can yield the
 * possibility of updating DQM state on number of GWS queues.
 *
B
Ben Goz 已提交
457 458 459 460 461 462 463 464 465
 * @vmid: If the scheduling mode is no cp scheduling the field defines the vmid
 * of the queue.
 *
 * This structure represents the queue properties for each queue no matter if
 * it's user mode or kernel mode queue.
 *
 */
struct queue_properties {
	enum kfd_queue_type type;
B
Ben Goz 已提交
466
	enum kfd_queue_format format;
B
Ben Goz 已提交
467 468 469 470 471 472 473
	unsigned int queue_id;
	uint64_t queue_address;
	uint64_t  queue_size;
	uint32_t priority;
	uint32_t queue_percent;
	uint32_t *read_ptr;
	uint32_t *write_ptr;
474
	void __iomem *doorbell_ptr;
B
Ben Goz 已提交
475 476
	uint32_t doorbell_off;
	bool is_interop;
477
	bool is_evicted;
B
Ben Goz 已提交
478
	bool is_active;
479
	bool is_gws;
B
Ben Goz 已提交
480 481
	/* Not relevant for user mode queues in cp scheduling */
	unsigned int vmid;
B
Ben Goz 已提交
482 483 484 485
	/* Relevant only for sdma queues*/
	uint32_t sdma_engine_id;
	uint32_t sdma_queue_id;
	uint32_t sdma_vm_addr;
486 487 488 489 490
	/* Relevant only for VI */
	uint64_t eop_ring_buffer_address;
	uint32_t eop_ring_buffer_size;
	uint64_t ctx_save_restore_area_address;
	uint32_t ctx_save_restore_area_size;
F
Felix Kuehling 已提交
491 492 493
	uint32_t ctl_stack_size;
	uint64_t tba_addr;
	uint64_t tma_addr;
B
Ben Goz 已提交
494 495
};

496 497 498 499 500
#define QUEUE_IS_ACTIVE(q) ((q).queue_size > 0 &&	\
			    (q).queue_address != 0 &&	\
			    (q).queue_percent > 0 &&	\
			    !(q).is_evicted)

501 502 503 504 505 506 507 508 509 510 511 512 513
enum mqd_update_flag {
	UPDATE_FLAG_CU_MASK = 0,
};

struct mqd_update_info {
	union {
		struct {
			uint32_t count; /* Must be a multiple of 32 */
			uint32_t *ptr;
		} cu_mask;
	};
	enum mqd_update_flag update_flag;
};
514

B
Ben Goz 已提交
515 516 517 518 519
/**
 * struct queue
 *
 * @list: Queue linked list.
 *
520
 * @mqd: The queue MQD (memory queue descriptor).
B
Ben Goz 已提交
521 522 523 524 525 526 527 528
 *
 * @mqd_mem_obj: The MQD local gpu memory object.
 *
 * @gart_mqd_addr: The MQD gart mc address.
 *
 * @properties: The queue properties.
 *
 * @mec: Used only in no cp scheduling mode and identifies to micro engine id
529
 *	 that the queue should be executed on.
B
Ben Goz 已提交
530
 *
531 532
 * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe
 *	  id.
B
Ben Goz 已提交
533 534 535 536 537 538 539
 *
 * @queue: Used only in no cp scheduliong mode and identifies the queue's slot.
 *
 * @process: The kfd process that created this queue.
 *
 * @device: The kfd device that created this queue.
 *
540 541 542
 * @gws: Pointing to gws kgd_mem if this is a gws control queue; NULL
 * otherwise.
 *
B
Ben Goz 已提交
543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558
 * This structure represents user mode compute queues.
 * It contains all the necessary data to handle such queues.
 *
 */

struct queue {
	struct list_head list;
	void *mqd;
	struct kfd_mem_obj *mqd_mem_obj;
	uint64_t gart_mqd_addr;
	struct queue_properties properties;

	uint32_t mec;
	uint32_t pipe;
	uint32_t queue;

B
Ben Goz 已提交
559
	unsigned int sdma_id;
560
	unsigned int doorbell_id;
B
Ben Goz 已提交
561

B
Ben Goz 已提交
562 563
	struct kfd_process	*process;
	struct kfd_dev		*device;
564
	void *gws;
565 566 567

	/* procfs */
	struct kobject kobj;
B
Ben Goz 已提交
568 569
};

B
Ben Goz 已提交
570
enum KFD_MQD_TYPE {
571
	KFD_MQD_TYPE_HIQ = 0,		/* for hiq */
572 573
	KFD_MQD_TYPE_CP,		/* for cp queues and diq */
	KFD_MQD_TYPE_SDMA,		/* for sdma queues */
574
	KFD_MQD_TYPE_DIQ,		/* for diq */
B
Ben Goz 已提交
575 576 577
	KFD_MQD_TYPE_MAX
};

578 579 580 581 582 583
enum KFD_PIPE_PRIORITY {
	KFD_PIPE_PRIORITY_CS_LOW = 0,
	KFD_PIPE_PRIORITY_CS_MEDIUM,
	KFD_PIPE_PRIORITY_CS_HIGH
};

B
Ben Goz 已提交
584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611
struct scheduling_resources {
	unsigned int vmid_mask;
	enum kfd_queue_type type;
	uint64_t queue_mask;
	uint64_t gws_mask;
	uint32_t oac_mask;
	uint32_t gds_heap_base;
	uint32_t gds_heap_size;
};

struct process_queue_manager {
	/* data */
	struct kfd_process	*process;
	struct list_head	queues;
	unsigned long		*queue_slot_bitmap;
};

struct qcm_process_device {
	/* The Device Queue Manager that owns this data */
	struct device_queue_manager *dqm;
	struct process_queue_manager *pqm;
	/* Queues list */
	struct list_head queues_list;
	struct list_head priv_queue_list;

	unsigned int queue_count;
	unsigned int vmid;
	bool is_debug;
612
	unsigned int evicted; /* eviction counter, 0=active */
613 614 615 616 617 618

	/* This flag tells if we should reset all wavefronts on
	 * process termination
	 */
	bool reset_wavefronts;

619 620 621 622 623 624 625 626
	/* This flag tells us if this process has a GWS-capable
	 * queue that will be mapped into the runlist. It's
	 * possible to request a GWS BO, but not have the queue
	 * currently mapped, and this changes how the MAP_PROCESS
	 * PM4 packet is configured.
	 */
	bool mapped_gws_queue;

627
	/* All the memory management data should be here too */
B
Ben Goz 已提交
628
	uint64_t gds_context_area;
629
	/* Contains page table flags such as AMDGPU_PTE_VALID since gfx9 */
630
	uint64_t page_table_base;
B
Ben Goz 已提交
631 632 633 634 635 636 637
	uint32_t sh_mem_config;
	uint32_t sh_mem_bases;
	uint32_t sh_mem_ape1_base;
	uint32_t sh_mem_ape1_limit;
	uint32_t gds_size;
	uint32_t num_gws;
	uint32_t num_oac;
638
	uint32_t sh_hidden_private_base;
F
Felix Kuehling 已提交
639 640

	/* CWSR memory */
641
	struct kgd_mem *cwsr_mem;
F
Felix Kuehling 已提交
642
	void *cwsr_kaddr;
643
	uint64_t cwsr_base;
F
Felix Kuehling 已提交
644 645
	uint64_t tba_addr;
	uint64_t tma_addr;
646 647

	/* IB memory */
648
	struct kgd_mem *ib_mem;
649
	uint64_t ib_base;
650
	void *ib_kaddr;
651 652 653

	/* doorbell resources per process per device */
	unsigned long *doorbell_bitmap;
B
Ben Goz 已提交
654 655
};

656 657 658 659 660 661 662 663 664
/* KFD Memory Eviction */

/* Approx. wait time before attempting to restore evicted BOs */
#define PROCESS_RESTORE_TIME_MS 100
/* Approx. back off time if restore fails due to lack of memory */
#define PROCESS_BACK_OFF_TIME_MS 100
/* Approx. time before evicting the process again */
#define PROCESS_ACTIVE_TIME_MS 10

665 666 667 668 669 670 671 672
/* 8 byte handle containing GPU ID in the most significant 4 bytes and
 * idr_handle in the least significant 4 bytes
 */
#define MAKE_HANDLE(gpu_id, idr_handle) \
	(((uint64_t)(gpu_id) << 32) + idr_handle)
#define GET_GPU_ID(handle) (handle >> 32)
#define GET_IDR_HANDLE(handle) (handle & 0xFFFFFFFF)

673 674 675 676 677 678
enum kfd_pdd_bound {
	PDD_UNBOUND = 0,
	PDD_BOUND,
	PDD_BOUND_SUSPENDED,
};

679
#define MAX_SYSFS_FILENAME_LEN 15
680 681 682 683 684 685 686

/*
 * SDMA counter runs at 100MHz frequency.
 * We display SDMA activity in microsecond granularity in sysfs.
 * As a result, the divisor is 100.
 */
#define SDMA_ACTIVITY_DIVISOR  100
687

688 689 690 691 692
/* Data that is per-process-per device. */
struct kfd_process_device {
	/* The device that owns this data. */
	struct kfd_dev *dev;

693 694
	/* The process that owns this kfd_process_device. */
	struct kfd_process *process;
695

696 697 698
	/* per-process-per device QCM data structure */
	struct qcm_process_device qpd;

699 700 701 702 703 704 705 706
	/*Apertures*/
	uint64_t lds_base;
	uint64_t lds_limit;
	uint64_t gpuvm_base;
	uint64_t gpuvm_limit;
	uint64_t scratch_base;
	uint64_t scratch_limit;

707
	/* VM context for GPUVM allocations */
708
	struct file *drm_file;
709
	void *drm_priv;
710

711 712 713
	/* GPUVM allocations storage */
	struct idr alloc_idr;

714 715 716 717
	/* Flag used to tell the pdd has dequeued from the dqm.
	 * This is used to prevent dev->dqm->ops.process_termination() from
	 * being called twice when it is already called in IOMMU callback
	 * function.
718
	 */
719
	bool already_dequeued;
720
	bool runtime_inuse;
721 722 723

	/* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */
	enum kfd_pdd_bound bound;
724 725 726 727

	/* VRAM usage */
	uint64_t vram_usage;
	struct attribute attr_vram;
728 729 730 731 732 733
	char vram_filename[MAX_SYSFS_FILENAME_LEN];

	/* SDMA activity tracking */
	uint64_t sdma_past_activity_counter;
	struct attribute attr_sdma;
	char sdma_filename[MAX_SYSFS_FILENAME_LEN];
734 735 736 737 738 739 740

	/* Eviction activity tracking */
	uint64_t last_evict_timestamp;
	atomic64_t evict_duration_counter;
	struct attribute attr_evict;

	struct kobject *kobj_stats;
741
	unsigned int doorbell_index;
742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766

	/*
	 * @cu_occupancy: Reports occupancy of Compute Units (CU) of a process
	 * that is associated with device encoded by "this" struct instance. The
	 * value reflects CU usage by all of the waves launched by this process
	 * on this device. A very important property of occupancy parameter is
	 * that its value is a snapshot of current use.
	 *
	 * Following is to be noted regarding how this parameter is reported:
	 *
	 *  The number of waves that a CU can launch is limited by couple of
	 *  parameters. These are encoded by struct amdgpu_cu_info instance
	 *  that is part of every device definition. For GFX9 devices this
	 *  translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves
	 *  do not use scratch memory and 32 waves (max_scratch_slots_per_cu)
	 *  when they do use scratch memory. This could change for future
	 *  devices and therefore this example should be considered as a guide.
	 *
	 *  All CU's of a device are available for the process. This may not be true
	 *  under certain conditions - e.g. CU masking.
	 *
	 *  Finally number of CU's that are occupied by a process is affected by both
	 *  number of CU's a device has along with number of other competing processes
	 */
	struct attribute attr_cu_occupancy;
767 768 769 770 771 772 773 774 775

	/* sysfs counters for GPU retry fault and page migration tracking */
	struct kobject *kobj_counters;
	struct attribute attr_faults;
	struct attribute attr_page_in;
	struct attribute attr_page_out;
	uint64_t faults;
	uint64_t page_in;
	uint64_t page_out;
776 777
};

778 779
#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)

P
Philip Yang 已提交
780 781 782 783
struct svm_range_list {
	struct mutex			lock;
	struct rb_root_cached		objects;
	struct list_head		list;
P
Philip Yang 已提交
784 785 786
	struct work_struct		deferred_list_work;
	struct list_head		deferred_range_list;
	spinlock_t			deferred_list_lock;
787
	atomic_t			evicted_ranges;
788
	atomic_t			drain_pagefaults;
789
	struct delayed_work		restore_work;
790
	DECLARE_BITMAP(bitmap_supported, MAX_GPU_INSTANCE);
791
	struct task_struct 		*faulting_task;
P
Philip Yang 已提交
792 793
};

O
Oded Gabbay 已提交
794 795
/* Process data */
struct kfd_process {
796 797 798 799 800 801
	/*
	 * kfd_process are stored in an mm_struct*->kfd_process*
	 * hash table (kfd_processes in kfd_process.c)
	 */
	struct hlist_node kfd_processes;

802 803 804 805 806 807
	/*
	 * Opaque pointer to mm_struct. We don't hold a reference to
	 * it so it should never be dereferenced from here. This is
	 * only used for looking up processes by their mm.
	 */
	void *mm;
808

809 810 811
	struct kref ref;
	struct work_struct release_work;

812 813 814 815 816 817
	struct mutex mutex;

	/*
	 * In any process, the thread that started main() is the lead
	 * thread and outlives the rest.
	 * It is here because amd_iommu_bind_pasid wants a task_struct.
F
Felix Kuehling 已提交
818 819
	 * It can also be used for safely getting a reference to the
	 * mm_struct of the process.
820 821 822 823 824 825
	 */
	struct task_struct *lead_thread;

	/* We want to receive a notification when the mm_struct is destroyed */
	struct mmu_notifier mmu_notifier;

826
	u32 pasid;
827 828

	/*
829
	 * Array of kfd_process_device pointers,
830 831
	 * one for each device the process is using.
	 */
832 833
	struct kfd_process_device *pdds[MAX_GPU_INSTANCE];
	uint32_t n_pdds;
834

835 836
	struct process_queue_manager pqm;

837 838
	/*Is the user space process 32 bit?*/
	bool is_32bit_user_mode;
839 840 841

	/* Event-related data */
	struct mutex event_mutex;
842 843
	/* Event ID allocator and lookup */
	struct idr event_idr;
844
	/* Event page */
845
	u64 signal_handle;
846
	struct kfd_signal_page *signal_page;
847
	size_t signal_mapped_size;
848
	size_t signal_event_count;
849
	bool signal_event_limit_reached;
850 851 852 853 854 855 856 857

	/* Information used for memory eviction */
	void *kgd_process_info;
	/* Eviction fence that is attached to all the BOs of this process. The
	 * fence will be triggered during eviction and new one will be created
	 * during restore
	 */
	struct dma_fence *ef;
858 859 860 861 862 863 864 865 866 867

	/* Work items for evicting and restoring BOs */
	struct delayed_work eviction_work;
	struct delayed_work restore_work;
	/* seqno of the last scheduled eviction */
	unsigned int last_eviction_seqno;
	/* Approx. the last timestamp (in jiffies) when the process was
	 * restored after an eviction
	 */
	unsigned long last_restore_timestamp;
868 869 870

	/* Kobj for our procfs */
	struct kobject *kobj;
871
	struct kobject *kobj_queues;
872
	struct attribute attr_pasid;
P
Philip Yang 已提交
873

P
Philip Yang 已提交
874 875
	/* shared virtual memory registered by this process */
	struct svm_range_list svms;
876 877

	bool xnack_enabled;
878 879

	atomic_t poison;
O
Oded Gabbay 已提交
880 881
};

882 883 884 885
#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
extern DECLARE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
extern struct srcu_struct kfd_processes_srcu;

886
/**
887 888 889 890 891
 * typedef amdkfd_ioctl_t - typedef for ioctl function pointer.
 *
 * @filep: pointer to file structure.
 * @p: amdkfd process pointer.
 * @data: pointer to arg that was copied from user.
892
 *
893
 * Return: returns ioctl completion code.
894 895 896 897 898 899 900 901 902 903 904
 */
typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p,
				void *data);

struct amdkfd_ioctl_desc {
	unsigned int cmd;
	int flags;
	amdkfd_ioctl_t *func;
	unsigned int cmd_drv;
	const char *name;
};
905
bool kfd_dev_is_large_bar(struct kfd_dev *dev);
906

907
int kfd_process_create_wq(void);
908
void kfd_process_destroy_wq(void);
F
Felix Kuehling 已提交
909
struct kfd_process *kfd_create_process(struct file *filep);
910
struct kfd_process *kfd_get_process(const struct task_struct *);
911
struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid);
912
struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
913 914

int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id);
915
int kfd_process_gpuid_from_adev(struct kfd_process *p,
916 917
			       struct amdgpu_device *adev, uint32_t *gpuid,
			       uint32_t *gpuidx);
918 919 920 921 922 923 924 925 926
static inline int kfd_process_gpuid_from_gpuidx(struct kfd_process *p,
				uint32_t gpuidx, uint32_t *gpuid) {
	return gpuidx < p->n_pdds ? p->pdds[gpuidx]->dev->id : -EINVAL;
}
static inline struct kfd_process_device *kfd_process_device_from_gpuidx(
				struct kfd_process *p, uint32_t gpuidx) {
	return gpuidx < p->n_pdds ? p->pdds[gpuidx] : NULL;
}

927
void kfd_unref_process(struct kfd_process *p);
928 929
int kfd_process_evict_queues(struct kfd_process *p);
int kfd_process_restore_queues(struct kfd_process *p);
930 931
void kfd_suspend_all_processes(void);
int kfd_resume_all_processes(void);
932

933 934
int kfd_process_device_init_vm(struct kfd_process_device *pdd,
			       struct file *drm_file);
935
struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
936
						struct kfd_process *p);
937
struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
938 939 940
							struct kfd_process *p);
struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
							struct kfd_process *p);
941

942 943
bool kfd_process_xnack_mode(struct kfd_process *p, bool supported);

944
int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
F
Felix Kuehling 已提交
945 946
			  struct vm_area_struct *vma);

947 948 949 950 951 952 953 954
/* KFD process API for creating and translating handles */
int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
					void *mem);
void *kfd_process_device_translate_handle(struct kfd_process_device *p,
					int handle);
void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
					int handle);

955 956 957 958 959
/* PASIDs */
int kfd_pasid_init(void);
void kfd_pasid_exit(void);
bool kfd_set_pasid_limit(unsigned int new_limit);
unsigned int kfd_get_pasid_limit(void);
960 961
u32 kfd_pasid_alloc(void);
void kfd_pasid_free(u32 pasid);
962 963

/* Doorbells */
964
size_t kfd_doorbell_process_slice(struct kfd_dev *kfd);
965 966
int kfd_doorbell_init(struct kfd_dev *kfd);
void kfd_doorbell_fini(struct kfd_dev *kfd);
967 968
int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
		      struct vm_area_struct *vma);
969
void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
970 971 972
					unsigned int *doorbell_off);
void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr);
u32 read_kernel_doorbell(u32 __iomem *db);
973
void write_kernel_doorbell(void __iomem *db, u32 value);
974
void write_kernel_doorbell64(void __iomem *db, u64 value);
975
unsigned int kfd_get_doorbell_dw_offset_in_bar(struct kfd_dev *kfd,
976
					struct kfd_process_device *pdd,
977
					unsigned int doorbell_id);
978 979 980 981 982
phys_addr_t kfd_get_process_doorbells(struct kfd_process_device *pdd);
int kfd_alloc_process_doorbells(struct kfd_dev *kfd,
				unsigned int *doorbell_index);
void kfd_free_process_doorbells(struct kfd_dev *kfd,
				unsigned int doorbell_index);
983 984 985 986 987 988 989
/* GTT Sub-Allocator */

int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
			struct kfd_mem_obj **mem_obj);

int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj);

O
Oded Gabbay 已提交
990 991
extern struct device *kfd_device;

992 993 994
/* KFD's procfs */
void kfd_procfs_init(void);
void kfd_procfs_shutdown(void);
995 996
int kfd_procfs_add_queue(struct queue *q);
void kfd_procfs_del_queue(struct queue *q);
997

998 999 1000 1001 1002
/* Topology */
int kfd_topology_init(void);
void kfd_topology_shutdown(void);
int kfd_topology_add_device(struct kfd_dev *gpu);
int kfd_topology_remove_device(struct kfd_dev *gpu);
1003 1004
struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
						uint32_t proximity_domain);
1005
struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id);
1006 1007
struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
1008
struct kfd_dev *kfd_device_by_adev(const struct amdgpu_device *adev);
1009
int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev);
1010
int kfd_numa_node_to_apic_id(int numa_node_id);
1011
void kfd_double_confirm_iommu_support(struct kfd_dev *gpu);
1012

O
Oded Gabbay 已提交
1013
/* Interrupts */
1014 1015 1016
int kfd_interrupt_init(struct kfd_dev *dev);
void kfd_interrupt_exit(struct kfd_dev *dev);
bool enqueue_ih_ring_entry(struct kfd_dev *kfd,	const void *ih_ring_entry);
1017 1018 1019
bool interrupt_is_wanted(struct kfd_dev *dev,
				const uint32_t *ih_ring_entry,
				uint32_t *patched_ihre, bool *flag);
O
Oded Gabbay 已提交
1020

1021 1022 1023
/* amdkfd Apertures */
int kfd_init_apertures(struct kfd_process *process);

1024 1025 1026 1027
void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
				  uint64_t tba_addr,
				  uint64_t tma_addr);

1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071
/* CRIU */
/*
 * Need to increment KFD_CRIU_PRIV_VERSION each time a change is made to any of the CRIU private
 * structures:
 * kfd_criu_process_priv_data
 * kfd_criu_device_priv_data
 * kfd_criu_bo_priv_data
 * kfd_criu_queue_priv_data
 * kfd_criu_event_priv_data
 * kfd_criu_svm_range_priv_data
 */

#define KFD_CRIU_PRIV_VERSION 1

struct kfd_criu_process_priv_data {
	uint32_t version;
};

struct kfd_criu_device_priv_data {
	/* For future use */
	uint64_t reserved;
};

struct kfd_criu_bo_priv_data {
	uint64_t reserved;
};

struct kfd_criu_svm_range_priv_data {
	uint32_t object_type;
	uint32_t reserved;
};

struct kfd_criu_queue_priv_data {
	uint32_t object_type;
	uint32_t reserved;
};

struct kfd_criu_event_priv_data {
	uint32_t object_type;
	uint32_t reserved;
};

/* CRIU - End */

B
Ben Goz 已提交
1072
/* Queue Context Management */
1073
int init_queue(struct queue **q, const struct queue_properties *properties);
B
Ben Goz 已提交
1074
void uninit_queue(struct queue *q);
1075
void print_queue_properties(struct queue_properties *q);
B
Ben Goz 已提交
1076 1077
void print_queue(struct queue *q);

1078 1079
struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
1080 1081
struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
1082 1083
struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
1084 1085
struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
1086 1087
struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
1088 1089
struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
1090 1091
struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
void device_queue_manager_uninit(struct device_queue_manager *dqm);
B
Ben Goz 已提交
1092 1093
struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
					enum kfd_queue_type type);
1094
void kernel_queue_uninit(struct kernel_queue *kq, bool hanging);
1095
int kfd_process_vm_fault(struct device_queue_manager *dqm, u32 pasid);
B
Ben Goz 已提交
1096

1097 1098 1099 1100 1101 1102 1103
/* Process Queue Manager */
struct process_queue_node {
	struct queue *q;
	struct kernel_queue *kq;
	struct list_head process_queue_list;
};

1104 1105
void kfd_process_dequeue_from_device(struct kfd_process_device *pdd);
void kfd_process_dequeue_from_all_devices(struct kfd_process *p);
1106 1107 1108 1109 1110 1111
int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p);
void pqm_uninit(struct process_queue_manager *pqm);
int pqm_create_queue(struct process_queue_manager *pqm,
			    struct kfd_dev *dev,
			    struct file *f,
			    struct queue_properties *properties,
1112 1113
			    unsigned int *qid,
			    uint32_t *p_doorbell_offset_in_process);
1114
int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
1115
int pqm_update_queue_properties(struct process_queue_manager *pqm, unsigned int qid,
1116
			struct queue_properties *p);
1117 1118
int pqm_update_mqd(struct process_queue_manager *pqm, unsigned int qid,
			struct mqd_update_info *minfo);
1119 1120
int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
			void *gws);
1121 1122
struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
						unsigned int qid);
1123 1124
struct queue *pqm_get_user_queue(struct process_queue_manager *pqm,
						unsigned int qid);
1125 1126 1127 1128 1129
int pqm_get_wave_state(struct process_queue_manager *pqm,
		       unsigned int qid,
		       void __user *ctl_stack,
		       u32 *ctl_stack_used_size,
		       u32 *save_area_used_size);
1130

1131 1132
int amdkfd_fence_wait_timeout(uint64_t *fence_addr,
			      uint64_t fence_value,
1133
			      unsigned int timeout_ms);
1134

B
Ben Goz 已提交
1135 1136
/* Packet Manager */

1137 1138
#define KFD_FENCE_COMPLETED (100)
#define KFD_FENCE_INIT   (10)
B
Ben Goz 已提交
1139

B
Ben Goz 已提交
1140 1141 1142 1143 1144 1145
struct packet_manager {
	struct device_queue_manager *dqm;
	struct kernel_queue *priv_queue;
	struct mutex lock;
	bool allocated;
	struct kfd_mem_obj *ib_buffer_obj;
1146
	unsigned int ib_size_bytes;
1147
	bool is_over_subscription;
1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167

	const struct packet_manager_funcs *pmf;
};

struct packet_manager_funcs {
	/* Support ASIC-specific packet formats for PM4 packets */
	int (*map_process)(struct packet_manager *pm, uint32_t *buffer,
			struct qcm_process_device *qpd);
	int (*runlist)(struct packet_manager *pm, uint32_t *buffer,
			uint64_t ib, size_t ib_size_in_dwords, bool chain);
	int (*set_resources)(struct packet_manager *pm, uint32_t *buffer,
			struct scheduling_resources *res);
	int (*map_queues)(struct packet_manager *pm, uint32_t *buffer,
			struct queue *q, bool is_static);
	int (*unmap_queues)(struct packet_manager *pm, uint32_t *buffer,
			enum kfd_queue_type type,
			enum kfd_unmap_queues_filter mode,
			uint32_t filter_param, bool reset,
			unsigned int sdma_engine);
	int (*query_status)(struct packet_manager *pm, uint32_t *buffer,
1168
			uint64_t fence_address,	uint64_t fence_value);
1169 1170 1171 1172 1173 1174 1175 1176 1177 1178
	int (*release_mem)(uint64_t gpu_addr, uint32_t *buffer);

	/* Packet sizes */
	int map_process_size;
	int runlist_size;
	int set_resources_size;
	int map_queues_size;
	int unmap_queues_size;
	int query_status_size;
	int release_mem_size;
B
Ben Goz 已提交
1179 1180
};

1181
extern const struct packet_manager_funcs kfd_vi_pm_funcs;
1182
extern const struct packet_manager_funcs kfd_v9_pm_funcs;
1183
extern const struct packet_manager_funcs kfd_aldebaran_pm_funcs;
1184

1185
int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
1186
void pm_uninit(struct packet_manager *pm, bool hanging);
1187 1188 1189 1190
int pm_send_set_resources(struct packet_manager *pm,
				struct scheduling_resources *res);
int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues);
int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
1191
				uint64_t fence_value);
1192 1193

int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
1194
			enum kfd_unmap_queues_filter mode,
1195 1196 1197
			uint32_t filter_param, bool reset,
			unsigned int sdma_engine);

B
Ben Goz 已提交
1198 1199
void pm_release_ib(struct packet_manager *pm);

1200 1201 1202
/* Following PM funcs can be shared among VI and AI */
unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size);

1203 1204
uint64_t kfd_get_number_elems(struct kfd_dev *kfd);

1205 1206
/* Events */
extern const struct kfd_event_interrupt_class event_interrupt_class_cik;
1207 1208
extern const struct kfd_event_interrupt_class event_interrupt_class_v9;

1209
extern const struct kfd_device_global_init_class device_global_init_class_cik;
1210 1211 1212 1213 1214

void kfd_event_init_process(struct kfd_process *p);
void kfd_event_free_process(struct kfd_process *p);
int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma);
int kfd_wait_on_events(struct kfd_process *p,
1215
		       uint32_t num_events, void __user *data,
1216
		       bool all, uint32_t user_timeout_ms,
1217
		       uint32_t *wait_result);
1218
void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
1219
				uint32_t valid_id_bits);
1220
void kfd_signal_iommu_event(struct kfd_dev *dev,
1221 1222 1223
			    u32 pasid, unsigned long address,
			    bool is_write_requested, bool is_execute_requested);
void kfd_signal_hw_exception_event(u32 pasid);
1224 1225
int kfd_set_event(struct kfd_process *p, uint32_t event_id);
int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
1226 1227
int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
		       uint64_t size);
1228 1229 1230 1231 1232 1233
int kfd_event_create(struct file *devkfd, struct kfd_process *p,
		     uint32_t event_type, bool auto_reset, uint32_t node_id,
		     uint32_t *event_id, uint32_t *event_trigger_data,
		     uint64_t *event_page_offset, uint32_t *event_slot_index);
int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);

1234
void kfd_signal_vm_fault_event(struct kfd_dev *dev, u32 pasid,
S
shaoyunl 已提交
1235 1236
				struct kfd_vm_fault_info *info);

1237 1238
void kfd_signal_reset_event(struct kfd_dev *dev);

1239 1240
void kfd_signal_poison_consumed_event(struct kfd_dev *dev, u32 pasid);

1241
void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type);
1242

1243 1244
int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p);

1245 1246
bool kfd_is_locked(void);

1247 1248 1249 1250
/* Compute profile */
void kfd_inc_compute_active(struct kfd_dev *dev);
void kfd_dec_compute_active(struct kfd_dev *dev);

1251 1252 1253 1254
/* Cgroup Support */
/* Check with device cgroup if @kfd device is accessible */
static inline int kfd_devcgroup_check_permission(struct kfd_dev *kfd)
{
1255
#if defined(CONFIG_CGROUP_DEVICE) || defined(CONFIG_CGROUP_BPF)
1256 1257
	struct drm_device *ddev = kfd->ddev;

1258
	return devcgroup_check_permission(DEVCG_DEV_CHAR, DRM_MAJOR,
1259 1260 1261 1262 1263 1264 1265
					  ddev->render->index,
					  DEVCG_ACC_WRITE | DEVCG_ACC_READ);
#else
	return 0;
#endif
}

1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277
/* Debugfs */
#if defined(CONFIG_DEBUG_FS)

void kfd_debugfs_init(void);
void kfd_debugfs_fini(void);
int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data);
int pqm_debugfs_mqds(struct seq_file *m, void *data);
int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data);
int dqm_debugfs_hqds(struct seq_file *m, void *data);
int kfd_debugfs_rls_by_device(struct seq_file *m, void *data);
int pm_debugfs_runlist(struct seq_file *m, void *data);

1278 1279
int kfd_debugfs_hang_hws(struct kfd_dev *dev);
int pm_debugfs_hang_hws(struct packet_manager *pm);
1280
int dqm_debugfs_hang_hws(struct device_queue_manager *dqm);
1281

1282 1283 1284 1285 1286 1287 1288
#else

static inline void kfd_debugfs_init(void) {}
static inline void kfd_debugfs_fini(void) {}

#endif

O
Oded Gabbay 已提交
1289
#endif