kfd_priv.h 30.1 KB
Newer Older
O
Oded Gabbay 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
/*
 * Copyright 2014 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

#ifndef KFD_PRIV_H_INCLUDED
#define KFD_PRIV_H_INCLUDED

#include <linux/hashtable.h>
#include <linux/mmu_notifier.h>
#include <linux/mutex.h>
#include <linux/types.h>
#include <linux/atomic.h>
#include <linux/workqueue.h>
#include <linux/spinlock.h>
33
#include <linux/kfd_ioctl.h>
34
#include <linux/idr.h>
35
#include <linux/kfifo.h>
36
#include <linux/seq_file.h>
37
#include <linux/kref.h>
O
Oded Gabbay 已提交
38 39
#include <kgd_kfd_interface.h>

40 41
#include "amd_shared.h"

L
Laura Abbott 已提交
42 43
#define KFD_MAX_RING_ENTRY_SIZE	8

44 45
#define KFD_SYSFS_FILE_MODE 0444

46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72
/* GPU ID hash width in bits */
#define KFD_GPU_ID_HASH_WIDTH 16

/* Use upper bits of mmap offset to store KFD driver specific information.
 * BITS[63:62] - Encode MMAP type
 * BITS[61:46] - Encode gpu_id. To identify to which GPU the offset belongs to
 * BITS[45:0]  - MMAP offset value
 *
 * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these
 *  defines are w.r.t to PAGE_SIZE
 */
#define KFD_MMAP_TYPE_SHIFT	(62 - PAGE_SHIFT)
#define KFD_MMAP_TYPE_MASK	(0x3ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_TYPE_DOORBELL	(0x3ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_TYPE_EVENTS	(0x2ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_TYPE_RESERVED_MEM	(0x1ULL << KFD_MMAP_TYPE_SHIFT)

#define KFD_MMAP_GPU_ID_SHIFT (46 - PAGE_SHIFT)
#define KFD_MMAP_GPU_ID_MASK (((1ULL << KFD_GPU_ID_HASH_WIDTH) - 1) \
				<< KFD_MMAP_GPU_ID_SHIFT)
#define KFD_MMAP_GPU_ID(gpu_id) ((((uint64_t)gpu_id) << KFD_MMAP_GPU_ID_SHIFT)\
				& KFD_MMAP_GPU_ID_MASK)
#define KFD_MMAP_GPU_ID_GET(offset)    ((offset & KFD_MMAP_GPU_ID_MASK) \
				>> KFD_MMAP_GPU_ID_SHIFT)

#define KFD_MMAP_OFFSET_VALUE_MASK	(0x3FFFFFFFFFFFULL >> PAGE_SHIFT)
#define KFD_MMAP_OFFSET_VALUE_GET(offset) (offset & KFD_MMAP_OFFSET_VALUE_MASK)
73

B
Ben Goz 已提交
74 75
/*
 * When working with cp scheduler we should assign the HIQ manually or via
76
 * the amdgpu driver to a fixed hqd slot, here are the fixed HIQ hqd slot
B
Ben Goz 已提交
77 78 79 80 81 82 83
 * definitions for Kaveri. In Kaveri only the first ME queues participates
 * in the cp scheduling taking that in mind we set the HIQ slot in the
 * second ME.
 */
#define KFD_CIK_HIQ_PIPE 4
#define KFD_CIK_HIQ_QUEUE 0

84 85 86 87
/* Macro for allocating structures */
#define kfd_alloc_struct(ptr_to_struct)	\
	((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL))

88
#define KFD_MAX_NUM_OF_PROCESSES 512
89
#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
90

F
Felix Kuehling 已提交
91 92 93 94 95 96 97 98 99
/*
 * Size of the per-process TBA+TMA buffer: 2 pages
 *
 * The first page is the TBA used for the CWSR ISA code. The second
 * page is used as TMA for daisy changing a user-mode trap handler.
 */
#define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2)
#define KFD_CWSR_TMA_OFFSET PAGE_SIZE

100 101 102 103 104 105
#define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE		\
	(KFD_MAX_NUM_OF_PROCESSES *			\
			KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)

#define KFD_KERNEL_QUEUE_SIZE 2048

106 107 108 109 110 111 112 113 114 115 116
/*
 * 512 = 0x200
 * The doorbell index distance between SDMA RLC (2*i) and (2*i+1) in the
 * same SDMA engine on SOC15, which has 8-byte doorbells for SDMA.
 * 512 8-byte doorbell distance (i.e. one page away) ensures that SDMA RLC
 * (2*i+1) doorbells (in terms of the lower 12 bit address) lie exactly in
 * the OFFSET and SIZE set in registers like BIF_SDMA0_DOORBELL_RANGE.
 */
#define KFD_QUEUE_DOORBELL_MIRROR_OFFSET 512


117
/*
118 119
 * Kernel module parameter to specify maximum number of supported queues per
 * device
120
 */
121
extern int max_num_of_queues_per_device;
122

B
Ben Goz 已提交
123

124 125 126
/* Kernel module parameter to specify the scheduling policy */
extern int sched_policy;

127 128 129 130 131 132
/*
 * Kernel module parameter to specify the maximum process
 * number per HW scheduler
 */
extern int hws_max_conc_proc;

F
Felix Kuehling 已提交
133 134
extern int cwsr_enable;

135 136 137 138 139 140
/*
 * Kernel module parameter to specify whether to send sigterm to HSA process on
 * unhandled exception
 */
extern int send_sigterm;

141 142 143 144 145 146
/*
 * This kernel module is used to simulate large bar machine on non-large bar
 * enabled machines.
 */
extern int debug_largebar;

147 148 149 150 151 152
/*
 * Ignore CRAT table during KFD initialization, can be used to work around
 * broken CRAT tables on some AMD systems
 */
extern int ignore_crat;

153 154 155
/*
 * Set sh_mem_config.retry_disable on Vega10
 */
156
extern int noretry;
157

158 159 160 161 162
/*
 * Halt if HWS hang is detected
 */
extern int halt_if_hws_hang;

B
Ben Goz 已提交
163 164 165 166 167
enum cache_policy {
	cache_policy_coherent,
	cache_policy_noncoherent
};

168 169
#define KFD_IS_SOC15(chip) ((chip) >= CHIP_VEGA10)

170 171
struct kfd_event_interrupt_class {
	bool (*interrupt_isr)(struct kfd_dev *dev,
172 173
			const uint32_t *ih_ring_entry, uint32_t *patched_ihre,
			bool *patched_flag);
174
	void (*interrupt_wq)(struct kfd_dev *dev,
175
			const uint32_t *ih_ring_entry);
176 177
};

O
Oded Gabbay 已提交
178
struct kfd_device_info {
179
	enum amd_asic_type asic_family;
180
	const struct kfd_event_interrupt_class *event_interrupt_class;
O
Oded Gabbay 已提交
181
	unsigned int max_pasid_bits;
182
	unsigned int max_no_of_hqd;
183
	unsigned int doorbell_size;
O
Oded Gabbay 已提交
184
	size_t ih_ring_entry_size;
185
	uint8_t num_of_watch_points;
186
	uint16_t mqd_size_aligned;
F
Felix Kuehling 已提交
187
	bool supports_cwsr;
188
	bool needs_iommu_device;
189
	bool needs_pci_atomics;
190
	unsigned int num_sdma_engines;
191
	unsigned int num_sdma_queues_per_engine;
O
Oded Gabbay 已提交
192 193
};

194 195 196 197 198
struct kfd_mem_obj {
	uint32_t range_start;
	uint32_t range_end;
	uint64_t gpu_addr;
	uint32_t *cpu_ptr;
199
	void *gtt_mem;
200 201
};

202 203 204 205 206 207
struct kfd_vmid_info {
	uint32_t first_vmid_kfd;
	uint32_t last_vmid_kfd;
	uint32_t vmid_num_kfd;
};

O
Oded Gabbay 已提交
208 209 210 211 212 213 214 215
struct kfd_dev {
	struct kgd_dev *kgd;

	const struct kfd_device_info *device_info;
	struct pci_dev *pdev;

	unsigned int id;		/* topology stub index */

216 217 218 219 220 221 222 223 224 225 226 227
	phys_addr_t doorbell_base;	/* Start of actual doorbells used by
					 * KFD. It is aligned for mapping
					 * into user mode
					 */
	size_t doorbell_id_offset;	/* Doorbell offset (from KFD doorbell
					 * to HW doorbell, GFX reserved some
					 * at the start)
					 */
	u32 __iomem *doorbell_kernel_ptr; /* This is a pointer for a doorbells
					   * page used by kernel queue
					   */

O
Oded Gabbay 已提交
228
	struct kgd2kfd_shared_resources shared_resources;
229
	struct kfd_vmid_info vm_info;
O
Oded Gabbay 已提交
230

231 232
	const struct kfd2kgd_calls *kfd2kgd;
	struct mutex doorbell_mutex;
J
Joe Perches 已提交
233 234
	DECLARE_BITMAP(doorbell_available_index,
			KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
235

236 237 238 239 240 241 242 243
	void *gtt_mem;
	uint64_t gtt_start_gpu_addr;
	void *gtt_start_cpu_ptr;
	void *gtt_sa_bitmap;
	struct mutex gtt_sa_lock;
	unsigned int gtt_sa_chunk_size;
	unsigned int gtt_sa_num_of_chunks;

244
	/* Interrupts */
245
	struct kfifo ih_fifo;
246
	struct workqueue_struct *ih_wq;
247 248 249
	struct work_struct interrupt_work;
	spinlock_t interrupt_lock;

B
Ben Goz 已提交
250 251
	/* QCM Device instance */
	struct device_queue_manager *dqm;
O
Oded Gabbay 已提交
252

B
Ben Goz 已提交
253
	bool init_complete;
254 255 256 257 258
	/*
	 * Interrupts of interest to KFD are copied
	 * from the HW ring into a SW ring.
	 */
	bool interrupts_active;
259 260

	/* Debug manager */
K
Kent Russell 已提交
261
	struct kfd_dbgmgr *dbgmgr;
F
Felix Kuehling 已提交
262

263 264 265 266
	/* Firmware versions */
	uint16_t mec_fw_version;
	uint16_t sdma_fw_version;

267 268 269
	/* Maximum process number mapped to HW scheduler */
	unsigned int max_proc_per_quantum;

F
Felix Kuehling 已提交
270 271 272 273
	/* CWSR */
	bool cwsr_enabled;
	const void *cwsr_isa;
	unsigned int cwsr_isa_size;
274 275 276

	/* xGMI */
	uint64_t hive_id;
277 278

	bool pci_atomic_requested;
279 280 281

	/* SRAM ECC flag */
	atomic_t sram_ecc_flag;
282 283 284

	/* Compute Profile ref. count */
	atomic_t compute_profile;
O
Oded Gabbay 已提交
285 286
};

287 288 289 290 291 292
enum kfd_mempool {
	KFD_MEMPOOL_SYSTEM_CACHEABLE = 1,
	KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2,
	KFD_MEMPOOL_FRAMEBUFFER = 3,
};

O
Oded Gabbay 已提交
293 294 295 296 297
/* Character device interface */
int kfd_chardev_init(void);
void kfd_chardev_exit(void);
struct device *kfd_chardev(void);

B
Ben Goz 已提交
298
/**
299
 * enum kfd_unmap_queues_filter
B
Ben Goz 已提交
300
 *
301
 * @KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE: Preempts single queue.
B
Ben Goz 已提交
302
 *
303
 * @KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES: Preempts all queues in the
B
Ben Goz 已提交
304 305
 *						running queues list.
 *
306
 * @KFD_UNMAP_QUEUES_FILTER_BY_PASID: Preempts queues that belongs to
B
Ben Goz 已提交
307 308 309
 *						specific process.
 *
 */
310 311 312 313 314
enum kfd_unmap_queues_filter {
	KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE,
	KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
	KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
	KFD_UNMAP_QUEUES_FILTER_BY_PASID
B
Ben Goz 已提交
315
};
316

B
Ben Goz 已提交
317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
/**
 * enum kfd_queue_type
 *
 * @KFD_QUEUE_TYPE_COMPUTE: Regular user mode queue type.
 *
 * @KFD_QUEUE_TYPE_SDMA: Sdma user mode queue type.
 *
 * @KFD_QUEUE_TYPE_HIQ: HIQ queue type.
 *
 * @KFD_QUEUE_TYPE_DIQ: DIQ queue type.
 */
enum kfd_queue_type  {
	KFD_QUEUE_TYPE_COMPUTE,
	KFD_QUEUE_TYPE_SDMA,
	KFD_QUEUE_TYPE_HIQ,
	KFD_QUEUE_TYPE_DIQ
};

B
Ben Goz 已提交
335 336 337 338 339
enum kfd_queue_format {
	KFD_QUEUE_FORMAT_PM4,
	KFD_QUEUE_FORMAT_AQL
};

B
Ben Goz 已提交
340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366
/**
 * struct queue_properties
 *
 * @type: The queue type.
 *
 * @queue_id: Queue identifier.
 *
 * @queue_address: Queue ring buffer address.
 *
 * @queue_size: Queue ring buffer size.
 *
 * @priority: Defines the queue priority relative to other queues in the
 * process.
 * This is just an indication and HW scheduling may override the priority as
 * necessary while keeping the relative prioritization.
 * the priority granularity is from 0 to f which f is the highest priority.
 * currently all queues are initialized with the highest priority.
 *
 * @queue_percent: This field is partially implemented and currently a zero in
 * this field defines that the queue is non active.
 *
 * @read_ptr: User space address which points to the number of dwords the
 * cp read from the ring buffer. This field updates automatically by the H/W.
 *
 * @write_ptr: Defines the number of dwords written to the ring buffer.
 *
 * @doorbell_ptr: This field aim is to notify the H/W of new packet written to
367 368
 * the queue ring buffer. This field should be similar to write_ptr and the
 * user should update this field after he updated the write_ptr.
B
Ben Goz 已提交
369 370 371
 *
 * @doorbell_off: The doorbell offset in the doorbell pci-bar.
 *
372 373
 * @is_interop: Defines if this is a interop queue. Interop queue means that
 * the queue can access both graphics and compute resources.
B
Ben Goz 已提交
374
 *
375 376 377 378 379
 * @is_evicted: Defines if the queue is evicted. Only active queues
 * are evicted, rendering them inactive.
 *
 * @is_active: Defines if the queue is active or not. @is_active and
 * @is_evicted are protected by the DQM lock.
B
Ben Goz 已提交
380 381 382 383 384 385 386 387 388 389
 *
 * @vmid: If the scheduling mode is no cp scheduling the field defines the vmid
 * of the queue.
 *
 * This structure represents the queue properties for each queue no matter if
 * it's user mode or kernel mode queue.
 *
 */
struct queue_properties {
	enum kfd_queue_type type;
B
Ben Goz 已提交
390
	enum kfd_queue_format format;
B
Ben Goz 已提交
391 392 393 394 395 396 397
	unsigned int queue_id;
	uint64_t queue_address;
	uint64_t  queue_size;
	uint32_t priority;
	uint32_t queue_percent;
	uint32_t *read_ptr;
	uint32_t *write_ptr;
398
	void __iomem *doorbell_ptr;
B
Ben Goz 已提交
399 400
	uint32_t doorbell_off;
	bool is_interop;
401
	bool is_evicted;
B
Ben Goz 已提交
402 403 404
	bool is_active;
	/* Not relevant for user mode queues in cp scheduling */
	unsigned int vmid;
B
Ben Goz 已提交
405 406 407 408
	/* Relevant only for sdma queues*/
	uint32_t sdma_engine_id;
	uint32_t sdma_queue_id;
	uint32_t sdma_vm_addr;
409 410 411 412 413
	/* Relevant only for VI */
	uint64_t eop_ring_buffer_address;
	uint32_t eop_ring_buffer_size;
	uint64_t ctx_save_restore_area_address;
	uint32_t ctx_save_restore_area_size;
F
Felix Kuehling 已提交
414 415 416
	uint32_t ctl_stack_size;
	uint64_t tba_addr;
	uint64_t tma_addr;
417 418 419
	/* Relevant for CU */
	uint32_t cu_mask_count; /* Must be a multiple of 32 */
	uint32_t *cu_mask;
B
Ben Goz 已提交
420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
};

/**
 * struct queue
 *
 * @list: Queue linked list.
 *
 * @mqd: The queue MQD.
 *
 * @mqd_mem_obj: The MQD local gpu memory object.
 *
 * @gart_mqd_addr: The MQD gart mc address.
 *
 * @properties: The queue properties.
 *
 * @mec: Used only in no cp scheduling mode and identifies to micro engine id
436
 *	 that the queue should be execute on.
B
Ben Goz 已提交
437
 *
438 439
 * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe
 *	  id.
B
Ben Goz 已提交
440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462
 *
 * @queue: Used only in no cp scheduliong mode and identifies the queue's slot.
 *
 * @process: The kfd process that created this queue.
 *
 * @device: The kfd device that created this queue.
 *
 * This structure represents user mode compute queues.
 * It contains all the necessary data to handle such queues.
 *
 */

struct queue {
	struct list_head list;
	void *mqd;
	struct kfd_mem_obj *mqd_mem_obj;
	uint64_t gart_mqd_addr;
	struct queue_properties properties;

	uint32_t mec;
	uint32_t pipe;
	uint32_t queue;

B
Ben Goz 已提交
463
	unsigned int sdma_id;
464
	unsigned int doorbell_id;
B
Ben Goz 已提交
465

B
Ben Goz 已提交
466 467 468 469
	struct kfd_process	*process;
	struct kfd_dev		*device;
};

B
Ben Goz 已提交
470 471 472 473
/*
 * Please read the kfd_mqd_manager.h description.
 */
enum KFD_MQD_TYPE {
474 475 476 477
	KFD_MQD_TYPE_COMPUTE = 0,	/* for no cp scheduling */
	KFD_MQD_TYPE_HIQ,		/* for hiq */
	KFD_MQD_TYPE_CP,		/* for cp queues and diq */
	KFD_MQD_TYPE_SDMA,		/* for sdma queues */
478
	KFD_MQD_TYPE_DIQ,		/* for diq */
B
Ben Goz 已提交
479 480 481
	KFD_MQD_TYPE_MAX
};

B
Ben Goz 已提交
482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509
struct scheduling_resources {
	unsigned int vmid_mask;
	enum kfd_queue_type type;
	uint64_t queue_mask;
	uint64_t gws_mask;
	uint32_t oac_mask;
	uint32_t gds_heap_base;
	uint32_t gds_heap_size;
};

struct process_queue_manager {
	/* data */
	struct kfd_process	*process;
	struct list_head	queues;
	unsigned long		*queue_slot_bitmap;
};

struct qcm_process_device {
	/* The Device Queue Manager that owns this data */
	struct device_queue_manager *dqm;
	struct process_queue_manager *pqm;
	/* Queues list */
	struct list_head queues_list;
	struct list_head priv_queue_list;

	unsigned int queue_count;
	unsigned int vmid;
	bool is_debug;
510
	unsigned int evicted; /* eviction counter, 0=active */
511 512 513 514 515 516

	/* This flag tells if we should reset all wavefronts on
	 * process termination
	 */
	bool reset_wavefronts;

B
Ben Goz 已提交
517 518 519 520
	/*
	 * All the memory management data should be here too
	 */
	uint64_t gds_context_area;
521
	/* Contains page table flags such as AMDGPU_PTE_VALID since gfx9 */
522
	uint64_t page_table_base;
B
Ben Goz 已提交
523 524 525 526 527 528 529
	uint32_t sh_mem_config;
	uint32_t sh_mem_bases;
	uint32_t sh_mem_ape1_base;
	uint32_t sh_mem_ape1_limit;
	uint32_t gds_size;
	uint32_t num_gws;
	uint32_t num_oac;
530
	uint32_t sh_hidden_private_base;
F
Felix Kuehling 已提交
531 532 533

	/* CWSR memory */
	void *cwsr_kaddr;
534
	uint64_t cwsr_base;
F
Felix Kuehling 已提交
535 536
	uint64_t tba_addr;
	uint64_t tma_addr;
537 538 539

	/* IB memory */
	uint64_t ib_base;
540
	void *ib_kaddr;
541 542 543

	/* doorbell resources per process per device */
	unsigned long *doorbell_bitmap;
B
Ben Goz 已提交
544 545
};

546 547 548 549 550 551 552 553 554
/* KFD Memory Eviction */

/* Approx. wait time before attempting to restore evicted BOs */
#define PROCESS_RESTORE_TIME_MS 100
/* Approx. back off time if restore fails due to lack of memory */
#define PROCESS_BACK_OFF_TIME_MS 100
/* Approx. time before evicting the process again */
#define PROCESS_ACTIVE_TIME_MS 10

555 556 557 558 559 560 561 562
/* 8 byte handle containing GPU ID in the most significant 4 bytes and
 * idr_handle in the least significant 4 bytes
 */
#define MAKE_HANDLE(gpu_id, idr_handle) \
	(((uint64_t)(gpu_id) << 32) + idr_handle)
#define GET_GPU_ID(handle) (handle >> 32)
#define GET_IDR_HANDLE(handle) (handle & 0xFFFFFFFF)

563 564 565 566 567 568
enum kfd_pdd_bound {
	PDD_UNBOUND = 0,
	PDD_BOUND,
	PDD_BOUND_SUSPENDED,
};

569 570 571 572 573 574 575 576 577 578 579
/* Data that is per-process-per device. */
struct kfd_process_device {
	/*
	 * List of all per-device data for a process.
	 * Starts from kfd_process.per_device_data.
	 */
	struct list_head per_device_list;

	/* The device that owns this data. */
	struct kfd_dev *dev;

580 581
	/* The process that owns this kfd_process_device. */
	struct kfd_process *process;
582

583 584 585
	/* per-process-per device QCM data structure */
	struct qcm_process_device qpd;

586 587 588 589 590 591 592 593
	/*Apertures*/
	uint64_t lds_base;
	uint64_t lds_limit;
	uint64_t gpuvm_base;
	uint64_t gpuvm_limit;
	uint64_t scratch_base;
	uint64_t scratch_limit;

594
	/* VM context for GPUVM allocations */
595
	struct file *drm_file;
596 597
	void *vm;

598 599 600
	/* GPUVM allocations storage */
	struct idr alloc_idr;

601 602 603 604
	/* Flag used to tell the pdd has dequeued from the dqm.
	 * This is used to prevent dev->dqm->ops.process_termination() from
	 * being called twice when it is already called in IOMMU callback
	 * function.
605
	 */
606
	bool already_dequeued;
607 608 609

	/* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */
	enum kfd_pdd_bound bound;
610 611
};

612 613
#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)

O
Oded Gabbay 已提交
614 615
/* Process data */
struct kfd_process {
616 617 618 619 620 621
	/*
	 * kfd_process are stored in an mm_struct*->kfd_process*
	 * hash table (kfd_processes in kfd_process.c)
	 */
	struct hlist_node kfd_processes;

622 623 624 625 626 627
	/*
	 * Opaque pointer to mm_struct. We don't hold a reference to
	 * it so it should never be dereferenced from here. This is
	 * only used for looking up processes by their mm.
	 */
	void *mm;
628

629 630 631
	struct kref ref;
	struct work_struct release_work;

632 633 634 635 636 637
	struct mutex mutex;

	/*
	 * In any process, the thread that started main() is the lead
	 * thread and outlives the rest.
	 * It is here because amd_iommu_bind_pasid wants a task_struct.
F
Felix Kuehling 已提交
638 639
	 * It can also be used for safely getting a reference to the
	 * mm_struct of the process.
640 641 642 643 644 645 646 647 648 649
	 */
	struct task_struct *lead_thread;

	/* We want to receive a notification when the mm_struct is destroyed */
	struct mmu_notifier mmu_notifier;

	/* Use for delayed freeing of kfd_process structure */
	struct rcu_head	rcu;

	unsigned int pasid;
650
	unsigned int doorbell_index;
651 652 653 654 655 656 657

	/*
	 * List of kfd_process_device structures,
	 * one for each device the process is using.
	 */
	struct list_head per_device_data;

658 659
	struct process_queue_manager pqm;

660 661
	/*Is the user space process 32 bit?*/
	bool is_32bit_user_mode;
662 663 664

	/* Event-related data */
	struct mutex event_mutex;
665 666
	/* Event ID allocator and lookup */
	struct idr event_idr;
667 668
	/* Event page */
	struct kfd_signal_page *signal_page;
669
	size_t signal_mapped_size;
670
	size_t signal_event_count;
671
	bool signal_event_limit_reached;
672 673 674 675 676 677 678 679

	/* Information used for memory eviction */
	void *kgd_process_info;
	/* Eviction fence that is attached to all the BOs of this process. The
	 * fence will be triggered during eviction and new one will be created
	 * during restore
	 */
	struct dma_fence *ef;
680 681 682 683 684 685 686 687 688 689

	/* Work items for evicting and restoring BOs */
	struct delayed_work eviction_work;
	struct delayed_work restore_work;
	/* seqno of the last scheduled eviction */
	unsigned int last_eviction_seqno;
	/* Approx. the last timestamp (in jiffies) when the process was
	 * restored after an eviction
	 */
	unsigned long last_restore_timestamp;
O
Oded Gabbay 已提交
690 691
};

692 693 694 695
#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
extern DECLARE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
extern struct srcu_struct kfd_processes_srcu;

696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712
/**
 * Ioctl function type.
 *
 * \param filep pointer to file structure.
 * \param p amdkfd process pointer.
 * \param data pointer to arg that was copied from user.
 */
typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p,
				void *data);

struct amdkfd_ioctl_desc {
	unsigned int cmd;
	int flags;
	amdkfd_ioctl_t *func;
	unsigned int cmd_drv;
	const char *name;
};
713
bool kfd_dev_is_large_bar(struct kfd_dev *dev);
714

715
int kfd_process_create_wq(void);
716
void kfd_process_destroy_wq(void);
F
Felix Kuehling 已提交
717
struct kfd_process *kfd_create_process(struct file *filep);
718
struct kfd_process *kfd_get_process(const struct task_struct *);
719
struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid);
720
struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
721
void kfd_unref_process(struct kfd_process *p);
722 723
int kfd_process_evict_queues(struct kfd_process *p);
int kfd_process_restore_queues(struct kfd_process *p);
724 725
void kfd_suspend_all_processes(void);
int kfd_resume_all_processes(void);
726

727 728
int kfd_process_device_init_vm(struct kfd_process_device *pdd,
			       struct file *drm_file);
729
struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
730
						struct kfd_process *p);
731
struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
732 733 734
							struct kfd_process *p);
struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
							struct kfd_process *p);
735

736
int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
F
Felix Kuehling 已提交
737 738
			  struct vm_area_struct *vma);

739 740 741 742 743 744 745 746
/* KFD process API for creating and translating handles */
int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
					void *mem);
void *kfd_process_device_translate_handle(struct kfd_process_device *p,
					int handle);
void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
					int handle);

747
/* Process device data iterator */
748 749 750 751
struct kfd_process_device *kfd_get_first_process_device_data(
							struct kfd_process *p);
struct kfd_process_device *kfd_get_next_process_device_data(
						struct kfd_process *p,
752 753 754
						struct kfd_process_device *pdd);
bool kfd_has_process_device_data(struct kfd_process *p);

755 756 757 758 759 760 761 762 763
/* PASIDs */
int kfd_pasid_init(void);
void kfd_pasid_exit(void);
bool kfd_set_pasid_limit(unsigned int new_limit);
unsigned int kfd_get_pasid_limit(void);
unsigned int kfd_pasid_alloc(void);
void kfd_pasid_free(unsigned int pasid);

/* Doorbells */
764
size_t kfd_doorbell_process_slice(struct kfd_dev *kfd);
765 766
int kfd_doorbell_init(struct kfd_dev *kfd);
void kfd_doorbell_fini(struct kfd_dev *kfd);
767 768
int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
		      struct vm_area_struct *vma);
769
void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
770 771 772
					unsigned int *doorbell_off);
void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr);
u32 read_kernel_doorbell(u32 __iomem *db);
773
void write_kernel_doorbell(void __iomem *db, u32 value);
774
void write_kernel_doorbell64(void __iomem *db, u64 value);
775
unsigned int kfd_doorbell_id_to_offset(struct kfd_dev *kfd,
776
					struct kfd_process *process,
777
					unsigned int doorbell_id);
778 779 780 781
phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
					struct kfd_process *process);
int kfd_alloc_process_doorbells(struct kfd_process *process);
void kfd_free_process_doorbells(struct kfd_process *process);
782

783 784 785 786 787 788 789
/* GTT Sub-Allocator */

int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
			struct kfd_mem_obj **mem_obj);

int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj);

O
Oded Gabbay 已提交
790 791
extern struct device *kfd_device;

792 793 794 795 796
/* Topology */
int kfd_topology_init(void);
void kfd_topology_shutdown(void);
int kfd_topology_add_device(struct kfd_dev *gpu);
int kfd_topology_remove_device(struct kfd_dev *gpu);
797 798
struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
						uint32_t proximity_domain);
799
struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id);
800 801
struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
802
struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd);
803
int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev);
804
int kfd_numa_node_to_apic_id(int numa_node_id);
805

O
Oded Gabbay 已提交
806
/* Interrupts */
807 808 809
int kfd_interrupt_init(struct kfd_dev *dev);
void kfd_interrupt_exit(struct kfd_dev *dev);
bool enqueue_ih_ring_entry(struct kfd_dev *kfd,	const void *ih_ring_entry);
810 811 812
bool interrupt_is_wanted(struct kfd_dev *dev,
				const uint32_t *ih_ring_entry,
				uint32_t *patched_ihre, bool *flag);
O
Oded Gabbay 已提交
813

814 815 816
/* amdkfd Apertures */
int kfd_init_apertures(struct kfd_process *process);

B
Ben Goz 已提交
817
/* Queue Context Management */
818
int init_queue(struct queue **q, const struct queue_properties *properties);
B
Ben Goz 已提交
819
void uninit_queue(struct queue *q);
820
void print_queue_properties(struct queue_properties *q);
B
Ben Goz 已提交
821 822
void print_queue(struct queue *q);

823 824
struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
825 826
struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
827 828
struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
829 830
struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
831 832
struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
		struct kfd_dev *dev);
833 834
struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
void device_queue_manager_uninit(struct device_queue_manager *dqm);
B
Ben Goz 已提交
835 836 837
struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
					enum kfd_queue_type type);
void kernel_queue_uninit(struct kernel_queue *kq);
S
shaoyunl 已提交
838
int kfd_process_vm_fault(struct device_queue_manager *dqm, unsigned int pasid);
B
Ben Goz 已提交
839

840 841 842 843 844 845 846
/* Process Queue Manager */
struct process_queue_node {
	struct queue *q;
	struct kernel_queue *kq;
	struct list_head process_queue_list;
};

847 848
void kfd_process_dequeue_from_device(struct kfd_process_device *pdd);
void kfd_process_dequeue_from_all_devices(struct kfd_process *p);
849 850 851 852 853 854 855 856 857 858
int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p);
void pqm_uninit(struct process_queue_manager *pqm);
int pqm_create_queue(struct process_queue_manager *pqm,
			    struct kfd_dev *dev,
			    struct file *f,
			    struct queue_properties *properties,
			    unsigned int *qid);
int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
			struct queue_properties *p);
859 860
int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
			struct queue_properties *p);
861 862
struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
						unsigned int qid);
863 864 865 866 867
int pqm_get_wave_state(struct process_queue_manager *pqm,
		       unsigned int qid,
		       void __user *ctl_stack,
		       u32 *ctl_stack_used_size,
		       u32 *save_area_used_size);
868

869 870
int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
				unsigned int fence_value,
871
				unsigned int timeout_ms);
872

B
Ben Goz 已提交
873 874
/* Packet Manager */

875 876
#define KFD_FENCE_COMPLETED (100)
#define KFD_FENCE_INIT   (10)
B
Ben Goz 已提交
877

B
Ben Goz 已提交
878 879 880 881 882 883
struct packet_manager {
	struct device_queue_manager *dqm;
	struct kernel_queue *priv_queue;
	struct mutex lock;
	bool allocated;
	struct kfd_mem_obj *ib_buffer_obj;
884
	unsigned int ib_size_bytes;
885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915

	const struct packet_manager_funcs *pmf;
};

struct packet_manager_funcs {
	/* Support ASIC-specific packet formats for PM4 packets */
	int (*map_process)(struct packet_manager *pm, uint32_t *buffer,
			struct qcm_process_device *qpd);
	int (*runlist)(struct packet_manager *pm, uint32_t *buffer,
			uint64_t ib, size_t ib_size_in_dwords, bool chain);
	int (*set_resources)(struct packet_manager *pm, uint32_t *buffer,
			struct scheduling_resources *res);
	int (*map_queues)(struct packet_manager *pm, uint32_t *buffer,
			struct queue *q, bool is_static);
	int (*unmap_queues)(struct packet_manager *pm, uint32_t *buffer,
			enum kfd_queue_type type,
			enum kfd_unmap_queues_filter mode,
			uint32_t filter_param, bool reset,
			unsigned int sdma_engine);
	int (*query_status)(struct packet_manager *pm, uint32_t *buffer,
			uint64_t fence_address,	uint32_t fence_value);
	int (*release_mem)(uint64_t gpu_addr, uint32_t *buffer);

	/* Packet sizes */
	int map_process_size;
	int runlist_size;
	int set_resources_size;
	int map_queues_size;
	int unmap_queues_size;
	int query_status_size;
	int release_mem_size;
B
Ben Goz 已提交
916 917
};

918
extern const struct packet_manager_funcs kfd_vi_pm_funcs;
919
extern const struct packet_manager_funcs kfd_v9_pm_funcs;
920

921 922 923 924 925 926 927 928 929
int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
void pm_uninit(struct packet_manager *pm);
int pm_send_set_resources(struct packet_manager *pm,
				struct scheduling_resources *res);
int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues);
int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
				uint32_t fence_value);

int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
930
			enum kfd_unmap_queues_filter mode,
931 932 933
			uint32_t filter_param, bool reset,
			unsigned int sdma_engine);

B
Ben Goz 已提交
934 935
void pm_release_ib(struct packet_manager *pm);

936 937 938 939 940
/* Following PM funcs can be shared among VI and AI */
unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size);
int pm_set_resources_vi(struct packet_manager *pm, uint32_t *buffer,
				struct scheduling_resources *res);

941 942
uint64_t kfd_get_number_elems(struct kfd_dev *kfd);

943 944
/* Events */
extern const struct kfd_event_interrupt_class event_interrupt_class_cik;
945 946
extern const struct kfd_event_interrupt_class event_interrupt_class_v9;

947
extern const struct kfd_device_global_init_class device_global_init_class_cik;
948 949 950 951 952

void kfd_event_init_process(struct kfd_process *p);
void kfd_event_free_process(struct kfd_process *p);
int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma);
int kfd_wait_on_events(struct kfd_process *p,
953
		       uint32_t num_events, void __user *data,
954
		       bool all, uint32_t user_timeout_ms,
955
		       uint32_t *wait_result);
956 957
void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
				uint32_t valid_id_bits);
958 959 960
void kfd_signal_iommu_event(struct kfd_dev *dev,
		unsigned int pasid, unsigned long address,
		bool is_write_requested, bool is_execute_requested);
961
void kfd_signal_hw_exception_event(unsigned int pasid);
962 963
int kfd_set_event(struct kfd_process *p, uint32_t event_id);
int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
964 965
int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
		       uint64_t size);
966 967 968 969 970 971
int kfd_event_create(struct file *devkfd, struct kfd_process *p,
		     uint32_t event_type, bool auto_reset, uint32_t node_id,
		     uint32_t *event_id, uint32_t *event_trigger_data,
		     uint64_t *event_page_offset, uint32_t *event_slot_index);
int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);

S
shaoyunl 已提交
972 973 974
void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
				struct kfd_vm_fault_info *info);

975 976
void kfd_signal_reset_event(struct kfd_dev *dev);

977 978
void kfd_flush_tlb(struct kfd_process_device *pdd);

979 980
int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p);

981 982
bool kfd_is_locked(void);

983 984 985 986
/* Compute profile */
void kfd_inc_compute_active(struct kfd_dev *dev);
void kfd_dec_compute_active(struct kfd_dev *dev);

987 988 989 990 991 992 993 994 995 996 997 998
/* Debugfs */
#if defined(CONFIG_DEBUG_FS)

void kfd_debugfs_init(void);
void kfd_debugfs_fini(void);
int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data);
int pqm_debugfs_mqds(struct seq_file *m, void *data);
int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data);
int dqm_debugfs_hqds(struct seq_file *m, void *data);
int kfd_debugfs_rls_by_device(struct seq_file *m, void *data);
int pm_debugfs_runlist(struct seq_file *m, void *data);

999 1000 1001 1002
int kfd_debugfs_hang_hws(struct kfd_dev *dev);
int pm_debugfs_hang_hws(struct packet_manager *pm);
int dqm_debugfs_execute_queues(struct device_queue_manager *dqm);

1003 1004 1005 1006 1007 1008 1009
#else

static inline void kfd_debugfs_init(void) {}
static inline void kfd_debugfs_fini(void) {}

#endif

O
Oded Gabbay 已提交
1010
#endif