habanalabs.h 68.3 KB
Newer Older
O
Oded Gabbay 已提交
1 2 3 4 5 6 7 8 9 10
/* SPDX-License-Identifier: GPL-2.0
 *
 * Copyright 2016-2019 HabanaLabs, Ltd.
 * All Rights Reserved.
 *
 */

#ifndef HABANALABSP_H_
#define HABANALABSP_H_

11 12
#include "../include/common/armcp_if.h"
#include "../include/common/qman_if.h"
13
#include <uapi/misc/habanalabs.h>
14

O
Oded Gabbay 已提交
15
#include <linux/cdev.h>
16
#include <linux/iopoll.h>
17
#include <linux/irqreturn.h>
18 19
#include <linux/dma-direction.h>
#include <linux/scatterlist.h>
20
#include <linux/hashtable.h>
O
Oded Gabbay 已提交
21 22 23

#define HL_NAME				"habanalabs"

24 25
#define HL_MMAP_CB_MASK			(0x8000000000000000ull >> PAGE_SHIFT)

26
#define HL_PENDING_RESET_PER_SEC	30
27

28 29
#define HL_HARD_RESET_MAX_TIMEOUT	120

30 31
#define HL_DEVICE_TIMEOUT_USEC		1000000 /* 1 s */

32 33
#define HL_HEARTBEAT_PER_USEC		5000000 /* 5 s */

34 35
#define HL_PLL_LOW_JOB_FREQ_USEC	5000000 /* 5 s */

36 37 38
#define HL_ARMCP_INFO_TIMEOUT_USEC	10000000 /* 10s */
#define HL_ARMCP_EEPROM_TIMEOUT_USEC	10000000 /* 10s */

39 40
#define HL_PCI_ELBI_TIMEOUT_MSEC	10 /* 10ms */

O
Oded Gabbay 已提交
41 42
#define HL_SIM_MAX_TIMEOUT_US		10000000 /* 10s */

43 44
#define HL_IDLE_BUSY_TS_ARR_SIZE	4096

45 46 47 48 49 50
/* Memory */
#define MEM_HASH_TABLE_BITS		7 /* 1 << 7 buckets */

/* MMU */
#define MMU_HASH_TABLE_BITS		7 /* 1 << 7 buckets */

51 52 53 54
/*
 * HL_RSVD_SOBS 'sync stream' reserved sync objects per QMAN stream
 * HL_RSVD_MONS 'sync stream' reserved monitors per QMAN stream
 */
55 56 57 58 59 60 61 62
#define HL_RSVD_SOBS			4
#define HL_RSVD_MONS			2

#define HL_RSVD_SOBS_IN_USE		2
#define HL_RSVD_MONS_IN_USE		1

#define HL_MAX_SOB_VAL			(1 << 15)

63 64 65
#define IS_POWER_OF_2(n)		(n != 0 && ((n & (n - 1)) == 0))
#define IS_MAX_PENDING_CS_VALID(n)	(IS_POWER_OF_2(n) && (n > 1))

O
Ofir Bitton 已提交
66 67
#define HL_PCI_NUM_BARS			6

68 69
/**
 * struct pgt_info - MMU hop page info.
70 71 72
 * @node: hash linked-list node for the pgts shadow hash of pgts.
 * @phys_addr: physical address of the pgt.
 * @shadow_addr: shadow hop in the host.
73 74 75 76 77 78 79 80 81
 * @ctx: pointer to the owner ctx.
 * @num_of_ptes: indicates how many ptes are used in the pgt.
 *
 * The MMU page tables hierarchy is placed on the DRAM. When a new level (hop)
 * is needed during mapping, a new page is allocated and this structure holds
 * its essential information. During unmapping, if no valid PTEs remained in the
 * page, it is freed with its pgt_info structure.
 */
struct pgt_info {
82 83 84 85 86
	struct hlist_node	node;
	u64			phys_addr;
	u64			shadow_addr;
	struct hl_ctx		*ctx;
	int			num_of_ptes;
87 88
};

O
Oded Gabbay 已提交
89
struct hl_device;
90
struct hl_fpriv;
O
Oded Gabbay 已提交
91

O
Ofir Bitton 已提交
92 93 94 95 96 97 98 99 100 101
/**
 * enum hl_pci_match_mode - pci match mode per region
 * @PCI_ADDRESS_MATCH_MODE: address match mode
 * @PCI_BAR_MATCH_MODE: bar match mode
 */
enum hl_pci_match_mode {
	PCI_ADDRESS_MATCH_MODE,
	PCI_BAR_MATCH_MODE
};

102 103 104 105 106 107 108 109 110 111
/**
 * enum hl_fw_component - F/W components to read version through registers.
 * @FW_COMP_UBOOT: u-boot.
 * @FW_COMP_PREBOOT: preboot.
 */
enum hl_fw_component {
	FW_COMP_UBOOT,
	FW_COMP_PREBOOT
};

O
Oded Gabbay 已提交
112 113 114 115 116 117 118 119
/**
 * enum hl_queue_type - Supported QUEUE types.
 * @QUEUE_TYPE_NA: queue is not available.
 * @QUEUE_TYPE_EXT: external queue which is a DMA channel that may access the
 *                  host.
 * @QUEUE_TYPE_INT: internal queue that performs DMA inside the device's
 *			memories and/or operates the compute engines.
 * @QUEUE_TYPE_CPU: S/W queue for communication with the device's CPU.
T
Tomer Tayar 已提交
120 121
 * @QUEUE_TYPE_HW: queue of DMA and compute engines jobs, for which completion
 *                 notifications are sent by H/W.
O
Oded Gabbay 已提交
122 123 124 125 126
 */
enum hl_queue_type {
	QUEUE_TYPE_NA,
	QUEUE_TYPE_EXT,
	QUEUE_TYPE_INT,
T
Tomer Tayar 已提交
127 128
	QUEUE_TYPE_CPU,
	QUEUE_TYPE_HW
O
Oded Gabbay 已提交
129 130
};

131 132 133 134 135 136
enum hl_cs_type {
	CS_TYPE_DEFAULT,
	CS_TYPE_SIGNAL,
	CS_TYPE_WAIT
};

O
Ofir Bitton 已提交
137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162
/*
 * struct hl_inbound_pci_region - inbound region descriptor
 * @mode: pci match mode for this region
 * @addr: region target address
 * @size: region size in bytes
 * @offset_in_bar: offset within bar (address match mode)
 * @bar: bar id
 */
struct hl_inbound_pci_region {
	enum hl_pci_match_mode	mode;
	u64			addr;
	u64			size;
	u64			offset_in_bar;
	u8			bar;
};

/*
 * struct hl_outbound_pci_region - outbound region descriptor
 * @addr: region target address
 * @size: region size in bytes
 */
struct hl_outbound_pci_region {
	u64	addr;
	u64	size;
};

163 164 165 166 167 168 169 170 171 172 173 174 175 176
/*
 * struct hl_hw_sob - H/W SOB info.
 * @hdev: habanalabs device structure.
 * @kref: refcount of this SOB. The SOB will reset once the refcount is zero.
 * @sob_id: id of this SOB.
 * @q_idx: the H/W queue that uses this SOB.
 */
struct hl_hw_sob {
	struct hl_device	*hdev;
	struct kref		kref;
	u32			sob_id;
	u32			q_idx;
};

O
Oded Gabbay 已提交
177 178 179
/**
 * struct hw_queue_properties - queue information.
 * @type: queue type.
180 181
 * @driver_only: true if only the driver is allowed to send a job to this queue,
 *               false otherwise.
182 183
 * @requires_kernel_cb: true if a CB handle must be provided for jobs on this
 *                      queue, false otherwise (a CB address must be provided).
184
 * @supports_sync_stream: True if queue supports sync stream
O
Oded Gabbay 已提交
185 186 187
 */
struct hw_queue_properties {
	enum hl_queue_type	type;
188
	u8			driver_only;
189
	u8			requires_kernel_cb;
190
	u8			supports_sync_stream;
O
Oded Gabbay 已提交
191
};
O
Oded Gabbay 已提交
192

193 194 195
/**
 * enum vm_type_t - virtual memory mapping request information.
 * @VM_TYPE_USERPTR: mapping of user memory to device virtual address.
196
 * @VM_TYPE_PHYS_PACK: mapping of DRAM memory to device virtual address.
197 198
 */
enum vm_type_t {
199 200
	VM_TYPE_USERPTR = 0x1,
	VM_TYPE_PHYS_PACK = 0x2
201 202
};

203 204 205 206 207 208 209 210 211 212 213 214
/**
 * enum hl_device_hw_state - H/W device state. use this to understand whether
 *                           to do reset before hw_init or not
 * @HL_DEVICE_HW_STATE_CLEAN: H/W state is clean. i.e. after hard reset
 * @HL_DEVICE_HW_STATE_DIRTY: H/W state is dirty. i.e. we started to execute
 *                            hw_init
 */
enum hl_device_hw_state {
	HL_DEVICE_HW_STATE_CLEAN = 0,
	HL_DEVICE_HW_STATE_DIRTY
};

215 216
/**
 * struct hl_mmu_properties - ASIC specific MMU address translation properties.
217 218
 * @start_addr: virtual start address of the memory region.
 * @end_addr: virtual end address of the memory region.
219 220 221 222 223 224 225 226 227 228 229 230 231
 * @hop0_shift: shift of hop 0 mask.
 * @hop1_shift: shift of hop 1 mask.
 * @hop2_shift: shift of hop 2 mask.
 * @hop3_shift: shift of hop 3 mask.
 * @hop4_shift: shift of hop 4 mask.
 * @hop0_mask: mask to get the PTE address in hop 0.
 * @hop1_mask: mask to get the PTE address in hop 1.
 * @hop2_mask: mask to get the PTE address in hop 2.
 * @hop3_mask: mask to get the PTE address in hop 3.
 * @hop4_mask: mask to get the PTE address in hop 4.
 * @page_size: default page size used to allocate memory.
 */
struct hl_mmu_properties {
232 233
	u64	start_addr;
	u64	end_addr;
234 235 236 237 238 239 240 241 242 243 244 245 246
	u64	hop0_shift;
	u64	hop1_shift;
	u64	hop2_shift;
	u64	hop3_shift;
	u64	hop4_shift;
	u64	hop0_mask;
	u64	hop1_mask;
	u64	hop2_mask;
	u64	hop3_mask;
	u64	hop4_mask;
	u32	page_size;
};

O
Oded Gabbay 已提交
247 248
/**
 * struct asic_fixed_properties - ASIC specific immutable properties.
O
Oded Gabbay 已提交
249
 * @hw_queues_props: H/W queues properties.
250
 * @armcp_info: received various information from ArmCP regarding the H/W, e.g.
251
 *		available sensors.
252 253
 * @uboot_ver: F/W U-boot version.
 * @preboot_ver: F/W Preboot version.
254 255
 * @dmmu: DRAM MMU address translation properties.
 * @pmmu: PCI (host) MMU address translation properties.
256 257
 * @pmmu_huge: PCI (host) MMU address translation properties for memory
 *              allocated with huge pages.
O
Oded Gabbay 已提交
258 259 260 261 262 263 264 265
 * @sram_base_address: SRAM physical start address.
 * @sram_end_address: SRAM physical end address.
 * @sram_user_base_address - SRAM physical start address for user access.
 * @dram_base_address: DRAM physical start address.
 * @dram_end_address: DRAM physical end address.
 * @dram_user_base_address: DRAM physical start address for user access.
 * @dram_size: DRAM total size.
 * @dram_pci_bar_size: size of PCI bar towards DRAM.
266
 * @max_power_default: max power of the device after reset
267 268
 * @dram_size_for_default_page_mapping: DRAM size needed to map to avoid page
 *                                      fault.
269 270
 * @pcie_dbi_base_address: Base address of the PCIE_DBI block.
 * @pcie_aux_dbi_reg_addr: Address of the PCIE_AUX DBI register.
271
 * @mmu_pgt_addr: base physical address in DRAM of MMU page tables.
272
 * @mmu_dram_default_page_addr: DRAM default page physical address.
273 274 275 276 277
 * @mmu_pgt_size: MMU page tables total size.
 * @mmu_pte_size: PTE size in MMU page tables.
 * @mmu_hop_table_size: MMU hop table size.
 * @mmu_hop0_tables_total_size: total size of MMU hop0 tables.
 * @dram_page_size: page size for MMU DRAM allocation.
O
Oded Gabbay 已提交
278 279 280
 * @cfg_size: configuration space size on SRAM.
 * @sram_size: total size of SRAM.
 * @max_asid: maximum number of open contexts (ASIDs).
281
 * @num_of_events: number of possible internal H/W IRQs.
282 283 284 285
 * @psoc_pci_pll_nr: PCI PLL NR value.
 * @psoc_pci_pll_nf: PCI PLL NF value.
 * @psoc_pci_pll_od: PCI PLL OD value.
 * @psoc_pci_pll_div_factor: PCI PLL DIV FACTOR 1 value.
286
 * @psoc_timestamp_frequency: frequency of the psoc timestamp clock.
O
Oded Gabbay 已提交
287
 * @high_pll: high PLL frequency used by the device.
288 289
 * @cb_pool_cb_cnt: number of CBs in the CB pool.
 * @cb_pool_cb_size: size of each CB in the CB pool.
290 291
 * @max_pending_cs: maximum of concurrent pending command submissions
 * @max_queues: maximum amount of queues in the system
292 293 294
 * @sync_stream_first_sob: first sync object available for sync stream use
 * @sync_stream_first_mon: first monitor available for sync stream use
 * @tpc_enabled_mask: which TPCs are enabled.
295
 * @completion_queues_count: number of completion queues.
O
Oded Gabbay 已提交
296 297
 */
struct asic_fixed_properties {
298
	struct hw_queue_properties	*hw_queues_props;
299 300 301 302 303
	struct armcp_info		armcp_info;
	char				uboot_ver[VERSION_MAX_LEN];
	char				preboot_ver[VERSION_MAX_LEN];
	struct hl_mmu_properties	dmmu;
	struct hl_mmu_properties	pmmu;
304
	struct hl_mmu_properties	pmmu_huge;
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331
	u64				sram_base_address;
	u64				sram_end_address;
	u64				sram_user_base_address;
	u64				dram_base_address;
	u64				dram_end_address;
	u64				dram_user_base_address;
	u64				dram_size;
	u64				dram_pci_bar_size;
	u64				max_power_default;
	u64				dram_size_for_default_page_mapping;
	u64				pcie_dbi_base_address;
	u64				pcie_aux_dbi_reg_addr;
	u64				mmu_pgt_addr;
	u64				mmu_dram_default_page_addr;
	u32				mmu_pgt_size;
	u32				mmu_pte_size;
	u32				mmu_hop_table_size;
	u32				mmu_hop0_tables_total_size;
	u32				dram_page_size;
	u32				cfg_size;
	u32				sram_size;
	u32				max_asid;
	u32				num_of_events;
	u32				psoc_pci_pll_nr;
	u32				psoc_pci_pll_nf;
	u32				psoc_pci_pll_od;
	u32				psoc_pci_pll_div_factor;
332
	u32				psoc_timestamp_frequency;
333 334 335
	u32				high_pll;
	u32				cb_pool_cb_cnt;
	u32				cb_pool_cb_size;
336
	u32				max_pending_cs;
337
	u32				max_queues;
338 339
	u16				sync_stream_first_sob;
	u16				sync_stream_first_mon;
340 341
	u8				tpc_enabled_mask;
	u8				completion_queues_count;
O
Oded Gabbay 已提交
342 343
};

344 345 346 347 348 349 350 351 352 353 354 355 356
/**
 * struct hl_fence - software synchronization primitive
 * @completion: fence is implemented using completion
 * @refcount: refcount for this fence
 * @error: mark this fence with error
 *
 */
struct hl_fence {
	struct completion	completion;
	struct kref		refcount;
	int			error;
};

357
/**
358
 * struct hl_cs_compl - command submission completion object.
359
 * @base_fence: hl fence object.
360 361
 * @lock: spinlock to protect fence.
 * @hdev: habanalabs device structure.
362
 * @hw_sob: the H/W SOB used in this signal/wait CS.
363
 * @cs_seq: command submission sequence number.
364 365
 * @type: type of the CS - signal/wait.
 * @sob_val: the SOB value that is used in this signal/wait CS.
366
 */
367
struct hl_cs_compl {
368
	struct hl_fence		base_fence;
369 370
	spinlock_t		lock;
	struct hl_device	*hdev;
371
	struct hl_hw_sob	*hw_sob;
372
	u64			cs_seq;
373 374
	enum hl_cs_type		type;
	u16			sob_val;
375
};
O
Oded Gabbay 已提交
376

377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395
/*
 * Command Buffers
 */

/**
 * struct hl_cb_mgr - describes a Command Buffer Manager.
 * @cb_lock: protects cb_handles.
 * @cb_handles: an idr to hold all command buffer handles.
 */
struct hl_cb_mgr {
	spinlock_t		cb_lock;
	struct idr		cb_handles; /* protected by cb_lock */
};

/**
 * struct hl_cb - describes a Command Buffer.
 * @refcount: reference counter for usage of the CB.
 * @hdev: pointer to device this CB belongs to.
 * @lock: spinlock to protect mmap/cs flows.
O
Oded Gabbay 已提交
396
 * @debugfs_list: node in debugfs list of command buffers.
397 398 399 400 401 402
 * @pool_list: node in pool list of command buffers.
 * @kernel_address: Holds the CB's kernel virtual address.
 * @bus_address: Holds the CB's DMA address.
 * @mmap_size: Holds the CB's size that was mmaped.
 * @size: holds the CB's size.
 * @id: the CB's ID.
403
 * @cs_cnt: holds number of CS that this CB participates in.
404 405 406
 * @ctx_id: holds the ID of the owner's context.
 * @mmap: true if the CB is currently mmaped to user.
 * @is_pool: true if CB was acquired from the pool, false otherwise.
407
 * @is_internal: internaly allocated
408 409 410 411 412
 */
struct hl_cb {
	struct kref		refcount;
	struct hl_device	*hdev;
	spinlock_t		lock;
O
Oded Gabbay 已提交
413
	struct list_head	debugfs_list;
414 415 416 417 418 419
	struct list_head	pool_list;
	u64			kernel_address;
	dma_addr_t		bus_address;
	u32			mmap_size;
	u32			size;
	u32			id;
420
	u32			cs_cnt;
421 422 423
	u32			ctx_id;
	u8			mmap;
	u8			is_pool;
424
	u8			is_internal;
425 426 427
};


O
Oded Gabbay 已提交
428 429 430 431 432 433
/*
 * QUEUES
 */

struct hl_cs_job;

O
Ofir Bitton 已提交
434 435
/* Queue length of external and HW queues */
#define HL_QUEUE_LENGTH			4096
O
Oded Gabbay 已提交
436 437
#define HL_QUEUE_SIZE_IN_BYTES		(HL_QUEUE_LENGTH * HL_BD_SIZE)

438 439 440 441
#if (HL_MAX_JOBS_PER_CS > HL_QUEUE_LENGTH)
#error "HL_QUEUE_LENGTH must be greater than HL_MAX_JOBS_PER_CS"
#endif

O
Ofir Bitton 已提交
442
/* HL_CQ_LENGTH is in units of struct hl_cq_entry */
O
Oded Gabbay 已提交
443 444 445
#define HL_CQ_LENGTH			HL_QUEUE_LENGTH
#define HL_CQ_SIZE_IN_BYTES		(HL_CQ_LENGTH * HL_CQ_ENTRY_SIZE)

O
Ofir Bitton 已提交
446
/* Must be power of 2 */
447 448
#define HL_EQ_LENGTH			64
#define HL_EQ_SIZE_IN_BYTES		(HL_EQ_LENGTH * HL_EQ_ENTRY_SIZE)
O
Oded Gabbay 已提交
449

450
/* Host <-> ArmCP shared memory size */
451
#define HL_CPU_ACCESSIBLE_MEM_SIZE	SZ_2M
O
Oded Gabbay 已提交
452 453 454

/**
 * struct hl_hw_queue - describes a H/W transport queue.
455
 * @hw_sob: array of the used H/W SOBs by this H/W queue.
O
Oded Gabbay 已提交
456 457 458 459 460 461 462
 * @shadow_queue: pointer to a shadow queue that holds pointers to jobs.
 * @queue_type: type of queue.
 * @kernel_address: holds the queue's kernel virtual address.
 * @bus_address: holds the queue's DMA address.
 * @pi: holds the queue's pi value.
 * @ci: holds the queue's ci value, AS CALCULATED BY THE DRIVER (not real ci).
 * @hw_queue_id: the id of the H/W queue.
463 464
 * @cq_id: the id for the corresponding CQ for this H/W queue.
 * @msi_vec: the IRQ number of the H/W queue.
O
Oded Gabbay 已提交
465
 * @int_queue_len: length of internal queue (number of entries).
466 467 468
 * @next_sob_val: the next value to use for the currently used SOB.
 * @base_sob_id: the base SOB id of the SOBs used by this queue.
 * @base_mon_id: the base MON id of the MONs used by this queue.
O
Oded Gabbay 已提交
469
 * @valid: is the queue valid (we have array of 32 queues, not all of them
470 471 472
 *         exist).
 * @curr_sob_offset: the id offset to the currently used SOB from the
 *                   HL_RSVD_SOBS that are being used by this queue.
473
 * @supports_sync_stream: True if queue supports sync stream
O
Oded Gabbay 已提交
474 475
 */
struct hl_hw_queue {
476
	struct hl_hw_sob	hw_sob[HL_RSVD_SOBS];
O
Oded Gabbay 已提交
477 478 479 480 481
	struct hl_cs_job	**shadow_queue;
	enum hl_queue_type	queue_type;
	u64			kernel_address;
	dma_addr_t		bus_address;
	u32			pi;
482
	atomic_t		ci;
O
Oded Gabbay 已提交
483
	u32			hw_queue_id;
484 485
	u32			cq_id;
	u32			msi_vec;
O
Oded Gabbay 已提交
486
	u16			int_queue_len;
487 488 489
	u16			next_sob_val;
	u16			base_sob_id;
	u16			base_mon_id;
O
Oded Gabbay 已提交
490
	u8			valid;
491
	u8			curr_sob_offset;
492
	u8			supports_sync_stream;
O
Oded Gabbay 已提交
493 494 495 496 497 498 499
};

/**
 * struct hl_cq - describes a completion queue
 * @hdev: pointer to the device structure
 * @kernel_address: holds the queue's kernel virtual address
 * @bus_address: holds the queue's DMA address
500
 * @cq_idx: completion queue index in array
O
Oded Gabbay 已提交
501 502 503 504 505 506 507 508 509
 * @hw_queue_id: the id of the matching H/W queue
 * @ci: ci inside the queue
 * @pi: pi inside the queue
 * @free_slots_cnt: counter of free slots in queue
 */
struct hl_cq {
	struct hl_device	*hdev;
	u64			kernel_address;
	dma_addr_t		bus_address;
510
	u32			cq_idx;
O
Oded Gabbay 已提交
511 512 513 514 515
	u32			hw_queue_id;
	u32			ci;
	u32			pi;
	atomic_t		free_slots_cnt;
};
516

517 518 519 520 521 522 523 524 525 526 527 528 529 530
/**
 * struct hl_eq - describes the event queue (single one per device)
 * @hdev: pointer to the device structure
 * @kernel_address: holds the queue's kernel virtual address
 * @bus_address: holds the queue's DMA address
 * @ci: ci inside the queue
 */
struct hl_eq {
	struct hl_device	*hdev;
	u64			kernel_address;
	dma_addr_t		bus_address;
	u32			ci;
};

531

O
Oded Gabbay 已提交
532 533 534 535 536 537 538
/*
 * ASICs
 */

/**
 * enum hl_asic_type - supported ASIC types.
 * @ASIC_INVALID: Invalid ASIC type.
539
 * @ASIC_GOYA: Goya device.
540
 * @ASIC_GAUDI: Gaudi device.
O
Oded Gabbay 已提交
541 542
 */
enum hl_asic_type {
543
	ASIC_INVALID,
544 545
	ASIC_GOYA,
	ASIC_GAUDI
O
Oded Gabbay 已提交
546 547
};

548 549
struct hl_cs_parser;

550 551
/**
 * enum hl_pm_mng_profile - power management profile.
552
 * @PM_AUTO: internal clock is set by the Linux driver.
553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573
 * @PM_MANUAL: internal clock is set by the user.
 * @PM_LAST: last power management type.
 */
enum hl_pm_mng_profile {
	PM_AUTO = 1,
	PM_MANUAL,
	PM_LAST
};

/**
 * enum hl_pll_frequency - PLL frequency.
 * @PLL_HIGH: high frequency.
 * @PLL_LOW: low frequency.
 * @PLL_LAST: last frequency values that were configured by the user.
 */
enum hl_pll_frequency {
	PLL_HIGH = 1,
	PLL_LOW,
	PLL_LAST
};

574 575 576 577 578 579 580 581 582
#define PLL_REF_CLK 50

enum div_select_defs {
	DIV_SEL_REF_CLK = 0,
	DIV_SEL_PLL_CLK = 1,
	DIV_SEL_DIVIDED_REF = 2,
	DIV_SEL_DIVIDED_PLL = 3,
};

O
Oded Gabbay 已提交
583 584 585 586 587
/**
 * struct hl_asic_funcs - ASIC specific functions that are can be called from
 *                        common code.
 * @early_init: sets up early driver state (pre sw_init), doesn't configure H/W.
 * @early_fini: tears down what was done in early_init.
588 589
 * @late_init: sets up late driver/hw state (post hw_init) - Optional.
 * @late_fini: tears down what was done in late_init (pre hw_fini) - Optional.
O
Oded Gabbay 已提交
590 591
 * @sw_init: sets up driver state, does not configure H/W.
 * @sw_fini: tears down driver state, does not configure H/W.
592 593
 * @hw_init: sets up the H/W state.
 * @hw_fini: tears down the H/W state.
594 595 596
 * @halt_engines: halt engines, needed for reset sequence. This also disables
 *                interrupts from the device. Should be called before
 *                hw_fini and before CS rollback.
O
Oded Gabbay 已提交
597 598
 * @suspend: handles IP specific H/W or SW changes for suspend.
 * @resume: handles IP specific H/W or SW changes for resume.
599
 * @cb_mmap: maps a CB.
O
Oded Gabbay 已提交
600
 * @ring_doorbell: increment PI on a given QMAN.
601 602 603 604 605
 * @pqe_write: Write the PQ entry to the PQ. This is ASIC-specific
 *             function because the PQs are located in different memory areas
 *             per ASIC (SRAM, DRAM, Host memory) and therefore, the method of
 *             writing the PQE must match the destination memory area
 *             properties.
606 607 608 609 610 611 612 613
 * @asic_dma_alloc_coherent: Allocate coherent DMA memory by calling
 *                           dma_alloc_coherent(). This is ASIC function because
 *                           its implementation is not trivial when the driver
 *                           is loaded in simulation mode (not upstreamed).
 * @asic_dma_free_coherent:  Free coherent DMA memory by calling
 *                           dma_free_coherent(). This is ASIC function because
 *                           its implementation is not trivial when the driver
 *                           is loaded in simulation mode (not upstreamed).
O
Oded Gabbay 已提交
614 615
 * @get_int_queue_base: get the internal queue base address.
 * @test_queues: run simple test on all queues for sanity check.
616 617 618
 * @asic_dma_pool_zalloc: small DMA allocation of coherent memory from DMA pool.
 *                        size of allocation is HL_DMA_POOL_BLK_SIZE.
 * @asic_dma_pool_free: free small DMA allocation from pool.
O
Oded Gabbay 已提交
619 620
 * @cpu_accessible_dma_pool_alloc: allocate CPU PQ packet from DMA pool.
 * @cpu_accessible_dma_pool_free: free CPU PQ packet from DMA pool.
621 622 623 624 625
 * @hl_dma_unmap_sg: DMA unmap scatter-gather list.
 * @cs_parser: parse Command Submission.
 * @asic_dma_map_sg: DMA map scatter-gather list.
 * @get_dma_desc_list_size: get number of LIN_DMA packets required for CB.
 * @add_end_of_cb_packets: Add packets to the end of CB, if device requires it.
626
 * @update_eq_ci: update event queue CI.
627 628
 * @context_switch: called upon ASID context switch.
 * @restore_phase_topology: clear all SOBs amd MONs.
O
Oded Gabbay 已提交
629 630
 * @debugfs_read32: debug interface for reading u32 from DRAM/SRAM.
 * @debugfs_write32: debug interface for writing u32 to DRAM/SRAM.
631
 * @add_device_attr: add ASIC specific device attributes.
632
 * @handle_eqe: handle event queue entry (IRQ) from ArmCP.
633
 * @set_pll_profile: change PLL profile (manual/automatic).
634
 * @get_events_stat: retrieve event queue entries histogram.
635 636
 * @read_pte: read MMU page table entry from DRAM.
 * @write_pte: write MMU page table entry to DRAM.
637 638
 * @mmu_invalidate_cache: flush MMU STLB host/DRAM cache, either with soft
 *                        (L1 only) or hard (L0 & L1) flush.
639 640
 * @mmu_invalidate_cache_range: flush specific MMU STLB cache lines with
 *                              ASID-VA-size mask.
641
 * @send_heartbeat: send is-alive packet to ArmCP and verify response.
642 643 644
 * @set_clock_gating: enable/disable clock gating per engine according to
 *                    clock gating mask in hdev
 * @disable_clock_gating: disable clock gating completely
645
 * @debug_coresight: perform certain actions on Coresight for debugging.
646
 * @is_device_idle: return true if device is idle, false otherwise.
647
 * @soft_reset_late_init: perform certain actions needed after soft reset.
O
Oded Gabbay 已提交
648 649
 * @hw_queues_lock: acquire H/W queues lock.
 * @hw_queues_unlock: release H/W queues lock.
O
Oded Gabbay 已提交
650
 * @get_pci_id: retrieve PCI ID.
651
 * @get_eeprom_data: retrieve EEPROM data from F/W.
652 653 654 655 656
 * @send_cpu_message: send message to F/W. If the message is timedout, the
 *                    driver will eventually reset the device. The timeout can
 *                    be determined by the calling function or it can be 0 and
 *                    then the timeout is the default timeout for the specific
 *                    ASIC
657
 * @get_hw_state: retrieve the H/W state
658
 * @pci_bars_map: Map PCI BARs.
659 660
 * @set_dram_bar_base: Set DRAM BAR to map specific device address. Returns
 *                     old address the bar pointed to or U64_MAX for failure
661
 * @init_iatu: Initialize the iATU unit inside the PCI controller.
662 663
 * @rreg: Read a register. Needed for simulator support.
 * @wreg: Write a register. Needed for simulator support.
664
 * @halt_coresight: stop the ETF and ETR traces.
665
 * @ctx_init: context dependent initialization.
666
 * @get_clk_rate: Retrieve the ASIC current and maximum clock rate in MHz
667
 * @get_queue_id_for_cq: Get the H/W queue id related to the given CQ index.
668 669 670
 * @read_device_fw_version: read the device's firmware versions that are
 *                          contained in registers
 * @load_firmware_to_device: load the firmware to the device's memory
671
 * @load_boot_fit_to_device: load boot fit to device's memory
672 673 674 675 676
 * @get_signal_cb_size: Get signal CB size.
 * @get_wait_cb_size: Get wait CB size.
 * @gen_signal_cb: Generate a signal CB.
 * @gen_wait_cb: Generate a wait CB.
 * @reset_sob: Reset a SOB.
677 678
 * @set_dma_mask_from_fw: set the DMA mask in the driver according to the
 *                        firmware configuration
679
 * @get_device_time: Get the device time.
O
Oded Gabbay 已提交
680 681 682 683
 */
struct hl_asic_funcs {
	int (*early_init)(struct hl_device *hdev);
	int (*early_fini)(struct hl_device *hdev);
684 685
	int (*late_init)(struct hl_device *hdev);
	void (*late_fini)(struct hl_device *hdev);
O
Oded Gabbay 已提交
686 687
	int (*sw_init)(struct hl_device *hdev);
	int (*sw_fini)(struct hl_device *hdev);
688 689
	int (*hw_init)(struct hl_device *hdev);
	void (*hw_fini)(struct hl_device *hdev, bool hard_reset);
690
	void (*halt_engines)(struct hl_device *hdev, bool hard_reset);
O
Oded Gabbay 已提交
691 692
	int (*suspend)(struct hl_device *hdev);
	int (*resume)(struct hl_device *hdev);
693 694
	int (*cb_mmap)(struct hl_device *hdev, struct vm_area_struct *vma,
			u64 kaddress, phys_addr_t paddress, u32 size);
O
Oded Gabbay 已提交
695
	void (*ring_doorbell)(struct hl_device *hdev, u32 hw_queue_id, u32 pi);
696 697
	void (*pqe_write)(struct hl_device *hdev, __le64 *pqe,
			struct hl_bd *bd);
698
	void* (*asic_dma_alloc_coherent)(struct hl_device *hdev, size_t size,
O
Oded Gabbay 已提交
699
					dma_addr_t *dma_handle, gfp_t flag);
700
	void (*asic_dma_free_coherent)(struct hl_device *hdev, size_t size,
O
Oded Gabbay 已提交
701
					void *cpu_addr, dma_addr_t dma_handle);
O
Oded Gabbay 已提交
702 703 704
	void* (*get_int_queue_base)(struct hl_device *hdev, u32 queue_id,
				dma_addr_t *dma_handle, u16 *queue_len);
	int (*test_queues)(struct hl_device *hdev);
705
	void* (*asic_dma_pool_zalloc)(struct hl_device *hdev, size_t size,
O
Oded Gabbay 已提交
706
				gfp_t mem_flags, dma_addr_t *dma_handle);
707
	void (*asic_dma_pool_free)(struct hl_device *hdev, void *vaddr,
O
Oded Gabbay 已提交
708 709 710 711 712
				dma_addr_t dma_addr);
	void* (*cpu_accessible_dma_pool_alloc)(struct hl_device *hdev,
				size_t size, dma_addr_t *dma_handle);
	void (*cpu_accessible_dma_pool_free)(struct hl_device *hdev,
				size_t size, void *vaddr);
713
	void (*hl_dma_unmap_sg)(struct hl_device *hdev,
714
				struct scatterlist *sgl, int nents,
715 716 717
				enum dma_data_direction dir);
	int (*cs_parser)(struct hl_device *hdev, struct hl_cs_parser *parser);
	int (*asic_dma_map_sg)(struct hl_device *hdev,
718
				struct scatterlist *sgl, int nents,
719 720 721
				enum dma_data_direction dir);
	u32 (*get_dma_desc_list_size)(struct hl_device *hdev,
					struct sg_table *sgt);
722 723
	void (*add_end_of_cb_packets)(struct hl_device *hdev,
					u64 kernel_address, u32 len,
724 725
					u64 cq_addr, u32 cq_val, u32 msix_num,
					bool eb);
726
	void (*update_eq_ci)(struct hl_device *hdev, u32 val);
727 728
	int (*context_switch)(struct hl_device *hdev, u32 asid);
	void (*restore_phase_topology)(struct hl_device *hdev);
O
Oded Gabbay 已提交
729 730
	int (*debugfs_read32)(struct hl_device *hdev, u64 addr, u32 *val);
	int (*debugfs_write32)(struct hl_device *hdev, u64 addr, u32 val);
731 732
	int (*debugfs_read64)(struct hl_device *hdev, u64 addr, u64 *val);
	int (*debugfs_write64)(struct hl_device *hdev, u64 addr, u64 val);
733 734
	void (*add_device_attr)(struct hl_device *hdev,
				struct attribute_group *dev_attr_grp);
735 736
	void (*handle_eqe)(struct hl_device *hdev,
				struct hl_eq_entry *eq_entry);
737 738
	void (*set_pll_profile)(struct hl_device *hdev,
			enum hl_pll_frequency freq);
739 740
	void* (*get_events_stat)(struct hl_device *hdev, bool aggregate,
				u32 *size);
741 742
	u64 (*read_pte)(struct hl_device *hdev, u64 addr);
	void (*write_pte)(struct hl_device *hdev, u64 addr, u64 val);
743
	int (*mmu_invalidate_cache)(struct hl_device *hdev, bool is_hard,
744
					u32 flags);
745
	int (*mmu_invalidate_cache_range)(struct hl_device *hdev, bool is_hard,
746
			u32 asid, u64 va, u64 size);
747
	int (*send_heartbeat)(struct hl_device *hdev);
748
	void (*set_clock_gating)(struct hl_device *hdev);
749
	void (*disable_clock_gating)(struct hl_device *hdev);
750
	int (*debug_coresight)(struct hl_device *hdev, void *data);
751 752
	bool (*is_device_idle)(struct hl_device *hdev, u32 *mask,
				struct seq_file *s);
753
	int (*soft_reset_late_init)(struct hl_device *hdev);
O
Oded Gabbay 已提交
754 755
	void (*hw_queues_lock)(struct hl_device *hdev);
	void (*hw_queues_unlock)(struct hl_device *hdev);
O
Oded Gabbay 已提交
756
	u32 (*get_pci_id)(struct hl_device *hdev);
757 758
	int (*get_eeprom_data)(struct hl_device *hdev, void *data,
				size_t max_size);
O
Oded Gabbay 已提交
759 760
	int (*send_cpu_message)(struct hl_device *hdev, u32 *msg,
				u16 len, u32 timeout, long *result);
761
	enum hl_device_hw_state (*get_hw_state)(struct hl_device *hdev);
762
	int (*pci_bars_map)(struct hl_device *hdev);
763
	u64 (*set_dram_bar_base)(struct hl_device *hdev, u64 addr);
764
	int (*init_iatu)(struct hl_device *hdev);
765 766
	u32 (*rreg)(struct hl_device *hdev, u32 reg);
	void (*wreg)(struct hl_device *hdev, u32 reg, u32 val);
767
	void (*halt_coresight)(struct hl_device *hdev);
768
	int (*ctx_init)(struct hl_ctx *ctx);
769
	int (*get_clk_rate)(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk);
770
	u32 (*get_queue_id_for_cq)(struct hl_device *hdev, u32 cq_idx);
771 772 773
	void (*read_device_fw_version)(struct hl_device *hdev,
					enum hl_fw_component fwc);
	int (*load_firmware_to_device)(struct hl_device *hdev);
774
	int (*load_boot_fit_to_device)(struct hl_device *hdev);
775 776 777 778 779 780
	u32 (*get_signal_cb_size)(struct hl_device *hdev);
	u32 (*get_wait_cb_size)(struct hl_device *hdev);
	void (*gen_signal_cb)(struct hl_device *hdev, void *data, u16 sob_id);
	void (*gen_wait_cb)(struct hl_device *hdev, void *data, u16 sob_id,
				u16 sob_val, u16 mon_id, u32 q_idx);
	void (*reset_sob)(struct hl_device *hdev, void *data);
781
	void (*set_dma_mask_from_fw)(struct hl_device *hdev);
782
	u64 (*get_device_time)(struct hl_device *hdev);
O
Oded Gabbay 已提交
783
};
O
Oded Gabbay 已提交
784

785 786 787 788 789 790 791

/*
 * CONTEXTS
 */

#define HL_KERNEL_ASID_ID	0

792 793 794 795 796 797 798 799 800 801 802 803 804 805
/**
 * struct hl_va_range - virtual addresses range.
 * @lock: protects the virtual addresses list.
 * @list: list of virtual addresses blocks available for mappings.
 * @start_addr: range start address.
 * @end_addr: range end address.
 */
struct hl_va_range {
	struct mutex		lock;
	struct list_head	list;
	u64			start_addr;
	u64			end_addr;
};

806 807
/**
 * struct hl_ctx - user/kernel context.
808 809
 * @mem_hash: holds mapping from virtual address to virtual memory area
 *		descriptor (hl_vm_phys_pg_list or hl_userptr).
810
 * @mmu_shadow_hash: holds a mapping from shadow address to pgt_info structure.
811
 * @hpriv: pointer to the private (Kernel Driver) data of the process (fd).
812 813 814
 * @hdev: pointer to the device structure.
 * @refcount: reference counter for the context. Context is released only when
 *		this hits 0l. It is incremented on CS and CS_WAIT.
815
 * @cs_pending: array of hl fence objects representing pending CS.
816
 * @host_va_range: holds available virtual addresses for host mappings.
817 818
 * @host_huge_va_range: holds available virtual addresses for host mappings
 *                      with huge pages.
819 820
 * @dram_va_range: holds available virtual addresses for DRAM mappings.
 * @mem_hash_lock: protects the mem_hash.
821 822
 * @mmu_lock: protects the MMU page tables. Any change to the PGT, modifying the
 *            MMU hash or walking the PGT requires talking this lock.
O
Oded Gabbay 已提交
823
 * @debugfs_list: node in debugfs list of contexts.
824 825 826
 * @cs_sequence: sequence number for CS. Value is assigned to a CS and passed
 *			to user so user could inquire about CS. It is used as
 *			index to cs_pending array.
827 828
 * @dram_default_hops: array that holds all hops addresses needed for default
 *                     DRAM mapping.
829
 * @cs_lock: spinlock to protect cs_sequence.
830
 * @dram_phys_mem: amount of used physical DRAM memory by this context.
831 832 833 834 835 836 837
 * @thread_ctx_switch_token: token to prevent multiple threads of the same
 *				context	from running the context switch phase.
 *				Only a single thread should run it.
 * @thread_ctx_switch_wait_token: token to prevent the threads that didn't run
 *				the context switch phase from moving to their
 *				execution phase before the context switch phase
 *				has finished.
838
 * @asid: context's unique address space ID in the device's MMU.
839
 * @handle: context's opaque handle for user
840 841
 */
struct hl_ctx {
842
	DECLARE_HASHTABLE(mem_hash, MEM_HASH_TABLE_BITS);
843
	DECLARE_HASHTABLE(mmu_shadow_hash, MMU_HASH_TABLE_BITS);
844 845 846
	struct hl_fpriv		*hpriv;
	struct hl_device	*hdev;
	struct kref		refcount;
847
	struct hl_fence		**cs_pending;
848 849 850
	struct hl_va_range	*host_va_range;
	struct hl_va_range	*host_huge_va_range;
	struct hl_va_range	*dram_va_range;
851 852
	struct mutex		mem_hash_lock;
	struct mutex		mmu_lock;
O
Oded Gabbay 已提交
853
	struct list_head	debugfs_list;
854
	struct hl_cs_counters	cs_counters;
855
	u64			cs_sequence;
856
	u64			*dram_default_hops;
857
	spinlock_t		cs_lock;
858
	atomic64_t		dram_phys_mem;
859 860
	atomic_t		thread_ctx_switch_token;
	u32			thread_ctx_switch_wait_token;
861
	u32			asid;
862
	u32			handle;
863 864 865 866 867 868 869 870 871 872 873 874 875
};

/**
 * struct hl_ctx_mgr - for handling multiple contexts.
 * @ctx_lock: protects ctx_handles.
 * @ctx_handles: idr to hold all ctx handles.
 */
struct hl_ctx_mgr {
	struct mutex		ctx_lock;
	struct idr		ctx_handles;
};


876 877 878 879 880 881 882 883 884 885 886 887 888

/*
 * COMMAND SUBMISSIONS
 */

/**
 * struct hl_userptr - memory mapping chunk information
 * @vm_type: type of the VM.
 * @job_node: linked-list node for hanging the object on the Job's list.
 * @vec: pointer to the frame vector.
 * @sgt: pointer to the scatter-gather table that holds the pages.
 * @dir: for DMA unmapping, the direction must be supplied, so save it.
 * @debugfs_list: node in debugfs list of command submissions.
889
 * @addr: user-space virtual address of the start of the memory area.
890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912
 * @size: size of the memory area to pin & map.
 * @dma_mapped: true if the SG was mapped to DMA addresses, false otherwise.
 */
struct hl_userptr {
	enum vm_type_t		vm_type; /* must be first */
	struct list_head	job_node;
	struct frame_vector	*vec;
	struct sg_table		*sgt;
	enum dma_data_direction dir;
	struct list_head	debugfs_list;
	u64			addr;
	u32			size;
	u8			dma_mapped;
};

/**
 * struct hl_cs - command submission.
 * @jobs_in_queue_cnt: per each queue, maintain counter of submitted jobs.
 * @ctx: the context this CS belongs to.
 * @job_list: list of the CS's jobs in the various queues.
 * @job_lock: spinlock for the CS's jobs list. Needed for free_job.
 * @refcount: reference counter for usage of the CS.
 * @fence: pointer to the fence object of this CS.
913 914
 * @signal_fence: pointer to the fence object of the signal CS (used by wait
 *                CS only).
915
 * @finish_work: workqueue object to run when CS is completed by H/W.
916 917
 * @work_tdr: delayed work node for TDR.
 * @mirror_node : node in device mirror list of command submissions.
O
Oded Gabbay 已提交
918
 * @debugfs_list: node in debugfs list of command submissions.
919
 * @sequence: the sequence number of this CS.
920
 * @type: CS_TYPE_*.
921 922 923 924 925 926 927 928
 * @submitted: true if CS was submitted to H/W.
 * @completed: true if CS was completed by device.
 * @timedout : true if CS was timedout.
 * @tdr_active: true if TDR was activated for this CS (to prevent
 *		double TDR activation).
 * @aborted: true if CS was aborted due to some device error.
 */
struct hl_cs {
929
	u16			*jobs_in_queue_cnt;
930 931 932 933
	struct hl_ctx		*ctx;
	struct list_head	job_list;
	spinlock_t		job_lock;
	struct kref		refcount;
934 935
	struct hl_fence		*fence;
	struct hl_fence		*signal_fence;
936
	struct work_struct	finish_work;
937 938
	struct delayed_work	work_tdr;
	struct list_head	mirror_node;
O
Oded Gabbay 已提交
939
	struct list_head	debugfs_list;
940
	u64			sequence;
941
	enum hl_cs_type		type;
942 943 944 945 946 947 948
	u8			submitted;
	u8			completed;
	u8			timedout;
	u8			tdr_active;
	u8			aborted;
};

O
Oded Gabbay 已提交
949 950
/**
 * struct hl_cs_job - command submission job.
951 952 953 954 955
 * @cs_node: the node to hang on the CS jobs list.
 * @cs: the CS this job belongs to.
 * @user_cb: the CB we got from the user.
 * @patched_cb: in case of patching, this is internal CB which is submitted on
 *		the queue instead of the CB we got from the IOCTL.
O
Oded Gabbay 已提交
956
 * @finish_work: workqueue object to run when job is completed.
957 958
 * @userptr_list: linked-list of userptr mappings that belong to this job and
 *			wait for completion.
O
Oded Gabbay 已提交
959
 * @debugfs_list: node in debugfs list of command submission jobs.
T
Tomer Tayar 已提交
960
 * @queue_type: the type of the H/W queue this job is submitted to.
O
Oded Gabbay 已提交
961
 * @id: the id of this job inside a CS.
962 963 964
 * @hw_queue_id: the id of the H/W queue this job is submitted to.
 * @user_cb_size: the actual size of the CB we got from the user.
 * @job_cb_size: the actual size of the CB that we put on the queue.
T
Tomer Tayar 已提交
965 966 967
 * @is_kernel_allocated_cb: true if the CB handle we got from the user holds a
 *                          handle to a kernel-allocated CB object, false
 *                          otherwise (SRAM/DRAM/host address).
968 969 970 971 972 973
 * @contains_dma_pkt: whether the JOB contains at least one DMA packet. This
 *                    info is needed later, when adding the 2xMSG_PROT at the
 *                    end of the JOB, to know which barriers to put in the
 *                    MSG_PROT packets. Relevant only for GAUDI as GOYA doesn't
 *                    have streams so the engine can't be busy by another
 *                    stream.
O
Oded Gabbay 已提交
974 975
 */
struct hl_cs_job {
976 977 978 979
	struct list_head	cs_node;
	struct hl_cs		*cs;
	struct hl_cb		*user_cb;
	struct hl_cb		*patched_cb;
O
Oded Gabbay 已提交
980
	struct work_struct	finish_work;
981
	struct list_head	userptr_list;
O
Oded Gabbay 已提交
982
	struct list_head	debugfs_list;
T
Tomer Tayar 已提交
983
	enum hl_queue_type	queue_type;
O
Oded Gabbay 已提交
984
	u32			id;
985 986 987
	u32			hw_queue_id;
	u32			user_cb_size;
	u32			job_cb_size;
T
Tomer Tayar 已提交
988
	u8			is_kernel_allocated_cb;
989
	u8			contains_dma_pkt;
990 991 992
};

/**
T
Tomer Tayar 已提交
993
 * struct hl_cs_parser - command submission parser properties.
994 995 996 997 998 999
 * @user_cb: the CB we got from the user.
 * @patched_cb: in case of patching, this is internal CB which is submitted on
 *		the queue instead of the CB we got from the IOCTL.
 * @job_userptr_list: linked-list of userptr mappings that belong to the related
 *			job and wait for completion.
 * @cs_sequence: the sequence number of the related CS.
T
Tomer Tayar 已提交
1000
 * @queue_type: the type of the H/W queue this job is submitted to.
1001 1002 1003 1004 1005
 * @ctx_id: the ID of the context the related CS belongs to.
 * @hw_queue_id: the id of the H/W queue this job is submitted to.
 * @user_cb_size: the actual size of the CB we got from the user.
 * @patched_cb_size: the size of the CB after parsing.
 * @job_id: the id of the related job inside the related CS.
T
Tomer Tayar 已提交
1006 1007 1008
 * @is_kernel_allocated_cb: true if the CB handle we got from the user holds a
 *                          handle to a kernel-allocated CB object, false
 *                          otherwise (SRAM/DRAM/host address).
1009 1010 1011 1012 1013 1014
 * @contains_dma_pkt: whether the JOB contains at least one DMA packet. This
 *                    info is needed later, when adding the 2xMSG_PROT at the
 *                    end of the JOB, to know which barriers to put in the
 *                    MSG_PROT packets. Relevant only for GAUDI as GOYA doesn't
 *                    have streams so the engine can't be busy by another
 *                    stream.
1015 1016 1017 1018 1019 1020
 */
struct hl_cs_parser {
	struct hl_cb		*user_cb;
	struct hl_cb		*patched_cb;
	struct list_head	*job_userptr_list;
	u64			cs_sequence;
T
Tomer Tayar 已提交
1021
	enum hl_queue_type	queue_type;
1022 1023 1024 1025 1026
	u32			ctx_id;
	u32			hw_queue_id;
	u32			user_cb_size;
	u32			patched_cb_size;
	u8			job_id;
T
Tomer Tayar 已提交
1027
	u8			is_kernel_allocated_cb;
1028
	u8			contains_dma_pkt;
O
Oded Gabbay 已提交
1029
};
1030 1031


1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053
/*
 * MEMORY STRUCTURE
 */

/**
 * struct hl_vm_hash_node - hash element from virtual address to virtual
 *				memory area descriptor (hl_vm_phys_pg_list or
 *				hl_userptr).
 * @node: node to hang on the hash table in context object.
 * @vaddr: key virtual address.
 * @ptr: value pointer (hl_vm_phys_pg_list or hl_userptr).
 */
struct hl_vm_hash_node {
	struct hlist_node	node;
	u64			vaddr;
	void			*ptr;
};

/**
 * struct hl_vm_phys_pg_pack - physical page pack.
 * @vm_type: describes the type of the virtual area descriptor.
 * @pages: the physical page array.
1054 1055
 * @npages: num physical pages in the pack.
 * @total_size: total size of all the pages in this list.
1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067
 * @mapping_cnt: number of shared mappings.
 * @asid: the context related to this list.
 * @page_size: size of each page in the pack.
 * @flags: HL_MEM_* flags related to this list.
 * @handle: the provided handle related to this list.
 * @offset: offset from the first page.
 * @contiguous: is contiguous physical memory.
 * @created_from_userptr: is product of host virtual address.
 */
struct hl_vm_phys_pg_pack {
	enum vm_type_t		vm_type; /* must be first */
	u64			*pages;
1068 1069
	u64			npages;
	u64			total_size;
1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110
	atomic_t		mapping_cnt;
	u32			asid;
	u32			page_size;
	u32			flags;
	u32			handle;
	u32			offset;
	u8			contiguous;
	u8			created_from_userptr;
};

/**
 * struct hl_vm_va_block - virtual range block information.
 * @node: node to hang on the virtual range list in context object.
 * @start: virtual range start address.
 * @end: virtual range end address.
 * @size: virtual range size.
 */
struct hl_vm_va_block {
	struct list_head	node;
	u64			start;
	u64			end;
	u64			size;
};

/**
 * struct hl_vm - virtual memory manager for MMU.
 * @dram_pg_pool: pool for DRAM physical pages of 2MB.
 * @dram_pg_pool_refcount: reference counter for the pool usage.
 * @idr_lock: protects the phys_pg_list_handles.
 * @phys_pg_pack_handles: idr to hold all device allocations handles.
 * @init_done: whether initialization was done. We need this because VM
 *		initialization might be skipped during device initialization.
 */
struct hl_vm {
	struct gen_pool		*dram_pg_pool;
	struct kref		dram_pg_pool_refcount;
	spinlock_t		idr_lock;
	struct idr		phys_pg_pack_handles;
	u8			init_done;
};

1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133

/*
 * DEBUG, PROFILING STRUCTURE
 */

/**
 * struct hl_debug_params - Coresight debug parameters.
 * @input: pointer to component specific input parameters.
 * @output: pointer to component specific output parameters.
 * @output_size: size of output buffer.
 * @reg_idx: relevant register ID.
 * @op: component operation to execute.
 * @enable: true if to enable component debugging, false otherwise.
 */
struct hl_debug_params {
	void *input;
	void *output;
	u32 output_size;
	u32 reg_idx;
	u32 op;
	bool enable;
};

O
Oded Gabbay 已提交
1134 1135 1136 1137 1138 1139 1140 1141 1142
/*
 * FILE PRIVATE STRUCTURE
 */

/**
 * struct hl_fpriv - process information stored in FD private data.
 * @hdev: habanalabs device structure.
 * @filp: pointer to the given file structure.
 * @taskpid: current process ID.
1143
 * @ctx: current executing context. TODO: remove for multiple ctx per process
1144
 * @ctx_mgr: context manager to handle multiple context for this FD.
1145
 * @cb_mgr: command buffer manager to handle multiple buffers for this FD.
O
Oded Gabbay 已提交
1146
 * @debugfs_list: list of relevant ASIC debugfs.
1147
 * @dev_node: node in the device list of file private data
O
Oded Gabbay 已提交
1148
 * @refcount: number of related contexts.
1149
 * @restore_phase_mutex: lock for context switch and restore phase.
1150
 * @is_control: true for control device, false otherwise
O
Oded Gabbay 已提交
1151 1152 1153 1154 1155
 */
struct hl_fpriv {
	struct hl_device	*hdev;
	struct file		*filp;
	struct pid		*taskpid;
1156
	struct hl_ctx		*ctx;
1157
	struct hl_ctx_mgr	ctx_mgr;
1158
	struct hl_cb_mgr	cb_mgr;
O
Oded Gabbay 已提交
1159
	struct list_head	debugfs_list;
1160
	struct list_head	dev_node;
O
Oded Gabbay 已提交
1161
	struct kref		refcount;
1162
	struct mutex		restore_phase_mutex;
1163
	u8			is_control;
O
Oded Gabbay 已提交
1164 1165 1166
};


O
Oded Gabbay 已提交
1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
/*
 * DebugFS
 */

/**
 * struct hl_info_list - debugfs file ops.
 * @name: file name.
 * @show: function to output information.
 * @write: function to write to the file.
 */
struct hl_info_list {
	const char	*name;
	int		(*show)(struct seq_file *s, void *data);
	ssize_t		(*write)(struct file *file, const char __user *buf,
				size_t count, loff_t *f_pos);
};

/**
 * struct hl_debugfs_entry - debugfs dentry wrapper.
 * @dent: base debugfs entry structure.
 * @info_ent: dentry realted ops.
 * @dev_entry: ASIC specific debugfs manager.
 */
struct hl_debugfs_entry {
	struct dentry			*dent;
	const struct hl_info_list	*info_ent;
	struct hl_dbg_device_entry	*dev_entry;
};

/**
 * struct hl_dbg_device_entry - ASIC specific debugfs manager.
 * @root: root dentry.
 * @hdev: habanalabs device structure.
 * @entry_arr: array of available hl_debugfs_entry.
 * @file_list: list of available debugfs files.
 * @file_mutex: protects file_list.
 * @cb_list: list of available CBs.
 * @cb_spinlock: protects cb_list.
 * @cs_list: list of available CSs.
 * @cs_spinlock: protects cs_list.
 * @cs_job_list: list of available CB jobs.
 * @cs_job_spinlock: protects cs_job_list.
 * @userptr_list: list of available userptrs (virtual memory chunk descriptor).
 * @userptr_spinlock: protects userptr_list.
 * @ctx_mem_hash_list: list of available contexts with MMU mappings.
 * @ctx_mem_hash_spinlock: protects cb_list.
 * @addr: next address to read/write from/to in read/write32.
 * @mmu_addr: next virtual address to translate to physical address in mmu_show.
 * @mmu_asid: ASID to use while translating in mmu_show.
 * @i2c_bus: generic u8 debugfs file for bus value to use in i2c_data_read.
 * @i2c_bus: generic u8 debugfs file for address value to use in i2c_data_read.
 * @i2c_bus: generic u8 debugfs file for register value to use in i2c_data_read.
 */
struct hl_dbg_device_entry {
	struct dentry			*root;
	struct hl_device		*hdev;
	struct hl_debugfs_entry		*entry_arr;
	struct list_head		file_list;
	struct mutex			file_mutex;
	struct list_head		cb_list;
	spinlock_t			cb_spinlock;
	struct list_head		cs_list;
	spinlock_t			cs_spinlock;
	struct list_head		cs_job_list;
	spinlock_t			cs_job_spinlock;
	struct list_head		userptr_list;
	spinlock_t			userptr_spinlock;
	struct list_head		ctx_mem_hash_list;
	spinlock_t			ctx_mem_hash_spinlock;
	u64				addr;
	u64				mmu_addr;
	u32				mmu_asid;
	u8				i2c_bus;
	u8				i2c_addr;
	u8				i2c_reg;
};


O
Oded Gabbay 已提交
1245 1246 1247 1248 1249
/*
 * DEVICES
 */

/* Theoretical limit only. A single host can only contain up to 4 or 8 PCIe
1250
 * x16 cards. In extreme cases, there are hosts that can accommodate 16 cards.
O
Oded Gabbay 已提交
1251 1252 1253
 */
#define HL_MAX_MINORS	256

O
Oded Gabbay 已提交
1254 1255 1256 1257 1258 1259 1260
/*
 * Registers read & write functions.
 */

u32 hl_rreg(struct hl_device *hdev, u32 reg);
void hl_wreg(struct hl_device *hdev, u32 reg, u32 val);

1261 1262
#define RREG32(reg) hdev->asic_funcs->rreg(hdev, (reg))
#define WREG32(reg, v) hdev->asic_funcs->wreg(hdev, (reg), (v))
O
Oded Gabbay 已提交
1263
#define DREG32(reg) pr_info("REGISTER: " #reg " : 0x%08X\n",	\
1264
			hdev->asic_funcs->rreg(hdev, (reg)))
O
Oded Gabbay 已提交
1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275

#define WREG32_P(reg, val, mask)				\
	do {							\
		u32 tmp_ = RREG32(reg);				\
		tmp_ &= (mask);					\
		tmp_ |= ((val) & ~(mask));			\
		WREG32(reg, tmp_);				\
	} while (0)
#define WREG32_AND(reg, and) WREG32_P(reg, 0, and)
#define WREG32_OR(reg, or) WREG32_P(reg, or, ~(or))

1276 1277 1278 1279 1280 1281 1282 1283 1284 1285
#define RMWREG32(reg, val, mask)				\
	do {							\
		u32 tmp_ = RREG32(reg);				\
		tmp_ &= ~(mask);				\
		tmp_ |= ((val) << __ffs(mask));			\
		WREG32(reg, tmp_);				\
	} while (0)

#define RREG32_MASK(reg, mask) ((RREG32(reg) & mask) >> __ffs(mask))

O
Oded Gabbay 已提交
1286 1287
#define REG_FIELD_SHIFT(reg, field) reg##_##field##_SHIFT
#define REG_FIELD_MASK(reg, field) reg##_##field##_MASK
1288 1289 1290 1291
#define WREG32_FIELD(reg, offset, field, val)	\
	WREG32(mm##reg + offset, (RREG32(mm##reg + offset) & \
				~REG_FIELD_MASK(reg, field)) | \
				(val) << REG_FIELD_SHIFT(reg, field))
O
Oded Gabbay 已提交
1292

O
Oded Gabbay 已提交
1293 1294 1295
/* Timeout should be longer when working with simulator but cap the
 * increased timeout to some maximum
 */
1296 1297
#define hl_poll_timeout(hdev, addr, val, cond, sleep_us, timeout_us) \
({ \
1298 1299 1300 1301
	ktime_t __timeout; \
	if (hdev->pdev) \
		__timeout = ktime_add_us(ktime_get(), timeout_us); \
	else \
O
Oded Gabbay 已提交
1302 1303 1304
		__timeout = ktime_add_us(ktime_get(),\
				min((u64)(timeout_us * 10), \
					(u64) HL_SIM_MAX_TIMEOUT_US)); \
1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319
	might_sleep_if(sleep_us); \
	for (;;) { \
		(val) = RREG32(addr); \
		if (cond) \
			break; \
		if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \
			(val) = RREG32(addr); \
			break; \
		} \
		if (sleep_us) \
			usleep_range((sleep_us >> 2) + 1, sleep_us); \
	} \
	(cond) ? 0 : -ETIMEDOUT; \
})

1320 1321 1322
/*
 * address in this macro points always to a memory location in the
 * host's (server's) memory. That location is updated asynchronously
1323 1324 1325 1326 1327 1328 1329 1330
 * either by the direct access of the device or by another core.
 *
 * To work both in LE and BE architectures, we need to distinguish between the
 * two states (device or another core updates the memory location). Therefore,
 * if mem_written_by_device is true, the host memory being polled will be
 * updated directly by the device. If false, the host memory being polled will
 * be updated by host CPU. Required so host knows whether or not the memory
 * might need to be byte-swapped before returning value to caller.
1331
 */
1332 1333
#define hl_poll_timeout_memory(hdev, addr, val, cond, sleep_us, timeout_us, \
				mem_written_by_device) \
1334 1335 1336 1337 1338
({ \
	ktime_t __timeout; \
	if (hdev->pdev) \
		__timeout = ktime_add_us(ktime_get(), timeout_us); \
	else \
O
Oded Gabbay 已提交
1339 1340 1341
		__timeout = ktime_add_us(ktime_get(),\
				min((u64)(timeout_us * 10), \
					(u64) HL_SIM_MAX_TIMEOUT_US)); \
1342 1343 1344 1345 1346
	might_sleep_if(sleep_us); \
	for (;;) { \
		/* Verify we read updates done by other cores or by device */ \
		mb(); \
		(val) = *((u32 *) (uintptr_t) (addr)); \
1347
		if (mem_written_by_device) \
1348
			(val) = le32_to_cpu(*(__le32 *) &(val)); \
1349 1350 1351 1352
		if (cond) \
			break; \
		if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \
			(val) = *((u32 *) (uintptr_t) (addr)); \
1353
			if (mem_written_by_device) \
1354
				(val) = le32_to_cpu(*(__le32 *) &(val)); \
1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369
			break; \
		} \
		if (sleep_us) \
			usleep_range((sleep_us >> 2) + 1, sleep_us); \
	} \
	(cond) ? 0 : -ETIMEDOUT; \
})

#define hl_poll_timeout_device_memory(hdev, addr, val, cond, sleep_us, \
					timeout_us) \
({ \
	ktime_t __timeout; \
	if (hdev->pdev) \
		__timeout = ktime_add_us(ktime_get(), timeout_us); \
	else \
O
Oded Gabbay 已提交
1370 1371 1372
		__timeout = ktime_add_us(ktime_get(),\
				min((u64)(timeout_us * 10), \
					(u64) HL_SIM_MAX_TIMEOUT_US)); \
1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386
	might_sleep_if(sleep_us); \
	for (;;) { \
		(val) = readl(addr); \
		if (cond) \
			break; \
		if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \
			(val) = readl(addr); \
			break; \
		} \
		if (sleep_us) \
			usleep_range((sleep_us >> 2) + 1, sleep_us); \
	} \
	(cond) ? 0 : -ETIMEDOUT; \
})
1387

1388 1389
struct hwmon_chip_info;

1390 1391 1392 1393 1394 1395 1396 1397 1398 1399
/**
 * struct hl_device_reset_work - reset workqueue task wrapper.
 * @reset_work: reset work to be done.
 * @hdev: habanalabs device structure.
 */
struct hl_device_reset_work {
	struct work_struct		reset_work;
	struct hl_device		*hdev;
};

1400 1401 1402 1403 1404 1405 1406 1407 1408 1409
/**
 * struct hl_device_idle_busy_ts - used for calculating device utilization rate.
 * @idle_to_busy_ts: timestamp where device changed from idle to busy.
 * @busy_to_idle_ts: timestamp where device changed from busy to idle.
 */
struct hl_device_idle_busy_ts {
	ktime_t				idle_to_busy_ts;
	ktime_t				busy_to_idle_ts;
};

O
Oded Gabbay 已提交
1410 1411 1412
/**
 * struct hl_device - habanalabs device structure.
 * @pdev: pointer to PCI device, can be NULL in case of simulator device.
O
Ofir Bitton 已提交
1413 1414 1415
 * @pcie_bar_phys: array of available PCIe bars physical addresses.
 *		   (required only for PCI address match mode)
 * @pcie_bar: array of available PCIe bars virtual addresses.
O
Oded Gabbay 已提交
1416
 * @rmmio: configuration area address on SRAM.
O
Oded Gabbay 已提交
1417
 * @cdev: related char device.
1418 1419 1420
 * @cdev_ctrl: char device for control operations only (INFO IOCTL)
 * @dev: related kernel basic device structure.
 * @dev_ctrl: related kernel device structure for the control device
1421
 * @work_freq: delayed work to lower device frequency if possible.
1422
 * @work_heartbeat: delayed work for ArmCP is-alive check.
O
Oded Gabbay 已提交
1423 1424
 * @asic_name: ASIC specific nmae.
 * @asic_type: ASIC specific type.
O
Oded Gabbay 已提交
1425
 * @completion_queue: array of hl_cq.
1426 1427
 * @cq_wq: work queues of completion queues for executing work in process
 *         context.
O
Oded Gabbay 已提交
1428
 * @eq_wq: work queue of event queue for executing work in process context.
1429
 * @kernel_ctx: Kernel driver context structure.
O
Oded Gabbay 已提交
1430
 * @kernel_queues: array of hl_hw_queue.
1431 1432
 * @hw_queues_mirror_list: CS mirror list for TDR.
 * @hw_queues_mirror_lock: protects hw_queues_mirror_list.
1433
 * @kernel_cb_mgr: command buffer manager for creating/destroying/handling CGs.
1434
 * @event_queue: event queue for IRQ from ArmCP.
O
Oded Gabbay 已提交
1435
 * @dma_pool: DMA pool for small allocations.
1436 1437 1438
 * @cpu_accessible_dma_mem: Host <-> ArmCP shared memory CPU address.
 * @cpu_accessible_dma_address: Host <-> ArmCP shared memory DMA address.
 * @cpu_accessible_dma_pool: Host <-> ArmCP shared memory pool.
1439 1440
 * @asid_bitmap: holds used/available ASIDs.
 * @asid_mutex: protects asid_bitmap.
1441
 * @send_cpu_message_lock: enforces only one message in Host <-> ArmCP queue.
1442
 * @debug_lock: protects critical section of setting debug mode for device
O
Oded Gabbay 已提交
1443 1444 1445
 * @asic_prop: ASIC specific immutable properties.
 * @asic_funcs: ASIC specific functions.
 * @asic_specific: ASIC specific information to use only from ASIC files.
1446 1447
 * @mmu_pgt_pool: pool of available MMU hops.
 * @vm: virtual memory manager for MMU.
1448 1449
 * @mmu_cache_lock: protects MMU cache invalidation as it can serve one context.
 * @mmu_shadow_hop0: shadow mapping of the MMU hop 0 zone.
1450 1451 1452
 * @hwmon_dev: H/W monitor device.
 * @pm_mng_profile: current power management profile.
 * @hl_chip_info: ASIC's sensors information.
O
Oded Gabbay 已提交
1453
 * @hl_debugfs: device's debugfs manager.
1454 1455
 * @cb_pool: list of preallocated CBs.
 * @cb_pool_lock: protects the CB pool.
1456 1457 1458 1459
 * @internal_cb_pool_virt_addr: internal command buffer pool virtual address.
 * @internal_cb_pool_dma_addr: internal command buffer pool dma address.
 * @internal_cb_pool: internal command buffer memory pool.
 * @internal_cb_va_base: internal cb pool mmu virtual address base
1460 1461 1462
 * @fpriv_list: list of file private data structures. Each structure is created
 *              when a user opens the device
 * @fpriv_list_lock: protects the fpriv_list
1463
 * @compute_ctx: current compute context executing.
1464 1465
 * @idle_busy_ts_arr: array to hold time stamps of transitions from idle to busy
 *                    and vice-versa
1466
 * @aggregated_cs_counters: aggregated cs counters among all contexts
1467
 * @dram_used_mem: current DRAM memory consumption.
1468
 * @timeout_jiffies: device CS timeout value.
1469
 * @max_power: the max power of the device, as configured by the sysadmin. This
1470 1471
 *             value is saved so in case of hard-reset, the driver will restore
 *             this value and update the F/W after the re-initialization
1472 1473 1474
 * @clock_gating_mask: is clock gating enabled. bitmask that represents the
 *                     different engines. See debugfs-driver-habanalabs for
 *                     details.
1475
 * @in_reset: is device in reset flow.
1476
 * @curr_pll_profile: current PLL profile.
1477 1478
 * @card_type: Various ASICs have several card types. This indicates the card
 *             type of the current device.
1479 1480
 * @cs_active_cnt: number of active command submissions on this device (active
 *                 means already in H/W queues)
1481
 * @major: habanalabs kernel driver major.
1482
 * @high_pll: high PLL profile frequency.
1483 1484
 * @soft_reset_cnt: number of soft reset since the driver was loaded.
 * @hard_reset_cnt: number of hard reset since the driver was loaded.
1485
 * @idle_busy_ts_idx: index of current entry in idle_busy_ts_arr
O
Oded Gabbay 已提交
1486
 * @id: device minor.
1487
 * @id_control: minor of the control device
1488 1489
 * @cpu_pci_msb_addr: 50-bit extension bits for the device CPU's 40-bit
 *                    addresses.
O
Oded Gabbay 已提交
1490
 * @disabled: is device disabled.
1491 1492
 * @late_init_done: is late init stage was done during initialization.
 * @hwmon_initialized: is H/W monitor sensors was initialized.
1493 1494
 * @hard_reset_pending: is there a hard reset work pending.
 * @heartbeat: is heartbeat sanity check towards ArmCP enabled.
1495 1496
 * @reset_on_lockup: true if a reset should be done in case of stuck CS, false
 *                   otherwise.
1497
 * @dram_supports_virtual_memory: is MMU enabled towards DRAM.
1498
 * @dram_default_page_mapping: is DRAM default page mapping enabled.
1499 1500
 * @pmmu_huge_range: is a different virtual addresses range used for PMMU with
 *                   huge pages.
1501
 * @init_done: is the initialization of the device done.
1502
 * @mmu_enable: is MMU enabled.
1503
 * @mmu_huge_page_opt: is MMU huge pages optimization enabled.
1504
 * @device_cpu_disabled: is the device CPU disabled (due to timeouts)
1505
 * @dma_mask: the dma mask that was set for this device
1506
 * @in_debug: is device under debug. This, together with fpriv_list, enforces
1507
 *            that only a single user is configuring the debug infrastructure.
1508 1509
 * @power9_64bit_dma_enable: true to enable 64-bit DMA mask support. Relevant
 *                           only to POWER9 machines.
1510
 * @cdev_sysfs_created: were char devices and sysfs nodes created.
1511
 * @stop_on_err: true if engines should stop on error.
1512
 * @supports_sync_stream: is sync stream supported.
1513
 * @sync_stream_queue_idx: helper index for sync stream queues initialization.
1514
 * @supports_coresight: is CoreSight supported.
1515
 * @supports_soft_reset: is soft reset supported.
O
Oded Gabbay 已提交
1516 1517 1518
 */
struct hl_device {
	struct pci_dev			*pdev;
O
Ofir Bitton 已提交
1519 1520
	u64				pcie_bar_phys[HL_PCI_NUM_BARS];
	void __iomem			*pcie_bar[HL_PCI_NUM_BARS];
O
Oded Gabbay 已提交
1521
	void __iomem			*rmmio;
O
Oded Gabbay 已提交
1522
	struct cdev			cdev;
1523
	struct cdev			cdev_ctrl;
O
Oded Gabbay 已提交
1524
	struct device			*dev;
1525
	struct device			*dev_ctrl;
1526
	struct delayed_work		work_freq;
1527
	struct delayed_work		work_heartbeat;
1528
	char				asic_name[32];
O
Oded Gabbay 已提交
1529
	enum hl_asic_type		asic_type;
O
Oded Gabbay 已提交
1530
	struct hl_cq			*completion_queue;
1531
	struct workqueue_struct		**cq_wq;
1532
	struct workqueue_struct		*eq_wq;
1533
	struct hl_ctx			*kernel_ctx;
O
Oded Gabbay 已提交
1534
	struct hl_hw_queue		*kernel_queues;
1535 1536
	struct list_head		hw_queues_mirror_list;
	spinlock_t			hw_queues_mirror_lock;
1537
	struct hl_cb_mgr		kernel_cb_mgr;
1538
	struct hl_eq			event_queue;
O
Oded Gabbay 已提交
1539 1540 1541 1542
	struct dma_pool			*dma_pool;
	void				*cpu_accessible_dma_mem;
	dma_addr_t			cpu_accessible_dma_address;
	struct gen_pool			*cpu_accessible_dma_pool;
1543 1544
	unsigned long			*asid_bitmap;
	struct mutex			asid_mutex;
O
Oded Gabbay 已提交
1545
	struct mutex			send_cpu_message_lock;
1546
	struct mutex			debug_lock;
O
Oded Gabbay 已提交
1547 1548 1549
	struct asic_fixed_properties	asic_prop;
	const struct hl_asic_funcs	*asic_funcs;
	void				*asic_specific;
1550 1551 1552
	struct gen_pool			*mmu_pgt_pool;
	struct hl_vm			vm;
	struct mutex			mmu_cache_lock;
1553
	void				*mmu_shadow_hop0;
1554 1555 1556
	struct device			*hwmon_dev;
	enum hl_pm_mng_profile		pm_mng_profile;
	struct hwmon_chip_info		*hl_chip_info;
1557

O
Oded Gabbay 已提交
1558 1559
	struct hl_dbg_device_entry	hl_debugfs;

1560 1561 1562
	struct list_head		cb_pool;
	spinlock_t			cb_pool_lock;

1563 1564 1565 1566 1567
	void				*internal_cb_pool_virt_addr;
	dma_addr_t			internal_cb_pool_dma_addr;
	struct gen_pool			*internal_cb_pool;
	u64				internal_cb_va_base;

1568 1569 1570
	struct list_head		fpriv_list;
	struct mutex			fpriv_list_lock;

1571
	struct hl_ctx			*compute_ctx;
1572

1573 1574
	struct hl_device_idle_busy_ts	*idle_busy_ts_arr;

1575 1576
	struct hl_cs_counters		aggregated_cs_counters;

1577
	atomic64_t			dram_used_mem;
1578 1579
	u64				timeout_jiffies;
	u64				max_power;
1580
	u64				clock_gating_mask;
1581
	atomic_t			in_reset;
1582
	enum hl_pll_frequency		curr_pll_profile;
1583
	enum armcp_card_types		card_type;
1584
	int				cs_active_cnt;
O
Oded Gabbay 已提交
1585
	u32				major;
1586
	u32				high_pll;
1587 1588
	u32				soft_reset_cnt;
	u32				hard_reset_cnt;
1589
	u32				idle_busy_ts_idx;
O
Oded Gabbay 已提交
1590
	u16				id;
1591
	u16				id_control;
1592
	u16				cpu_pci_msb_addr;
O
Oded Gabbay 已提交
1593
	u8				disabled;
1594 1595
	u8				late_init_done;
	u8				hwmon_initialized;
1596 1597
	u8				hard_reset_pending;
	u8				heartbeat;
1598
	u8				reset_on_lockup;
1599
	u8				dram_supports_virtual_memory;
1600
	u8				dram_default_page_mapping;
1601
	u8				pmmu_huge_range;
1602
	u8				init_done;
1603
	u8				device_cpu_disabled;
1604
	u8				dma_mask;
1605
	u8				in_debug;
1606
	u8				power9_64bit_dma_enable;
1607
	u8				cdev_sysfs_created;
1608
	u8				stop_on_err;
1609
	u8				supports_sync_stream;
1610
	u8				sync_stream_queue_idx;
1611
	u8				supports_coresight;
1612
	u8				supports_soft_reset;
O
Oded Gabbay 已提交
1613 1614

	/* Parameters for bring-up */
1615
	u8				mmu_enable;
1616
	u8				mmu_huge_page_opt;
1617
	u8				cpu_enable;
O
Oded Gabbay 已提交
1618
	u8				reset_pcilink;
O
Oded Gabbay 已提交
1619
	u8				cpu_queues_enable;
1620 1621
	u8				fw_loading;
	u8				pldm;
1622 1623 1624 1625 1626 1627
	u8				axi_drain;
	u8				sram_scrambler_enable;
	u8				dram_scrambler_enable;
	u8				hard_reset_on_fw_events;
	u8				bmc_enable;
	u8				rl_enable;
O
Oded Gabbay 已提交
1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659
};


/*
 * IOCTLs
 */

/**
 * typedef hl_ioctl_t - typedef for ioctl function in the driver
 * @hpriv: pointer to the FD's private data, which contains state of
 *		user process
 * @data: pointer to the input/output arguments structure of the IOCTL
 *
 * Return: 0 for success, negative value for error
 */
typedef int hl_ioctl_t(struct hl_fpriv *hpriv, void *data);

/**
 * struct hl_ioctl_desc - describes an IOCTL entry of the driver.
 * @cmd: the IOCTL code as created by the kernel macros.
 * @func: pointer to the driver's function that should be called for this IOCTL.
 */
struct hl_ioctl_desc {
	unsigned int cmd;
	hl_ioctl_t *func;
};


/*
 * Kernel module functions that can be accessed by entire module
 */

1660 1661 1662 1663 1664 1665 1666 1667 1668
/**
 * hl_mem_area_inside_range() - Checks whether address+size are inside a range.
 * @address: The start address of the area we want to validate.
 * @size: The size in bytes of the area we want to validate.
 * @range_start_address: The start address of the valid range.
 * @range_end_address: The end address of the valid range.
 *
 * Return: true if the area is inside the valid range, false otherwise.
 */
1669
static inline bool hl_mem_area_inside_range(u64 address, u64 size,
1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711
				u64 range_start_address, u64 range_end_address)
{
	u64 end_address = address + size;

	if ((address >= range_start_address) &&
			(end_address <= range_end_address) &&
			(end_address > address))
		return true;

	return false;
}

/**
 * hl_mem_area_crosses_range() - Checks whether address+size crossing a range.
 * @address: The start address of the area we want to validate.
 * @size: The size in bytes of the area we want to validate.
 * @range_start_address: The start address of the valid range.
 * @range_end_address: The end address of the valid range.
 *
 * Return: true if the area overlaps part or all of the valid range,
 *		false otherwise.
 */
static inline bool hl_mem_area_crosses_range(u64 address, u32 size,
				u64 range_start_address, u64 range_end_address)
{
	u64 end_address = address + size;

	if ((address >= range_start_address) &&
			(address < range_end_address))
		return true;

	if ((end_address >= range_start_address) &&
			(end_address < range_end_address))
		return true;

	if ((address < range_start_address) &&
			(end_address >= range_end_address))
		return true;

	return false;
}

O
Oded Gabbay 已提交
1712
int hl_device_open(struct inode *inode, struct file *filp);
1713
int hl_device_open_ctrl(struct inode *inode, struct file *filp);
1714
bool hl_device_disabled_or_in_reset(struct hl_device *hdev);
1715
enum hl_device_status hl_device_status(struct hl_device *hdev);
1716
int hl_device_set_debug_mode(struct hl_device *hdev, bool enable);
O
Oded Gabbay 已提交
1717 1718 1719
int create_hdev(struct hl_device **dev, struct pci_dev *pdev,
		enum hl_asic_type asic_type, int minor);
void destroy_hdev(struct hl_device *hdev);
O
Oded Gabbay 已提交
1720 1721 1722 1723
int hl_hw_queues_create(struct hl_device *hdev);
void hl_hw_queues_destroy(struct hl_device *hdev);
int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
				u32 cb_size, u64 cb_ptr);
1724
int hl_hw_queue_schedule_cs(struct hl_cs *cs);
O
Oded Gabbay 已提交
1725 1726
u32 hl_hw_queue_add_ptr(u32 ptr, u16 val);
void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id);
1727
void hl_int_hw_queue_update_ci(struct hl_cs *cs);
1728
void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset);
O
Oded Gabbay 已提交
1729 1730 1731 1732 1733 1734

#define hl_queue_inc_ptr(p)		hl_hw_queue_add_ptr(p, 1)
#define hl_pi_2_offset(pi)		((pi) & (HL_QUEUE_LENGTH - 1))

int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id);
void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q);
1735 1736
int hl_eq_init(struct hl_device *hdev, struct hl_eq *q);
void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q);
1737 1738
void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q);
void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q);
1739 1740
irqreturn_t hl_irq_handler_cq(int irq, void *arg);
irqreturn_t hl_irq_handler_eq(int irq, void *arg);
1741 1742
u32 hl_cq_inc_ptr(u32 ptr);

1743 1744 1745 1746 1747 1748 1749 1750
int hl_asid_init(struct hl_device *hdev);
void hl_asid_fini(struct hl_device *hdev);
unsigned long hl_asid_alloc(struct hl_device *hdev);
void hl_asid_free(struct hl_device *hdev, unsigned long asid);

int hl_ctx_create(struct hl_device *hdev, struct hl_fpriv *hpriv);
void hl_ctx_free(struct hl_device *hdev, struct hl_ctx *ctx);
int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx);
1751 1752
void hl_ctx_do_release(struct kref *ref);
void hl_ctx_get(struct hl_device *hdev,	struct hl_ctx *ctx);
1753
int hl_ctx_put(struct hl_ctx *ctx);
1754
struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq);
1755 1756
void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr);
void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr);
1757

O
Oded Gabbay 已提交
1758 1759 1760 1761
int hl_device_init(struct hl_device *hdev, struct class *hclass);
void hl_device_fini(struct hl_device *hdev);
int hl_device_suspend(struct hl_device *hdev);
int hl_device_resume(struct hl_device *hdev);
1762 1763
int hl_device_reset(struct hl_device *hdev, bool hard_reset,
			bool from_hard_reset_thread);
1764 1765
void hl_hpriv_get(struct hl_fpriv *hpriv);
void hl_hpriv_put(struct hl_fpriv *hpriv);
1766
int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq);
1767
uint32_t hl_device_utilization(struct hl_device *hdev, uint32_t period_ms);
1768

1769 1770 1771 1772 1773 1774 1775 1776
int hl_build_hwmon_channel_info(struct hl_device *hdev,
		struct armcp_sensor *sensors_arr);

int hl_sysfs_init(struct hl_device *hdev);
void hl_sysfs_fini(struct hl_device *hdev);

int hl_hwmon_init(struct hl_device *hdev);
void hl_hwmon_fini(struct hl_device *hdev);
O
Oded Gabbay 已提交
1777

1778
int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr, u32 cb_size,
1779
		u64 *handle, int ctx_id, bool internal_cb);
1780 1781 1782 1783 1784 1785 1786
int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle);
int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma);
struct hl_cb *hl_cb_get(struct hl_device *hdev,	struct hl_cb_mgr *mgr,
			u32 handle);
void hl_cb_put(struct hl_cb *cb);
void hl_cb_mgr_init(struct hl_cb_mgr *mgr);
void hl_cb_mgr_fini(struct hl_device *hdev, struct hl_cb_mgr *mgr);
1787 1788
struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size,
					bool internal_cb);
1789 1790 1791
int hl_cb_pool_init(struct hl_device *hdev);
int hl_cb_pool_fini(struct hl_device *hdev);

1792
void hl_cs_rollback_all(struct hl_device *hdev);
T
Tomer Tayar 已提交
1793 1794
struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev,
		enum hl_queue_type queue_type, bool is_kernel_allocated_cb);
1795
void hl_sob_reset_error(struct kref *ref);
1796 1797
void hl_fence_put(struct hl_fence *fence);
void hl_fence_get(struct hl_fence *fence);
1798

O
Oded Gabbay 已提交
1799
void goya_set_asic_funcs(struct hl_device *hdev);
1800
void gaudi_set_asic_funcs(struct hl_device *hdev);
O
Oded Gabbay 已提交
1801

1802 1803 1804 1805 1806 1807
int hl_vm_ctx_init(struct hl_ctx *ctx);
void hl_vm_ctx_fini(struct hl_ctx *ctx);

int hl_vm_init(struct hl_device *hdev);
void hl_vm_fini(struct hl_device *hdev);

1808
int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size,
1809
			struct hl_userptr *userptr);
1810
void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr);
1811 1812 1813 1814 1815 1816
void hl_userptr_delete_list(struct hl_device *hdev,
				struct list_head *userptr_list);
bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr, u32 size,
				struct list_head *userptr_list,
				struct hl_userptr **userptr);

1817 1818
int hl_mmu_init(struct hl_device *hdev);
void hl_mmu_fini(struct hl_device *hdev);
1819
int hl_mmu_ctx_init(struct hl_ctx *ctx);
1820
void hl_mmu_ctx_fini(struct hl_ctx *ctx);
1821 1822 1823 1824
int hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,
		u32 page_size, bool flush_pte);
int hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr, u32 page_size,
		bool flush_pte);
1825 1826 1827
void hl_mmu_swap_out(struct hl_ctx *ctx);
void hl_mmu_swap_in(struct hl_ctx *ctx);

1828
int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
1829 1830 1831 1832
				void __iomem *dst);
int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode);
int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
				u16 len, u32 timeout, long *result);
1833 1834 1835
int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type);
int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
		size_t irq_arr_size);
1836 1837 1838 1839 1840 1841 1842 1843
int hl_fw_test_cpu_queue(struct hl_device *hdev);
void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
						dma_addr_t *dma_handle);
void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
					void *vaddr);
int hl_fw_send_heartbeat(struct hl_device *hdev);
int hl_fw_armcp_info_get(struct hl_device *hdev);
int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size);
1844
int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
1845 1846 1847
			u32 msg_to_cpu_reg, u32 cpu_msg_status_reg,
			u32 boot_err0_reg, bool skip_bmc,
			u32 cpu_timeout, u32 boot_fit_timeout);
1848

1849 1850 1851 1852 1853
int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3],
			bool is_wc[3]);
int hl_pci_iatu_write(struct hl_device *hdev, u32 addr, u32 data);
int hl_pci_set_dram_bar_base(struct hl_device *hdev, u8 inbound_region, u8 bar,
				u64 addr);
O
Ofir Bitton 已提交
1854 1855 1856 1857
int hl_pci_set_inbound_region(struct hl_device *hdev, u8 region,
		struct hl_inbound_pci_region *pci_region);
int hl_pci_set_outbound_region(struct hl_device *hdev,
		struct hl_outbound_pci_region *pci_region);
1858
int hl_pci_init(struct hl_device *hdev);
1859 1860
void hl_pci_fini(struct hl_device *hdev);

1861 1862
long hl_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr);
void hl_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq);
1863 1864
int hl_get_temperature(struct hl_device *hdev,
		       int sensor_index, u32 attr, long *value);
1865
int hl_set_temperature(struct hl_device *hdev,
1866 1867 1868 1869 1870 1871 1872 1873 1874
		       int sensor_index, u32 attr, long value);
int hl_get_voltage(struct hl_device *hdev,
		   int sensor_index, u32 attr, long *value);
int hl_get_current(struct hl_device *hdev,
		   int sensor_index, u32 attr, long *value);
int hl_get_fan_speed(struct hl_device *hdev,
		     int sensor_index, u32 attr, long *value);
int hl_get_pwm_info(struct hl_device *hdev,
		    int sensor_index, u32 attr, long *value);
1875 1876 1877
void hl_set_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr,
			long value);
u64 hl_get_max_power(struct hl_device *hdev);
1878
void hl_set_max_power(struct hl_device *hdev);
1879 1880 1881 1882
int hl_set_voltage(struct hl_device *hdev,
			int sensor_index, u32 attr, long value);
int hl_set_current(struct hl_device *hdev,
			int sensor_index, u32 attr, long value);
1883

O
Oded Gabbay 已提交
1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977
#ifdef CONFIG_DEBUG_FS

void hl_debugfs_init(void);
void hl_debugfs_fini(void);
void hl_debugfs_add_device(struct hl_device *hdev);
void hl_debugfs_remove_device(struct hl_device *hdev);
void hl_debugfs_add_file(struct hl_fpriv *hpriv);
void hl_debugfs_remove_file(struct hl_fpriv *hpriv);
void hl_debugfs_add_cb(struct hl_cb *cb);
void hl_debugfs_remove_cb(struct hl_cb *cb);
void hl_debugfs_add_cs(struct hl_cs *cs);
void hl_debugfs_remove_cs(struct hl_cs *cs);
void hl_debugfs_add_job(struct hl_device *hdev, struct hl_cs_job *job);
void hl_debugfs_remove_job(struct hl_device *hdev, struct hl_cs_job *job);
void hl_debugfs_add_userptr(struct hl_device *hdev, struct hl_userptr *userptr);
void hl_debugfs_remove_userptr(struct hl_device *hdev,
				struct hl_userptr *userptr);
void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx);
void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx);

#else

static inline void __init hl_debugfs_init(void)
{
}

static inline void hl_debugfs_fini(void)
{
}

static inline void hl_debugfs_add_device(struct hl_device *hdev)
{
}

static inline void hl_debugfs_remove_device(struct hl_device *hdev)
{
}

static inline void hl_debugfs_add_file(struct hl_fpriv *hpriv)
{
}

static inline void hl_debugfs_remove_file(struct hl_fpriv *hpriv)
{
}

static inline void hl_debugfs_add_cb(struct hl_cb *cb)
{
}

static inline void hl_debugfs_remove_cb(struct hl_cb *cb)
{
}

static inline void hl_debugfs_add_cs(struct hl_cs *cs)
{
}

static inline void hl_debugfs_remove_cs(struct hl_cs *cs)
{
}

static inline void hl_debugfs_add_job(struct hl_device *hdev,
					struct hl_cs_job *job)
{
}

static inline void hl_debugfs_remove_job(struct hl_device *hdev,
					struct hl_cs_job *job)
{
}

static inline void hl_debugfs_add_userptr(struct hl_device *hdev,
					struct hl_userptr *userptr)
{
}

static inline void hl_debugfs_remove_userptr(struct hl_device *hdev,
					struct hl_userptr *userptr)
{
}

static inline void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev,
					struct hl_ctx *ctx)
{
}

static inline void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev,
					struct hl_ctx *ctx)
{
}

#endif

1978 1979
/* IOCTLs */
long hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg);
1980
long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg);
1981
int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data);
1982 1983
int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data);
int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data);
1984
int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data);
1985

O
Oded Gabbay 已提交
1986
#endif /* HABANALABSP_H_ */