habanalabs.h 57.9 KB
Newer Older
O
Oded Gabbay 已提交
1 2 3 4 5 6 7 8 9 10
/* SPDX-License-Identifier: GPL-2.0
 *
 * Copyright 2016-2019 HabanaLabs, Ltd.
 * All Rights Reserved.
 *
 */

#ifndef HABANALABSP_H_
#define HABANALABSP_H_

11
#include "include/armcp_if.h"
O
Oded Gabbay 已提交
12
#include "include/qman_if.h"
13

O
Oded Gabbay 已提交
14
#include <linux/cdev.h>
15
#include <linux/iopoll.h>
16
#include <linux/irqreturn.h>
17 18 19
#include <linux/dma-fence.h>
#include <linux/dma-direction.h>
#include <linux/scatterlist.h>
20
#include <linux/hashtable.h>
O
Oded Gabbay 已提交
21 22 23

#define HL_NAME				"habanalabs"

24 25
#define HL_MMAP_CB_MASK			(0x8000000000000000ull >> PAGE_SHIFT)

26 27
#define HL_PENDING_RESET_PER_SEC	5

28 29
#define HL_DEVICE_TIMEOUT_USEC		1000000 /* 1 s */

30 31
#define HL_HEARTBEAT_PER_USEC		5000000 /* 5 s */

32 33
#define HL_PLL_LOW_JOB_FREQ_USEC	5000000 /* 5 s */

34 35 36
#define HL_ARMCP_INFO_TIMEOUT_USEC	10000000 /* 10s */
#define HL_ARMCP_EEPROM_TIMEOUT_USEC	10000000 /* 10s */

37 38
#define HL_PCI_ELBI_TIMEOUT_MSEC	10 /* 10ms */

O
Oded Gabbay 已提交
39 40
#define HL_SIM_MAX_TIMEOUT_US		10000000 /* 10s */

O
Oded Gabbay 已提交
41 42
#define HL_MAX_QUEUES			128

43 44 45 46 47
#define HL_MAX_JOBS_PER_CS		64

/* MUST BE POWER OF 2 and larger than 1 */
#define HL_MAX_PENDING_CS		64

48 49
#define HL_IDLE_BUSY_TS_ARR_SIZE	4096

50 51 52 53 54 55 56 57
/* Memory */
#define MEM_HASH_TABLE_BITS		7 /* 1 << 7 buckets */

/* MMU */
#define MMU_HASH_TABLE_BITS		7 /* 1 << 7 buckets */

/**
 * struct pgt_info - MMU hop page info.
58 59 60
 * @node: hash linked-list node for the pgts shadow hash of pgts.
 * @phys_addr: physical address of the pgt.
 * @shadow_addr: shadow hop in the host.
61 62 63 64 65 66 67 68 69
 * @ctx: pointer to the owner ctx.
 * @num_of_ptes: indicates how many ptes are used in the pgt.
 *
 * The MMU page tables hierarchy is placed on the DRAM. When a new level (hop)
 * is needed during mapping, a new page is allocated and this structure holds
 * its essential information. During unmapping, if no valid PTEs remained in the
 * page, it is freed with its pgt_info structure.
 */
struct pgt_info {
70 71 72 73 74
	struct hlist_node	node;
	u64			phys_addr;
	u64			shadow_addr;
	struct hl_ctx		*ctx;
	int			num_of_ptes;
75 76
};

O
Oded Gabbay 已提交
77
struct hl_device;
78
struct hl_fpriv;
O
Oded Gabbay 已提交
79

O
Oded Gabbay 已提交
80 81 82 83 84 85 86 87
/**
 * enum hl_queue_type - Supported QUEUE types.
 * @QUEUE_TYPE_NA: queue is not available.
 * @QUEUE_TYPE_EXT: external queue which is a DMA channel that may access the
 *                  host.
 * @QUEUE_TYPE_INT: internal queue that performs DMA inside the device's
 *			memories and/or operates the compute engines.
 * @QUEUE_TYPE_CPU: S/W queue for communication with the device's CPU.
T
Tomer Tayar 已提交
88 89
 * @QUEUE_TYPE_HW: queue of DMA and compute engines jobs, for which completion
 *                 notifications are sent by H/W.
O
Oded Gabbay 已提交
90 91 92 93 94
 */
enum hl_queue_type {
	QUEUE_TYPE_NA,
	QUEUE_TYPE_EXT,
	QUEUE_TYPE_INT,
T
Tomer Tayar 已提交
95 96
	QUEUE_TYPE_CPU,
	QUEUE_TYPE_HW
O
Oded Gabbay 已提交
97 98 99 100 101
};

/**
 * struct hw_queue_properties - queue information.
 * @type: queue type.
102 103
 * @driver_only: true if only the driver is allowed to send a job to this queue,
 *               false otherwise.
104 105
 * @requires_kernel_cb: true if a CB handle must be provided for jobs on this
 *                      queue, false otherwise (a CB address must be provided).
O
Oded Gabbay 已提交
106 107 108
 */
struct hw_queue_properties {
	enum hl_queue_type	type;
109
	u8			driver_only;
110
	u8			requires_kernel_cb;
O
Oded Gabbay 已提交
111
};
O
Oded Gabbay 已提交
112

113 114 115
/**
 * enum vm_type_t - virtual memory mapping request information.
 * @VM_TYPE_USERPTR: mapping of user memory to device virtual address.
116
 * @VM_TYPE_PHYS_PACK: mapping of DRAM memory to device virtual address.
117 118 119
 */
enum vm_type_t {
	VM_TYPE_USERPTR,
120
	VM_TYPE_PHYS_PACK
121 122
};

123 124 125 126 127 128 129 130 131 132 133 134
/**
 * enum hl_device_hw_state - H/W device state. use this to understand whether
 *                           to do reset before hw_init or not
 * @HL_DEVICE_HW_STATE_CLEAN: H/W state is clean. i.e. after hard reset
 * @HL_DEVICE_HW_STATE_DIRTY: H/W state is dirty. i.e. we started to execute
 *                            hw_init
 */
enum hl_device_hw_state {
	HL_DEVICE_HW_STATE_CLEAN = 0,
	HL_DEVICE_HW_STATE_DIRTY
};

O
Oded Gabbay 已提交
135 136
/**
 * struct asic_fixed_properties - ASIC specific immutable properties.
O
Oded Gabbay 已提交
137
 * @hw_queues_props: H/W queues properties.
138
 * @armcp_info: received various information from ArmCP regarding the H/W, e.g.
139
 *		available sensors.
140 141
 * @uboot_ver: F/W U-boot version.
 * @preboot_ver: F/W Preboot version.
O
Oded Gabbay 已提交
142 143 144 145 146 147 148 149
 * @sram_base_address: SRAM physical start address.
 * @sram_end_address: SRAM physical end address.
 * @sram_user_base_address - SRAM physical start address for user access.
 * @dram_base_address: DRAM physical start address.
 * @dram_end_address: DRAM physical end address.
 * @dram_user_base_address: DRAM physical start address for user access.
 * @dram_size: DRAM total size.
 * @dram_pci_bar_size: size of PCI bar towards DRAM.
150
 * @max_power_default: max power of the device after reset
O
Oded Gabbay 已提交
151 152 153 154 155 156 157 158
 * @va_space_host_start_address: base address of virtual memory range for
 *                               mapping host memory.
 * @va_space_host_end_address: end address of virtual memory range for
 *                             mapping host memory.
 * @va_space_dram_start_address: base address of virtual memory range for
 *                               mapping DRAM memory.
 * @va_space_dram_end_address: end address of virtual memory range for
 *                             mapping DRAM memory.
159 160
 * @dram_size_for_default_page_mapping: DRAM size needed to map to avoid page
 *                                      fault.
161 162
 * @pcie_dbi_base_address: Base address of the PCIE_DBI block.
 * @pcie_aux_dbi_reg_addr: Address of the PCIE_AUX DBI register.
163
 * @mmu_pgt_addr: base physical address in DRAM of MMU page tables.
164
 * @mmu_dram_default_page_addr: DRAM default page physical address.
165 166 167 168 169
 * @mmu_pgt_size: MMU page tables total size.
 * @mmu_pte_size: PTE size in MMU page tables.
 * @mmu_hop_table_size: MMU hop table size.
 * @mmu_hop0_tables_total_size: total size of MMU hop0 tables.
 * @dram_page_size: page size for MMU DRAM allocation.
O
Oded Gabbay 已提交
170 171 172
 * @cfg_size: configuration space size on SRAM.
 * @sram_size: total size of SRAM.
 * @max_asid: maximum number of open contexts (ASIDs).
173
 * @num_of_events: number of possible internal H/W IRQs.
174 175 176 177
 * @psoc_pci_pll_nr: PCI PLL NR value.
 * @psoc_pci_pll_nf: PCI PLL NF value.
 * @psoc_pci_pll_od: PCI PLL OD value.
 * @psoc_pci_pll_div_factor: PCI PLL DIV FACTOR 1 value.
O
Oded Gabbay 已提交
178 179
 * @completion_queues_count: number of completion queues.
 * @high_pll: high PLL frequency used by the device.
180 181
 * @cb_pool_cb_cnt: number of CBs in the CB pool.
 * @cb_pool_cb_size: size of each CB in the CB pool.
O
Oded Gabbay 已提交
182 183 184
 * @tpc_enabled_mask: which TPCs are enabled.
 */
struct asic_fixed_properties {
O
Oded Gabbay 已提交
185
	struct hw_queue_properties	hw_queues_props[HL_MAX_QUEUES];
186
	struct armcp_info	armcp_info;
187 188
	char			uboot_ver[VERSION_MAX_LEN];
	char			preboot_ver[VERSION_MAX_LEN];
O
Oded Gabbay 已提交
189 190 191 192 193 194 195 196
	u64			sram_base_address;
	u64			sram_end_address;
	u64			sram_user_base_address;
	u64			dram_base_address;
	u64			dram_end_address;
	u64			dram_user_base_address;
	u64			dram_size;
	u64			dram_pci_bar_size;
197
	u64			max_power_default;
O
Oded Gabbay 已提交
198 199 200 201
	u64			va_space_host_start_address;
	u64			va_space_host_end_address;
	u64			va_space_dram_start_address;
	u64			va_space_dram_end_address;
202
	u64			dram_size_for_default_page_mapping;
203 204
	u64			pcie_dbi_base_address;
	u64			pcie_aux_dbi_reg_addr;
205
	u64			mmu_pgt_addr;
206
	u64			mmu_dram_default_page_addr;
207 208 209 210 211
	u32			mmu_pgt_size;
	u32			mmu_pte_size;
	u32			mmu_hop_table_size;
	u32			mmu_hop0_tables_total_size;
	u32			dram_page_size;
O
Oded Gabbay 已提交
212 213 214
	u32			cfg_size;
	u32			sram_size;
	u32			max_asid;
215
	u32			num_of_events;
216 217 218 219
	u32			psoc_pci_pll_nr;
	u32			psoc_pci_pll_nf;
	u32			psoc_pci_pll_od;
	u32			psoc_pci_pll_div_factor;
O
Oded Gabbay 已提交
220
	u32			high_pll;
221 222
	u32			cb_pool_cb_cnt;
	u32			cb_pool_cb_size;
O
Oded Gabbay 已提交
223 224 225 226
	u8			completion_queues_count;
	u8			tpc_enabled_mask;
};

227 228 229 230 231 232 233 234 235 236 237 238 239
/**
 * struct hl_dma_fence - wrapper for fence object used by command submissions.
 * @base_fence: kernel fence object.
 * @lock: spinlock to protect fence.
 * @hdev: habanalabs device structure.
 * @cs_seq: command submission sequence number.
 */
struct hl_dma_fence {
	struct dma_fence	base_fence;
	spinlock_t		lock;
	struct hl_device	*hdev;
	u64			cs_seq;
};
O
Oded Gabbay 已提交
240

241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261
/*
 * Command Buffers
 */

#define HL_MAX_CB_SIZE		0x200000	/* 2MB */

/**
 * struct hl_cb_mgr - describes a Command Buffer Manager.
 * @cb_lock: protects cb_handles.
 * @cb_handles: an idr to hold all command buffer handles.
 */
struct hl_cb_mgr {
	spinlock_t		cb_lock;
	struct idr		cb_handles; /* protected by cb_lock */
};

/**
 * struct hl_cb - describes a Command Buffer.
 * @refcount: reference counter for usage of the CB.
 * @hdev: pointer to device this CB belongs to.
 * @lock: spinlock to protect mmap/cs flows.
O
Oded Gabbay 已提交
262
 * @debugfs_list: node in debugfs list of command buffers.
263 264 265 266 267 268
 * @pool_list: node in pool list of command buffers.
 * @kernel_address: Holds the CB's kernel virtual address.
 * @bus_address: Holds the CB's DMA address.
 * @mmap_size: Holds the CB's size that was mmaped.
 * @size: holds the CB's size.
 * @id: the CB's ID.
269
 * @cs_cnt: holds number of CS that this CB participates in.
270 271 272 273 274 275 276 277
 * @ctx_id: holds the ID of the owner's context.
 * @mmap: true if the CB is currently mmaped to user.
 * @is_pool: true if CB was acquired from the pool, false otherwise.
 */
struct hl_cb {
	struct kref		refcount;
	struct hl_device	*hdev;
	spinlock_t		lock;
O
Oded Gabbay 已提交
278
	struct list_head	debugfs_list;
279 280 281 282 283 284
	struct list_head	pool_list;
	u64			kernel_address;
	dma_addr_t		bus_address;
	u32			mmap_size;
	u32			size;
	u32			id;
285
	u32			cs_cnt;
286 287 288 289 290 291
	u32			ctx_id;
	u8			mmap;
	u8			is_pool;
};


O
Oded Gabbay 已提交
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318
/*
 * QUEUES
 */

struct hl_cs_job;

/*
 * Currently, there are two limitations on the maximum length of a queue:
 *
 * 1. The memory footprint of the queue. The current allocated space for the
 *    queue is PAGE_SIZE. Because each entry in the queue is HL_BD_SIZE,
 *    the maximum length of the queue can be PAGE_SIZE / HL_BD_SIZE,
 *    which currently is 4096/16 = 256 entries.
 *
 *    To increase that, we need either to decrease the size of the
 *    BD (difficult), or allocate more than a single page (easier).
 *
 * 2. Because the size of the JOB handle field in the BD CTL / completion queue
 *    is 10-bit, we can have up to 1024 open jobs per hardware queue.
 *    Therefore, each queue can hold up to 1024 entries.
 *
 * HL_QUEUE_LENGTH is in units of struct hl_bd.
 * HL_QUEUE_LENGTH * sizeof(struct hl_bd) should be <= HL_PAGE_SIZE
 */

#define HL_PAGE_SIZE			4096 /* minimum page size */
/* Must be power of 2 (HL_PAGE_SIZE / HL_BD_SIZE) */
O
Oded Gabbay 已提交
319
#define HL_QUEUE_LENGTH			256
O
Oded Gabbay 已提交
320 321 322 323 324 325 326 327 328
#define HL_QUEUE_SIZE_IN_BYTES		(HL_QUEUE_LENGTH * HL_BD_SIZE)

/*
 * HL_CQ_LENGTH is in units of struct hl_cq_entry.
 * HL_CQ_LENGTH should be <= HL_PAGE_SIZE
 */
#define HL_CQ_LENGTH			HL_QUEUE_LENGTH
#define HL_CQ_SIZE_IN_BYTES		(HL_CQ_LENGTH * HL_CQ_ENTRY_SIZE)

329 330 331
/* Must be power of 2 (HL_PAGE_SIZE / HL_EQ_ENTRY_SIZE) */
#define HL_EQ_LENGTH			64
#define HL_EQ_SIZE_IN_BYTES		(HL_EQ_LENGTH * HL_EQ_ENTRY_SIZE)
O
Oded Gabbay 已提交
332

333
/* Host <-> ArmCP shared memory size */
334
#define HL_CPU_ACCESSIBLE_MEM_SIZE	SZ_2M
O
Oded Gabbay 已提交
335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379

/**
 * struct hl_hw_queue - describes a H/W transport queue.
 * @shadow_queue: pointer to a shadow queue that holds pointers to jobs.
 * @queue_type: type of queue.
 * @kernel_address: holds the queue's kernel virtual address.
 * @bus_address: holds the queue's DMA address.
 * @pi: holds the queue's pi value.
 * @ci: holds the queue's ci value, AS CALCULATED BY THE DRIVER (not real ci).
 * @hw_queue_id: the id of the H/W queue.
 * @int_queue_len: length of internal queue (number of entries).
 * @valid: is the queue valid (we have array of 32 queues, not all of them
 *		exists).
 */
struct hl_hw_queue {
	struct hl_cs_job	**shadow_queue;
	enum hl_queue_type	queue_type;
	u64			kernel_address;
	dma_addr_t		bus_address;
	u32			pi;
	u32			ci;
	u32			hw_queue_id;
	u16			int_queue_len;
	u8			valid;
};

/**
 * struct hl_cq - describes a completion queue
 * @hdev: pointer to the device structure
 * @kernel_address: holds the queue's kernel virtual address
 * @bus_address: holds the queue's DMA address
 * @hw_queue_id: the id of the matching H/W queue
 * @ci: ci inside the queue
 * @pi: pi inside the queue
 * @free_slots_cnt: counter of free slots in queue
 */
struct hl_cq {
	struct hl_device	*hdev;
	u64			kernel_address;
	dma_addr_t		bus_address;
	u32			hw_queue_id;
	u32			ci;
	u32			pi;
	atomic_t		free_slots_cnt;
};
380

381 382 383 384 385 386 387 388 389 390 391 392 393 394
/**
 * struct hl_eq - describes the event queue (single one per device)
 * @hdev: pointer to the device structure
 * @kernel_address: holds the queue's kernel virtual address
 * @bus_address: holds the queue's DMA address
 * @ci: ci inside the queue
 */
struct hl_eq {
	struct hl_device	*hdev;
	u64			kernel_address;
	dma_addr_t		bus_address;
	u32			ci;
};

395

O
Oded Gabbay 已提交
396 397 398 399 400 401 402
/*
 * ASICs
 */

/**
 * enum hl_asic_type - supported ASIC types.
 * @ASIC_INVALID: Invalid ASIC type.
403
 * @ASIC_GOYA: Goya device.
O
Oded Gabbay 已提交
404 405
 */
enum hl_asic_type {
406 407
	ASIC_INVALID,
	ASIC_GOYA
O
Oded Gabbay 已提交
408 409
};

410 411
struct hl_cs_parser;

412 413
/**
 * enum hl_pm_mng_profile - power management profile.
414
 * @PM_AUTO: internal clock is set by the Linux driver.
415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
 * @PM_MANUAL: internal clock is set by the user.
 * @PM_LAST: last power management type.
 */
enum hl_pm_mng_profile {
	PM_AUTO = 1,
	PM_MANUAL,
	PM_LAST
};

/**
 * enum hl_pll_frequency - PLL frequency.
 * @PLL_HIGH: high frequency.
 * @PLL_LOW: low frequency.
 * @PLL_LAST: last frequency values that were configured by the user.
 */
enum hl_pll_frequency {
	PLL_HIGH = 1,
	PLL_LOW,
	PLL_LAST
};

O
Oded Gabbay 已提交
436 437 438 439 440
/**
 * struct hl_asic_funcs - ASIC specific functions that are can be called from
 *                        common code.
 * @early_init: sets up early driver state (pre sw_init), doesn't configure H/W.
 * @early_fini: tears down what was done in early_init.
441 442
 * @late_init: sets up late driver/hw state (post hw_init) - Optional.
 * @late_fini: tears down what was done in late_init (pre hw_fini) - Optional.
O
Oded Gabbay 已提交
443 444
 * @sw_init: sets up driver state, does not configure H/W.
 * @sw_fini: tears down driver state, does not configure H/W.
445 446
 * @hw_init: sets up the H/W state.
 * @hw_fini: tears down the H/W state.
447 448 449
 * @halt_engines: halt engines, needed for reset sequence. This also disables
 *                interrupts from the device. Should be called before
 *                hw_fini and before CS rollback.
O
Oded Gabbay 已提交
450 451
 * @suspend: handles IP specific H/W or SW changes for suspend.
 * @resume: handles IP specific H/W or SW changes for resume.
452
 * @cb_mmap: maps a CB.
O
Oded Gabbay 已提交
453
 * @ring_doorbell: increment PI on a given QMAN.
454 455 456 457 458
 * @pqe_write: Write the PQ entry to the PQ. This is ASIC-specific
 *             function because the PQs are located in different memory areas
 *             per ASIC (SRAM, DRAM, Host memory) and therefore, the method of
 *             writing the PQE must match the destination memory area
 *             properties.
459 460 461 462 463 464 465 466
 * @asic_dma_alloc_coherent: Allocate coherent DMA memory by calling
 *                           dma_alloc_coherent(). This is ASIC function because
 *                           its implementation is not trivial when the driver
 *                           is loaded in simulation mode (not upstreamed).
 * @asic_dma_free_coherent:  Free coherent DMA memory by calling
 *                           dma_free_coherent(). This is ASIC function because
 *                           its implementation is not trivial when the driver
 *                           is loaded in simulation mode (not upstreamed).
O
Oded Gabbay 已提交
467 468
 * @get_int_queue_base: get the internal queue base address.
 * @test_queues: run simple test on all queues for sanity check.
469 470 471
 * @asic_dma_pool_zalloc: small DMA allocation of coherent memory from DMA pool.
 *                        size of allocation is HL_DMA_POOL_BLK_SIZE.
 * @asic_dma_pool_free: free small DMA allocation from pool.
O
Oded Gabbay 已提交
472 473
 * @cpu_accessible_dma_pool_alloc: allocate CPU PQ packet from DMA pool.
 * @cpu_accessible_dma_pool_free: free CPU PQ packet from DMA pool.
474 475 476 477 478
 * @hl_dma_unmap_sg: DMA unmap scatter-gather list.
 * @cs_parser: parse Command Submission.
 * @asic_dma_map_sg: DMA map scatter-gather list.
 * @get_dma_desc_list_size: get number of LIN_DMA packets required for CB.
 * @add_end_of_cb_packets: Add packets to the end of CB, if device requires it.
479
 * @update_eq_ci: update event queue CI.
480 481
 * @context_switch: called upon ASID context switch.
 * @restore_phase_topology: clear all SOBs amd MONs.
O
Oded Gabbay 已提交
482 483
 * @debugfs_read32: debug interface for reading u32 from DRAM/SRAM.
 * @debugfs_write32: debug interface for writing u32 to DRAM/SRAM.
484
 * @add_device_attr: add ASIC specific device attributes.
485
 * @handle_eqe: handle event queue entry (IRQ) from ArmCP.
486
 * @set_pll_profile: change PLL profile (manual/automatic).
487
 * @get_events_stat: retrieve event queue entries histogram.
488 489 490 491 492 493
 * @read_pte: read MMU page table entry from DRAM.
 * @write_pte: write MMU page table entry to DRAM.
 * @mmu_invalidate_cache: flush MMU STLB cache, either with soft (L1 only) or
 *                        hard (L0 & L1) flush.
 * @mmu_invalidate_cache_range: flush specific MMU STLB cache lines with
 *                              ASID-VA-size mask.
494
 * @send_heartbeat: send is-alive packet to ArmCP and verify response.
495
 * @debug_coresight: perform certain actions on Coresight for debugging.
496
 * @is_device_idle: return true if device is idle, false otherwise.
497
 * @soft_reset_late_init: perform certain actions needed after soft reset.
O
Oded Gabbay 已提交
498 499
 * @hw_queues_lock: acquire H/W queues lock.
 * @hw_queues_unlock: release H/W queues lock.
O
Oded Gabbay 已提交
500
 * @get_pci_id: retrieve PCI ID.
501
 * @get_eeprom_data: retrieve EEPROM data from F/W.
O
Oded Gabbay 已提交
502
 * @send_cpu_message: send buffer to ArmCP.
503
 * @get_hw_state: retrieve the H/W state
504
 * @pci_bars_map: Map PCI BARs.
505 506
 * @set_dram_bar_base: Set DRAM BAR to map specific device address. Returns
 *                     old address the bar pointed to or U64_MAX for failure
507
 * @init_iatu: Initialize the iATU unit inside the PCI controller.
508 509
 * @rreg: Read a register. Needed for simulator support.
 * @wreg: Write a register. Needed for simulator support.
510
 * @halt_coresight: stop the ETF and ETR traces.
O
Oded Gabbay 已提交
511 512 513 514
 */
struct hl_asic_funcs {
	int (*early_init)(struct hl_device *hdev);
	int (*early_fini)(struct hl_device *hdev);
515 516
	int (*late_init)(struct hl_device *hdev);
	void (*late_fini)(struct hl_device *hdev);
O
Oded Gabbay 已提交
517 518
	int (*sw_init)(struct hl_device *hdev);
	int (*sw_fini)(struct hl_device *hdev);
519 520
	int (*hw_init)(struct hl_device *hdev);
	void (*hw_fini)(struct hl_device *hdev, bool hard_reset);
521
	void (*halt_engines)(struct hl_device *hdev, bool hard_reset);
O
Oded Gabbay 已提交
522 523
	int (*suspend)(struct hl_device *hdev);
	int (*resume)(struct hl_device *hdev);
524 525
	int (*cb_mmap)(struct hl_device *hdev, struct vm_area_struct *vma,
			u64 kaddress, phys_addr_t paddress, u32 size);
O
Oded Gabbay 已提交
526
	void (*ring_doorbell)(struct hl_device *hdev, u32 hw_queue_id, u32 pi);
527 528
	void (*pqe_write)(struct hl_device *hdev, __le64 *pqe,
			struct hl_bd *bd);
529
	void* (*asic_dma_alloc_coherent)(struct hl_device *hdev, size_t size,
O
Oded Gabbay 已提交
530
					dma_addr_t *dma_handle, gfp_t flag);
531
	void (*asic_dma_free_coherent)(struct hl_device *hdev, size_t size,
O
Oded Gabbay 已提交
532
					void *cpu_addr, dma_addr_t dma_handle);
O
Oded Gabbay 已提交
533 534 535
	void* (*get_int_queue_base)(struct hl_device *hdev, u32 queue_id,
				dma_addr_t *dma_handle, u16 *queue_len);
	int (*test_queues)(struct hl_device *hdev);
536
	void* (*asic_dma_pool_zalloc)(struct hl_device *hdev, size_t size,
O
Oded Gabbay 已提交
537
				gfp_t mem_flags, dma_addr_t *dma_handle);
538
	void (*asic_dma_pool_free)(struct hl_device *hdev, void *vaddr,
O
Oded Gabbay 已提交
539 540 541 542 543
				dma_addr_t dma_addr);
	void* (*cpu_accessible_dma_pool_alloc)(struct hl_device *hdev,
				size_t size, dma_addr_t *dma_handle);
	void (*cpu_accessible_dma_pool_free)(struct hl_device *hdev,
				size_t size, void *vaddr);
544
	void (*hl_dma_unmap_sg)(struct hl_device *hdev,
545
				struct scatterlist *sgl, int nents,
546 547 548
				enum dma_data_direction dir);
	int (*cs_parser)(struct hl_device *hdev, struct hl_cs_parser *parser);
	int (*asic_dma_map_sg)(struct hl_device *hdev,
549
				struct scatterlist *sgl, int nents,
550 551 552
				enum dma_data_direction dir);
	u32 (*get_dma_desc_list_size)(struct hl_device *hdev,
					struct sg_table *sgt);
553 554 555
	void (*add_end_of_cb_packets)(struct hl_device *hdev,
					u64 kernel_address, u32 len,
					u64 cq_addr, u32 cq_val, u32 msix_num);
556
	void (*update_eq_ci)(struct hl_device *hdev, u32 val);
557 558
	int (*context_switch)(struct hl_device *hdev, u32 asid);
	void (*restore_phase_topology)(struct hl_device *hdev);
O
Oded Gabbay 已提交
559 560
	int (*debugfs_read32)(struct hl_device *hdev, u64 addr, u32 *val);
	int (*debugfs_write32)(struct hl_device *hdev, u64 addr, u32 val);
561 562
	void (*add_device_attr)(struct hl_device *hdev,
				struct attribute_group *dev_attr_grp);
563 564
	void (*handle_eqe)(struct hl_device *hdev,
				struct hl_eq_entry *eq_entry);
565 566
	void (*set_pll_profile)(struct hl_device *hdev,
			enum hl_pll_frequency freq);
567 568
	void* (*get_events_stat)(struct hl_device *hdev, bool aggregate,
				u32 *size);
569 570 571 572 573
	u64 (*read_pte)(struct hl_device *hdev, u64 addr);
	void (*write_pte)(struct hl_device *hdev, u64 addr, u64 val);
	void (*mmu_invalidate_cache)(struct hl_device *hdev, bool is_hard);
	void (*mmu_invalidate_cache_range)(struct hl_device *hdev, bool is_hard,
			u32 asid, u64 va, u64 size);
574
	int (*send_heartbeat)(struct hl_device *hdev);
575
	int (*debug_coresight)(struct hl_device *hdev, void *data);
576 577
	bool (*is_device_idle)(struct hl_device *hdev, u32 *mask,
				struct seq_file *s);
578
	int (*soft_reset_late_init)(struct hl_device *hdev);
O
Oded Gabbay 已提交
579 580
	void (*hw_queues_lock)(struct hl_device *hdev);
	void (*hw_queues_unlock)(struct hl_device *hdev);
O
Oded Gabbay 已提交
581
	u32 (*get_pci_id)(struct hl_device *hdev);
582 583
	int (*get_eeprom_data)(struct hl_device *hdev, void *data,
				size_t max_size);
O
Oded Gabbay 已提交
584 585
	int (*send_cpu_message)(struct hl_device *hdev, u32 *msg,
				u16 len, u32 timeout, long *result);
586
	enum hl_device_hw_state (*get_hw_state)(struct hl_device *hdev);
587
	int (*pci_bars_map)(struct hl_device *hdev);
588
	u64 (*set_dram_bar_base)(struct hl_device *hdev, u64 addr);
589
	int (*init_iatu)(struct hl_device *hdev);
590 591
	u32 (*rreg)(struct hl_device *hdev, u32 reg);
	void (*wreg)(struct hl_device *hdev, u32 reg, u32 val);
592
	void (*halt_coresight)(struct hl_device *hdev);
O
Oded Gabbay 已提交
593
};
O
Oded Gabbay 已提交
594

595 596 597 598 599 600 601

/*
 * CONTEXTS
 */

#define HL_KERNEL_ASID_ID	0

602 603 604 605 606 607 608 609 610 611 612 613 614 615
/**
 * struct hl_va_range - virtual addresses range.
 * @lock: protects the virtual addresses list.
 * @list: list of virtual addresses blocks available for mappings.
 * @start_addr: range start address.
 * @end_addr: range end address.
 */
struct hl_va_range {
	struct mutex		lock;
	struct list_head	list;
	u64			start_addr;
	u64			end_addr;
};

616 617
/**
 * struct hl_ctx - user/kernel context.
618 619
 * @mem_hash: holds mapping from virtual address to virtual memory area
 *		descriptor (hl_vm_phys_pg_list or hl_userptr).
620 621
 * @mmu_phys_hash: holds a mapping from physical address to pgt_info structure.
 * @mmu_shadow_hash: holds a mapping from shadow address to pgt_info structure.
622
 * @hpriv: pointer to the private (Kernel Driver) data of the process (fd).
623 624 625
 * @hdev: pointer to the device structure.
 * @refcount: reference counter for the context. Context is released only when
 *		this hits 0l. It is incremented on CS and CS_WAIT.
626
 * @cs_pending: array of DMA fence objects representing pending CS.
627 628 629 630 631
 * @host_va_range: holds available virtual addresses for host mappings.
 * @dram_va_range: holds available virtual addresses for DRAM mappings.
 * @mem_hash_lock: protects the mem_hash.
 * @mmu_lock: protects the MMU page tables. Any change to the PGT, modifing the
 *            MMU hash or walking the PGT requires talking this lock
O
Oded Gabbay 已提交
632
 * @debugfs_list: node in debugfs list of contexts.
633 634 635
 * @cs_sequence: sequence number for CS. Value is assigned to a CS and passed
 *			to user so user could inquire about CS. It is used as
 *			index to cs_pending array.
636 637
 * @dram_default_hops: array that holds all hops addresses needed for default
 *                     DRAM mapping.
638
 * @cs_lock: spinlock to protect cs_sequence.
639
 * @dram_phys_mem: amount of used physical DRAM memory by this context.
640 641 642 643 644 645 646
 * @thread_ctx_switch_token: token to prevent multiple threads of the same
 *				context	from running the context switch phase.
 *				Only a single thread should run it.
 * @thread_ctx_switch_wait_token: token to prevent the threads that didn't run
 *				the context switch phase from moving to their
 *				execution phase before the context switch phase
 *				has finished.
647
 * @asid: context's unique address space ID in the device's MMU.
648
 * @handle: context's opaque handle for user
649 650
 */
struct hl_ctx {
651
	DECLARE_HASHTABLE(mem_hash, MEM_HASH_TABLE_BITS);
652 653
	DECLARE_HASHTABLE(mmu_phys_hash, MMU_HASH_TABLE_BITS);
	DECLARE_HASHTABLE(mmu_shadow_hash, MMU_HASH_TABLE_BITS);
654 655 656
	struct hl_fpriv		*hpriv;
	struct hl_device	*hdev;
	struct kref		refcount;
657
	struct dma_fence	*cs_pending[HL_MAX_PENDING_CS];
658 659 660 661
	struct hl_va_range	host_va_range;
	struct hl_va_range	dram_va_range;
	struct mutex		mem_hash_lock;
	struct mutex		mmu_lock;
O
Oded Gabbay 已提交
662
	struct list_head	debugfs_list;
663
	u64			cs_sequence;
664
	u64			*dram_default_hops;
665
	spinlock_t		cs_lock;
666
	atomic64_t		dram_phys_mem;
667 668
	atomic_t		thread_ctx_switch_token;
	u32			thread_ctx_switch_wait_token;
669
	u32			asid;
670
	u32			handle;
671 672 673 674 675 676 677 678 679 680 681 682 683
};

/**
 * struct hl_ctx_mgr - for handling multiple contexts.
 * @ctx_lock: protects ctx_handles.
 * @ctx_handles: idr to hold all ctx handles.
 */
struct hl_ctx_mgr {
	struct mutex		ctx_lock;
	struct idr		ctx_handles;
};


684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722

/*
 * COMMAND SUBMISSIONS
 */

/**
 * struct hl_userptr - memory mapping chunk information
 * @vm_type: type of the VM.
 * @job_node: linked-list node for hanging the object on the Job's list.
 * @vec: pointer to the frame vector.
 * @sgt: pointer to the scatter-gather table that holds the pages.
 * @dir: for DMA unmapping, the direction must be supplied, so save it.
 * @debugfs_list: node in debugfs list of command submissions.
 * @addr: user-space virtual pointer to the start of the memory area.
 * @size: size of the memory area to pin & map.
 * @dma_mapped: true if the SG was mapped to DMA addresses, false otherwise.
 */
struct hl_userptr {
	enum vm_type_t		vm_type; /* must be first */
	struct list_head	job_node;
	struct frame_vector	*vec;
	struct sg_table		*sgt;
	enum dma_data_direction dir;
	struct list_head	debugfs_list;
	u64			addr;
	u32			size;
	u8			dma_mapped;
};

/**
 * struct hl_cs - command submission.
 * @jobs_in_queue_cnt: per each queue, maintain counter of submitted jobs.
 * @ctx: the context this CS belongs to.
 * @job_list: list of the CS's jobs in the various queues.
 * @job_lock: spinlock for the CS's jobs list. Needed for free_job.
 * @refcount: reference counter for usage of the CS.
 * @fence: pointer to the fence object of this CS.
 * @work_tdr: delayed work node for TDR.
 * @mirror_node : node in device mirror list of command submissions.
O
Oded Gabbay 已提交
723
 * @debugfs_list: node in debugfs list of command submissions.
724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740
 * @sequence: the sequence number of this CS.
 * @submitted: true if CS was submitted to H/W.
 * @completed: true if CS was completed by device.
 * @timedout : true if CS was timedout.
 * @tdr_active: true if TDR was activated for this CS (to prevent
 *		double TDR activation).
 * @aborted: true if CS was aborted due to some device error.
 */
struct hl_cs {
	u8			jobs_in_queue_cnt[HL_MAX_QUEUES];
	struct hl_ctx		*ctx;
	struct list_head	job_list;
	spinlock_t		job_lock;
	struct kref		refcount;
	struct dma_fence	*fence;
	struct delayed_work	work_tdr;
	struct list_head	mirror_node;
O
Oded Gabbay 已提交
741
	struct list_head	debugfs_list;
742 743 744 745 746 747 748 749
	u64			sequence;
	u8			submitted;
	u8			completed;
	u8			timedout;
	u8			tdr_active;
	u8			aborted;
};

O
Oded Gabbay 已提交
750 751
/**
 * struct hl_cs_job - command submission job.
752 753 754 755 756
 * @cs_node: the node to hang on the CS jobs list.
 * @cs: the CS this job belongs to.
 * @user_cb: the CB we got from the user.
 * @patched_cb: in case of patching, this is internal CB which is submitted on
 *		the queue instead of the CB we got from the IOCTL.
O
Oded Gabbay 已提交
757
 * @finish_work: workqueue object to run when job is completed.
758 759
 * @userptr_list: linked-list of userptr mappings that belong to this job and
 *			wait for completion.
O
Oded Gabbay 已提交
760
 * @debugfs_list: node in debugfs list of command submission jobs.
T
Tomer Tayar 已提交
761
 * @queue_type: the type of the H/W queue this job is submitted to.
O
Oded Gabbay 已提交
762
 * @id: the id of this job inside a CS.
763 764 765
 * @hw_queue_id: the id of the H/W queue this job is submitted to.
 * @user_cb_size: the actual size of the CB we got from the user.
 * @job_cb_size: the actual size of the CB that we put on the queue.
T
Tomer Tayar 已提交
766 767 768
 * @is_kernel_allocated_cb: true if the CB handle we got from the user holds a
 *                          handle to a kernel-allocated CB object, false
 *                          otherwise (SRAM/DRAM/host address).
O
Oded Gabbay 已提交
769 770
 */
struct hl_cs_job {
771 772 773 774
	struct list_head	cs_node;
	struct hl_cs		*cs;
	struct hl_cb		*user_cb;
	struct hl_cb		*patched_cb;
O
Oded Gabbay 已提交
775
	struct work_struct	finish_work;
776
	struct list_head	userptr_list;
O
Oded Gabbay 已提交
777
	struct list_head	debugfs_list;
T
Tomer Tayar 已提交
778
	enum hl_queue_type	queue_type;
O
Oded Gabbay 已提交
779
	u32			id;
780 781 782
	u32			hw_queue_id;
	u32			user_cb_size;
	u32			job_cb_size;
T
Tomer Tayar 已提交
783
	u8			is_kernel_allocated_cb;
784 785 786
};

/**
T
Tomer Tayar 已提交
787
 * struct hl_cs_parser - command submission parser properties.
788 789 790 791 792 793
 * @user_cb: the CB we got from the user.
 * @patched_cb: in case of patching, this is internal CB which is submitted on
 *		the queue instead of the CB we got from the IOCTL.
 * @job_userptr_list: linked-list of userptr mappings that belong to the related
 *			job and wait for completion.
 * @cs_sequence: the sequence number of the related CS.
T
Tomer Tayar 已提交
794
 * @queue_type: the type of the H/W queue this job is submitted to.
795 796 797 798 799
 * @ctx_id: the ID of the context the related CS belongs to.
 * @hw_queue_id: the id of the H/W queue this job is submitted to.
 * @user_cb_size: the actual size of the CB we got from the user.
 * @patched_cb_size: the size of the CB after parsing.
 * @job_id: the id of the related job inside the related CS.
T
Tomer Tayar 已提交
800 801 802
 * @is_kernel_allocated_cb: true if the CB handle we got from the user holds a
 *                          handle to a kernel-allocated CB object, false
 *                          otherwise (SRAM/DRAM/host address).
803 804 805 806 807 808
 */
struct hl_cs_parser {
	struct hl_cb		*user_cb;
	struct hl_cb		*patched_cb;
	struct list_head	*job_userptr_list;
	u64			cs_sequence;
T
Tomer Tayar 已提交
809
	enum hl_queue_type	queue_type;
810 811 812 813 814
	u32			ctx_id;
	u32			hw_queue_id;
	u32			user_cb_size;
	u32			patched_cb_size;
	u8			job_id;
T
Tomer Tayar 已提交
815
	u8			is_kernel_allocated_cb;
O
Oded Gabbay 已提交
816
};
817 818


819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840
/*
 * MEMORY STRUCTURE
 */

/**
 * struct hl_vm_hash_node - hash element from virtual address to virtual
 *				memory area descriptor (hl_vm_phys_pg_list or
 *				hl_userptr).
 * @node: node to hang on the hash table in context object.
 * @vaddr: key virtual address.
 * @ptr: value pointer (hl_vm_phys_pg_list or hl_userptr).
 */
struct hl_vm_hash_node {
	struct hlist_node	node;
	u64			vaddr;
	void			*ptr;
};

/**
 * struct hl_vm_phys_pg_pack - physical page pack.
 * @vm_type: describes the type of the virtual area descriptor.
 * @pages: the physical page array.
841 842
 * @npages: num physical pages in the pack.
 * @total_size: total size of all the pages in this list.
843 844 845 846 847 848 849 850 851 852 853 854
 * @mapping_cnt: number of shared mappings.
 * @asid: the context related to this list.
 * @page_size: size of each page in the pack.
 * @flags: HL_MEM_* flags related to this list.
 * @handle: the provided handle related to this list.
 * @offset: offset from the first page.
 * @contiguous: is contiguous physical memory.
 * @created_from_userptr: is product of host virtual address.
 */
struct hl_vm_phys_pg_pack {
	enum vm_type_t		vm_type; /* must be first */
	u64			*pages;
855 856
	u64			npages;
	u64			total_size;
857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897
	atomic_t		mapping_cnt;
	u32			asid;
	u32			page_size;
	u32			flags;
	u32			handle;
	u32			offset;
	u8			contiguous;
	u8			created_from_userptr;
};

/**
 * struct hl_vm_va_block - virtual range block information.
 * @node: node to hang on the virtual range list in context object.
 * @start: virtual range start address.
 * @end: virtual range end address.
 * @size: virtual range size.
 */
struct hl_vm_va_block {
	struct list_head	node;
	u64			start;
	u64			end;
	u64			size;
};

/**
 * struct hl_vm - virtual memory manager for MMU.
 * @dram_pg_pool: pool for DRAM physical pages of 2MB.
 * @dram_pg_pool_refcount: reference counter for the pool usage.
 * @idr_lock: protects the phys_pg_list_handles.
 * @phys_pg_pack_handles: idr to hold all device allocations handles.
 * @init_done: whether initialization was done. We need this because VM
 *		initialization might be skipped during device initialization.
 */
struct hl_vm {
	struct gen_pool		*dram_pg_pool;
	struct kref		dram_pg_pool_refcount;
	spinlock_t		idr_lock;
	struct idr		phys_pg_pack_handles;
	u8			init_done;
};

898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920

/*
 * DEBUG, PROFILING STRUCTURE
 */

/**
 * struct hl_debug_params - Coresight debug parameters.
 * @input: pointer to component specific input parameters.
 * @output: pointer to component specific output parameters.
 * @output_size: size of output buffer.
 * @reg_idx: relevant register ID.
 * @op: component operation to execute.
 * @enable: true if to enable component debugging, false otherwise.
 */
struct hl_debug_params {
	void *input;
	void *output;
	u32 output_size;
	u32 reg_idx;
	u32 op;
	bool enable;
};

O
Oded Gabbay 已提交
921 922 923 924 925 926 927 928 929
/*
 * FILE PRIVATE STRUCTURE
 */

/**
 * struct hl_fpriv - process information stored in FD private data.
 * @hdev: habanalabs device structure.
 * @filp: pointer to the given file structure.
 * @taskpid: current process ID.
930
 * @ctx: current executing context. TODO: remove for multiple ctx per process
931
 * @ctx_mgr: context manager to handle multiple context for this FD.
932
 * @cb_mgr: command buffer manager to handle multiple buffers for this FD.
O
Oded Gabbay 已提交
933
 * @debugfs_list: list of relevant ASIC debugfs.
934
 * @dev_node: node in the device list of file private data
O
Oded Gabbay 已提交
935
 * @refcount: number of related contexts.
936
 * @restore_phase_mutex: lock for context switch and restore phase.
937
 * @is_control: true for control device, false otherwise
O
Oded Gabbay 已提交
938 939 940 941 942
 */
struct hl_fpriv {
	struct hl_device	*hdev;
	struct file		*filp;
	struct pid		*taskpid;
943
	struct hl_ctx		*ctx;
944
	struct hl_ctx_mgr	ctx_mgr;
945
	struct hl_cb_mgr	cb_mgr;
O
Oded Gabbay 已提交
946
	struct list_head	debugfs_list;
947
	struct list_head	dev_node;
O
Oded Gabbay 已提交
948
	struct kref		refcount;
949
	struct mutex		restore_phase_mutex;
950
	u8			is_control;
O
Oded Gabbay 已提交
951 952 953
};


O
Oded Gabbay 已提交
954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
/*
 * DebugFS
 */

/**
 * struct hl_info_list - debugfs file ops.
 * @name: file name.
 * @show: function to output information.
 * @write: function to write to the file.
 */
struct hl_info_list {
	const char	*name;
	int		(*show)(struct seq_file *s, void *data);
	ssize_t		(*write)(struct file *file, const char __user *buf,
				size_t count, loff_t *f_pos);
};

/**
 * struct hl_debugfs_entry - debugfs dentry wrapper.
 * @dent: base debugfs entry structure.
 * @info_ent: dentry realted ops.
 * @dev_entry: ASIC specific debugfs manager.
 */
struct hl_debugfs_entry {
	struct dentry			*dent;
	const struct hl_info_list	*info_ent;
	struct hl_dbg_device_entry	*dev_entry;
};

/**
 * struct hl_dbg_device_entry - ASIC specific debugfs manager.
 * @root: root dentry.
 * @hdev: habanalabs device structure.
 * @entry_arr: array of available hl_debugfs_entry.
 * @file_list: list of available debugfs files.
 * @file_mutex: protects file_list.
 * @cb_list: list of available CBs.
 * @cb_spinlock: protects cb_list.
 * @cs_list: list of available CSs.
 * @cs_spinlock: protects cs_list.
 * @cs_job_list: list of available CB jobs.
 * @cs_job_spinlock: protects cs_job_list.
 * @userptr_list: list of available userptrs (virtual memory chunk descriptor).
 * @userptr_spinlock: protects userptr_list.
 * @ctx_mem_hash_list: list of available contexts with MMU mappings.
 * @ctx_mem_hash_spinlock: protects cb_list.
 * @addr: next address to read/write from/to in read/write32.
 * @mmu_addr: next virtual address to translate to physical address in mmu_show.
 * @mmu_asid: ASID to use while translating in mmu_show.
 * @i2c_bus: generic u8 debugfs file for bus value to use in i2c_data_read.
 * @i2c_bus: generic u8 debugfs file for address value to use in i2c_data_read.
 * @i2c_bus: generic u8 debugfs file for register value to use in i2c_data_read.
 */
struct hl_dbg_device_entry {
	struct dentry			*root;
	struct hl_device		*hdev;
	struct hl_debugfs_entry		*entry_arr;
	struct list_head		file_list;
	struct mutex			file_mutex;
	struct list_head		cb_list;
	spinlock_t			cb_spinlock;
	struct list_head		cs_list;
	spinlock_t			cs_spinlock;
	struct list_head		cs_job_list;
	spinlock_t			cs_job_spinlock;
	struct list_head		userptr_list;
	spinlock_t			userptr_spinlock;
	struct list_head		ctx_mem_hash_list;
	spinlock_t			ctx_mem_hash_spinlock;
	u64				addr;
	u64				mmu_addr;
	u32				mmu_asid;
	u8				i2c_bus;
	u8				i2c_addr;
	u8				i2c_reg;
};


O
Oded Gabbay 已提交
1032 1033 1034 1035 1036
/*
 * DEVICES
 */

/* Theoretical limit only. A single host can only contain up to 4 or 8 PCIe
1037
 * x16 cards. In extreme cases, there are hosts that can accommodate 16 cards.
O
Oded Gabbay 已提交
1038 1039 1040
 */
#define HL_MAX_MINORS	256

O
Oded Gabbay 已提交
1041 1042 1043 1044 1045 1046 1047
/*
 * Registers read & write functions.
 */

u32 hl_rreg(struct hl_device *hdev, u32 reg);
void hl_wreg(struct hl_device *hdev, u32 reg, u32 val);

1048 1049
#define RREG32(reg) hdev->asic_funcs->rreg(hdev, (reg))
#define WREG32(reg, v) hdev->asic_funcs->wreg(hdev, (reg), (v))
O
Oded Gabbay 已提交
1050
#define DREG32(reg) pr_info("REGISTER: " #reg " : 0x%08X\n",	\
1051
			hdev->asic_funcs->rreg(hdev, (reg)))
O
Oded Gabbay 已提交
1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068

#define WREG32_P(reg, val, mask)				\
	do {							\
		u32 tmp_ = RREG32(reg);				\
		tmp_ &= (mask);					\
		tmp_ |= ((val) & ~(mask));			\
		WREG32(reg, tmp_);				\
	} while (0)
#define WREG32_AND(reg, and) WREG32_P(reg, 0, and)
#define WREG32_OR(reg, or) WREG32_P(reg, or, ~(or))

#define REG_FIELD_SHIFT(reg, field) reg##_##field##_SHIFT
#define REG_FIELD_MASK(reg, field) reg##_##field##_MASK
#define WREG32_FIELD(reg, field, val)	\
	WREG32(mm##reg, (RREG32(mm##reg) & ~REG_FIELD_MASK(reg, field)) | \
			(val) << REG_FIELD_SHIFT(reg, field))

O
Oded Gabbay 已提交
1069 1070 1071
/* Timeout should be longer when working with simulator but cap the
 * increased timeout to some maximum
 */
1072 1073
#define hl_poll_timeout(hdev, addr, val, cond, sleep_us, timeout_us) \
({ \
1074 1075 1076 1077
	ktime_t __timeout; \
	if (hdev->pdev) \
		__timeout = ktime_add_us(ktime_get(), timeout_us); \
	else \
O
Oded Gabbay 已提交
1078 1079 1080
		__timeout = ktime_add_us(ktime_get(),\
				min((u64)(timeout_us * 10), \
					(u64) HL_SIM_MAX_TIMEOUT_US)); \
1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095
	might_sleep_if(sleep_us); \
	for (;;) { \
		(val) = RREG32(addr); \
		if (cond) \
			break; \
		if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \
			(val) = RREG32(addr); \
			break; \
		} \
		if (sleep_us) \
			usleep_range((sleep_us >> 2) + 1, sleep_us); \
	} \
	(cond) ? 0 : -ETIMEDOUT; \
})

1096 1097 1098
/*
 * address in this macro points always to a memory location in the
 * host's (server's) memory. That location is updated asynchronously
1099 1100 1101 1102 1103 1104 1105 1106
 * either by the direct access of the device or by another core.
 *
 * To work both in LE and BE architectures, we need to distinguish between the
 * two states (device or another core updates the memory location). Therefore,
 * if mem_written_by_device is true, the host memory being polled will be
 * updated directly by the device. If false, the host memory being polled will
 * be updated by host CPU. Required so host knows whether or not the memory
 * might need to be byte-swapped before returning value to caller.
1107
 */
1108 1109
#define hl_poll_timeout_memory(hdev, addr, val, cond, sleep_us, timeout_us, \
				mem_written_by_device) \
1110 1111 1112 1113 1114
({ \
	ktime_t __timeout; \
	if (hdev->pdev) \
		__timeout = ktime_add_us(ktime_get(), timeout_us); \
	else \
O
Oded Gabbay 已提交
1115 1116 1117
		__timeout = ktime_add_us(ktime_get(),\
				min((u64)(timeout_us * 10), \
					(u64) HL_SIM_MAX_TIMEOUT_US)); \
1118 1119 1120 1121 1122
	might_sleep_if(sleep_us); \
	for (;;) { \
		/* Verify we read updates done by other cores or by device */ \
		mb(); \
		(val) = *((u32 *) (uintptr_t) (addr)); \
1123
		if (mem_written_by_device) \
1124
			(val) = le32_to_cpu(*(__le32 *) &(val)); \
1125 1126 1127 1128
		if (cond) \
			break; \
		if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \
			(val) = *((u32 *) (uintptr_t) (addr)); \
1129
			if (mem_written_by_device) \
1130
				(val) = le32_to_cpu(*(__le32 *) &(val)); \
1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145
			break; \
		} \
		if (sleep_us) \
			usleep_range((sleep_us >> 2) + 1, sleep_us); \
	} \
	(cond) ? 0 : -ETIMEDOUT; \
})

#define hl_poll_timeout_device_memory(hdev, addr, val, cond, sleep_us, \
					timeout_us) \
({ \
	ktime_t __timeout; \
	if (hdev->pdev) \
		__timeout = ktime_add_us(ktime_get(), timeout_us); \
	else \
O
Oded Gabbay 已提交
1146 1147 1148
		__timeout = ktime_add_us(ktime_get(),\
				min((u64)(timeout_us * 10), \
					(u64) HL_SIM_MAX_TIMEOUT_US)); \
1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162
	might_sleep_if(sleep_us); \
	for (;;) { \
		(val) = readl(addr); \
		if (cond) \
			break; \
		if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \
			(val) = readl(addr); \
			break; \
		} \
		if (sleep_us) \
			usleep_range((sleep_us >> 2) + 1, sleep_us); \
	} \
	(cond) ? 0 : -ETIMEDOUT; \
})
1163

1164 1165
struct hwmon_chip_info;

1166 1167 1168 1169 1170 1171 1172 1173 1174 1175
/**
 * struct hl_device_reset_work - reset workqueue task wrapper.
 * @reset_work: reset work to be done.
 * @hdev: habanalabs device structure.
 */
struct hl_device_reset_work {
	struct work_struct		reset_work;
	struct hl_device		*hdev;
};

1176 1177 1178 1179 1180 1181 1182 1183 1184 1185
/**
 * struct hl_device_idle_busy_ts - used for calculating device utilization rate.
 * @idle_to_busy_ts: timestamp where device changed from idle to busy.
 * @busy_to_idle_ts: timestamp where device changed from busy to idle.
 */
struct hl_device_idle_busy_ts {
	ktime_t				idle_to_busy_ts;
	ktime_t				busy_to_idle_ts;
};

O
Oded Gabbay 已提交
1186 1187 1188
/**
 * struct hl_device - habanalabs device structure.
 * @pdev: pointer to PCI device, can be NULL in case of simulator device.
O
Oded Gabbay 已提交
1189 1190
 * @pcie_bar: array of available PCIe bars.
 * @rmmio: configuration area address on SRAM.
O
Oded Gabbay 已提交
1191
 * @cdev: related char device.
1192 1193 1194
 * @cdev_ctrl: char device for control operations only (INFO IOCTL)
 * @dev: related kernel basic device structure.
 * @dev_ctrl: related kernel device structure for the control device
1195
 * @work_freq: delayed work to lower device frequency if possible.
1196
 * @work_heartbeat: delayed work for ArmCP is-alive check.
O
Oded Gabbay 已提交
1197 1198
 * @asic_name: ASIC specific nmae.
 * @asic_type: ASIC specific type.
O
Oded Gabbay 已提交
1199 1200 1201
 * @completion_queue: array of hl_cq.
 * @cq_wq: work queue of completion queues for executing work in process context
 * @eq_wq: work queue of event queue for executing work in process context.
1202
 * @kernel_ctx: Kernel driver context structure.
O
Oded Gabbay 已提交
1203
 * @kernel_queues: array of hl_hw_queue.
1204 1205
 * @hw_queues_mirror_list: CS mirror list for TDR.
 * @hw_queues_mirror_lock: protects hw_queues_mirror_list.
1206
 * @kernel_cb_mgr: command buffer manager for creating/destroying/handling CGs.
1207
 * @event_queue: event queue for IRQ from ArmCP.
O
Oded Gabbay 已提交
1208
 * @dma_pool: DMA pool for small allocations.
1209 1210 1211
 * @cpu_accessible_dma_mem: Host <-> ArmCP shared memory CPU address.
 * @cpu_accessible_dma_address: Host <-> ArmCP shared memory DMA address.
 * @cpu_accessible_dma_pool: Host <-> ArmCP shared memory pool.
1212 1213
 * @asid_bitmap: holds used/available ASIDs.
 * @asid_mutex: protects asid_bitmap.
1214
 * @send_cpu_message_lock: enforces only one message in Host <-> ArmCP queue.
1215
 * @debug_lock: protects critical section of setting debug mode for device
O
Oded Gabbay 已提交
1216 1217 1218
 * @asic_prop: ASIC specific immutable properties.
 * @asic_funcs: ASIC specific functions.
 * @asic_specific: ASIC specific information to use only from ASIC files.
1219 1220
 * @mmu_pgt_pool: pool of available MMU hops.
 * @vm: virtual memory manager for MMU.
1221 1222
 * @mmu_cache_lock: protects MMU cache invalidation as it can serve one context.
 * @mmu_shadow_hop0: shadow mapping of the MMU hop 0 zone.
1223 1224 1225
 * @hwmon_dev: H/W monitor device.
 * @pm_mng_profile: current power management profile.
 * @hl_chip_info: ASIC's sensors information.
O
Oded Gabbay 已提交
1226
 * @hl_debugfs: device's debugfs manager.
1227 1228
 * @cb_pool: list of preallocated CBs.
 * @cb_pool_lock: protects the CB pool.
1229 1230 1231
 * @fpriv_list: list of file private data structures. Each structure is created
 *              when a user opens the device
 * @fpriv_list_lock: protects the fpriv_list
1232
 * @compute_ctx: current compute context executing.
1233 1234
 * @idle_busy_ts_arr: array to hold time stamps of transitions from idle to busy
 *                    and vice-versa
1235
 * @dram_used_mem: current DRAM memory consumption.
1236
 * @timeout_jiffies: device CS timeout value.
1237
 * @max_power: the max power of the device, as configured by the sysadmin. This
1238 1239
 *             value is saved so in case of hard-reset, the driver will restore
 *             this value and update the F/W after the re-initialization
1240
 * @in_reset: is device in reset flow.
1241
 * @curr_pll_profile: current PLL profile.
1242 1243
 * @cs_active_cnt: number of active command submissions on this device (active
 *                 means already in H/W queues)
1244
 * @major: habanalabs kernel driver major.
1245
 * @high_pll: high PLL profile frequency.
1246 1247
 * @soft_reset_cnt: number of soft reset since the driver was loaded.
 * @hard_reset_cnt: number of hard reset since the driver was loaded.
1248
 * @idle_busy_ts_idx: index of current entry in idle_busy_ts_arr
O
Oded Gabbay 已提交
1249
 * @id: device minor.
1250
 * @id_control: minor of the control device
O
Oded Gabbay 已提交
1251
 * @disabled: is device disabled.
1252 1253
 * @late_init_done: is late init stage was done during initialization.
 * @hwmon_initialized: is H/W monitor sensors was initialized.
1254 1255
 * @hard_reset_pending: is there a hard reset work pending.
 * @heartbeat: is heartbeat sanity check towards ArmCP enabled.
1256 1257
 * @reset_on_lockup: true if a reset should be done in case of stuck CS, false
 *                   otherwise.
1258
 * @dram_supports_virtual_memory: is MMU enabled towards DRAM.
1259
 * @dram_default_page_mapping: is DRAM default page mapping enabled.
1260
 * @init_done: is the initialization of the device done.
1261
 * @mmu_enable: is MMU enabled.
1262
 * @device_cpu_disabled: is the device CPU disabled (due to timeouts)
1263
 * @dma_mask: the dma mask that was set for this device
1264
 * @in_debug: is device under debug. This, together with fpriv_list, enforces
1265
 *            that only a single user is configuring the debug infrastructure.
1266
 * @cdev_sysfs_created: were char devices and sysfs nodes created.
O
Oded Gabbay 已提交
1267 1268 1269
 */
struct hl_device {
	struct pci_dev			*pdev;
O
Oded Gabbay 已提交
1270 1271
	void __iomem			*pcie_bar[6];
	void __iomem			*rmmio;
O
Oded Gabbay 已提交
1272
	struct cdev			cdev;
1273
	struct cdev			cdev_ctrl;
O
Oded Gabbay 已提交
1274
	struct device			*dev;
1275
	struct device			*dev_ctrl;
1276
	struct delayed_work		work_freq;
1277
	struct delayed_work		work_heartbeat;
O
Oded Gabbay 已提交
1278 1279
	char				asic_name[16];
	enum hl_asic_type		asic_type;
O
Oded Gabbay 已提交
1280 1281
	struct hl_cq			*completion_queue;
	struct workqueue_struct		*cq_wq;
1282
	struct workqueue_struct		*eq_wq;
1283
	struct hl_ctx			*kernel_ctx;
O
Oded Gabbay 已提交
1284
	struct hl_hw_queue		*kernel_queues;
1285 1286
	struct list_head		hw_queues_mirror_list;
	spinlock_t			hw_queues_mirror_lock;
1287
	struct hl_cb_mgr		kernel_cb_mgr;
1288
	struct hl_eq			event_queue;
O
Oded Gabbay 已提交
1289 1290 1291 1292
	struct dma_pool			*dma_pool;
	void				*cpu_accessible_dma_mem;
	dma_addr_t			cpu_accessible_dma_address;
	struct gen_pool			*cpu_accessible_dma_pool;
1293 1294
	unsigned long			*asid_bitmap;
	struct mutex			asid_mutex;
O
Oded Gabbay 已提交
1295
	struct mutex			send_cpu_message_lock;
1296
	struct mutex			debug_lock;
O
Oded Gabbay 已提交
1297 1298 1299
	struct asic_fixed_properties	asic_prop;
	const struct hl_asic_funcs	*asic_funcs;
	void				*asic_specific;
1300 1301 1302
	struct gen_pool			*mmu_pgt_pool;
	struct hl_vm			vm;
	struct mutex			mmu_cache_lock;
1303
	void				*mmu_shadow_hop0;
1304 1305 1306
	struct device			*hwmon_dev;
	enum hl_pm_mng_profile		pm_mng_profile;
	struct hwmon_chip_info		*hl_chip_info;
1307

O
Oded Gabbay 已提交
1308 1309
	struct hl_dbg_device_entry	hl_debugfs;

1310 1311 1312
	struct list_head		cb_pool;
	spinlock_t			cb_pool_lock;

1313 1314 1315
	struct list_head		fpriv_list;
	struct mutex			fpriv_list_lock;

1316
	struct hl_ctx			*compute_ctx;
1317

1318 1319
	struct hl_device_idle_busy_ts	*idle_busy_ts_arr;

1320
	atomic64_t			dram_used_mem;
1321 1322
	u64				timeout_jiffies;
	u64				max_power;
1323
	atomic_t			in_reset;
1324
	enum hl_pll_frequency		curr_pll_profile;
1325
	int				cs_active_cnt;
O
Oded Gabbay 已提交
1326
	u32				major;
1327
	u32				high_pll;
1328 1329
	u32				soft_reset_cnt;
	u32				hard_reset_cnt;
1330
	u32				idle_busy_ts_idx;
O
Oded Gabbay 已提交
1331
	u16				id;
1332
	u16				id_control;
O
Oded Gabbay 已提交
1333
	u8				disabled;
1334 1335
	u8				late_init_done;
	u8				hwmon_initialized;
1336 1337
	u8				hard_reset_pending;
	u8				heartbeat;
1338
	u8				reset_on_lockup;
1339
	u8				dram_supports_virtual_memory;
1340
	u8				dram_default_page_mapping;
1341
	u8				init_done;
1342
	u8				device_cpu_disabled;
1343
	u8				dma_mask;
1344
	u8				in_debug;
1345
	u8				cdev_sysfs_created;
O
Oded Gabbay 已提交
1346 1347

	/* Parameters for bring-up */
1348
	u8				mmu_enable;
1349
	u8				cpu_enable;
O
Oded Gabbay 已提交
1350
	u8				reset_pcilink;
O
Oded Gabbay 已提交
1351
	u8				cpu_queues_enable;
1352 1353
	u8				fw_loading;
	u8				pldm;
O
Oded Gabbay 已提交
1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385
};


/*
 * IOCTLs
 */

/**
 * typedef hl_ioctl_t - typedef for ioctl function in the driver
 * @hpriv: pointer to the FD's private data, which contains state of
 *		user process
 * @data: pointer to the input/output arguments structure of the IOCTL
 *
 * Return: 0 for success, negative value for error
 */
typedef int hl_ioctl_t(struct hl_fpriv *hpriv, void *data);

/**
 * struct hl_ioctl_desc - describes an IOCTL entry of the driver.
 * @cmd: the IOCTL code as created by the kernel macros.
 * @func: pointer to the driver's function that should be called for this IOCTL.
 */
struct hl_ioctl_desc {
	unsigned int cmd;
	hl_ioctl_t *func;
};


/*
 * Kernel module functions that can be accessed by entire module
 */

1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437
/**
 * hl_mem_area_inside_range() - Checks whether address+size are inside a range.
 * @address: The start address of the area we want to validate.
 * @size: The size in bytes of the area we want to validate.
 * @range_start_address: The start address of the valid range.
 * @range_end_address: The end address of the valid range.
 *
 * Return: true if the area is inside the valid range, false otherwise.
 */
static inline bool hl_mem_area_inside_range(u64 address, u32 size,
				u64 range_start_address, u64 range_end_address)
{
	u64 end_address = address + size;

	if ((address >= range_start_address) &&
			(end_address <= range_end_address) &&
			(end_address > address))
		return true;

	return false;
}

/**
 * hl_mem_area_crosses_range() - Checks whether address+size crossing a range.
 * @address: The start address of the area we want to validate.
 * @size: The size in bytes of the area we want to validate.
 * @range_start_address: The start address of the valid range.
 * @range_end_address: The end address of the valid range.
 *
 * Return: true if the area overlaps part or all of the valid range,
 *		false otherwise.
 */
static inline bool hl_mem_area_crosses_range(u64 address, u32 size,
				u64 range_start_address, u64 range_end_address)
{
	u64 end_address = address + size;

	if ((address >= range_start_address) &&
			(address < range_end_address))
		return true;

	if ((end_address >= range_start_address) &&
			(end_address < range_end_address))
		return true;

	if ((address < range_start_address) &&
			(end_address >= range_end_address))
		return true;

	return false;
}

O
Oded Gabbay 已提交
1438
int hl_device_open(struct inode *inode, struct file *filp);
1439
int hl_device_open_ctrl(struct inode *inode, struct file *filp);
1440
bool hl_device_disabled_or_in_reset(struct hl_device *hdev);
1441
enum hl_device_status hl_device_status(struct hl_device *hdev);
1442
int hl_device_set_debug_mode(struct hl_device *hdev, bool enable);
O
Oded Gabbay 已提交
1443 1444 1445
int create_hdev(struct hl_device **dev, struct pci_dev *pdev,
		enum hl_asic_type asic_type, int minor);
void destroy_hdev(struct hl_device *hdev);
O
Oded Gabbay 已提交
1446 1447 1448 1449
int hl_hw_queues_create(struct hl_device *hdev);
void hl_hw_queues_destroy(struct hl_device *hdev);
int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
				u32 cb_size, u64 cb_ptr);
1450
int hl_hw_queue_schedule_cs(struct hl_cs *cs);
O
Oded Gabbay 已提交
1451 1452
u32 hl_hw_queue_add_ptr(u32 ptr, u16 val);
void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id);
1453
void hl_int_hw_queue_update_ci(struct hl_cs *cs);
1454
void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset);
O
Oded Gabbay 已提交
1455 1456 1457 1458 1459 1460

#define hl_queue_inc_ptr(p)		hl_hw_queue_add_ptr(p, 1)
#define hl_pi_2_offset(pi)		((pi) & (HL_QUEUE_LENGTH - 1))

int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id);
void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q);
1461 1462
int hl_eq_init(struct hl_device *hdev, struct hl_eq *q);
void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q);
1463 1464
void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q);
void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q);
1465 1466
irqreturn_t hl_irq_handler_cq(int irq, void *arg);
irqreturn_t hl_irq_handler_eq(int irq, void *arg);
1467 1468
u32 hl_cq_inc_ptr(u32 ptr);

1469 1470 1471 1472 1473 1474 1475 1476
int hl_asid_init(struct hl_device *hdev);
void hl_asid_fini(struct hl_device *hdev);
unsigned long hl_asid_alloc(struct hl_device *hdev);
void hl_asid_free(struct hl_device *hdev, unsigned long asid);

int hl_ctx_create(struct hl_device *hdev, struct hl_fpriv *hpriv);
void hl_ctx_free(struct hl_device *hdev, struct hl_ctx *ctx);
int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx);
1477 1478
void hl_ctx_do_release(struct kref *ref);
void hl_ctx_get(struct hl_device *hdev,	struct hl_ctx *ctx);
1479
int hl_ctx_put(struct hl_ctx *ctx);
1480
struct dma_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq);
1481 1482
void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr);
void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr);
1483

O
Oded Gabbay 已提交
1484 1485 1486 1487
int hl_device_init(struct hl_device *hdev, struct class *hclass);
void hl_device_fini(struct hl_device *hdev);
int hl_device_suspend(struct hl_device *hdev);
int hl_device_resume(struct hl_device *hdev);
1488 1489
int hl_device_reset(struct hl_device *hdev, bool hard_reset,
			bool from_hard_reset_thread);
1490 1491
void hl_hpriv_get(struct hl_fpriv *hpriv);
void hl_hpriv_put(struct hl_fpriv *hpriv);
1492
int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq);
1493
uint32_t hl_device_utilization(struct hl_device *hdev, uint32_t period_ms);
1494

1495 1496 1497 1498 1499 1500 1501 1502
int hl_build_hwmon_channel_info(struct hl_device *hdev,
		struct armcp_sensor *sensors_arr);

int hl_sysfs_init(struct hl_device *hdev);
void hl_sysfs_fini(struct hl_device *hdev);

int hl_hwmon_init(struct hl_device *hdev);
void hl_hwmon_fini(struct hl_device *hdev);
O
Oded Gabbay 已提交
1503

1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516
int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr, u32 cb_size,
		u64 *handle, int ctx_id);
int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle);
int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma);
struct hl_cb *hl_cb_get(struct hl_device *hdev,	struct hl_cb_mgr *mgr,
			u32 handle);
void hl_cb_put(struct hl_cb *cb);
void hl_cb_mgr_init(struct hl_cb_mgr *mgr);
void hl_cb_mgr_fini(struct hl_device *hdev, struct hl_cb_mgr *mgr);
struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size);
int hl_cb_pool_init(struct hl_device *hdev);
int hl_cb_pool_fini(struct hl_device *hdev);

1517
void hl_cs_rollback_all(struct hl_device *hdev);
T
Tomer Tayar 已提交
1518 1519
struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev,
		enum hl_queue_type queue_type, bool is_kernel_allocated_cb);
1520

O
Oded Gabbay 已提交
1521 1522
void goya_set_asic_funcs(struct hl_device *hdev);

1523 1524 1525 1526 1527 1528
int hl_vm_ctx_init(struct hl_ctx *ctx);
void hl_vm_ctx_fini(struct hl_ctx *ctx);

int hl_vm_init(struct hl_device *hdev);
void hl_vm_fini(struct hl_device *hdev);

1529
int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size,
1530 1531 1532 1533 1534 1535 1536 1537
			struct hl_userptr *userptr);
int hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr);
void hl_userptr_delete_list(struct hl_device *hdev,
				struct list_head *userptr_list);
bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr, u32 size,
				struct list_head *userptr_list,
				struct hl_userptr **userptr);

1538 1539
int hl_mmu_init(struct hl_device *hdev);
void hl_mmu_fini(struct hl_device *hdev);
1540
int hl_mmu_ctx_init(struct hl_ctx *ctx);
1541 1542 1543 1544 1545 1546
void hl_mmu_ctx_fini(struct hl_ctx *ctx);
int hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size);
int hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr, u32 page_size);
void hl_mmu_swap_out(struct hl_ctx *ctx);
void hl_mmu_swap_in(struct hl_ctx *ctx);

1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560
int hl_fw_push_fw_to_device(struct hl_device *hdev, const char *fw_name,
				void __iomem *dst);
int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode);
int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
				u16 len, u32 timeout, long *result);
int hl_fw_test_cpu_queue(struct hl_device *hdev);
void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
						dma_addr_t *dma_handle);
void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
					void *vaddr);
int hl_fw_send_heartbeat(struct hl_device *hdev);
int hl_fw_armcp_info_get(struct hl_device *hdev);
int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size);

1561 1562 1563 1564 1565 1566
int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3],
			bool is_wc[3]);
int hl_pci_iatu_write(struct hl_device *hdev, u32 addr, u32 data);
int hl_pci_set_dram_bar_base(struct hl_device *hdev, u8 inbound_region, u8 bar,
				u64 addr);
int hl_pci_init_iatu(struct hl_device *hdev, u64 sram_base_address,
1567 1568
			u64 dram_base_address, u64 host_phys_base_address,
			u64 host_phys_size);
1569
int hl_pci_init(struct hl_device *hdev, u8 dma_mask);
1570
void hl_pci_fini(struct hl_device *hdev);
1571
int hl_pci_set_dma_mask(struct hl_device *hdev, u8 dma_mask);
1572

1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584
long hl_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr);
void hl_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq);
long hl_get_temperature(struct hl_device *hdev, int sensor_index, u32 attr);
long hl_get_voltage(struct hl_device *hdev, int sensor_index, u32 attr);
long hl_get_current(struct hl_device *hdev, int sensor_index, u32 attr);
long hl_get_fan_speed(struct hl_device *hdev, int sensor_index, u32 attr);
long hl_get_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr);
void hl_set_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr,
			long value);
u64 hl_get_max_power(struct hl_device *hdev);
void hl_set_max_power(struct hl_device *hdev, u64 value);

O
Oded Gabbay 已提交
1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678
#ifdef CONFIG_DEBUG_FS

void hl_debugfs_init(void);
void hl_debugfs_fini(void);
void hl_debugfs_add_device(struct hl_device *hdev);
void hl_debugfs_remove_device(struct hl_device *hdev);
void hl_debugfs_add_file(struct hl_fpriv *hpriv);
void hl_debugfs_remove_file(struct hl_fpriv *hpriv);
void hl_debugfs_add_cb(struct hl_cb *cb);
void hl_debugfs_remove_cb(struct hl_cb *cb);
void hl_debugfs_add_cs(struct hl_cs *cs);
void hl_debugfs_remove_cs(struct hl_cs *cs);
void hl_debugfs_add_job(struct hl_device *hdev, struct hl_cs_job *job);
void hl_debugfs_remove_job(struct hl_device *hdev, struct hl_cs_job *job);
void hl_debugfs_add_userptr(struct hl_device *hdev, struct hl_userptr *userptr);
void hl_debugfs_remove_userptr(struct hl_device *hdev,
				struct hl_userptr *userptr);
void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx);
void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx);

#else

static inline void __init hl_debugfs_init(void)
{
}

static inline void hl_debugfs_fini(void)
{
}

static inline void hl_debugfs_add_device(struct hl_device *hdev)
{
}

static inline void hl_debugfs_remove_device(struct hl_device *hdev)
{
}

static inline void hl_debugfs_add_file(struct hl_fpriv *hpriv)
{
}

static inline void hl_debugfs_remove_file(struct hl_fpriv *hpriv)
{
}

static inline void hl_debugfs_add_cb(struct hl_cb *cb)
{
}

static inline void hl_debugfs_remove_cb(struct hl_cb *cb)
{
}

static inline void hl_debugfs_add_cs(struct hl_cs *cs)
{
}

static inline void hl_debugfs_remove_cs(struct hl_cs *cs)
{
}

static inline void hl_debugfs_add_job(struct hl_device *hdev,
					struct hl_cs_job *job)
{
}

static inline void hl_debugfs_remove_job(struct hl_device *hdev,
					struct hl_cs_job *job)
{
}

static inline void hl_debugfs_add_userptr(struct hl_device *hdev,
					struct hl_userptr *userptr)
{
}

static inline void hl_debugfs_remove_userptr(struct hl_device *hdev,
					struct hl_userptr *userptr)
{
}

static inline void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev,
					struct hl_ctx *ctx)
{
}

static inline void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev,
					struct hl_ctx *ctx)
{
}

#endif

1679 1680
/* IOCTLs */
long hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg);
1681
long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg);
1682
int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data);
1683 1684
int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data);
int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data);
1685
int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data);
1686

O
Oded Gabbay 已提交
1687
#endif /* HABANALABSP_H_ */