tlb_uv.c 20.3 KB
Newer Older
1 2 3 4 5 6 7 8
/*
 *	SGI UltraViolet TLB flush routines.
 *
 *	(c) 2008 Cliff Wickman <cpw@sgi.com>, SGI.
 *
 *	This code is released under the GNU General Public License version 2 or
 *	later.
 */
9
#include <linux/seq_file.h>
10 11 12 13 14
#include <linux/proc_fs.h>
#include <linux/kernel.h>

#include <asm/mmu_context.h>
#include <asm/uv/uv_mmrs.h>
15
#include <asm/uv/uv_hub.h>
16
#include <asm/uv/uv_bau.h>
17 18
#include <asm/genapic.h>
#include <asm/idle.h>
19
#include <asm/tsc.h>
20
#include <asm/irq_vectors.h>
21

22 23
#include <mach_apic.h>

24 25 26 27 28 29 30
static struct bau_control	**uv_bau_table_bases __read_mostly;
static int			uv_bau_retry_limit __read_mostly;

/* position of pnode (which is nasid>>1): */
static int			uv_nshift __read_mostly;

static unsigned long		uv_mmask __read_mostly;
31

32 33
static DEFINE_PER_CPU(struct ptc_stats, ptcstats);
static DEFINE_PER_CPU(struct bau_control, bau_control);
34 35 36 37 38 39 40 41 42

/*
 * Free a software acknowledge hardware resource by clearing its Pending
 * bit. This will return a reply to the sender.
 * If the message has timed out, a reply has already been sent by the
 * hardware but the resource has not been released. In that case our
 * clear of the Timeout bit (as well) will free the resource. No reply will
 * be sent (the hardware will only do one reply per message).
 */
43
static void uv_reply_to_message(int resource,
44 45
				struct bau_payload_queue_entry *msg,
				struct bau_msg_status *msp)
46
{
47
	unsigned long dw;
48

49
	dw = (1 << (resource + UV_SW_ACK_NPENDING)) | (1 << resource);
50 51 52 53
	msg->replied_to = 1;
	msg->sw_ack_vector = 0;
	if (msp)
		msp->seen_by.bits = 0;
54
	uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, dw);
55 56 57 58 59 60
}

/*
 * Do all the things a cpu should do for a TLB shootdown message.
 * Other cpu's may come here at the same time for this message.
 */
61
static void uv_bau_process_message(struct bau_payload_queue_entry *msg,
62
				   int msg_slot, int sw_ack_slot)
63 64 65
{
	unsigned long this_cpu_mask;
	struct bau_msg_status *msp;
66
	int cpu;
67 68 69 70 71

	msp = __get_cpu_var(bau_control).msg_statuses + msg_slot;
	cpu = uv_blade_processor_id();
	msg->number_of_cpus =
	    uv_blade_nr_online_cpus(uv_node_to_blade_id(numa_node_id()));
72
	this_cpu_mask = 1UL << cpu;
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95
	if (msp->seen_by.bits & this_cpu_mask)
		return;
	atomic_or_long(&msp->seen_by.bits, this_cpu_mask);

	if (msg->replied_to == 1)
		return;

	if (msg->address == TLB_FLUSH_ALL) {
		local_flush_tlb();
		__get_cpu_var(ptcstats).alltlb++;
	} else {
		__flush_tlb_one(msg->address);
		__get_cpu_var(ptcstats).onetlb++;
	}

	__get_cpu_var(ptcstats).requestee++;

	atomic_inc_short(&msg->acknowledge_count);
	if (msg->number_of_cpus == msg->acknowledge_count)
		uv_reply_to_message(sw_ack_slot, msg, msp);
}

/*
96
 * Examine the payload queue on one distribution node to see
97 98 99 100
 * which messages have not been seen, and which cpu(s) have not seen them.
 *
 * Returns the number of cpu's that have not responded.
 */
101
static int uv_examine_destination(struct bau_control *bau_tablesp, int sender)
102 103 104
{
	struct bau_payload_queue_entry *msg;
	struct bau_msg_status *msp;
105 106 107
	int count = 0;
	int i;
	int j;
108

109 110 111 112 113 114 115 116 117
	for (msg = bau_tablesp->va_queue_first, i = 0; i < DEST_Q_SIZE;
	     msg++, i++) {
		if ((msg->sending_cpu == sender) && (!msg->replied_to)) {
			msp = bau_tablesp->msg_statuses + i;
			printk(KERN_DEBUG
			       "blade %d: address:%#lx %d of %d, not cpu(s): ",
			       i, msg->address, msg->acknowledge_count,
			       msg->number_of_cpus);
			for (j = 0; j < msg->number_of_cpus; j++) {
118
				if (!((1L << j) & msp->seen_by.bits)) {
119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
					count++;
					printk("%d ", j);
				}
			}
			printk("\n");
		}
	}
	return count;
}

/*
 * Examine the payload queue on all the distribution nodes to see
 * which messages have not been seen, and which cpu(s) have not seen them.
 *
 * Returns the number of cpu's that have not responded.
 */
static int uv_examine_destinations(struct bau_target_nodemask *distribution)
{
	int sender;
	int i;
	int count = 0;

141
	sender = smp_processor_id();
142
	for (i = 0; i < sizeof(struct bau_target_nodemask) * BITSPERBYTE; i++) {
143 144
		if (!bau_node_isset(i, distribution))
			continue;
145
		count += uv_examine_destination(uv_bau_table_bases[i], sender);
146 147 148 149
	}
	return count;
}

150 151 152 153 154
/*
 * wait for completion of a broadcast message
 *
 * return COMPLETE, RETRY or GIVEUP
 */
155
static int uv_wait_completion(struct bau_desc *bau_desc,
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
			      unsigned long mmr_offset, int right_shift)
{
	int exams = 0;
	long destination_timeouts = 0;
	long source_timeouts = 0;
	unsigned long descriptor_status;

	while ((descriptor_status = (((unsigned long)
		uv_read_local_mmr(mmr_offset) >>
			right_shift) & UV_ACT_STATUS_MASK)) !=
			DESC_STATUS_IDLE) {
		if (descriptor_status == DESC_STATUS_SOURCE_TIMEOUT) {
			source_timeouts++;
			if (source_timeouts > SOURCE_TIMEOUT_LIMIT)
				source_timeouts = 0;
			__get_cpu_var(ptcstats).s_retry++;
			return FLUSH_RETRY;
		}
		/*
		 * spin here looking for progress at the destinations
		 */
		if (descriptor_status == DESC_STATUS_DESTINATION_TIMEOUT) {
			destination_timeouts++;
			if (destination_timeouts > DESTINATION_TIMEOUT_LIMIT) {
				/*
				 * returns number of cpus not responding
				 */
				if (uv_examine_destinations
				    (&bau_desc->distribution) == 0) {
					__get_cpu_var(ptcstats).d_retry++;
					return FLUSH_RETRY;
				}
				exams++;
				if (exams >= uv_bau_retry_limit) {
					printk(KERN_DEBUG
					       "uv_flush_tlb_others");
					printk("giving up on cpu %d\n",
					       smp_processor_id());
					return FLUSH_GIVEUP;
				}
				/*
				 * delays can hang the simulator
				   udelay(1000);
				 */
				destination_timeouts = 0;
			}
		}
	}
	return FLUSH_COMPLETE;
}

/**
 * uv_flush_send_and_wait
 *
 * Send a broadcast and wait for a broadcast message to complete.
 *
 * The cpumaskp mask contains the cpus the broadcast was sent to.
 *
 * Returns 1 if all remote flushing was done. The mask is zeroed.
 * Returns 0 if some remote flushing remains to be done. The mask is left
 * unchanged.
 */
218 219
int uv_flush_send_and_wait(int cpu, int this_blade, struct bau_desc *bau_desc,
			   cpumask_t *cpumaskp)
220 221 222 223
{
	int completion_status = 0;
	int right_shift;
	int tries = 0;
224 225
	int blade;
	int bit;
226
	unsigned long mmr_offset;
227
	unsigned long index;
228 229 230 231 232 233 234 235 236 237 238 239 240 241
	cycles_t time1;
	cycles_t time2;

	if (cpu < UV_CPUS_PER_ACT_STATUS) {
		mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_0;
		right_shift = cpu * UV_ACT_STATUS_SIZE;
	} else {
		mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_1;
		right_shift =
		    ((cpu - UV_CPUS_PER_ACT_STATUS) * UV_ACT_STATUS_SIZE);
	}
	time1 = get_cycles();
	do {
		tries++;
242 243
		index = (1UL << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) |
			cpu;
244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276
		uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index);
		completion_status = uv_wait_completion(bau_desc, mmr_offset,
					right_shift);
	} while (completion_status == FLUSH_RETRY);
	time2 = get_cycles();
	__get_cpu_var(ptcstats).sflush += (time2 - time1);
	if (tries > 1)
		__get_cpu_var(ptcstats).retriesok++;

	if (completion_status == FLUSH_GIVEUP) {
		/*
		 * Cause the caller to do an IPI-style TLB shootdown on
		 * the cpu's, all of which are still in the mask.
		 */
		__get_cpu_var(ptcstats).ptc_i++;
		return 0;
	}

	/*
	 * Success, so clear the remote cpu's from the mask so we don't
	 * use the IPI method of shootdown on them.
	 */
	for_each_cpu_mask(bit, *cpumaskp) {
		blade = uv_cpu_to_blade_id(bit);
		if (blade == this_blade)
			continue;
		cpu_clear(bit, *cpumaskp);
	}
	if (!cpus_empty(*cpumaskp))
		return 0;
	return 1;
}

277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
/**
 * uv_flush_tlb_others - globally purge translation cache of a virtual
 * address or all TLB's
 * @cpumaskp: mask of all cpu's in which the address is to be removed
 * @mm: mm_struct containing virtual address range
 * @va: virtual address to be removed (or TLB_FLUSH_ALL for all TLB's on cpu)
 *
 * This is the entry point for initiating any UV global TLB shootdown.
 *
 * Purges the translation caches of all specified processors of the given
 * virtual address, or purges all TLB's on specified processors.
 *
 * The caller has derived the cpumaskp from the mm_struct and has subtracted
 * the local cpu from the mask.  This function is called only if there
 * are bits set in the mask. (e.g. flush_tlb_page())
 *
 * The cpumaskp is converted into a nodemask of the nodes containing
 * the cpus.
295 296 297
 *
 * Returns 1 if all remote flushing was done.
 * Returns 0 if some remote flushing remains to be done.
298
 */
299
int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm,
300
			unsigned long va)
301 302
{
	int i;
303
	int bit;
304 305 306
	int blade;
	int cpu;
	int this_blade;
307
	int locals = 0;
308
	struct bau_desc *bau_desc;
309 310 311 312

	cpu = uv_blade_processor_id();
	this_blade = uv_numa_blade_id();
	bau_desc = __get_cpu_var(bau_control).descriptor_base;
313
	bau_desc += UV_ITEMS_PER_DESCRIPTOR * cpu;
314 315 316 317 318 319

	bau_nodes_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE);

	i = 0;
	for_each_cpu_mask(bit, *cpumaskp) {
		blade = uv_cpu_to_blade_id(bit);
320
		BUG_ON(blade > (UV_DISTRIBUTION_SIZE - 1));
321 322
		if (blade == this_blade) {
			locals++;
323
			continue;
324
		}
325 326 327
		bau_node_set(blade, &bau_desc->distribution);
		i++;
	}
328 329 330 331 332 333 334 335 336
	if (i == 0) {
		/*
		 * no off_node flushing; return status for local node
		 */
		if (locals)
			return 0;
		else
			return 1;
	}
337 338 339 340 341 342
	__get_cpu_var(ptcstats).requestor++;
	__get_cpu_var(ptcstats).ntargeted += i;

	bau_desc->payload.address = va;
	bau_desc->payload.sending_cpu = smp_processor_id();

343
	return uv_flush_send_and_wait(cpu, this_blade, bau_desc, cpumaskp);
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359
}

/*
 * The BAU message interrupt comes here. (registered by set_intr_gate)
 * See entry_64.S
 *
 * We received a broadcast assist message.
 *
 * Interrupts may have been disabled; this interrupt could represent
 * the receipt of several messages.
 *
 * All cores/threads on this node get this interrupt.
 * The last one to see it does the s/w ack.
 * (the resource will not be freed until noninterruptable cpus see this
 *  interrupt; hardware will timeout the s/w ack and reply ERROR)
 */
360
void uv_bau_message_interrupt(struct pt_regs *regs)
361
{
362 363
	struct bau_payload_queue_entry *va_queue_first;
	struct bau_payload_queue_entry *va_queue_last;
364
	struct bau_payload_queue_entry *msg;
365
	struct pt_regs *old_regs = set_irq_regs(regs);
366 367
	cycles_t time1;
	cycles_t time2;
368 369 370 371 372 373 374 375 376 377
	int msg_slot;
	int sw_ack_slot;
	int fw;
	int count = 0;
	unsigned long local_pnode;

	ack_APIC_irq();
	exit_idle();
	irq_enter();

378
	time1 = get_cycles();
379 380 381

	local_pnode = uv_blade_to_pnode(uv_numa_blade_id());

382
	va_queue_first = __get_cpu_var(bau_control).va_queue_first;
383
	va_queue_last = __get_cpu_var(bau_control).va_queue_last;
384

385 386 387 388
	msg = __get_cpu_var(bau_control).bau_msg_head;
	while (msg->sw_ack_vector) {
		count++;
		fw = msg->sw_ack_vector;
389
		msg_slot = msg - va_queue_first;
390 391 392 393 394
		sw_ack_slot = ffs(fw) - 1;

		uv_bau_process_message(msg, msg_slot, sw_ack_slot);

		msg++;
395 396
		if (msg > va_queue_last)
			msg = va_queue_first;
397 398 399 400 401 402 403
		__get_cpu_var(bau_control).bau_msg_head = msg;
	}
	if (!count)
		__get_cpu_var(ptcstats).nomsg++;
	else if (count > 1)
		__get_cpu_var(ptcstats).multmsg++;

404 405
	time2 = get_cycles();
	__get_cpu_var(ptcstats).dflush += (time2 - time1);
406 407 408 409 410

	irq_exit();
	set_irq_regs(old_regs);
}

411
static void uv_enable_timeouts(void)
412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431
{
	int i;
	int blade;
	int last_blade;
	int pnode;
	int cur_cpu = 0;
	unsigned long apicid;

	last_blade = -1;
	for_each_online_node(i) {
		blade = uv_node_to_blade_id(i);
		if (blade == last_blade)
			continue;
		last_blade = blade;
		apicid = per_cpu(x86_cpu_to_apicid, cur_cpu);
		pnode = uv_blade_to_pnode(blade);
		cur_cpu += uv_blade_nr_possible_cpus(i);
	}
}

432
static void *uv_ptc_seq_start(struct seq_file *file, loff_t *offset)
433 434 435 436 437 438
{
	if (*offset < num_possible_cpus())
		return offset;
	return NULL;
}

439
static void *uv_ptc_seq_next(struct seq_file *file, void *data, loff_t *offset)
440 441 442 443 444 445 446
{
	(*offset)++;
	if (*offset < num_possible_cpus())
		return offset;
	return NULL;
}

447
static void uv_ptc_seq_stop(struct seq_file *file, void *data)
448 449 450 451 452 453 454
{
}

/*
 * Display the statistics thru /proc
 * data points to the cpu number
 */
455
static int uv_ptc_seq_show(struct seq_file *file, void *data)
456 457 458 459 460 461 462 463 464 465
{
	struct ptc_stats *stat;
	int cpu;

	cpu = *(loff_t *)data;

	if (!cpu) {
		seq_printf(file,
		"# cpu requestor requestee one all sretry dretry ptc_i ");
		seq_printf(file,
466
		"sw_ack sflush dflush sok dnomsg dmult starget\n");
467 468 469 470 471 472 473 474 475 476 477
	}
	if (cpu < num_possible_cpus() && cpu_online(cpu)) {
		stat = &per_cpu(ptcstats, cpu);
		seq_printf(file, "cpu %d %ld %ld %ld %ld %ld %ld %ld ",
			   cpu, stat->requestor,
			   stat->requestee, stat->onetlb, stat->alltlb,
			   stat->s_retry, stat->d_retry, stat->ptc_i);
		seq_printf(file, "%lx %ld %ld %ld %ld %ld %ld\n",
			   uv_read_global_mmr64(uv_blade_to_pnode
					(uv_cpu_to_blade_id(cpu)),
					UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE),
478
			   stat->sflush, stat->dflush,
479 480 481 482 483 484 485 486 487 488 489
			   stat->retriesok, stat->nomsg,
			   stat->multmsg, stat->ntargeted);
	}

	return 0;
}

/*
 *  0: display meaning of the statistics
 * >0: retry limit
 */
490
static ssize_t uv_ptc_proc_write(struct file *file, const char __user *user,
491
				 size_t count, loff_t *data)
492 493 494 495
{
	long newmode;
	char optstr[64];

496
	if (count == 0 || count > sizeof(optstr))
497
		return -EINVAL;
498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
	if (copy_from_user(optstr, user, count))
		return -EFAULT;
	optstr[count - 1] = '\0';
	if (strict_strtoul(optstr, 10, &newmode) < 0) {
		printk(KERN_DEBUG "%s is invalid\n", optstr);
		return -EINVAL;
	}

	if (newmode == 0) {
		printk(KERN_DEBUG "# cpu:      cpu number\n");
		printk(KERN_DEBUG
		"requestor:  times this cpu was the flush requestor\n");
		printk(KERN_DEBUG
		"requestee:  times this cpu was requested to flush its TLBs\n");
		printk(KERN_DEBUG
		"one:        times requested to flush a single address\n");
		printk(KERN_DEBUG
		"all:        times requested to flush all TLB's\n");
		printk(KERN_DEBUG
		"sretry:     number of retries of source-side timeouts\n");
		printk(KERN_DEBUG
		"dretry:     number of retries of destination-side timeouts\n");
		printk(KERN_DEBUG
		"ptc_i:      times UV fell through to IPI-style flushes\n");
		printk(KERN_DEBUG
		"sw_ack:     image of UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE\n");
		printk(KERN_DEBUG
525
		"sflush_us:  cycles spent in uv_flush_tlb_others()\n");
526
		printk(KERN_DEBUG
527
		"dflush_us:  cycles spent in handling flush requests\n");
528 529 530 531 532 533 534 535 536 537 538 539 540 541 542
		printk(KERN_DEBUG "sok:        successes on retry\n");
		printk(KERN_DEBUG "dnomsg:     interrupts with no message\n");
		printk(KERN_DEBUG
		"dmult:      interrupts with multiple messages\n");
		printk(KERN_DEBUG "starget:    nodes targeted\n");
	} else {
		uv_bau_retry_limit = newmode;
		printk(KERN_DEBUG "timeout retry limit:%d\n",
		       uv_bau_retry_limit);
	}

	return count;
}

static const struct seq_operations uv_ptc_seq_ops = {
543 544 545 546
	.start		= uv_ptc_seq_start,
	.next		= uv_ptc_seq_next,
	.stop		= uv_ptc_seq_stop,
	.show		= uv_ptc_seq_show
547 548
};

549
static int uv_ptc_proc_open(struct inode *inode, struct file *file)
550 551 552 553 554
{
	return seq_open(file, &uv_ptc_seq_ops);
}

static const struct file_operations proc_uv_ptc_operations = {
555 556 557 558 559
	.open		= uv_ptc_proc_open,
	.read		= seq_read,
	.write		= uv_ptc_proc_write,
	.llseek		= seq_lseek,
	.release	= seq_release,
560 561
};

562
static int __init uv_ptc_init(void)
563
{
564
	struct proc_dir_entry *proc_uv_ptc;
565 566 567 568 569 570 571 572 573 574 575 576 577 578

	if (!is_uv_system())
		return 0;

	proc_uv_ptc = create_proc_entry(UV_PTC_BASENAME, 0444, NULL);
	if (!proc_uv_ptc) {
		printk(KERN_ERR "unable to create %s proc entry\n",
		       UV_PTC_BASENAME);
		return -EINVAL;
	}
	proc_uv_ptc->proc_fops = &proc_uv_ptc_operations;
	return 0;
}

579 580 581 582
/*
 * begin the initialization of the per-blade control structures
 */
static struct bau_control * __init uv_table_bases_init(int blade, int node)
583
{
584 585
	int i;
	struct bau_msg_status *msp;
586
	struct bau_control *bau_tabp;
587

588
	bau_tabp =
589
	    kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, node);
590
	BUG_ON(!bau_tabp);
591

592
	bau_tabp->msg_statuses =
593
	    kmalloc_node(sizeof(struct bau_msg_status) *
594 595
			 DEST_Q_SIZE, GFP_KERNEL, node);
	BUG_ON(!bau_tabp->msg_statuses);
596

597
	for (i = 0, msp = bau_tabp->msg_statuses; i < DEST_Q_SIZE; i++, msp++)
598 599
		bau_cpubits_clear(&msp->seen_by, (int)
				  uv_blade_nr_possible_cpus(blade));
600

601
	uv_bau_table_bases[blade] = bau_tabp;
602

603
	return bau_tabp;
604 605
}

606 607 608
/*
 * finish the initialization of the per-blade control structures
 */
609 610 611 612
static void __init
uv_table_bases_finish(int blade, int node, int cur_cpu,
		      struct bau_control *bau_tablesp,
		      struct bau_desc *adp)
613 614
{
	struct bau_control *bcp;
615
	int i;
616

617
	for (i = cur_cpu; i < cur_cpu + uv_blade_nr_possible_cpus(blade); i++) {
618
		bcp = (struct bau_control *)&per_cpu(bau_control, i);
619 620 621 622 623 624

		bcp->bau_msg_head	= bau_tablesp->va_queue_first;
		bcp->va_queue_first	= bau_tablesp->va_queue_first;
		bcp->va_queue_last	= bau_tablesp->va_queue_last;
		bcp->msg_statuses	= bau_tablesp->msg_statuses;
		bcp->descriptor_base	= adp;
625 626
	}
}
627 628

/*
629
 * initialize the sending side's sending buffers
630
 */
631
static struct bau_desc * __init
632
uv_activation_descriptor_init(int node, int pnode)
633 634 635 636
{
	int i;
	unsigned long pa;
	unsigned long m;
637
	unsigned long n;
638
	unsigned long mmr_image;
639 640
	struct bau_desc *adp;
	struct bau_desc *ad2;
641

642
	adp = (struct bau_desc *)
643
	    kmalloc_node(16384, GFP_KERNEL, node);
644
	BUG_ON(!adp);
645

646 647 648
	pa = __pa((unsigned long)adp);
	n = pa >> uv_nshift;
	m = pa & uv_mmask;
649

650
	mmr_image = uv_read_global_mmr64(pnode, UVH_LB_BAU_SB_DESCRIPTOR_BASE);
651
	if (mmr_image) {
652 653 654
		uv_write_global_mmr64(pnode, (unsigned long)
				      UVH_LB_BAU_SB_DESCRIPTOR_BASE,
				      (n << UV_DESC_BASE_PNODE_SHIFT | m));
655 656
	}

657
	for (i = 0, ad2 = adp; i < UV_ACTIVATION_DESCRIPTOR_SIZE; i++, ad2++) {
658
		memset(ad2, 0, sizeof(struct bau_desc));
659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674
		ad2->header.sw_ack_flag = 1;
		ad2->header.base_dest_nodeid =
		    uv_blade_to_pnode(uv_cpu_to_blade_id(0));
		ad2->header.command = UV_NET_ENDPOINT_INTD;
		ad2->header.int_both = 1;
		/*
		 * all others need to be set to zero:
		 *   fairness chaining multilevel count replied_to
		 */
	}
	return adp;
}

/*
 * initialize the destination side's receiving buffers
 */
675 676
static struct bau_payload_queue_entry * __init
uv_payload_queue_init(int node, int pnode, struct bau_control *bau_tablesp)
677
{
678
	struct bau_payload_queue_entry *pqp;
679
	char *cp;
680

681 682 683 684
	pqp = (struct bau_payload_queue_entry *) kmalloc_node(
		(DEST_Q_SIZE + 1) * sizeof(struct bau_payload_queue_entry),
		GFP_KERNEL, node);
	BUG_ON(!pqp);
685

686 687 688 689 690 691 692 693 694 695
	cp = (char *)pqp + 31;
	pqp = (struct bau_payload_queue_entry *)(((unsigned long)cp >> 5) << 5);
	bau_tablesp->va_queue_first = pqp;
	uv_write_global_mmr64(pnode,
			      UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST,
			      ((unsigned long)pnode <<
			       UV_PAYLOADQ_PNODE_SHIFT) |
			      uv_physnodeaddr(pqp));
	uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL,
			      uv_physnodeaddr(pqp));
696
	bau_tablesp->va_queue_last = pqp + (DEST_Q_SIZE - 1);
697 698 699
	uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST,
			      (unsigned long)
			      uv_physnodeaddr(bau_tablesp->va_queue_last));
700
	memset(pqp, 0, sizeof(struct bau_payload_queue_entry) * DEST_Q_SIZE);
701

702 703
	return pqp;
}
704

705 706 707 708 709 710 711 712
/*
 * Initialization of each UV blade's structures
 */
static int __init uv_init_blade(int blade, int node, int cur_cpu)
{
	int pnode;
	unsigned long pa;
	unsigned long apicid;
713
	struct bau_desc *adp;
714 715
	struct bau_payload_queue_entry *pqp;
	struct bau_control *bau_tablesp;
716

717 718 719 720 721 722 723 724 725 726 727 728 729 730
	bau_tablesp = uv_table_bases_init(blade, node);
	pnode = uv_blade_to_pnode(blade);
	adp = uv_activation_descriptor_init(node, pnode);
	pqp = uv_payload_queue_init(node, pnode, bau_tablesp);
	uv_table_bases_finish(blade, node, cur_cpu, bau_tablesp, adp);
	/*
	 * the below initialization can't be in firmware because the
	 * messaging IRQ will be determined by the OS
	 */
	apicid = per_cpu(x86_cpu_to_apicid, cur_cpu);
	pa = uv_read_global_mmr64(pnode, UVH_BAU_DATA_CONFIG);
	if ((pa & 0xff) != UV_BAU_MESSAGE) {
		uv_write_global_mmr64(pnode, UVH_BAU_DATA_CONFIG,
				      ((apicid << 32) | UV_BAU_MESSAGE));
731
	}
732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
	return 0;
}

/*
 * Initialization of BAU-related structures
 */
static int __init uv_bau_init(void)
{
	int blade;
	int node;
	int nblades;
	int last_blade;
	int cur_cpu = 0;

	if (!is_uv_system())
		return 0;
748

749
	uv_bau_retry_limit = 1;
750
	uv_nshift = uv_hub_info->n_val;
751
	uv_mmask = (1UL << uv_hub_info->n_val) - 1;
752 753
	nblades = 0;
	last_blade = -1;
754 755
	for_each_online_node(node) {
		blade = uv_node_to_blade_id(node);
756 757 758 759 760 761 762
		if (blade == last_blade)
			continue;
		last_blade = blade;
		nblades++;
	}
	uv_bau_table_bases = (struct bau_control **)
	    kmalloc(nblades * sizeof(struct bau_control *), GFP_KERNEL);
763
	BUG_ON(!uv_bau_table_bases);
764

765
	last_blade = -1;
766 767
	for_each_online_node(node) {
		blade = uv_node_to_blade_id(node);
768 769 770
		if (blade == last_blade)
			continue;
		last_blade = blade;
771 772
		uv_init_blade(blade, node, cur_cpu);
		cur_cpu += uv_blade_nr_possible_cpus(blade);
773
	}
774
	alloc_intr_gate(UV_BAU_MESSAGE, uv_bau_message_intr1);
775
	uv_enable_timeouts();
776

777 778 779
	return 0;
}
__initcall(uv_bau_init);
780
__initcall(uv_ptc_init);