spu_task_sync.c

/*
 * Cell Broadband Engine OProfile Support
 *
 * (C) Copyright IBM Corporation 2006
 *
 * Author: Maynard Johnson <maynardj@us.ibm.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

/* The purpose of this file is to handle SPU event task switching
 * and to record SPU context information into the OProfile
 * event buffer.
 *
 * Additionally, the spu_sync_buffer function is provided as a helper
 * for recoding actual SPU program counter samples to the event buffer.
 */
#include <linux/dcookies.h>
#include <linux/kref.h>
#include <linux/mm.h>
#include <linux/fs.h>
#include <linux/module.h>
#include <linux/notifier.h>
#include <linux/numa.h>
#include <linux/oprofile.h>
#include <linux/spinlock.h>
#include "pr_util.h"

#define RELEASE_ALL 9999

static DEFINE_SPINLOCK(buffer_lock);
static DEFINE_SPINLOCK(cache_lock);
static int num_spu_nodes;
int spu_prof_num_nodes;
int last_guard_val[MAX_NUMNODES * 8];

/* Container for caching information about an active SPU task. */
struct cached_info {
	struct vma_to_fileoffset_map *map;
	struct spu *the_spu;	/* needed to access pointer to local_store */
	struct kref cache_ref;
};

static struct cached_info *spu_info[MAX_NUMNODES * 8];

static void destroy_cached_info(struct kref *kref)
{
	struct cached_info *info;

	info = container_of(kref, struct cached_info, cache_ref);
	vma_map_free(info->map);
	kfree(info);
	module_put(THIS_MODULE);
}

/* Return the cached_info for the passed SPU number.
 * ATTENTION:  Callers are responsible for obtaining the
 *	       cache_lock if needed prior to invoking this function.
 */
static struct cached_info *get_cached_info(struct spu *the_spu, int spu_num)
{
	struct kref *ref;
	struct cached_info *ret_info;

	if (spu_num >= num_spu_nodes) {
		printk(KERN_ERR "SPU_PROF: "
		       "%s, line %d: Invalid index %d into spu info cache\n",
		       __func__, __LINE__, spu_num);
		ret_info = NULL;
		goto out;
	}
	if (!spu_info[spu_num] && the_spu) {
		ref = spu_get_profile_private_kref(the_spu->ctx);
		if (ref) {
			spu_info[spu_num] = container_of(ref, struct cached_info, cache_ref);
			kref_get(&spu_info[spu_num]->cache_ref);
		}
	}

	ret_info = spu_info[spu_num];
 out:
	return ret_info;
}


/* Looks for cached info for the passed spu.  If not found, the
 * cached info is created for the passed spu.
 * Returns 0 for success; otherwise, -1 for error.
 */
static int
prepare_cached_spu_info(struct spu *spu, unsigned long objectId)
{
	unsigned long flags;
	struct vma_to_fileoffset_map *new_map;
	int retval = 0;
	struct cached_info *info;

	/* We won't bother getting cache_lock here since
	 * don't do anything with the cached_info that's returned.
	 */
	info = get_cached_info(spu, spu->number);

	if (info) {
		pr_debug("Found cached SPU info.\n");
		goto out;
	}

	/* Create cached_info and set spu_info[spu->number] to point to it.
	 * spu->number is a system-wide value, not a per-node value.
	 */
	info = kzalloc(sizeof(struct cached_info), GFP_KERNEL);
	if (!info) {
		printk(KERN_ERR "SPU_PROF: "
		       "%s, line %d: create vma_map failed\n",
		       __func__, __LINE__);
		retval = -ENOMEM;
		goto err_alloc;
	}
	new_map = create_vma_map(spu, objectId);
	if (!new_map) {
		printk(KERN_ERR "SPU_PROF: "
		       "%s, line %d: create vma_map failed\n",
		       __func__, __LINE__);
		retval = -ENOMEM;
		goto err_alloc;
	}

	pr_debug("Created vma_map\n");
	info->map = new_map;
	info->the_spu = spu;
	kref_init(&info->cache_ref);
	spin_lock_irqsave(&cache_lock, flags);
	spu_info[spu->number] = info;
	/* Increment count before passing off ref to SPUFS. */
	kref_get(&info->cache_ref);

	/* We increment the module refcount here since SPUFS is
	 * responsible for the final destruction of the cached_info,
	 * and it must be able to access the destroy_cached_info()
	 * function defined in the OProfile module.  We decrement
	 * the module refcount in destroy_cached_info.
	 */
	try_module_get(THIS_MODULE);
	spu_set_profile_private_kref(spu->ctx, &info->cache_ref,
				destroy_cached_info);
	spin_unlock_irqrestore(&cache_lock, flags);
	goto out;

err_alloc:
	kfree(info);
out:
	return retval;
}

/*
 * NOTE:  The caller is responsible for locking the
 *	  cache_lock prior to calling this function.
 */
static int release_cached_info(int spu_index)
{
	int index, end;

	if (spu_index == RELEASE_ALL) {
		end = num_spu_nodes;
		index = 0;
	} else {
		if (spu_index >= num_spu_nodes) {
			printk(KERN_ERR "SPU_PROF: "
				"%s, line %d: "
				"Invalid index %d into spu info cache\n",
				__func__, __LINE__, spu_index);
			goto out;
		}
		end = spu_index + 1;
		index = spu_index;
	}
	for (; index < end; index++) {
		if (spu_info[index]) {
			kref_put(&spu_info[index]->cache_ref,
				 destroy_cached_info);
			spu_info[index] = NULL;
		}
	}

out:
	return 0;
}

/* The source code for fast_get_dcookie was "borrowed"
 * from drivers/oprofile/buffer_sync.c.
 */

/* Optimisation. We can manage without taking the dcookie sem
 * because we cannot reach this code without at least one
 * dcookie user still being registered (namely, the reader
 * of the event buffer).
 */
static inline unsigned long fast_get_dcookie(struct path *path)
{
	unsigned long cookie;

	if (path->dentry->d_cookie)
		return (unsigned long)path->dentry;
	get_dcookie(path, &cookie);
	return cookie;
}

/* Look up the dcookie for the task's first VM_EXECUTABLE mapping,
 * which corresponds loosely to "application name". Also, determine
 * the offset for the SPU ELF object.  If computed offset is
 * non-zero, it implies an embedded SPU object; otherwise, it's a
 * separate SPU binary, in which case we retrieve it's dcookie.
 * For the embedded case, we must determine if SPU ELF is embedded
 * in the executable application or another file (i.e., shared lib).
 * If embedded in a shared lib, we must get the dcookie and return
 * that to the caller.
 */
static unsigned long
get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp,
			    unsigned long *spu_bin_dcookie,
			    unsigned long spu_ref)
{
	unsigned long app_cookie = 0;
	unsigned int my_offset = 0;
	struct file *app = NULL;
	struct vm_area_struct *vma;
	struct mm_struct *mm = spu->mm;

	if (!mm)
		goto out;

	down_read(&mm->mmap_sem);

	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		if (!vma->vm_file)
			continue;
		if (!(vma->vm_flags & VM_EXECUTABLE))
			continue;
		app_cookie = fast_get_dcookie(&vma->vm_file->f_path);
		pr_debug("got dcookie for %s\n",
			 vma->vm_file->f_dentry->d_name.name);
		app = vma->vm_file;
		break;
	}

	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		if (vma->vm_start > spu_ref || vma->vm_end <= spu_ref)
			continue;
		my_offset = spu_ref - vma->vm_start;
		if (!vma->vm_file)
			goto fail_no_image_cookie;

		pr_debug("Found spu ELF at %X(object-id:%lx) for file %s\n",
			 my_offset, spu_ref,
			 vma->vm_file->f_dentry->d_name.name);
		*offsetp = my_offset;
		break;
	}

	*spu_bin_dcookie = fast_get_dcookie(&vma->vm_file->f_path);
	pr_debug("got dcookie for %s\n", vma->vm_file->f_dentry->d_name.name);

	up_read(&mm->mmap_sem);

out:
	return app_cookie;

fail_no_image_cookie:
	up_read(&mm->mmap_sem);

	printk(KERN_ERR "SPU_PROF: "
		"%s, line %d: Cannot find dcookie for SPU binary\n",
		__func__, __LINE__);
	goto out;
}



/* This function finds or creates cached context information for the
 * passed SPU and records SPU context information into the OProfile
 * event buffer.
 */
static int process_context_switch(struct spu *spu, unsigned long objectId)
{
	unsigned long flags;
	int retval;
	unsigned int offset = 0;
	unsigned long spu_cookie = 0, app_dcookie;

	retval = prepare_cached_spu_info(spu, objectId);
	if (retval)
		goto out;

	/* Get dcookie first because a mutex_lock is taken in that
	 * code path, so interrupts must not be disabled.
	 */
	app_dcookie = get_exec_dcookie_and_offset(spu, &offset, &spu_cookie, objectId);
	if (!app_dcookie || !spu_cookie) {
		retval  = -ENOENT;
		goto out;
	}

	/* Record context info in event buffer */
	spin_lock_irqsave(&buffer_lock, flags);
	add_event_entry(ESCAPE_CODE);
	add_event_entry(SPU_CTX_SWITCH_CODE);
	add_event_entry(spu->number);
	add_event_entry(spu->pid);
	add_event_entry(spu->tgid);
	add_event_entry(app_dcookie);
	add_event_entry(spu_cookie);
	add_event_entry(offset);
	spin_unlock_irqrestore(&buffer_lock, flags);
	smp_wmb();	/* insure spu event buffer updates are written */
			/* don't want entries intermingled... */
out:
	return retval;
}

/*
 * This function is invoked on either a bind_context or unbind_context.
 * If called for an unbind_context, the val arg is 0; otherwise,
 * it is the object-id value for the spu context.
 * The data arg is of type 'struct spu *'.
 */
static int spu_active_notify(struct notifier_block *self, unsigned long val,
				void *data)
{
	int retval;
	unsigned long flags;
	struct spu *the_spu = data;

	pr_debug("SPU event notification arrived\n");
	if (!val) {
		spin_lock_irqsave(&cache_lock, flags);
		retval = release_cached_info(the_spu->number);
		spin_unlock_irqrestore(&cache_lock, flags);
	} else {
		retval = process_context_switch(the_spu, val);
	}
	return retval;
}

static struct notifier_block spu_active = {
	.notifier_call = spu_active_notify,
};

static int number_of_online_nodes(void)
{
        u32 cpu; u32 tmp;
        int nodes = 0;
        for_each_online_cpu(cpu) {
                tmp = cbe_cpu_to_node(cpu) + 1;
                if (tmp > nodes)
                        nodes++;
        }
        return nodes;
}

/* The main purpose of this function is to synchronize
 * OProfile with SPUFS by registering to be notified of
 * SPU task switches.
 *
 * NOTE: When profiling SPUs, we must ensure that only
 * spu_sync_start is invoked and not the generic sync_start
 * in drivers/oprofile/oprof.c.	 A return value of
 * SKIP_GENERIC_SYNC or SYNC_START_ERROR will
 * accomplish this.
 */
int spu_sync_start(void)
{
	int k;
	int ret = SKIP_GENERIC_SYNC;
	int register_ret;
	unsigned long flags = 0;

	spu_prof_num_nodes = number_of_online_nodes();
	num_spu_nodes = spu_prof_num_nodes * 8;

	spin_lock_irqsave(&buffer_lock, flags);
	add_event_entry(ESCAPE_CODE);
	add_event_entry(SPU_PROFILING_CODE);
	add_event_entry(num_spu_nodes);
	spin_unlock_irqrestore(&buffer_lock, flags);

	/* Register for SPU events  */
	register_ret = spu_switch_event_register(&spu_active);
	if (register_ret) {
		ret = SYNC_START_ERROR;
		goto out;
	}

	for (k = 0; k < (MAX_NUMNODES * 8); k++)
		last_guard_val[k] = 0;
	pr_debug("spu_sync_start -- running.\n");
out:
	return ret;
}

/* Record SPU program counter samples to the oprofile event buffer. */
void spu_sync_buffer(int spu_num, unsigned int *samples,
		     int num_samples)
{
	unsigned long long file_offset;
	unsigned long flags;
	int i;
	struct vma_to_fileoffset_map *map;
	struct spu *the_spu;
	unsigned long long spu_num_ll = spu_num;
	unsigned long long spu_num_shifted = spu_num_ll << 32;
	struct cached_info *c_info;

	/* We need to obtain the cache_lock here because it's
	 * possible that after getting the cached_info, the SPU job
	 * corresponding to this cached_info may end, thus resulting
	 * in the destruction of the cached_info.
	 */
	spin_lock_irqsave(&cache_lock, flags);
	c_info = get_cached_info(NULL, spu_num);
	if (!c_info) {
		/* This legitimately happens when the SPU task ends before all
		 * samples are recorded.
		 * No big deal -- so we just drop a few samples.
		 */
		pr_debug("SPU_PROF: No cached SPU contex "
			  "for SPU #%d. Dropping samples.\n", spu_num);
		goto out;
	}

	map = c_info->map;
	the_spu = c_info->the_spu;
	spin_lock(&buffer_lock);
	for (i = 0; i < num_samples; i++) {
		unsigned int sample = *(samples+i);
		int grd_val = 0;
		file_offset = 0;
		if (sample == 0)
			continue;
		file_offset = vma_map_lookup( map, sample, the_spu, &grd_val);

		/* If overlays are used by this SPU application, the guard
		 * value is non-zero, indicating which overlay section is in
		 * use.	 We need to discard samples taken during the time
		 * period which an overlay occurs (i.e., guard value changes).
		 */
		if (grd_val && grd_val != last_guard_val[spu_num]) {
			last_guard_val[spu_num] = grd_val;
			/* Drop the rest of the samples. */
			break;
		}

		add_event_entry(file_offset | spu_num_shifted);
	}
	spin_unlock(&buffer_lock);
out:
	spin_unlock_irqrestore(&cache_lock, flags);
}


int spu_sync_stop(void)
{
	unsigned long flags = 0;
	int ret = spu_switch_event_unregister(&spu_active);
	if (ret) {
		printk(KERN_ERR "SPU_PROF: "
			"%s, line %d: spu_switch_event_unregister returned %d\n",
			__func__, __LINE__, ret);
		goto out;
	}

	spin_lock_irqsave(&cache_lock, flags);
	ret = release_cached_info(RELEASE_ALL);
	spin_unlock_irqrestore(&cache_lock, flags);
out:
	pr_debug("spu_sync_stop -- done.\n");
	return ret;
}