diff -uNrp --exclude CVS linux-head/Makefile linux-2.6.x/Makefile
--- linux-head/Makefile	2006-08-28 10:57:59.000000000 +0200
+++ linux-2.6.x/Makefile	2006-10-12 21:58:07.000000000 +0200
@@ -511,6 +511,10 @@ CFLAGS		+= $(call add-align,CONFIG_CC_AL
 CFLAGS		+= $(call add-align,CONFIG_CC_ALIGN_LOOPS,-loops)
 CFLAGS		+= $(call add-align,CONFIG_CC_ALIGN_JUMPS,-jumps)
 
+ifdef CONFIG_IPIPE_TRACE_MCOUNT
+CFLAGS          += -pg
+endif
+
 ifdef CONFIG_FRAME_POINTER
 CFLAGS		+= -fno-omit-frame-pointer $(call cc-option,-fno-optimize-sibling-calls,)
 else
diff -uNrp --exclude CVS linux-head/arch/blackfin/Kconfig linux-2.6.x/arch/blackfin/Kconfig
--- linux-head/arch/blackfin/Kconfig	2006-10-08 19:13:07.000000000 +0200
+++ linux-2.6.x/arch/blackfin/Kconfig	2006-10-09 22:36:50.000000000 +0200
@@ -752,6 +752,8 @@ endchoice
 
 endmenu
 
+source "kernel/ipipe/Kconfig"
+
 if (BF537 || BF533)
 
 menu "CPU Frequency scaling"
diff -uNrp --exclude CVS linux-head/arch/blackfin/kernel/Makefile linux-2.6.x/arch/blackfin/kernel/Makefile
--- linux-head/arch/blackfin/kernel/Makefile	2006-10-08 19:13:07.000000000 +0200
+++ linux-2.6.x/arch/blackfin/kernel/Makefile	2006-10-12 21:58:39.000000000 +0200
@@ -10,4 +10,6 @@ obj-y := \
 
 obj-$(CONFIG_MODULES)                += module.o
 obj-$(CONFIG_BFIN_DMA_5XX)           += bfin_dma_5xx.o
+obj-$(CONFIG_IPIPE)                  += ipipe-core.o ipipe-root.o
+obj-$(CONFIG_IPIPE_TRACE_MCOUNT)     += ipipe-mcount.o
 obj-$(CONFIG_DUAL_CORE_TEST_MODULE)  += dualcore_test.o
diff -uNrp --exclude CVS linux-head/arch/blackfin/kernel/entry.S linux-2.6.x/arch/blackfin/kernel/entry.S
--- linux-head/arch/blackfin/kernel/entry.S	2006-10-08 19:13:08.000000000 +0200
+++ linux-2.6.x/arch/blackfin/kernel/entry.S	2006-10-09 22:40:13.000000000 +0200
@@ -6,7 +6,7 @@
  * Created:
  * Description:
  *
- * Rev:          $Id: entry.S,v 1.25 2006/09/29 05:35:01 rgetz Exp $
+ * Rev:          $Id: entry.S,v 1.5 2006/10/09 20:40:13 rpm Exp $
  *
  * Modified:
  *               Copyright 2004-2006 Analog Devices Inc.
@@ -37,6 +37,10 @@
 .text
 
 ENTRY(_ret_from_fork)
+#ifdef CONFIG_IPIPE
+	[--sp] = reti; 		/* IRQs on. */
+	SP += 4;
+#endif /* CONFIG_IPIPE */	
 	SP += -12;
 	call _schedule_tail;
 	SP += 12;
diff -uNrp --exclude CVS linux-head/arch/blackfin/kernel/ipipe-core.c linux-2.6.x/arch/blackfin/kernel/ipipe-core.c
--- linux-head/arch/blackfin/kernel/ipipe-core.c	1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.x/arch/blackfin/kernel/ipipe-core.c	2006-10-09 22:33:18.000000000 +0200
@@ -0,0 +1,215 @@
+/* -*- linux-c -*-
+ * linux/arch/blackfin/kernel/ipipe-core.c
+ *
+ * Copyright (C) 2005 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-dependent I-PIPE core support for the Blackfin.
+ */
+
+#include <linux/config.h>
+#include <linux/kernel.h>
+#include <linux/smp.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/bitops.h>
+#include <linux/module.h>
+#include <asm/system.h>
+#include <asm/atomic.h>
+#include <asm/irqchip.h>
+#include <asm/io.h>
+
+extern struct irqdesc irq_desc[];
+
+struct pt_regs __ipipe_tick_regs[IPIPE_NR_CPUS];
+
+static void __ipipe_no_irqtail(void);
+
+unsigned long __ipipe_irq_tail_hook = (unsigned long)&__ipipe_no_irqtail;
+
+#ifdef CONFIG_SMP
+
+static cpumask_t __ipipe_cpu_sync_map;
+
+static cpumask_t __ipipe_cpu_lock_map;
+
+static ipipe_spinlock_t __ipipe_cpu_barrier = IPIPE_SPIN_LOCK_UNLOCKED;
+
+static atomic_t __ipipe_critical_count = ATOMIC_INIT(0);
+
+static void (*__ipipe_cpu_sync) (void);
+
+/* Always called with hw interrupts off. */
+
+void __ipipe_do_critical_sync(unsigned irq)
+{
+	ipipe_declare_cpuid;
+
+	ipipe_load_cpuid();
+
+	cpu_set(cpuid, __ipipe_cpu_sync_map);
+
+	/* Now we are in sync with the lock requestor running on another
+	   CPU. Enter a spinning wait until he releases the global
+	   lock. */
+	spin_lock_hw(&__ipipe_cpu_barrier);
+
+	/* Got it. Now get out. */
+
+	if (__ipipe_cpu_sync)
+		/* Call the sync routine if any. */
+		__ipipe_cpu_sync();
+
+	spin_unlock_hw(&__ipipe_cpu_barrier);
+
+	cpu_clear(cpuid, __ipipe_cpu_sync_map);
+}
+
+#endif	/* CONFIG_SMP */
+
+unsigned long ipipe_critical_enter(void (*syncfn) (void))
+{
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
+
+#ifdef CONFIG_SMP
+	if (num_online_cpus() > 1) {	/* We might be running a SMP-kernel on a UP box... */
+		ipipe_declare_cpuid;
+		cpumask_t lock_map;
+
+		ipipe_load_cpuid();
+
+		if (!cpu_test_and_set(cpuid, __ipipe_cpu_lock_map)) {
+			while (cpu_test_and_set
+			       (BITS_PER_LONG - 1, __ipipe_cpu_lock_map)) {
+				int n = 0;
+				do {
+					cpu_relax();
+				} while (++n < cpuid);
+			}
+
+			spin_lock_hw(&__ipipe_cpu_barrier);
+
+			__ipipe_cpu_sync = syncfn;
+
+			/* Send the sync IPI to all processors but the current one. */
+			send_IPI_allbutself(IPIPE_CRITICAL_VECTOR);
+
+			cpus_andnot(lock_map, cpu_online_map,
+				    __ipipe_cpu_lock_map);
+
+			while (!cpus_equal(__ipipe_cpu_sync_map, lock_map))
+				cpu_relax();
+		}
+
+		atomic_inc(&__ipipe_critical_count);
+	}
+#endif	/* CONFIG_SMP */
+
+	return flags;
+}
+
+void ipipe_critical_exit(unsigned long flags)
+{
+#ifdef CONFIG_SMP
+	if (num_online_cpus() > 1) {	/* We might be running a SMP-kernel on a UP box... */
+		ipipe_declare_cpuid;
+
+		ipipe_load_cpuid();
+
+		if (atomic_dec_and_test(&__ipipe_critical_count)) {
+			spin_unlock_hw(&__ipipe_cpu_barrier);
+
+			while (!cpus_empty(__ipipe_cpu_sync_map))
+				cpu_relax();
+
+			cpu_clear(cpuid, __ipipe_cpu_lock_map);
+			cpu_clear(BITS_PER_LONG - 1, __ipipe_cpu_lock_map);
+		}
+	}
+#endif	/* CONFIG_SMP */
+
+	local_irq_restore_hw(flags);
+}
+
+static void __ipipe_no_irqtail(void)
+{
+}
+
+int ipipe_get_sysinfo(struct ipipe_sysinfo *info)
+{
+	info->ncpus = num_online_cpus();
+	info->cpufreq = ipipe_cpu_freq();
+	info->archdep.tmirq = IPIPE_TIMER_IRQ;
+	info->archdep.tmfreq = info->cpufreq;
+
+	return 0;
+}
+
+/*
+ * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline
+ * just like if it has been actually received from a hw source. Also
+ * works for virtual interrupts.
+ */
+int ipipe_trigger_irq(unsigned irq)
+{
+	unsigned long flags;
+
+	if (irq >= IPIPE_NR_IRQS ||
+	    (ipipe_virtual_irq_p(irq)
+	     && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)))
+		return -EINVAL;
+
+	local_irq_save_hw(flags);
+
+	__ipipe_handle_irq(irq, NULL);
+
+	local_irq_restore_hw(flags);
+
+	return 1;
+}
+
+int ipipe_tune_timer(unsigned long ns, int flags)
+{
+	unsigned long x, hz;
+
+	x = ipipe_critical_enter(NULL);
+
+	bfin_write_TIMER_DISABLE(1);	
+	__builtin_bfin_ssync();
+	bfin_write_TIMER0_CONFIG(0x59);	/* IRQ enable, periodic, PWM_OUT, SCLKed, OUT PAD disabled */
+	__builtin_bfin_ssync();
+	hz = (flags & IPIPE_RESET_TIMER) ? HZ : 1000000000L / ns;
+	bfin_write_TIMER0_PERIOD(get_sclk() / hz);
+	__builtin_bfin_ssync();
+	bfin_write_TIMER0_WIDTH(1);
+	__builtin_bfin_ssync();
+	bfin_write_TIMER_ENABLE(1);
+	__builtin_bfin_ssync();
+
+	ipipe_critical_exit(x);
+
+	return 0;
+}
+
+EXPORT_SYMBOL(__ipipe_irq_tail_hook);
+EXPORT_SYMBOL(ipipe_critical_enter);
+EXPORT_SYMBOL(ipipe_critical_exit);
+EXPORT_SYMBOL(ipipe_trigger_irq);
+EXPORT_SYMBOL(ipipe_get_sysinfo);
+EXPORT_SYMBOL(ipipe_tune_timer);
diff -uNrp --exclude CVS linux-head/arch/blackfin/kernel/ipipe-mcount.S linux-2.6.x/arch/blackfin/kernel/ipipe-mcount.S
--- linux-head/arch/blackfin/kernel/ipipe-mcount.S	1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.x/arch/blackfin/kernel/ipipe-mcount.S	2006-10-12 22:41:52.000000000 +0200
@@ -0,0 +1,35 @@
+/*
+ * linux/arch/blackfin/ipipe-mcount.S
+ *
+ * Copyright (C) 2006 Analog Devices Inc.
+ */
+
+#include <linux/config.h>
+#include <linux/linkage.h>
+#include <asm/blackfin.h>
+
+.text
+
+.align 4 	/* just in case */
+
+ENTRY(_mcount)
+	[--sp] = ( r7:0, p5:0 );
+	[--sp] = ASTAT;
+	p1.L = _ipipe_trace_enable;
+	p1.H = _ipipe_trace_enable;
+	r7 = [p1];
+	CC = r7==0;
+	if CC jump out;	
+	link 0x10;
+	r0 = 0x0;	
+	[sp + 0xc] = r0; /* v */
+	r0 = 0x0;	/* type: IPIPE_TRACE_FN */
+	r1 = rets;
+	p0 = [fp];	/* p0: Prior FP */
+	r2 = [p0 + 4];	/* r2: Prior RETS */
+	call ___ipipe_trace;
+	unlink;
+out:
+	ASTAT = [sp++];
+	( r7:0, p5:0 ) = [sp++];
+	rts;
diff -uNrp --exclude CVS linux-head/arch/blackfin/kernel/ipipe-root.c linux-2.6.x/arch/blackfin/kernel/ipipe-root.c
--- linux-head/arch/blackfin/kernel/ipipe-root.c	1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.x/arch/blackfin/kernel/ipipe-root.c	2006-10-12 22:09:06.000000000 +0200
@@ -0,0 +1,286 @@
+/* -*- linux-c -*-
+ * linux/arch/blackfin/kernel/ipipe-root.c
+ *
+ * Copyright (C) 2005 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-dependent I-pipe support for the Blackfin.
+ */
+
+#include <linux/config.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/slab.h>
+#include <linux/errno.h>
+#include <asm/unistd.h>
+#include <asm/system.h>
+#include <asm/atomic.h>
+#include <asm/io.h>
+
+/* __ipipe_sync_stage -> asm_do_IRQ -> do_simple/edge/level_IRQ -> __do_irq -> action */
+asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs);
+
+extern struct irqdesc irq_desc[];
+
+unsigned long __ipipe_core_clock;
+
+unsigned long __ipipe_freq_scale;
+
+/*
+ * __ipipe_enable_pipeline() -- We are running on the boot CPU, hw
+ * interrupts are off, and secondary CPUs are still lost in space.
+ */
+void __ipipe_enable_pipeline(void)
+{
+	unsigned irq;
+
+	__ipipe_core_clock = get_cclk(); /* Fetch this once. */
+	__ipipe_freq_scale = 1000000000UL / __ipipe_core_clock;
+
+	for (irq = 0; irq < NR_IRQS; irq++) {
+		if (irq != IRQ_SW_INT1 && irq != IRQ_SW_INT2)
+			ipipe_virtualize_irq(ipipe_root_domain,
+					     irq,
+					     (ipipe_irq_handler_t)&asm_do_IRQ,
+					     NULL,
+					     &__ipipe_ack_irq,
+					     IPIPE_HANDLE_MASK | IPIPE_PASS_MASK);
+	}
+}
+
+int __ipipe_ack_irq(unsigned irq)
+{
+	struct irqdesc *desc = irq_desc + irq;
+	unsigned long flags;
+	ipipe_declare_cpuid;
+
+	if (irq == IRQ_SYSTMR) {
+		/* Clear interrupt latch for TIMER0, don't mask. */
+		bfin_write_TIMER_STATUS(1);
+		__builtin_bfin_ssync();
+		return 1;
+	}
+
+	/*
+	 * No need to mask IRQs at hw level: we are always called from
+	 * __ipipe_handle_irq(), so interrupts are already off. We
+	 * stall the pipeline so that spin_lock_irq*() ops won't
+	 * unintentionally flush it, since this could cause infinite
+	 * recursion.
+	 */
+
+	ipipe_load_cpuid();
+	flags = ipipe_test_and_stall_pipeline();
+	preempt_disable();
+	desc->chip->ack(irq);
+	preempt_enable_no_resched();
+	ipipe_restore_pipeline_nosync(per_cpu(ipipe_percpu_domain, cpuid), flags, cpuid);
+
+	return 1;
+}
+
+/*
+ * __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic
+ * interrupt protection log is maintained here for each domain. Hw
+ * interrupts are masked on entry.
+ */
+void __ipipe_handle_irq(unsigned irq, struct pt_regs *regs)
+{
+	struct ipipe_domain *this_domain, *next_domain;
+	struct list_head *head, *pos;
+	int m_ack, s_ack, s = -1;
+	ipipe_declare_cpuid;
+
+	m_ack = (regs == NULL);	/* Software-triggered IRQs do not need
+				 * any ack. */
+	ipipe_load_cpuid();
+
+	this_domain = per_cpu(ipipe_percpu_domain, cpuid);
+
+	if (test_bit(IPIPE_STICKY_FLAG, &this_domain->irqs[irq].control))
+		head = &this_domain->p_link;
+	else {
+		head = __ipipe_pipeline.next;
+		next_domain = list_entry(head, struct ipipe_domain, p_link);
+		if (likely(test_bit(IPIPE_WIRED_FLAG, &next_domain->irqs[irq].control))) {
+			if (!m_ack && next_domain->irqs[irq].acknowledge != NULL)
+				next_domain->irqs[irq].acknowledge(irq);
+			if (test_bit(IPIPE_ROOTLOCK_FLAG, &ipipe_root_domain->flags))
+				s = __test_and_set_bit(IPIPE_STALL_FLAG,
+						       &ipipe_root_domain->cpudata[cpuid].status);
+			if (likely(__ipipe_dispatch_wired(next_domain, irq)))
+				goto finalize;
+			return;
+		}
+	}
+
+	/* Ack the interrupt. */
+
+	s_ack = m_ack;
+	pos = head;
+
+	while (pos != &__ipipe_pipeline) {
+		next_domain = list_entry(pos, struct ipipe_domain, p_link);
+
+		/*
+		 * For each domain handling the incoming IRQ, mark it as
+		 * pending in its log.
+		 */
+		if (test_bit(IPIPE_HANDLE_FLAG,
+			     &next_domain->irqs[irq].control)) {
+			/*
+			 * Domains that handle this IRQ are polled for
+			 * acknowledging it by decreasing priority order. The
+			 * interrupt must be made pending _first_ in the
+			 * domain's status flags before the PIC is unlocked.
+			 */
+
+			next_domain->cpudata[cpuid].irq_counters[irq].total_hits++;
+			next_domain->cpudata[cpuid].irq_counters[irq].pending_hits++;
+			__ipipe_set_irq_bit(next_domain, cpuid, irq);
+
+			/*
+			 * Always get the first master acknowledge available.
+			 * Once we've got it, allow slave acknowledge
+			 * handlers to run (until one of them stops us).
+			 */
+			if (next_domain->irqs[irq].acknowledge != NULL) {
+				if (!m_ack)
+					m_ack = next_domain->irqs[irq].acknowledge(irq);
+				else if (test_bit
+					 (IPIPE_SHARED_FLAG,
+					  &next_domain->irqs[irq].control) && !s_ack)
+					s_ack = next_domain->irqs[irq].acknowledge(irq);
+			}
+		}
+
+		/*
+		 * If the domain does not want the IRQ to be passed down the
+		 * interrupt pipe, exit the loop now.
+		 */
+
+		if (!test_bit(IPIPE_PASS_FLAG, &next_domain->irqs[irq].control))
+			break;
+
+		pos = next_domain->p_link.next;
+	}
+
+	/*
+	 * Now walk the pipeline, yielding control to the highest
+	 * priority domain that has pending interrupt(s) or
+	 * immediately to the current domain if the interrupt has been
+	 * marked as 'sticky'. This search does not go beyond the
+	 * current domain in the pipeline. We also enforce the
+	 * additional root stage lock (blackfin-specific). */
+
+	if (test_bit(IPIPE_ROOTLOCK_FLAG, &ipipe_root_domain->flags))
+		s = __test_and_set_bit(IPIPE_STALL_FLAG,
+				       &ipipe_root_domain->cpudata[cpuid].status);
+finalize:
+	__ipipe_walk_pipeline(head, cpuid);
+
+	if (!s)
+		__clear_bit(IPIPE_STALL_FLAG,
+			    &ipipe_root_domain->cpudata[cpuid].status);
+}
+
+int __ipipe_check_root(void)
+{
+	ipipe_declare_cpuid;
+	/*
+	 * SMP: This routine is called with hw interrupts off, so no
+	 * migration can occur while checking the identity of the
+	 * current domain.
+	 */
+	ipipe_load_cpuid();
+	return per_cpu(ipipe_percpu_domain, cpuid) == ipipe_root_domain;
+}
+
+void __ipipe_stall_root_raw(void)
+{
+	ipipe_declare_cpuid;
+
+	ipipe_load_cpuid();
+
+	__set_bit(IPIPE_STALL_FLAG,
+		  &ipipe_root_domain->cpudata[cpuid].status);
+}
+
+void __ipipe_unstall_root_raw(void)
+{
+	ipipe_declare_cpuid;
+
+	ipipe_load_cpuid();
+
+	__clear_bit(IPIPE_STALL_FLAG,
+		    &ipipe_root_domain->cpudata[cpuid].status);
+}
+
+int __ipipe_syscall_root(struct pt_regs *regs)
+{
+	ipipe_declare_cpuid;
+	unsigned long flags;
+
+	/* We need to run the IRQ tail hook whenever we don't
+	 * propagate a syscall to higher domains, because we know that
+	 * important operations might be pending there (e.g. Xenomai
+	 * deferred rescheduling). */
+
+	if (!__ipipe_syscall_watched_p(current, regs->orig_p0)) {
+		void (*hook)(void) = (void (*)(void))__ipipe_irq_tail_hook;
+		hook();
+		return 0;
+	}
+
+	/*
+	 * This routine either returns:
+	 * 0 -- if the syscall is to be passed to Linux;
+	 * 1 -- if the syscall should not be passed to Linux, and no
+	 * tail work should be performed;
+	 * -1 -- if the syscall should not be passed to Linux but the
+	 * tail work has to be performed (for handling signals etc).
+	 */
+
+	if (__ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL) &&
+	    __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL,regs) > 0) {
+		if (ipipe_current_domain == ipipe_root_domain && !in_atomic()) {
+			/*
+			 * Sync pending VIRQs before _TIF_NEED_RESCHED
+			 * is tested.
+			 */
+			ipipe_lock_cpu(flags);
+			if ((ipipe_root_domain->cpudata[cpuid].irq_pending_hi & IPIPE_IRQMASK_VIRT) != 0)
+				__ipipe_sync_pipeline(IPIPE_IRQMASK_VIRT);
+			ipipe_unlock_cpu(flags);
+			return -1;
+		}
+		return 1;
+	}
+
+	return 0;
+}
+
+EXPORT_SYMBOL(__ipipe_core_clock);
+EXPORT_SYMBOL(__ipipe_freq_scale);
+EXPORT_SYMBOL(show_stack);
+
+#ifdef CONFIG_IPIPE_TRACE_MCOUNT
+void notrace mcount(void);
+EXPORT_SYMBOL(mcount);
+#endif /* CONFIG_IPIPE_TRACE_MCOUNT */
diff -uNrp --exclude CVS linux-head/arch/blackfin/kernel/irqchip.c linux-2.6.x/arch/blackfin/kernel/irqchip.c
--- linux-head/arch/blackfin/kernel/irqchip.c	2006-10-08 19:13:08.000000000 +0200
+++ linux-2.6.x/arch/blackfin/kernel/irqchip.c	2006-10-09 22:40:13.000000000 +0200
@@ -6,7 +6,7 @@
  * Created:
  * Description:  This file contains the simple DMA Implementation for Blackfin
  *
- * Rev:          $Id: irqchip.c,v 1.22 2006/09/25 07:49:02 magicyang Exp $
+ * Rev:          $Id: irqchip.c,v 1.5 2006/10/09 20:40:13 rpm Exp $
  *
  * Modified:
  *               Copyright 2004-2006 Analog Devices Inc.
@@ -53,6 +53,93 @@ static LIST_HEAD(irq_pending);
 
 struct irqdesc irq_desc[NR_IRQS];
 
+#ifdef CONFIG_IPIPE
+
+/* Implement a threaded interrupt model a la PREEMPT_RT on top of the
+   I-pipe, so that Linux device IRQ handlers cannot defer the
+   interrupt tail code for too long. */
+
+#include <linux/kthread.h>
+
+static int create_irq_threads;
+
+static int do_irqd(void * __desc)
+{
+	struct irqdesc *desc = __desc;
+	unsigned irq = desc - irq_desc;
+	int cpu = smp_processor_id();
+	cpumask_t cpumask;
+
+	sigfillset(&current->blocked);
+	current->flags |= PF_NOFREEZE;
+	cpumask = cpumask_of_cpu(cpu);
+	set_cpus_allowed(current, cpumask);
+
+	ipipe_setscheduler_root(current,SCHED_FIFO,
+				50 + IVG7 - __ipipe_get_irq_priority(irq));
+
+	while (!kthread_should_stop()) {
+		if (!down_interruptible(&desc->thrsem)) {
+			local_irq_disable();
+			desc->thrhandler(irq,&__ipipe_tick_regs[cpu]);
+			local_irq_enable();
+		}
+	}
+	__set_current_state(TASK_RUNNING);
+	return 0;
+}
+
+static void kick_irqd(unsigned irq, void *cookie)
+{
+	struct irqdesc *desc = irq_desc + irq;
+	up(&desc->thrsem);
+}
+
+static int start_irq_thread(unsigned irq, struct irqdesc *desc)
+{
+	if (desc->thread || !create_irq_threads)
+		return 0;
+
+	sema_init(&desc->thrsem,0);
+	desc->thread = kthread_create(do_irqd, desc, "IRQ %d", irq);
+
+	if (!desc->thread) {
+		printk(KERN_ERR "irqd: could not create IRQ thread %d!\n", irq);
+		return -ENOMEM;
+	}
+
+	wake_up_process(desc->thread);
+
+	desc->thrhandler = (void (*)(unsigned,struct pt_regs *))ipipe_root_domain->irqs[irq].handler;
+	ipipe_root_domain->irqs[irq].handler = &kick_irqd;
+
+	return 0;
+}
+
+int __init init_irqthreads(void)
+{
+	unsigned irq;
+
+	create_irq_threads = 1;
+
+	for (irq = 0; irq < NR_IRQS; irq++) {
+		struct irqdesc *desc = irq_desc + irq;
+		if (desc->action)
+		    start_irq_thread(irq, desc);
+	}
+
+	return 0;
+}
+
+#else /* CONFIG_IPIPE */
+
+int __init init_irqthreads(void)
+{
+    return 0;
+}
+
+#endif /* CONFIG_IPIPE */
+
 /*
  * Dummy mask/unmask handler
  */
@@ -212,6 +299,7 @@ int show_interrupts(struct seq_file *p, 
  */
 static int check_irq_lock(struct irqdesc *desc, int irq, struct pt_regs *regs)
 {
+#ifndef CONFIG_IPIPE /* This check raises false positives over the I-pipe */
 	unsigned long instr_ptr = instruction_pointer(regs);
 
 	if (desc->lck_jif == jiffies &&
@@ -229,6 +317,7 @@ static int check_irq_lock(struct irqdesc
 		desc->lck_pc = instruction_pointer(regs);
 		desc->lck_jif = jiffies;
 	}
+#endif /* CONFIG_IPIPE */
 	return 0;
 }
 
@@ -308,6 +397,11 @@ void do_simple_IRQ(unsigned int irq, str
 		int ret = __do_irq(irq, action, regs);
 		if (ret != IRQ_HANDLED)
 			report_bad_irq(irq, regs, desc, ret);
+#ifdef CONFIG_IPIPE
+		if (likely(!desc->disable_depth &&
+			   !check_irq_lock(desc, irq, regs)))
+			desc->chip->unmask(irq);
+#endif /* CONFIG_IPIPE */
 	}
 }
 
@@ -332,7 +426,9 @@ void do_edge_IRQ(unsigned int irq, struc
 	/*
 	 * Acknowledge and clear the IRQ, but don't mask it.
 	 */
+#ifndef CONFIG_IPIPE
 	desc->chip->ack(irq);
+#endif /* CONFIG_IPIPE */
 
 	/*
 	 * Mark the IRQ currently in progress.
@@ -375,7 +471,9 @@ void do_edge_IRQ(unsigned int irq, struc
 	 */
 	desc->pending = 1;
 	desc->chip->mask(irq);
+#ifndef CONFIG_IPIPE
 	desc->chip->ack(irq);
+#endif /* CONFIG_IPIPE */
 }
 
 /*
@@ -391,7 +489,9 @@ void do_level_IRQ(unsigned int irq, stru
 	/*
 	 * Acknowledge, clear _AND_ disable the interrupt.
 	 */
+#ifndef CONFIG_IPIPE
 	desc->chip->ack(irq);
+#endif /* CONFIG_IPIPE */
 
 	if (likely(!desc->disable_depth)) {
 		kstat_cpu(cpu).irqs[irq]++;
@@ -477,6 +577,7 @@ asmlinkage void asm_do_IRQ(unsigned int 
 
 	spin_unlock(&irq_controller_lock);
 
+#ifndef CONFIG_IPIPE	/* Useless and bugous over the I-pipe: IRQs are threaded. */
 	/* If we're the only interrupt running (ignoring IRQ15 which is for
 	 * syscalls), lower our priority to IRQ14 so that softirqs run at
 	 * that level.  If there's another, lower-level interrupt, irq_exit
@@ -487,6 +588,7 @@ asmlinkage void asm_do_IRQ(unsigned int 
 	other_ints = pending & (pending - 1);
 	if (other_ints == 0)
 		lower_to_irq14();
+#endif /* !CONFIG_IPIPE */
 	irq_exit();
 }
 
@@ -613,6 +715,10 @@ int setup_irq(unsigned int irq, struct i
 	 * The following block of code has to be executed atomically
 	 */
 	desc = irq_desc + irq;
+#ifdef CONFIG_IPIPE
+	if (start_irq_thread(irq, desc))
+	    return -ENOMEM;
+#endif /* CONFIG_IPIPE */
 	spin_lock_irqsave(&irq_controller_lock, flags);
 	p = &desc->action;
 	if ((old = *p) != NULL) {
diff -uNrp --exclude CVS linux-head/arch/blackfin/kernel/process.c linux-2.6.x/arch/blackfin/kernel/process.c
--- linux-head/arch/blackfin/kernel/process.c	2006-10-08 19:13:08.000000000 +0200
+++ linux-2.6.x/arch/blackfin/kernel/process.c	2006-10-09 22:40:13.000000000 +0200
@@ -6,7 +6,7 @@
  * Created:
  * Description:  Blackfin architecture-dependent process handling.
  *
- * Rev:          $Id: process.c,v 1.43 2006/09/27 17:47:51 rgetz Exp $
+ * Rev:          $Id: process.c,v 1.8 2006/10/09 20:40:13 rpm Exp $
  *
  * Modified:
  *               Copyright 2004-2006 Analog Devices Inc.
@@ -101,6 +101,7 @@ static inline void default_idle(void)
 {
 	while (!need_resched()) {
 		leds_switch(LED_OFF);
+		ipipe_suspend_domain();
 		local_irq_disable();
 		if (likely(!need_resched()))
 			idle_with_irq_disabled();
diff -uNrp --exclude CVS linux-head/arch/blackfin/kernel/time.c linux-2.6.x/arch/blackfin/kernel/time.c
--- linux-head/arch/blackfin/kernel/time.c	2006-10-08 19:13:09.000000000 +0200
+++ linux-2.6.x/arch/blackfin/kernel/time.c	2006-10-09 22:40:13.000000000 +0200
@@ -7,7 +7,7 @@
  * Description:  This file contains the bfin-specific time handling details.
  *               Most of the stuff is located in the machine specific files.
  *
- * Rev:          $Id: time.c,v 1.44 2006/09/27 15:36:50 rgetz Exp $
+ * Rev:          $Id: time.c,v 1.9 2006/10/09 20:40:13 rpm Exp $
  *
  * Modified:
  *               Copyright 2004-2006 Analog Devices Inc.
@@ -139,6 +139,27 @@ static struct irqaction bfin_timer_irq =
 static void
 time_sched_init(irqreturn_t(*timer_routine) (int, void *, struct pt_regs *))
 {
+#ifdef CONFIG_IPIPE
+	/* Use builtin TIMER0 for time source instead of the high
+	 * priority core timer; this makes the latter available to
+	 * client RTOS running over the I-pipe. Xenomai will make good
+	 * use of this. */
+
+	/* Power down the core timer, just to play safe. */
+	bfin_write_TCNTL(0);
+	__builtin_bfin_csync();
+	/* We use TIMER0 in PWM_OUT, periodic mode. */
+	bfin_write_TIMER_DISABLE(1);    /* Disable TIMER0 for now. */
+	__builtin_bfin_ssync();
+	bfin_write_TIMER0_CONFIG(0x59); /* IRQ enable, periodic, PWM_OUT, SCLKed, OUT PAD disabled */
+	__builtin_bfin_ssync();
+	bfin_write_TIMER0_PERIOD(get_sclk() / HZ);
+	__builtin_bfin_ssync();
+	bfin_write_TIMER0_WIDTH(1);
+	__builtin_bfin_ssync();
+	bfin_write_TIMER_ENABLE(1);     /* Enable TIMER0. */
+	__builtin_bfin_ssync();
+#else /* !CONFIG_IPIPE */
 	u32 tcount;
 
 	/* power up the timer, but don't enable it just yet */
@@ -158,12 +179,11 @@ time_sched_init(irqreturn_t(*timer_routi
 	__builtin_bfin_csync();
 
 	bfin_write_TCNTL(7);
-
+#endif /* CONFIG_IPIPE */
 	bfin_timer_irq.handler = timer_routine;
-	/* call setup_irq instead of request_irq because request_irq calls
-	 * kmalloc which has not been initialized yet
-	 */
-	setup_irq(IRQ_CORETMR, &bfin_timer_irq);
+	/* call setup_irq instead of request_irq because request_irq
+	   calls kmalloc which has not been initialized yet */
+	setup_irq(IRQ_SYSTMR, &bfin_timer_irq);
 }
 
 /*
@@ -174,17 +194,22 @@ static unsigned long gettimeoffset(void)
 	unsigned long offset;
 	unsigned long clocks_per_jiffy;
 
+#ifdef CONFIG_IPIPE
+	/* When the I-pipe is active, Linux uses TMR0 and not the core
+	   timer, which is then reserved to any real-time subsystem
+	that may rely on Adeos for interrupt virtualization. */
+	clocks_per_jiffy =  bfin_read_TIMER0_PERIOD();
+	offset =  (clocks_per_jiffy - bfin_read_TIMER0_COUNTER()) 
+			/ (((clocks_per_jiffy + 1) * HZ) / USEC_PER_SEC);
+#else /* !CONFIG_IPIPE */
 	clocks_per_jiffy = bfin_read_TPERIOD();
 	offset =
 	    (clocks_per_jiffy -
 	     bfin_read_TCOUNT()) / (((clocks_per_jiffy + 1) * HZ) /
 				    USEC_PER_SEC);
-
-	/* Check if we just wrapped the counters and maybe missed a tick */
-	if ((bfin_read_ILAT() & (1 << IRQ_CORETMR))
-	    && (offset < (100000 / HZ / 2)))
+	if ((bfin_read_ILAT() & (1 << IRQ_PRIOTMR)) && (offset < (100000 / HZ / 2)))
 		offset += (USEC_PER_SEC / HZ);
-
+#endif /* CONFIG_IPIPE */
 	return offset;
 }
 
diff -uNrp --exclude CVS linux-head/arch/blackfin/kernel/traps.c linux-2.6.x/arch/blackfin/kernel/traps.c
--- linux-head/arch/blackfin/kernel/traps.c	2006-10-08 19:13:09.000000000 +0200
+++ linux-2.6.x/arch/blackfin/kernel/traps.c	2006-10-09 22:40:13.000000000 +0200
@@ -6,7 +6,7 @@
  * Created:
  * Description:  uses S/W interrupt 15 for the system calls
  *
- * Rev:          $Id: traps.c,v 1.77 2006/10/02 23:44:53 vapier Exp $
+ * Rev:          $Id: traps.c,v 1.8 2006/10/09 20:40:13 rpm Exp $
  *
  * Modified:
  *               Copyright 2004-2006 Analog Devices Inc.
@@ -362,6 +362,9 @@ asmlinkage void trap_c(struct pt_regs *f
 		break;
 	}
 
+	if (ipipe_trap_notify(fp->seqstat & 0x3f,fp))
+		goto nsig;
+
 	info.si_signo = sig;
 	info.si_errno = 0;
 	info.si_addr = (void *)fp->pc;
@@ -391,7 +394,7 @@ asmlinkage void trap_c(struct pt_regs *f
 			panic("Help- I have fallen and can't get up\n");
 		}
 	}
-
+nsig:
 	trace_buffer_restore(j);
 	return;
 }
diff -uNrp --exclude CVS linux-head/arch/blackfin/lib/ins.S linux-2.6.x/arch/blackfin/lib/ins.S
--- linux-head/arch/blackfin/lib/ins.S	2006-10-08 19:13:10.000000000 +0200
+++ linux-2.6.x/arch/blackfin/lib/ins.S	2006-10-09 22:40:13.000000000 +0200
@@ -6,7 +6,7 @@
  * Created:      Tue Mar 22 15:27:24 CEST 2005
  * Description:  Implementation of ins{bwl} for BlackFin processors using zero overhead loops.
  *
- * Rev:          $Id: ins.S,v 1.10 2006/09/23 06:11:30 vapier Exp $
+ * Rev:          $Id: ins.S,v 1.4 2006/10/09 20:40:13 rpm Exp $
  *
  * Modified:
  *               Copyright 2004-2006 Analog Devices Inc.
@@ -38,7 +38,16 @@ ENTRY(_insl)
 	P0 = R0;	/* P0 = port */
 	P1 = R1;	/* P1 = address */
 	P2 = R2;	/* P2 = count */
+#ifdef CONFIG_IPIPE
+	[--sp] = rets
+	[--sp] = (P5:0);
+	sp += -12
+	call ___ipipe_stall_root_raw
+	sp += 12
+	(P5:0) = [sp++];
+#else /* !CONFIG_IPIPE */
 	cli R2;
+#endif /* CONFIG_IPIPE */
 	NOP;
 	NOP;
 
@@ -46,14 +55,30 @@ ENTRY(_insl)
 .Llong_loop_s: R0 = [P0];
 .Llong_loop_e: [P1++] = R0;
 
+#ifdef CONFIG_IPIPE
+	sp += -12
+	call ___ipipe_unstall_root_raw
+	sp += 12
+	rets = [sp++]
+#else /* !CONFIG_IPIPE */
 	sti R2;
+#endif /* CONFIG_IPIPE */
 	RTS;
 
 ENTRY(_insw)
 	P0 = R0;	/* P0 = port */
 	P1 = R1;	/* P1 = address */
 	P2 = R2;	/* P2 = count */
+#ifdef CONFIG_IPIPE
+	[--sp] = rets
+	[--sp] = (P5:0);
+	sp += -12
+	call ___ipipe_stall_root_raw
+	sp += 12
+	(P5:0) = [sp++];
+#else /* !CONFIG_IPIPE */
 	cli R2;
+#endif
 	NOP;
 	NOP;
 
@@ -61,19 +86,42 @@ ENTRY(_insw)
 .Lword_loop_s: R0 = W[P0];
 .Lword_loop_e: W[P1++] = R0;
 
+#ifdef CONFIG_IPIPE
+	sp += -12
+	call ___ipipe_unstall_root_raw
+	sp += 12
+	rets = [sp++]
+#else /* !CONFIG_IPIPE */
 	sti R2;
+#endif /* CONFIG_IPIPE */
 	RTS;
 
 ENTRY(_insb)
 	P0 = R0;	/* P0 = port */
 	P1 = R1;	/* P1 = address */
 	P2 = R2;	/* P2 = count */
+#ifdef CONFIG_IPIPE
+	[--sp] = rets
+	[--sp] = (P5:0);
+	sp += -12
+	call ___ipipe_stall_root_raw
+	sp += 12
+	(P5:0) = [sp++];
+#else		
 	cli R2;
+#endif	
 	NOP;
 	NOP;
 
 	LSETUP( .Lbyte_loop_s, .Lbyte_loop_e) LC0 = P2;
 .Lbyte_loop_s: R0 = B[P0];
 .Lbyte_loop_e: B[P1++] = R0;
+#ifdef CONFIG_IPIPE
+	sp += -12
+	call ___ipipe_unstall_root_raw
+	sp += 12
+	rets = [sp++]
+#else		
 	sti R2;
+#endif	
 	RTS;
diff -uNrp --exclude CVS linux-head/arch/blackfin/mach-bf533/Kconfig linux-2.6.x/arch/blackfin/mach-bf533/Kconfig
--- linux-head/arch/blackfin/mach-bf533/Kconfig	2006-10-08 19:13:12.000000000 +0200
+++ linux-2.6.x/arch/blackfin/mach-bf533/Kconfig	2006-10-09 22:36:50.000000000 +0200
@@ -57,7 +57,7 @@ config DMA7_UARTTX
 	default 10
 config TIMER0
 	int "TIMER0"
-	default 11
+	default 7
 config TIMER1
 	int "TIMER1"
 	default 11
diff -uNrp --exclude CVS linux-head/arch/blackfin/mach-common/entry.S linux-2.6.x/arch/blackfin/mach-common/entry.S
--- linux-head/arch/blackfin/mach-common/entry.S	2006-10-08 19:13:18.000000000 +0200
+++ linux-2.6.x/arch/blackfin/mach-common/entry.S	2006-10-11 09:46:22.000000000 +0200
@@ -8,7 +8,7 @@
  *               This also contains the timer-interrupt handler, as well as all
  *               interrupts and faults that can result in a task-switch.
  *
- * Rev:          $Id: entry.S,v 1.49 2006/09/29 05:35:31 rgetz Exp $
+ * Rev:          $Id: entry.S,v 1.6 2006/10/09 20:40:13 rpm Exp $
  *
  * Modified:
  *               Copyright 2004-2006 Analog Devices Inc.
@@ -405,6 +405,19 @@ ENTRY(_system_call)
 	p2 = [p2];
 
 	[p2+(TASK_THREAD+THREAD_KSP)] = sp;
+#ifdef CONFIG_IPIPE
+	r0 = sp; 
+	SP += -12;
+	call ___ipipe_syscall_root; 
+	SP += 12;
+	cc = r0 == 1;
+	if cc jump .Lsyscall_really_exit;
+	cc = r0 == -1;
+	if cc jump .Lresume_userspace;
+	r3 = [sp + PT_R3]; 
+	r4 = [sp + PT_R4]; 
+	p0 = [sp + PT_ORIG_P0]; 
+#endif /* CONFIG_IPIPE */
 
 	/* Check the System Call */
 	r7 = __NR_syscall;
@@ -488,6 +501,11 @@ ENTRY(_system_call)
 	SP += 12;
 
 .Lsyscall_really_exit:
+syscall_really_exit:
+#ifdef CONFIG_IPIPE
+	[--sp] = reti;
+	r5 = [sp++];
+#endif /* CONFIG_IPIPE */
 	r5 = [sp + PT_RESERVED];
 	rets = r5;
 	rts;
@@ -570,6 +588,16 @@ _new_old_task:
 	rts;
 
 ENTRY(_ret_from_exception)
+#ifdef CONFIG_IPIPE
+	[--sp] = rets;
+	SP += -12;
+	call ___ipipe_check_root
+	SP += 12
+	rets = [sp++];
+	cc = r0 == 0;
+	if cc jump 4f;                /* not on behalf of Linux, get out */
+#endif /* CONFIG_IPIPE */
+	
 	p2.l = lo(IPEND);
 	p2.h = hi(IPEND);
 
@@ -669,7 +697,7 @@ ENTRY(_return_from_int)
 2:
 	rts;
 
-ENTRY(_lower_to_irq14)
+ENTRY(_lower_to_irq14) /* Not called when the I-pipe is enabled. */
 #if defined(ANOMALY_05000281)
 	r0.l = lo(CONFIG_BOOT_LOAD);
 	r0.h = hi(CONFIG_BOOT_LOAD);
diff -uNrp --exclude CVS linux-head/arch/blackfin/mach-common/interrupt.S linux-2.6.x/arch/blackfin/mach-common/interrupt.S
--- linux-head/arch/blackfin/mach-common/interrupt.S	2006-10-08 19:13:19.000000000 +0200
+++ linux-2.6.x/arch/blackfin/mach-common/interrupt.S	2006-10-12 22:15:33.000000000 +0200
@@ -7,7 +7,7 @@
  * Created:      ?
  * Description:  Interrupt Entries
  *
- * Rev:          $Id: interrupt.S,v 1.33 2006/09/23 06:16:47 vapier Exp $
+ * Rev:          $Id: interrupt.S,v 1.9 2006/10/12 20:15:33 rpm Exp $
  *
  * Modified:
  *               Copyright 2004-2006 Analog Devices Inc.
@@ -148,8 +148,15 @@ __common_int_entry:
 #endif
 	r1 =  sp;
 	SP += -12;
+#ifdef CONFIG_IPIPE
+	call ___ipipe_grab_irq
+	SP += 12;
+	cc = r0 == 0;
+	if cc jump .Lcommon_restore_context;
+#else /* CONFIG_IPIPE */
 	call _do_irq;
 	SP += 12;
+#endif /* CONFIG_IPIPE */
 	call _return_from_int;
 .Lcommon_restore_context:
 	RESTORE_CONTEXT
@@ -251,3 +258,53 @@ ENTRY(_evt_system_call)
 #endif
 	call _system_call;
 	jump .Lcommon_restore_context;
+
+#ifdef CONFIG_IPIPE
+
+ENTRY(___ipipe_call_irqtail)
+
+	r0.l = 1f;
+	r0.h = 1f;
+	reti = r0;
+	rti
+1:
+	[--sp] = rets;
+	[--sp] = ( r7:4, p5:3 );
+	p0.l = ___ipipe_irq_tail_hook;
+	p0.h = ___ipipe_irq_tail_hook;
+	p0 = [p0] ;
+	sp += -12 ;
+	call (p0) ;
+	sp += 12;
+	( r7:4, p5:3 ) = [sp++];
+	rets = [sp++];
+
+	[--sp] = reti;
+	reti = [sp++];          /* IRQs are off. */
+	r0.h = 3f;
+	r0.l = 3f;
+	p0.l = lo(EVT14);
+	p0.h = hi(EVT14);
+	[p0] = r0;
+	csync;
+	r0 = 0x401f;
+	sti r0;
+	raise 14;
+	[--sp] = reti;          /* IRQs on. */
+2:
+	jump 2b                 /* Likely paranoid. */
+3:
+	sp += 4;                /* Discard saved RETI */
+	r0.h = _evt14_softirq;
+	r0.l = _evt14_softirq;
+	p0.l = lo(EVT14);
+	p0.h = hi(EVT14);
+	[p0] = r0;
+	csync
+	p0.l = _irq_flags;
+	p0.h = _irq_flags;
+	r0 = [p0];
+	sti r0
+	rts;
+
+#endif /* CONFIG_IPIPE */
diff -uNrp --exclude CVS linux-head/arch/blackfin/mach-common/ints-priority-sc.c linux-2.6.x/arch/blackfin/mach-common/ints-priority-sc.c
--- linux-head/arch/blackfin/mach-common/ints-priority-sc.c	2006-10-08 19:13:20.000000000 +0200
+++ linux-2.6.x/arch/blackfin/mach-common/ints-priority-sc.c	2006-10-11 23:14:17.000000000 +0200
@@ -6,7 +6,7 @@
  * Created:      ?
  * Description:  Set up the interupt priorities
  *
- * Rev:          $Id: ints-priority-sc.c,v 1.45 2006/09/26 11:05:35 hennerich Exp $
+ * Rev:          $Id: ints-priority-sc.c,v 1.8 2006/10/09 20:40:13 rpm Exp $
  *
  * Modified:
  *               1996 Roman Zippel
@@ -132,8 +132,8 @@ static void ack_noop(unsigned int irq)
 static void bf533_core_mask_irq(unsigned int irq)
 {
 	irq_flags &= ~(1 << irq);
-	if (!irqs_disabled())
-		local_irq_enable();
+	if (!irqs_disabled_hw())
+		local_irq_enable_hw();
 }
 
 static void bf533_core_unmask_irq(unsigned int irq)
@@ -148,8 +148,8 @@ static void bf533_core_unmask_irq(unsign
 	 * local_irq_enable just does "STI irq_flags", so it's exactly
 	 * what we need.
 	 */
-	if (!irqs_disabled())
-		local_irq_enable();
+	if (!irqs_disabled_hw())
+		local_irq_enable_hw();
 	return;
 }
 
@@ -168,13 +168,21 @@ static void bf533_internal_unmask_irq(un
 }
 
 static struct irqchip bf533_core_irqchip = {
+#ifdef CONFIG_IPIPE
+	.ack = bf533_core_mask_irq,
+#else /* !CONFIG_IPIPE */
 	.ack = ack_noop,
+#endif /* CONFIG_IPIPE */
 	.mask = bf533_core_mask_irq,
 	.unmask = bf533_core_unmask_irq,
 };
 
 static struct irqchip bf533_internal_irqchip = {
+#ifdef CONFIG_IPIPE
+	.ack = bf533_internal_mask_irq,
+#else /* !CONFIG_IPIPE */
 	.ack = ack_noop,
+#endif /* CONFIG_IPIPE */
 	.mask = bf533_internal_mask_irq,
 	.unmask = bf533_internal_unmask_irq,
 };
@@ -192,23 +200,23 @@ static void bf537_generic_error_mask_irq
 	error_int_mask &= ~(1L << (irq - IRQ_PPI_ERROR));
 
 	if (!error_int_mask) {
-		local_irq_disable();
+		local_irq_disable_hw();
 		bfin_write_SIC_IMASK(bfin_read_SIC_IMASK() &
 				     ~(1 <<
 				       (IRQ_GENERIC_ERROR -
 					(IRQ_CORETMR + 1))));
 		__builtin_bfin_ssync();
-		local_irq_enable();
+		local_irq_enable_hw();
 	}
 }
 
 static void bf537_generic_error_unmask_irq(unsigned int irq)
 {
-	local_irq_disable();
+	local_irq_disable_hw();
 	bfin_write_SIC_IMASK(bfin_read_SIC_IMASK() | 1 <<
 			     (IRQ_GENERIC_ERROR - (IRQ_CORETMR + 1)));
 	__builtin_bfin_ssync();
-	local_irq_enable();
+	local_irq_enable_hw();
 
 	error_int_mask |= 1L << (irq - IRQ_PPI_ERROR);
 }
@@ -613,6 +621,10 @@ static void bf533_gpio_ack_irq(unsigned 
 	int gpionr = irq - IRQ_PF0;
 	int mask = (1L << gpionr);
 	bfin_write_FIO_FLAG_C(mask);
+#ifdef CONFIG_IPIPE
+	__builtin_bfin_ssync();
+	bfin_write_FIO_MASKB_C(mask);
+#endif /* CONFIG_IPIPE */
 /*	if (gpio_edge_triggered & mask) {
 		* ack *
 	} else {
@@ -937,3 +949,75 @@ void bfin_gpio_interrupt_setup(int irq, 
 }
 
 EXPORT_SYMBOL(bfin_gpio_interrupt_setup);
+
+#ifdef CONFIG_IPIPE
+
+int __ipipe_get_irq_priority(unsigned irq)
+{
+	int ient, prio;
+
+	if (irq <= IRQ_CORETMR)
+		return irq;
+
+	for (ient = 0; ient < NR_PERI_INTS; ient++) {
+		struct ivgx *ivg = ivg_table + ient;
+		if (ivg->irqno == irq) {
+			for (prio = 0; prio <= IVG13-IVG7; prio++) {
+				if (ivg7_13[prio].ifirst <= ivg &&
+				    ivg7_13[prio].istop > ivg) {
+					return prio + IVG7;
+				}
+			}
+		}
+	}
+
+	return IVG13;	/* Paranoïd. */
+}
+
+/* Hw interrupts are disabled on entry (check SAVE_CONTEXT). */
+
+asmlinkage int __ipipe_grab_irq(int vec, struct pt_regs *regs)
+{
+	struct ivgx *ivg_stop = ivg7_13[vec-IVG7].istop;
+	struct ivgx *ivg = ivg7_13[vec-IVG7].ifirst;
+	unsigned long sic_status;
+	ipipe_declare_cpuid;
+	int irq;
+
+	if (likely(vec == EVT_IVTMR_P)) {
+		irq = IRQ_CORETMR;
+		goto handle_irq;
+	}
+
+	__builtin_bfin_ssync();
+ 	sic_status = bfin_read_SIC_IMASK() & bfin_read_SIC_ISR();
+
+	for(;; ivg++) {
+		if (ivg >= ivg_stop)  {
+			num_spurious++;
+			return 0;
+		}
+		else if (sic_status & ivg->isrflag)
+			break;
+	}
+
+	irq = ivg->irqno;
+
+	ipipe_load_cpuid();
+
+	if (irq == IRQ_SYSTMR) {
+	    bfin_write_TIMER_STATUS(1); /* Latch TIMIL0 */
+	    /* for update_process_times() */
+	    __ipipe_tick_regs[cpuid].ipend = regs->ipend;
+	}
+
+handle_irq:
+
+	__ipipe_handle_irq(irq, regs);
+
+	return (per_cpu(ipipe_percpu_domain, cpuid) == ipipe_root_domain &&
+		!test_bit(IPIPE_STALL_FLAG,
+			  &ipipe_root_domain->cpudata[cpuid].status));
+}
+
+#endif /* CONFIG_IPIPE */
diff -uNrp --exclude CVS linux-head/include/asm-blackfin/bitops.h linux-2.6.x/include/asm-blackfin/bitops.h
--- linux-head/include/asm-blackfin/bitops.h	2006-03-22 08:43:31.000000000 +0100
+++ linux-2.6.x/include/asm-blackfin/bitops.h	2006-10-09 22:36:50.000000000 +0200
@@ -121,9 +121,9 @@ static __inline__ void set_bit(int nr, v
 
 	a += nr >> 5;
 	mask = 1 << (nr & 0x1f);
-	local_irq_save(flags);
+	local_irq_save_hw(flags);
 	*a |= mask;
-	local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 }
 
 static __inline__ void __set_bit(int nr, volatile unsigned long *addr)
@@ -149,9 +149,9 @@ static __inline__ void clear_bit(int nr,
 	unsigned long flags;
 	a += nr >> 5;
 	mask = 1 << (nr & 0x1f);
-	local_irq_save(flags);
+	local_irq_save_hw(flags);
 	*a &= ~mask;
-	local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 }
 
 static __inline__ void __clear_bit(int nr, volatile unsigned long *addr)
@@ -171,9 +171,9 @@ static __inline__ void change_bit(int nr
 
 	ADDR += nr >> 5;
 	mask = 1 << (nr & 31);
-	local_irq_save(flags);
+	local_irq_save_hw(flags);
 	*ADDR ^= mask;
-	local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 }
 
 static __inline__ void __change_bit(int nr, volatile unsigned long *addr)
@@ -194,10 +194,10 @@ static __inline__ int test_and_set_bit(i
 
 	a += nr >> 5;
 	mask = 1 << (nr & 0x1f);
-	local_irq_save(flags);
+	local_irq_save_hw(flags);
 	retval = (mask & *a) != 0;
 	*a |= mask;
-	local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return retval;
 }
@@ -222,10 +222,10 @@ static __inline__ int test_and_clear_bit
 
 	a += nr >> 5;
 	mask = 1 << (nr & 0x1f);
-	local_irq_save(flags);
+	local_irq_save_hw(flags);
 	retval = (mask & *a) != 0;
 	*a &= ~mask;
-	local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return retval;
 }
@@ -250,10 +250,10 @@ static __inline__ int test_and_change_bi
 
 	a += nr >> 5;
 	mask = 1 << (nr & 0x1f);
-	local_irq_save(flags);
+	local_irq_save_hw(flags);
 	retval = (mask & *a) != 0;
 	*a ^= mask;
-	local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 	return retval;
 }
 
diff -uNrp --exclude CVS linux-head/include/asm-blackfin/entry.h linux-2.6.x/include/asm-blackfin/entry.h
--- linux-head/include/asm-blackfin/entry.h	2006-10-08 19:19:40.000000000 +0200
+++ linux-2.6.x/include/asm-blackfin/entry.h	2006-10-08 19:04:03.000000000 +0200
@@ -23,6 +23,19 @@
 #define SAVE_ALL_SYS		save_context_no_interrupts
 /* This is used for all normal interrupts.  It saves a minimum of registers
    to the stack, loads the IRQ number, and jumps to common code.  */
+#ifdef CONFIG_IPIPE
+#define INTERRUPT_ENTRY(N)						\
+    [--sp] = SYSCFG;							\
+									\
+    [--sp] = P0;	/*orig_p0*/					\
+    [--sp] = R0;	/*orig_r0*/					\
+    [--sp] = (R7:0,P5:0);						\
+    R0 = (N);								\
+    p0.l = lo(IPEND);							\
+    p0.h = hi(IPEND);							\
+    r1 = [p0];								\
+    jump __common_int_entry;
+#else /* !CONFIG_IPIPE */
 #define INTERRUPT_ENTRY(N)						\
     [--sp] = SYSCFG;							\
 									\
@@ -31,6 +44,7 @@
     [--sp] = (R7:0,P5:0);						\
     R0 = (N);								\
     jump __common_int_entry;
+#endif /* CONFIG_IPIPE */
 
 /* For timer interrupts, we need to save IPEND, since the user_mode
 	   macro accesses it to determine where to account time.  */
diff -uNrp --exclude CVS linux-head/include/asm-blackfin/ipipe.h linux-2.6.x/include/asm-blackfin/ipipe.h
--- linux-head/include/asm-blackfin/ipipe.h	1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.x/include/asm-blackfin/ipipe.h	2006-10-12 22:35:42.000000000 +0200
@@ -0,0 +1,252 @@
+/*   -*- linux-c -*-
+ *   include/asm-blackfin/ipipe.h
+ *
+ *   Copyright (C) 2002-2005 Philippe Gerum.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ *   USA; either version 2 of the License, or (at your option) any later
+ *   version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the Free Software
+ *   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __BFINNOMMU_IPIPE_H
+#define __NBINNOMMU_IPIPE_H
+
+#include <linux/config.h>
+
+#ifdef CONFIG_IPIPE
+
+#include <linux/cpumask.h>
+#include <linux/list.h>
+#include <linux/threads.h>
+#include <asm/ptrace.h>
+#include <asm/mach/irq.h>
+#include <asm/bitops.h>
+#include <asm/traps.h>
+#include <asm/irqchip.h>
+
+#define IPIPE_ARCH_STRING     "1.5-00"
+#define IPIPE_MAJOR_NUMBER    1
+#define IPIPE_MINOR_NUMBER    5
+#define IPIPE_PATCH_NUMBER    0
+
+#define IPIPE_NR_XIRQS		NR_IRQS
+#define IPIPE_IRQ_ISHIFT	5	/* 2^5 for 32bits arch. */
+
+/* Blackfin-specific, global domain flags */
+#define IPIPE_ROOTLOCK_FLAG	1	/* Lock pipeline for root */
+
+#ifdef CONFIG_SMP
+#error "I-pipe/blackfin: SMP not implemented"
+#else /* !CONFIG_SMP */
+#define ipipe_processor_id()	0
+#endif	/* CONFIG_SMP */
+
+#define prepare_arch_switch(next)		\
+do {						\
+	ipipe_schedule_notify(current, next);	\
+	local_irq_disable_hw();			\
+} while(0)
+
+#define task_hijacked(p)	\
+  ({ int x = ipipe_current_domain != ipipe_root_domain; \
+     __clear_bit(IPIPE_SYNC_FLAG,&ipipe_root_domain->cpudata[task_cpu(p)].status); \
+     local_irq_enable_hw(); x; })
+
+ /* Blackfin traps -- i.e. exception vector numbers */
+#define IPIPE_NR_FAULTS		52 /* We leave a gap after VEC_ILL_RES. */
+/* Pseudo-vectors used for kernel events */
+#define IPIPE_FIRST_EVENT	IPIPE_NR_FAULTS
+#define IPIPE_EVENT_SYSCALL	(IPIPE_FIRST_EVENT)
+#define IPIPE_EVENT_SCHEDULE	(IPIPE_FIRST_EVENT + 1)
+#define IPIPE_EVENT_SIGWAKE	(IPIPE_FIRST_EVENT + 2)
+#define IPIPE_EVENT_SETSCHED	(IPIPE_FIRST_EVENT + 3)
+#define IPIPE_EVENT_INIT	(IPIPE_FIRST_EVENT + 4)
+#define IPIPE_EVENT_EXIT	(IPIPE_FIRST_EVENT + 5)
+#define IPIPE_EVENT_CLEANUP	(IPIPE_FIRST_EVENT + 6)
+#define IPIPE_LAST_EVENT	IPIPE_EVENT_CLEANUP
+#define IPIPE_NR_EVENTS		(IPIPE_LAST_EVENT + 1)
+
+struct ipipe_domain;
+
+struct ipipe_sysinfo {
+
+	int ncpus;		/* Number of CPUs on board */
+	u64 cpufreq;		/* CPU frequency (in Hz) */
+
+	/* Arch-dependent block */
+
+	struct {
+		unsigned tmirq;	/* Timer tick IRQ */
+		u64 tmfreq;	/* Timer frequency */
+	} archdep;
+};
+
+#define ipipe_read_tsc(t)					\
+	({							\
+	unsigned long __cy2;					\
+	__asm__ __volatile__ (	"1: %0 = CYCLES2\n"		\
+				"%1 = CYCLES\n"			\
+				"%2 = CYCLES2\n"		\
+				"CC = %2 == %0\n"		\
+				"if ! CC jump 1b\n"		\
+				:"=r" (((unsigned long *)&t)[1]),	\
+				"=r" (((unsigned long *)&t)[0]),	\
+				"=r" (__cy2)				\
+				: /*no input*/ : "CC");			\
+	t;								\
+	})
+
+#define ipipe_cpu_freq()	__ipipe_core_clock
+#define ipipe_tsc2ns(_t)	(((unsigned long)(_t)) * __ipipe_freq_scale)
+#define ipipe_tsc2us(_t)	(ipipe_tsc2ns(_t) / 1000 + 1)
+
+/* Private interface -- Internal use only */
+
+#define __ipipe_check_platform()	do { } while(0)
+
+#define __ipipe_init_platform()	do { } while(0)
+
+#define __ipipe_enable_irqdesc(irq)	do { irq_desc[irq].disable_depth = 0; } while(0)
+
+#define __ipipe_enable_irq(irq)	irq_desc[irq].chip->unmask(irq)
+
+#define __ipipe_disable_irq(irq)	irq_desc[irq].chip->mask(irq)
+
+#define __ipipe_lock_root() \
+	set_bit(IPIPE_ROOTLOCK_FLAG, &ipipe_root_domain->flags)
+
+#define __ipipe_unlock_root() \
+	clear_bit(IPIPE_ROOTLOCK_FLAG, &ipipe_root_domain->flags)
+
+void __ipipe_enable_pipeline(void);
+
+#define __ipipe_sync_pipeline(syncmask)				\
+do {								\
+	struct ipipe_domain *ipd = ipipe_current_domain;	\
+	if (likely(ipd != ipipe_root_domain || !test_bit(IPIPE_ROOTLOCK_FLAG, &ipd->flags))) \
+		__ipipe_sync_stage(syncmask);			\
+} while(0)
+
+int __ipipe_ack_irq(unsigned irq);
+
+void __ipipe_do_IRQ(unsigned irq,
+		    struct pt_regs *regs);
+
+void __ipipe_do_timer(unsigned irq,
+		      struct pt_regs *regs);
+
+void __ipipe_handle_irq(unsigned irq,
+			struct pt_regs *regs);
+
+int __ipipe_get_irq_priority(unsigned irq);
+
+void __ipipe_stall_root_raw(void);
+
+void __ipipe_unstall_root_raw(void);
+
+extern struct pt_regs __ipipe_tick_regs[];
+
+extern unsigned long __ipipe_core_clock;
+
+extern unsigned long __ipipe_freq_scale;
+
+extern unsigned long __ipipe_irq_tail_hook;
+
+#define IPIPE_TIMER_IRQ  IRQ_CORETMR
+
+#define IRQ_SYSTMR     IRQ_TMR0
+
+#ifdef CONFIG_BF533
+#define IRQ_PRIOTMR    CONFIG_TIMER0
+#else
+#define IRQ_PRIOTMR    CONFIG_IRQ_TMR0
+#endif
+
+unsigned long get_cclk(void);	/* Core clock freq (HZ) */
+
+unsigned long get_sclk(void);	/* System clock freq (HZ) */
+
+static inline unsigned long __ipipe_ffnz(unsigned long ul)
+{
+	return ffs(ul) - 1;
+}
+
+#define __ipipe_run_irqtail()  /* Must be a macro */	\
+do {							\
+	asmlinkage void __ipipe_call_irqtail(void);	\
+	unsigned long pending;				\
+	__builtin_bfin_csync();				\
+	pending = bfin_read_IPEND();			\
+	if (pending & 0x8000) {				\
+		pending &= ~0x8010;				\
+		if (pending && (pending & (pending - 1)) == 0)	\
+			__ipipe_call_irqtail();			\
+	}							\
+} while(0)
+
+#define __ipipe_run_isr(ipd, irq, cpuid) 	\
+do {					 	\
+	if (ipd == ipipe_root_domain) {		\
+	/*							\
+	 * Note: the I-pipe implements a threaded interrupt model on	\
+	 * this arch for Linux external IRQs. The interrupt handler we	\
+	 * call here only wakes up the associated IRQ thread.		\
+	 */								\
+		if (ipipe_virtual_irq_p(irq)) {				\
+			/* No irqtail here; virtual interrupts have no effect \
+			   on IPEND so there is no need for processing	\
+			   deferral. */					\
+			local_irq_enable_nohead(ipd);			\
+			ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie); \
+			local_irq_disable_nohead(ipd);			\
+		} else							\
+			/* No need to run the irqtail here either; we	\
+			   are not preemptable by hw IRQs, so		\
+			   non-Linux IRQs cannot stack over the short	\
+			   thread wakeup code. Which in turn means	\
+			   that no irqtail condition could be pending	\
+			   for domains above Linux in the pipeline. */	\
+			((void (*)(unsigned, struct pt_regs *))		\
+			 ipd->irqs[irq].handler) (irq, __ipipe_tick_regs + cpuid); \
+	} else {							\
+		__clear_bit(IPIPE_SYNC_FLAG, &cpudata->status);		\
+		local_irq_enable_nohead(ipd);				\
+		ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie);	\
+		/* Attempt to exit the outer interrupt level before	\
+		 * starting the deferred IRQ processing. */		\
+		local_irq_disable_nohead(ipd);				\
+		__ipipe_run_irqtail();					\
+		__set_bit(IPIPE_SYNC_FLAG, &cpudata->status);		\
+	}								\
+} while(0)
+
+#define __ipipe_syscall_watched_p(p, sc)	\
+	(((p)->flags & PF_EVNOTIFY) || (unsigned long)sc >= NR_syscalls)
+
+#else /* !CONFIG_IPIPE */
+
+#define task_hijacked(p)		0
+#define ipipe_trap_notify(t,r)  	0
+
+#define __ipipe_stall_root_raw()	do { } while(0)
+#define __ipipe_unstall_root_raw()	do { } while(0)
+
+#define IRQ_SYSTMR     IRQ_CORETMR
+#define IRQ_PRIOTMR    IRQ_CORETMR
+
+#endif /* CONFIG_IPIPE */
+
+int init_irqthreads(void);
+
+#endif	/* !__BFINNOMMU_IPIPE_H */
diff -uNrp --exclude CVS linux-head/include/asm-blackfin/irqchip.h linux-2.6.x/include/asm-blackfin/irqchip.h
--- linux-head/include/asm-blackfin/irqchip.h	2005-10-28 05:01:42.000000000 +0200
+++ linux-2.6.x/include/asm-blackfin/irqchip.h	2006-10-09 22:36:50.000000000 +0200
@@ -14,6 +14,10 @@
 struct irqdesc;
 struct pt_regs;
 struct seq_file;
+#ifdef CONFIG_IPIPE
+#include <asm/semaphore.h>
+struct task_struct;
+#endif /* CONFIG_IPIPE */
 
 typedef void (*irq_handler_t) (unsigned int, struct irqdesc *,
 			       struct pt_regs *);
@@ -59,6 +63,11 @@ struct irqdesc {
 	void *chipdata;
 	void *data;
 	unsigned int disable_depth;
+#ifdef CONFIG_IPIPE
+	struct task_struct *thread;
+	struct semaphore thrsem;
+	void (*thrhandler)(unsigned irq, struct pt_regs *);
+#endif /* CONFIG_IPIPE */
 
 	unsigned int triggered:1;	/* IRQ has occurred           */
 	unsigned int running:1;	/* IRQ is running             */
diff -uNrp --exclude CVS linux-head/include/asm-blackfin/system.h linux-2.6.x/include/asm-blackfin/system.h
--- linux-head/include/asm-blackfin/system.h	2006-10-08 19:19:40.000000000 +0200
+++ linux-2.6.x/include/asm-blackfin/system.h	2006-10-12 20:08:50.000000000 +0200
@@ -9,7 +9,7 @@
  * Created:     25Jan2001 - Tony Kou
  * Description: system.h include file
  *
- * Rev:       $Id: system.h,v 1.25 2006/09/29 05:36:56 rgetz Exp $
+ * Rev:       $Id: system.h,v 1.5 2006/10/12 18:08:50 rpm Exp $
  *
  * Modified:     22Sep2006 - Robin Getz
  *                - move include blackfin.h down, so I can get access to
@@ -46,6 +46,148 @@
 
 extern unsigned long irq_flags;
 
+#ifdef CONFIG_IPIPE
+
+void __ipipe_stall_root(void);
+
+void __ipipe_unstall_root(void);
+
+unsigned long __ipipe_test_root(void);
+
+unsigned long __ipipe_test_and_stall_root(void);
+
+void __ipipe_restore_root(unsigned long flags);
+
+#ifdef CONFIG_DEBUG_HWERR
+#define __all_masked_irq_flags  0x3f
+#define __save_and_cli_hw(x) do {		\
+	__asm__ __volatile__ (			\
+		"cli %0;\n\tsti %1;"		\
+		:"=&d"(x): "d" (0x3F));		\
+} while (0)
+#else
+#define __all_masked_irq_flags  0x1f
+#define __save_and_cli_hw(x) do {	\
+	__asm__ __volatile__ (          \
+		"cli %0;"		\
+		:"=&d"(x):);		\
+} while (0)
+#endif
+
+#define irqs_enabled_from_flags_hw(x) ((x) != __all_masked_irq_flags)
+
+#define local_test_iflag_hw(x) irqs_enabled_from_flags_hw(x)
+
+#define local_save_flags(x)						\
+do {								\
+	(x) = __ipipe_test_root()?__all_masked_irq_flags:irq_flags;	\
+} while(0)
+
+#define local_irq_save(x)						\
+do {								\
+	(x) = __ipipe_test_and_stall_root()?__all_masked_irq_flags:irq_flags; \
+} while(0)
+
+#define local_irq_restore(x)	__ipipe_restore_root((x) == __all_masked_irq_flags)
+#define local_irq_disable()	__ipipe_stall_root()
+#define local_irq_enable()	__ipipe_unstall_root()
+#define irqs_disabled()	__ipipe_test_root()
+
+#define local_save_flags_hw(x) asm volatile ("cli %0;"  \
+					     "sti %0;"	\
+					     :"=d"(x):)
+
+#define	irqs_disabled_hw()			\
+({						\
+	unsigned long flags;			\
+	local_save_flags_hw(flags);		\
+	!irqs_enabled_from_flags_hw(flags);	\
+})
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+
+#include <linux/ipipe_trace.h>
+
+#define local_irq_disable_hw()		\
+do {						\
+	int _tmp_dummy;				\
+	if (!irqs_disabled_hw())		\
+		ipipe_trace_begin(0x80000000);	\
+	__asm__ __volatile__ ( "cli %0;":"=d" (_tmp_dummy):);	\
+} while (0)
+
+#define local_irq_enable_hw()			\
+do {						\
+	if (irqs_disabled_hw())	     		\
+		ipipe_trace_end(0x80000000);	\
+	__asm__ __volatile__ ("sti %0;"::"d"(irq_flags)); \
+} while (0)
+
+#define local_irq_save_hw(x)			\
+do {						\
+	__save_and_cli_hw(x);			\
+	if (local_test_iflag_hw(x))		\
+		ipipe_trace_begin(0x80000001);	\
+} while (0)
+
+#define local_irq_restore_hw(x)		\
+do {						\
+	if (local_test_iflag_hw(x)) {		\
+		ipipe_trace_end(0x80000001);	\
+		local_irq_enable_hw_notrace();	\
+	}					\
+} while (0)
+
+#define local_irq_disable_hw_notrace()  \
+do {					\
+	int _tmp_dummy;						\
+	__asm__ __volatile__ ( "cli %0;":"=d" (_tmp_dummy):);	\
+} while(0)
+
+#define local_irq_enable_hw_notrace() \
+	__asm__ __volatile__ ("sti %0;"::"d"(irq_flags))
+
+#define local_irq_save_hw_notrace(x) __save_and_cli_hw(x)
+
+#define local_irq_restore_hw_notrace(x)	\
+do {						\
+	if (local_test_iflag_hw(x))		\
+		local_irq_enable_hw_notrace();	\
+} while (0)
+
+#else /* CONFIG_IPIPE_TRACE_IRQSOFF */
+
+#define local_irq_enable_hw() 	\
+do {					\
+	__asm__ __volatile__ (		\
+		"sti %0;"		\
+		::"d"(irq_flags));	\
+} while(0)
+
+#define local_irq_disable_hw() 	\
+do {					\
+	int _tmp_dummy;			\
+	__asm__ __volatile__ (		\
+		"cli %0;"		\
+		:"=d" (_tmp_dummy):);	\
+} while(0)
+
+#define local_irq_restore_hw(x) do {			\
+	if (irqs_enabled_from_flags_hw(x))		\
+		local_irq_enable_hw();			\
+} while (0)
+
+#define local_irq_save_hw(x) __save_and_cli_hw(x)
+
+#define local_irq_disable_hw_notrace()	local_irq_disable_hw()
+#define local_irq_enable_hw_notrace()	local_irq_enable_hw()
+#define local_irq_save_hw_notrace(flags)	local_irq_save_hw(flags)
+#define local_irq_restore_hw_notrace(flags)	local_irq_restore_hw(flags)
+
+#endif  /* CONFIG_IPIPE_TRACE_IRQSOFF */
+
+#else /* !CONFIG_IPIPE */
+
 #define local_irq_enable() do {		\
 	__asm__ __volatile__ (		\
 		"sti %0;"		\
@@ -59,22 +201,6 @@ extern unsigned long irq_flags;
 		:"=d" (_tmp_dummy):);	\
 } while (0)
 
-#if defined(ANOMALY_05000244) && defined (CONFIG_BLKFIN_CACHE)
-#define idle_with_irq_disabled() do {   \
-        __asm__ __volatile__ (          \
-                "nop; nop;\n"           \
-                ".align 8;\n"           \
-                "sti %0; idle;\n"       \
-                ::"d" (irq_flags));     \
-} while (0)
-#else
-#define idle_with_irq_disabled() do {   \
-	__asm__ __volatile__ (          \
-		".align 8;\n"           \
-		"sti %0; idle;\n"       \
-		::"d" (irq_flags));     \
-} while (0)
-#endif
 
 #ifdef CONFIG_DEBUG_HWERR
 #define __save_and_cli(x) do {			\
@@ -115,6 +241,31 @@ extern unsigned long irq_flags;
 	!irqs_enabled_from_flags(flags);	\
 })
 
+#define local_irq_save_hw(flags)	local_irq_save(flags)
+#define local_irq_restore_hw(flags)	local_irq_restore(flags)
+#define local_irq_enable_hw()		local_irq_enable()
+#define local_irq_disable_hw(flags)	local_irq_disable()
+#define irqs_disabled_hw(flags)	irqs_disabled()
+
+#endif /* CONFIG_IPIPE */
+
+#if defined(ANOMALY_05000244) && defined (CONFIG_BLKFIN_CACHE)
+#define idle_with_irq_disabled() do {   \
+        __asm__ __volatile__ (          \
+                "nop; nop;\n"           \
+                ".align 8;\n"           \
+                "sti %0; idle;\n"       \
+                ::"d" (irq_flags));     \
+} while (0)
+#else
+#define idle_with_irq_disabled() do {   \
+	__asm__ __volatile__ (          \
+		".align 8;\n"           \
+		"sti %0; idle;\n"       \
+		::"d" (irq_flags));     \
+} while (0)
+#endif
+
 /*
  * Force strict CPU ordering.
  */
@@ -154,7 +305,7 @@ static inline unsigned long __xchg(unsig
 	unsigned long tmp = 0;
 	unsigned long flags = 0;
 
-	local_irq_save(flags);
+	local_irq_save_hw(flags);
 
 	switch (size) {
 	case 1:
@@ -176,7 +327,7 @@ static inline unsigned long __xchg(unsig
 			 : "=&d" (tmp) : "d" (x), "m" (*__xg(ptr)) : "memory");
 		break;
 	}
-	local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 	return tmp;
 }
 
@@ -191,7 +342,7 @@ static inline unsigned long __cmpxchg(vo
 	unsigned long tmp = 0;
 	unsigned long flags = 0;
 
-	local_irq_save(flags);
+	local_irq_save_hw(flags);
 
 	switch (size) {
 	case 1:
@@ -222,7 +373,7 @@ static inline unsigned long __cmpxchg(vo
 			 : "=&d" (tmp) : "d" (old), "d" (new), "m" (*__xg(ptr)) : "memory");
 		break;
 	}
-	local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 	return tmp;
 }
 
diff -uNrp --exclude CVS linux-head/include/linux/hardirq.h linux-2.6.x/include/linux/hardirq.h
--- linux-head/include/linux/hardirq.h	2006-03-23 06:54:07.000000000 +0100
+++ linux-2.6.x/include/linux/hardirq.h	2006-10-09 22:36:50.000000000 +0200
@@ -87,8 +87,21 @@ extern void synchronize_irq(unsigned int
 # define synchronize_irq(irq)	barrier()
 #endif
 
+#ifdef CONFIG_IPIPE
+#define nmi_enter() \
+do { \
+    if (ipipe_current_domain == ipipe_root_domain) \
+	irq_enter(); \
+} while(0)
+#define nmi_exit() \
+do { \
+    if (ipipe_current_domain == ipipe_root_domain) \
+	sub_preempt_count(HARDIRQ_OFFSET); \
+} while(0)
+#else /* !CONFIG_IPIPE */
 #define nmi_enter()		irq_enter()
 #define nmi_exit()		sub_preempt_count(HARDIRQ_OFFSET)
+#endif /* CONFIG_IPIPE */
 
 struct task_struct;
 
diff -uNrp --exclude CVS linux-head/include/linux/ipipe.h linux-2.6.x/include/linux/ipipe.h
--- linux-head/include/linux/ipipe.h	1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.x/include/linux/ipipe.h	2006-10-15 16:24:07.000000000 +0200
@@ -0,0 +1,770 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe.h
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_H
+#define __LINUX_IPIPE_H
+
+#include <linux/config.h>
+#include <linux/spinlock.h>
+#include <linux/cache.h>
+#include <linux/percpu.h>
+#include <asm/ipipe.h>
+
+#ifdef CONFIG_IPIPE
+
+#define IPIPE_VERSION_STRING	IPIPE_ARCH_STRING
+#define IPIPE_RELEASE_NUMBER	((IPIPE_MAJOR_NUMBER << 16) | \
+				 (IPIPE_MINOR_NUMBER <<  8) | \
+				 (IPIPE_PATCH_NUMBER))
+
+#ifndef BROKEN_BUILTIN_RETURN_ADDRESS
+#define __BUILTIN_RETURN_ADDRESS0 ((unsigned long)__builtin_return_address(0))
+#define __BUILTIN_RETURN_ADDRESS1 ((unsigned long)__builtin_return_address(1))
+#endif /* !BUILTIN_RETURN_ADDRESS */
+
+#define IPIPE_ROOT_PRIO		100
+#define IPIPE_ROOT_ID		0
+#define IPIPE_ROOT_NPTDKEYS	4	/* Must be <= BITS_PER_LONG */
+
+#define IPIPE_RESET_TIMER	0x1
+#define IPIPE_GRAB_TIMER	0x2
+
+/* Global domain flags */
+#define IPIPE_SPRINTK_FLAG	0	/* Synchronous printk() allowed */
+#define IPIPE_AHEAD_FLAG	1	/* Domain always heads the pipeline */
+
+/* Per-cpu pipeline status */
+#define IPIPE_STALL_FLAG	0	/* Stalls a pipeline stage -- guaranteed at bit #0 */
+#define IPIPE_SYNC_FLAG		1	/* The interrupt syncer is running for the domain */
+#define IPIPE_NOSTACK_FLAG	2	/* Domain currently runs on a foreign stack */
+
+#define IPIPE_SYNC_MASK		(1 << IPIPE_SYNC_FLAG)
+
+/* Interrupt control bits */
+#define IPIPE_HANDLE_FLAG	0
+#define IPIPE_PASS_FLAG		1
+#define IPIPE_ENABLE_FLAG	2
+#define IPIPE_DYNAMIC_FLAG	IPIPE_HANDLE_FLAG
+#define IPIPE_STICKY_FLAG	3
+#define IPIPE_SYSTEM_FLAG	4
+#define IPIPE_LOCK_FLAG		5
+#define IPIPE_SHARED_FLAG	6
+#define IPIPE_WIRED_FLAG	7
+#define IPIPE_EXCLUSIVE_FLAG	8
+
+#define IPIPE_HANDLE_MASK	(1 << IPIPE_HANDLE_FLAG)
+#define IPIPE_PASS_MASK		(1 << IPIPE_PASS_FLAG)
+#define IPIPE_ENABLE_MASK	(1 << IPIPE_ENABLE_FLAG)
+#define IPIPE_DYNAMIC_MASK	IPIPE_HANDLE_MASK
+#define IPIPE_STICKY_MASK	(1 << IPIPE_STICKY_FLAG)
+#define IPIPE_SYSTEM_MASK	(1 << IPIPE_SYSTEM_FLAG)
+#define IPIPE_LOCK_MASK		(1 << IPIPE_LOCK_FLAG)
+#define IPIPE_SHARED_MASK	(1 << IPIPE_SHARED_FLAG)
+#define IPIPE_WIRED_MASK	(1 << IPIPE_WIRED_FLAG)
+#define IPIPE_EXCLUSIVE_MASK	(1 << IPIPE_EXCLUSIVE_FLAG)
+
+#define IPIPE_DEFAULT_MASK	(IPIPE_HANDLE_MASK|IPIPE_PASS_MASK)
+#define IPIPE_STDROOT_MASK	(IPIPE_HANDLE_MASK|IPIPE_PASS_MASK|IPIPE_SYSTEM_MASK)
+
+#define IPIPE_EVENT_SELF        0x80000000
+
+/* Number of virtual IRQs */
+#define IPIPE_NR_VIRQS		BITS_PER_LONG
+/* First virtual IRQ # */
+#define IPIPE_VIRQ_BASE		(((IPIPE_NR_XIRQS + BITS_PER_LONG - 1) / BITS_PER_LONG) * BITS_PER_LONG)
+/* Total number of IRQ slots */
+#define IPIPE_NR_IRQS		(IPIPE_VIRQ_BASE + IPIPE_NR_VIRQS)
+/* Number of indirect words needed to map the whole IRQ space. */
+#define IPIPE_IRQ_IWORDS	((IPIPE_NR_IRQS + BITS_PER_LONG - 1) / BITS_PER_LONG)
+#define IPIPE_IRQ_IMASK		(BITS_PER_LONG - 1)
+#define IPIPE_IRQMASK_ANY	(~0L)
+#define IPIPE_IRQMASK_VIRT	(IPIPE_IRQMASK_ANY << (IPIPE_VIRQ_BASE / BITS_PER_LONG))
+
+#ifdef CONFIG_SMP
+
+#define IPIPE_NR_CPUS		NR_CPUS
+#define ipipe_declare_cpuid	int cpuid
+#define ipipe_load_cpuid()	do { \
+					(cpuid) = ipipe_processor_id();	\
+				} while(0)
+#define ipipe_lock_cpu(flags)	do { \
+					local_irq_save_hw(flags); \
+					(cpuid) = ipipe_processor_id(); \
+				} while(0)
+#define ipipe_unlock_cpu(flags)	local_irq_restore_hw(flags)
+#define ipipe_get_cpu(flags)	ipipe_lock_cpu(flags)
+#define ipipe_put_cpu(flags)	ipipe_unlock_cpu(flags)
+#define ipipe_current_domain	per_cpu(ipipe_percpu_domain, ipipe_processor_id())
+
+#else /* !CONFIG_SMP */
+
+#define IPIPE_NR_CPUS		1
+#define ipipe_declare_cpuid	const int cpuid = 0
+#define ipipe_load_cpuid()	do { } while(0)
+#define ipipe_lock_cpu(flags)	local_irq_save_hw(flags)
+#define ipipe_unlock_cpu(flags)	local_irq_restore_hw(flags)
+#define ipipe_get_cpu(flags)	do { (void)(flags); } while(0)
+#define ipipe_put_cpu(flags)	do { } while(0)
+#define ipipe_current_domain	per_cpu(ipipe_percpu_domain, 0)
+
+#endif /* CONFIG_SMP */
+
+#define ipipe_virtual_irq_p(irq)	((irq) >= IPIPE_VIRQ_BASE && \
+					 (irq) < IPIPE_NR_IRQS)
+
+typedef void (*ipipe_irq_handler_t)(unsigned irq,
+				    void *cookie);
+
+#define IPIPE_SAME_HANDLER	((ipipe_irq_handler_t)(-1))
+
+typedef int (*ipipe_irq_ackfn_t)(unsigned irq);
+
+typedef int (*ipipe_event_handler_t)(unsigned event,
+				     struct ipipe_domain *from,
+				     void *data);
+struct ipipe_domain {
+
+	struct list_head p_link;	/* Link in pipeline */
+
+	struct ipcpudata {
+		unsigned long status;
+		unsigned long irq_pending_hi;
+		unsigned long irq_pending_lo[IPIPE_IRQ_IWORDS];
+		struct ipirqcnt {
+			unsigned long pending_hits;
+			unsigned long total_hits;
+		} irq_counters[IPIPE_NR_IRQS];
+		unsigned long long evsync;
+	} ____cacheline_aligned_in_smp cpudata[IPIPE_NR_CPUS];
+
+	struct {
+		unsigned long control;
+		ipipe_irq_ackfn_t acknowledge;
+		ipipe_irq_handler_t handler;
+		void *cookie;
+	} ____cacheline_aligned irqs[IPIPE_NR_IRQS];
+
+	ipipe_event_handler_t evhand[IPIPE_NR_EVENTS]; /* Event handlers. */
+	unsigned long long evself;	/* Self-monitored event bits. */
+	unsigned long flags;
+	unsigned domid;
+	const char *name;
+	int priority;
+	void *pdd;
+};
+
+#define IPIPE_HEAD_PRIORITY	(-1) /* For domains always heading the pipeline */
+
+struct ipipe_domain_attr {
+
+	unsigned domid;		/* Domain identifier -- Magic value set by caller */
+	const char *name;	/* Domain name -- Warning: won't be dup'ed! */
+	int priority;		/* Priority in interrupt pipeline */
+	void (*entry) (void);	/* Domain entry point */
+	void *pdd;		/* Per-domain (opaque) data pointer */
+};
+
+/* The following macros must be used hw interrupts off. */
+
+#define __ipipe_irq_cookie(ipd,irq)	(ipd)->irqs[irq].cookie
+#define __ipipe_irq_handler(ipd,irq)	(ipd)->irqs[irq].handler
+
+#define __ipipe_cpudata_irq_hits(ipd,cpuid,irq)	((ipd)->cpudata[cpuid].irq_counters[irq].total_hits)
+
+#define __ipipe_set_irq_bit(ipd,cpuid,irq) \
+do { \
+	if (!test_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control)) { \
+		__set_bit(irq & IPIPE_IRQ_IMASK,&(ipd)->cpudata[cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT]); \
+		__set_bit(irq >> IPIPE_IRQ_ISHIFT,&(ipd)->cpudata[cpuid].irq_pending_hi); \
+	} \
+} while(0)
+
+#define __ipipe_clear_pend(ipd,cpuid,irq) \
+do { \
+	__clear_bit(irq & IPIPE_IRQ_IMASK,&(ipd)->cpudata[cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT]); \
+	if ((ipd)->cpudata[cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT] == 0) \
+		__clear_bit(irq >> IPIPE_IRQ_ISHIFT,&(ipd)->cpudata[cpuid].irq_pending_hi); \
+} while(0)
+
+#define __ipipe_lock_irq(ipd,cpuid,irq) \
+do { \
+	if (!test_and_set_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control)) \
+		__ipipe_clear_pend(ipd,cpuid,irq); \
+} while(0)
+
+#define __ipipe_unlock_irq(ipd,irq) \
+do { \
+	int __cpuid, __nr_cpus = num_online_cpus(); \
+	if (test_and_clear_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control)) \
+		for (__cpuid = 0; __cpuid < __nr_cpus; __cpuid++) \
+			if ((ipd)->cpudata[__cpuid].irq_counters[irq].pending_hits > 0) { /* We need atomic ops next. */ \
+				set_bit(irq & IPIPE_IRQ_IMASK,&(ipd)->cpudata[__cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT]); \
+				set_bit(irq >> IPIPE_IRQ_ISHIFT,&(ipd)->cpudata[__cpuid].irq_pending_hi); \
+			} \
+} while(0)
+
+#define __ipipe_clear_irq(ipd,irq) \
+do { \
+	int __cpuid, __nr_cpus = num_online_cpus(); \
+	clear_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control); \
+	for (__cpuid = 0; __cpuid < __nr_cpus; __cpuid++) { \
+		(ipd)->cpudata[__cpuid].irq_counters[irq].pending_hits = 0; \
+		__ipipe_clear_pend(ipd,__cpuid,irq); \
+	} \
+} while(0)
+
+#ifdef __RAW_SPIN_LOCK_UNLOCKED
+#define spin_lock_hw(x)		__raw_spin_lock(&(x)->raw_lock)
+#define spin_trylock_hw(x)		__raw_spin_trylock(&(x)->raw_lock)
+#define spin_unlock_hw(x)		__raw_spin_unlock(&(x)->raw_lock)
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
+#define write_lock_hw(x)		__raw_write_lock(&(x)->raw_lock)
+#define write_trylock_hw(x)		__raw_write_trylock(&(x)->raw_lock)
+#define write_unlock_hw(x)		__raw_write_unlock(&(x)->raw_lock)
+#define read_lock_hw(x)		__raw_read_lock(&(x)->raw_lock)
+#define read_trylock_hw(x)		__raw_read_trylock(&(x)->raw_lock)
+#define read_unlock_hw(x)		__raw_read_unlock(&(x)->raw_lock)
+#else /* UP non-debug */
+#define write_lock_hw(lock)		do { (void)(lock); } while (0)
+#define write_trylock_hw(lock)	({ (void)(lock); 1; })
+#define write_unlock_hw(lock)		do { (void)(lock); } while (0)
+#define read_lock_hw(lock)		do { (void)(lock); } while (0)
+#define read_trylock_hw(lock)		({ (void)(lock); 1; })
+#define read_unlock_hw(lock)		do { (void)(lock); } while (0)
+#endif /* CONFIG_SMP || CONFIG_DEBUG_SPINLOCK */
+#else	/* !__RAW_SPIN_LOCK_UNLOCKED */
+#define spin_lock_hw(x)		_raw_spin_lock(x)
+#define spin_unlock_hw(x)		_raw_spin_unlock(x)
+#define spin_trylock_hw(x)		_raw_spin_trylock(x)
+#define write_lock_hw(x)		_raw_write_lock(x)
+#define write_unlock_hw(x)		_raw_write_unlock(x)
+#define write_trylock_hw(x)		_raw_write_trylock(x)
+#define read_lock_hw(x)		_raw_read_lock(x)
+#define read_unlock_hw(x)		_raw_read_unlock(x)
+#endif	/* __RAW_SPIN_LOCK_UNLOCKED */
+
+typedef spinlock_t			ipipe_spinlock_t;
+typedef rwlock_t			ipipe_rwlock_t;
+#define IPIPE_SPIN_LOCK_UNLOCKED	SPIN_LOCK_UNLOCKED
+#define IPIPE_RW_LOCK_UNLOCKED		RW_LOCK_UNLOCKED
+
+#define spin_lock_irqsave_hw(x,flags)		\
+do {						\
+	local_irq_save_hw(flags);		\
+	spin_lock_hw(x);			\
+} while (0)
+
+#define spin_unlock_irqrestore_hw(x,flags)	\
+do {						\
+	spin_unlock_hw(x);			\
+	local_irq_restore_hw(flags);		\
+} while (0)
+
+#define spin_lock_irq_hw(x)			\
+do {						\
+	local_irq_disable_hw();			\
+	spin_lock_hw(x);			\
+} while (0)
+
+#define spin_unlock_irq_hw(x)			\
+do {						\
+	spin_unlock_hw(x);			\
+	local_irq_enable_hw();			\
+} while (0)
+
+#define read_lock_irqsave_hw(lock, flags)	\
+do {						\
+	local_irq_save_hw(flags);		\
+	read_lock_hw(lock);			\
+} while (0)
+
+#define read_unlock_irqrestore_hw(lock, flags)	\
+do {						\
+	read_unlock_hw(lock);			\
+	local_irq_restore_hw(flags);		\
+} while (0)
+
+#define write_lock_irqsave_hw(lock, flags)	\
+do {						\
+	local_irq_save_hw(flags);		\
+	write_lock_hw(lock);			\
+} while (0)
+
+#define write_unlock_irqrestore_hw(lock, flags)	\
+do {						\
+	write_unlock_hw(lock);			\
+	local_irq_restore_hw(flags);		\
+} while (0)
+
+DECLARE_PER_CPU(struct ipipe_domain *, ipipe_percpu_domain);
+
+extern struct ipipe_domain ipipe_root;
+
+#define ipipe_root_domain (&ipipe_root)
+
+extern unsigned __ipipe_printk_virq;
+
+extern unsigned long __ipipe_virtual_irq_map;
+
+extern struct list_head __ipipe_pipeline;
+
+extern ipipe_spinlock_t __ipipe_pipelock;
+
+extern int __ipipe_event_monitors[];
+
+/* Private interface */
+
+void ipipe_init(void);
+
+#ifdef CONFIG_PROC_FS
+void ipipe_init_proc(void);
+
+#ifdef CONFIG_IPIPE_TRACE
+void __ipipe_init_tracer(void);
+#else /* !CONFIG_IPIPE_TRACE */
+#define __ipipe_init_tracer()       do { } while(0)
+#endif /* CONFIG_IPIPE_TRACE */
+
+#else	/* !CONFIG_PROC_FS */
+#define ipipe_init_proc()	do { } while(0)
+#endif	/* CONFIG_PROC_FS */
+
+void __ipipe_init_stage(struct ipipe_domain *ipd);
+
+void __ipipe_cleanup_domain(struct ipipe_domain *ipd);
+
+void __ipipe_add_domain_proc(struct ipipe_domain *ipd);
+
+void __ipipe_remove_domain_proc(struct ipipe_domain *ipd);
+
+void __ipipe_flush_printk(unsigned irq, void *cookie);
+
+void __ipipe_stall_root(void);
+
+void __ipipe_unstall_root(void);
+
+unsigned long __ipipe_test_root(void);
+
+unsigned long __ipipe_test_and_stall_root(void);
+
+void fastcall __ipipe_walk_pipeline(struct list_head *pos, int cpuid);
+
+void fastcall __ipipe_restore_root(unsigned long x);
+
+int fastcall __ipipe_schedule_irq(unsigned irq, struct list_head *head);
+
+int fastcall __ipipe_dispatch_event(unsigned event, void *data);
+
+int fastcall __ipipe_dispatch_wired(struct ipipe_domain *head, unsigned irq);
+
+void fastcall __ipipe_sync_stage(unsigned long syncmask);
+
+#ifndef __ipipe_sync_pipeline
+#define __ipipe_sync_pipeline(syncmask) __ipipe_sync_stage(syncmask)
+#endif
+
+#ifndef __ipipe_run_irqtail
+#define __ipipe_run_irqtail() do { } while(0)
+#endif
+
+#define __ipipe_pipeline_head_p(ipd) (&(ipd)->p_link == __ipipe_pipeline.next)
+
+/*
+ * Keep the following as a macro, so that client code could check for
+ * the support of the invariant pipeline head optimization.
+ */
+#define __ipipe_pipeline_head() list_entry(__ipipe_pipeline.next,struct ipipe_domain,p_link)
+
+#define __ipipe_event_monitored_p(ev) \
+	(__ipipe_event_monitors[ev] > 0 || (ipipe_current_domain->evself & (1LL << ev)))
+
+#ifdef CONFIG_SMP
+
+cpumask_t __ipipe_set_irq_affinity(unsigned irq,
+				   cpumask_t cpumask);
+
+int fastcall __ipipe_send_ipi(unsigned ipi,
+			      cpumask_t cpumask);
+
+#endif /* CONFIG_SMP */
+
+/* Called with hw interrupts off. */
+static inline void __ipipe_switch_to(struct ipipe_domain *out,
+				     struct ipipe_domain *in, int cpuid)
+{
+	void ipipe_suspend_domain(void);
+
+	/*
+	 * "in" is guaranteed to be closer than "out" from the head of the
+	 * pipeline (and obviously different).
+	 */
+
+	out->cpudata[cpuid].evsync = 0;
+	per_cpu(ipipe_percpu_domain, cpuid) = in;
+
+	ipipe_suspend_domain();	/* Sync stage and propagate interrupts. */
+	ipipe_load_cpuid();	/* Processor might have changed. */
+
+	if (per_cpu(ipipe_percpu_domain, cpuid) == in)
+		/*
+		 * Otherwise, something has changed the current domain under
+		 * our feet recycling the register set; do not override.
+		 */
+		per_cpu(ipipe_percpu_domain, cpuid) = out;
+}
+
+#define ipipe_sigwake_notify(p)	\
+do {					\
+	if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_SIGWAKE)) \
+		__ipipe_dispatch_event(IPIPE_EVENT_SIGWAKE,p);		\
+} while(0)
+
+#define ipipe_exit_notify(p)	\
+do {				\
+	if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_EXIT)) \
+		__ipipe_dispatch_event(IPIPE_EVENT_EXIT,p);		\
+} while(0)
+
+#define ipipe_setsched_notify(p)	\
+do {					\
+	if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_SETSCHED)) \
+		__ipipe_dispatch_event(IPIPE_EVENT_SETSCHED,p);		\
+} while(0)
+
+#define ipipe_schedule_notify(prev, next)				\
+do {									\
+	if ((((prev)->flags|(next)->flags) & PF_EVNOTIFY) &&		\
+	    __ipipe_event_monitored_p(IPIPE_EVENT_SCHEDULE))		\
+		__ipipe_dispatch_event(IPIPE_EVENT_SCHEDULE,next);	\
+} while(0)
+
+#define ipipe_trap_notify(ex, regs)		\
+({						\
+	ipipe_declare_cpuid;			\
+	int ret = 0;				\
+	ipipe_load_cpuid();			\
+	if ((test_bit(IPIPE_NOSTACK_FLAG, &ipipe_current_domain->cpudata[cpuid].status) || \
+	     ((current)->flags & PF_EVNOTIFY)) &&			\
+	    __ipipe_event_monitored_p(ex))				\
+		ret = __ipipe_dispatch_event(ex, regs);			\
+	ret;								\
+})
+
+static inline void ipipe_init_notify(struct task_struct *p)
+{
+	if (__ipipe_event_monitored_p(IPIPE_EVENT_INIT))
+		__ipipe_dispatch_event(IPIPE_EVENT_INIT,p);
+}
+
+struct mm_struct;
+
+static inline void ipipe_cleanup_notify(struct mm_struct *mm)
+{
+	if (__ipipe_event_monitored_p(IPIPE_EVENT_CLEANUP))
+		__ipipe_dispatch_event(IPIPE_EVENT_CLEANUP,mm);
+}
+
+/* Public interface */
+
+int ipipe_register_domain(struct ipipe_domain *ipd,
+			  struct ipipe_domain_attr *attr);
+
+int ipipe_unregister_domain(struct ipipe_domain *ipd);
+
+void ipipe_suspend_domain(void);
+
+int ipipe_virtualize_irq(struct ipipe_domain *ipd,
+			 unsigned irq,
+			 ipipe_irq_handler_t handler,
+			 void *cookie,
+			 ipipe_irq_ackfn_t acknowledge,
+			 unsigned modemask);
+
+static inline int ipipe_share_irq(unsigned irq,
+				  ipipe_irq_ackfn_t acknowledge)
+{
+	return ipipe_virtualize_irq(ipipe_current_domain,
+				    irq,
+				    IPIPE_SAME_HANDLER,
+				    NULL,
+				    acknowledge,
+				    IPIPE_SHARED_MASK | IPIPE_HANDLE_MASK |
+				    IPIPE_PASS_MASK);
+}
+
+int ipipe_control_irq(unsigned irq,
+		      unsigned clrmask,
+		      unsigned setmask);
+
+unsigned ipipe_alloc_virq(void);
+
+int ipipe_free_virq(unsigned virq);
+
+int fastcall ipipe_trigger_irq(unsigned irq);
+
+static inline int ipipe_propagate_irq(unsigned irq)
+{
+	return __ipipe_schedule_irq(irq, ipipe_current_domain->p_link.next);
+}
+
+static inline int ipipe_schedule_irq(unsigned irq)
+{
+	return __ipipe_schedule_irq(irq, &ipipe_current_domain->p_link);
+}
+
+void fastcall ipipe_stall_pipeline_from(struct ipipe_domain *ipd);
+
+unsigned long fastcall ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd);
+
+void fastcall ipipe_unstall_pipeline_from(struct ipipe_domain *ipd);
+
+unsigned long fastcall ipipe_test_and_unstall_pipeline_from(struct ipipe_domain *ipd);
+
+void fastcall ipipe_restore_pipeline_from(struct ipipe_domain *ipd,
+					  unsigned long x);
+
+static inline unsigned long ipipe_test_pipeline_from(struct ipipe_domain *ipd)
+{
+	unsigned long flags, x;
+	ipipe_declare_cpuid;
+
+	ipipe_get_cpu(flags);
+	x = test_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+	ipipe_put_cpu(flags);
+
+	return x;
+}
+
+static inline void ipipe_restore_pipeline_nosync(struct ipipe_domain *ipd,
+						 unsigned long x, int cpuid)
+{
+	/*
+	 * If cpuid is current, then it must be held on entry
+	 * (ipipe_get_cpu/local_irq_save_hw/local_irq_disable_hw).
+	 */
+
+	if (x)
+		__set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+	else
+		__clear_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+}
+
+static inline void ipipe_stall_pipeline_head(void)
+{
+	ipipe_declare_cpuid;
+	unsigned long flags;
+
+	ipipe_lock_cpu(flags);
+	__set_bit(IPIPE_STALL_FLAG, &__ipipe_pipeline_head()->cpudata[cpuid].status);
+}
+
+static inline unsigned long ipipe_test_and_stall_pipeline_head(void)
+{
+	unsigned long flags;
+	ipipe_declare_cpuid;
+
+	ipipe_lock_cpu(flags);
+	return __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_pipeline_head()->cpudata[cpuid].status);
+}
+
+void ipipe_unstall_pipeline_head(void);
+
+void fastcall __ipipe_restore_pipeline_head(struct ipipe_domain *head,
+					    unsigned long x);
+
+static inline void ipipe_restore_pipeline_head(unsigned long x)
+{
+	struct ipipe_domain *head = __ipipe_pipeline_head();
+	/* On some archs, __test_and_set_bit() might return different
+	 * truth value than test_bit(), so we test the exclusive OR of
+	 * both statuses, assuming that the lowest bit is always set in
+	 * the truth value (if this is wrong, the failed optimization will
+	 * be caught in __ipipe_restore_pipeline_head() if
+	 * CONFIG_DEBUG_KERNEL is set). */
+	if ((x ^ test_bit(IPIPE_STALL_FLAG, &head->cpudata[ipipe_processor_id()].status)) & 1)
+		__ipipe_restore_pipeline_head(head,x);
+}
+
+#define ipipe_unstall_pipeline() \
+	ipipe_unstall_pipeline_from(ipipe_current_domain)
+
+#define ipipe_test_and_unstall_pipeline() \
+	ipipe_test_and_unstall_pipeline_from(ipipe_current_domain)
+
+#define ipipe_test_pipeline() \
+	ipipe_test_pipeline_from(ipipe_current_domain)
+
+#define ipipe_test_and_stall_pipeline() \
+	ipipe_test_and_stall_pipeline_from(ipipe_current_domain)
+
+#define ipipe_stall_pipeline() \
+	ipipe_stall_pipeline_from(ipipe_current_domain)
+
+#define ipipe_restore_pipeline(x) \
+	ipipe_restore_pipeline_from(ipipe_current_domain, (x))
+
+void ipipe_init_attr(struct ipipe_domain_attr *attr);
+
+int ipipe_get_sysinfo(struct ipipe_sysinfo *sysinfo);
+
+int ipipe_tune_timer(unsigned long ns,
+		     int flags);
+
+unsigned long ipipe_critical_enter(void (*syncfn) (void));
+
+void ipipe_critical_exit(unsigned long flags);
+
+static inline void ipipe_set_printk_sync(struct ipipe_domain *ipd)
+{
+	set_bit(IPIPE_SPRINTK_FLAG, &ipd->flags);
+}
+
+static inline void ipipe_set_printk_async(struct ipipe_domain *ipd)
+{
+	clear_bit(IPIPE_SPRINTK_FLAG, &ipd->flags);
+}
+
+static inline void ipipe_set_foreign_stack(struct ipipe_domain *ipd)
+{
+	/* Must be called hw interrupts off. */
+	ipipe_declare_cpuid;
+	ipipe_load_cpuid();
+	__set_bit(IPIPE_NOSTACK_FLAG, &ipd->cpudata[cpuid].status);
+}
+
+static inline void ipipe_clear_foreign_stack(struct ipipe_domain *ipd)
+{
+	/* Must be called hw interrupts off. */
+	ipipe_declare_cpuid;
+	ipipe_load_cpuid();
+	__clear_bit(IPIPE_NOSTACK_FLAG, &ipd->cpudata[cpuid].status);
+}
+
+#define ipipe_safe_current()					\
+({								\
+	ipipe_declare_cpuid;					\
+	struct task_struct *p;					\
+	ipipe_load_cpuid();					\
+	p = test_bit(IPIPE_NOSTACK_FLAG,			\
+		     &per_cpu(ipipe_percpu_domain, cpuid)->cpudata[cpuid].status) ? &init_task : current; \
+	p; \
+})
+
+ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd,
+					unsigned event,
+					ipipe_event_handler_t handler);
+
+cpumask_t ipipe_set_irq_affinity(unsigned irq,
+				 cpumask_t cpumask);
+
+int fastcall ipipe_send_ipi(unsigned ipi,
+			    cpumask_t cpumask);
+
+int ipipe_setscheduler_root(struct task_struct *p,
+			    int policy,
+			    int prio);
+
+int ipipe_reenter_root(struct task_struct *prev,
+		       int policy,
+		       int prio);
+
+int ipipe_alloc_ptdkey(void);
+
+int ipipe_free_ptdkey(int key);
+
+int fastcall ipipe_set_ptd(int key,
+			   void *value);
+
+void fastcall *ipipe_get_ptd(int key);
+
+#define local_irq_enable_hw_cond()		local_irq_enable_hw()
+#define local_irq_disable_hw_cond()		local_irq_disable_hw()
+#define local_irq_save_hw_cond(flags)	local_irq_save_hw(flags)
+#define local_irq_restore_hw_cond(flags)	local_irq_restore_hw(flags)
+
+#define local_irq_enable_nohead(ipd)			\
+	do {						\
+		if (!__ipipe_pipeline_head_p(ipd))	\
+			local_irq_enable_hw();		\
+	} while(0)
+
+#define local_irq_disable_nohead(ipd)		\
+	do {						\
+		if (!__ipipe_pipeline_head_p(ipd))	\
+			local_irq_disable_hw();		\
+	} while(0)
+
+#define spin_lock_irqsave_hw_cond(lock,flags)	spin_lock_irqsave_hw(lock,flags)
+#define spin_unlock_irqrestore_hw_cond(lock,flags)	spin_unlock_irqrestore_hw(lock,flags)
+#define smp_processor_id_hw()				ipipe_processor_id()
+
+#define ipipe_irq_lock(irq)						\
+	do {								\
+		ipipe_declare_cpuid;					\
+		ipipe_load_cpuid();					\
+		__ipipe_lock_irq(per_cpu(ipipe_percpu_domain, cpuid), cpuid, irq);\
+	} while(0)
+
+#define ipipe_irq_unlock(irq)						\
+	do {								\
+		ipipe_declare_cpuid;					\
+		ipipe_load_cpuid();					\
+		__ipipe_unlock_irq(per_cpu(ipipe_percpu_domain, cpuid), irq);	\
+	} while(0)
+
+#define ipipe_root_domain_p		(ipipe_current_domain == ipipe_root_domain)
+
+#else	/* !CONFIG_IPIPE */
+
+#define ipipe_init()			do { } while(0)
+#define ipipe_suspend_domain()	do { } while(0)
+#define ipipe_sigwake_notify(p)	do { } while(0)
+#define ipipe_setsched_notify(p)	do { } while(0)
+#define ipipe_init_notify(p)		do { } while(0)
+#define ipipe_exit_notify(p)		do { } while(0)
+#define ipipe_cleanup_notify(mm)	do { } while(0)
+#define ipipe_trap_notify(t,r)	0
+#define ipipe_init_proc()		do { } while(0)
+
+#define spin_lock_hw(lock)			spin_lock(lock)
+#define spin_unlock_hw(lock)			spin_unlock(lock)
+#define spin_lock_irq_hw(lock)		spin_lock_irq(lock)
+#define spin_unlock_irq_hw(lock)		spin_unlock_irq(lock)
+#define spin_lock_irqsave_hw(lock,flags)	spin_lock_irqsave(lock, flags)
+#define spin_unlock_irqrestore_hw(lock,flags)	spin_unlock_irqrestore(lock, flags)
+
+#define local_irq_enable_hw_cond()		do { } while(0)
+#define local_irq_disable_hw_cond()		do { } while(0)
+#define local_irq_save_hw_cond(flags)	do { (void)(flags); } while(0)
+#define local_irq_restore_hw_cond(flags)	do { } while(0)
+#define spin_lock_irqsave_hw_cond(lock,flags)	do { (void)(flags); spin_lock(lock); } while(0)
+#define spin_unlock_irqrestore_hw_cond(lock,flags)	spin_unlock(lock)
+#define smp_processor_id_hw()			smp_processor_id()
+
+#define ipipe_irq_lock(irq)		do { } while(0)
+#define ipipe_irq_unlock(irq)		do { } while(0)
+
+#define ipipe_root_domain_p		1
+#define ipipe_safe_current		current
+
+#endif	/* CONFIG_IPIPE */
+
+#endif	/* !__LINUX_IPIPE_H */
diff -uNrp --exclude CVS linux-head/include/linux/ipipe_trace.h linux-2.6.x/include/linux/ipipe_trace.h
--- linux-head/include/linux/ipipe_trace.h	1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.x/include/linux/ipipe_trace.h	2006-10-12 19:43:06.000000000 +0200
@@ -0,0 +1,40 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_trace.h
+ *
+ * Copyright (C) 2005 Luotao Fu.
+ *               2005, 2006 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef _LINUX_IPIPE_TRACE_H
+#define _LINUX_IPIPE_TRACE_H
+
+#include <linux/types.h>
+
+void ipipe_trace_begin(unsigned long v);
+void ipipe_trace_end(unsigned long v);
+void ipipe_trace_freeze(unsigned long v);
+void ipipe_trace_special(unsigned char special_id, unsigned long v);
+void ipipe_trace_pid(pid_t pid, short prio);
+
+int ipipe_trace_max_reset(void);
+int ipipe_trace_frozen_reset(void);
+
+void ipipe_trace_panic_freeze(void);
+void ipipe_trace_panic_dump(void);
+
+#endif	/* !__LINUX_IPIPE_H */
diff -uNrp --exclude CVS linux-head/include/linux/linkage.h linux-2.6.x/include/linux/linkage.h
--- linux-head/include/linux/linkage.h	2006-03-23 06:54:07.000000000 +0100
+++ linux-2.6.x/include/linux/linkage.h	2006-10-12 19:28:34.000000000 +0200
@@ -51,4 +51,8 @@
 #define fastcall
 #endif
 
+#ifndef notrace
+#define notrace		__attribute__((no_instrument_function))
+#endif
+
 #endif
diff -uNrp --exclude CVS linux-head/include/linux/preempt.h linux-2.6.x/include/linux/preempt.h
--- linux-head/include/linux/preempt.h	2006-03-23 06:54:07.000000000 +0100
+++ linux-2.6.x/include/linux/preempt.h	2006-10-09 22:36:50.000000000 +0200
@@ -27,29 +27,44 @@
 
 asmlinkage void preempt_schedule(void);
 
-#define preempt_disable() \
-do { \
-	inc_preempt_count(); \
-	barrier(); \
+#ifdef CONFIG_IPIPE
+#include <asm/ipipe.h>
+DECLARE_PER_CPU(struct ipipe_domain *, ipipe_percpu_domain);
+extern struct ipipe_domain ipipe_root;
+#define ipipe_preempt_guard() (per_cpu(ipipe_percpu_domain, ipipe_processor_id()) == &ipipe_root)
+#else  /* !CONFIG_IPIPE */
+#define ipipe_preempt_guard()	1
+#endif /* CONFIG_IPIPE */
+
+#define preempt_disable()						\
+do {									\
+	if (ipipe_preempt_guard()) {					\
+		inc_preempt_count();					\
+		barrier();						\
+	}								\
 } while (0)
 
-#define preempt_enable_no_resched() \
-do { \
-	barrier(); \
-	dec_preempt_count(); \
+#define preempt_enable_no_resched()					\
+do {									\
+	if (ipipe_preempt_guard()) {					\
+		barrier();						\
+		dec_preempt_count();					\
+	}								\
 } while (0)
 
-#define preempt_check_resched() \
-do { \
-	if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \
-		preempt_schedule(); \
+#define preempt_check_resched()						\
+do {									\
+	if (ipipe_preempt_guard()) {					\
+		if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))	\
+			preempt_schedule();				\
+	}								\
 } while (0)
 
-#define preempt_enable() \
-do { \
-	preempt_enable_no_resched(); \
+#define preempt_enable()						\
+do {									\
+	preempt_enable_no_resched();					\
 	barrier(); \
-	preempt_check_resched(); \
+	preempt_check_resched();					\
 } while (0)
 
 #else
diff -uNrp --exclude CVS linux-head/include/linux/sched.h linux-2.6.x/include/linux/sched.h
--- linux-head/include/linux/sched.h	2006-03-23 06:54:07.000000000 +0100
+++ linux-2.6.x/include/linux/sched.h	2006-10-09 22:36:50.000000000 +0200
@@ -4,6 +4,7 @@
 #include <asm/param.h>	/* for HZ */
 
 #include <linux/config.h>
+#include <linux/ipipe.h>
 #include <linux/capability.h>
 #include <linux/threads.h>
 #include <linux/kernel.h>
@@ -129,6 +130,13 @@ extern unsigned long nr_iowait(void);
 #define EXIT_DEAD		32
 /* in tsk->state again */
 #define TASK_NONINTERACTIVE	64
+#ifdef CONFIG_IPIPE
+#define TASK_ATOMICSWITCH	512
+#define TASK_NOWAKEUP          1024
+#else  /* !CONFIG_IPIPE */
+#define TASK_ATOMICSWITCH	0
+#define TASK_NOWAKEUP          0
+#endif /* CONFIG_IPIPE */
 
 #define __set_task_state(tsk, state_value)		\
 	do { (tsk)->state = (state_value); } while (0)
@@ -871,6 +879,9 @@ struct task_struct {
 #endif
 	atomic_t fs_excl;	/* holding fs exclusive resources */
 	struct rcu_head rcu;
+#ifdef CONFIG_IPIPE
+	void *ptd[IPIPE_ROOT_NPTDKEYS];
+#endif
 };
 
 static inline pid_t process_group(struct task_struct *tsk)
@@ -928,6 +939,11 @@ static inline void put_task_struct(struc
 #define PF_BORROWED_MM	0x00400000	/* I am a kthread doing use_mm */
 #define PF_RANDOMIZE	0x00800000	/* randomize virtual address space */
 #define PF_SWAPWRITE	0x01000000	/* Allowed to write to swap */
+#ifdef CONFIG_IPIPE
+#define PF_EVNOTIFY    0x40000000	/* Notify other domains about internal events */
+#else
+#define PF_EVNOTIFY    0
+#endif /* CONFIG_IPIPE */
 
 /*
  * Only the _current_ task can read/write to tsk->flags, but other
diff -uNrp --exclude CVS linux-head/init/Kconfig linux-2.6.x/init/Kconfig
--- linux-head/init/Kconfig	2006-06-19 09:42:35.000000000 +0200
+++ linux-2.6.x/init/Kconfig	2006-10-15 21:37:21.000000000 +0200
@@ -58,6 +58,7 @@ menu "General setup"
 
 config LOCALVERSION
 	string "Local version - append to kernel release"
+	default "-ipipe"
 	help
 	  Append an extra string to the end of your kernel version.
 	  This will show up when you type uname, for example.
diff -uNrp --exclude CVS linux-head/init/main.c linux-2.6.x/init/main.c
--- linux-head/init/main.c	2006-03-22 03:34:07.000000000 +0100
+++ linux-2.6.x/init/main.c	2006-10-09 22:36:50.000000000 +0200
@@ -486,6 +486,11 @@ asmlinkage void __init start_kernel(void
 	hrtimers_init();
 	softirq_init();
 	time_init();
+	/*
+	 * We need to wait for the interrupt and time subsystems to be
+	 * initialized before enabling the pipeline.
+	 */
+ 	ipipe_init();
 
 	/*
 	 * HACK ALERT! This is early. We're enabling the console before
@@ -611,6 +616,7 @@ static void __init do_basic_setup(void)
 #ifdef CONFIG_SYSCTL
 	sysctl_init();
 #endif
+	ipipe_init_proc();
 
 	do_initcalls();
 }
@@ -670,6 +676,8 @@ static int init(void * unused)
 
 	smp_prepare_cpus(max_cpus);
 
+	init_irqthreads();
+
 	do_pre_smp_initcalls();
 
 	fixup_cpu_present_map();
diff -uNrp --exclude CVS linux-head/kernel/Makefile linux-2.6.x/kernel/Makefile
--- linux-head/kernel/Makefile	2006-03-22 03:36:56.000000000 +0100
+++ linux-2.6.x/kernel/Makefile	2006-10-15 21:37:28.000000000 +0200
@@ -34,6 +34,7 @@ obj-$(CONFIG_DETECT_SOFTLOCKUP) += softl
 obj-$(CONFIG_GENERIC_HARDIRQS) += irq/
 obj-$(CONFIG_SECCOMP) += seccomp.o
 obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
+obj-$(CONFIG_IPIPE) += ipipe/
 
 ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y)
 # According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is
diff -uNrp --exclude CVS linux-head/kernel/exit.c linux-2.6.x/kernel/exit.c
--- linux-head/kernel/exit.c	2006-08-28 07:52:06.000000000 +0200
+++ linux-2.6.x/kernel/exit.c	2006-10-09 22:36:50.000000000 +0200
@@ -844,6 +844,7 @@ fastcall NORET_TYPE void do_exit(long co
 		exit_itimers(tsk->signal);
 		acct_process(code);
 	}
+ 	ipipe_exit_notify(tsk);
 	exit_mm(tsk);
 
 	exit_sem(tsk);
diff -uNrp --exclude CVS linux-head/kernel/fork.c linux-2.6.x/kernel/fork.c
--- linux-head/kernel/fork.c	2006-04-27 17:03:18.000000000 +0200
+++ linux-2.6.x/kernel/fork.c	2006-10-09 22:36:50.000000000 +0200
@@ -371,6 +371,7 @@ void fastcall __mmdrop(struct mm_struct 
 void mmput(struct mm_struct *mm)
 {
 	if (atomic_dec_and_test(&mm->mm_users)) {
+		ipipe_cleanup_notify(mm);
 		exit_aio(mm);
 		exit_mmap(mm);
 		if (!list_empty(&mm->mmlist)) {
@@ -885,7 +886,7 @@ static inline void copy_flags(unsigned l
 {
 	unsigned long new_flags = p->flags;
 
-	new_flags &= ~(PF_SUPERPRIV | PF_NOFREEZE);
+	new_flags &= ~(PF_SUPERPRIV | PF_NOFREEZE | PF_EVNOTIFY);
 	new_flags |= PF_FORKNOEXEC;
 	if (!(clone_flags & CLONE_PTRACE))
 		p->ptrace = 0;
@@ -1198,6 +1199,14 @@ static task_t *copy_process(unsigned lon
 	spin_unlock(&current->sighand->siglock);
 	write_unlock_irq(&tasklist_lock);
 	proc_fork_connector(p);
+#ifdef CONFIG_IPIPE
+	{
+	int k;
+
+	for (k = 0; k < IPIPE_ROOT_NPTDKEYS; k++)
+		p->ptd[k] = NULL;
+	}
+#endif /* CONFIG_IPIPE */
 	return p;
 
 bad_fork_cleanup_namespace:
diff -uNrp --exclude CVS linux-head/kernel/ipipe/Kconfig linux-2.6.x/kernel/ipipe/Kconfig
--- linux-head/kernel/ipipe/Kconfig	1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.x/kernel/ipipe/Kconfig	2006-10-09 22:33:18.000000000 +0200
@@ -0,0 +1,6 @@
+config IPIPE
+	bool "Interrupt pipeline"
+	default y
+	---help---
+	  Activate this option if you want the interrupt pipeline to be
+	  compiled in.
diff -uNrp --exclude CVS linux-head/kernel/ipipe/Kconfig.debug linux-2.6.x/kernel/ipipe/Kconfig.debug
--- linux-head/kernel/ipipe/Kconfig.debug	1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.x/kernel/ipipe/Kconfig.debug	2006-10-02 07:15:26.000000000 +0200
@@ -0,0 +1,69 @@
+config IPIPE_DEBUG
+	bool "I-pipe debugging"
+	depends on IPIPE
+
+config IPIPE_TRACE
+	bool "Latency tracing"
+	depends on IPIPE_DEBUG
+	select FRAME_POINTER
+	select KALLSYMS
+	select PROC_FS
+	---help---
+	  Activate this option if you want to use per-function tracing of
+	  the kernel. The tracer will collect data via instrumentation
+	  features like the one below or with the help of explicite calls
+	  of ipipe_trace_xxx(). See include/linux/ipipe_trace.h for the
+	  in-kernel tracing API. The collected data and runtime control
+	  is available via /proc/ipipe/trace/*.
+
+config IPIPE_TRACE_ENABLE
+	bool "Enable tracing on boot"
+	depends on IPIPE_TRACE
+	default y
+	---help---
+	  Disable this option if you want to arm the tracer after booting
+	  manually ("echo 1 > /proc/ipipe/tracer/enable"). This can reduce
+	  boot time on slow embedded devices due to the tracer overhead.
+
+config IPIPE_TRACE_MCOUNT
+	bool "Intrument function entries"
+	depends on IPIPE_TRACE
+	default y
+	---help---
+	  When enabled, records every kernel function entry in the tracer
+	  log. While this slows down the system noticeably, it provides
+	  the highest level of information about the flow of events.
+	  However, it can be switch off in order to record only explicit
+	  I-pipe trace points.
+
+config IPIPE_TRACE_IRQSOFF
+	bool "Trace IRQs-off times"
+	depends on IPIPE_TRACE
+	default y
+	---help---
+	  Activate this option if I-pipe shall trace the longest path
+	  with hard-IRQs switched off.
+
+config IPIPE_TRACE_SHIFT
+	int "Depth of trace log (14 => 16Kpoints, 15 => 32Kpoints)"
+	range 10 18
+	default 14
+	depends on IPIPE_TRACE
+	---help---
+	  The number of trace points to hold tracing data for each
+	  trace path, as a power of 2.
+
+config IPIPE_TRACE_VMALLOC
+	bool "Use vmalloc'ed trace buffer"
+	depends on IPIPE_TRACE
+	---help---
+	  Instead of reserving static kernel data, the required buffer
+	  is allocated via vmalloc during boot-up when this option is
+	  enabled. This can help to start systems that are low on memory,
+	  but it slightly degrades overall performance. Try this option
+	  when a traced kernel hangs unexpectedly at boot time.
+
+config IPIPE_TRACE_ENABLE_VALUE
+	int
+	default 0 if !IPIPE_TRACE_ENABLE
+	default 1 if IPIPE_TRACE_ENABLE
diff -uNrp --exclude CVS linux-head/kernel/ipipe/Makefile linux-2.6.x/kernel/ipipe/Makefile
--- linux-head/kernel/ipipe/Makefile	1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.x/kernel/ipipe/Makefile	2006-10-09 22:33:18.000000000 +0200
@@ -0,0 +1,3 @@
+
+obj-$(CONFIG_IPIPE)	+= core.o generic.o
+obj-$(CONFIG_IPIPE_TRACE) += tracer.o
diff -uNrp --exclude CVS linux-head/kernel/ipipe/core.c linux-2.6.x/kernel/ipipe/core.c
--- linux-head/kernel/ipipe/core.c	1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.x/kernel/ipipe/core.c	2006-10-12 22:27:51.000000000 +0200
@@ -0,0 +1,1049 @@
+/* -*- linux-c -*-
+ * linux/kernel/ipipe/core.c
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-independent I-PIPE core support.
+ */
+
+#include <linux/sched.h>
+#include <linux/module.h>
+#include <linux/kallsyms.h>
+#include <linux/interrupt.h>
+#ifdef CONFIG_PROC_FS
+#include <linux/proc_fs.h>
+#endif	/* CONFIG_PROC_FS */
+
+struct ipipe_domain ipipe_root =
+	{ .cpudata = {[0 ... IPIPE_NR_CPUS-1] =
+		{ .status = (1<<IPIPE_STALL_FLAG) } } };
+
+DEFINE_PER_CPU(struct ipipe_domain *, ipipe_percpu_domain) =
+	{ &ipipe_root };
+
+ipipe_spinlock_t __ipipe_pipelock = IPIPE_SPIN_LOCK_UNLOCKED;
+
+LIST_HEAD(__ipipe_pipeline);
+
+unsigned long __ipipe_virtual_irq_map;
+
+#ifdef CONFIG_PRINTK
+unsigned __ipipe_printk_virq;
+#endif /* CONFIG_PRINTK */
+
+int __ipipe_event_monitors[IPIPE_NR_EVENTS];
+
+/*
+ * ipipe_init() -- Initialization routine of the IPIPE layer. Called
+ * by the host kernel early during the boot procedure.
+ */
+void ipipe_init(void)
+{
+	struct ipipe_domain *ipd = &ipipe_root;
+
+	__ipipe_check_platform();	/* Do platform dependent checks first. */
+
+	/*
+	 * A lightweight registration code for the root domain. We are
+	 * running on the boot CPU, hw interrupts are off, and
+	 * secondary CPUs are still lost in space.
+	 */
+
+	ipd->name = "Linux";
+	ipd->domid = IPIPE_ROOT_ID;
+	ipd->priority = IPIPE_ROOT_PRIO;
+
+	__ipipe_init_stage(ipd);
+
+	INIT_LIST_HEAD(&ipd->p_link);
+	list_add_tail(&ipd->p_link, &__ipipe_pipeline);
+
+	__ipipe_init_platform();
+
+#ifdef CONFIG_PRINTK
+	__ipipe_printk_virq = ipipe_alloc_virq();	/* Cannot fail here. */
+	ipd->irqs[__ipipe_printk_virq].handler = &__ipipe_flush_printk;
+	ipd->irqs[__ipipe_printk_virq].cookie = NULL;
+	ipd->irqs[__ipipe_printk_virq].acknowledge = NULL;
+	ipd->irqs[__ipipe_printk_virq].control = IPIPE_HANDLE_MASK;
+#endif /* CONFIG_PRINTK */
+
+	__ipipe_enable_pipeline();
+
+	printk(KERN_INFO "I-pipe %s: pipeline enabled.\n",
+	       IPIPE_VERSION_STRING);
+}
+
+void __ipipe_init_stage(struct ipipe_domain *ipd)
+{
+	int cpuid, n;
+
+	for (cpuid = 0; cpuid < IPIPE_NR_CPUS; cpuid++) {
+		ipd->cpudata[cpuid].irq_pending_hi = 0;
+
+		for (n = 0; n < IPIPE_IRQ_IWORDS; n++)
+			ipd->cpudata[cpuid].irq_pending_lo[n] = 0;
+
+		for (n = 0; n < IPIPE_NR_IRQS; n++) {
+			ipd->cpudata[cpuid].irq_counters[n].total_hits = 0;
+			ipd->cpudata[cpuid].irq_counters[n].pending_hits = 0;
+		}
+
+		ipd->cpudata[cpuid].evsync = 0;
+	}
+
+	for (n = 0; n < IPIPE_NR_IRQS; n++) {
+		ipd->irqs[n].acknowledge = NULL;
+		ipd->irqs[n].handler = NULL;
+		ipd->irqs[n].control = IPIPE_PASS_MASK;	/* Pass but don't handle */
+	}
+
+	for (n = 0; n < IPIPE_NR_EVENTS; n++)
+		ipd->evhand[n] = NULL;
+
+	ipd->evself = 0LL;
+
+#ifdef CONFIG_SMP
+	ipd->irqs[IPIPE_CRITICAL_IPI].acknowledge = &__ipipe_ack_system_irq;
+	ipd->irqs[IPIPE_CRITICAL_IPI].handler = &__ipipe_do_critical_sync;
+	ipd->irqs[IPIPE_CRITICAL_IPI].cookie = NULL;
+	/* Immediately handle in the current domain but *never* pass */
+	ipd->irqs[IPIPE_CRITICAL_IPI].control =
+		IPIPE_HANDLE_MASK|IPIPE_STICKY_MASK|IPIPE_SYSTEM_MASK;
+#endif	/* CONFIG_SMP */
+}
+
+void __ipipe_stall_root(void)
+{
+	ipipe_declare_cpuid;
+	unsigned long flags;
+
+	ipipe_get_cpu(flags); /* Care for migration. */
+	set_bit(IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status);
+	ipipe_put_cpu(flags);
+}
+
+void __ipipe_cleanup_domain(struct ipipe_domain *ipd)
+{
+	ipipe_unstall_pipeline_from(ipd);
+
+#ifdef CONFIG_SMP
+	{
+		int cpu;
+
+		for_each_online_cpu(cpu) {
+			while (ipd->cpudata[cpu].irq_pending_hi != 0)
+				cpu_relax();
+		}
+	}
+#endif	/* CONFIG_SMP */
+}
+
+void __ipipe_unstall_root(void)
+{
+	ipipe_declare_cpuid;
+
+	local_irq_disable_hw();
+
+	ipipe_load_cpuid();
+
+	__clear_bit(IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status);
+
+	if (unlikely(ipipe_root_domain->cpudata[cpuid].irq_pending_hi != 0))
+		__ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+
+	local_irq_enable_hw();
+}
+
+unsigned long __ipipe_test_root(void)
+{
+	unsigned long flags, x;
+	ipipe_declare_cpuid;
+
+	ipipe_get_cpu(flags); /* Care for migration. */
+	x = test_bit(IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status);
+	ipipe_put_cpu(flags);
+
+	return x;
+}
+
+unsigned long __ipipe_test_and_stall_root(void)
+{
+	unsigned long flags, x;
+	ipipe_declare_cpuid;
+
+	ipipe_get_cpu(flags); /* Care for migration. */
+	x = test_and_set_bit(IPIPE_STALL_FLAG,
+			     &ipipe_root_domain->cpudata[cpuid].status);
+	ipipe_put_cpu(flags);
+
+	return x;
+}
+
+void fastcall __ipipe_restore_root(unsigned long x)
+{
+	if (x)
+		__ipipe_stall_root();
+	else
+		__ipipe_unstall_root();
+}
+
+void fastcall ipipe_stall_pipeline_from(struct ipipe_domain *ipd)
+{
+	ipipe_declare_cpuid;
+#ifdef CONFIG_SMP
+	unsigned long flags;
+
+	ipipe_lock_cpu(flags); /* Care for migration. */
+
+	__set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+
+	if (!__ipipe_pipeline_head_p(ipd))
+		ipipe_unlock_cpu(flags);
+#else	/* CONFIG_SMP */
+	set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+
+	if (__ipipe_pipeline_head_p(ipd))
+		local_irq_disable_hw();
+#endif	/* CONFIG_SMP */
+}
+
+unsigned long fastcall ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd)
+{
+	ipipe_declare_cpuid;
+	unsigned long s;
+#ifdef CONFIG_SMP
+	unsigned long flags;
+
+	ipipe_lock_cpu(flags); /* Care for migration. */
+
+	s = __test_and_set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+
+	if (!__ipipe_pipeline_head_p(ipd))
+		ipipe_unlock_cpu(flags);
+#else	/* CONFIG_SMP */
+	s = test_and_set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+
+	if (__ipipe_pipeline_head_p(ipd))
+		local_irq_disable_hw();
+#endif	/* CONFIG_SMP */
+
+	return s;
+}
+
+/*
+ * ipipe_unstall_pipeline_from() -- Unstall the pipeline and
+ * synchronize pending interrupts for a given domain. See
+ * __ipipe_walk_pipeline() for more information.
+ */
+void fastcall ipipe_unstall_pipeline_from(struct ipipe_domain *ipd)
+{
+	struct list_head *pos;
+	unsigned long flags;
+	ipipe_declare_cpuid;
+
+	ipipe_lock_cpu(flags);
+
+	__clear_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+
+	if (ipd == per_cpu(ipipe_percpu_domain, cpuid))
+		pos = &ipd->p_link;
+	else
+		pos = __ipipe_pipeline.next;
+
+	__ipipe_walk_pipeline(pos, cpuid);
+
+	if (__ipipe_pipeline_head_p(ipd))
+		local_irq_enable_hw();
+	else
+		ipipe_unlock_cpu(flags);
+}
+
+unsigned long fastcall ipipe_test_and_unstall_pipeline_from(struct ipipe_domain *ipd)
+{
+	unsigned long flags, x;
+	ipipe_declare_cpuid;
+
+	ipipe_get_cpu(flags);
+	x = test_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+	ipipe_unstall_pipeline_from(ipd);
+	ipipe_put_cpu(flags);
+
+	return x;
+}
+
+void fastcall ipipe_restore_pipeline_from(struct ipipe_domain *ipd,
+					  unsigned long x)
+{
+	if (x)
+		ipipe_stall_pipeline_from(ipd);
+	else
+		ipipe_unstall_pipeline_from(ipd);
+}
+
+void ipipe_unstall_pipeline_head(void)
+{
+	struct ipipe_domain *head;
+	unsigned long flags;
+	ipipe_declare_cpuid;
+
+	ipipe_lock_cpu(flags);
+	head = __ipipe_pipeline_head();
+	__clear_bit(IPIPE_STALL_FLAG, &head->cpudata[cpuid].status);
+
+	if (unlikely(head->cpudata[cpuid].irq_pending_hi != 0)) {
+		if (likely(head == per_cpu(ipipe_percpu_domain, cpuid)))
+			__ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+		else
+			__ipipe_walk_pipeline(&head->p_link, cpuid);
+        }
+
+	local_irq_enable_hw();
+}
+
+void fastcall __ipipe_restore_pipeline_head(struct ipipe_domain *head, unsigned long x)
+{
+	ipipe_declare_cpuid;
+	unsigned long flags;
+
+	ipipe_lock_cpu(flags);
+
+	if (x) {
+#ifdef CONFIG_DEBUG_KERNEL
+		static int warned;
+		if (!warned && test_and_set_bit(IPIPE_STALL_FLAG, &head->cpudata[cpuid].status)) {
+			/*
+			 * Already stalled albeit ipipe_restore_pipeline_head()
+			 * should have detected it? Send a warning once.\n");
+			 */
+			warned = 1;
+			printk(KERN_WARNING
+				   "I-pipe: ipipe_restore_pipeline_head() optimization failed.\n");
+			dump_stack();
+		}
+#else /* !CONFIG_DEBUG_KERNEL */
+		set_bit(IPIPE_STALL_FLAG, &head->cpudata[cpuid].status);
+#endif /* CONFIG_DEBUG_KERNEL */
+	}
+	else {
+		__clear_bit(IPIPE_STALL_FLAG, &head->cpudata[cpuid].status);
+		if (unlikely(head->cpudata[cpuid].irq_pending_hi != 0)) {
+			if (likely(head == per_cpu(ipipe_percpu_domain, cpuid)))
+				__ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+			else
+				__ipipe_walk_pipeline(&head->p_link, cpuid);
+		}
+		local_irq_enable_hw();
+	}
+}
+
+/* __ipipe_walk_pipeline(): Plays interrupts pending in the log. Must
+   be called with local hw interrupts disabled. */
+
+void fastcall __ipipe_walk_pipeline(struct list_head *pos, int cpuid)
+{
+	struct ipipe_domain *this_domain = per_cpu(ipipe_percpu_domain, cpuid);
+
+	while (pos != &__ipipe_pipeline) {
+		struct ipipe_domain *next_domain =
+		    list_entry(pos, struct ipipe_domain, p_link);
+
+		if (test_bit
+		    (IPIPE_STALL_FLAG, &next_domain->cpudata[cpuid].status))
+			break;	/* Stalled stage -- do not go further. */
+
+		if (next_domain->cpudata[cpuid].irq_pending_hi != 0) {
+
+			if (next_domain == this_domain)
+				__ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+			else {
+				__ipipe_switch_to(this_domain, next_domain,
+						  cpuid);
+
+				ipipe_load_cpuid();	/* Processor might have changed. */
+
+				if (this_domain->cpudata[cpuid].
+				    irq_pending_hi != 0
+				    && !test_bit(IPIPE_STALL_FLAG,
+						 &this_domain->cpudata[cpuid].status))
+					__ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+			}
+
+			break;
+		} else if (next_domain == this_domain)
+			break;
+
+		pos = next_domain->p_link.next;
+	}
+}
+
+/*
+ * ipipe_suspend_domain() -- Suspend the current domain, switching to
+ * the next one which has pending work down the pipeline.
+ */
+void ipipe_suspend_domain(void)
+{
+	struct ipipe_domain *this_domain, *next_domain;
+	struct list_head *ln;
+	unsigned long flags;
+	ipipe_declare_cpuid;
+
+	ipipe_lock_cpu(flags);
+
+	this_domain = next_domain = per_cpu(ipipe_percpu_domain, cpuid);
+
+	__clear_bit(IPIPE_STALL_FLAG, &this_domain->cpudata[cpuid].status);
+
+	if (this_domain->cpudata[cpuid].irq_pending_hi != 0)
+		goto sync_stage;
+
+	for (;;) {
+		ln = next_domain->p_link.next;
+
+		if (ln == &__ipipe_pipeline)
+			break;
+
+		next_domain = list_entry(ln, struct ipipe_domain, p_link);
+
+		if (test_bit(IPIPE_STALL_FLAG,
+			     &next_domain->cpudata[cpuid].status))
+			break;
+
+		if (next_domain->cpudata[cpuid].irq_pending_hi == 0)
+			continue;
+
+		per_cpu(ipipe_percpu_domain, cpuid) = next_domain;
+
+sync_stage:
+		__ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+
+		ipipe_load_cpuid();	/* Processor might have changed. */
+
+		if (per_cpu(ipipe_percpu_domain, cpuid) != next_domain)
+			/*
+			 * Something has changed the current domain under our
+			 * feet, recycling the register set; take note.
+			 */
+			this_domain = per_cpu(ipipe_percpu_domain, cpuid);
+	}
+
+	per_cpu(ipipe_percpu_domain, cpuid) = this_domain;
+
+	ipipe_unlock_cpu(flags);
+}
+
+/* ipipe_alloc_virq() -- Allocate a pipelined virtual/soft interrupt.
+ * Virtual interrupts are handled in exactly the same way than their
+ * hw-generated counterparts wrt pipelining.
+ */
+unsigned ipipe_alloc_virq(void)
+{
+	unsigned long flags, irq = 0;
+	int ipos;
+
+	spin_lock_irqsave_hw(&__ipipe_pipelock, flags);
+
+	if (__ipipe_virtual_irq_map != ~0) {
+		ipos = ffz(__ipipe_virtual_irq_map);
+		set_bit(ipos, &__ipipe_virtual_irq_map);
+		irq = ipos + IPIPE_VIRQ_BASE;
+	}
+
+	spin_unlock_irqrestore_hw(&__ipipe_pipelock, flags);
+
+	return irq;
+}
+
+/* ipipe_virtualize_irq() -- Attach a handler (and optionally a hw
+   acknowledge routine) to an interrupt for a given domain. */
+
+int ipipe_virtualize_irq(struct ipipe_domain *ipd,
+			 unsigned irq,
+			 ipipe_irq_handler_t handler,
+			 void *cookie,
+			 ipipe_irq_ackfn_t acknowledge,
+			 unsigned modemask)
+{
+	unsigned long flags;
+	int err;
+
+	if (irq >= IPIPE_NR_IRQS)
+		return -EINVAL;
+
+	if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK)
+		return -EPERM;
+
+	if (!test_bit(IPIPE_AHEAD_FLAG, &ipd->flags))
+		/* Silently unwire interrupts for non-heading domains. */
+		modemask &= ~IPIPE_WIRED_MASK;
+
+	spin_lock_irqsave_hw(&__ipipe_pipelock, flags);
+
+	if (handler != NULL) {
+
+		if (handler == IPIPE_SAME_HANDLER) {
+			handler = ipd->irqs[irq].handler;
+			cookie = ipd->irqs[irq].cookie;
+
+			if (handler == NULL) {
+				err = -EINVAL;
+				goto unlock_and_exit;
+			}
+		} else if ((modemask & IPIPE_EXCLUSIVE_MASK) != 0 &&
+			   ipd->irqs[irq].handler != NULL) {
+			err = -EBUSY;
+			goto unlock_and_exit;
+		}
+
+		if ((modemask & (IPIPE_SHARED_MASK | IPIPE_PASS_MASK)) ==
+		    IPIPE_SHARED_MASK) {
+			err = -EINVAL;
+			goto unlock_and_exit;
+		}
+
+		/* Wired interrupts can only be delivered to domains
+		 * always heading the pipeline. */
+
+		if ((modemask & IPIPE_WIRED_MASK) != 0) {
+			if ((modemask & (IPIPE_SHARED_MASK | IPIPE_PASS_MASK | IPIPE_STICKY_MASK)) != 0) {
+				err = -EINVAL;
+				goto unlock_and_exit;
+			}
+			modemask |= (IPIPE_HANDLE_MASK);
+		}
+
+		if ((modemask & IPIPE_STICKY_MASK) != 0)
+			modemask |= IPIPE_HANDLE_MASK;
+	} else
+		modemask &=
+		    ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK |
+		      IPIPE_SHARED_MASK | IPIPE_EXCLUSIVE_MASK | IPIPE_WIRED_MASK);
+
+	if (acknowledge == NULL) {
+		if ((modemask & IPIPE_SHARED_MASK) == 0) {
+			if (!ipipe_virtual_irq_p(irq)) {
+				/* Acknowledge handler unspecified for a hw
+				   interrupt -- this is ok in non-shared
+				   management mode, but we will force the use
+				   of the Linux-defined handler instead. */
+				acknowledge = ipipe_root_domain->irqs[irq].acknowledge;
+			}
+		}
+		else {
+			/* A valid acknowledge handler to be called in shared mode
+			   is required when declaring a shared IRQ. */
+			err = -EINVAL;
+			goto unlock_and_exit;
+		}
+	}
+
+	ipd->irqs[irq].handler = handler;
+	ipd->irqs[irq].cookie = cookie;
+	ipd->irqs[irq].acknowledge = acknowledge;
+	ipd->irqs[irq].control = modemask;
+
+	if (irq < NR_IRQS && handler != NULL && !ipipe_virtual_irq_p(irq)) {
+		__ipipe_enable_irqdesc(irq);
+
+		if ((modemask & IPIPE_ENABLE_MASK) != 0) {
+			if (ipd != ipipe_current_domain) {
+				/* IRQ enable/disable state is domain-sensitive, so we may
+				   not change it for another domain. What is allowed
+				   however is forcing some domain to handle an interrupt
+				   source, by passing the proper 'ipd' descriptor which
+				   thus may be different from ipipe_current_domain. */
+				err = -EPERM;
+				goto unlock_and_exit;
+			}
+			
+			__ipipe_enable_irq(irq);
+		}
+	}
+
+	err = 0;
+
+      unlock_and_exit:
+
+	spin_unlock_irqrestore_hw(&__ipipe_pipelock, flags);
+
+	return err;
+}
+
+/* ipipe_control_irq() -- Change modes of a pipelined interrupt for
+ * the current domain. */
+
+int ipipe_control_irq(unsigned irq, unsigned clrmask, unsigned setmask)
+{
+	struct ipipe_domain *ipd;
+	unsigned long flags;
+
+	if (irq >= IPIPE_NR_IRQS)
+		return -EINVAL;
+
+	ipd = ipipe_current_domain;
+
+	if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK)
+		return -EPERM;
+
+	if (((setmask | clrmask) & IPIPE_SHARED_MASK) != 0)
+		return -EINVAL;
+
+	if (ipd->irqs[irq].handler == NULL)
+		setmask &= ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK);
+
+	if ((setmask & IPIPE_STICKY_MASK) != 0)
+		setmask |= IPIPE_HANDLE_MASK;
+
+	if ((clrmask & (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK)) != 0)	/* If one goes, both go. */
+		clrmask |= (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK);
+
+	spin_lock_irqsave_hw(&__ipipe_pipelock, flags);
+
+	ipd->irqs[irq].control &= ~clrmask;
+	ipd->irqs[irq].control |= setmask;
+
+	if ((setmask & IPIPE_ENABLE_MASK) != 0)
+		__ipipe_enable_irq(irq);
+	else if ((clrmask & IPIPE_ENABLE_MASK) != 0)
+		__ipipe_disable_irq(irq);
+
+	spin_unlock_irqrestore_hw(&__ipipe_pipelock, flags);
+
+	return 0;
+}
+
+/* __ipipe_dispatch_event() -- Low-level event dispatcher. */
+
+int fastcall __ipipe_dispatch_event (unsigned event, void *data)
+{
+	struct ipipe_domain *start_domain, *this_domain, *next_domain;
+	ipipe_event_handler_t evhand;
+	struct list_head *pos, *npos;
+	unsigned long flags;
+	ipipe_declare_cpuid;
+	int propagate = 1;
+
+	ipipe_lock_cpu(flags);
+
+	start_domain = this_domain = per_cpu(ipipe_percpu_domain, cpuid);
+
+	list_for_each_safe(pos,npos,&__ipipe_pipeline) {
+
+		/*
+		 * Note: Domain migration may occur while running
+		 * event or interrupt handlers, in which case the
+		 * current register set is going to be recycled for a
+		 * different domain than the initiating one. We do
+		 * care for that, always tracking the current domain
+		 * descriptor upon return from those handlers.
+		 */
+		next_domain = list_entry(pos,struct ipipe_domain,p_link);
+
+		/*
+		 * Keep a cached copy of the handler's address since
+		 * ipipe_catch_event() may clear it under our feet.
+		 */
+
+		evhand = next_domain->evhand[event];
+
+		if (evhand != NULL) {
+			per_cpu(ipipe_percpu_domain, cpuid) = next_domain;
+			next_domain->cpudata[cpuid].evsync |= (1LL << event);
+			ipipe_unlock_cpu(flags);
+			propagate = !evhand(event,start_domain,data);
+			ipipe_lock_cpu(flags);
+			next_domain->cpudata[cpuid].evsync &= ~(1LL << event);
+			if (per_cpu(ipipe_percpu_domain, cpuid) != next_domain)
+				this_domain = per_cpu(ipipe_percpu_domain, cpuid);
+		}
+
+		if (next_domain != ipipe_root_domain &&	/* NEVER sync the root stage here. */
+		    next_domain->cpudata[cpuid].irq_pending_hi != 0 &&
+		    !test_bit(IPIPE_STALL_FLAG,&next_domain->cpudata[cpuid].status)) {
+			per_cpu(ipipe_percpu_domain, cpuid) = next_domain;
+			__ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+			ipipe_load_cpuid();
+			if (per_cpu(ipipe_percpu_domain, cpuid) != next_domain)
+				this_domain = per_cpu(ipipe_percpu_domain, cpuid);
+		}
+
+		per_cpu(ipipe_percpu_domain, cpuid) = this_domain;
+
+		if (next_domain == this_domain || !propagate)
+			break;
+	}
+
+	ipipe_unlock_cpu(flags);
+
+	return !propagate;
+}
+
+/*
+ * __ipipe_dispatch_wired -- Wired interrupt dispatcher. Wired
+ * interrupts are immediately and unconditionally delivered to the
+ * domain heading the pipeline upon receipt, and such domain must have
+ * been registered as an invariant head for the system (priority ==
+ * IPIPE_HEAD_PRIORITY). The motivation for using wired interrupts is
+ * to get an extra-fast dispatching path for those IRQs, by relying on
+ * a straightforward logic based on assumptions that must always be
+ * true for invariant head domains.  The following assumptions are
+ * made when dealing with such interrupts:
+ *
+ * 1- Wired interrupts are purely dynamic, i.e. the decision to
+ * propagate them down the pipeline must be done from the head domain
+ * ISR.
+ * 2- Wired interrupts cannot be shared or sticky.
+ * 3- The root domain cannot be an invariant pipeline head, in
+ * consequence of what the root domain cannot handle wired
+ * interrupts.
+ * 4- Wired interrupts must have a valid acknowledge handler for the
+ * head domain (if needed), and in any case, must not rely on handlers
+ * provided by lower priority domains during the acknowledge cycle
+ * (see __ipipe_handle_irq).
+ *
+ * Called with hw interrupts off.
+ */
+
+int fastcall __ipipe_dispatch_wired(struct ipipe_domain *head, unsigned irq)
+{
+	struct ipcpudata *cpudata;
+	struct ipipe_domain *old;
+	ipipe_declare_cpuid;
+
+	ipipe_load_cpuid();
+	cpudata = &head->cpudata[cpuid];
+	cpudata->irq_counters[irq].total_hits++;
+
+	if (test_bit(IPIPE_LOCK_FLAG, &head->irqs[irq].control)) {
+		/* If we can't process this IRQ right now, we must
+		 * mark it as pending, so that it will get played
+		 * during normal log sync when the corresponding
+		 * interrupt source is eventually unlocked. */
+		cpudata->irq_counters[irq].pending_hits++;
+		return 0;
+	}
+
+	if (test_bit(IPIPE_STALL_FLAG, &cpudata->status)) {
+		cpudata->irq_counters[irq].pending_hits++;
+		__ipipe_set_irq_bit(head, cpuid, irq);
+		return 0;
+	}
+
+	old = per_cpu(ipipe_percpu_domain, cpuid);
+	per_cpu(ipipe_percpu_domain, cpuid) = head; /* Switch to the head domain. */
+
+	__set_bit(IPIPE_STALL_FLAG, &cpudata->status);
+	head->irqs[irq].handler(irq,head->irqs[irq].cookie); /* Call the ISR. */
+	__ipipe_run_irqtail();
+	__clear_bit(IPIPE_STALL_FLAG, &cpudata->status);
+
+	/* We expect the caller to start a complete pipeline walk upon
+	 * return, so that propagated interrupts will get played. */
+
+	if (per_cpu(ipipe_percpu_domain, cpuid) == head)
+		per_cpu(ipipe_percpu_domain, cpuid) = old; /* Back to the preempted domain. */
+
+	return 1;
+}
+
+/*
+ * __ipipe_sync_stage() -- Flush the pending IRQs for the current
+ * domain (and processor). This routine flushes the interrupt log
+ * (see "Optimistic interrupt protection" from D. Stodolsky et al. for
+ * more on the deferred interrupt scheme). Every interrupt that
+ * occurred while the pipeline was stalled gets played. WARNING:
+ * callers on SMP boxen should always check for CPU migration on
+ * return of this routine. One can control the kind of interrupts
+ * which are going to be sync'ed using the syncmask
+ * parameter. IPIPE_IRQMASK_ANY plays them all, IPIPE_IRQMASK_VIRT
+ * plays virtual interrupts only.
+ *
+ * This routine must be called with hw interrupts off.
+ */
+void fastcall __ipipe_sync_stage(unsigned long syncmask)
+{
+	unsigned long mask, submask;
+	struct ipcpudata *cpudata;
+	struct ipipe_domain *ipd;
+	ipipe_declare_cpuid;
+	int level, rank;
+	unsigned irq;
+
+	ipipe_load_cpuid();
+	ipd = per_cpu(ipipe_percpu_domain, cpuid);
+	cpudata = &ipd->cpudata[cpuid];
+
+	if (__test_and_set_bit(IPIPE_SYNC_FLAG, &cpudata->status))
+		return;
+
+	/*
+	 * The policy here is to keep the dispatching code interrupt-free
+	 * by stalling the current stage. If the upper domain handler
+	 * (which we call) wants to re-enable interrupts while in a safe
+	 * portion of the code (e.g. SA_INTERRUPT flag unset for Linux's
+	 * sigaction()), it will have to unstall (then stall again before
+	 * returning to us!) the stage when it sees fit.
+	 */
+	while ((mask = (cpudata->irq_pending_hi & syncmask)) != 0) {
+		level = __ipipe_ffnz(mask);
+		__clear_bit(level, &cpudata->irq_pending_hi);
+
+		while ((submask = cpudata->irq_pending_lo[level]) != 0) {
+			rank = __ipipe_ffnz(submask);
+			irq = (level << IPIPE_IRQ_ISHIFT) + rank;
+
+			if (test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control)) {
+				__clear_bit(rank, &cpudata->irq_pending_lo[level]);
+				continue;
+			}
+
+			if (--cpudata->irq_counters[irq].pending_hits == 0)
+				__clear_bit(rank, &cpudata->irq_pending_lo[level]);
+
+			__set_bit(IPIPE_STALL_FLAG, &cpudata->status);
+			__ipipe_run_isr(ipd, irq, cpuid);
+#ifdef CONFIG_SMP
+			{
+				int _cpuid = ipipe_processor_id();
+
+				if (_cpuid != cpuid) {	/* Handle CPU migration. */
+					/*
+					 * We expect any domain to clear the SYNC bit each
+					 * time it switches in a new task, so that preemptions
+					 * and/or CPU migrations (in the SMP case) over the
+					 * ISR do not lock out the log syncer for some
+					 * indefinite amount of time. In the Linux case,
+					 * schedule() handles this (see kernel/sched.c). For
+					 * this reason, we don't bother clearing it here for
+					 * the source CPU in the migration handling case,
+					 * since it must have scheduled another task in by
+					 * now.
+					 */
+					cpuid = _cpuid;
+					cpudata = &ipd->cpudata[cpuid];
+					__set_bit(IPIPE_SYNC_FLAG, &cpudata->status);
+				}
+			}
+#endif	/* CONFIG_SMP */
+
+			__clear_bit(IPIPE_STALL_FLAG, &cpudata->status);
+		}
+	}
+
+	__clear_bit(IPIPE_SYNC_FLAG, &cpudata->status);
+}
+
+#ifdef CONFIG_PROC_FS
+
+#include <linux/proc_fs.h>
+
+struct proc_dir_entry *ipipe_proc_root;
+
+static int __ipipe_version_info_proc(char *page,
+				     char **start,
+				     off_t off, int count, int *eof, void *data)
+{
+	int len = sprintf(page, "%s\n", IPIPE_VERSION_STRING);
+
+	len -= off;
+
+	if (len <= off + count)
+		*eof = 1;
+
+	*start = page + off;
+
+	if(len > count)
+		len = count;
+
+	if(len < 0)
+		len = 0;
+
+	return len;
+}
+
+static int __ipipe_common_info_proc(char *page,
+				    char **start,
+				    off_t off, int count, int *eof, void *data)
+{
+	struct ipipe_domain *ipd = (struct ipipe_domain *)data;
+	unsigned long ctlbits;
+	unsigned irq, _irq;
+	char *p = page;
+	int len;
+
+	spin_lock(&__ipipe_pipelock);
+
+	if (test_bit(IPIPE_AHEAD_FLAG,&ipd->flags))
+		p += sprintf(p, "Invariant head");
+	else
+		p += sprintf(p, "Priority=%d", ipd->priority);
+
+	p += sprintf(p, ", Id=0x%.8x\n", ipd->domid);
+
+	irq = 0;
+
+	while (irq < IPIPE_NR_IRQS) {
+		ctlbits =
+			(ipd->irqs[irq].
+			 control & (IPIPE_HANDLE_MASK | IPIPE_PASS_MASK |
+				    IPIPE_STICKY_MASK | IPIPE_WIRED_MASK));
+		if (irq >= IPIPE_NR_XIRQS && !ipipe_virtual_irq_p(irq)) {
+			/*
+			 * There might be a hole between the last external
+			 * IRQ and the first virtual one; skip it.
+			 */
+			irq++;
+			continue;
+		}
+
+		if (ipipe_virtual_irq_p(irq)
+		    && !test_bit(irq - IPIPE_VIRQ_BASE,
+				 &__ipipe_virtual_irq_map)) {
+			/* Non-allocated virtual IRQ; skip it. */
+			irq++;
+			continue;
+		}
+
+		/*
+		 * Attempt to group consecutive IRQ numbers having the
+		 * same virtualization settings in a single line.
+		 */
+
+		_irq = irq;
+
+		while (++_irq < IPIPE_NR_IRQS) {
+			if (ipipe_virtual_irq_p(_irq) !=
+			    ipipe_virtual_irq_p(irq)
+			    || (ipipe_virtual_irq_p(_irq)
+				&& !test_bit(_irq - IPIPE_VIRQ_BASE,
+					     &__ipipe_virtual_irq_map))
+			    || ctlbits != (ipd->irqs[_irq].
+			     control & (IPIPE_HANDLE_MASK |
+					IPIPE_PASS_MASK |
+					IPIPE_STICKY_MASK)))
+				break;
+		}
+
+		if (_irq == irq + 1)
+			p += sprintf(p, "irq%u: ", irq);
+		else
+			p += sprintf(p, "irq%u-%u: ", irq, _irq - 1);
+
+		/*
+		 * Statuses are as follows:
+		 * o "accepted" means handled _and_ passed down the pipeline.
+		 * o "grabbed" means handled, but the interrupt might be
+		 * terminated _or_ passed down the pipeline depending on
+		 * what the domain handler asks for to the I-pipe.
+		 * o "wired" is basically the same as "grabbed", except that
+		 * the interrupt is unconditionally delivered to an invariant
+		 * pipeline head domain.
+		 * o "passed" means unhandled by the domain but passed
+		 * down the pipeline.
+		 * o "discarded" means unhandled and _not_ passed down the
+		 * pipeline. The interrupt merely disappears from the
+		 * current domain down to the end of the pipeline.
+		 */
+		if (ctlbits & IPIPE_HANDLE_MASK) {
+			if (ctlbits & IPIPE_PASS_MASK)
+				p += sprintf(p, "accepted");
+			else if (ctlbits & IPIPE_WIRED_MASK)
+				p += sprintf(p, "wired");
+			else
+				p += sprintf(p, "grabbed");
+		} else if (ctlbits & IPIPE_PASS_MASK)
+			p += sprintf(p, "passed");
+		else
+			p += sprintf(p, "discarded");
+
+		if (ctlbits & IPIPE_STICKY_MASK)
+			p += sprintf(p, ", sticky");
+
+		if (ipipe_virtual_irq_p(irq))
+			p += sprintf(p, ", virtual");
+
+		p += sprintf(p, "\n");
+
+		irq = _irq;
+	}
+
+	spin_unlock(&__ipipe_pipelock);
+
+	len = p - page;
+
+	if (len <= off + count)
+		*eof = 1;
+
+	*start = page + off;
+
+	len -= off;
+
+	if (len > count)
+		len = count;
+
+	if (len < 0)
+		len = 0;
+
+	return len;
+}
+
+void __ipipe_add_domain_proc(struct ipipe_domain *ipd)
+{
+	create_proc_read_entry(ipd->name,0444,ipipe_proc_root,&__ipipe_common_info_proc,ipd);
+}
+
+void __ipipe_remove_domain_proc(struct ipipe_domain *ipd)
+{
+	remove_proc_entry(ipd->name,ipipe_proc_root);
+}
+
+void ipipe_init_proc(void)
+{
+	ipipe_proc_root = create_proc_entry("ipipe",S_IFDIR, 0);
+	create_proc_read_entry("version",0444,ipipe_proc_root,&__ipipe_version_info_proc,NULL);
+	__ipipe_add_domain_proc(ipipe_root_domain);
+
+	__ipipe_init_tracer();
+}
+
+#endif	/* CONFIG_PROC_FS */
+
+EXPORT_SYMBOL(ipipe_virtualize_irq);
+EXPORT_SYMBOL(ipipe_control_irq);
+EXPORT_SYMBOL(ipipe_suspend_domain);
+EXPORT_SYMBOL(ipipe_alloc_virq);
+EXPORT_PER_CPU_SYMBOL(ipipe_percpu_domain);
+EXPORT_SYMBOL(ipipe_root);
+EXPORT_SYMBOL(ipipe_stall_pipeline_from);
+EXPORT_SYMBOL(ipipe_test_and_stall_pipeline_from);
+EXPORT_SYMBOL(ipipe_unstall_pipeline_from);
+EXPORT_SYMBOL(ipipe_restore_pipeline_from);
+EXPORT_SYMBOL(ipipe_test_and_unstall_pipeline_from);
+EXPORT_SYMBOL(ipipe_unstall_pipeline_head);
+EXPORT_SYMBOL(__ipipe_restore_pipeline_head);
+EXPORT_SYMBOL(__ipipe_unstall_root);
+EXPORT_SYMBOL(__ipipe_stall_root);
+EXPORT_SYMBOL(__ipipe_restore_root);
+EXPORT_SYMBOL(__ipipe_test_and_stall_root);
+EXPORT_SYMBOL(__ipipe_test_root);
+EXPORT_SYMBOL(__ipipe_dispatch_event);
+EXPORT_SYMBOL(__ipipe_dispatch_wired);
+EXPORT_SYMBOL(__ipipe_sync_stage);
+EXPORT_SYMBOL(__ipipe_pipeline);
+EXPORT_SYMBOL(__ipipe_pipelock);
+EXPORT_SYMBOL(__ipipe_virtual_irq_map);
diff -uNrp --exclude CVS linux-head/kernel/ipipe/generic.c linux-2.6.x/kernel/ipipe/generic.c
--- linux-head/kernel/ipipe/generic.c	1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.x/kernel/ipipe/generic.c	2006-10-12 22:27:51.000000000 +0200
@@ -0,0 +1,424 @@
+/* -*- linux-c -*-
+ * linux/kernel/ipipe/generic.c
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-independent I-PIPE services.
+ */
+
+#include <linux/version.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#ifdef CONFIG_PROC_FS
+#include <linux/proc_fs.h>
+#endif	/* CONFIG_PROC_FS */
+
+MODULE_DESCRIPTION("I-pipe");
+MODULE_LICENSE("GPL");
+
+static int __ipipe_ptd_key_count;
+
+static unsigned long __ipipe_ptd_key_map;
+
+/* ipipe_register_domain() -- Link a new domain to the pipeline. */
+
+int ipipe_register_domain(struct ipipe_domain *ipd,
+			  struct ipipe_domain_attr *attr)
+{
+	struct list_head *pos;
+	unsigned long flags;
+
+	if (ipipe_current_domain != ipipe_root_domain) {
+		printk(KERN_WARNING
+		       "I-pipe: Only the root domain may register a new domain.\n");
+		return -EPERM;
+	}
+
+	if (attr->priority == IPIPE_HEAD_PRIORITY &&
+	    test_bit(IPIPE_AHEAD_FLAG,&__ipipe_pipeline_head()->flags))
+		return -EAGAIN;	/* Cannot override current head. */
+
+	flags = ipipe_critical_enter(NULL);
+
+	list_for_each(pos, &__ipipe_pipeline) {
+		struct ipipe_domain *_ipd =
+			list_entry(pos, struct ipipe_domain, p_link);
+		if (_ipd->domid == attr->domid)
+			break;
+	}
+
+	ipipe_critical_exit(flags);
+
+	if (pos != &__ipipe_pipeline)
+		/* A domain with the given id already exists -- fail. */
+		return -EBUSY;
+
+	ipd->name = attr->name;
+	ipd->domid = attr->domid;
+	ipd->pdd = attr->pdd;
+	ipd->flags = 0;
+
+	if (attr->priority == IPIPE_HEAD_PRIORITY) {
+		ipd->priority = INT_MAX;
+		__set_bit(IPIPE_AHEAD_FLAG,&ipd->flags);
+	}
+	else
+		ipd->priority = attr->priority;
+
+	__ipipe_init_stage(ipd);
+
+	INIT_LIST_HEAD(&ipd->p_link);
+
+#ifdef CONFIG_PROC_FS
+	__ipipe_add_domain_proc(ipd);
+#endif /* CONFIG_PROC_FS */
+
+	flags = ipipe_critical_enter(NULL);
+
+	list_for_each(pos, &__ipipe_pipeline) {
+		struct ipipe_domain *_ipd =
+			list_entry(pos, struct ipipe_domain, p_link);
+		if (ipd->priority > _ipd->priority)
+			break;
+	}
+
+	list_add_tail(&ipd->p_link, pos);
+
+	ipipe_critical_exit(flags);
+
+	printk(KERN_WARNING "I-pipe: Domain %s registered.\n", ipd->name);
+
+	/*
+	 * Finally, allow the new domain to perform its initialization
+	 * chores.
+	 */
+
+	if (attr->entry != NULL) {
+		ipipe_declare_cpuid;
+
+		ipipe_lock_cpu(flags);
+
+		per_cpu(ipipe_percpu_domain, cpuid) = ipd;
+		attr->entry();
+		per_cpu(ipipe_percpu_domain, cpuid) = ipipe_root_domain;
+
+		ipipe_load_cpuid();	/* Processor might have changed. */
+
+		if (ipipe_root_domain->cpudata[cpuid].irq_pending_hi != 0 &&
+		    !test_bit(IPIPE_STALL_FLAG,
+			      &ipipe_root_domain->cpudata[cpuid].status))
+			__ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+
+		ipipe_unlock_cpu(flags);
+	}
+
+	return 0;
+}
+
+/* ipipe_unregister_domain() -- Remove a domain from the pipeline. */
+
+int ipipe_unregister_domain(struct ipipe_domain *ipd)
+{
+	unsigned long flags;
+
+	if (ipipe_current_domain != ipipe_root_domain) {
+		printk(KERN_WARNING
+		       "I-pipe: Only the root domain may unregister a domain.\n");
+		return -EPERM;
+	}
+
+	if (ipd == ipipe_root_domain) {
+		printk(KERN_WARNING
+		       "I-pipe: Cannot unregister the root domain.\n");
+		return -EPERM;
+	}
+#ifdef CONFIG_SMP
+	{
+		int nr_cpus = num_online_cpus(), _cpuid;
+		unsigned irq;
+
+		/*
+		 * In the SMP case, wait for the logged events to drain on
+		 * other processors before eventually removing the domain
+		 * from the pipeline.
+		 */
+
+		ipipe_unstall_pipeline_from(ipd);
+
+		flags = ipipe_critical_enter(NULL);
+
+		for (irq = 0; irq < IPIPE_NR_IRQS; irq++) {
+			clear_bit(IPIPE_HANDLE_FLAG, &ipd->irqs[irq].control);
+			clear_bit(IPIPE_STICKY_FLAG, &ipd->irqs[irq].control);
+			set_bit(IPIPE_PASS_FLAG, &ipd->irqs[irq].control);
+		}
+
+		ipipe_critical_exit(flags);
+
+		for (_cpuid = 0; _cpuid < nr_cpus; _cpuid++)
+			for (irq = 0; irq < IPIPE_NR_IRQS; irq++)
+				while (ipd->cpudata[_cpuid].irq_counters[irq].pending_hits > 0)
+					cpu_relax();
+	}
+#endif	/* CONFIG_SMP */
+
+#ifdef CONFIG_PROC_FS
+	__ipipe_remove_domain_proc(ipd);
+#endif /* CONFIG_PROC_FS */
+
+	/*
+	 * Simply remove the domain from the pipeline and we are almost done.
+	 */
+
+	flags = ipipe_critical_enter(NULL);
+	list_del_init(&ipd->p_link);
+	ipipe_critical_exit(flags);
+
+	__ipipe_cleanup_domain(ipd);
+
+	printk(KERN_WARNING "I-pipe: Domain %s unregistered.\n", ipd->name);
+
+	return 0;
+}
+
+/*
+ * ipipe_propagate_irq() -- Force a given IRQ propagation on behalf of
+ * a running interrupt handler to the next domain down the pipeline.
+ * ipipe_schedule_irq() -- Does almost the same as above, but attempts
+ * to pend the interrupt for the current domain first.
+ */
+int fastcall __ipipe_schedule_irq(unsigned irq, struct list_head *head)
+{
+	struct list_head *ln;
+	unsigned long flags;
+	ipipe_declare_cpuid;
+
+	if (irq >= IPIPE_NR_IRQS ||
+	    (ipipe_virtual_irq_p(irq)
+	     && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)))
+		return -EINVAL;
+
+	ipipe_lock_cpu(flags);
+
+	ln = head;
+
+	while (ln != &__ipipe_pipeline) {
+		struct ipipe_domain *ipd =
+			list_entry(ln, struct ipipe_domain, p_link);
+
+		if (test_bit(IPIPE_HANDLE_FLAG, &ipd->irqs[irq].control)) {
+			ipd->cpudata[cpuid].irq_counters[irq].total_hits++;
+			ipd->cpudata[cpuid].irq_counters[irq].pending_hits++;
+			__ipipe_set_irq_bit(ipd, cpuid, irq);
+			ipipe_unlock_cpu(flags);
+			return 1;
+		}
+
+		ln = ipd->p_link.next;
+	}
+
+	ipipe_unlock_cpu(flags);
+
+	return 0;
+}
+
+/* ipipe_free_virq() -- Release a virtual/soft interrupt. */
+
+int ipipe_free_virq(unsigned virq)
+{
+	if (!ipipe_virtual_irq_p(virq))
+		return -EINVAL;
+
+	clear_bit(virq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map);
+
+	return 0;
+}
+
+void ipipe_init_attr(struct ipipe_domain_attr *attr)
+{
+	attr->name = "anon";
+	attr->domid = 1;
+	attr->entry = NULL;
+	attr->priority = IPIPE_ROOT_PRIO;
+	attr->pdd = NULL;
+}
+
+/*
+ * ipipe_catch_event() -- Interpose or remove an event handler for a
+ * given domain.
+ */
+ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd,
+					unsigned event,
+					ipipe_event_handler_t handler)
+{
+	ipipe_event_handler_t old_handler;
+	unsigned long flags;
+	int self = 0, cpuid;
+
+	if (event & IPIPE_EVENT_SELF) {
+		event &= ~IPIPE_EVENT_SELF;
+		self = 1;
+	}
+
+	if (event >= IPIPE_NR_EVENTS)
+		return NULL;
+
+	flags = ipipe_critical_enter(NULL);
+
+	if (!(old_handler = xchg(&ipd->evhand[event],handler)))	{
+		if (handler) {
+			if (self)
+				ipd->evself |= (1LL << event);
+			else
+				__ipipe_event_monitors[event]++;
+		}
+	}
+	else if (!handler) {
+		if (ipd->evself & (1LL << event))
+			ipd->evself &= ~(1LL << event);
+		else
+			__ipipe_event_monitors[event]--;
+	} else if ((ipd->evself & (1LL << event)) && !self) {
+			__ipipe_event_monitors[event]++;
+			ipd->evself &= ~(1LL << event);
+	} else if (!(ipd->evself & (1LL << event)) && self) {
+			__ipipe_event_monitors[event]--;
+			ipd->evself |= (1LL << event);
+	}
+	
+	ipipe_critical_exit(flags);
+
+	if (!handler && ipipe_root_domain_p) {
+		/*
+		 * If we cleared a handler on behalf of the root
+		 * domain, we have to wait for any current invocation
+		 * to drain, since our caller might subsequently unmap
+		 * the target domain. To this aim, this code
+		 * synchronizes with __ipipe_dispatch_event(),
+		 * guaranteeing that either the dispatcher sees a null
+		 * handler in which case it discards the invocation
+		 * (which also prevents from entering a livelock), or
+		 * finds a valid handler and calls it. Symmetrically,
+		 * ipipe_catch_event() ensures that the called code
+		 * won't be unmapped under our feet until the event
+		 * synchronization flag is cleared for the given event
+		 * on all CPUs.
+		 */
+
+		for_each_online_cpu(cpuid) {
+			while (ipd->cpudata[cpuid].evsync & (1LL << event))
+				schedule_timeout_interruptible(HZ / 50);
+		}
+	}
+
+	return old_handler;
+}
+
+cpumask_t ipipe_set_irq_affinity (unsigned irq, cpumask_t cpumask)
+{
+#ifdef CONFIG_SMP
+	if (irq >= IPIPE_NR_XIRQS)
+		/* Allow changing affinity of external IRQs only. */
+		return CPU_MASK_NONE;
+
+	if (num_online_cpus() > 1)
+		return __ipipe_set_irq_affinity(irq,cpumask);
+#endif /* CONFIG_SMP */
+
+	return CPU_MASK_NONE;
+}
+
+int fastcall ipipe_send_ipi (unsigned ipi, cpumask_t cpumask)
+
+{
+#ifdef CONFIG_SMP
+	return __ipipe_send_ipi(ipi,cpumask);
+#else /* !CONFIG_SMP */
+	return -EINVAL;
+#endif /* CONFIG_SMP */
+}
+
+int ipipe_alloc_ptdkey (void)
+{
+	unsigned long flags;
+	int key = -1;
+
+	spin_lock_irqsave_hw(&__ipipe_pipelock,flags);
+
+	if (__ipipe_ptd_key_count < IPIPE_ROOT_NPTDKEYS) {
+		key = ffz(__ipipe_ptd_key_map);
+		set_bit(key,&__ipipe_ptd_key_map);
+		__ipipe_ptd_key_count++;
+	}
+
+	spin_unlock_irqrestore_hw(&__ipipe_pipelock,flags);
+
+	return key;
+}
+
+int ipipe_free_ptdkey (int key)
+{
+	unsigned long flags;
+
+	if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+		return -EINVAL;
+
+	spin_lock_irqsave_hw(&__ipipe_pipelock,flags);
+
+	if (test_and_clear_bit(key,&__ipipe_ptd_key_map))
+		__ipipe_ptd_key_count--;
+
+	spin_unlock_irqrestore_hw(&__ipipe_pipelock,flags);
+
+	return 0;
+}
+
+int fastcall ipipe_set_ptd (int key, void *value)
+
+{
+	if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+		return -EINVAL;
+
+	current->ptd[key] = value;
+
+	return 0;
+}
+
+void fastcall *ipipe_get_ptd (int key)
+
+{
+	if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+		return NULL;
+
+	return current->ptd[key];
+}
+
+EXPORT_SYMBOL(ipipe_register_domain);
+EXPORT_SYMBOL(ipipe_unregister_domain);
+EXPORT_SYMBOL(ipipe_free_virq);
+EXPORT_SYMBOL(ipipe_init_attr);
+EXPORT_SYMBOL(ipipe_catch_event);
+EXPORT_SYMBOL(ipipe_alloc_ptdkey);
+EXPORT_SYMBOL(ipipe_free_ptdkey);
+EXPORT_SYMBOL(ipipe_set_ptd);
+EXPORT_SYMBOL(ipipe_get_ptd);
+EXPORT_SYMBOL(ipipe_set_irq_affinity);
+EXPORT_SYMBOL(ipipe_send_ipi);
+EXPORT_SYMBOL(__ipipe_schedule_irq);
diff -uNrp --exclude CVS linux-head/kernel/ipipe/tracer.c linux-2.6.x/kernel/ipipe/tracer.c
--- linux-head/kernel/ipipe/tracer.c	1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.x/kernel/ipipe/tracer.c	2006-10-12 22:27:51.000000000 +0200
@@ -0,0 +1,1243 @@
+/* -*- linux-c -*-
+ * kernel/ipipe/tracer.c
+ *
+ * Copyright (C) 2005 Luotao Fu.
+ *               2005, 2006 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/kallsyms.h>
+#include <linux/seq_file.h>
+#include <linux/proc_fs.h>
+#include <linux/ctype.h>
+#include <linux/vmalloc.h>
+#include <linux/ipipe_trace.h>
+#include <asm/uaccess.h>
+
+#define IPIPE_TRACE_PATHS           4 /* <!> Do not lower below 3 */
+#define IPIPE_DEFAULT_ACTIVE        0
+#define IPIPE_DEFAULT_MAX           1
+#define IPIPE_DEFAULT_FROZEN        2
+
+#define IPIPE_TRACE_POINTS          (1 << CONFIG_IPIPE_TRACE_SHIFT)
+#define WRAP_POINT_NO(point)        ((point) & (IPIPE_TRACE_POINTS-1))
+
+#define IPIPE_DEFAULT_PRE_TRACE     10
+#define IPIPE_DEFAULT_POST_TRACE    10
+#define IPIPE_DEFAULT_BACK_TRACE    30
+
+#define IPIPE_DELAY_NOTE            1000  /* in nanoseconds */
+#define IPIPE_DELAY_WARN            10000 /* in nanoseconds */
+
+#define IPIPE_TFLG_NMI_LOCK         0x0001
+#define IPIPE_TFLG_NMI_HIT          0x0002
+#define IPIPE_TFLG_NMI_FREEZE_REQ   0x0004
+
+#define IPIPE_TFLG_HWIRQ_OFF        0x0100
+#define IPIPE_TFLG_FREEZING         0x0200
+#define IPIPE_TFLG_CURRDOM_SHIFT    10   /* bits 10..11: current domain */
+#define IPIPE_TFLG_CURRDOM_MASK     0x0C00
+#define IPIPE_TFLG_DOMSTATE_SHIFT   12   /* bits 12..15: domain stalled? */
+#define IPIPE_TFLG_DOMSTATE_BITS    3
+
+#define IPIPE_TFLG_DOMAIN_STALLED(point, n) \
+	(point->flags & (1 << (n + IPIPE_TFLG_DOMSTATE_SHIFT)))
+#define IPIPE_TFLG_CURRENT_DOMAIN(point) \
+	((point->flags & IPIPE_TFLG_CURRDOM_MASK) >> IPIPE_TFLG_CURRDOM_SHIFT)
+
+
+struct ipipe_trace_point{
+	short type;
+	short flags;
+	unsigned long eip;
+	unsigned long parent_eip;
+	unsigned long v;
+	unsigned long long timestamp;
+};
+
+struct ipipe_trace_path{
+	volatile int flags;
+	int dump_lock; /* separated from flags due to cross-cpu access */
+	int trace_pos; /* next point to fill */
+	int begin, end; /* finalised path begin and end */
+	int post_trace; /* non-zero when in post-trace phase */
+	unsigned long long length; /* max path length in cycles */
+	unsigned long nmi_saved_eip; /* for deferred requests from NMIs */
+	unsigned long nmi_saved_parent_eip;
+	unsigned long nmi_saved_v;
+	struct ipipe_trace_point point[IPIPE_TRACE_POINTS];
+} ____cacheline_aligned_in_smp;
+
+enum ipipe_trace_type
+{
+	IPIPE_TRACE_FUNC = 0,
+	IPIPE_TRACE_BEGIN,
+	IPIPE_TRACE_END,
+	IPIPE_TRACE_FREEZE,
+	IPIPE_TRACE_SPECIAL,
+	IPIPE_TRACE_PID,
+};
+
+#define IPIPE_TYPE_MASK             0x0007
+#define IPIPE_TYPE_BITS             3
+
+
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
+#define IPIPE_DEFAULT_TRACE_STATE   0
+
+static struct ipipe_trace_path *trace_paths[NR_CPUS];
+
+#else /* !CONFIG_IPIPE_TRACE_VMALLOC */
+#define IPIPE_DEFAULT_TRACE_STATE   CONFIG_IPIPE_TRACE_ENABLE_VALUE
+
+static struct ipipe_trace_path trace_paths[NR_CPUS][IPIPE_TRACE_PATHS] =
+	{ [0 ... NR_CPUS-1] =
+		{ [0 ... IPIPE_TRACE_PATHS-1] =
+			{ .begin = -1, .end = -1 }
+		}
+	};
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
+
+int ipipe_trace_enable = IPIPE_DEFAULT_TRACE_STATE;
+
+static int active_path[NR_CPUS] =
+	{ [0 ... NR_CPUS-1] = IPIPE_DEFAULT_ACTIVE };
+static int max_path[NR_CPUS] =
+	{ [0 ... NR_CPUS-1] = IPIPE_DEFAULT_MAX };
+static int frozen_path[NR_CPUS] =
+	{ [0 ... NR_CPUS-1] = IPIPE_DEFAULT_FROZEN };
+static ipipe_spinlock_t global_path_lock = IPIPE_SPIN_LOCK_UNLOCKED;
+static int pre_trace = IPIPE_DEFAULT_PRE_TRACE;
+static int post_trace = IPIPE_DEFAULT_POST_TRACE;
+static int back_trace = IPIPE_DEFAULT_BACK_TRACE;
+static int verbose_trace = 0;
+
+static DECLARE_MUTEX(out_mutex);
+static struct ipipe_trace_path *print_path;
+static struct ipipe_trace_path *panic_path;
+static int print_pre_trace;
+static int print_post_trace;
+
+
+static long __ipipe_signed_tsc2us(long long tsc);
+static void
+__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point);
+static void __ipipe_print_symname(struct seq_file *m, unsigned long eip);
+
+
+static notrace void
+__ipipe_store_domain_states(struct ipipe_trace_point *point, int cpu_id)
+{
+	struct list_head *pos;
+	int i = 0;
+
+	list_for_each_prev(pos, &__ipipe_pipeline) {
+		struct ipipe_domain *ipd =
+			list_entry(pos, struct ipipe_domain, p_link);
+
+		if (test_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpu_id].status))
+			point->flags |= 1 << (i + IPIPE_TFLG_DOMSTATE_SHIFT);
+
+		if (ipd == per_cpu(ipipe_percpu_domain, cpu_id))
+			point->flags |= i << IPIPE_TFLG_CURRDOM_SHIFT;
+
+		if (++i > IPIPE_TFLG_DOMSTATE_BITS)
+			break;
+	}
+}
+
+static notrace int __ipipe_get_free_trace_path(int old, int cpu_id)
+{
+	int new_active = old;
+	struct ipipe_trace_path *tp;
+
+	do {
+		if (++new_active == IPIPE_TRACE_PATHS)
+			new_active = 0;
+		tp = &trace_paths[cpu_id][new_active];
+	} while ((new_active == max_path[cpu_id]) ||
+	         (new_active == frozen_path[cpu_id]) ||
+	         tp->dump_lock);
+
+	return new_active;
+}
+
+static notrace void
+__ipipe_migrate_pre_trace(struct ipipe_trace_path *new_tp,
+                          struct ipipe_trace_path *old_tp, int old_pos)
+{
+	int i;
+
+	new_tp->trace_pos = pre_trace+1;
+
+	for (i = new_tp->trace_pos; i > 0; i--)
+		memcpy(&new_tp->point[WRAP_POINT_NO(new_tp->trace_pos-i)],
+		       &old_tp->point[WRAP_POINT_NO(old_pos-i)],
+		       sizeof(struct ipipe_trace_point));
+
+	/* mark the end (i.e. the point before point[0]) invalid */
+	new_tp->point[IPIPE_TRACE_POINTS-1].eip = 0;
+}
+
+static notrace struct ipipe_trace_path *
+__ipipe_trace_end(int cpu_id, struct ipipe_trace_path *tp, int pos)
+{
+	struct ipipe_trace_path *old_tp = tp;
+	long active = active_path[cpu_id];
+	unsigned long long length;
+
+	/* do we have a new worst case? */
+	length = tp->point[tp->end].timestamp -
+	         tp->point[tp->begin].timestamp;
+	if (length > (trace_paths[cpu_id][max_path[cpu_id]]).length) {
+		/* we need protection here against other cpus trying
+		   to start a proc dump */
+		spin_lock_hw(&global_path_lock);
+
+		/* active path holds new worst case */
+		tp->length = length;
+		max_path[cpu_id] = active;
+
+		/* find next unused trace path */
+		active = __ipipe_get_free_trace_path(active, cpu_id);
+
+		spin_unlock_hw(&global_path_lock);
+
+		tp = &trace_paths[cpu_id][active];
+
+		/* migrate last entries for pre-tracing */
+		__ipipe_migrate_pre_trace(tp, old_tp, pos);
+	}
+
+	return tp;
+}
+
+static notrace struct ipipe_trace_path *
+__ipipe_trace_freeze(int cpu_id, struct ipipe_trace_path *tp, int pos)
+{
+	struct ipipe_trace_path *old_tp = tp;
+	long active = active_path[cpu_id];
+	int i;
+
+	/* frozen paths have no core (begin=end) */
+	tp->begin = tp->end;
+
+	/* we need protection here against other cpus trying
+	 * to set their frozen path or to start a proc dump */
+	spin_lock_hw(&global_path_lock);
+
+	frozen_path[cpu_id] = active;
+
+	/* find next unused trace path */
+	active = __ipipe_get_free_trace_path(active, cpu_id);
+
+	/* check if this is the first frozen path */
+	for_each_online_cpu(i) {
+		if ((i != cpu_id) &&
+		    (trace_paths[i][frozen_path[i]].end >= 0))
+			tp->end = -1;
+	}
+
+	spin_unlock_hw(&global_path_lock);
+
+	tp = &trace_paths[cpu_id][active];
+
+	/* migrate last entries for pre-tracing */
+	__ipipe_migrate_pre_trace(tp, old_tp, pos);
+
+	return tp;
+}
+
+void notrace
+__ipipe_trace(enum ipipe_trace_type type, unsigned long eip,
+              unsigned long parent_eip, unsigned long v)
+{
+	struct ipipe_trace_path *tp, *old_tp;
+	int pos, next_pos, begin;
+	struct ipipe_trace_point *point;
+	unsigned long flags;
+	int cpu_id;
+
+	local_irq_save_hw_notrace(flags);
+
+	cpu_id = ipipe_processor_id();
+restart:
+	tp = old_tp = &trace_paths[cpu_id][active_path[cpu_id]];
+
+	/* here starts a race window with NMIs - catched below */
+
+	/* check for NMI recursion */
+	if (unlikely(tp->flags & IPIPE_TFLG_NMI_LOCK)) {
+		tp->flags |= IPIPE_TFLG_NMI_HIT;
+
+		/* first freeze request from NMI context? */
+		if ((type == IPIPE_TRACE_FREEZE) &&
+		    !(tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)) {
+			/* save arguments and mark deferred freezing */
+			tp->flags |= IPIPE_TFLG_NMI_FREEZE_REQ;
+			tp->nmi_saved_eip = eip;
+			tp->nmi_saved_parent_eip = parent_eip;
+			tp->nmi_saved_v = v;
+		}
+		return; /* no need for restoring flags inside IRQ */
+	}
+
+	/* clear NMI events and set lock (atomically per cpu) */
+	tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT |
+	                           IPIPE_TFLG_NMI_FREEZE_REQ))
+	                       | IPIPE_TFLG_NMI_LOCK;
+
+	/* check active_path again - some nasty NMI may have switched
+	 * it meanwhile */
+	if (unlikely(tp != &trace_paths[cpu_id][active_path[cpu_id]])) {
+		/* release lock on wrong path and restart */
+		tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+		/* there is no chance that the NMI got deferred
+		 * => no need to check for pending freeze requests */
+		goto restart;
+	}
+
+	/* get the point buffer */
+	pos = tp->trace_pos;
+	point = &tp->point[pos];
+
+	/* store all trace point data */
+	point->type = type;
+	point->flags = local_test_iflag_hw(flags) ? 0 : IPIPE_TFLG_HWIRQ_OFF;
+	point->eip = eip;
+	point->parent_eip = parent_eip;
+	point->v = v;
+	ipipe_read_tsc(point->timestamp);
+
+	__ipipe_store_domain_states(point, cpu_id);
+
+	/* forward to next point buffer */
+	next_pos = WRAP_POINT_NO(pos+1);
+	tp->trace_pos = next_pos;
+
+	/* only mark beginning if we haven't started yet */
+	begin = tp->begin;
+	if (unlikely(type == IPIPE_TRACE_BEGIN) && (begin < 0))
+		tp->begin = pos;
+
+	/* end of critical path, start post-trace if not already started */
+	if (unlikely(type == IPIPE_TRACE_END) &&
+	    (begin >= 0) && !tp->post_trace)
+		tp->post_trace = post_trace + 1;
+
+	/* freeze only if the slot is free and we are not already freezing */
+	if (unlikely(type == IPIPE_TRACE_FREEZE) &&
+	    (trace_paths[cpu_id][frozen_path[cpu_id]].begin < 0) &&
+	    !(tp->flags & IPIPE_TFLG_FREEZING)) {
+		tp->post_trace = post_trace + 1;
+		tp->flags |= IPIPE_TFLG_FREEZING;
+	}
+
+	/* enforce end of trace in case of overflow */
+	if (unlikely(WRAP_POINT_NO(next_pos + 1) == begin)) {
+		tp->end = pos;
+		goto enforce_end;
+	}
+
+	/* stop tracing this path if we are in post-trace and
+	 *  a) that phase is over now or
+	 *  b) a new TRACE_BEGIN came in but we are not freezing this path */
+	if (unlikely((tp->post_trace > 0) && ((--tp->post_trace == 0) ||
+	             ((type == IPIPE_TRACE_BEGIN) &&
+	              !(tp->flags & IPIPE_TFLG_FREEZING))))) {
+		/* store the path's end (i.e. excluding post-trace) */
+		tp->end = WRAP_POINT_NO(pos - post_trace + tp->post_trace);
+
+enforce_end:
+		if (tp->flags & IPIPE_TFLG_FREEZING)
+			tp = __ipipe_trace_freeze(cpu_id, tp, pos);
+		else
+			tp = __ipipe_trace_end(cpu_id, tp, pos);
+
+		/* reset the active path, maybe already start a new one */
+		tp->begin = (type == IPIPE_TRACE_BEGIN) ?
+			WRAP_POINT_NO(tp->trace_pos - 1) : -1;
+		tp->end = -1;
+		tp->post_trace = 0;
+		tp->flags = 0;
+
+		/* update active_path not earlier to avoid races with NMIs */
+		active_path[cpu_id] = tp - trace_paths[cpu_id];
+	}
+
+	/* we still have old_tp and point,
+	 * let's reset NMI lock and check for catches */
+	old_tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+	if (unlikely(old_tp->flags & IPIPE_TFLG_NMI_HIT)) {
+		/* well, this late tagging may not immediately be visible for
+		 * other cpus already dumping this path - a minor issue */
+		point->flags |= IPIPE_TFLG_NMI_HIT;
+
+		/* handle deferred freezing from NMI context */
+		if (old_tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)
+			__ipipe_trace(IPIPE_TRACE_FREEZE, old_tp->nmi_saved_eip,
+			              old_tp->nmi_saved_parent_eip,
+			              old_tp->nmi_saved_v);
+	}
+
+	local_irq_restore_hw_notrace(flags);
+}
+
+static unsigned long __ipipe_global_path_lock(void)
+{
+	unsigned long flags;
+	int cpu_id;
+	struct ipipe_trace_path *tp;
+
+	spin_lock_irqsave_hw(&global_path_lock, flags);
+
+	cpu_id = ipipe_processor_id();
+restart:
+	tp = &trace_paths[cpu_id][active_path[cpu_id]];
+
+	/* here is small race window with NMIs - catched below */
+
+	/* clear NMI events and set lock (atomically per cpu) */
+	tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT |
+	                           IPIPE_TFLG_NMI_FREEZE_REQ))
+	                       | IPIPE_TFLG_NMI_LOCK;
+
+	/* check active_path again - some nasty NMI may have switched
+	 * it meanwhile */
+	if (tp != &trace_paths[cpu_id][active_path[cpu_id]]) {
+		/* release lock on wrong path and restart */
+		tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+		/* there is no chance that the NMI got deferred
+		 * => no need to check for pending freeze requests */
+		goto restart;
+	}
+
+	return flags;
+}
+
+static void __ipipe_global_path_unlock(unsigned long flags)
+{
+	int cpu_id;
+	struct ipipe_trace_path *tp;
+
+	/* release spinlock first - it's not involved in the NMI issue */
+	spin_unlock_hw(&global_path_lock);
+
+	cpu_id = ipipe_processor_id();
+	tp = &trace_paths[cpu_id][active_path[cpu_id]];
+
+	tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+	/* handle deferred freezing from NMI context */
+	if (tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)
+		__ipipe_trace(IPIPE_TRACE_FREEZE, tp->nmi_saved_eip,
+		              tp->nmi_saved_parent_eip, tp->nmi_saved_v);
+
+	local_irq_restore_hw(flags);
+}
+
+void notrace ipipe_trace_begin(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_BEGIN, __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_begin);
+
+void notrace ipipe_trace_end(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_END, __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_end);
+
+void notrace ipipe_trace_freeze(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_FREEZE, __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_freeze);
+
+void notrace ipipe_trace_special(unsigned char id, unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_SPECIAL | (id << IPIPE_TYPE_BITS),
+	              __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_special);
+
+void notrace ipipe_trace_pid(pid_t pid, short prio)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_PID | (prio << IPIPE_TYPE_BITS),
+	              __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, pid);
+}
+EXPORT_SYMBOL(ipipe_trace_pid);
+
+int ipipe_trace_max_reset(void)
+{
+	int cpu_id;
+	unsigned long flags;
+	struct ipipe_trace_path *path;
+	int ret = 0;
+
+	flags = __ipipe_global_path_lock();
+
+	for_each_cpu(cpu_id) {
+		path = &trace_paths[cpu_id][max_path[cpu_id]];
+
+		if (path->dump_lock) {
+			ret = -EBUSY;
+			break;
+		}
+
+		path->begin     = -1;
+		path->end       = -1;
+		path->trace_pos = 0;
+		path->length    = 0;
+	}
+
+	__ipipe_global_path_unlock(flags);
+
+	return ret;
+}
+EXPORT_SYMBOL(ipipe_trace_max_reset);
+
+int ipipe_trace_frozen_reset(void)
+{
+	int cpu_id;
+	unsigned long flags;
+	struct ipipe_trace_path *path;
+	int ret = 0;
+
+	flags = __ipipe_global_path_lock();
+
+	for_each_cpu(cpu_id) {
+		path = &trace_paths[cpu_id][frozen_path[cpu_id]];
+
+		if (path->dump_lock) {
+			ret = -EBUSY;
+			break;
+		}
+
+		path->begin = -1;
+		path->end = -1;
+		path->trace_pos = 0;
+		path->length    = 0;
+	}
+
+	__ipipe_global_path_unlock(flags);
+
+	return ret;
+}
+EXPORT_SYMBOL(ipipe_trace_frozen_reset);
+
+void ipipe_trace_panic_freeze(void)
+{
+	unsigned long flags;
+	int cpu_id;
+
+	ipipe_trace_enable = 0;
+	local_irq_save_hw_notrace(flags);
+
+	cpu_id = ipipe_processor_id();
+
+	panic_path = &trace_paths[cpu_id][active_path[cpu_id]];
+
+	local_irq_restore_hw(flags);
+}
+EXPORT_SYMBOL(ipipe_trace_panic_freeze);
+
+static void
+__ipipe_get_task_info(char *task_info, struct ipipe_trace_point *point,
+                      int trylock)
+{
+	struct task_struct *task = NULL;
+	char buf[8];
+	int i;
+	int locked = 1;
+
+	if (trylock && !read_trylock(&tasklist_lock))
+		locked = 0;
+	else
+		read_lock(&tasklist_lock);
+
+	if (locked)
+		task = find_task_by_pid((pid_t)point->v);
+
+	if (task)
+		strncpy(task_info, task->comm, 11);
+	else
+		strcpy(task_info, "-<?>-");
+
+	if (locked)
+		read_unlock(&tasklist_lock);
+
+	for (i = strlen(task_info); i < 11; i++)
+		task_info[i] = ' ';
+
+	sprintf(buf, " %d ", point->type >> IPIPE_TYPE_BITS);
+	strcpy(task_info + (11 - strlen(buf)), buf);
+}
+
+void ipipe_trace_panic_dump(void)
+{
+	int cnt = back_trace;
+	int start, pos;
+	char task_info[12];
+
+	printk("I-pipe tracer log (%d points):\n", cnt);
+
+	start = pos = WRAP_POINT_NO(panic_path->trace_pos-1);
+
+	while (cnt-- > 0) {
+		struct ipipe_trace_point *point = &panic_path->point[pos];
+		long time;
+		char buf[16];
+
+		if (!point->eip)
+			printk("-<invalid>-\n");
+		else {
+			__ipipe_trace_point_type(buf, point);
+			printk(buf);
+
+			switch (point->type & IPIPE_TYPE_MASK) {
+				case IPIPE_TRACE_FUNC:
+					printk("           ");
+					break;
+
+				case IPIPE_TRACE_PID:
+					__ipipe_get_task_info(task_info,
+					                      point, 1);
+					printk(task_info);
+					break;
+
+				default:
+					printk("0x%08lx ", point->v);
+			}
+
+			time = __ipipe_signed_tsc2us(point->timestamp -
+				panic_path->point[start].timestamp);
+			printk(" %5ld ", time);
+
+			__ipipe_print_symname(NULL, point->eip);
+			printk(" (");
+			__ipipe_print_symname(NULL, point->parent_eip);
+			printk(")\n");
+		}
+		pos = WRAP_POINT_NO(pos - 1);
+	}
+}
+EXPORT_SYMBOL(ipipe_trace_panic_dump);
+
+
+/* --- /proc output --- */
+
+static notrace int __ipipe_in_critical_trpath(long point_no)
+{
+	return ((WRAP_POINT_NO(point_no-print_path->begin) <
+	         WRAP_POINT_NO(print_path->end-print_path->begin)) ||
+	        ((print_path->end == print_path->begin) &&
+	         (WRAP_POINT_NO(point_no-print_path->end) >
+	          print_post_trace)));
+}
+
+static long __ipipe_signed_tsc2us(long long tsc)
+{
+        unsigned long long abs_tsc;
+        long us;
+
+	/* ipipe_tsc2us works on unsigned => handle sign separately */
+        abs_tsc = (tsc >= 0) ? tsc : -tsc;
+        us = ipipe_tsc2us(abs_tsc);
+        if (tsc < 0)
+                return -us;
+        else
+                return us;
+}
+
+static void
+__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point)
+{
+	switch (point->type & IPIPE_TYPE_MASK) {
+		case IPIPE_TRACE_FUNC:
+			strcpy(buf, "func    ");
+			break;
+
+		case IPIPE_TRACE_BEGIN:
+			strcpy(buf, "begin   ");
+			break;
+
+		case IPIPE_TRACE_END:
+			strcpy(buf, "end     ");
+			break;
+
+		case IPIPE_TRACE_FREEZE:
+			strcpy(buf, "freeze  ");
+			break;
+
+		case IPIPE_TRACE_SPECIAL:
+			sprintf(buf, "(0x%02x)  ",
+				point->type >> IPIPE_TYPE_BITS);
+			break;
+
+		case IPIPE_TRACE_PID:
+			sprintf(buf, "[%5d] ", (pid_t)point->v);
+			break;
+	}
+}
+
+static void
+__ipipe_print_pathmark(struct seq_file *m, struct ipipe_trace_point *point)
+{
+	char mark = ' ';
+	int point_no = point - print_path->point;
+	int i;
+
+	if (print_path->end == point_no)
+		mark = '<';
+	else if (print_path->begin == point_no)
+		mark = '>';
+	else if (__ipipe_in_critical_trpath(point_no))
+		mark = ':';
+	seq_printf(m, "%c%c", mark,
+	           (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' ');
+
+	if (!verbose_trace)
+		return;
+
+	for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--)
+		seq_printf(m, "%c",
+			(IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ?
+			    (IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
+				'#' : '+') :
+			(IPIPE_TFLG_DOMAIN_STALLED(point, i) ? '*' : ' '));
+}
+
+static void
+__ipipe_print_delay(struct seq_file *m, struct ipipe_trace_point *point)
+{
+	unsigned long delay = 0;
+	int next;
+	char *mark = "  ";
+
+	next = WRAP_POINT_NO(point+1 - print_path->point);
+
+	if (next != print_path->trace_pos)
+		delay = ipipe_tsc2ns(print_path->point[next].timestamp -
+		                     point->timestamp);
+
+	if (__ipipe_in_critical_trpath(point - print_path->point)) {
+		if (delay > IPIPE_DELAY_WARN)
+			mark = "! ";
+		else if (delay > IPIPE_DELAY_NOTE)
+			mark = "+ ";
+	}
+	seq_puts(m, mark);
+
+	if (verbose_trace)
+		seq_printf(m, "%3lu.%03lu%c ", delay/1000, delay%1000,
+		           (point->flags & IPIPE_TFLG_NMI_HIT) ? 'N' : ' ');
+	else
+		seq_puts(m, " ");
+}
+
+static void __ipipe_print_symname(struct seq_file *m, unsigned long eip)
+{
+	char namebuf[KSYM_NAME_LEN+1];
+	unsigned long size, offset;
+	const char *sym_name;
+	char *modname;
+
+	sym_name = kallsyms_lookup(eip, &size, &offset, &modname, namebuf);
+
+	/* printing to /proc? */
+	if (m) {
+		if (sym_name) {
+			if (verbose_trace) {
+				seq_printf(m, "%s+0x%lx", sym_name, offset);
+				if (modname)
+					seq_printf(m, " [%s]", modname);
+			} else
+				seq_puts(m, sym_name);
+		} else
+			seq_printf(m, "<%08lx>", eip);
+	} else {
+		/* panic dump */
+		if (sym_name) {
+			printk("%s+0x%lx", sym_name, offset);
+			if (modname)
+				printk(" [%s]", modname);
+		}
+	}
+}
+
+#if defined(CONFIG_XENO_OPT_DEBUG) || defined(CONFIG_DEBUG_PREEMPT)
+static void __ipipe_print_dbgwarning(struct seq_file *m)
+{
+	seq_puts(m, "\n******** WARNING ********\n"
+		"The following debugging options will increase the observed "
+		"latencies:\n"
+#ifdef CONFIG_XENO_OPT_DEBUG
+		" o CONFIG_XENO_OPT_DEBUG\n"
+#endif /* CONFIG_XENO_OPT_DEBUG */
+#ifdef CONFIG_XENO_OPT_DEBUG_QUEUES
+		" o CONFIG_XENO_OPT_DEBUG_QUEUES (very costly)\n"
+#endif /* CONFIG_XENO_OPT_DEBUG */
+#ifdef CONFIG_DEBUG_PREEMPT
+		" o CONFIG_DEBUG_PREEMPT\n"
+#endif /* CONFIG_DEBUG_PREEMPT */
+		"\n");
+}
+#else /* !WARN_ON_DEBUGGING_LATENCIES */
+# define __ipipe_print_dbgwarning(m)
+#endif /* WARN_ON_DEBUGGING_LATENCIES */
+
+static void __ipipe_print_headline(struct seq_file *m)
+{
+	if (verbose_trace) {
+		const char *name[4] = { [0 ... 3] = "<unused>" };
+		struct list_head *pos;
+		int i = 0;
+
+		list_for_each_prev(pos, &__ipipe_pipeline) {
+			struct ipipe_domain *ipd =
+				list_entry(pos, struct ipipe_domain, p_link);
+
+			name[i] = ipd->name;
+			if (++i > 3)
+				break;
+		}
+
+		seq_printf(m,
+		           " +----- Hard IRQs ('|': locked)\n"
+		           " |+---- %s\n"
+		           " ||+--- %s\n"
+		           " |||+-- %s\n"
+		           " ||||+- %s%s\n"
+		           " |||||                        +---------- "
+		               "Delay flag ('+': > %d us, '!': > %d us)\n"
+		           " |||||                        |        +- "
+		               "NMI noise ('N')\n"
+		           " |||||                        |        |\n"
+		           "      Type    User Val.   Time    Delay  Function "
+		               "(Parent)\n",
+		           name[3], name[2], name[1], name[0],
+		           name[0] ? " ('*': domain stalled, '+': current, "
+		               "'#': current+stalled)" : "",
+		           IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000);
+	} else
+		seq_printf(m,
+		           " +--------------- Hard IRQs ('|': locked)\n"
+		           " |             +- Delay flag "
+		               "('+': > %d us, '!': > %d us)\n"
+		           " |             |\n"
+		           "  Type     Time   Function (Parent)\n",
+		           IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000);
+}
+
+static void *__ipipe_max_prtrace_start(struct seq_file *m, loff_t *pos)
+{
+	loff_t n = *pos;
+
+	down(&out_mutex);
+
+	if (!n) {
+		struct ipipe_trace_path *path;
+		unsigned long length_usecs;
+		int points, i;
+		unsigned long flags;
+
+		/* protect against max_path/frozen_path updates while we
+		 * haven't locked our target path, also avoid recursively
+		 * taking global_path_lock from NMI context */
+		flags = __ipipe_global_path_lock();
+
+		/* find the longest of all per-cpu paths */
+		print_path = NULL;
+		for_each_online_cpu(i) {
+			path = &trace_paths[i][max_path[i]];
+			if ((print_path == NULL) ||
+			    (path->length > print_path->length))
+				print_path = path;
+		}
+		print_path->dump_lock = 1;
+
+		__ipipe_global_path_unlock(flags);
+
+		/* does this path actually contain data? */
+		if (print_path->end == print_path->begin)
+			return NULL;
+
+		/* number of points inside the critical path */
+		points = WRAP_POINT_NO(print_path->end-print_path->begin+1);
+
+		/* pre- and post-tracing length, post-trace length was frozen
+		   in __ipipe_trace, pre-trace may have to be reduced due to
+		   buffer overrun */
+		print_pre_trace  = pre_trace;
+		print_post_trace = WRAP_POINT_NO(print_path->trace_pos -
+		                                 print_path->end - 1);
+		if (points+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1)
+			print_pre_trace = IPIPE_TRACE_POINTS - 1 - points -
+				print_post_trace;
+
+		length_usecs = ipipe_tsc2us(print_path->length);
+		seq_printf(m, "I-pipe worst-case tracing service on %s/ipipe-%s\n"
+			"------------------------------------------------------------\n",
+			UTS_RELEASE, IPIPE_ARCH_STRING);
+		__ipipe_print_dbgwarning(m);
+		seq_printf(m, "Begin: %lld cycles, Trace Points: %d (-%d/+%d), "
+			"Length: %lu us\n\n",
+			print_path->point[print_path->begin].timestamp,
+			points, print_pre_trace, print_post_trace, length_usecs);
+		__ipipe_print_headline(m);
+	}
+
+	/* check if we are inside the trace range */
+	if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 +
+	                       print_pre_trace + print_post_trace))
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin -
+	                                        print_pre_trace + n)];
+}
+
+static void *__ipipe_prtrace_next(struct seq_file *m, void *p, loff_t *pos)
+{
+	loff_t n = ++*pos;
+
+	/* check if we are inside the trace range with the next entry */
+	if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 +
+	                       print_pre_trace + print_post_trace))
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin -
+	                                        print_pre_trace + *pos)];
+}
+
+static void __ipipe_prtrace_stop(struct seq_file *m, void *p)
+{
+	if (print_path)
+		print_path->dump_lock = 0;
+	up(&out_mutex);
+}
+
+static int __ipipe_prtrace_show(struct seq_file *m, void *p)
+{
+	long time;
+	struct ipipe_trace_point *point = p;
+	char buf[16];
+
+	if (!point->eip) {
+		seq_puts(m, "-<invalid>-\n");
+		return 0;
+	}
+
+	__ipipe_print_pathmark(m, point);
+	__ipipe_trace_point_type(buf, point);
+	seq_puts(m, buf);
+	if (verbose_trace)
+		switch (point->type & IPIPE_TYPE_MASK) {
+			case IPIPE_TRACE_FUNC:
+				seq_puts(m, "           ");
+				break;
+
+			case IPIPE_TRACE_PID:
+				__ipipe_get_task_info(buf, point, 0);
+				seq_puts(m, buf);
+				break;
+
+			default:
+				seq_printf(m, "0x%08lx ", point->v);
+		}
+
+	time = __ipipe_signed_tsc2us(point->timestamp -
+		print_path->point[print_path->begin].timestamp);
+	seq_printf(m, "%5ld", time);
+
+	__ipipe_print_delay(m, point);
+	__ipipe_print_symname(m, point->eip);
+	seq_puts(m, " (");
+	__ipipe_print_symname(m, point->parent_eip);
+	seq_puts(m, ")\n");
+
+	return 0;
+}
+
+static struct seq_operations __ipipe_max_ptrace_ops = {
+	.start = __ipipe_max_prtrace_start,
+	.next  = __ipipe_prtrace_next,
+	.stop  = __ipipe_prtrace_stop,
+	.show  = __ipipe_prtrace_show
+};
+
+static int __ipipe_max_prtrace_open(struct inode *inode, struct file *file)
+{
+	return seq_open(file, &__ipipe_max_ptrace_ops);
+}
+
+static ssize_t
+__ipipe_max_reset(struct file *file, const char __user *pbuffer,
+                  size_t count, loff_t *data)
+{
+	down(&out_mutex);
+	ipipe_trace_max_reset();
+	up(&out_mutex);
+
+	return count;
+}
+
+struct file_operations __ipipe_max_prtrace_fops = {
+	.open       = __ipipe_max_prtrace_open,
+	.read       = seq_read,
+	.write      = __ipipe_max_reset,
+	.llseek     = seq_lseek,
+	.release    = seq_release,
+};
+
+static void *__ipipe_frozen_prtrace_start(struct seq_file *m, loff_t *pos)
+{	
+	loff_t n = *pos;
+
+	down(&out_mutex);
+
+	if (!n) {
+		struct ipipe_trace_path *path;
+		int i;
+		unsigned long flags;
+
+		/* protect against max_path/frozen_path updates while we
+		 * haven't locked our target path, also avoid recursively
+		 * taking global_path_lock from NMI context */
+		flags = __ipipe_global_path_lock();
+
+		/* find the first of all per-cpu frozen paths */
+		print_path = NULL;
+		for_each_online_cpu(i) {
+			path = &trace_paths[i][frozen_path[i]];
+			if (path->end >= 0)
+				print_path = path;
+		}
+		if (print_path)
+			print_path->dump_lock = 1;
+
+		__ipipe_global_path_unlock(flags);
+
+		if (!print_path)
+			return NULL;
+
+		/* back- and post-tracing length, post-trace length was frozen
+		   in __ipipe_trace, back-trace may have to be reduced due to
+		   buffer overrun */
+		print_pre_trace  = back_trace-1; /* substract freeze point */
+		print_post_trace = WRAP_POINT_NO(print_path->trace_pos -
+		                                 print_path->end - 1);
+		if (1+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1)
+			print_pre_trace = IPIPE_TRACE_POINTS - 2 -
+				print_post_trace;
+
+		seq_printf(m, "I-pipe frozen back-tracing service on %s/ipipe-%s\n"
+			"------------------------------------------------------"
+			"------\n",
+			UTS_RELEASE, IPIPE_ARCH_STRING);
+		__ipipe_print_dbgwarning(m);
+		seq_printf(m, "Freeze: %lld cycles, Trace Points: %d (+%d)\n\n",
+			print_path->point[print_path->begin].timestamp,
+			print_pre_trace+1, print_post_trace);
+		__ipipe_print_headline(m);
+	}
+
+	/* check if we are inside the trace range */
+	if (n >= print_pre_trace + 1 + print_post_trace)
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin-
+	                                        print_pre_trace+n)];
+}
+
+static struct seq_operations __ipipe_frozen_ptrace_ops = {
+	.start = __ipipe_frozen_prtrace_start,
+	.next  = __ipipe_prtrace_next,
+	.stop  = __ipipe_prtrace_stop,
+	.show  = __ipipe_prtrace_show
+};
+
+static int __ipipe_frozen_prtrace_open(struct inode *inode, struct file *file)
+{
+	return seq_open(file, &__ipipe_frozen_ptrace_ops);
+}
+
+static ssize_t
+__ipipe_frozen_ctrl(struct file *file, const char __user *pbuffer,
+                    size_t count, loff_t *data)
+{
+	char *end, buf[16];
+	int val;
+	int n;
+
+	n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+	if (copy_from_user(buf, pbuffer, n))
+		return -EFAULT;
+
+	buf[n] = '\0';
+	val = simple_strtol(buf, &end, 0);
+
+	if (((*end != '\0') && !isspace(*end)) || (val < 0))
+		return -EINVAL;
+
+	down(&out_mutex);
+	ipipe_trace_frozen_reset();
+	if (val > 0)
+		ipipe_trace_freeze(-1);
+	up(&out_mutex);
+
+	return count;
+}
+
+struct file_operations __ipipe_frozen_prtrace_fops = {
+	.open       = __ipipe_frozen_prtrace_open,
+	.read       = seq_read,
+	.write      = __ipipe_frozen_ctrl,
+	.llseek     = seq_lseek,
+	.release    = seq_release,
+};
+
+static int __ipipe_rd_proc_val(char *page, char **start, off_t off,
+                               int count, int *eof, void *data)
+{
+	int len;
+
+	len = sprintf(page, "%u\n", *(int *)data);
+	len -= off;
+	if (len <= off + count)
+		*eof = 1;
+	*start = page + off;
+	if (len > count)
+		len = count;
+	if (len < 0)
+		len = 0;
+
+	return len;
+}
+
+static int __ipipe_wr_proc_val(struct file *file, const char __user *buffer,
+                               unsigned long count, void *data)
+{
+	char *end, buf[16];
+	int val;
+	int n;
+
+	n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+	if (copy_from_user(buf, buffer, n))
+		return -EFAULT;
+
+	buf[n] = '\0';
+	val = simple_strtol(buf, &end, 0);
+
+	if (((*end != '\0') && !isspace(*end)) || (val < 0))
+		return -EINVAL;
+
+	down(&out_mutex);
+	*(int *)data = val;
+	up(&out_mutex);
+
+	return count;
+}
+
+extern struct proc_dir_entry *ipipe_proc_root;
+
+static void __init
+__ipipe_create_trace_proc_val(struct proc_dir_entry *trace_dir,
+                              const char *name, int *value_ptr)
+{
+	struct proc_dir_entry *entry;
+
+	entry = create_proc_entry(name, 0644, trace_dir);
+	if (entry) {
+		entry->data = value_ptr;
+		entry->read_proc = __ipipe_rd_proc_val;
+		entry->write_proc = __ipipe_wr_proc_val;
+		entry->owner = THIS_MODULE;
+	}
+}
+
+void __init __ipipe_init_tracer(void)
+{
+	struct proc_dir_entry *trace_dir;
+	struct proc_dir_entry *entry;
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
+	int cpu, path;
+
+	for (cpu = 0; cpu < NR_CPUS; cpu++) {
+		trace_paths[cpu] = vmalloc(
+			sizeof(struct ipipe_trace_path) * IPIPE_TRACE_PATHS);
+		if (!trace_paths) {
+			printk(KERN_ERR "I-pipe: "
+			       "insufficient memory for trace buffer.\n");
+			return;
+		}
+		memset(trace_paths[cpu], 0,
+			sizeof(struct ipipe_trace_path) * IPIPE_TRACE_PATHS);
+		for (path = 0; path < IPIPE_TRACE_PATHS; path++) {
+			trace_paths[cpu][path].begin = -1;
+			trace_paths[cpu][path].end   = -1;
+		}
+	}
+	ipipe_trace_enable = CONFIG_IPIPE_TRACE_ENABLE_VALUE;
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
+
+	trace_dir = create_proc_entry("trace", S_IFDIR, ipipe_proc_root);
+
+	entry = create_proc_entry("max", 0644, trace_dir);
+	if (entry)
+		entry->proc_fops = &__ipipe_max_prtrace_fops;
+
+	entry = create_proc_entry("frozen", 0644, trace_dir);
+	if (entry)
+		entry->proc_fops = &__ipipe_frozen_prtrace_fops;
+
+	__ipipe_create_trace_proc_val(trace_dir, "pre_trace_points",
+	                              &pre_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "post_trace_points",
+	                              &post_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "back_trace_points",
+	                              &back_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "verbose",
+	                              &verbose_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "enable",
+	                              &ipipe_trace_enable);
+}
diff -uNrp --exclude CVS linux-head/kernel/irq/handle.c linux-2.6.x/kernel/irq/handle.c
--- linux-head/kernel/irq/handle.c	2006-03-22 03:36:57.000000000 +0100
+++ linux-2.6.x/kernel/irq/handle.c	2006-10-09 22:36:50.000000000 +0200
@@ -81,6 +81,17 @@ fastcall int handle_IRQ_event(unsigned i
 {
 	int ret, retval = 0, status = 0;
 
+#ifdef CONFIG_IPIPE
+	/*
+	 * If processing a timer tick, pass the original regs as
+	 * collected during preemption and not our phony - always
+	 * kernel-originated - frame, so that we don't wreck the
+	 * profiling code.
+	 */
+	if (__ipipe_tick_irq == irq)
+		regs = __ipipe_tick_regs + smp_processor_id();
+#endif /* CONFIG_IPIPE */
+
 	if (!(action->flags & SA_INTERRUPT))
 		local_irq_enable();
 
@@ -117,16 +128,20 @@ fastcall unsigned int __do_IRQ(unsigned 
 		/*
 		 * No locking required for CPU-local interrupts:
 		 */
+#ifndef CONFIG_IPIPE
 		if (desc->handler->ack)
 			desc->handler->ack(irq);
+#endif /* !CONFIG_IPIPE */
 		action_ret = handle_IRQ_event(irq, regs, desc->action);
 		desc->handler->end(irq);
 		return 1;
 	}
 
 	spin_lock(&desc->lock);
+#ifndef CONFIG_IPIPE
 	if (desc->handler->ack)
 		desc->handler->ack(irq);
+#endif /* !CONFIG_IPIPE */
 	/*
 	 * REPLAY is when Linux resends an IRQ that was dropped earlier
 	 * WAITING is used by probe to mark irqs that are being tested
diff -uNrp --exclude CVS linux-head/kernel/printk.c linux-2.6.x/kernel/printk.c
--- linux-head/kernel/printk.c	2006-03-22 03:36:56.000000000 +0100
+++ linux-2.6.x/kernel/printk.c	2006-10-09 22:36:50.000000000 +0200
@@ -511,6 +511,78 @@ __attribute__((weak)) unsigned long long
  * printf(3)
  */
 
+#ifdef CONFIG_IPIPE
+
+static ipipe_spinlock_t __ipipe_printk_lock = IPIPE_SPIN_LOCK_UNLOCKED;
+
+static int __ipipe_printk_fill;
+
+static char __ipipe_printk_buf[__LOG_BUF_LEN];
+
+void __ipipe_flush_printk (unsigned virq, void *cookie)
+{
+	char *p = __ipipe_printk_buf;
+	int len, lmax, out = 0;
+	unsigned long flags;
+
+	goto start;
+
+	do {
+		spin_unlock_irqrestore_hw(&__ipipe_printk_lock,flags);
+ start:
+		lmax = __ipipe_printk_fill;
+		while (out < lmax) {
+			len = strlen(p) + 1;
+			printk("%s",p);
+			p += len;
+			out += len;
+		}
+		spin_lock_irqsave_hw(&__ipipe_printk_lock,flags);
+	}
+	while (__ipipe_printk_fill != lmax);
+
+	__ipipe_printk_fill = 0;
+
+	spin_unlock_irqrestore_hw(&__ipipe_printk_lock,flags);
+}
+
+asmlinkage int printk(const char *fmt, ...)
+{
+	int r, fbytes, oldcount;
+    	unsigned long flags;
+	va_list args;
+
+	va_start(args, fmt);
+
+	if (ipipe_current_domain == ipipe_root_domain ||
+	    test_bit(IPIPE_SPRINTK_FLAG,&ipipe_current_domain->flags) ||
+	    oops_in_progress) {
+		r = vprintk(fmt, args);
+		goto out;
+	}
+
+	spin_lock_irqsave_hw(&__ipipe_printk_lock,flags);
+
+	oldcount = __ipipe_printk_fill;
+	fbytes = __LOG_BUF_LEN - oldcount;
+
+	if (fbytes > 1)	{
+		r = vscnprintf(__ipipe_printk_buf + __ipipe_printk_fill,
+			       fbytes, fmt, args) + 1; /* account for the null byte */
+		__ipipe_printk_fill += r;
+	} else
+		r = 0;
+
+	spin_unlock_irqrestore_hw(&__ipipe_printk_lock,flags);
+
+	if (oldcount == 0)
+		ipipe_trigger_irq(__ipipe_printk_virq);
+out: 
+	va_end(args);
+
+	return r;
+}
+#else /* !CONFIG_IPIPE */
 asmlinkage int printk(const char *fmt, ...)
 {
 	va_list args;
@@ -522,6 +594,7 @@ asmlinkage int printk(const char *fmt, .
 
 	return r;
 }
+#endif /* CONFIG_IPIPE */
 
 /* cpu currently holding logbuf_lock */
 static volatile unsigned int printk_cpu = UINT_MAX;
diff -uNrp --exclude CVS linux-head/kernel/sched.c linux-2.6.x/kernel/sched.c
--- linux-head/kernel/sched.c	2006-04-27 17:03:18.000000000 +0200
+++ linux-2.6.x/kernel/sched.c	2006-10-09 22:36:50.000000000 +0200
@@ -1169,7 +1169,7 @@ static int try_to_wake_up(task_t *p, uns
 
 	rq = task_rq_lock(p, &flags);
 	old_state = p->state;
-	if (!(old_state & state))
+	if (!(old_state & state) || (old_state & TASK_NOWAKEUP))
 		goto out;
 
 	if (p->array)
@@ -1571,6 +1571,8 @@ asmlinkage void schedule_tail(task_t *pr
 #endif
 	if (current->set_child_tid)
 		put_user(current->pid, current->set_child_tid);
+
+	ipipe_init_notify(current);
 }
 
 /*
@@ -2872,13 +2874,17 @@ asmlinkage void __sched schedule(void)
 	unsigned long run_time;
 	int cpu, idx, new_prio;
 
+#ifdef CONFIG_IPIPE
+	if (unlikely(!ipipe_root_domain_p))
+		return;
+#endif /* CONFIG_IPIPE */
 	/*
 	 * Test if we are atomic.  Since do_exit() needs to call into
 	 * schedule() atomically, we ignore that path for now.
 	 * Otherwise, whine if we are scheduling when we should not be.
 	 */
 	if (likely(!current->exit_state)) {
-		if (unlikely(in_atomic())) {
+		if (unlikely(!(current->state & TASK_ATOMICSWITCH) && in_atomic())) {
 			printk(KERN_ERR "scheduling while atomic: "
 				"%s/0x%08x/%d\n",
 				current->comm, preempt_count(), current->pid);
@@ -2887,8 +2893,13 @@ asmlinkage void __sched schedule(void)
 	}
 	profile_hit(SCHED_PROFILING, __builtin_return_address(0));
 
+	if (unlikely(current->state & TASK_ATOMICSWITCH)) {
+		current->state &= ~TASK_ATOMICSWITCH;
+		goto need_resched_nodisable;
+	}
 need_resched:
 	preempt_disable();
+need_resched_nodisable:
 	prev = current;
 	release_kernel_lock(prev);
 need_resched_nonpreemptible:
@@ -3027,6 +3038,8 @@ switch_tasks:
 		prepare_task_switch(rq, next);
 		prev = context_switch(rq, prev, next);
 		barrier();
+ 		if (task_hijacked(prev))
+ 		    return;
 		/*
 		 * this_rq must be evaluated again because prev may have moved
 		 * CPUs since it called schedule(), thus the 'rq' on its stack
@@ -3040,7 +3053,7 @@ switch_tasks:
 	if (unlikely(reacquire_kernel_lock(prev) < 0))
 		goto need_resched_nonpreemptible;
 	preempt_enable_no_resched();
-	if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
+	if (unlikely(test_thread_flag(TIF_NEED_RESCHED) && ipipe_root_domain_p))
 		goto need_resched;
 }
 
@@ -3059,6 +3072,11 @@ asmlinkage void __sched preempt_schedule
 	struct task_struct *task = current;
 	int saved_lock_depth;
 #endif
+#ifdef CONFIG_IPIPE
+	/* Do not reschedule over non-Linux domains. */
+	if (unlikely(!ipipe_root_domain_p))
+ 		return;
+#endif /* CONFIG_IPIPE */
 	/*
 	 * If there is a non-zero preempt_count or interrupts are disabled,
 	 * we do not want to preempt the current task.  Just return..
@@ -3697,6 +3715,7 @@ recheck:
 		deactivate_task(p, rq);
 	oldprio = p->prio;
 	__setscheduler(p, policy, param->sched_priority);
+	ipipe_setsched_notify(p);
 	if (array) {
 		__activate_task(p, rq);
 		/*
@@ -6165,3 +6184,50 @@ void set_curr_task(int cpu, task_t *p)
 }
 
 #endif
+
+#ifdef CONFIG_IPIPE
+
+int ipipe_setscheduler_root (struct task_struct *p, int policy, int prio)
+{
+	prio_array_t *array;
+	unsigned long flags;
+	runqueue_t *rq;
+	int oldprio;
+
+	rq = task_rq_lock(p, &flags);
+	array = p->array;
+	if (array)
+		deactivate_task(p, rq);
+	oldprio = p->prio;
+	__setscheduler(p, policy, prio);
+	if (array) {
+		__activate_task(p, rq);
+		if (task_running(rq, p)) {
+			if (p->prio > oldprio)
+				resched_task(rq->curr);
+		} else if (TASK_PREEMPTS_CURR(p, rq))
+			resched_task(rq->curr);
+	}
+	task_rq_unlock(rq, &flags);
+
+	return 0;
+}
+
+EXPORT_SYMBOL(ipipe_setscheduler_root);
+
+int ipipe_reenter_root (struct task_struct *prev, int policy, int prio)
+{
+	finish_task_switch(this_rq(), prev);
+	if (reacquire_kernel_lock(current) < 0)
+		;
+	preempt_enable_no_resched();
+
+	if (current->policy != policy || current->rt_priority != prio)
+		return ipipe_setscheduler_root(current,policy,prio);
+
+	return 0;
+}
+
+EXPORT_SYMBOL(ipipe_reenter_root);
+
+#endif /* CONFIG_IPIPE */
diff -uNrp --exclude CVS linux-head/kernel/signal.c linux-2.6.x/kernel/signal.c
--- linux-head/kernel/signal.c	2006-04-27 17:03:18.000000000 +0200
+++ linux-2.6.x/kernel/signal.c	2006-10-09 22:36:50.000000000 +0200
@@ -604,6 +604,7 @@ void signal_wake_up(struct task_struct *
 	unsigned int mask;
 
 	set_tsk_thread_flag(t, TIF_SIGPENDING);
+	ipipe_sigwake_notify(t); /* TIF_SIGPENDING must be set first. */
 
 	/*
 	 * For SIGKILL, we want to wake it up in the stopped/traced case.
diff -uNrp --exclude CVS linux-head/lib/Kconfig.debug linux-2.6.x/lib/Kconfig.debug
--- linux-head/lib/Kconfig.debug	2006-04-11 04:34:15.000000000 +0200
+++ linux-2.6.x/lib/Kconfig.debug	2006-10-15 12:27:20.000000000 +0200
@@ -23,6 +23,8 @@ config MAGIC_SYSRQ
 	  keys are documented in <file:Documentation/sysrq.txt>. Don't say Y
 	  unless you really know what this hack does.
 
+#source "kernel/ipipe/Kconfig.debug"
+
 config DEBUG_KERNEL
 	bool "Kernel debugging"
 	help
diff -uNrp --exclude CVS linux-head/lib/smp_processor_id.c linux-2.6.x/lib/smp_processor_id.c
--- linux-head/lib/smp_processor_id.c	2006-03-22 03:37:51.000000000 +0100
+++ linux-2.6.x/lib/smp_processor_id.c	2006-10-09 22:36:50.000000000 +0200
@@ -13,6 +13,11 @@ unsigned int debug_smp_processor_id(void
 	int this_cpu = raw_smp_processor_id();
 	cpumask_t this_mask;
 
+#ifdef CONFIG_IPIPE
+ 	if (ipipe_current_domain != ipipe_root_domain)
+	    return this_cpu;
+#endif /* CONFIG_IPIPE */
+
 	if (likely(preempt_count))
 		goto out;
 
diff -uNrp --exclude CVS linux-head/lib/spinlock_debug.c linux-2.6.x/lib/spinlock_debug.c
--- linux-head/lib/spinlock_debug.c	2006-03-22 03:37:51.000000000 +0100
+++ linux-2.6.x/lib/spinlock_debug.c	2006-10-09 22:36:50.000000000 +0200
@@ -10,6 +10,7 @@
 #include <linux/spinlock.h>
 #include <linux/interrupt.h>
 #include <linux/delay.h>
+#include <linux/module.h>
 
 static void spin_bug(spinlock_t *lock, const char *msg)
 {
@@ -96,6 +97,8 @@ void _raw_spin_lock(spinlock_t *lock)
 	debug_spin_lock_after(lock);
 }
 
+EXPORT_SYMBOL(_raw_spin_lock);
+
 int _raw_spin_trylock(spinlock_t *lock)
 {
 	int ret = __raw_spin_trylock(&lock->raw_lock);
@@ -111,12 +114,16 @@ int _raw_spin_trylock(spinlock_t *lock)
 	return ret;
 }
 
+EXPORT_SYMBOL(_raw_spin_trylock);
+
 void _raw_spin_unlock(spinlock_t *lock)
 {
 	debug_spin_unlock(lock);
 	__raw_spin_unlock(&lock->raw_lock);
 }
 
+EXPORT_SYMBOL(_raw_spin_unlock);
+
 static void rwlock_bug(rwlock_t *lock, const char *msg)
 {
 	static long print_once = 1;
@@ -167,6 +174,8 @@ void _raw_read_lock(rwlock_t *lock)
 		__read_lock_debug(lock);
 }
 
+EXPORT_SYMBOL(_raw_read_lock);
+
 int _raw_read_trylock(rwlock_t *lock)
 {
 	int ret = __raw_read_trylock(&lock->raw_lock);
@@ -180,12 +189,16 @@ int _raw_read_trylock(rwlock_t *lock)
 	return ret;
 }
 
+EXPORT_SYMBOL(_raw_read_trylock);
+
 void _raw_read_unlock(rwlock_t *lock)
 {
 	RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
 	__raw_read_unlock(&lock->raw_lock);
 }
 
+EXPORT_SYMBOL(_raw_read_unlock);
+
 static inline void debug_write_lock_before(rwlock_t *lock)
 {
 	RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
@@ -241,6 +254,8 @@ void _raw_write_lock(rwlock_t *lock)
 	debug_write_lock_after(lock);
 }
 
+EXPORT_SYMBOL(_raw_write_lock);
+
 int _raw_write_trylock(rwlock_t *lock)
 {
 	int ret = __raw_write_trylock(&lock->raw_lock);
@@ -256,8 +271,12 @@ int _raw_write_trylock(rwlock_t *lock)
 	return ret;
 }
 
+EXPORT_SYMBOL(_raw_write_trylock);
+
 void _raw_write_unlock(rwlock_t *lock)
 {
 	debug_write_unlock(lock);
 	__raw_write_unlock(&lock->raw_lock);
 }
+
+EXPORT_SYMBOL(_raw_write_unlock);
Binary files linux-head/linux and linux-2.6.x/linux differ
