diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 88eace4..5c53d0f 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -109,6 +109,7 @@ config PPC
 	select HAVE_MEMBLOCK_NODE_MAP
 	select HAVE_DMA_ATTRS
 	select HAVE_DMA_API_DEBUG
+ 	select HAVE_FUNCTION_TRACE_MCOUNT_TEST
 	select HAVE_OPROFILE
 	select HAVE_DEBUG_KMEMLEAK
 	select ARCH_HAS_SG_CHAIN
@@ -123,6 +124,7 @@ config PPC
 	select GENERIC_IRQ_SHOW
 	select GENERIC_IRQ_SHOW_LEVEL
 	select IRQ_FORCED_THREADING
+	select IPIPE_HAVE_HOSTRT if IPIPE
 	select HAVE_RCU_TABLE_FREE if SMP
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_BPF_JIT if PPC64
@@ -301,6 +303,13 @@ source "arch/powerpc/platforms/Kconfig"
 
 menu "Kernel options"
 
+source "kernel/ipipe/Kconfig"
+
+config IPIPE_HAVE_PREEMPTIBLE_SWITCH
+       bool
+       depends on IPIPE
+       default y
+
 config HIGHMEM
 	bool "High memory support"
 	depends on PPC32
diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile
index 8a5bc1c..7e5f382 100644
--- a/arch/powerpc/boot/Makefile
+++ b/arch/powerpc/boot/Makefile
@@ -36,6 +36,14 @@ ifdef CONFIG_DEBUG_INFO
 BOOTCFLAGS	+= -g
 endif
 
+ifdef CONFIG_IPIPE_TRACE
+# do not trace the boot loader
+nullstring :=
+space      := $(nullstring) # end of the line
+pg_flag     = $(nullstring) -pg # end of the line
+BOOTCFLAGS     := $(subst ${pg_flag},${space},${BOOTCFLAGS})
+endif
+
 ifeq ($(call cc-option-yn, -fstack-protector),y)
 BOOTCFLAGS	+= -fno-stack-protector
 endif
diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h
index 77f52b2..8eb3473 100644
--- a/arch/powerpc/include/asm/exception-64s.h
+++ b/arch/powerpc/include/asm/exception-64s.h
@@ -332,6 +332,20 @@ do_kvm_##n:								\
 	GET_CTR(r10, area);						   \
 	std	r10,_CTR(r1);
 
+#ifdef CONFIG_IPIPE
+/* Do NOT alter Rc(eq) in this code;  our caller uses it. */
+#define COPY_SOFTISTATE(mreg)			\
+	ld	mreg,PACAROOTPCPU(r13);		\
+	ld	mreg,0(mreg);			\
+	nor	mreg,mreg,mreg;			\
+	clrldi	mreg,mreg,63;			\
+	std	mreg,SOFTE(r1)
+#else /* !CONFIG_IPIPE */
+#define COPY_SOFTISTATE(mreg)			\
+	lbz	mreg,PACASOFTIRQEN(r13);	\
+	std	mreg,SOFTE(r1)
+#endif /* !CONFIG_IPIPE */
+
 #define EXCEPTION_PROLOG_COMMON_3(n)					   \
 	std	r2,GPR2(r1);		/* save r2 in stackframe	*/ \
 	SAVE_4GPRS(3, r1);		/* save r3 - r6 in stackframe   */ \
@@ -339,9 +353,8 @@ do_kvm_##n:								\
 	mflr	r9;			/* Get LR, later save to stack	*/ \
 	ld	r2,PACATOC(r13);	/* get kernel TOC into r2	*/ \
 	std	r9,_LINK(r1);						   \
-	lbz	r10,PACASOFTIRQEN(r13);				   \
+	COPY_SOFTISTATE(r10);						   \
 	mfspr	r11,SPRN_XER;		/* save XER in stackframe	*/ \
-	std	r10,SOFTE(r1);						   \
 	std	r11,_XER(r1);						   \
 	li	r9,(n)+1;						   \
 	std	r9,_TRAP(r1);		/* set trap number		*/ \
@@ -428,11 +441,15 @@ label##_relon_hv:						\
 #define SOFTEN_VALUE_0xe60	PACA_IRQ_HMI
 #define SOFTEN_VALUE_0xe62	PACA_IRQ_HMI
 
+#ifdef CONFIG_IPIPE
+#define __SOFTEN_TEST(h, vec)
+#else /* !CONFIG_IPIPE */
 #define __SOFTEN_TEST(h, vec)						\
 	lbz	r10,PACASOFTIRQEN(r13);					\
 	cmpwi	r10,0;							\
 	li	r10,SOFTEN_VALUE_##vec;					\
 	beq	masked_##h##interrupt
+#endif /* !CONFIG_IPIPE */
 #define _SOFTEN_TEST(h, vec)	__SOFTEN_TEST(h, vec)
 
 #define SOFTEN_TEST_PR(vec)						\
@@ -515,11 +532,63 @@ label##_relon_hv:							\
  * runlatch, etc...
  */
 
+.macro HARD_ENABLE_INTS tmp=r10
+#ifdef CONFIG_PPC_BOOK3E
+	wrteei	1
+#else
+	ld	\tmp,PACAKMSR(r13)
+	ori	\tmp,\tmp,MSR_EE
+	mtmsrd	\tmp,1
+#endif /* CONFIG_PPC_BOOK3E */
+.endm
+
+.macro HARD_DISABLE_INTS tmp=r10
+#ifdef CONFIG_PPC_BOOK3E
+	wrteei	0
+#else
+	ld	\tmp,PACAKMSR(r13) /* Get kernel MSR without EE */
+	mtmsrd	\tmp,1		  /* Update machine state */
+#endif /* CONFIG_PPC_BOOK3E */
+.endm
+
+.macro HARD_DISABLE_INTS_RI
+#ifdef CONFIG_PPC_BOOK3E
+	wrteei	0
+#else
+	/*
+	 * For performance reasons we clear RI the same time that we
+	 * clear EE. We only need to clear RI just before we restore r13
+	 * below, but batching it with EE saves us one expensive mtmsrd call.
+	 * We have to be careful to restore RI if we branch anywhere from
+	 * here (eg syscall_exit_work).
+	 *
+	 * CAUTION: using r9-r11 the way they are is assumed by the
+	 * caller.
+	 */
+	ld	r10,PACAKMSR(r13) /* Get kernel MSR without EE */
+	li	r9,MSR_RI
+	andc	r11,r10,r9
+	mtmsrd	r11,1		  /* Update machine state */
+#endif /* CONFIG_PPC_BOOK3E */
+.endm
+
 /*
  * This addition reconciles our actual IRQ state with the various software
  * flags that track it. This may call C code.
  */
+#ifdef CONFIG_IPIPE
+#define ADD_RECONCILE				\
+	ld	r10,PACAROOTPCPU(r13);		\
+	ld	r11,0(r10);			\
+	ori	r11,r11,1;			\
+	std	r11,0(r10);			\
+	mfmsr	r11;				\
+	ori	r11,r11,MSR_EE;			\
+	mtmsrd	r11,1;
+#define RECONCILE_IRQ_STATE(__rA, __rB)	HARD_DISABLE_INTS __rA
+#else /* !CONFIG_IPIPE */
 #define ADD_RECONCILE	RECONCILE_IRQ_STATE(r10,r11)
+#endif /* !CONFIG_IPIPE */
 
 #define ADD_NVGPRS				\
 	bl	save_nvgprs
@@ -552,9 +621,24 @@ label##_common:							\
  * in the idle task and therefore need the special idle handling
  * (finish nap and runlatch)
  */
+#ifdef CONFIG_IPIPE
+/*
+ * No NAP mode when pipelining, we don't want that extra latency.
+ * Runlatch will be considered later in __ipipe_exit_irq().
+ */
+#define STD_EXCEPTION_COMMON_ASYNC(trap, label, hdlr)		\
+	EXCEPTION_COMMON(trap, label, __ipipe_grab_irq,	\
+			 __ipipe_ret_from_except_lite, HARD_DISABLE_INTS)
+#define DECREMENTER_EXCEPTION(trap, label, hdlr)		  \
+	EXCEPTION_COMMON(trap, label, __ipipe_grab_timer,	  \
+			 __ipipe_ret_from_except_lite, HARD_DISABLE_INTS)
+#else
 #define STD_EXCEPTION_COMMON_ASYNC(trap, label, hdlr)		  \
 	EXCEPTION_COMMON(trap, label, hdlr, ret_from_except_lite, \
 			 FINISH_NAP;ADD_RECONCILE;RUNLATCH_ON)
+#define DECREMENTER_EXCEPTION(trap, label, hdlr)		  \
+	STD_EXCEPTION_COMMON_ASYNC(trap, label, hdlr)
+#endif
 
 /*
  * When the idle code in power4_idle puts the CPU into NAP mode,
diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h
index b59ac27..9cc1d25 100644
--- a/arch/powerpc/include/asm/hw_irq.h
+++ b/arch/powerpc/include/asm/hw_irq.h
@@ -41,6 +41,26 @@ extern void unknown_exception(struct pt_regs *regs);
 #ifdef CONFIG_PPC64
 #include <asm/paca.h>
 
+#ifdef CONFIG_PPC_BOOK3E
+#define __hard_irq_enable()	asm volatile("wrteei 1" : : : "memory")
+#define __hard_irq_disable()	asm volatile("wrteei 0" : : : "memory")
+#else
+#define __hard_irq_enable()	__mtmsrd(local_paca->kernel_msr | MSR_EE, 1)
+#define __hard_irq_disable()	__mtmsrd(local_paca->kernel_msr, 1)
+#endif
+
+#ifndef CONFIG_IPIPE
+
+#define hard_irq_disable()	do {			\
+	u8 _was_enabled;				\
+	__hard_irq_disable();				\
+	_was_enabled = local_paca->soft_enabled;	\
+	local_paca->soft_enabled = 0;			\
+	local_paca->irq_happened |= PACA_IRQ_HARD_DIS;	\
+	if (_was_enabled)				\
+		trace_hardirqs_off();			\
+} while(0)
+
 static inline unsigned long arch_local_save_flags(void)
 {
 	unsigned long flags;
@@ -88,24 +108,6 @@ static inline bool arch_irqs_disabled(void)
 	return arch_irqs_disabled_flags(arch_local_save_flags());
 }
 
-#ifdef CONFIG_PPC_BOOK3E
-#define __hard_irq_enable()	asm volatile("wrteei 1" : : : "memory")
-#define __hard_irq_disable()	asm volatile("wrteei 0" : : : "memory")
-#else
-#define __hard_irq_enable()	__mtmsrd(local_paca->kernel_msr | MSR_EE, 1)
-#define __hard_irq_disable()	__mtmsrd(local_paca->kernel_msr, 1)
-#endif
-
-#define hard_irq_disable()	do {			\
-	u8 _was_enabled;				\
-	__hard_irq_disable();				\
-	_was_enabled = local_paca->soft_enabled;	\
-	local_paca->soft_enabled = 0;			\
-	local_paca->irq_happened |= PACA_IRQ_HARD_DIS;	\
-	if (_was_enabled)				\
-		trace_hardirqs_off();			\
-} while(0)
-
 static inline bool lazy_irq_pending(void)
 {
 	return !!(get_paca()->irq_happened & ~PACA_IRQ_HARD_DIS);
@@ -123,6 +125,24 @@ static inline void may_hard_irq_enable(void)
 		__hard_irq_enable();
 }
 
+#else /* CONFIG_IPIPE */
+
+/*
+ * The built-in soft disabling mechanism is diverted to the pipeline
+ * when CONFIG_IPIPE is enabled, therefore we won't hard disable
+ * waiting for soft enabling.
+ */
+static inline bool lazy_irq_pending(void)
+{
+	return false;
+}
+
+static inline void may_hard_irq_enable(void) { }
+
+#define hard_irq_disable()	hard_local_irq_disable()
+
+#endif /* CONFIG_IPIPE */
+
 static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
 {
 	return !regs->softe;
@@ -134,6 +154,8 @@ extern bool prep_irq_for_idle(void);
 
 #define SET_MSR_EE(x)	mtmsr(x)
 
+#ifndef CONFIG_IPIPE
+
 static inline unsigned long arch_local_save_flags(void)
 {
 	return mfmsr();
@@ -188,7 +210,9 @@ static inline bool arch_irqs_disabled(void)
 	return arch_irqs_disabled_flags(arch_local_save_flags());
 }
 
-#define hard_irq_disable()		arch_local_irq_disable()
+#endif /* !CONFIG_IPIPE */
+
+#define hard_irq_disable()		hard_local_irq_disable()
 
 static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
 {
@@ -201,6 +225,8 @@ static inline void may_hard_irq_enable(void) { }
 
 #define ARCH_IRQ_INIT_FLAGS	IRQ_NOREQUEST
 
+#include <asm/ipipe_hwirq.h>
+
 /*
  * interrupt-retrigger: should we handle this via lost interrupts and IPIs
  * or should we not care like we do now ? --BenH.
diff --git a/arch/powerpc/include/asm/ipipe.h b/arch/powerpc/include/asm/ipipe.h
new file mode 100644
index 0000000..40aca5b
--- /dev/null
+++ b/arch/powerpc/include/asm/ipipe.h
@@ -0,0 +1,157 @@
+/*
+ *   include/asm-powerpc/ipipe.h
+ *
+ *   I-pipe 32/64bit merge - Copyright (C) 2007 Philippe Gerum.
+ *   I-pipe PA6T support - Copyright (C) 2007 Philippe Gerum.
+ *   I-pipe 64-bit PowerPC port - Copyright (C) 2005 Heikki Lindholm.
+ *   I-pipe PowerPC support - Copyright (C) 2002-2005 Philippe Gerum.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ *   USA; either version 2 of the License, or (at your option) any later
+ *   version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the Free Software
+ *   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __ASM_POWERPC_IPIPE_H
+#define __ASM_POWERPC_IPIPE_H
+
+#ifdef CONFIG_IPIPE
+
+#include <asm/ptrace.h>
+#include <asm/hw_irq.h>
+#include <asm/irq.h>
+#include <asm/bitops.h>
+#include <asm/time.h>
+#include <linux/ipipe_domain.h>
+#include <linux/irq.h>
+#include <linux/list.h>
+#include <linux/cpumask.h>
+#include <linux/cache.h>
+#include <linux/threads.h>
+
+#define IPIPE_CORE_RELEASE	8
+
+struct ipipe_domain;
+
+struct ipipe_arch_sysinfo {
+};
+
+#ifdef CONFIG_DEBUGGER
+extern cpumask_t __ipipe_dbrk_pending;
+#endif
+
+extern unsigned long __ipipe_hrtimer_freq;
+
+#define __ipipe_hrclock_freq	ppc_tb_freq
+#define __ipipe_cpu_freq	ppc_proc_freq
+
+#ifdef CONFIG_PPC64
+#define ipipe_read_tsc(t)	(t = mftb())
+#define ipipe_tsc2ns(t)		(((t) * 1000UL) / (ppc_tb_freq / 1000000UL))
+#define ipipe_tsc2us(t)		((t) / (ppc_tb_freq / 1000000UL))
+#else /* CONFIG_PPC32 */
+#define ipipe_read_tsc(t)					\
+	({							\
+		unsigned long __tbu;				\
+		__asm__ __volatile__ ("1: mftbu %0\n"		\
+				      "mftb %1\n"		\
+				      "mftbu %2\n"		\
+				      "cmpw %2,%0\n"		\
+				      "bne- 1b\n"		\
+				      :"=r" (((unsigned long *)&t)[0]),	\
+				       "=r" (((unsigned long *)&t)[1]),	\
+				       "=r" (__tbu));			\
+		t;							\
+	})
+
+#define ipipe_tsc2ns(t)	\
+	((((unsigned long)(t)) * 1000) / (ppc_tb_freq / 1000000))
+
+#define ipipe_tsc2us(t)						\
+	({							\
+		unsigned long long delta = (t);			\
+		do_div(delta, ppc_tb_freq/1000000+1);		\
+		(unsigned long)delta;				\
+	})
+#endif /* CONFIG_PPC32 */
+
+static inline const char *ipipe_clock_name(void)
+{
+	return "timebase";
+}
+
+/* Private interface -- Internal use only */
+
+#define __ipipe_enable_irq(irq)			enable_irq(irq)
+#define __ipipe_disable_irq(irq)		disable_irq(irq)
+#define __ipipe_enable_irqdesc(ipd, irq)	do { } while(0)
+#define __ipipe_disable_irqdesc(ipd, irq)	do { } while(0)
+
+void __ipipe_early_core_setup(void);
+
+void __ipipe_enable_pipeline(void);
+
+#ifdef CONFIG_SMP
+struct ipipe_ipi_struct {
+	volatile unsigned long value;
+} ____cacheline_aligned;
+
+void __ipipe_hook_critical_ipi(struct ipipe_domain *ipd);
+
+void __ipipe_register_ipi(unsigned int irq);
+#else
+#define __ipipe_hook_critical_ipi(ipd)	do { } while(0)
+#endif /* CONFIG_SMP */
+
+void __ipipe_dispatch_irq(unsigned int irq, int flags);
+
+static inline void __ipipe_handle_irq(unsigned int irq, struct pt_regs *regs)
+{
+	/* NULL regs means software-triggered, no ack needed. */
+	__ipipe_dispatch_irq(irq, regs ? 0 : IPIPE_IRQF_NOACK);
+}
+
+struct irq_desc;
+void __ipipe_ack_level_irq(unsigned irq, struct irq_desc *desc);
+void __ipipe_end_level_irq(unsigned irq, struct irq_desc *desc);
+void __ipipe_ack_edge_irq(unsigned irq, struct irq_desc *desc);
+void __ipipe_end_edge_irq(unsigned irq, struct irq_desc *desc);
+
+#ifdef CONFIG_IPIPE_DEBUG
+void __ipipe_serial_debug(const char *fmt, ...);
+#else
+#define __ipipe_serial_debug(fmt, args...)	do { } while (0)
+#endif
+
+static inline unsigned long __ipipe_ffnz(unsigned long ul)
+{
+#ifdef CONFIG_PPC64
+	__asm__ __volatile__("cntlzd %0, %1":"=r"(ul):"r"(ul & (-ul)));
+	return 63 - ul;
+#else
+	__asm__ __volatile__("cntlzw %0, %1":"=r"(ul):"r"(ul & (-ul)));
+	return 31 - ul;
+#endif
+}
+
+#define __ipipe_root_tick_p(regs)	((regs)->msr & MSR_EE)
+
+static inline void ipipe_mute_pic(void) { }
+
+static inline void ipipe_unmute_pic(void) { }
+
+static inline void ipipe_notify_root_preemption(void) { }
+
+#endif /* !CONFIG_IPIPE */
+
+#endif /* !__ASM_POWERPC_IPIPE_H */
diff --git a/arch/powerpc/include/asm/ipipe_base.h b/arch/powerpc/include/asm/ipipe_base.h
new file mode 100644
index 0000000..167a70d
--- /dev/null
+++ b/arch/powerpc/include/asm/ipipe_base.h
@@ -0,0 +1,135 @@
+/* -*- linux-c -*-
+ * include/asm-powerpc/ipipe_base.h
+ *
+ * Copyright (C) 2007-2012 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __ASM_POWERPC_IPIPE_BASE_H
+#define __ASM_POWERPC_IPIPE_BASE_H
+
+#ifdef CONFIG_IPIPE
+
+#define IPIPE_NR_XIRQS		CONFIG_NR_IRQS
+#ifdef CONFIG_PPC64
+#define IPIPE_IRQ_ISHIFT	6		/* 64-bit arch. */
+#else
+#define IPIPE_IRQ_ISHIFT	5		/* 32-bit arch. */
+#endif
+
+/* 
+ * The first virtual interrupt is reserved for the timer (see
+ * __ipipe_early_core_setup).
+ */
+#define IPIPE_TIMER_VIRQ	IPIPE_VIRQ_BASE
+
+#ifdef CONFIG_SMP
+/* 
+ * These are virtual IPI numbers. The OpenPIC supports only 4 IPIs and
+ * all are already used by Linux. The virtualization layer is
+ * implemented by piggybacking the debugger break IPI 0x3,
+ * which is demultiplexed in __ipipe_ipi_demux().
+ */
+/* these are bit numbers in practice */
+#define IPIPE_MSG_CRITICAL_IPI		0
+#define IPIPE_MSG_HRTIMER_IPI		(IPIPE_MSG_CRITICAL_IPI + 1)
+#define IPIPE_MSG_RESCHEDULE_IPI	(IPIPE_MSG_CRITICAL_IPI + 2)
+
+#define IPIPE_MSG_IPI_MASK	((1UL << IPIPE_MSG_CRITICAL_IPI) |	\
+				 (1UL << IPIPE_MSG_HRTIMER_IPI) |	\
+				 (1UL << IPIPE_MSG_RESCHEDULE_IPI))
+
+#define IPIPE_CRITICAL_IPI	(IPIPE_VIRQ_BASE + 1)
+#define IPIPE_HRTIMER_IPI	(IPIPE_CRITICAL_IPI + 1)
+#define IPIPE_RESCHEDULE_IPI	(IPIPE_CRITICAL_IPI + 2)
+
+#define IPIPE_BASE_IPI_OFFSET	IPIPE_CRITICAL_IPI
+
+#define ipipe_processor_id()	raw_smp_processor_id()
+
+#else  /* !CONFIG_SMP */
+#define ipipe_processor_id()	0
+#endif /* CONFIG_SMP */
+
+/* traps */
+#define IPIPE_TRAP_ACCESS	 0	/* Data or instruction access exception */
+#define IPIPE_TRAP_ALIGNMENT	 1	/* Alignment exception */
+#define IPIPE_TRAP_ALTUNAVAIL	 2	/* Altivec unavailable */
+#define IPIPE_TRAP_PCE		 3	/* Program check exception */
+#define IPIPE_TRAP_MCE		 4	/* Machine check exception */
+#define IPIPE_TRAP_UNKNOWN	 5	/* Unknown exception */
+#define IPIPE_TRAP_IABR		 6	/* Instruction breakpoint */
+#define IPIPE_TRAP_RM		 7	/* Run mode exception */
+#define IPIPE_TRAP_SSTEP	 8	/* Single-step exception */
+#define IPIPE_TRAP_NREC		 9	/* Non-recoverable exception */
+#define IPIPE_TRAP_SOFTEMU	10	/* Software emulation */
+#define IPIPE_TRAP_DEBUG	11	/* Debug exception */
+#define IPIPE_TRAP_SPE		12	/* SPE exception */
+#define IPIPE_TRAP_ALTASSIST	13	/* Altivec assist exception */
+#define IPIPE_TRAP_CACHE	14	/* Cache-locking exception (FSL) */
+#define IPIPE_TRAP_KFPUNAVAIL	15	/* FP unavailable exception */
+#define IPIPE_TRAP_MAYDAY	16	/* Internal recovery trap */
+#define IPIPE_NR_FAULTS		17
+
+#ifndef __ASSEMBLY__
+
+#ifdef CONFIG_SMP
+
+void ipipe_stall_root(void);
+
+unsigned long ipipe_test_and_stall_root(void);
+
+unsigned long ipipe_test_root(void);
+
+#else /* !CONFIG_SMP */
+
+#include <linux/bitops.h>
+
+extern unsigned long __ipipe_root_status;
+
+static __inline__ void ipipe_stall_root(void)
+{
+	volatile unsigned long *p = &__ipipe_root_status;
+	ipipe_root_only();
+	set_bit(0, p);
+}
+
+static __inline__ unsigned long ipipe_test_and_stall_root(void)
+{
+	volatile unsigned long *p = &__ipipe_root_status;
+	ipipe_root_only();
+	return test_and_set_bit(0, p);
+}
+
+static __inline__ unsigned long ipipe_test_root(void)
+{
+	volatile unsigned long *p = &__ipipe_root_status;
+	return test_bit(0, p);
+}
+
+#endif /* !CONFIG_SMP */
+
+#endif /* !__ASSEMBLY__ */
+
+#ifdef CONFIG_IPIPE_LEGACY
+#define __IPIPE_FEATURE_PREEMPTIBLE_SWITCH	1
+#define __IPIPE_FEATURE_HARDENED_SWITCHMM	1
+#endif
+
+#endif /* !CONFIG_IPIPE */
+
+#endif	/* !__ASM_POWERPC_IPIPE_BASE_H */
diff --git a/arch/powerpc/include/asm/ipipe_hwirq.h b/arch/powerpc/include/asm/ipipe_hwirq.h
new file mode 100644
index 0000000..2bc8217
--- /dev/null
+++ b/arch/powerpc/include/asm/ipipe_hwirq.h
@@ -0,0 +1,252 @@
+/* -*- linux-c -*-
+ * include/asm-powerpc/ipipe_hwirq.h
+ *
+ * Copyright (C) 2009 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef _ASM_POWERPC_IPIPE_HWIRQ_H
+
+#ifdef CONFIG_PPC32
+
+#if defined(CONFIG_BOOKE)
+#define hard_local_irq_restore_notrace(x)	__asm__ __volatile__("wrtee %0" : : "r" (x) : "memory")
+#else
+#define hard_local_irq_restore_notrace(x)	mtmsr(x)
+#endif
+
+static inline void hard_local_irq_disable_notrace(void)
+{
+#ifdef CONFIG_BOOKE
+	__asm__ __volatile__("wrteei 0": : :"memory");
+#else
+	unsigned long msr = mfmsr();
+	mtmsr(msr & ~MSR_EE);
+#endif
+}
+
+static inline void hard_local_irq_enable_notrace(void)
+{
+#ifdef CONFIG_BOOKE
+	__asm__ __volatile__("wrteei 1": : :"memory");
+#else
+	unsigned long msr = mfmsr();
+	mtmsr(msr | MSR_EE);
+#endif
+}
+
+static inline unsigned long hard_local_irq_save_notrace(void)
+{
+	unsigned long msr = mfmsr();
+#ifdef CONFIG_BOOKE
+	__asm__ __volatile__("wrteei 0": : :"memory");
+#else
+	mtmsr(msr & ~MSR_EE);
+#endif
+	return msr;
+}
+
+#else /* CONFIG_PPC64 */
+
+#include <asm/paca.h>
+
+#ifdef CONFIG_PPC_BOOK3E
+static inline void hard_local_irq_disable_notrace(void)
+{
+	__asm__ __volatile__("wrteei 0": : :"memory");
+}
+
+static inline void hard_local_irq_enable_notrace(void)
+{
+	__asm__ __volatile__("wrteei 1": : :"memory");
+}
+#else /* !CONFIG_PPC_BOOK3E */
+static inline void hard_local_irq_disable_notrace(void)
+{
+	__mtmsrd(mfmsr() & ~MSR_EE, 1);
+}
+
+static inline void hard_local_irq_enable_notrace(void)
+{
+	__mtmsrd(mfmsr() | MSR_EE, 1);
+}
+#endif /* !CONFIG_PPC_BOOK3E */
+
+static inline unsigned long hard_local_irq_save_notrace(void)
+{
+	unsigned long msr = mfmsr();
+	hard_local_irq_disable_notrace();
+	return msr;
+}
+
+#define hard_local_irq_restore_notrace(x)	__mtmsrd(x, 1)
+
+#endif /* CONFIG_PPC64 */
+
+#ifdef CONFIG_IPIPE
+
+#include <linux/ipipe_base.h>
+#include <linux/ipipe_trace.h>
+
+static inline int arch_irqs_disabled_flags(unsigned long flags)
+{
+	return (flags & MSR_EE) == 0;
+}
+
+static inline unsigned long arch_local_irq_disable(void)
+{
+	unsigned long flags;
+
+	flags = (!ipipe_test_and_stall_root()) << MSR_EE_LG;
+	barrier();
+
+	return flags;
+}
+
+static inline void arch_local_irq_enable(void)
+{
+	barrier();
+	ipipe_unstall_root();
+}
+
+static inline void arch_local_irq_restore(unsigned long flags)
+{
+	barrier();
+	if (!arch_irqs_disabled_flags(flags))
+		ipipe_unstall_root();
+}
+
+static inline unsigned long arch_local_irq_save(void)
+{
+	return arch_local_irq_disable();
+}
+
+static inline unsigned long arch_local_save_flags(void)
+{
+	return (!ipipe_test_root()) << MSR_EE_LG;
+}
+
+static inline int arch_irqs_disabled(void)
+{
+	unsigned long flags = arch_local_save_flags();
+
+	return arch_irqs_disabled_flags(flags);
+}
+
+static inline unsigned long arch_mangle_irq_bits(int stalled, unsigned long msr)
+{
+	/* Merge virtual and real interrupt mask bits. */
+	return (msr & ~MSR_VIRTEE) | ((long)(stalled == 0) << MSR_VIRTEE_LG);
+}
+
+static inline int arch_demangle_irq_bits(unsigned long *flags)
+{
+	int stalled = (*flags & MSR_VIRTEE) == 0;
+
+	*flags &= ~MSR_VIRTEE;
+
+	return stalled;
+}
+
+static inline unsigned long hard_local_save_flags(void)
+{
+	return mfmsr();
+}
+
+static inline int hard_irqs_disabled_flags(unsigned long flags)
+{
+	return (flags & MSR_EE) == 0;
+}
+
+static inline int hard_irqs_disabled(void)
+{
+	unsigned long flags = hard_local_save_flags();
+
+	return hard_irqs_disabled_flags(flags);
+}
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+
+static inline void hard_local_irq_disable(void)
+{
+	if (!hard_irqs_disabled()) {
+		hard_local_irq_disable_notrace();
+		ipipe_trace_begin(0x80000000);
+	}
+}
+
+static inline void hard_local_irq_enable(void)
+{
+	if (hard_irqs_disabled()) {
+		ipipe_trace_end(0x80000000);
+		hard_local_irq_enable_notrace();
+	}
+}
+
+static inline unsigned long hard_local_irq_save(void)
+{
+	unsigned long flags;
+
+	flags = hard_local_irq_save_notrace();
+	if (flags & MSR_EE)
+		ipipe_trace_begin(0x80000001);
+
+	return flags;
+}
+
+static inline void hard_local_irq_restore(unsigned long flags)
+{
+	if (flags & MSR_EE)
+		ipipe_trace_end(0x80000001);
+
+	hard_local_irq_restore_notrace(flags);
+}
+
+#else /* !CONFIG_IPIPE_TRACE_IRQSOFF */
+
+#define hard_local_irq_disable    hard_local_irq_disable_notrace
+#define hard_local_irq_enable     hard_local_irq_enable_notrace
+#define hard_local_irq_save       hard_local_irq_save_notrace
+#define hard_local_irq_restore    hard_local_irq_restore_notrace
+
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+
+#else /* !CONFIG_IPIPE */
+
+#define hard_local_irq_save()		arch_local_irq_save()
+#define hard_local_irq_restore(x)	arch_local_irq_restore(x)
+#define hard_local_irq_enable()		arch_local_irq_enable()
+#define hard_local_irq_disable()	arch_local_irq_disable()
+#define hard_irqs_disabled()		arch_irqs_disabled()
+#define hard_irqs_disabled_flags(flags)	arch_irqs_disabled_flags(flags)
+
+#define hard_cond_local_irq_enable()		do { } while(0)
+#define hard_cond_local_irq_disable()		do { } while(0)
+#define hard_cond_local_irq_save()		0
+#define hard_cond_local_irq_restore(flags)	do { (void)(flags); } while(0)
+
+#endif /* !CONFIG_IPIPE */
+
+#if defined(CONFIG_SMP) && defined(CONFIG_IPIPE)
+#define hard_smp_local_irq_save()		hard_local_irq_save()
+#define hard_smp_local_irq_restore(flags)	hard_local_irq_restore(flags)
+#else /* !CONFIG_SMP */
+#define hard_smp_local_irq_save()		0
+#define hard_smp_local_irq_restore(flags)	do { (void)(flags); } while(0)
+#endif /* CONFIG_SMP */
+
+#endif /* !_ASM_POWERPC_IPIPE_HWIRQ_H */
diff --git a/arch/powerpc/include/asm/irq.h b/arch/powerpc/include/asm/irq.h
index e8e3a0a..90fe40a 100644
--- a/arch/powerpc/include/asm/irq.h
+++ b/arch/powerpc/include/asm/irq.h
@@ -67,8 +67,17 @@ extern void call_do_softirq(struct thread_info *tp);
 extern void call_do_irq(struct pt_regs *regs, struct thread_info *tp);
 extern void do_IRQ(struct pt_regs *regs);
 extern void __do_irq(struct pt_regs *regs);
+extern void ___do_irq(unsigned int irq, struct pt_regs *regs);
 
 int irq_choose_cpu(const struct cpumask *mask);
 
+#if defined(CONFIG_DEBUG_STACKOVERFLOW) && !defined(CONFIG_IPIPE_LEGACY)
+void check_stack_overflow(void);
+#else
+static inline void check_stack_overflow(void)
+{
+}
+#endif
+
 #endif /* _ASM_IRQ_H */
 #endif /* __KERNEL__ */
diff --git a/arch/powerpc/include/asm/irqflags.h b/arch/powerpc/include/asm/irqflags.h
index f214906..093829c 100644
--- a/arch/powerpc/include/asm/irqflags.h
+++ b/arch/powerpc/include/asm/irqflags.h
@@ -45,6 +45,7 @@
  * NB: This may call C code, so the caller must be prepared for volatiles to
  * be clobbered.
  */
+#ifndef CONFIG_IPIPE
 #define RECONCILE_IRQ_STATE(__rA, __rB)		\
 	lbz	__rA,PACASOFTIRQEN(r13);	\
 	lbz	__rB,PACAIRQHAPPENED(r13);	\
@@ -56,17 +57,21 @@
 	stb	__rA,PACASOFTIRQEN(r13);	\
 	TRACE_DISABLE_INTS;			\
 44:
+#endif /* !CONFIG_IPIPE */
 
 #else
 #define TRACE_ENABLE_INTS
 #define TRACE_DISABLE_INTS
 
+#ifndef CONFIG_IPIPE
 #define RECONCILE_IRQ_STATE(__rA, __rB)		\
 	lbz	__rA,PACAIRQHAPPENED(r13);	\
 	li	__rB,0;				\
 	ori	__rA,__rA,PACA_IRQ_HARD_DIS;	\
 	stb	__rB,PACASOFTIRQEN(r13);	\
 	stb	__rA,PACAIRQHAPPENED(r13)
+#endif /* !CONFIG_IPIPE */
+
 #endif
 #endif
 
diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h
index a6dcdb6..a383e04 100644
--- a/arch/powerpc/include/asm/kvm_ppc.h
+++ b/arch/powerpc/include/asm/kvm_ppc.h
@@ -647,7 +647,7 @@ static inline void kvmppc_fix_ee_before_entry(void)
 {
 	trace_hardirqs_on();
 
-#ifdef CONFIG_PPC64
+#if defined(CONFIG_PPC64) && !defined(CONFIG_IPIPE)
 	/*
 	 * To avoid races, the caller must have gone directly from having
 	 * interrupts fully-enabled to hard-disabled.
diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h
index 73382eb..0f42e5a 100644
--- a/arch/powerpc/include/asm/mmu_context.h
+++ b/arch/powerpc/include/asm/mmu_context.h
@@ -35,15 +35,39 @@ extern void switch_cop(struct mm_struct *next);
 extern int use_cop(unsigned long acop, struct mm_struct *mm);
 extern void drop_cop(unsigned long acop, struct mm_struct *mm);
 
+#if !defined(CONFIG_IPIPE) || defined(CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH)
+
+#define ipipe_mm_switch_protect(flags)		\
+  do { (void)(flags); } while (0)
+
+#define ipipe_mm_switch_unprotect(flags)	\
+  do { (void)(flags); } while (0)
+
+#else /* CONFIG_IPIPE && !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */
+
+#define ipipe_mm_switch_protect(flags)		\
+  do {						\
+    	(flags) = hard_cond_local_irq_save();	\
+	barrier();				\
+  } while (0)					\
+
+#define ipipe_mm_switch_unprotect(flags)	\
+  do {						\
+	barrier();				\
+    	hard_cond_local_irq_restore(flags);	\
+  } while (0)					\
+
+#endif /* CONFIG_IPIPE && !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */
+
 /*
  * switch_mm is the entry point called from the architecture independent
  * code in kernel/sched/core.c
  */
-static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
-			     struct task_struct *tsk)
+static inline void __do_switch_mm(struct mm_struct *prev, struct mm_struct *next,
+				  struct task_struct *tsk, bool irq_sync_p)
 {
 	/* Mark this context has been used on the new CPU */
-	cpumask_set_cpu(smp_processor_id(), mm_cpumask(next));
+	cpumask_set_cpu(ipipe_processor_id(), mm_cpumask(next));
 
 	/* 32-bit keeps track of the current PGDIR in the thread struct */
 #ifdef CONFIG_PPC32
@@ -58,6 +82,9 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
 	if (prev == next)
 		return;
 
+	if (irq_sync_p)
+		hard_local_irq_enable();
+	
 #ifdef CONFIG_PPC_ICSWX
 	/* Switch coprocessor context only if prev or next uses a coprocessor */
 	if (prev->context.acop || next->context.acop)
@@ -81,7 +108,71 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
 	/* Out of line for now */
 	switch_mmu_context(prev, next);
 #endif
+	if (irq_sync_p)
+		hard_local_irq_disable();
+}
+
+static inline void __switch_mm(struct mm_struct *prev, struct mm_struct *next,
+			       struct task_struct *tsk)
+{
+#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH
+#if defined(CONFIG_PPC_MMU_NOHASH) && defined(CONFIG_SMP)
+ /*
+  * mmu_context_nohash in SMP mode is tracking an activity counter
+  * into the mm struct. Therefore, we make sure the kernel always sees
+  * the ipipe_percpu.active_mm update and the actual switch as a
+  * single atomic operation. Since the related code already requires
+  * to hard disable irqs all through the switch, there is no
+  * additional penalty anyway.
+  */
+#define mmswitch_irq_sync false
+#else
+#define mmswitch_irq_sync true
+#endif
+	IPIPE_WARN_ONCE(hard_irqs_disabled());
+	for (;;) {
+		hard_local_irq_disable();
+		__this_cpu_write(ipipe_percpu.active_mm, NULL);
+		barrier();
+		__do_switch_mm(prev, next, tsk, mmswitch_irq_sync);
+		if (!test_and_clear_thread_flag(TIF_MMSWITCH_INT)) {
+			__this_cpu_write(ipipe_percpu.active_mm, next);
+			hard_local_irq_enable();
+			return;
+		}
+	}
+#else /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */
+	IPIPE_WARN_ONCE(!hard_irqs_disabled());
+	__do_switch_mm(prev, next, tsk, false);
+#endif /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */
+}
 
+#ifdef CONFIG_IPIPE
+/*
+ * ipipe_switch_mm_head() is reserved to the head domain for switching
+ * mmu context.
+ */
+static inline
+void ipipe_switch_mm_head(struct mm_struct *prev, struct mm_struct *next,
+			  struct task_struct *tsk)
+{
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	__do_switch_mm(prev, next, tsk, false);
+	hard_local_irq_restore(flags);
+}
+
+#endif /* CONFIG_IPIPE */
+
+static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
+			     struct task_struct *tsk)
+{
+	unsigned long flags;
+
+	ipipe_mm_switch_protect(flags);
+	__switch_mm(prev, next, tsk);
+	ipipe_mm_switch_unprotect(flags);
 }
 
 #define deactivate_mm(tsk,mm)	do { } while (0)
@@ -92,11 +183,15 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
  */
 static inline void activate_mm(struct mm_struct *prev, struct mm_struct *next)
 {
+#ifndef CONFIG_IPIPE
 	unsigned long flags;
 
 	local_irq_save(flags);
+#endif
 	switch_mm(prev, next, current);
+#ifndef CONFIG_IPIPE
 	local_irq_restore(flags);
+#endif
 }
 
 /* We don't currently use enter_lazy_tlb() for anything */
diff --git a/arch/powerpc/include/asm/mpic.h b/arch/powerpc/include/asm/mpic.h
index 754f93d..b570ddf 100644
--- a/arch/powerpc/include/asm/mpic.h
+++ b/arch/powerpc/include/asm/mpic.h
@@ -296,7 +296,7 @@ struct mpic
 #ifdef CONFIG_MPIC_U3_HT_IRQS
 	/* The fixup table */
 	struct mpic_irq_fixup	*fixups;
-	raw_spinlock_t	fixup_lock;
+	ipipe_spinlock_t	fixup_lock;
 #endif
 
 	/* Register access method */
diff --git a/arch/powerpc/include/asm/nmi.h b/arch/powerpc/include/asm/nmi.h
new file mode 100644
index 0000000..0d4a0a4
--- /dev/null
+++ b/arch/powerpc/include/asm/nmi.h
@@ -0,0 +1,9 @@
+/*
+ *  linux/include/asm-powerpc/nmi.h
+ */
+#ifndef ASM_NMI_H
+#define ASM_NMI_H
+
+extern void touch_nmi_watchdog(void);
+
+#endif /* ASM_NMI_H */
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index a5139ea..1c93874 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -43,6 +43,7 @@ extern unsigned int debug_smp_processor_id(void); /* from linux/smp.h */
 
 struct task_struct;
 struct opal_machine_check_event;
+struct ipipe_percpu_domain_data;
 
 /*
  * Defines the layout of the paca.
@@ -143,8 +144,12 @@ struct paca_struct {
 	u64 saved_r1;			/* r1 save for RTAS calls or PM */
 	u64 saved_msr;			/* MSR saved here by enter_rtas */
 	u16 trap_save;			/* Used when bad stack is encountered */
+#ifdef CONFIG_IPIPE
+	u64 ipipe_statp;		/* Pointer to I-pipe status word */
+#else
 	u8 soft_enabled;		/* irq soft-enable flag */
 	u8 irq_happened;		/* irq happened while soft-disabled */
+#endif
 	u8 io_sync;			/* writel() needs spin_unlock sync */
 	u8 irq_work_pending;		/* IRQ_WORK interrupt while soft-disable */
 	u8 nap_state_lost;		/* NV GPR values lost in power7_idle */
@@ -170,6 +175,10 @@ struct paca_struct {
 	u8 hmi_event_available;		 /* HMI event is available */
 #endif
 
+#ifdef CONFIG_IPIPE
+	struct ipipe_percpu_domain_data *root_context;	/* Address of root context data */
+#endif /* CONFIG_IPIPE */
+
 	/* Stuff for accurate time accounting */
 	u64 user_time;			/* accumulated usermode TB ticks */
 	u64 system_time;		/* accumulated system TB ticks */
diff --git a/arch/powerpc/include/asm/qe_ic.h b/arch/powerpc/include/asm/qe_ic.h
index 25784cc..f29c8cf 100644
--- a/arch/powerpc/include/asm/qe_ic.h
+++ b/arch/powerpc/include/asm/qe_ic.h
@@ -85,7 +85,7 @@ static inline void qe_ic_cascade_low_ipic(unsigned int irq,
 	unsigned int cascade_irq = qe_ic_get_low_irq(qe_ic);
 
 	if (cascade_irq != NO_IRQ)
-		generic_handle_irq(cascade_irq);
+		ipipe_handle_demuxed_irq(cascade_irq);
 }
 
 static inline void qe_ic_cascade_high_ipic(unsigned int irq,
@@ -95,7 +95,7 @@ static inline void qe_ic_cascade_high_ipic(unsigned int irq,
 	unsigned int cascade_irq = qe_ic_get_high_irq(qe_ic);
 
 	if (cascade_irq != NO_IRQ)
-		generic_handle_irq(cascade_irq);
+		ipipe_handle_demuxed_irq(cascade_irq);
 }
 
 static inline void qe_ic_cascade_low_mpic(unsigned int irq,
@@ -106,7 +106,7 @@ static inline void qe_ic_cascade_low_mpic(unsigned int irq,
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 
 	if (cascade_irq != NO_IRQ)
-		generic_handle_irq(cascade_irq);
+		ipipe_handle_demuxed_irq(cascade_irq);
 
 	chip->irq_eoi(&desc->irq_data);
 }
@@ -119,7 +119,7 @@ static inline void qe_ic_cascade_high_mpic(unsigned int irq,
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 
 	if (cascade_irq != NO_IRQ)
-		generic_handle_irq(cascade_irq);
+		ipipe_handle_demuxed_irq(cascade_irq);
 
 	chip->irq_eoi(&desc->irq_data);
 }
@@ -136,7 +136,7 @@ static inline void qe_ic_cascade_muxed_mpic(unsigned int irq,
 		cascade_irq = qe_ic_get_low_irq(qe_ic);
 
 	if (cascade_irq != NO_IRQ)
-		generic_handle_irq(cascade_irq);
+		ipipe_handle_demuxed_irq(cascade_irq);
 
 	chip->irq_eoi(&desc->irq_data);
 }
diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h
index a68ee15..85e3e82 100644
--- a/arch/powerpc/include/asm/reg.h
+++ b/arch/powerpc/include/asm/reg.h
@@ -34,6 +34,7 @@
 #define MSR_TS_LG	33		/* Trans Mem state (2 bits) */
 #define MSR_TM_LG	32		/* Trans Mem Available */
 #define MSR_VEC_LG	25	        /* Enable AltiVec */
+#define MSR_VIRTEE_LG	29		/* I-pipe stall bit */
 #define MSR_VSX_LG	23		/* Enable VSX */
 #define MSR_POW_LG	18		/* Enable Power Management */
 #define MSR_WE_LG	18		/* Wait State Enable */
@@ -110,6 +111,10 @@
 #define MSR_TM_ACTIVE(x) (((x) & MSR_TS_MASK) != 0) /* Transaction active? */
 #define MSR_TM_TRANSACTIONAL(x)	(((x) & MSR_TS_MASK) == MSR_TS_T)
 #define MSR_TM_SUSPENDED(x)	(((x) & MSR_TS_MASK) == MSR_TS_S)
+/*
+ * CONFIG_IPIPE only. We divert the unused bit #29 from the MSR.
+ */
+#define MSR_VIRTEE	__MASK(MSR_VIRTEE_LG)
 
 #if defined(CONFIG_PPC_BOOK3S_64)
 #define MSR_64BIT	MSR_SF
diff --git a/arch/powerpc/include/asm/smp.h b/arch/powerpc/include/asm/smp.h
index 5a6614a..5956e5b 100644
--- a/arch/powerpc/include/asm/smp.h
+++ b/arch/powerpc/include/asm/smp.h
@@ -77,8 +77,14 @@ int generic_check_cpu_restart(unsigned int cpu);
 /* 32-bit */
 extern int smp_hw_index[];
 
+#ifdef CONFIG_IPIPE_LEGACY
+extern int smp_logical_index[];
+#define raw_smp_processor_id()		(smp_logical_index[mfspr(SPRN_PIR)])
+#define hard_smp_processor_id() 	(smp_hw_index[raw_smp_processor_id()])
+#else
 #define raw_smp_processor_id()	(current_thread_info()->cpu)
 #define hard_smp_processor_id() 	(smp_hw_index[smp_processor_id()])
+#endif
 
 static inline int get_hard_smp_processor_id(int cpu)
 {
@@ -88,6 +94,10 @@ static inline int get_hard_smp_processor_id(int cpu)
 static inline void set_hard_smp_processor_id(int cpu, int phys)
 {
 	smp_hw_index[cpu] = phys;
+#ifdef CONFIG_IPIPE_LEGACY
+	BUG_ON(phys >= NR_CPUS);
+	smp_logical_index[phys] = cpu;
+#endif
 }
 #endif
 
@@ -114,6 +124,7 @@ extern int cpu_to_core_id(int cpu);
 #define PPC_MSG_RESCHEDULE      1
 #define PPC_MSG_TICK_BROADCAST	2
 #define PPC_MSG_DEBUGGER_BREAK  3
+#define PPC_MSG_IPIPE_DEMUX     PPC_MSG_DEBUGGER_BREAK
 
 /* for irq controllers that have dedicated ipis per message (4) */
 extern int smp_request_message_ipi(int virq, int message);
diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h
index b034ecd..b41266f 100644
--- a/arch/powerpc/include/asm/thread_info.h
+++ b/arch/powerpc/include/asm/thread_info.h
@@ -33,6 +33,7 @@
 #include <asm/processor.h>
 #include <asm/page.h>
 #include <linux/stringify.h>
+#include <ipipe/thread_info.h>
 
 /*
  * low level task data.
@@ -48,6 +49,10 @@ struct thread_info {
 
 	/* low level flags - has atomic operations done on it */
 	unsigned long	flags ____cacheline_aligned_in_smp;
+#ifdef CONFIG_IPIPE
+	unsigned long ipipe_flags;
+#endif
+	struct ipipe_threadinfo ipipe_data;
 };
 
 /*
@@ -107,6 +112,7 @@ static inline struct thread_info *current_thread_info(void)
 #if defined(CONFIG_PPC64)
 #define TIF_ELF2ABI		18	/* function descriptors must die! */
 #endif
+#define TIF_MMSWITCH_INT	20	/* MMU context switch interrupted */
 
 /* as above, but as bit values */
 #define _TIF_SYSCALL_TRACE	(1<<TIF_SYSCALL_TRACE)
@@ -125,6 +131,7 @@ static inline struct thread_info *current_thread_info(void)
 #define _TIF_SYSCALL_TRACEPOINT	(1<<TIF_SYSCALL_TRACEPOINT)
 #define _TIF_EMULATE_STACK_STORE	(1<<TIF_EMULATE_STACK_STORE)
 #define _TIF_NOHZ		(1<<TIF_NOHZ)
+#define _TIF_MMSWITCH_INT	(1<<TIF_MMSWITCH_INT)
 #define _TIF_SYSCALL_T_OR_A	(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
 				 _TIF_SECCOMP | _TIF_SYSCALL_TRACEPOINT | \
 				 _TIF_NOHZ)
@@ -134,6 +141,15 @@ static inline struct thread_info *current_thread_info(void)
 				 _TIF_RESTORE_TM)
 #define _TIF_PERSYSCALL_MASK	(_TIF_RESTOREALL|_TIF_NOERROR)
 
+/* ti->ipipe_flags */
+#define TIP_MAYDAY	0	/* MAYDAY call is pending */
+#define TIP_NOTIFY	1	/* Notify head domain about kernel events */
+#define TIP_HEAD	2	/* Runs in head domain */
+
+#define _TIP_MAYDAY	(1<<TIP_MAYDAY)
+#define _TIP_NOTIFY	(1<<TIP_NOTIFY)
+#define _TIP_HEAD	(1<<TIP_HEAD)
+
 /* Bits in local_flags */
 /* Don't move TLF_NAPPING without adjusting the code in entry_32.S */
 #define TLF_NAPPING		0	/* idle thread enabled NAP mode */
diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h
index 9485b43..e6fc6f2 100644
--- a/arch/powerpc/include/asm/uaccess.h
+++ b/arch/powerpc/include/asm/uaccess.h
@@ -117,6 +117,21 @@ struct exception_table_entry {
 #define __get_user_unaligned __get_user
 #define __put_user_unaligned __put_user
 
+#ifdef CONFIG_IPIPE
+/*
+ * Calling from atomic context should be allowed for all __nocheck
+ * accessors. The head domain may do this, provided page faulting is
+ * duly prevented.
+ */
+#define __chk_might_fault(__pu)	do { } while (0)
+#else
+#define __chk_might_fault(__pu)					\
+	do {							\
+		if (!is_kernel_addr((unsigned long)__pu))	\
+			might_fault();				\
+	} while (0)
+#endif
+
 extern long __put_user_bad(void);
 
 /*
@@ -177,8 +192,7 @@ do {								\
 ({								\
 	long __pu_err;						\
 	__typeof__(*(ptr)) __user *__pu_addr = (ptr);		\
-	if (!is_kernel_addr((unsigned long)__pu_addr))		\
-		might_fault();					\
+	__chk_might_fault(__pu_addr);				\
 	__chk_user_ptr(ptr);					\
 	__put_user_size((x), __pu_addr, (size), __pu_err);	\
 	__pu_err;						\
@@ -267,8 +281,7 @@ do {								\
 	unsigned long __gu_val;					\
 	const __typeof__(*(ptr)) __user *__gu_addr = (ptr);	\
 	__chk_user_ptr(ptr);					\
-	if (!is_kernel_addr((unsigned long)__gu_addr))		\
-		might_fault();					\
+	__chk_might_fault(__gu_addr);				\
 	__get_user_size(__gu_val, __gu_addr, (size), __gu_err);	\
 	(x) = (__typeof__(*(ptr)))__gu_val;			\
 	__gu_err;						\
@@ -281,8 +294,7 @@ do {								\
 	long long __gu_val;					\
 	const __typeof__(*(ptr)) __user *__gu_addr = (ptr);	\
 	__chk_user_ptr(ptr);					\
-	if (!is_kernel_addr((unsigned long)__gu_addr))		\
-		might_fault();					\
+	__chk_might_fault(__gu_addr);				\
 	__get_user_size(__gu_val, __gu_addr, (size), __gu_err);	\
 	(x) = (__typeof__(*(ptr)))__gu_val;			\
 	__gu_err;						\
diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile
index 502cf69..0e29ef7 100644
--- a/arch/powerpc/kernel/Makefile
+++ b/arch/powerpc/kernel/Makefile
@@ -45,6 +45,7 @@ obj64-$(CONFIG_RELOCATABLE)	+= reloc_64.o
 obj-$(CONFIG_PPC_BOOK3E_64)	+= exceptions-64e.o idle_book3e.o
 obj-$(CONFIG_PPC64)		+= vdso64/
 obj-$(CONFIG_ALTIVEC)		+= vecemu.o
+obj-$(CONFIG_IPIPE)		+= ipipe.o
 obj-$(CONFIG_PPC_970_NAP)	+= idle_power4.o
 obj-$(CONFIG_PPC_P7_NAP)	+= idle_power7.o
 obj-$(CONFIG_PPC_OF)		+= of_platform.o prom_parse.o
diff --git a/arch/powerpc/kernel/align.c b/arch/powerpc/kernel/align.c
index 34f5552..8ffb7f2 100644
--- a/arch/powerpc/kernel/align.c
+++ b/arch/powerpc/kernel/align.c
@@ -739,7 +739,7 @@ int fix_alignment(struct pt_regs *regs)
 	unsigned int reg, areg;
 	unsigned int dsisr;
 	unsigned char __user *addr;
-	unsigned long p, swiz;
+	unsigned long p, swiz, irqflags __maybe_unused;
 	int ret, i;
 	union data {
 		u64 ll;
@@ -957,10 +957,10 @@ int fix_alignment(struct pt_regs *regs)
 		if (flags & S) {
 			/* Single-precision FP store requires conversion... */
 #ifdef CONFIG_PPC_FPU
-			preempt_disable();
+			irqflags = hard_preempt_disable();
 			enable_kernel_fp();
 			cvt_df(&data.dd, (float *)&data.x32.low32);
-			preempt_enable();
+			hard_preempt_enable(irqflags);
 #else
 			return 0;
 #endif
@@ -997,10 +997,10 @@ int fix_alignment(struct pt_regs *regs)
 	/* Single-precision FP load requires conversion... */
 	case LD+F+S:
 #ifdef CONFIG_PPC_FPU
-		preempt_disable();
+		irqflags = hard_preempt_disable();
 		enable_kernel_fp();
 		cvt_fd((float *)&data.x32.low32, &data.dd);
-		preempt_enable();
+		hard_preempt_enable(irqflags);
 #else
 		return 0;
 #endif
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 9d7dede..aad5e9d 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -158,6 +158,9 @@ int main(void)
 
 	DEFINE(TI_FLAGS, offsetof(struct thread_info, flags));
 	DEFINE(TI_LOCAL_FLAGS, offsetof(struct thread_info, local_flags));
+#ifdef CONFIG_IPIPE
+	DEFINE(TI_IPIPE, offsetof(struct thread_info, ipipe_flags));
+#endif
 	DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count));
 	DEFINE(TI_TASK, offsetof(struct thread_info, task));
 	DEFINE(TI_CPU, offsetof(struct thread_info, cpu));
@@ -182,8 +185,12 @@ int main(void)
 	DEFINE(PACATOC, offsetof(struct paca_struct, kernel_toc));
 	DEFINE(PACAKBASE, offsetof(struct paca_struct, kernelbase));
 	DEFINE(PACAKMSR, offsetof(struct paca_struct, kernel_msr));
+#ifdef CONFIG_IPIPE
+	DEFINE(PACAROOTPCPU, offsetof(struct paca_struct, ipipe_statp));
+#else
 	DEFINE(PACASOFTIRQEN, offsetof(struct paca_struct, soft_enabled));
 	DEFINE(PACAIRQHAPPENED, offsetof(struct paca_struct, irq_happened));
+#endif
 	DEFINE(PACACONTEXTID, offsetof(struct paca_struct, context.id));
 #ifdef CONFIG_PPC_MM_SLICES
 	DEFINE(PACALOWSLICESPSIZE, offsetof(struct paca_struct,
diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c
index 8084059..83b9229 100644
--- a/arch/powerpc/kernel/cputable.c
+++ b/arch/powerpc/kernel/cputable.c
@@ -2156,7 +2156,7 @@ static struct cpu_spec * __init setup_cpu_spec(unsigned long offset,
 	return t;
 }
 
-struct cpu_spec * __init identify_cpu(unsigned long offset, unsigned int pvr)
+notrace struct cpu_spec * __init identify_cpu(unsigned long offset, unsigned int pvr)
 {
 	struct cpu_spec *s = cpu_specs;
 	int i;
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index 22b45a4..e973007 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -182,9 +182,11 @@ transfer_to_handler:
 2:	/* if from kernel, check interrupted DOZE/NAP mode and
          * check for stack overflow
          */
+#ifndef CONFIG_IPIPE_LEGACY	
 	lwz	r9,KSP_LIMIT(r12)
 	cmplw	r1,r9			/* if r1 <= ksp_limit */
 	ble-	stack_ovf		/* then the kernel stack overflowed */
+#endif	
 5:
 #if defined(CONFIG_6xx) || defined(CONFIG_E500)
 	CURRENT_THREAD_INFO(r9, r1)
@@ -307,6 +309,68 @@ _GLOBAL(DoSyscall)
 	lwz	r11,_CCR(r1)	/* Clear SO bit in CR */
 	rlwinm	r11,r11,0,4,2
 	stw	r11,_CCR(r1)
+#ifdef CONFIG_IPIPE
+	CURRENT_THREAD_INFO(r10, r1)
+#ifndef CONFIG_IPIPE_LEGACY	
+	cmplwi	cr0,r0,NR_syscalls
+	blt	slow_path
+	lwz	r11,TI_IPIPE(r10)
+	andi.	r12,r11,_TIP_HEAD /* implies that TIP_NOTIFY if set */
+	beq	slow_path_check
+	addi	r3,r1,GPR0
+	bl	ipipe_fastcall_hook /* __IPIPE_SYSCALL_E is assumed */
+	cmpwi	r3,0
+	blt	no_fastcall
+	CURRENT_THREAD_INFO(r10, r1)
+	lwz	r11,TI_IPIPE(r10)
+	andi.	r12,r11,_TIP_HEAD
+	bne+	fastcall_exit_check
+	bl	__ipipe_root_sync
+	lwz	r3,GPR3(r1)
+	b	ret_from_syscall
+no_fastcall:
+	/* fastcall handler not implemented */
+	lwz	r0,GPR0(r1)
+slow_path:
+#endif /* !CONFIG_IPIPE_LEGACY */
+	lwz	r11,TI_IPIPE(r10)
+slow_path_check:
+	andi.	r12,r11,_TIP_NOTIFY
+	bne	pipeline_syscall
+	cmplwi	cr0,r0,NR_syscalls
+	blt	root_syscall
+pipeline_syscall:
+	addi	r3,r1,GPR0
+	bl	__ipipe_notify_syscall
+	cmpwi	cr1,r3,0
+	CURRENT_THREAD_INFO(r10, r1)
+	lwz	r11,TI_IPIPE(r10)
+	andi.	r12,r11,_TIP_HEAD
+	bne	fastcall_exit
+	beq	cr1,root_syscall
+	lwz	r3,GPR3(r1)		/* syscall return value */
+	b	ret_from_syscall
+fastcall_exit_check:
+	andi.	r12,r11,_TIP_MAYDAY
+	beq+	fastcall_exit
+	addi	r3,r1,GPR0
+	bl	__ipipe_call_mayday
+fastcall_exit:
+	lwz	r3,GPR3(r1)		/* syscall return value */
+	LOAD_MSR_KERNEL(r10,MSR_KERNEL)	/* clear MSR_EE */
+	SYNC
+	MTMSRD(r10)
+	b	syscall_exit_cont
+root_syscall:
+	lwz	r0,GPR0(r1)
+	lwz	r3,GPR3(r1)
+	lwz	r4,GPR4(r1)
+	lwz	r5,GPR5(r1)
+	lwz	r6,GPR6(r1)
+	lwz	r7,GPR7(r1)
+	lwz	r8,GPR8(r1)
+	lwz	r9,GPR9(r1)
+#endif /* CONFIG_IPIPE */
 #ifdef SHOW_SYSCALLS
 	bl	do_show_syscall
 #endif /* SHOW_SYSCALLS */
@@ -430,6 +494,21 @@ END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
 
 	.globl	ret_from_fork
 ret_from_fork:
+#ifdef CONFIG_IPIPE
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+	stwu	r1,-4(r1)
+	stw	r3,0(r1)
+	lis	r3,(0x80000000)@h
+	ori	r3,r3,(0x80000000)@l
+	bl	ipipe_trace_end
+	lwz	r3,0(r1)
+	addi	r1,r1,4
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+	LOAD_MSR_KERNEL(r10,MSR_KERNEL)
+	ori	r10,r10,MSR_EE
+	SYNC
+	MTMSRD(r10)
+#endif /* CONFIG_IPIPE */
 	REST_NVGPRS(r1)
 	bl	schedule_tail
 	li	r3,0
@@ -437,6 +516,21 @@ ret_from_fork:
 
 	.globl	ret_from_kernel_thread
 ret_from_kernel_thread:
+#ifdef CONFIG_IPIPE
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+	stwu	r1,-4(r1)
+	stw	r3,0(r1)
+	lis	r3,(0x80000000)@h
+	ori	r3,r3,(0x80000000)@l
+	bl	ipipe_trace_end
+	lwz	r3,0(r1)
+	addi	r1,r1,4
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+	LOAD_MSR_KERNEL(r10,MSR_KERNEL)
+	ori	r10,r10,MSR_EE
+	SYNC
+	MTMSRD(r10)
+#endif /* CONFIG_IPIPE */
 	REST_NVGPRS(r1)
 	bl	schedule_tail
 	mtlr	r14
@@ -822,6 +916,19 @@ ret_from_except:
 	SYNC			/* Some chip revs have problems here... */
 	MTMSRD(r10)		/* disable interrupts */
 
+#ifdef CONFIG_IPIPE
+#ifdef CONFIG_IPIPE_LEGACY
+        bl __ipipe_check_root
+        cmpwi   r3, 0
+	mfmsr	r10		/* this is used later, might be messed */
+        beq- restore
+#else	
+	CURRENT_THREAD_INFO(r9, r1)
+	lwz	r0,TI_IPIPE(r9)
+	andi.	r0,r0,_TIP_HEAD
+        bne-    restore
+#endif	
+#endif /* CONFIG_IPIPE */
 	lwz	r3,_MSR(r1)	/* Returning to user mode? */
 	andi.	r0,r3,MSR_PR
 	beq	resume_kernel
@@ -844,6 +951,12 @@ restore_user:
 
 	b	restore
 
+#ifdef CONFIG_IPIPE
+#define PREEMPT_SCHEDULE_IRQ	__ipipe_preempt_schedule_irq
+#else
+#define PREEMPT_SCHEDULE_IRQ	preempt_schedule_irq
+#endif
+
 /* N.B. the only way to get here is from the beq following ret_from_except. */
 resume_kernel:
 	/* check current_thread_info, _TIF_EMULATE_STACK_STORE */
@@ -901,7 +1014,7 @@ resume_kernel:
 	 */
 	bl	trace_hardirqs_off
 #endif
-1:	bl	preempt_schedule_irq
+1:	bl	PREEMPT_SCHEDULE_IRQ
 	CURRENT_THREAD_INFO(r9, r1)
 	lwz	r3,TI_FLAGS(r9)
 	andi.	r0,r3,_TIF_NEED_RESCHED
@@ -1310,6 +1423,13 @@ ee_restarts:
 	.space	4
 	.previous
 
+#ifdef CONFIG_IPIPE
+_GLOBAL(__ipipe_ret_from_except)
+        cmpwi   r3, 0
+        bne+ ret_from_except
+        b restore
+#endif /* CONFIG_IPIPE */
+
 /*
  * PROM code for specific machines follows.  Put it
  * here so it's easy to add arch-specific sections later.
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index 0905c8d..f9293c9 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -33,6 +33,7 @@
 #include <asm/irqflags.h>
 #include <asm/ftrace.h>
 #include <asm/hw_irq.h>
+#include <asm/exception-64s.h>
 #include <asm/context_tracking.h>
 
 /*
@@ -121,27 +122,86 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
 	 * of irq tracing is used, we additionally check that condition
 	 * is correct
 	 */
-#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
+#if defined(CONFIG_TRACE_IRQFLAGS) && \
+	defined(CONFIG_BUG) && !defined(CONFIG_IPIPE)
 	lbz	r10,PACASOFTIRQEN(r13)
 	xori	r10,r10,1
 1:	tdnei	r10,0
 	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
 #endif
 
-#ifdef CONFIG_PPC_BOOK3E
-	wrteei	1
-#else
-	ld	r11,PACAKMSR(r13)
-	ori	r11,r11,MSR_EE
-	mtmsrd	r11,1
-#endif /* CONFIG_PPC_BOOK3E */
-
+#ifdef CONFIG_IPIPE
+	COPY_SOFTISTATE(r11)
+	HARD_ENABLE_INTS
+	CURRENT_THREAD_INFO(r10, r1)
+#ifndef CONFIG_IPIPE_LEGACY
+	cmpldi	cr0,r0,NR_syscalls
+	blt	slow_path
+	ld	r11,TI_IPIPE(r10)
+	andi.	r12,r11,_TIP_HEAD /* implies that TIP_NOTIFY if set */
+	beq	slow_path_check
+	addi	r3,r1,GPR0
+	bl	ipipe_fastcall_hook /* __IPIPE_SYSCALL_E is assumed */
+	cmpwi	r3,0
+	blt	no_fastcall
+	CURRENT_THREAD_INFO(r10, r1)
+	ld	r11,TI_IPIPE(r10)
+	andi.	r12,r11,_TIP_HEAD
+	bne+	fastcall_exit_check
+	bl	__ipipe_root_sync
+	ld	r3,GPR3(r1)
+	b	syscall_exit
+no_fastcall:
+	/* fastcall handler not implemented */
+	ld	r0,GPR0(r1)
+slow_path:
+#endif /* !CONFIG_IPIPE_LEGACY */
+	ld	r11,TI_IPIPE(r10)
+slow_path_check:
+	andi.	r12,r11,_TIP_NOTIFY
+	bne	pipeline_syscall
+	cmpldi	cr0,r0,NR_syscalls
+	blt	root_syscall
+pipeline_syscall:
+	addi	r3,r1,GPR0
+	bl	__ipipe_notify_syscall
+	cmpwi	cr1,r3,0
+	CURRENT_THREAD_INFO(r10, r1)
+	ld	r11,TI_IPIPE(r10)
+	andi.	r12,r11,_TIP_HEAD
+	bne	fastcall_exit
+	beq	cr1,root_syscall
+	ld	r3,GPR3(r1)	/* syscall return value */
+	b	syscall_exit
+fastcall_exit_check:
+	andi.	r12,r11,_TIP_MAYDAY
+	beq+	fastcall_exit
+	addi	r3,r1,GPR0
+	bl	__ipipe_call_mayday
+fastcall_exit:
+	HARD_DISABLE_INTS_RI
+	ld	r3,GPR3(r1)	/* syscall return value */
+	ld	r8,_MSR(r1)
+	ld	r5,_CCR(r1)
+	b	.Lsyscall_error_cont
+root_syscall:
+	ld	r0,GPR0(r1)
+	ld	r3,GPR3(r1)
+	ld	r4,GPR4(r1)
+	ld	r5,GPR5(r1)
+	ld	r6,GPR6(r1)
+	ld	r7,GPR7(r1)
+	ld	r8,GPR8(r1)
+#else /* !CONFIG_IPIPE */
+	HARD_ENABLE_INTS
 	/* We do need to set SOFTE in the stack frame or the return
 	 * from interrupt will be painful
 	 */
 	li	r10,1
 	std	r10,SOFTE(r1)
-
+#endif /* !CONFIG_IPIPE */
+	addi	r9,r1,STACK_FRAME_OVERHEAD
+	
 #ifdef SHOW_SYSCALLS
 	bl	do_show_syscall
 	REST_GPR(0,r1)
@@ -196,22 +256,7 @@ syscall_exit:
 	 * Disable interrupts so current_thread_info()->flags can't change,
 	 * and so that we don't get interrupted after loading SRR0/1.
 	 */
-#ifdef CONFIG_PPC_BOOK3E
-	wrteei	0
-#else
-	ld	r10,PACAKMSR(r13)
-	/*
-	 * For performance reasons we clear RI the same time that we
-	 * clear EE. We only need to clear RI just before we restore r13
-	 * below, but batching it with EE saves us one expensive mtmsrd call.
-	 * We have to be careful to restore RI if we branch anywhere from
-	 * here (eg syscall_exit_work).
-	 */
-	li	r9,MSR_RI
-	andc	r11,r10,r9
-	mtmsrd	r11,1
-#endif /* CONFIG_PPC_BOOK3E */
-
+	HARD_DISABLE_INTS_RI
 	ld	r9,TI_FLAGS(r12)
 	li	r11,-_LAST_ERRNO
 	andi.	r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
@@ -310,16 +355,9 @@ syscall_exit_work:
 	andi.	r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP)
 	beq	ret_from_except_lite
 
-	/* Re-enable interrupts */
-#ifdef CONFIG_PPC_BOOK3E
-	wrteei	1
-#else
-	ld	r10,PACAKMSR(r13)
-	ori	r10,r10,MSR_EE
-	mtmsrd	r10,1
-#endif /* CONFIG_PPC_BOOK3E */
-
 	bl	save_nvgprs
+	/* Re-enable interrupts */
+	HARD_ENABLE_INTS
 	addi	r3,r1,STACK_FRAME_OVERHEAD
 	bl	do_syscall_trace_leave
 	b	ret_from_except
@@ -370,12 +408,18 @@ _GLOBAL(ppc64_swapcontext)
 	b	syscall_exit
 
 _GLOBAL(ret_from_fork)
+#ifdef CONFIG_IPIPE
+	HARD_ENABLE_INTS
+#endif /* CONFIG_IPIPE */
 	bl	schedule_tail
 	REST_NVGPRS(r1)
 	li	r3,0
 	b	syscall_exit
 
 _GLOBAL(ret_from_kernel_thread)
+#ifdef CONFIG_IPIPE
+	HARD_ENABLE_INTS
+#endif /* CONFIG_IPIPE */
 	bl	schedule_tail
 	REST_NVGPRS(r1)
 	mtlr	r14
@@ -592,6 +636,13 @@ END_FTR_SECTION_IFSET(CPU_FTR_DSCR)
 	addi	r1,r1,SWITCH_FRAME_SIZE
 	blr
 
+#ifdef CONFIG_IPIPE
+_GLOBAL(__ipipe_ret_from_except_lite)
+	cmpwi	r3,0
+	bne+	.ret_from_except_lite
+	b	restore
+#endif /* CONFIG_IPIPE */
+
 	.align	7
 _GLOBAL(ret_from_except)
 	ld	r11,_TRAP(r1)
@@ -605,14 +656,20 @@ _GLOBAL(ret_from_except_lite)
 	 * can't change between when we test it and when we return
 	 * from the interrupt.
 	 */
-#ifdef CONFIG_PPC_BOOK3E
-	wrteei	0
-#else
-	ld	r10,PACAKMSR(r13) /* Get kernel MSR without EE */
-	mtmsrd	r10,1		  /* Update machine state */
-#endif /* CONFIG_PPC_BOOK3E */
-
+	HARD_DISABLE_INTS
 	CURRENT_THREAD_INFO(r9, r1)
+#ifdef CONFIG_IPIPE
+#ifdef CONFIG_IPIPE_LEGACY
+	bl	.__ipipe_check_root
+	cmpwi	r3,0
+	beq-	restore
+#else	
+	ld	r3,TI_IPIPE(r9)
+	andi.	r3,r3,_TIP_HEAD
+	bne-	restore
+#endif	
+#endif /* CONFIG_IPIPE */
+
 	ld	r3,_MSR(r1)
 #ifdef CONFIG_PPC_BOOK3E
 	ld	r10,PACACURRENT(r13)
@@ -721,8 +778,11 @@ resume_kernel:
 	 * sure we are soft-disabled first and reconcile irq state.
 	 */
 	RECONCILE_IRQ_STATE(r3,r4)
+#ifdef CONFIG_IPIPE
+1:	bl	__ipipe_preempt_schedule_irq
+#else
 1:	bl	preempt_schedule_irq
-
+#endif
 	/* Re-test flags and eventually loop */
 	CURRENT_THREAD_INFO(r9, r1)
 	ld	r4,TI_FLAGS(r9)
@@ -735,12 +795,7 @@ resume_kernel:
 	 * when we return from the interrupt, and so that we don't get
 	 * interrupted after loading SRR0/1.
 	 */
-#ifdef CONFIG_PPC_BOOK3E
-	wrteei	0
-#else
-	ld	r10,PACAKMSR(r13) /* Get kernel MSR without EE */
-	mtmsrd	r10,1		  /* Update machine state */
-#endif /* CONFIG_PPC_BOOK3E */
+	HARD_DISABLE_INTS
 #endif /* CONFIG_PREEMPT */
 
 	.globl	fast_exc_return_irq
@@ -751,6 +806,13 @@ restore:
 	 * are about to re-enable interrupts
 	 */
 	ld	r5,SOFTE(r1)
+#ifdef CONFIG_IPIPE
+	ld	r4,PACAROOTPCPU(r13)
+	cmpwi	cr0,r4,0
+	bne	1f
+	TRACE_ENABLE_INTS
+1:
+#else	/* !CONFIG_IPIPE */
 	lbz	r6,PACASOFTIRQEN(r13)
 	cmpwi	cr0,r5,0
 	beq	restore_irq_off
@@ -777,6 +839,7 @@ restore_no_replay:
 	TRACE_ENABLE_INTS
 	li	r0,1
 	stb	r0,PACASOFTIRQEN(r13);
+#endif	/* !CONFIG_IPIPE */
 
 	/*
 	 * Final return path. BookE is handled in a different file
@@ -867,6 +930,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 
 #endif /* CONFIG_PPC_BOOK3E */
 
+#ifndef CONFIG_IPIPE	
 	/*
 	 * We are returning to a context with interrupts soft disabled.
 	 *
@@ -948,6 +1012,8 @@ restore_check_irq_replay:
 	b	ret_from_except
 #endif /* CONFIG_PPC_DOORBELL */
 1:	b	ret_from_except /* What else to do here ? */
+
+#endif /* !CONFIG_IPIPE */	
  
 unrecov_restore:
 	addi	r3,r1,STACK_FRAME_OVERHEAD
@@ -994,7 +1060,7 @@ _GLOBAL(enter_rtas)
 	li	r0,0
 	mtcr	r0
 
-#ifdef CONFIG_BUG	
+#if defined(CONFIG_BUG) && !defined(CONFIG_IPIPE)
 	/* There is no way it is acceptable to get here with interrupts enabled,
 	 * check it with the asm equivalent of WARN_ON
 	 */
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 5e01984..6f68eff 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -644,9 +644,11 @@ masked_##_H##interrupt:					\
 	GET_SCRATCH0(r13);				\
 	##_H##rfid;					\
 	b	.
-	
+
+#ifndef CONFIG_IPIPE
 	MASKED_INTERRUPT()
 	MASKED_INTERRUPT(H)
+#endif
 
 /*
  * Called from arch_local_irq_enable when an interrupt needs
@@ -761,7 +763,7 @@ kvmppc_skip_Hinterrupt:
 	STD_EXCEPTION_COMMON(0x100, system_reset, system_reset_exception)
 
 	STD_EXCEPTION_COMMON_ASYNC(0x500, hardware_interrupt, do_IRQ)
-	STD_EXCEPTION_COMMON_ASYNC(0x900, decrementer, timer_interrupt)
+	DECREMENTER_EXCEPTION(0x900, decrementer, timer_interrupt)
 	STD_EXCEPTION_COMMON(0x980, hdecrementer, hdec_interrupt)
 #ifdef CONFIG_PPC_DOORBELL
 	STD_EXCEPTION_COMMON_ASYNC(0xa00, doorbell_super, doorbell_exception)
diff --git a/arch/powerpc/kernel/fpu.S b/arch/powerpc/kernel/fpu.S
index 9ad236e..3f38333 100644
--- a/arch/powerpc/kernel/fpu.S
+++ b/arch/powerpc/kernel/fpu.S
@@ -195,7 +195,17 @@ END_FTR_SECTION_IFSET(CPU_FTR_VSX)
  * Enables the FPU for use in the kernel on return.
  */
 _GLOBAL(giveup_fpu)
+#ifdef CONFIG_IPIPE
+	mfmsr	r10
+#ifdef CONFIG_PPC64	
+	rldicl	r5,r10,48,1		/* clear MSR_EE */
+	rotldi	r5,r5,16
+#else
+	rlwinm	r5,r10,0,17,15		/* clear MSR_EE */
+#endif	
+#else	
 	mfmsr	r5
+#endif	
 	ori	r5,r5,MSR_FP
 #ifdef CONFIG_VSX
 BEGIN_FTR_SECTION
@@ -208,7 +218,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_VSX)
 	SYNC_601
 	isync
 	PPC_LCMPI	0,r3,0
-	beqlr-				/* if no previous owner, done */
+	beq-	3f			/* if no previous owner, done */
 	addi	r3,r3,THREAD	        /* want THREAD of task */
 	PPC_LL	r6,THREAD_FPSAVEAREA(r3)
 	PPC_LL	r5,PT_REGS(r3)
@@ -235,6 +245,18 @@ END_FTR_SECTION_IFSET(CPU_FTR_VSX)
 	LOAD_REG_ADDRBASE(r4,last_task_used_math)
 	PPC_STL	r5,ADDROFF(last_task_used_math)(r4)
 #endif /* CONFIG_SMP */
+3:
+#ifdef CONFIG_IPIPE		/* restore interrupt state */
+	andi.	r10,r10,MSR_EE
+	beqlr
+	mfmsr	r5
+	ori	r5,r5,MSR_EE
+	SYNC_601
+	ISYNC_601
+	MTMSRD(r5)
+	SYNC_601
+	isync
+#endif
 	blr
 
 /*
diff --git a/arch/powerpc/kernel/head_32.S b/arch/powerpc/kernel/head_32.S
index dc0488b..6da2091 100644
--- a/arch/powerpc/kernel/head_32.S
+++ b/arch/powerpc/kernel/head_32.S
@@ -326,6 +326,12 @@ i##n:								\
 	EXC_XFER_TEMPLATE(n, hdlr, n, NOCOPY, transfer_to_handler_full,	\
 			  ret_from_except_full)
 
+#ifdef CONFIG_IPIPE
+#define EXC_XFER_IPIPE(n, hdlr)		\
+	EXC_XFER_TEMPLATE(n, hdlr, n+1, NOCOPY, transfer_to_handler, \
+			  __ipipe_ret_from_except)
+#endif /* CONFIG_IPIPE */
+
 #define EXC_XFER_LITE(n, hdlr)		\
 	EXC_XFER_TEMPLATE(n, hdlr, n+1, NOCOPY, transfer_to_handler, \
 			  ret_from_except)
@@ -413,7 +419,11 @@ InstructionAccess:
 	EXC_XFER_LITE(0x400, handle_page_fault)
 
 /* External interrupt */
+#ifdef CONFIG_IPIPE
+	EXCEPTION(0x500, HardwareInterrupt, __ipipe_grab_irq, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
 	EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
+#endif /* CONFIG_IPIPE */
 
 /* Alignment exception */
 	. = 0x600
@@ -449,7 +459,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_FPU_UNAVAILABLE)
 	EXC_XFER_EE_LITE(0x800, kernel_fp_unavailable_exception)
 
 /* Decrementer */
+#ifdef CONFIG_IPIPE
+	EXCEPTION(0x900, Decrementer, __ipipe_grab_timer, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
 	EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
+#endif /* CONFIG_IPIPE */
 
 	EXCEPTION(0xa00, Trap_0a, unknown_exception, EXC_XFER_EE)
 	EXCEPTION(0xb00, Trap_0b, unknown_exception, EXC_XFER_EE)
@@ -1019,6 +1033,12 @@ _ENTRY(switch_mmu_context)
 	lwz	r3,MMCONTEXTID(r4)
 	cmpwi	cr0,r3,0
 	blt-	4f
+#ifdef CONFIG_IPIPE
+	mfmsr	r7
+	rlwinm	r0,r7,0,17,15	/* clear MSR_EE in r0 */
+	mtmsr	r0
+	sync
+#endif	
 	mulli	r3,r3,897	/* multiply context by skew factor */
 	rlwinm	r3,r3,4,8,27	/* VSID = (context & 0xfffff) << 4 */
 	addis	r3,r3,0x6000	/* Set Ks, Ku bits */
@@ -1042,6 +1062,9 @@ _ENTRY(switch_mmu_context)
 	rlwinm	r3,r3,0,8,3	/* clear out any overflow from VSID field */
 	addis	r4,r4,0x1000	/* address of next segment */
 	bdnz	3b
+#ifdef CONFIG_IPIPE
+	mtmsr	r7
+#endif	
 	sync
 	isync
 	blr
diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S
index 7d7d863..1c01a90 100644
--- a/arch/powerpc/kernel/head_40x.S
+++ b/arch/powerpc/kernel/head_40x.S
@@ -229,6 +229,12 @@ label:
 	EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, NOCOPY, transfer_to_handler_full, \
 			  ret_from_except_full)
 
+#ifdef CONFIG_IPIPE
+#define EXC_XFER_IPIPE(n, hdlr)		\
+	EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, NOCOPY, transfer_to_handler, \
+			  __ipipe_ret_from_except)
+#endif /* CONFIG_IPIPE */
+
 #define EXC_XFER_LITE(n, hdlr)		\
 	EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, NOCOPY, transfer_to_handler, \
 			  ret_from_except)
@@ -397,7 +403,11 @@ label:
 	EXC_XFER_LITE(0x400, handle_page_fault)
 
 /* 0x0500 - External Interrupt Exception */
+#ifdef CONFIG_IPIPE
+	EXCEPTION(0x0500, HardwareInterrupt, __ipipe_grab_irq, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
 	EXCEPTION(0x0500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
+#endif /* CONFIG_IPIPE */
 
 /* 0x0600 - Alignment Exception */
 	START_EXCEPTION(0x0600, Alignment)
@@ -732,7 +742,11 @@ Decrementer:
 	lis	r0,TSR_PIS@h
 	mtspr	SPRN_TSR,r0		/* Clear the PIT exception */
 	addi	r3,r1,STACK_FRAME_OVERHEAD
+#ifdef CONFIG_IPIPE
+	EXC_XFER_IPIPE(0x1000, __ipipe_grab_timer)
+#else /* !CONFIG_IPIPE */
 	EXC_XFER_LITE(0x1000, timer_interrupt)
+#endif /* CONFIG_IPIPE */
 
 	/* Fixed Interval Timer (FIT) Exception. (from 0x1010) */
 FITException:
diff --git a/arch/powerpc/kernel/head_44x.S b/arch/powerpc/kernel/head_44x.S
index b5061ab..e69a9af 100644
--- a/arch/powerpc/kernel/head_44x.S
+++ b/arch/powerpc/kernel/head_44x.S
@@ -262,8 +262,13 @@ interrupt_base:
 	INSTRUCTION_STORAGE_EXCEPTION
 
 	/* External Input Interrupt */
+#ifdef CONFIG_IPIPE
+	EXCEPTION(0x0500, BOOKE_INTERRUPT_EXTERNAL, ExternalInput, \
+		  __ipipe_grab_irq, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
 	EXCEPTION(0x0500, BOOKE_INTERRUPT_EXTERNAL, ExternalInput, \
 		  do_IRQ, EXC_XFER_LITE)
+#endif /* CONFIG_IPIPE */
 
 	/* Alignment Interrupt */
 	ALIGNMENT_EXCEPTION
diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S
index d48125d..bd1b45e 100644
--- a/arch/powerpc/kernel/head_64.S
+++ b/arch/powerpc/kernel/head_64.S
@@ -596,6 +596,7 @@ _GLOBAL(pmac_secondary_start)
 	add	r13,r13,r4		/* for this processor.		*/
 	SET_PACA(r13)			/* Save vaddr of paca in an SPRG*/
 
+#ifndef CONFIG_IPIPE
 	/* Mark interrupts soft and hard disabled (they might be enabled
 	 * in the PACA when doing hotplug)
 	 */
@@ -603,6 +604,7 @@ _GLOBAL(pmac_secondary_start)
 	stb	r0,PACASOFTIRQEN(r13)
 	li	r0,PACA_IRQ_HARD_DIS
 	stb	r0,PACAIRQHAPPENED(r13)
+#endif	
 
 	/* Create a temp kernel stack for use before relocation is on.	*/
 	ld	r1,PACAEMERGSP(r13)
@@ -653,12 +655,14 @@ __secondary_start:
 	li	r7,0
 	mtlr	r7
 
+#ifndef CONFIG_IPIPE
 	/* Mark interrupts soft and hard disabled (they might be enabled
 	 * in the PACA when doing hotplug)
 	 */
 	stb	r7,PACASOFTIRQEN(r13)
 	li	r0,PACA_IRQ_HARD_DIS
 	stb	r0,PACAIRQHAPPENED(r13)
+#endif	
 
 	/* enable MMU and jump to start_secondary */
 	LOAD_REG_ADDR(r3, start_secondary_prolog)
@@ -821,6 +825,7 @@ start_here_common:
 	/* Do more system initializations in virtual mode */
 	bl	setup_system
 
+#ifndef CONFIG_IPIPE	
 	/* Mark interrupts soft and hard disabled (they might be enabled
 	 * in the PACA when doing hotplug)
 	 */
@@ -828,6 +833,7 @@ start_here_common:
 	stb	r0,PACASOFTIRQEN(r13)
 	li	r0,PACA_IRQ_HARD_DIS
 	stb	r0,PACAIRQHAPPENED(r13)
+#endif	
 
 	/* Generic kernel entry */
 	bl	start_kernel
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index fafff8d..63c87229 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -190,6 +190,12 @@ i##n:								\
 	EXC_XFER_TEMPLATE(n, hdlr, n, NOCOPY, transfer_to_handler_full,	\
 			  ret_from_except_full)
 
+#ifdef CONFIG_IPIPE
+#define EXC_XFER_IPIPE(n, hdlr)		\
+	EXC_XFER_TEMPLATE(n, hdlr, n+1, NOCOPY, transfer_to_handler, \
+			  __ipipe_ret_from_except)
+#endif /* CONFIG_IPIPE */
+
 #define EXC_XFER_LITE(n, hdlr)		\
 	EXC_XFER_TEMPLATE(n, hdlr, n+1, NOCOPY, transfer_to_handler, \
 			  ret_from_except)
@@ -245,7 +251,11 @@ InstructionAccess:
 	EXC_XFER_LITE(0x400, handle_page_fault)
 
 /* External interrupt */
+#ifdef CONFIG_IPIPE
+	EXCEPTION(0x500, HardwareInterrupt, __ipipe_grab_irq, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
 	EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
+#endif /* CONFIG_IPIPE */
 
 /* Alignment exception */
 	. = 0x600
@@ -268,7 +278,11 @@ Alignment:
 	EXCEPTION(0x800, FPUnavailable, unknown_exception, EXC_XFER_STD)
 
 /* Decrementer */
+#ifdef CONFIG_IPIPE
+	EXCEPTION(0x900, Decrementer, __ipipe_grab_timer, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
 	EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
+#endif /* CONFIG_IPIPE */
 
 	EXCEPTION(0xa00, Trap_0a, unknown_exception, EXC_XFER_EE)
 	EXCEPTION(0xb00, Trap_0b, unknown_exception, EXC_XFER_EE)
@@ -897,15 +911,24 @@ _GLOBAL(set_context)
  * It is important we get called with interrupts disabled.  I used to
  * do that, but it appears that all code that calls this already had
  * interrupt disabled.
+ * I-pipe: we enforce this back, we need to bypass virtualization
+ * here.
  */
 	.globl	set_dec_cpu6
 set_dec_cpu6:
+#ifdef CONFIG_IPIPE
+	mfmsr	r6
+	rlwinm	r5,r6,0,17,15	/* clear MSR_EE */
+#endif
 	lis	r7, cpu6_errata_word@h
 	ori	r7, r7, cpu6_errata_word@l
 	li	r4, 0x2c00
 	stw	r4, 8(r7)
 	lwz	r4, 8(r7)
         mtspr   22, r3		/* Update Decrementer */
+#ifdef CONFIG_IPIPE
+	MTMSRD(r6)
+#endif
 	SYNC
 	blr
 #endif
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index a620203..a67bb7c 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -240,6 +240,12 @@ label:
 	EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, NOCOPY, transfer_to_handler_full, \
 			  ret_from_except_full)
 
+#ifdef CONFIG_IPIPE
+#define EXC_XFER_IPIPE(n, hdlr)		\
+	EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, NOCOPY, transfer_to_handler, \
+			  __ipipe_ret_from_except)
+#endif /* CONFIG_IPIPE */
+
 #define EXC_XFER_LITE(n, hdlr)		\
 	EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, NOCOPY, transfer_to_handler, \
 			  ret_from_except)
@@ -404,6 +410,15 @@ label:
 	addi	r3,r1,STACK_FRAME_OVERHEAD;				      \
 	EXC_XFER_STD(0x0700, program_check_exception)
 
+#ifdef CONFIG_IPIPE
+#define DECREMENTER_EXCEPTION						      \
+	START_EXCEPTION(Decrementer)					      \
+	NORMAL_EXCEPTION_PROLOG(DECREMENTER);				      \
+	lis     r0,TSR_DIS@h;           /* Setup the DEC interrupt mask */    \
+	mtspr   SPRN_TSR,r0;		/* Clear the DEC interrupt */	      \
+	addi    r3,r1,STACK_FRAME_OVERHEAD;				      \
+	EXC_XFER_IPIPE(0x0900, __ipipe_grab_timer)
+#else /* !CONFIG_IPIPE */
 #define DECREMENTER_EXCEPTION						      \
 	START_EXCEPTION(Decrementer)					      \
 	NORMAL_EXCEPTION_PROLOG(DECREMENTER);		      \
@@ -411,6 +426,7 @@ label:
 	mtspr   SPRN_TSR,r0;		/* Clear the DEC interrupt */	      \
 	addi    r3,r1,STACK_FRAME_OVERHEAD;				      \
 	EXC_XFER_LITE(0x0900, timer_interrupt)
+#endif /* CONFIG_IPIPE */
 
 #define FP_UNAVAILABLE_EXCEPTION					      \
 	START_EXCEPTION(FloatingPointUnavailable)			      \
diff --git a/arch/powerpc/kernel/head_fsl_booke.S b/arch/powerpc/kernel/head_fsl_booke.S
index fffd1f9..66425e9 100644
--- a/arch/powerpc/kernel/head_fsl_booke.S
+++ b/arch/powerpc/kernel/head_fsl_booke.S
@@ -390,7 +390,11 @@ interrupt_base:
 	INSTRUCTION_STORAGE_EXCEPTION
 
 	/* External Input Interrupt */
+#ifdef CONFIG_IPIPE
+	EXCEPTION(0x0500, EXTERNAL, ExternalInput, __ipipe_grab_irq, EXC_XFER_LITE)
+#else /* !CONFIG_IPIPE */
 	EXCEPTION(0x0500, EXTERNAL, ExternalInput, do_IRQ, EXC_XFER_LITE)
+#endif /* CONFIG_IPIPE */
 
 	/* Alignment Interrupt */
 	ALIGNMENT_EXCEPTION
@@ -1017,10 +1021,14 @@ _GLOBAL(__setup_ehv_ivors)
 _GLOBAL(giveup_spe)
 	mfmsr	r5
 	oris	r5,r5,MSR_SPE@h
+#ifdef CONFIG_IPIPE
+	mr	r10,r5
+	rlwinm	r5,r5,0,17,15		/* clear MSR_EE */
+#endif
 	mtmsr	r5			/* enable use of SPE now */
 	isync
 	cmpi	0,r3,0
-	beqlr-				/* if no previous owner, done */
+	beq-	2f			/* if no previous owner, done */
 	addi	r3,r3,THREAD		/* want THREAD of task */
 	lwz	r5,PT_REGS(r3)
 	cmpi	0,r5,0
@@ -1040,6 +1048,15 @@ _GLOBAL(giveup_spe)
 	lis	r4,last_task_used_spe@ha
 	stw	r5,last_task_used_spe@l(r4)
 #endif /* !CONFIG_SMP */
+2:
+#ifdef CONFIG_IPIPE
+	andi.	r5,r10,MSR_EE
+	beqlr-
+	mfmsr	r5
+	ori	r5,r5,MSR_EE
+	mtmsr	r5
+	isync
+#endif
 	blr
 #endif /* CONFIG_SPE */
 
diff --git a/arch/powerpc/kernel/idle.c b/arch/powerpc/kernel/idle.c
index d7216c9..888dfd1 100644
--- a/arch/powerpc/kernel/idle.c
+++ b/arch/powerpc/kernel/idle.c
@@ -53,19 +53,47 @@ void arch_cpu_idle_dead(void)
 }
 #endif
 
+#ifdef CONFIG_IPIPE
+static void __ipipe_halt_root(void)
+{
+	struct ipipe_percpu_domain_data *p;
+
+	/*
+	 * Emulate idle entry sequence over the root domain, which is
+	 * stalled on entry.
+	 */
+	hard_local_irq_disable();
+
+	p = ipipe_this_cpu_root_context();
+	__clear_bit(IPIPE_STALL_FLAG, &p->status);
+
+	if (unlikely(__ipipe_ipending_p(p)))
+		__ipipe_sync_stage();
+	else
+		ppc_md.power_save();
+
+	hard_local_irq_enable();
+}
+#else
+static void __ipipe_halt_root(void)
+{
+	ppc_md.power_save();
+	/*
+	 * Some power_save functions return with
+	 * interrupts enabled, some don't.
+	 */
+	if (irqs_disabled())
+		local_irq_enable();
+}
+#endif
+
 void arch_cpu_idle(void)
 {
 	ppc64_runlatch_off();
 
-	if (ppc_md.power_save) {
-		ppc_md.power_save();
-		/*
-		 * Some power_save functions return with
-		 * interrupts enabled, some don't.
-		 */
-		if (irqs_disabled())
-			local_irq_enable();
-	} else {
+	if (ppc_md.power_save)
+		__ipipe_halt_root();
+	else {
 		local_irq_enable();
 		/*
 		 * Go into low thread priority and possibly
diff --git a/arch/powerpc/kernel/idle_power4.S b/arch/powerpc/kernel/idle_power4.S
index f57a193..ac19c01 100644
--- a/arch/powerpc/kernel/idle_power4.S
+++ b/arch/powerpc/kernel/idle_power4.S
@@ -36,10 +36,12 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CAN_NAP)
 	rotldi	r0,r0,16
 	mtmsrd	r0,1
 
+#ifndef CONFIG_IPIPE
 	/* Check if something happened while soft-disabled */
 	lbz	r0,PACAIRQHAPPENED(r13)
 	cmpwi	cr0,r0,0
 	bnelr
+#endif	
 
 	/* Soft-enable interrupts */
 #ifdef CONFIG_TRACE_IRQFLAGS
@@ -52,9 +54,10 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CAN_NAP)
 	mtlr	r0
 	mfmsr	r7
 #endif /* CONFIG_TRACE_IRQFLAGS */
-
+#ifndef CONFIG_IPIPE
 	li	r0,1
 	stb	r0,PACASOFTIRQEN(r13)	/* we'll hard-enable shortly */
+#endif	
 BEGIN_FTR_SECTION
 	DSSALL
 	sync
diff --git a/arch/powerpc/kernel/idle_power7.S b/arch/powerpc/kernel/idle_power7.S
index 283c603..c6d39bd 100644
--- a/arch/powerpc/kernel/idle_power7.S
+++ b/arch/powerpc/kernel/idle_power7.S
@@ -71,12 +71,14 @@ _GLOBAL(power7_powersave_common)
 	rotldi	r9,r9,16
 	mtmsrd	r9,1			/* hard-disable interrupts */
 
+#ifndef CONFIG_IPIPE	
 	/* Check if something happened while soft-disabled */
 	lbz	r0,PACAIRQHAPPENED(r13)
 	andi.	r0,r0,~PACA_IRQ_HARD_DIS@l
 	beq	1f
 	cmpwi	cr0,r4,0
 	beq	1f
+#endif	
 	addi	r1,r1,INT_FRAME_SIZE
 	ld	r0,16(r1)
 	mtlr	r0
@@ -86,8 +88,10 @@ _GLOBAL(power7_powersave_common)
 	 * be in when returning and we need to tell arch_local_irq_restore()
 	 * about it
 	 */
+#ifndef CONFIG_IPIPE	
 	li	r0,PACA_IRQ_HARD_DIS
 	stb	r0,PACAIRQHAPPENED(r13)
+#endif	
 
 	/* We haven't lost state ... yet */
 	li	r0,0
diff --git a/arch/powerpc/kernel/ipipe.c b/arch/powerpc/kernel/ipipe.c
new file mode 100644
index 0000000..292fed5
--- /dev/null
+++ b/arch/powerpc/kernel/ipipe.c
@@ -0,0 +1,372 @@
+/* -*- linux-c -*-
+ * linux/arch/powerpc/kernel/ipipe.c
+ *
+ * Copyright (C) 2005 Heikki Lindholm (PPC64 port).
+ * Copyright (C) 2004 Wolfgang Grandegger (Adeos/ppc port over 2.4).
+ * Copyright (C) 2002-2012 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-dependent I-PIPE core support for PowerPC 32/64bit.
+ */
+
+#include <linux/kernel.h>
+#include <linux/smp.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/bitops.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/module.h>
+#include <linux/kernel_stat.h>
+#include <linux/ipipe_tickdev.h>
+#include <asm/reg.h>
+#include <asm/switch_to.h>
+#include <asm/mmu_context.h>
+#include <asm/unistd.h>
+#include <asm/machdep.h>
+#include <asm/atomic.h>
+#include <asm/hardirq.h>
+#include <asm/io.h>
+#include <asm/time.h>
+#include <asm/runlatch.h>
+#include <asm/debug.h>
+
+static void __ipipe_do_IRQ(unsigned int irq, void *cookie);
+
+static void __ipipe_do_timer(unsigned int irq, void *cookie);
+
+#define DECREMENTER_MAX	0x7fffffff
+
+#ifdef CONFIG_SMP
+
+static DEFINE_PER_CPU(struct ipipe_ipi_struct, ipipe_ipi_message);
+
+unsigned int __ipipe_ipi_irq = NR_IRQS + 1; /* dummy value */
+
+#ifdef CONFIG_DEBUGGER
+cpumask_t __ipipe_dbrk_pending;	/* pending debugger break IPIs */
+#endif
+
+void __ipipe_hook_critical_ipi(struct ipipe_domain *ipd)
+{
+	unsigned int ipi = IPIPE_CRITICAL_IPI;
+
+	ipd->irqs[ipi].ackfn = NULL;
+	ipd->irqs[ipi].handler = __ipipe_do_critical_sync;
+	ipd->irqs[ipi].cookie = NULL;
+	ipd->irqs[ipi].control = IPIPE_HANDLE_MASK|IPIPE_STICKY_MASK;
+}
+
+void __ipipe_register_ipi(unsigned int irq)
+{
+	__ipipe_ipi_irq = irq;
+}
+
+static void __ipipe_ipi_demux(int irq, struct pt_regs *regs)
+{
+	struct irq_desc *desc = irq_to_desc(irq);
+	int ipi, cpu = ipipe_processor_id();
+
+	desc->ipipe_ack(irq, desc);
+
+	kstat_incr_irq_this_cpu(irq);
+
+	while (per_cpu(ipipe_ipi_message, cpu).value & IPIPE_MSG_IPI_MASK) {
+		for (ipi = IPIPE_MSG_CRITICAL_IPI; ipi <= IPIPE_MSG_RESCHEDULE_IPI; ++ipi) {
+			if (test_and_clear_bit(ipi, &per_cpu(ipipe_ipi_message, cpu).value)) {
+				mb();
+				__ipipe_handle_irq(ipi + IPIPE_BASE_IPI_OFFSET, NULL);
+			}
+		}
+	}
+
+#ifdef CONFIG_DEBUGGER
+	/*
+	 * The debugger IPI handler should be NMI-safe, so let's call
+	 * it immediately in case the IPI is pending.
+	 */
+	if (cpu_isset(cpu, __ipipe_dbrk_pending)) {
+		cpu_clear(cpu, __ipipe_dbrk_pending);
+		debugger_ipi(regs);
+	}
+#endif /* CONFIG_DEBUGGER */
+
+	ipipe_end_irq(irq);
+}
+
+void ipipe_set_irq_affinity(unsigned int irq, cpumask_t cpumask)
+{
+	if (WARN_ON_ONCE(irq_get_chip(irq)->irq_set_affinity == NULL))
+		return;
+
+	if (WARN_ON_ONCE(cpumask_any_and(&cpumask, cpu_online_mask) >= nr_cpu_ids))
+		return;
+
+	irq_get_chip(irq)->irq_set_affinity(irq_get_irq_data(irq), &cpumask, true);
+}
+EXPORT_SYMBOL_GPL(ipipe_set_irq_affinity);
+
+void ipipe_send_ipi(unsigned int ipi, cpumask_t cpumask)
+{
+	unsigned long flags;
+	int cpu, me;
+
+	flags = hard_local_irq_save();
+
+	ipi -= IPIPE_BASE_IPI_OFFSET;
+	for_each_online_cpu(cpu) {
+		if (cpu_isset(cpu, cpumask))
+			set_bit(ipi, &per_cpu(ipipe_ipi_message, cpu).value);
+	}
+	mb();
+
+	if (unlikely(cpus_empty(cpumask)))
+		goto out;
+
+	me = ipipe_processor_id();
+	for_each_cpu_mask_nr(cpu, cpumask) {
+		if (cpu != me)
+			smp_ops->message_pass(cpu, PPC_MSG_IPIPE_DEMUX);
+	}
+out:
+	hard_local_irq_restore(flags);
+}
+EXPORT_SYMBOL_GPL(ipipe_send_ipi);
+
+void ipipe_stall_root(void)
+{
+	unsigned long flags;
+
+	ipipe_root_only();
+	flags = hard_local_irq_save();
+	set_bit(IPIPE_STALL_FLAG, &__ipipe_root_status);
+	hard_local_irq_restore(flags);
+}
+EXPORT_SYMBOL_GPL(ipipe_stall_root);
+
+unsigned long ipipe_test_and_stall_root(void)
+{
+	unsigned long flags;
+	int x;
+
+	ipipe_root_only();
+	flags = hard_local_irq_save();
+	x = test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_root_status);
+	hard_local_irq_restore(flags);
+
+	return x;
+}
+EXPORT_SYMBOL(ipipe_test_and_stall_root);
+
+unsigned long ipipe_test_root(void)
+{
+	unsigned long flags;
+	int x;
+
+	flags = hard_local_irq_save();
+	x = test_bit(IPIPE_STALL_FLAG, &__ipipe_root_status);
+	hard_local_irq_restore(flags);
+
+	return x;
+}
+EXPORT_SYMBOL_GPL(ipipe_test_root);
+
+#endif	/* CONFIG_SMP */
+
+void __ipipe_early_core_setup(void)
+{
+	unsigned int virq;
+	/*
+	 * Allocate all the virtual IRQs we need. We expect fixed virq
+	 * numbers starting at IPIPE_VIRQ_BASE, so we request them
+	 * early.
+	 */
+	virq = ipipe_alloc_virq();
+	BUG_ON(virq != IPIPE_TIMER_VIRQ);
+#ifdef CONFIG_SMP
+	virq = ipipe_alloc_virq();
+	BUG_ON(virq != IPIPE_CRITICAL_IPI);
+	virq = ipipe_alloc_virq();
+	BUG_ON(virq != IPIPE_HRTIMER_IPI);
+	virq = ipipe_alloc_virq();
+	BUG_ON(virq != IPIPE_RESCHEDULE_IPI);
+#endif
+}
+
+/*
+ * __ipipe_enable_pipeline() -- We are running on the boot CPU, hw
+ * interrupts are off, and secondary CPUs are still lost in space.
+ */
+void __ipipe_enable_pipeline(void)
+{
+	unsigned long flags;
+	unsigned int irq;
+
+	flags = ipipe_critical_enter(NULL);
+
+	/* First, intercept all interrupts from the root
+	 * domain. Regular Linux interrupt handlers will receive
+	 * raw_cpu_ptr(&ipipe_percpu.tick_regs) for external IRQs,
+	 * whatever cookie is passed here.
+	 */
+	for (irq = 0; irq < NR_IRQS; irq++)
+		ipipe_request_irq(ipipe_root_domain,
+				  irq,
+				  __ipipe_do_IRQ, NULL,
+				  NULL);
+	/*
+	 * We use a virtual IRQ to handle the timer irq (decrementer
+	 * trap) which was allocated early in
+	 * __ipipe_early_core_setup().
+	 */
+	ipipe_request_irq(ipipe_root_domain,
+			  IPIPE_TIMER_VIRQ,
+			  __ipipe_do_timer, NULL,
+			  NULL);
+
+	ipipe_critical_exit(flags);
+}
+
+int ipipe_get_sysinfo(struct ipipe_sysinfo *info)
+{
+	info->sys_nr_cpus = num_online_cpus();
+	info->sys_cpu_freq = __ipipe_cpu_freq;
+	info->sys_hrtimer_irq = per_cpu(ipipe_percpu.hrtimer_irq, 0);
+	info->sys_hrtimer_freq = __ipipe_hrtimer_freq;
+	info->sys_hrclock_freq = __ipipe_hrclock_freq;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipipe_get_sysinfo);
+
+static int __ipipe_exit_irq(struct pt_regs *regs)
+{
+	int root = __ipipe_root_p;
+
+	if (root) {
+#ifdef CONFIG_PPC_970_NAP
+		struct thread_info *ti = current_thread_info();
+		/* Emulate the napping check when 100% sure we do run
+		 * over the root context. */
+		if (test_and_clear_bit(TLF_NAPPING, &ti->local_flags))
+			regs->nip = regs->link;
+#endif
+#ifdef CONFIG_PPC64
+		ppc64_runlatch_on();
+#endif
+	}
+
+	/*
+	 * Testing for user_regs() eliminates foreign stack contexts,
+	 * including from legacy domains (CONFIG_IPIPE_LEGACY) which
+	 * did not set the foreign stack bit (foreign stacks are
+	 * always kernel-based).
+	 */
+	if (user_mode(regs) && ipipe_test_thread_flag(TIP_MAYDAY))
+		__ipipe_call_mayday(regs);
+
+	if (root && !test_bit(IPIPE_STALL_FLAG, &__ipipe_root_status))
+		return 1;
+
+	return 0;
+}
+
+int __ipipe_grab_irq(struct pt_regs *regs)
+{
+	int irq;
+
+	irq = ppc_md.get_irq();
+	if (unlikely(irq == NO_IRQ)) {
+		__get_cpu_var(irq_stat).spurious_irqs++;
+		return __ipipe_exit_irq(regs);
+	}
+
+	if (likely(irq != NO_IRQ)) {
+		ipipe_trace_irq_entry(irq);
+#ifdef CONFIG_SMP
+		/* Check for cascaded I-pipe IPIs */
+		if (irq == __ipipe_ipi_irq)
+			__ipipe_ipi_demux(irq, regs);
+		else
+#endif /* CONFIG_SMP */
+			__ipipe_handle_irq(irq, regs);
+	}
+
+	ipipe_trace_irq_exit(irq);
+
+	return __ipipe_exit_irq(regs);
+}
+
+static void __ipipe_do_IRQ(unsigned int irq, void *cookie)
+{
+	struct pt_regs *regs, *old_regs;
+
+	/* Any sensible register frame will do for non-timer IRQs. */
+	regs = raw_cpu_ptr(&ipipe_percpu.tick_regs);
+	old_regs = set_irq_regs(regs);
+	___do_irq(irq, regs);
+	set_irq_regs(old_regs);
+}
+
+static void __ipipe_do_timer(unsigned int irq, void *cookie)
+{
+	check_stack_overflow();
+	timer_interrupt(raw_cpu_ptr(&ipipe_percpu.tick_regs));
+}
+
+int __ipipe_grab_timer(struct pt_regs *regs)
+{
+	struct pt_regs *tick_regs;
+	struct ipipe_domain *ipd;
+
+	ipd = __ipipe_current_domain;
+
+	set_dec(DECREMENTER_MAX);
+
+	ipipe_trace_irq_entry(IPIPE_TIMER_VIRQ);
+
+	tick_regs = raw_cpu_ptr(&ipipe_percpu.tick_regs);
+	tick_regs->msr = regs->msr;
+	tick_regs->nip = regs->nip;
+	if (ipd != &ipipe_root)
+		/* Tick should not be charged to Linux. */
+		tick_regs->msr &= ~MSR_EE;
+
+	__ipipe_handle_irq(IPIPE_TIMER_VIRQ, NULL);
+
+	ipipe_trace_irq_exit(IPIPE_TIMER_VIRQ);
+
+	return __ipipe_exit_irq(regs);
+}
+
+EXPORT_SYMBOL_GPL(show_stack);
+EXPORT_SYMBOL_GPL(_switch);
+#ifndef CONFIG_SMP
+EXPORT_SYMBOL_GPL(last_task_used_math);
+#endif
+#ifdef CONFIG_IPIPE_LEGACY
+#ifdef CONFIG_PPC64
+EXPORT_PER_CPU_SYMBOL(ppc64_tlb_batch);
+EXPORT_SYMBOL_GPL(switch_slb);
+EXPORT_SYMBOL_GPL(__flush_tlb_pending);
+#else  /* !CONFIG_PPC64 */
+void atomic_set_mask(unsigned long mask, unsigned long *ptr);
+void atomic_clear_mask(unsigned long mask, unsigned long *ptr);
+EXPORT_SYMBOL_GPL(atomic_set_mask);
+EXPORT_SYMBOL_GPL(atomic_clear_mask);
+#endif	/* !CONFIG_PPC64 */
+#endif /* !CONFIG_IPIPE_LEGACY */
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index c143835..82279ee 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -95,6 +95,36 @@ extern int tau_interrupts(int);
 
 int distribute_irqs = 1;
 
+/*
+ * This is specifically called by assembly code to re-enable interrupts
+ * if they are currently disabled. This is typically called before
+ * schedule() or do_signal() when returning to userspace. We do it
+ * in C to avoid the burden of dealing with lockdep etc...
+ *
+ * NOTE: This is called with interrupts hard disabled but not marked
+ * as such in paca->irq_happened, so we need to resync this.
+ */
+void notrace restore_interrupts(void)
+{
+	if (irqs_disabled()) {
+#ifndef CONFIG_IPIPE
+		local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
+#endif
+		local_irq_enable();
+	} else
+		__hard_irq_enable();
+}
+
+#ifndef CONFIG_IPIPE
+
+static inline notrace int decrementer_check_overflow(void)
+{
+ 	u64 now = get_tb_or_rtc();
+	u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
+ 
+	return now >= *next_tb;
+}
+
 static inline notrace unsigned long get_irq_happened(void)
 {
 	unsigned long happened;
@@ -111,14 +141,6 @@ static inline notrace void set_soft_enabled(unsigned long enable)
 	: : "r" (enable), "i" (offsetof(struct paca_struct, soft_enabled)));
 }
 
-static inline notrace int decrementer_check_overflow(void)
-{
- 	u64 now = get_tb_or_rtc();
- 	u64 *next_tb = &__get_cpu_var(decrementers_next_tb);
- 
-	return now >= *next_tb;
-}
-
 /* This is called whenever we are re-enabling interrupts
  * and returns either 0 (nothing to do) or 500/900/280/a00/e80 if
  * there's an EE, DEC or DBELL to generate.
@@ -278,23 +300,7 @@ notrace void arch_local_irq_restore(unsigned long en)
 }
 EXPORT_SYMBOL(arch_local_irq_restore);
 
-/*
- * This is specifically called by assembly code to re-enable interrupts
- * if they are currently disabled. This is typically called before
- * schedule() or do_signal() when returning to userspace. We do it
- * in C to avoid the burden of dealing with lockdep etc...
- *
- * NOTE: This is called with interrupts hard disabled but not marked
- * as such in paca->irq_happened, so we need to resync this.
- */
-void notrace restore_interrupts(void)
-{
-	if (irqs_disabled()) {
-		local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
-		local_irq_enable();
-	} else
-		__hard_irq_enable();
-}
+#endif /* !CONFIG_IPIPE */
 
 /*
  * This is a helper to use when about to go into idle low-power
@@ -329,6 +335,7 @@ bool prep_irq_for_idle(void)
 	/* Tell lockdep we are about to re-enable */
 	trace_hardirqs_on();
 
+#ifndef CONFIG_IPIPE
 	/*
 	 * Mark interrupts as soft-enabled and clear the
 	 * PACA_IRQ_HARD_DIS from the pending mask since we
@@ -337,6 +344,7 @@ bool prep_irq_for_idle(void)
 	 */
 	local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
 	local_paca->soft_enabled = 1;
+#endif
 
 	/* Tell the caller to enter the low power state */
 	return true;
@@ -461,9 +469,10 @@ void migrate_irqs(void)
 }
 #endif
 
-static inline void check_stack_overflow(void)
+#if defined(CONFIG_DEBUG_STACKOVERFLOW) && !defined(CONFIG_IPIPE_LEGACY)
+
+void check_stack_overflow(void)
 {
-#ifdef CONFIG_DEBUG_STACKOVERFLOW
 	long sp;
 
 	sp = current_stack_pointer() & (THREAD_SIZE-1);
@@ -474,26 +483,18 @@ static inline void check_stack_overflow(void)
 			sp - sizeof(struct thread_info));
 		dump_stack();
 	}
-#endif
 }
 
-void __do_irq(struct pt_regs *regs)
-{
-	unsigned int irq;
+#endif
 
+void ___do_irq(unsigned int irq, struct pt_regs *regs)
+{
 	irq_enter();
 
 	trace_irq_entry(regs);
 
 	check_stack_overflow();
 
-	/*
-	 * Query the platform PIC for the interrupt & ack it.
-	 *
-	 * This will typically lower the interrupt line to the CPU
-	 */
-	irq = ppc_md.get_irq();
-
 	/* We can hard enable interrupts now to allow perf interrupts */
 	may_hard_irq_enable();
 
@@ -508,9 +509,25 @@ void __do_irq(struct pt_regs *regs)
 	irq_exit();
 }
 
+void __do_irq(struct pt_regs *regs)
+{
+	unsigned int irq;
+
+	/*
+	 * Query the platform PIC for the interrupt & ack it.
+	 *
+	 * This will typically lower the interrupt line to the CPU
+	 */
+	irq = ppc_md.get_irq();
+	___do_irq(irq, regs);
+}
+
 void do_IRQ(struct pt_regs *regs)
 {
 	struct pt_regs *old_regs = set_irq_regs(regs);
+#ifdef CONFIG_IPIPE
+	__do_irq(regs);
+#else /* !CONFIG_IPIPE */
 	struct thread_info *curtp, *irqtp, *sirqtp;
 
 	/* Switch to the irq stack to handle this */
@@ -541,6 +558,7 @@ void do_IRQ(struct pt_regs *regs)
 	/* Copy back updates to the thread_info */
 	if (irqtp->flags)
 		set_bits(irqtp->flags, &curtp->flags);
+#endif /* !CONFIG_IPIPE */
 
 	set_irq_regs(old_regs);
 }
@@ -596,6 +614,19 @@ void exc_lvl_ctx_init(void)
 }
 #endif
 
+#ifdef CONFIG_IPIPE
+
+/* We don't switch stacks when the pipeline is enabled. */
+
+void irq_ctx_init(void) { }
+
+void do_softirq_own_stack(void)
+{
+	__do_softirq();
+}
+
+#else  /* !CONFIG_IPIPE */
+
 struct thread_info *softirq_ctx[NR_CPUS] __read_mostly;
 struct thread_info *hardirq_ctx[NR_CPUS] __read_mostly;
 
@@ -633,6 +664,8 @@ void do_softirq_own_stack(void)
 		set_bits(irqtp->flags, &curtp->flags);
 }
 
+#endif  /* !CONFIG_IPIPE */
+
 irq_hw_number_t virq_to_hw(unsigned int virq)
 {
 	struct irq_data *irq_data = irq_get_irq_data(virq);
diff --git a/arch/powerpc/kernel/misc_32.S b/arch/powerpc/kernel/misc_32.S
index 7c6bb4b..0a8604e 100644
--- a/arch/powerpc/kernel/misc_32.S
+++ b/arch/powerpc/kernel/misc_32.S
@@ -40,6 +40,7 @@
  * We store the saved ksp_limit in the unused part
  * of the STACK_FRAME_OVERHEAD
  */
+#ifndef CONFIG_IPIPE
 _GLOBAL(call_do_softirq)
 	mflr	r0
 	stw	r0,4(r1)
@@ -76,6 +77,7 @@ _GLOBAL(call_do_irq)
 	stw	r10,THREAD+KSP_LIMIT(r2)
 	mtlr	r0
 	blr
+#endif	
 
 /*
  * This returns the high 64 bits of the product of two 64-bit numbers.
diff --git a/arch/powerpc/kernel/misc_64.S b/arch/powerpc/kernel/misc_64.S
index 4e314b9..cf1555b 100644
--- a/arch/powerpc/kernel/misc_64.S
+++ b/arch/powerpc/kernel/misc_64.S
@@ -29,6 +29,7 @@
 
 	.text
 
+#ifndef CONFIG_IPIPE
 _GLOBAL(call_do_softirq)
 	mflr	r0
 	std	r0,16(r1)
@@ -50,6 +51,7 @@ _GLOBAL(call_do_irq)
 	ld	r0,16(r1)
 	mtlr	r0
 	blr
+#endif	
 
 	.section	".toc","aw"
 PPC64_CACHES:
diff --git a/arch/powerpc/kernel/ppc_ksyms_32.c b/arch/powerpc/kernel/ppc_ksyms_32.c
index 30ddd8a..5165097 100644
--- a/arch/powerpc/kernel/ppc_ksyms_32.c
+++ b/arch/powerpc/kernel/ppc_ksyms_32.c
@@ -23,6 +23,9 @@ EXPORT_SYMBOL(pci_dram_offset);
 
 #ifdef CONFIG_SMP
 EXPORT_SYMBOL(smp_hw_index);
+#ifdef CONFIG_IPIPE_LEGACY
+EXPORT_SYMBOL(smp_logical_index);
+#endif
 #endif
 
 long long __ashrdi3(long long, int);
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 923cd2d..94220ba 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -154,8 +154,12 @@ EXPORT_SYMBOL_GPL(flush_fp_to_thread);
 
 void enable_kernel_fp(void)
 {
+	unsigned long flags;
+
 	WARN_ON(preemptible());
 
+	flags = hard_cond_local_irq_save();
+
 #ifdef CONFIG_SMP
 	if (current->thread.regs && (current->thread.regs->msr & MSR_FP))
 		giveup_fpu_maybe_transactional(current);
@@ -164,6 +168,7 @@ void enable_kernel_fp(void)
 #else
 	giveup_fpu_maybe_transactional(last_task_used_math);
 #endif /* CONFIG_SMP */
+	hard_cond_local_irq_restore(flags);
 }
 EXPORT_SYMBOL(enable_kernel_fp);
 
@@ -631,8 +636,10 @@ void tm_recheckpoint(struct thread_struct *thread,
 	 * change and later in the trecheckpoint code, we have a userspace R1.
 	 * So let's hard disable over this region.
 	 */
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
+#ifndef CONFIG_IPIPE
 	hard_irq_disable();
+#endif
 
 	/* The TM SPRs are restored here, so that TEXASR.FS can be set
 	 * before the trecheckpoint and no explosion occurs.
@@ -641,7 +648,7 @@ void tm_recheckpoint(struct thread_struct *thread,
 
 	__tm_recheckpoint(thread, orig_msr);
 
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 static inline void tm_recheckpoint_new_task(struct task_struct *new)
@@ -753,6 +760,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
 #ifdef CONFIG_PPC_BOOK3S_64
 	struct ppc64_tlb_batch *batch;
 #endif
+	unsigned long flags;
 
 	WARN_ON(!irqs_disabled());
 
@@ -766,6 +774,8 @@ struct task_struct *__switch_to(struct task_struct *prev,
 	 */
 	save_early_sprs(&prev->thread);
 
+	flags = hard_local_irq_save();
+
 	__switch_to_tm(prev);
 
 #ifdef CONFIG_SMP
@@ -880,7 +890,9 @@ struct task_struct *__switch_to(struct task_struct *prev,
 	 * window where the kernel stack SLB and the kernel stack are out
 	 * of sync. Hard disable here.
 	 */
+#ifndef CONFIG_IPIPE
 	hard_irq_disable();
+#endif
 
 	tm_recheckpoint_new_task(new);
 
@@ -894,6 +906,8 @@ struct task_struct *__switch_to(struct task_struct *prev,
 	}
 #endif /* CONFIG_PPC_BOOK3S_64 */
 
+	hard_local_irq_restore(flags);
+
 	return last;
 }
 
@@ -1457,6 +1471,29 @@ int get_unalign_ctl(struct task_struct *tsk, unsigned long adr)
 	return put_user(tsk->thread.align_ctl, (unsigned int __user *)adr);
 }
 
+#ifdef CONFIG_IPIPE
+
+int validate_sp(unsigned long sp, struct task_struct *p,
+		       unsigned long nbytes)
+{
+	unsigned long stack_page = (unsigned long)task_stack_page(p);
+
+	/*
+	 * We can't always know the bounds of the current stack when
+	 * built in legacy mode due to potentially foreign stack
+	 * contexts, so let's pretend the stack pointer is fine
+	 * in this case.
+	 */
+	if (IS_ENABLED(CONFIG_IPIPE_LEGACY) ||
+	    (sp >= stack_page + sizeof(struct thread_struct)
+	     && sp <= stack_page + THREAD_SIZE - nbytes))
+		return 1;
+
+	return 0;
+}
+
+#else /* !CONFIG_IPIPE */
+
 static inline int valid_irq_stack(unsigned long sp, struct task_struct *p,
 				  unsigned long nbytes)
 {
@@ -1493,6 +1530,8 @@ int validate_sp(unsigned long sp, struct task_struct *p,
 	return valid_irq_stack(sp, p, nbytes);
 }
 
+#endif /* !CONFIG_IPIPE */
+
 EXPORT_SYMBOL(validate_sp);
 
 unsigned long get_wchan(struct task_struct *p)
diff --git a/arch/powerpc/kernel/setup_32.c b/arch/powerpc/kernel/setup_32.c
index 07831ed..d0fabed 100644
--- a/arch/powerpc/kernel/setup_32.c
+++ b/arch/powerpc/kernel/setup_32.c
@@ -48,6 +48,9 @@ int boot_cpuid_phys;
 EXPORT_SYMBOL_GPL(boot_cpuid_phys);
 
 int smp_hw_index[NR_CPUS];
+#ifdef CONFIG_IPIPE_LEGACY
+int smp_logical_index[NR_CPUS];
+#endif
 
 unsigned long ISA_DMA_THRESHOLD;
 unsigned int DMA_MODE_READ;
@@ -223,6 +226,9 @@ int __init ppc_init(void)
 
 arch_initcall(ppc_init);
 
+#ifdef CONFIG_IPIPE
+static inline void irqstack_early_init(void) { }
+#else
 static void __init irqstack_early_init(void)
 {
 	unsigned int i;
@@ -236,6 +242,7 @@ static void __init irqstack_early_init(void)
 			__va(memblock_alloc(THREAD_SIZE, THREAD_SIZE));
 	}
 }
+#endif
 
 #if defined(CONFIG_BOOKE) || defined(CONFIG_40x)
 static void __init exc_lvl_early_init(void)
diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c
index 4f3cfe1..8675ad3 100644
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -190,10 +190,14 @@ early_param("smt-enabled", early_smt_enabled);
 /** Fix up paca fields required for the boot cpu */
 static void fixup_boot_paca(void)
 {
-	/* The boot cpu is started */
-	get_paca()->cpu_start = 1;
+	/*
+	 * The boot cpu is started. We use the direct PACA accessor to
+	 * prevent access to uninit I-pipe globals when running the
+	 * debug code w/ CONFIG_DEBUG_PREEMPT.
+	 */
+	local_paca->cpu_start = 1;
 	/* Allow percpu accesses to work until we setup percpu data */
-	get_paca()->data_offset = 0;
+	local_paca->data_offset = 0;
 }
 
 static void cpu_ready_for_interrupts(void)
@@ -314,8 +318,10 @@ void __init early_setup(unsigned long dt_ptr)
 #ifdef CONFIG_SMP
 void early_setup_secondary(void)
 {
+#ifndef CONFIG_IPIPE
 	/* Mark interrupts enabled in PACA */
 	get_paca()->soft_enabled = 0;
+#endif
 
 	/* Initialize the hash table or TLB handling */
 	early_init_mmu_secondary();
@@ -444,6 +450,13 @@ static void __init initialize_cache_info(void)
  */
 void __init setup_system(void)
 {
+#ifdef CONFIG_IPIPE
+	/*
+	 * Early init before per-cpu areas are moved to their final
+	 * location.
+	 */
+	local_paca->ipipe_statp = (u64)&ipipe_percpu_context(&ipipe_root, 0)->status;
+#endif
 	DBG(" -> setup_system()\n");
 
 	/* Apply the CPUs-specific and firmware specific fixups to kernel
@@ -577,6 +590,9 @@ static u64 safe_stack_limit(void)
 #endif
 }
 
+#ifdef CONFIG_IPIPE
+static inline void irqstack_early_init(void) { }
+#else
 static void __init irqstack_early_init(void)
 {
 	u64 limit = safe_stack_limit();
@@ -595,6 +611,7 @@ static void __init irqstack_early_init(void)
 					    THREAD_SIZE, limit));
 	}
 }
+#endif
 
 #ifdef CONFIG_PPC_BOOK3E
 static void __init exc_lvl_early_init(void)
@@ -793,6 +810,10 @@ void __init setup_per_cpu_areas(void)
                 __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
 		paca[cpu].data_offset = __per_cpu_offset[cpu];
 	}
+#ifdef CONFIG_IPIPE
+	/* Reset pointer to the relocated per-cpu root domain data. */
+	local_paca->ipipe_statp = (u64)&ipipe_percpu_context(&ipipe_root, 0)->status;
+#endif
 }
 #endif
 
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index 71e186d..52d3531 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -179,7 +179,7 @@ const char *smp_ipi_name[] = {
 	[PPC_MSG_CALL_FUNCTION] =  "ipi call function",
 	[PPC_MSG_RESCHEDULE] = "ipi reschedule",
 	[PPC_MSG_TICK_BROADCAST] = "ipi tick-broadcast",
-	[PPC_MSG_DEBUGGER_BREAK] = "ipi debugger",
+	[PPC_MSG_DEBUGGER_BREAK] = "ipi I-pipe/debugger",
 };
 
 /* optional function to request ipi, for controllers with >= 4 ipis */
@@ -190,10 +190,10 @@ int smp_request_message_ipi(int virq, int msg)
 	if (msg < 0 || msg > PPC_MSG_DEBUGGER_BREAK) {
 		return -EINVAL;
 	}
-#if !defined(CONFIG_DEBUGGER) && !defined(CONFIG_KEXEC)
-	if (msg == PPC_MSG_DEBUGGER_BREAK) {
-		return 1;
-	}
+#ifdef CONFIG_IPIPE
+	if (msg == PPC_MSG_DEBUGGER_BREAK)
+		/* Piggyback the debugger IPI for the I-pipe. */
+		__ipipe_register_ipi(virq);
 #endif
 	err = request_irq(virq, smp_ipi_action[msg],
 			  IRQF_PERCPU | IRQF_NO_THREAD | IRQF_NO_SUSPEND,
@@ -314,8 +314,12 @@ void smp_send_debugger_break(void)
 		return;
 
 	for_each_online_cpu(cpu)
-		if (cpu != me)
+		if (cpu != me) {
+#ifdef CONFIG_IPIPE
+			cpu_set(cpu, __ipipe_dbrk_pending);
+#endif
 			do_message_pass(cpu, PPC_MSG_DEBUGGER_BREAK);
+		}
 }
 #endif
 
@@ -694,6 +698,9 @@ void start_secondary(void *unused)
 	unsigned int cpu = smp_processor_id();
 	int i, base;
 
+#if defined(CONFIG_IPIPE) && defined(CONFIG_PPC64)
+	local_paca->ipipe_statp = (u64)&ipipe_percpu_context(&ipipe_root, cpu)->status;
+#endif
 	atomic_inc(&init_mm.mm_count);
 	current->active_mm = &init_mm;
 
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index 7505599..9758941 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -54,6 +54,7 @@
 #include <linux/irq.h>
 #include <linux/delay.h>
 #include <linux/irq_work.h>
+#include <linux/ipipe_tickdev.h>
 #include <asm/trace.h>
 
 #include <asm/io.h>
@@ -113,6 +114,9 @@ EXPORT_SYMBOL(decrementer_clockevent);
 
 DEFINE_PER_CPU(u64, decrementers_next_tb);
 static DEFINE_PER_CPU(struct clock_event_device, decrementers);
+#ifdef CONFIG_IPIPE
+static DEFINE_PER_CPU(struct ipipe_timer, itimers);
+#endif /* CONFIG_IPIPE */
 
 #define XSEC_PER_SEC (1024*1024)
 
@@ -494,7 +498,7 @@ static void __timer_interrupt(void)
 	}
 
 	now = get_tb_or_rtc();
-	if (now >= *next_tb) {
+	if (clockevent_ipipe_stolen(evt) || now >= *next_tb) {
 		*next_tb = ~(u64)0;
 		if (evt->event_handler)
 			evt->event_handler(evt);
@@ -528,11 +532,15 @@ void timer_interrupt(struct pt_regs * regs)
 {
 	struct pt_regs *old_regs;
 	u64 *next_tb = &__get_cpu_var(decrementers_next_tb);
+#ifdef CONFIG_IPIPE
+	struct clock_event_device *evt = &__get_cpu_var(decrementers);
+#endif
 
 	/* Ensure a positive value is written to the decrementer, or else
 	 * some CPUs will continue to take decrementer exceptions.
 	 */
-	set_dec(DECREMENTER_MAX);
+	if (!clockevent_ipipe_stolen(evt))
+		set_dec(DECREMENTER_MAX);
 
 	/* Some implementations of hotplug will get timer interrupts while
 	 * offline, just ignore these and we also need to set
@@ -557,10 +565,18 @@ void timer_interrupt(struct pt_regs * regs)
 #endif
 
 	old_regs = set_irq_regs(regs);
+#ifndef CONFIG_IPIPE
+	/*
+	 * The timer interrupt is a virtual one when the I-pipe is
+	 * active, therefore we already called irq_enter() for it (see
+	 * __ipipe_run_isr).
+	 */
 	irq_enter();
-
+#endif
 	__timer_interrupt();
+#ifndef CONFIG_IPIPE
 	irq_exit();
+#endif
 	set_irq_regs(old_regs);
 }
 
@@ -740,8 +756,37 @@ static cycle_t timebase_read(struct clocksource *cs)
 	return (cycle_t)get_tb();
 }
 
+#ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD
+/*
+ * Expect build error once converted to the newest update_vsyscall()
+ * API.
+ */
+static inline void update_hostrt(struct timespec *wall_time, struct timespec *wtm,
+				 struct clocksource *clock, u32 mult, u32 shift)
+{
+	/*
+	 * This is a temporary work-around until powerpc implements
+	 * the latest update_vsyscall() interface. We only fill in the
+	 * timekeeper fields ipipe_update_hostrt() currently uses.
+	 */
+	struct timekeeper tk = {
+		.tkr = {
+			.clock = clock,
+			.shift = clock->shift,
+			.mult = mult,
+			.xtime_nsec = (u64)wall_time->tv_nsec << shift,
+		},
+		.xtime_sec = wall_time->tv_sec,
+		.wall_to_monotonic = timespec_to_timespec64(*wtm),
+	};
+
+	ipipe_update_hostrt(&tk);
+}
+
+#endif
+
 void update_vsyscall_old(struct timespec *wall_time, struct timespec *wtm,
-			 struct clocksource *clock, u32 mult, cycle_t cycle_last)
+			 struct clocksource *clock, u32 mult, u32 shift, cycle_t cycle_last)
 {
 	u64 new_tb_to_xs, new_stamp_xsec;
 	u32 frac_sec;
@@ -783,6 +828,8 @@ void update_vsyscall_old(struct timespec *wall_time, struct timespec *wtm,
 	vdso_data->stamp_sec_fraction = frac_sec;
 	smp_wmb();
 	++(vdso_data->tb_update_count);
+
+	update_hostrt(wall_time, wtm, clock, mult, shift);
 }
 
 void update_vsyscall_tz(void)
@@ -830,6 +877,22 @@ static void decrementer_set_mode(enum clock_event_mode mode,
 		decrementer_set_next_event(DECREMENTER_MAX, dev);
 }
 
+#ifdef CONFIG_IPIPE
+static int itimer_set(unsigned long evt, void *timer)
+{
+#ifndef CONFIG_40x
+	/*
+	 * Decrementer must be set to a positive 32bit value,
+	 * otherwise it would flood us with exceptions.
+	 */
+	if (evt > DECREMENTER_MAX)
+		evt = DECREMENTER_MAX;
+#endif /* CONFIG_40x */
+	set_dec((int)evt);
+	return 0;
+}
+#endif /* CONFIG_IPIPE */
+
 /* Interrupt handler for the timer broadcast IPI */
 void tick_broadcast_ipi_handler(void)
 {
@@ -849,6 +912,13 @@ static void register_decrementer_clockevent(int cpu)
 	printk_once(KERN_DEBUG "clockevent: %s mult[%x] shift[%d] cpu[%d]\n",
 		    dec->name, dec->mult, dec->shift, cpu);
 
+#ifdef CONFIG_IPIPE
+	dec->ipipe_timer = &per_cpu(itimers, cpu);
+	dec->ipipe_timer->irq = IPIPE_TIMER_VIRQ;
+	dec->ipipe_timer->set = itimer_set;
+	dec->ipipe_timer->min_delay_ticks = 3;
+#endif /* CONFIG_IPIPE */
+
 	clockevents_register_device(dec);
 }
 
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index 0dc43f9..c08f972 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -702,6 +702,9 @@ void machine_check_exception(struct pt_regs *regs)
 
 	__get_cpu_var(irq_stat).mce_exceptions++;
 
+	if (__ipipe_report_trap(IPIPE_TRAP_MCE, regs))
+	    	return;
+
 	/* See if any machine dependent calls. In theory, we would want
 	 * to call the CPU first, and call the ppc_md. one if the CPU
 	 * one returns a positive number. However there is existing code
@@ -769,6 +772,9 @@ void unknown_exception(struct pt_regs *regs)
 	printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n",
 	       regs->nip, regs->msr, regs->trap);
 
+	if (__ipipe_report_trap(IPIPE_TRAP_UNKNOWN, regs))
+	    	return;
+
 	_exception(SIGTRAP, regs, 0, 0);
 
 	exception_exit(prev_state);
@@ -778,6 +784,9 @@ void instruction_breakpoint_exception(struct pt_regs *regs)
 {
 	enum ctx_state prev_state = exception_enter();
 
+	if (__ipipe_report_trap(IPIPE_TRAP_IABR, regs))
+	    	return;
+
 	if (notify_die(DIE_IABR_MATCH, "iabr_match", regs, 5,
 					5, SIGTRAP) == NOTIFY_STOP)
 		goto bail;
@@ -791,6 +800,8 @@ bail:
 
 void RunModeException(struct pt_regs *regs)
 {
+	if (__ipipe_report_trap(IPIPE_TRAP_RM, regs))
+	    	return;
 	_exception(SIGTRAP, regs, 0, 0);
 }
 
@@ -800,6 +811,9 @@ void __kprobes single_step_exception(struct pt_regs *regs)
 
 	clear_single_step(regs);
 
+	if (__ipipe_report_trap(IPIPE_TRAP_SSTEP, regs))
+	    return;
+
 	if (notify_die(DIE_SSTEP, "single_step", regs, 5,
 					5, SIGTRAP) == NOTIFY_STOP)
 		goto bail;
@@ -1140,6 +1154,9 @@ void __kprobes program_check_exception(struct pt_regs *regs)
 	/* We can now get here via a FP Unavailable exception if the core
 	 * has no FPU, in that case the reason flags will be 0 */
 
+	if (__ipipe_report_trap(IPIPE_TRAP_PCE, regs))
+	    	return;
+
 	if (reason & REASON_FP) {
 		/* IEEE FP exception */
 		parse_fpe(regs);
@@ -1264,6 +1281,9 @@ void alignment_exception(struct pt_regs *regs)
 	if (!arch_irq_disabled_regs(regs))
 		local_irq_enable();
 
+	if (__ipipe_report_trap(IPIPE_TRAP_ALIGNMENT, regs))
+	    	return;
+
 	if (tm_abort_check(regs, TM_CAUSE_ALIGNMENT | TM_CAUSE_PERSISTENT))
 		goto bail;
 
@@ -1307,6 +1327,8 @@ void nonrecoverable_exception(struct pt_regs *regs)
 {
 	printk(KERN_ERR "Non-recoverable exception at PC=%lx MSR=%lx\n",
 	       regs->nip, regs->msr);
+	if (__ipipe_report_trap(IPIPE_TRAP_NREC, regs))
+	    	return;
 	debugger(regs);
 	die("nonrecoverable exception", regs, SIGKILL);
 }
@@ -1324,6 +1346,8 @@ void kernel_fp_unavailable_exception(struct pt_regs *regs)
 
 	printk(KERN_EMERG "Unrecoverable FP Unavailable Exception "
 			  "%lx at %lx\n", regs->trap, regs->nip);
+	if (__ipipe_report_trap(IPIPE_TRAP_KFPUNAVAIL, regs))
+	    	return;
 	die("Unrecoverable FP Unavailable Exception", regs, SIGABRT);
 
 	exception_exit(prev_state);
@@ -1333,6 +1357,9 @@ void altivec_unavailable_exception(struct pt_regs *regs)
 {
 	enum ctx_state prev_state = exception_enter();
 
+	if (__ipipe_report_trap(IPIPE_TRAP_ALTUNAVAIL, regs))
+	    	return;
+
 	if (user_mode(regs)) {
 		/* A user program has executed an altivec instruction,
 		   but this kernel doesn't support altivec. */
@@ -1527,6 +1554,10 @@ void performance_monitor_exception(struct pt_regs *regs)
 #ifdef CONFIG_8xx
 void SoftwareEmulation(struct pt_regs *regs)
 {
+
+	if (__ipipe_report_trap(IPIPE_TRAP_SOFTEMU, regs))
+	    	return;
+
 	CHECK_FULL_REGS(regs);
 
 	if (!user_mode(regs)) {
@@ -1604,6 +1635,9 @@ static void handle_debug(struct pt_regs *regs, unsigned long debug_status)
 
 void __kprobes DebugException(struct pt_regs *regs, unsigned long debug_status)
 {
+ 	if (__ipipe_report_trap(IPIPE_TRAP_DEBUG, regs))
+ 	    	return;
+
 	current->thread.debug.dbsr = debug_status;
 
 	/* Hack alert: On BookE, Branch Taken stops on the branch itself, while
@@ -1678,6 +1712,9 @@ void altivec_assist_exception(struct pt_regs *regs)
 {
 	int err;
 
+	if (__ipipe_report_trap(IPIPE_TRAP_ALTASSIST, regs))
+	    	return;
+
 	if (!user_mode(regs)) {
 		printk(KERN_EMERG "VMX/Altivec assist exception in kernel mode"
 		       " at %lx\n", regs->nip);
@@ -1730,8 +1767,11 @@ void CacheLockingException(struct pt_regs *regs, unsigned long address,
 	 * as priv ops, in the future we could try to do
 	 * something smarter
 	 */
-	if (error_code & (ESR_DLK|ESR_ILK))
+	if (error_code & (ESR_DLK|ESR_ILK)) {
+		if (__ipipe_report_trap(IPIPE_TRAP_CACHE, regs))
+			return;
 		_exception(SIGILL, regs, ILL_PRVOPC, regs->nip);
+	}
 	return;
 }
 #endif /* CONFIG_FSL_BOOKE */
@@ -1790,6 +1830,9 @@ void SPEFloatingPointRoundException(struct pt_regs *regs)
 	extern int speround_handler(struct pt_regs *regs);
 	int err;
 
+	if (__ipipe_report_trap(IPIPE_TRAP_SPE, regs))
+	    	return;
+
 	preempt_disable();
 	if (regs->msr & MSR_SPE)
 		giveup_spe(current);
@@ -1827,6 +1870,8 @@ void unrecoverable_exception(struct pt_regs *regs)
 {
 	printk(KERN_EMERG "Unrecoverable exception %lx at %lx\n",
 	       regs->trap, regs->nip);
+	if (__ipipe_report_trap(IPIPE_TRAP_NREC, regs))
+	    	return;
 	die("Unrecoverable exception", regs, SIGABRT);
 }
 
diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c
index d5edbeb..690ba26 100644
--- a/arch/powerpc/lib/code-patching.c
+++ b/arch/powerpc/lib/code-patching.c
@@ -16,6 +16,7 @@
 #include <asm/uaccess.h>
 
 
+notrace
 int patch_instruction(unsigned int *addr, unsigned int instr)
 {
 	int err;
@@ -27,11 +28,13 @@ int patch_instruction(unsigned int *addr, unsigned int instr)
 	return 0;
 }
 
+notrace
 int patch_branch(unsigned int *addr, unsigned long target, int flags)
 {
 	return patch_instruction(addr, create_branch(addr, target, flags));
 }
 
+notrace
 unsigned int create_branch(const unsigned int *addr,
 			   unsigned long target, int flags)
 {
@@ -52,6 +55,7 @@ unsigned int create_branch(const unsigned int *addr,
 	return instruction;
 }
 
+notrace
 unsigned int create_cond_branch(const unsigned int *addr,
 				unsigned long target, int flags)
 {
@@ -72,21 +76,25 @@ unsigned int create_cond_branch(const unsigned int *addr,
 	return instruction;
 }
 
+notrace
 static unsigned int branch_opcode(unsigned int instr)
 {
 	return (instr >> 26) & 0x3F;
 }
 
+notrace
 static int instr_is_branch_iform(unsigned int instr)
 {
 	return branch_opcode(instr) == 18;
 }
 
+notrace
 static int instr_is_branch_bform(unsigned int instr)
 {
 	return branch_opcode(instr) == 16;
 }
 
+notrace
 int instr_is_relative_branch(unsigned int instr)
 {
 	if (instr & BRANCH_ABSOLUTE)
@@ -95,6 +103,7 @@ int instr_is_relative_branch(unsigned int instr)
 	return instr_is_branch_iform(instr) || instr_is_branch_bform(instr);
 }
 
+notrace
 static unsigned long branch_iform_target(const unsigned int *instr)
 {
 	signed long imm;
@@ -111,6 +120,7 @@ static unsigned long branch_iform_target(const unsigned int *instr)
 	return (unsigned long)imm;
 }
 
+notrace
 static unsigned long branch_bform_target(const unsigned int *instr)
 {
 	signed long imm;
@@ -127,6 +137,7 @@ static unsigned long branch_bform_target(const unsigned int *instr)
 	return (unsigned long)imm;
 }
 
+notrace
 unsigned long branch_target(const unsigned int *instr)
 {
 	if (instr_is_branch_iform(*instr))
@@ -137,6 +148,7 @@ unsigned long branch_target(const unsigned int *instr)
 	return 0;
 }
 
+notrace
 int instr_is_branch_to_addr(const unsigned int *instr, unsigned long addr)
 {
 	if (instr_is_branch_iform(*instr) || instr_is_branch_bform(*instr))
@@ -145,6 +157,7 @@ int instr_is_branch_to_addr(const unsigned int *instr, unsigned long addr)
 	return 0;
 }
 
+notrace
 unsigned int translate_branch(const unsigned int *dest, const unsigned int *src)
 {
 	unsigned long target;
diff --git a/arch/powerpc/lib/feature-fixups.c b/arch/powerpc/lib/feature-fixups.c
index 7ce3870..c447b8d 100644
--- a/arch/powerpc/lib/feature-fixups.c
+++ b/arch/powerpc/lib/feature-fixups.c
@@ -31,6 +31,7 @@ struct fixup_entry {
 	long		alt_end_off;
 };
 
+notrace
 static unsigned int *calc_addr(struct fixup_entry *fcur, long offset)
 {
 	/*
@@ -41,6 +42,7 @@ static unsigned int *calc_addr(struct fixup_entry *fcur, long offset)
 	return (unsigned int *)((unsigned long)fcur + offset);
 }
 
+notrace
 static int patch_alt_instruction(unsigned int *src, unsigned int *dest,
 				 unsigned int *alt_start, unsigned int *alt_end)
 {
@@ -64,6 +66,7 @@ static int patch_alt_instruction(unsigned int *src, unsigned int *dest,
 	return 0;
 }
 
+notrace
 static int patch_feature_section(unsigned long value, struct fixup_entry *fcur)
 {
 	unsigned int *start, *end, *alt_start, *alt_end, *src, *dest;
@@ -93,6 +96,7 @@ static int patch_feature_section(unsigned long value, struct fixup_entry *fcur)
 	return 0;
 }
 
+notrace
 void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end)
 {
 	struct fixup_entry *fcur, *fend;
@@ -113,6 +117,7 @@ void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end)
 	}
 }
 
+notrace
 void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end)
 {
 	long *start, *end;
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index f06b56b..8259031 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -209,9 +209,9 @@ static int mm_fault_error(struct pt_regs *regs, unsigned long addr, int fault)
 int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,
 			    unsigned long error_code)
 {
-	enum ctx_state prev_state = exception_enter();
+	enum ctx_state prev_state;
 	struct vm_area_struct * vma;
-	struct mm_struct *mm = current->mm;
+	struct mm_struct *mm;
 	unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
 	int code = SEGV_MAPERR;
 	int is_write = 0;
@@ -220,6 +220,13 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,
 	int fault;
 	int rc = 0, store_update_sp = 0;
 
+	prev_state = exception_enter();
+
+	if (__ipipe_report_trap(IPIPE_TRAP_ACCESS, regs))
+		return 0;
+
+	mm = current->mm;
+
 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
 	/*
 	 * Fortunately the bit assignments in SRR1 for an instruction
diff --git a/arch/powerpc/mm/hash_low_32.S b/arch/powerpc/mm/hash_low_32.S
index 115347f..dce0f48 100644
--- a/arch/powerpc/mm/hash_low_32.S
+++ b/arch/powerpc/mm/hash_low_32.S
@@ -496,7 +496,11 @@ htab_hash_searches:
  *
  * We assume that there is a hash table in use (Hash != 0).
  */
+#ifdef CONFIG_IPIPE
+_GLOBAL(__flush_hash_pages)
+#else
 _GLOBAL(flush_hash_pages)
+#endif
 	tophys(r7,0)
 
 	/*
@@ -531,18 +535,9 @@ _GLOBAL(flush_hash_pages)
 	addi	r6,r6,-1
 	b	1b
 
-	/* Convert context and va to VSID */
-2:	mulli	r3,r3,897*16		/* multiply context by context skew */
-	rlwinm	r0,r4,4,28,31		/* get ESID (top 4 bits of va) */
-	mulli	r0,r0,0x111		/* multiply by ESID skew */
-	add	r3,r3,r0		/* note code below trims to 24 bits */
-
-	/* Construct the high word of the PPC-style PTE (r11) */
-	rlwinm	r11,r3,7,1,24		/* put VSID in 0x7fffff80 bits */
-	rlwimi	r11,r4,10,26,31		/* put in API (abbrev page index) */
-	SET_V(r11)			/* set V (valid) bit */
-
+2:
 #ifdef CONFIG_SMP
+	li	r11,0
 	addis	r9,r7,mmu_hash_lock@ha
 	addi	r9,r9,mmu_hash_lock@l
 	CURRENT_THREAD_INFO(r8, r1)
@@ -557,10 +552,36 @@ _GLOBAL(flush_hash_pages)
 11:	lwz	r0,0(r9)
 	cmpi	0,r0,0
 	beq	10b
+	mtmsr	r10
+	SYNC_601
+	isync
+	li	r11,1
+	rlwinm	r0,r10,0,17,15		/* clear bit 16 (MSR_EE) */
+	rlwinm	r0,r0,0,28,26		/* clear MSR_DR */
+	mtmsr	r0
+	SYNC_601
+	isync
 	b	11b
 12:	isync
+	cmpwi	r11,0
+	beq	13f
+	li	r0,0
+	stw	r0,0(r9)		/* clear mmu_hash_lock */
+	b	1b
+13:
 #endif
 
+	/* Convert context and va to VSID */
+	mulli	r3,r3,897*16		/* multiply context by context skew */
+	rlwinm	r0,r4,4,28,31		/* get ESID (top 4 bits of va) */
+	mulli	r0,r0,0x111		/* multiply by ESID skew */
+	add	r3,r3,r0		/* note code below trims to 24 bits */
+
+	/* Construct the high word of the PPC-style PTE (r11) */
+	rlwinm	r11,r3,7,1,24		/* put VSID in 0x7fffff80 bits */
+	rlwimi	r11,r4,10,26,31		/* put in API (abbrev page index) */
+	SET_V(r11)			/* set V (valid) bit */
+
 	/*
 	 * Check the _PAGE_HASHPTE bit in the linux PTE.  If it is
 	 * already clear, we're done (for this pte).  If not,
@@ -631,7 +652,7 @@ _GLOBAL(flush_hash_patch_B)
 
 19:	mtmsr	r10
 	SYNC_601
-	isync
+	sync
 	blr
 
 /*
diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index ae4962a..289de72 100644
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -196,7 +196,7 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
 			unsigned long vflags, int psize, int apsize, int ssize)
 {
 	struct hash_pte *hptep = htab_address + hpte_group;
-	unsigned long hpte_v, hpte_r;
+	unsigned long hpte_v, hpte_r, flags;
 	int i;
 
 	if (!(vflags & HPTE_V_BOLTED)) {
@@ -205,6 +205,8 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
 			hpte_group, vpn, pa, rflags, vflags, psize);
 	}
 
+	flags = hard_local_irq_save();
+
 	for (i = 0; i < HPTES_PER_GROUP; i++) {
 		if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID)) {
 			/* retry with lock held */
@@ -217,8 +219,28 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
 		hptep++;
 	}
 
-	if (i == HPTES_PER_GROUP)
+	if (i == HPTES_PER_GROUP) {
+		hard_local_irq_restore(flags);
 		return -1;
+	}
+
+#ifdef CONFIG_PPC_PASEMI_A2_WORKAROUNDS
+	/* Workaround for bug 4910: No non-guarded access over IOB */
+	if (pa >= 0x80000000 && pa < 0x100000000)
+		rflags |= _PAGE_GUARDED;
+#endif
+
+#ifdef CONFIG_PPC_PASEMI_A2_WORKAROUNDS
+	/* Workaround for bug 4910: No non-guarded access over IOB */
+	if (pa >= 0x80000000 && pa < 0x100000000)
+		rflags |= _PAGE_GUARDED;
+#endif
+
+#ifdef CONFIG_PPC_PASEMI_A2_WORKAROUNDS
+	/* Workaround for bug 4910: No non-guarded access over IOB */
+	if (pa >= 0x80000000 && pa < 0x100000000)
+		rflags |= _PAGE_GUARDED;
+#endif
 
 	hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
 	hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
@@ -237,6 +259,8 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
 	 */
 	hptep->v = cpu_to_be64(hpte_v);
 
+	hard_local_irq_restore(flags);
+
 	__asm__ __volatile__ ("ptesync" : : : "memory");
 
 	return i | (!!(vflags & HPTE_V_SECONDARY) << 3);
@@ -247,13 +271,15 @@ static long native_hpte_remove(unsigned long hpte_group)
 	struct hash_pte *hptep;
 	int i;
 	int slot_offset;
-	unsigned long hpte_v;
+	unsigned long hpte_v, flags;
 
 	DBG_LOW("    remove(group=%lx)\n", hpte_group);
 
 	/* pick a random entry to start at */
 	slot_offset = mftb() & 0x7;
 
+	flags = hard_local_irq_save();
+
 	for (i = 0; i < HPTES_PER_GROUP; i++) {
 		hptep = htab_address + hpte_group + slot_offset;
 		hpte_v = be64_to_cpu(hptep->v);
@@ -272,12 +298,16 @@ static long native_hpte_remove(unsigned long hpte_group)
 		slot_offset &= 0x7;
 	}
 
-	if (i == HPTES_PER_GROUP)
+	if (i == HPTES_PER_GROUP) {
+		hard_local_irq_restore(flags);
 		return -1;
+	}
 
 	/* Invalidate the hpte. NOTE: this also unlocks it */
 	hptep->v = 0;
 
+	hard_local_irq_restore(flags);
+
 	return i;
 }
 
@@ -286,7 +316,7 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
 				 int apsize, int ssize, int local)
 {
 	struct hash_pte *hptep = htab_address + slot;
-	unsigned long hpte_v, want_v;
+	unsigned long hpte_v, want_v, flags;
 	int ret = 0;
 
 	want_v = hpte_encode_avpn(vpn, bpsize, ssize);
@@ -294,6 +324,8 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
 	DBG_LOW("    update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)",
 		vpn, want_v & HPTE_V_AVPN, slot, newpp);
 
+	flags = hard_local_irq_save();
+
 	native_lock_hpte(hptep);
 
 	hpte_v = be64_to_cpu(hptep->v);
@@ -315,6 +347,8 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
 	}
 	native_unlock_hpte(hptep);
 
+	hard_local_irq_restore(flags);
+
 	/* Ensure it is out of the tlb too. */
 	tlbie(vpn, bpsize, apsize, ssize, local);
 
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index d5339a3..f6712c7 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -117,7 +117,7 @@ int mmu_ci_restrictions;
 #ifdef CONFIG_DEBUG_PAGEALLOC
 static u8 *linear_map_hash_slots;
 static unsigned long linear_map_hash_count;
-static DEFINE_SPINLOCK(linear_map_hash_lock);
+static IPIPE_DEFINE_SPINLOCK(linear_map_hash_lock);
 #endif /* CONFIG_DEBUG_PAGEALLOC */
 
 /* There are definitions of page sizes arrays to be used when none
@@ -970,6 +970,10 @@ void hash_failure_debug(unsigned long ea, unsigned long access,
 static void check_paca_psize(unsigned long ea, struct mm_struct *mm,
 			     int psize, bool user_region)
 {
+	unsigned long flags;
+	
+	flags = hard_local_irq_save();
+
 	if (user_region) {
 		if (psize != get_paca_psize(ea)) {
 			get_paca()->context = mm->context;
@@ -981,6 +985,8 @@ static void check_paca_psize(unsigned long ea, struct mm_struct *mm,
 			mmu_psize_defs[mmu_vmalloc_psize].sllp;
 		slb_vmalloc_update();
 	}
+
+	hard_local_irq_restore(flags);
 }
 
 /* Result code is:
@@ -999,6 +1005,7 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea, unsigned long access, u
 	const struct cpumask *tmp;
 	int rc, user_region = 0, local = 0;
 	int psize, ssize;
+	unsigned long flags __maybe_unused;
 
 	DBG_LOW("hash_page(ea=%016lx, access=%lx, trap=%lx\n",
 		ea, access, trap);
@@ -1227,7 +1234,7 @@ void hash_preload(struct mm_struct *mm, unsigned long ea,
 	 * Hash doesn't like irqs. Walking linux page table with irq disabled
 	 * saves us from holding multiple locks.
 	 */
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 
 	/*
 	 * THP pages use update_mmu_cache_pmd. We don't do
@@ -1271,7 +1278,7 @@ void hash_preload(struct mm_struct *mm, unsigned long ea,
 				   mm->context.user_psize,
 				   pte_val(*ptep));
 out_exit:
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 /* WARNING: This is called from hash_low_64.S, if you change this prototype,
@@ -1338,6 +1345,10 @@ void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc)
 {
 	enum ctx_state prev_state = exception_enter();
 
+	if (__ipipe_report_trap(IPIPE_TRAP_ACCESS, regs))
+		/* Not all access faults go through do_page_fault(). */
+	    	return;
+
 	if (user_mode(regs)) {
 #ifdef CONFIG_PPC_SUBPAGE_PROT
 		if (rc == -2)
@@ -1437,7 +1448,7 @@ void kernel_map_pages(struct page *page, int numpages, int enable)
 	unsigned long flags, vaddr, lmi;
 	int i;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	for (i = 0; i < numpages; i++, page++) {
 		vaddr = (unsigned long)page_address(page);
 		lmi = __pa(vaddr) >> PAGE_SHIFT;
@@ -1448,7 +1459,7 @@ void kernel_map_pages(struct page *page, int numpages, int enable)
 		else
 			kernel_unmap_linear_page(vaddr, lmi);
 	}
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 #endif /* CONFIG_DEBUG_PAGEALLOC */
 
diff --git a/arch/powerpc/mm/mmu_context_nohash.c b/arch/powerpc/mm/mmu_context_nohash.c
index 928ebe7..1c4d33d 100644
--- a/arch/powerpc/mm/mmu_context_nohash.c
+++ b/arch/powerpc/mm/mmu_context_nohash.c
@@ -57,7 +57,7 @@ static unsigned int next_context, nr_free_contexts;
 static unsigned long *context_map;
 static unsigned long *stale_map[NR_CPUS];
 static struct mm_struct **context_mm;
-static DEFINE_RAW_SPINLOCK(context_lock);
+static IPIPE_DEFINE_RAW_SPINLOCK(context_lock);
 
 #define CTX_MAP_SIZE	\
 	(sizeof(unsigned long) * (last_context / BITS_PER_LONG + 1))
@@ -141,7 +141,7 @@ static unsigned int steal_context_smp(unsigned int id)
 static unsigned int steal_context_up(unsigned int id)
 {
 	struct mm_struct *mm;
-	int cpu = smp_processor_id();
+	int cpu = ipipe_processor_id();
 
 	/* Pick up the victim mm */
 	mm = context_mm[id];
@@ -193,7 +193,7 @@ static void context_check_map(void) { }
 
 void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
 {
-	unsigned int i, id, cpu = smp_processor_id();
+	unsigned int i, id, cpu = ipipe_processor_id();
 	unsigned long *map;
 
 	/* No lockless fast path .. yet */
@@ -203,6 +203,7 @@ void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
 		cpu, next, next->context.active, next->context.id);
 
 #ifdef CONFIG_SMP
+	WARN_ON(!hard_irqs_disabled());
 	/* Mark us active and the previous one not anymore */
 	next->context.active++;
 	if (prev) {
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index 6e450ca..b54fb05 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -133,9 +133,13 @@ static void __slb_flush_and_rebolt(void)
 
 void slb_flush_and_rebolt(void)
 {
+	unsigned long flags;
 
-	WARN_ON(!irqs_disabled());
-
+#ifdef CONFIG_IPIPE
+	flags = hard_local_save_flags();
+#else
+ 	WARN_ON(!irqs_disabled());
+#endif
 	/*
 	 * We can't take a PMU exception in the following code, so hard
 	 * disable interrupts.
@@ -144,6 +148,7 @@ void slb_flush_and_rebolt(void)
 
 	__slb_flush_and_rebolt();
 	get_paca()->slb_cache_ptr = 0;
+	hard_cond_local_irq_restore(flags);
 }
 
 void slb_vmalloc_update(void)
@@ -192,6 +197,12 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
 	unsigned long pc = KSTK_EIP(tsk);
 	unsigned long stack = KSTK_ESP(tsk);
 	unsigned long exec_base;
+	unsigned long flags;
+#ifdef CONFIG_IPIPE
+	flags = hard_local_save_flags();
+#else
+ 	WARN_ON(!irqs_disabled());
+#endif
 
 	/*
 	 * We need interrupts hard-disabled here, not just soft-disabled,
@@ -225,6 +236,7 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
 	get_paca()->slb_cache_ptr = 0;
 	get_paca()->context = mm->context;
 
+	hard_cond_local_irq_restore(flags);
 	/*
 	 * preload some userspace segments into the SLB.
 	 * Almost all 32 and 64bit PowerPC executables are linked at
diff --git a/arch/powerpc/mm/tlb_hash32.c b/arch/powerpc/mm/tlb_hash32.c
index 558e30c..66e4d08 100644
--- a/arch/powerpc/mm/tlb_hash32.c
+++ b/arch/powerpc/mm/tlb_hash32.c
@@ -87,6 +87,37 @@ void tlb_flush(struct mmu_gather *tlb)
  *    -- Cort
  */
 
+#ifdef CONFIG_IPIPE
+
+int __flush_hash_pages(unsigned context, unsigned long va,
+		       unsigned long pmdval, int count);
+
+int flush_hash_pages(unsigned context, unsigned long va,
+		     unsigned long pmdval, int count)
+{
+	int bulk, ret = 0;
+	/*
+	 * Submitting flush requests on insanely large PTE counts
+	 * (e.g. HIGHMEM) may cause severe latency penalty on high
+	 * priority domains since this must be done with hw interrupts
+	 * off (typically, peaks over 400 us have been observed on
+	 * 864xD). We split flush requests in bulks of 64 PTEs to
+	 * prevent that; the modified assembly helper which performs
+	 * the actual flush (__flush_hash_pages()) will spin on the
+	 * mmu_lock with interrupts enabled to further reduce latency.
+	 */
+	while (count > 0) {
+		bulk = count > 64 ? 64 : count;
+		ret |= __flush_hash_pages(context, va, pmdval, bulk);
+		va += (bulk << PAGE_SHIFT);
+		count -= bulk;
+	}
+
+	return ret;
+}
+
+#endif /* CONFIG_IPIPE */
+
 static void flush_range(struct mm_struct *mm, unsigned long start,
 			unsigned long end)
 {
diff --git a/arch/powerpc/platforms/52xx/mpc52xx_pic.c b/arch/powerpc/platforms/52xx/mpc52xx_pic.c
index 2944bc8..2f305f5 100644
--- a/arch/powerpc/platforms/52xx/mpc52xx_pic.c
+++ b/arch/powerpc/platforms/52xx/mpc52xx_pic.c
@@ -142,35 +142,72 @@ static unsigned char mpc52xx_map_senses[4] = {
 };
 
 /* Utility functions */
-static inline void io_be_setbit(u32 __iomem *addr, int bitno)
+static inline void __io_be_setbit(u32 __iomem *addr, int bitno)
 {
 	out_be32(addr, in_be32(addr) | (1 << bitno));
 }
 
-static inline void io_be_clrbit(u32 __iomem *addr, int bitno)
+static inline void io_be_setbit(u32 __iomem *addr, int bitno)
+{
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	__io_be_setbit(addr, bitno);
+	hard_local_irq_restore(flags);
+}
+
+static inline void __io_be_clrbit(u32 __iomem *addr, int bitno)
 {
 	out_be32(addr, in_be32(addr) & ~(1 << bitno));
 }
 
+static inline void io_be_clrbit(u32 __iomem *addr, int bitno)
+{
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	__io_be_clrbit(addr, bitno);
+	hard_local_irq_restore(flags);
+}
+
 /*
  * IRQ[0-3] interrupt irq_chip
  */
 static void mpc52xx_extirq_mask(struct irq_data *d)
 {
 	int l2irq = irqd_to_hwirq(d) & MPC52xx_IRQ_L2_MASK;
-	io_be_clrbit(&intr->ctrl, 11 - l2irq);
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	__io_be_clrbit(&intr->ctrl, 11 - l2irq);
+	ipipe_lock_irq(d->irq);
+	hard_local_irq_restore(flags);
 }
 
 static void mpc52xx_extirq_unmask(struct irq_data *d)
 {
 	int l2irq = irqd_to_hwirq(d) & MPC52xx_IRQ_L2_MASK;
-	io_be_setbit(&intr->ctrl, 11 - l2irq);
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	__io_be_setbit(&intr->ctrl, 11 - l2irq);
+	ipipe_unlock_irq(d->irq);
+	hard_local_irq_restore(flags);
+}
+
+static void mpc52xx_extirq_mask_ack(struct irq_data *d)
+{
+	int l2irq = irqd_to_hwirq(d) & MPC52xx_IRQ_L2_MASK;
+
+	__io_be_clrbit(&intr->ctrl, 11 - l2irq);
+	__io_be_setbit(&intr->ctrl, 27 - l2irq);
 }
 
 static void mpc52xx_extirq_ack(struct irq_data *d)
 {
 	int l2irq = irqd_to_hwirq(d) & MPC52xx_IRQ_L2_MASK;
-	io_be_setbit(&intr->ctrl, 27-l2irq);
+
+	__io_be_setbit(&intr->ctrl, 27 - l2irq);
 }
 
 static int mpc52xx_extirq_set_type(struct irq_data *d, unsigned int flow_type)
@@ -205,6 +242,7 @@ static struct irq_chip mpc52xx_extirq_irqchip = {
 	.name = "MPC52xx External",
 	.irq_mask = mpc52xx_extirq_mask,
 	.irq_unmask = mpc52xx_extirq_unmask,
+	.irq_mask_ack = mpc52xx_extirq_mask_ack,
 	.irq_ack = mpc52xx_extirq_ack,
 	.irq_set_type = mpc52xx_extirq_set_type,
 };
@@ -217,22 +255,38 @@ static int mpc52xx_null_set_type(struct irq_data *d, unsigned int flow_type)
 	return 0; /* Do nothing so that the sense mask will get updated */
 }
 
+static void mpc52xx_main_mask_ack(struct irq_data *d)
+{
+	int l2irq = irqd_to_hwirq(d) & MPC52xx_IRQ_L2_MASK;
+	__io_be_setbit(&intr->main_mask, 16 - l2irq);
+}
+
 static void mpc52xx_main_mask(struct irq_data *d)
 {
 	int l2irq = irqd_to_hwirq(d) & MPC52xx_IRQ_L2_MASK;
-	io_be_setbit(&intr->main_mask, 16 - l2irq);
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	__io_be_setbit(&intr->main_mask, 16 - l2irq);
+	ipipe_lock_irq(d->irq);
+	hard_local_irq_restore(flags);
 }
 
 static void mpc52xx_main_unmask(struct irq_data *d)
 {
 	int l2irq = irqd_to_hwirq(d) & MPC52xx_IRQ_L2_MASK;
-	io_be_clrbit(&intr->main_mask, 16 - l2irq);
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	__io_be_clrbit(&intr->main_mask, 16 - l2irq);
+	ipipe_unlock_irq(d->irq);
+	hard_local_irq_restore(flags);
 }
 
 static struct irq_chip mpc52xx_main_irqchip = {
 	.name = "MPC52xx Main",
 	.irq_mask = mpc52xx_main_mask,
-	.irq_mask_ack = mpc52xx_main_mask,
+	.irq_mask_ack = mpc52xx_main_mask_ack,
 	.irq_unmask = mpc52xx_main_unmask,
 	.irq_set_type = mpc52xx_null_set_type,
 };
@@ -240,22 +294,38 @@ static struct irq_chip mpc52xx_main_irqchip = {
 /*
  * Peripherals interrupt irq_chip
  */
+static void mpc52xx_periph_mask_ack(struct irq_data *d)
+{
+	int l2irq = irqd_to_hwirq(d) & MPC52xx_IRQ_L2_MASK;
+	__io_be_setbit(&intr->per_mask, 31 - l2irq);
+}
+
 static void mpc52xx_periph_mask(struct irq_data *d)
 {
 	int l2irq = irqd_to_hwirq(d) & MPC52xx_IRQ_L2_MASK;
-	io_be_setbit(&intr->per_mask, 31 - l2irq);
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	__io_be_setbit(&intr->per_mask, 31 - l2irq);
+	ipipe_lock_irq(d->irq);
+	hard_local_irq_restore(flags);
 }
 
 static void mpc52xx_periph_unmask(struct irq_data *d)
 {
 	int l2irq = irqd_to_hwirq(d) & MPC52xx_IRQ_L2_MASK;
-	io_be_clrbit(&intr->per_mask, 31 - l2irq);
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	__io_be_clrbit(&intr->per_mask, 31 - l2irq);
+	ipipe_unlock_irq(d->irq);
+	hard_local_irq_restore(flags);
 }
 
 static struct irq_chip mpc52xx_periph_irqchip = {
 	.name = "MPC52xx Peripherals",
 	.irq_mask = mpc52xx_periph_mask,
-	.irq_mask_ack = mpc52xx_periph_mask,
+	.irq_mask_ack = mpc52xx_periph_mask_ack,
 	.irq_unmask = mpc52xx_periph_unmask,
 	.irq_set_type = mpc52xx_null_set_type,
 };
@@ -263,29 +333,40 @@ static struct irq_chip mpc52xx_periph_irqchip = {
 /*
  * SDMA interrupt irq_chip
  */
-static void mpc52xx_sdma_mask(struct irq_data *d)
+static void mpc52xx_sdma_mask_ack(struct irq_data *d)
 {
 	int l2irq = irqd_to_hwirq(d) & MPC52xx_IRQ_L2_MASK;
-	io_be_setbit(&sdma->IntMask, l2irq);
+	__io_be_setbit(&sdma->IntMask, l2irq);
+	out_be32(&sdma->IntPend, 1 << l2irq);
 }
 
-static void mpc52xx_sdma_unmask(struct irq_data *d)
+static void mpc52xx_sdma_mask(struct irq_data *d)
 {
 	int l2irq = irqd_to_hwirq(d) & MPC52xx_IRQ_L2_MASK;
-	io_be_clrbit(&sdma->IntMask, l2irq);
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	__io_be_setbit(&sdma->IntMask, l2irq);
+	ipipe_lock_irq(d->irq);
+	hard_local_irq_restore(flags);
 }
 
-static void mpc52xx_sdma_ack(struct irq_data *d)
+static void mpc52xx_sdma_unmask(struct irq_data *d)
 {
 	int l2irq = irqd_to_hwirq(d) & MPC52xx_IRQ_L2_MASK;
-	out_be32(&sdma->IntPend, 1 << l2irq);
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	__io_be_clrbit(&sdma->IntMask, l2irq);
+	ipipe_unlock_irq(d->irq);
+	hard_local_irq_restore(flags);
 }
 
 static struct irq_chip mpc52xx_sdma_irqchip = {
 	.name = "MPC52xx SDMA",
 	.irq_mask = mpc52xx_sdma_mask,
 	.irq_unmask = mpc52xx_sdma_unmask,
-	.irq_ack = mpc52xx_sdma_ack,
+	.irq_mask_ack = mpc52xx_sdma_mask_ack,
 	.irq_set_type = mpc52xx_null_set_type,
 };
 
diff --git a/arch/powerpc/platforms/82xx/pq2ads-pci-pic.c b/arch/powerpc/platforms/82xx/pq2ads-pci-pic.c
index 74861a7..0132f29 100644
--- a/arch/powerpc/platforms/82xx/pq2ads-pci-pic.c
+++ b/arch/powerpc/platforms/82xx/pq2ads-pci-pic.c
@@ -17,6 +17,7 @@
 #include <linux/irq.h>
 #include <linux/types.h>
 #include <linux/slab.h>
+#include <linux/ipipe.h>
 
 #include <asm/io.h>
 #include <asm/prom.h>
@@ -24,7 +25,7 @@
 
 #include "pq2.h"
 
-static DEFINE_RAW_SPINLOCK(pci_pic_lock);
+static IPIPE_DEFINE_RAW_SPINLOCK(pci_pic_lock);
 
 struct pq2ads_pci_pic {
 	struct device_node *node;
@@ -42,13 +43,14 @@ static void pq2ads_pci_mask_irq(struct irq_data *d)
 {
 	struct pq2ads_pci_pic *priv = irq_data_get_irq_chip_data(d);
 	int irq = NUM_IRQS - irqd_to_hwirq(d) - 1;
+	unsigned long flags;
 
 	if (irq != -1) {
-		unsigned long flags;
 		raw_spin_lock_irqsave(&pci_pic_lock, flags);
 
 		setbits32(&priv->regs->mask, 1 << irq);
 		mb();
+		ipipe_lock_irq(d->irq);
 
 		raw_spin_unlock_irqrestore(&pci_pic_lock, flags);
 	}
@@ -58,12 +60,13 @@ static void pq2ads_pci_unmask_irq(struct irq_data *d)
 {
 	struct pq2ads_pci_pic *priv = irq_data_get_irq_chip_data(d);
 	int irq = NUM_IRQS - irqd_to_hwirq(d) - 1;
+	unsigned long flags;
 
 	if (irq != -1) {
-		unsigned long flags;
 
 		raw_spin_lock_irqsave(&pci_pic_lock, flags);
 		clrbits32(&priv->regs->mask, 1 << irq);
+		ipipe_unlock_irq(d->irq);
 		raw_spin_unlock_irqrestore(&pci_pic_lock, flags);
 	}
 }
@@ -96,7 +99,7 @@ static void pq2ads_pci_irq_demux(unsigned int irq, struct irq_desc *desc)
 		for (bit = 0; pend != 0; ++bit, pend <<= 1) {
 			if (pend & 0x80000000) {
 				int virq = irq_linear_revmap(priv->host, bit);
-				generic_handle_irq(virq);
+				ipipe_handle_demuxed_irq(virq);
 			}
 		}
 	}
diff --git a/arch/powerpc/platforms/85xx/common.c b/arch/powerpc/platforms/85xx/common.c
index 4a9ad87..0c2962a 100644
--- a/arch/powerpc/platforms/85xx/common.c
+++ b/arch/powerpc/platforms/85xx/common.c
@@ -8,6 +8,7 @@
 
 #include <linux/of_irq.h>
 #include <linux/of_platform.h>
+#include <linux/ipipe.h>
 
 #include <asm/qe.h>
 #include <sysdev/cpm2_pic.h>
@@ -54,7 +55,7 @@ static void cpm2_cascade(unsigned int irq, struct irq_desc *desc)
 	int cascade_irq;
 
 	while ((cascade_irq = cpm2_get_irq()) >= 0)
-		generic_handle_irq(cascade_irq);
+		ipipe_handle_demuxed_irq(cascade_irq);
 
 	chip->irq_eoi(&desc->irq_data);
 }
diff --git a/arch/powerpc/platforms/8xx/m8xx_setup.c b/arch/powerpc/platforms/8xx/m8xx_setup.c
index d303774..b2d8550 100644
--- a/arch/powerpc/platforms/8xx/m8xx_setup.c
+++ b/arch/powerpc/platforms/8xx/m8xx_setup.c
@@ -168,6 +168,7 @@ int mpc8xx_set_rtc_time(struct rtc_time *tm)
 {
 	sitk8xx_t __iomem *sys_tmr1;
 	sit8xx_t __iomem *sys_tmr2;
+	unsigned long flags;
 	int time;
 
 	sys_tmr1 = immr_map(im_sitk);
@@ -175,9 +176,11 @@ int mpc8xx_set_rtc_time(struct rtc_time *tm)
 	time = mktime(tm->tm_year+1900, tm->tm_mon+1, tm->tm_mday,
 	              tm->tm_hour, tm->tm_min, tm->tm_sec);
 
+	flags = hard_cond_local_irq_save();
 	out_be32(&sys_tmr1->sitk_rtck, KAPWR_KEY);
 	out_be32(&sys_tmr2->sit_rtc, time);
 	out_be32(&sys_tmr1->sitk_rtck, ~KAPWR_KEY);
+	hard_cond_local_irq_restore(flags);
 
 	immr_unmap(sys_tmr2);
 	immr_unmap(sys_tmr1);
@@ -203,7 +206,7 @@ void mpc8xx_restart(char *cmd)
 	car8xx_t __iomem *clk_r = immr_map(im_clkrst);
 
 
-	local_irq_disable();
+	hard_local_irq_disable();
 
 	setbits32(&clk_r->car_plprcr, 0x00000080);
 	/* Clear the ME bit in MSR to cause checkstop on machine check
@@ -220,7 +223,7 @@ static void cpm_cascade(unsigned int irq, struct irq_desc *desc)
 	int cascade_irq = cpm_get_irq();
 
 	if (cascade_irq >= 0)
-		generic_handle_irq(cascade_irq);
+		ipipe_handle_demuxed_irq(cascade_irq);
 
 	chip->irq_eoi(&desc->irq_data);
 }
diff --git a/arch/powerpc/platforms/cell/spu_base.c b/arch/powerpc/platforms/cell/spu_base.c
index ffcbd24..42e6a33 100644
--- a/arch/powerpc/platforms/cell/spu_base.c
+++ b/arch/powerpc/platforms/cell/spu_base.c
@@ -59,7 +59,7 @@ EXPORT_SYMBOL_GPL(force_sig_info);
 /*
  * Protects cbe_spu_info and spu->number.
  */
-static DEFINE_SPINLOCK(spu_lock);
+static IPIPE_DEFINE_SPINLOCK(spu_lock);
 
 /*
  * List of all spus in the system.
diff --git a/arch/powerpc/platforms/powermac/pic.c b/arch/powerpc/platforms/powermac/pic.c
index 4c24bf6..6c4ddae 100644
--- a/arch/powerpc/platforms/powermac/pic.c
+++ b/arch/powerpc/platforms/powermac/pic.c
@@ -55,7 +55,7 @@ static volatile struct pmac_irq_hw __iomem *pmac_irq_hw[4];
 static int max_irqs;
 static int max_real_irqs;
 
-static DEFINE_RAW_SPINLOCK(pmac_pic_lock);
+static IPIPE_DEFINE_RAW_SPINLOCK(pmac_pic_lock);
 
 /* The max irq number this driver deals with is 128; see max_irqs */
 static DECLARE_BITMAP(ppc_lost_interrupts, 128);
diff --git a/arch/powerpc/platforms/ps3/htab.c b/arch/powerpc/platforms/ps3/htab.c
index 3e270e3..c3c20db7 100644
--- a/arch/powerpc/platforms/ps3/htab.c
+++ b/arch/powerpc/platforms/ps3/htab.c
@@ -42,7 +42,7 @@ enum ps3_lpar_vas_id {
 };
 
 
-static DEFINE_SPINLOCK(ps3_htab_lock);
+static IPIPE_DEFINE_SPINLOCK(ps3_htab_lock);
 
 static long ps3_hpte_insert(unsigned long hpte_group, unsigned long vpn,
 	unsigned long pa, unsigned long rflags, unsigned long vflags,
diff --git a/arch/powerpc/platforms/ps3/interrupt.c b/arch/powerpc/platforms/ps3/interrupt.c
index 5f3b232..284d267 100644
--- a/arch/powerpc/platforms/ps3/interrupt.c
+++ b/arch/powerpc/platforms/ps3/interrupt.c
@@ -87,7 +87,7 @@ struct ps3_bmp {
 
 struct ps3_private {
 	struct ps3_bmp bmp __attribute__ ((aligned (PS3_BMP_MINALIGN)));
-	spinlock_t bmp_lock;
+	ipipe_spinlock_t bmp_lock;
 	u64 ppe_id;
 	u64 thread_id;
 	unsigned long ipi_debug_brk_mask;
diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c
index f6880d2..6793dad 100644
--- a/arch/powerpc/platforms/pseries/lpar.c
+++ b/arch/powerpc/platforms/pseries/lpar.c
@@ -184,7 +184,7 @@ static long pSeries_lpar_hpte_insert(unsigned long hpte_group,
 	return (slot & 7) | (!!(vflags & HPTE_V_SECONDARY) << 3);
 }
 
-static DEFINE_SPINLOCK(pSeries_lpar_tlbie_lock);
+static IPIPE_DEFINE_SPINLOCK(pSeries_lpar_tlbie_lock);
 
 static long pSeries_lpar_hpte_remove(unsigned long hpte_group)
 {
diff --git a/arch/powerpc/sysdev/cpm1.c b/arch/powerpc/sysdev/cpm1.c
index 5e6ff38..1daea61 100644
--- a/arch/powerpc/sysdev/cpm1.c
+++ b/arch/powerpc/sysdev/cpm1.c
@@ -59,15 +59,23 @@ static struct irq_domain *cpm_pic_host;
 static void cpm_mask_irq(struct irq_data *d)
 {
 	unsigned int cpm_vec = (unsigned int)irqd_to_hwirq(d);
+	unsigned long flags;
 
+	flags = hard_cond_local_irq_save();
+	ipipe_lock_irq(d->irq);
 	clrbits32(&cpic_reg->cpic_cimr, (1 << cpm_vec));
+	hard_cond_local_irq_restore(flags);
 }
 
 static void cpm_unmask_irq(struct irq_data *d)
 {
 	unsigned int cpm_vec = (unsigned int)irqd_to_hwirq(d);
-
+	unsigned long flags;
+ 
+	flags = hard_cond_local_irq_save();
 	setbits32(&cpic_reg->cpic_cimr, (1 << cpm_vec));
+	ipipe_unlock_irq(d->irq);
+	hard_cond_local_irq_restore(flags);
 }
 
 static void cpm_end_irq(struct irq_data *d)
@@ -77,11 +85,40 @@ static void cpm_end_irq(struct irq_data *d)
 	out_be32(&cpic_reg->cpic_cisr, (1 << cpm_vec));
 }
 
+#ifdef CONFIG_IPIPE
+
+static void cpm_hold_irq(struct irq_data *d)
+{
+	unsigned int cpm_vec = (unsigned int)irqd_to_hwirq(d);
+	unsigned long flags;
+
+	flags = hard_cond_local_irq_save();
+	out_be32(&cpic_reg->cpic_cisr, (1 << cpm_vec));
+	clrbits32(&cpic_reg->cpic_cimr, (1 << cpm_vec));
+	hard_cond_local_irq_restore(flags);
+}
+
+static void cpm_release_irq(struct irq_data *d)
+{
+	unsigned int cpm_vec = (unsigned int)irqd_to_hwirq(d);
+	unsigned long flags;
+ 
+	flags = hard_cond_local_irq_save();
+	setbits32(&cpic_reg->cpic_cimr, (1 << cpm_vec));
+	hard_cond_local_irq_restore(flags);
+}
+
+#endif
+
 static struct irq_chip cpm_pic = {
 	.name = "CPM PIC",
 	.irq_mask = cpm_mask_irq,
 	.irq_unmask = cpm_unmask_irq,
 	.irq_eoi = cpm_end_irq,
+#ifdef CONFIG_IPIPE
+	.irq_hold = cpm_hold_irq,
+	.irq_release = cpm_release_irq,
+#endif
 };
 
 int cpm_get_irq(void)
diff --git a/arch/powerpc/sysdev/cpm2_pic.c b/arch/powerpc/sysdev/cpm2_pic.c
index a11bd1d..8244f9b 100644
--- a/arch/powerpc/sysdev/cpm2_pic.c
+++ b/arch/powerpc/sysdev/cpm2_pic.c
@@ -80,47 +80,63 @@ static void cpm2_mask_irq(struct irq_data *d)
 {
 	int	bit, word;
 	unsigned int irq_nr = irqd_to_hwirq(d);
+	unsigned long flags;
 
 	bit = irq_to_siubit[irq_nr];
 	word = irq_to_siureg[irq_nr];
 
+	flags = hard_cond_local_irq_save();
+	ipipe_lock_irq(d->irq);
 	ppc_cached_irq_mask[word] &= ~(1 << bit);
 	out_be32(&cpm2_intctl->ic_simrh + word, ppc_cached_irq_mask[word]);
+	hard_cond_local_irq_restore(flags);
 }
 
 static void cpm2_unmask_irq(struct irq_data *d)
 {
 	int	bit, word;
 	unsigned int irq_nr = irqd_to_hwirq(d);
+	unsigned long flags;
 
 	bit = irq_to_siubit[irq_nr];
 	word = irq_to_siureg[irq_nr];
 
+	flags = hard_cond_local_irq_save();
 	ppc_cached_irq_mask[word] |= 1 << bit;
 	out_be32(&cpm2_intctl->ic_simrh + word, ppc_cached_irq_mask[word]);
+	ipipe_unlock_irq(d->irq);
+	hard_cond_local_irq_restore(flags);
 }
 
-static void cpm2_ack(struct irq_data *d)
+static void cpm2_mask_ack(struct irq_data *d)
 {
 	int	bit, word;
 	unsigned int irq_nr = irqd_to_hwirq(d);
+	unsigned long flags;
 
 	bit = irq_to_siubit[irq_nr];
 	word = irq_to_siureg[irq_nr];
 
+	flags = hard_cond_local_irq_save();
+	ppc_cached_irq_mask[word] &= ~(1 << bit);
+	out_be32(&cpm2_intctl->ic_simrh + word, ppc_cached_irq_mask[word]);
 	out_be32(&cpm2_intctl->ic_sipnrh + word, 1 << bit);
+	hard_cond_local_irq_restore(flags);
 }
 
 static void cpm2_end_irq(struct irq_data *d)
 {
 	int	bit, word;
 	unsigned int irq_nr = irqd_to_hwirq(d);
+	unsigned long flags;
 
 	bit = irq_to_siubit[irq_nr];
 	word = irq_to_siureg[irq_nr];
 
+	flags = hard_cond_local_irq_save();
 	ppc_cached_irq_mask[word] |= 1 << bit;
 	out_be32(&cpm2_intctl->ic_simrh + word, ppc_cached_irq_mask[word]);
+	hard_cond_local_irq_restore(flags);
 
 	/*
 	 * Work around large numbers of spurious IRQs on PowerPC 82xx
@@ -191,8 +207,11 @@ static struct irq_chip cpm2_pic = {
 	.name = "CPM2 SIU",
 	.irq_mask = cpm2_mask_irq,
 	.irq_unmask = cpm2_unmask_irq,
-	.irq_ack = cpm2_ack,
+	.irq_mask_ack = cpm2_mask_ack,
 	.irq_eoi = cpm2_end_irq,
+#ifdef CONFIG_IPIPE
+	.irq_hold	= cpm2_end_irq,
+#endif
 	.irq_set_type = cpm2_set_irq_type,
 	.flags = IRQCHIP_EOI_IF_HANDLED,
 };
diff --git a/arch/powerpc/sysdev/fsl_mpic_err.c b/arch/powerpc/sysdev/fsl_mpic_err.c
index b83f325..782d29f 100644
--- a/arch/powerpc/sysdev/fsl_mpic_err.c
+++ b/arch/powerpc/sysdev/fsl_mpic_err.c
@@ -13,6 +13,7 @@
 #include <linux/irq.h>
 #include <linux/smp.h>
 #include <linux/interrupt.h>
+#include <linux/ipipe.h>
 
 #include <asm/io.h>
 #include <asm/irq.h>
@@ -39,10 +40,14 @@ static void fsl_mpic_mask_err(struct irq_data *d)
 	u32 eimr;
 	struct mpic *mpic = irq_data_get_irq_chip_data(d);
 	unsigned int src = virq_to_hw(d->irq) - mpic->err_int_vecs[0];
+	unsigned long flags;
 
+	flags = hard_cond_local_irq_save();
+	ipipe_lock_irq(d->irq);
 	eimr = mpic_fsl_err_read(mpic->err_regs, MPIC_ERR_INT_EIMR);
 	eimr |= (1 << (31 - src));
 	mpic_fsl_err_write(mpic->err_regs, eimr);
+	hard_cond_local_irq_restore(flags);
 }
 
 static void fsl_mpic_unmask_err(struct irq_data *d)
@@ -50,10 +55,14 @@ static void fsl_mpic_unmask_err(struct irq_data *d)
 	u32 eimr;
 	struct mpic *mpic = irq_data_get_irq_chip_data(d);
 	unsigned int src = virq_to_hw(d->irq) - mpic->err_int_vecs[0];
+	unsigned long flags;
 
+	flags = hard_cond_local_irq_save();
 	eimr = mpic_fsl_err_read(mpic->err_regs, MPIC_ERR_INT_EIMR);
 	eimr &= ~(1 << (31 - src));
 	mpic_fsl_err_write(mpic->err_regs, eimr);
+	ipipe_unlock_irq(d->irq);
+	hard_cond_local_irq_restore(flags);
 }
 
 static struct irq_chip fsl_mpic_err_chip = {
@@ -117,7 +126,7 @@ static irqreturn_t fsl_error_int_handler(int irq, void *data)
 				 mpic->err_int_vecs[errint]);
 		WARN_ON(cascade_irq == NO_IRQ);
 		if (cascade_irq != NO_IRQ) {
-			generic_handle_irq(cascade_irq);
+			ipipe_handle_demuxed_irq(cascade_irq);
 		} else {
 			eimr |=  1 << (31 - errint);
 			mpic_fsl_err_write(mpic->err_regs, eimr);
@@ -128,6 +137,13 @@ static irqreturn_t fsl_error_int_handler(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+static void __ipipe_error_cascade(unsigned int irq, struct irq_desc *desc)
+{
+#ifdef CONFIG_IPIPE
+	fsl_error_int_handler(irq, irq_get_handler_data(irq));
+#endif
+}
+
 void mpic_err_int_init(struct mpic *mpic, irq_hw_number_t irqnum)
 {
 	unsigned int virq;
@@ -142,8 +158,13 @@ void mpic_err_int_init(struct mpic *mpic, irq_hw_number_t irqnum)
 	/* Mask all error interrupts */
 	mpic_fsl_err_write(mpic->err_regs, ~0);
 
-	ret = request_irq(virq, fsl_error_int_handler, IRQF_NO_THREAD,
-		    "mpic-error-int", mpic);
-	if (ret)
-		pr_err("Failed to register error interrupt handler\n");
+	if (IS_ENABLED(CONFIG_IPIPE)) {
+		irq_set_chained_handler(virq, __ipipe_error_cascade);
+		irq_set_handler_data(virq, mpic);
+	} else {
+		ret = request_irq(virq, fsl_error_int_handler, IRQF_NO_THREAD,
+				  "mpic-error-int", mpic);
+		if (ret)
+			pr_err("Failed to register error interrupt handler\n");
+	}
 }
diff --git a/arch/powerpc/sysdev/fsl_msi.c b/arch/powerpc/sysdev/fsl_msi.c
index da08ed0..e0a8231 100644
--- a/arch/powerpc/sysdev/fsl_msi.c
+++ b/arch/powerpc/sysdev/fsl_msi.c
@@ -298,7 +298,7 @@ static irqreturn_t fsl_msi_cascade(int irq, void *data)
 				msi_hwirq(msi_data, msir_index,
 					  intr_index + have_shift));
 		if (cascade_irq != NO_IRQ) {
-			generic_handle_irq(cascade_irq);
+			ipipe_handle_demuxed_irq(cascade_irq);
 			ret = IRQ_HANDLED;
 		}
 		have_shift += intr_index + 1;
@@ -308,6 +308,13 @@ static irqreturn_t fsl_msi_cascade(int irq, void *data)
 	return ret;
 }
 
+static void __ipipe_msi_cascade(unsigned int irq, struct irq_desc *desc)
+{
+#ifdef CONFIG_IPIPE
+	fsl_msi_cascade(irq, irq_get_handler_data(irq));
+#endif
+}
+
 static int fsl_of_msi_remove(struct platform_device *ofdev)
 {
 	struct fsl_msi *msi = platform_get_drvdata(ofdev);
@@ -361,12 +368,17 @@ static int fsl_msi_setup_hwirq(struct fsl_msi *msi, struct platform_device *dev,
 	cascade_data->virq = virt_msir;
 	msi->cascade_array[irq_index] = cascade_data;
 
-	ret = request_irq(virt_msir, fsl_msi_cascade, IRQF_NO_THREAD,
-			  "fsl-msi-cascade", cascade_data);
-	if (ret) {
-		dev_err(&dev->dev, "failed to request_irq(%d), ret = %d\n",
-			virt_msir, ret);
-		return ret;
+	if (IS_ENABLED(CONFIG_IPIPE)) {
+		irq_set_chained_handler(virt_msir, __ipipe_msi_cascade);
+		irq_set_handler_data(virt_msir, cascade_data);
+	} else {
+		ret = request_irq(virt_msir, fsl_msi_cascade, IRQF_NO_THREAD,
+				  "fsl-msi-cascade", cascade_data);
+		if (ret) {
+			dev_err(&dev->dev, "failed to request_irq(%d), ret = %d\n",
+				virt_msir, ret);
+			return ret;
+		}
 	}
 
 	/* Release the hwirqs corresponding to this MSI register */
diff --git a/arch/powerpc/sysdev/i8259.c b/arch/powerpc/sysdev/i8259.c
index 45598da..c73bdd6 100644
--- a/arch/powerpc/sysdev/i8259.c
+++ b/arch/powerpc/sysdev/i8259.c
@@ -22,7 +22,7 @@ static unsigned char cached_8259[2] = { 0xff, 0xff };
 #define cached_A1 (cached_8259[0])
 #define cached_21 (cached_8259[1])
 
-static DEFINE_RAW_SPINLOCK(i8259_lock);
+static IPIPE_DEFINE_RAW_SPINLOCK(i8259_lock);
 
 static struct irq_domain *i8259_host;
 
diff --git a/arch/powerpc/sysdev/ipic.c b/arch/powerpc/sysdev/ipic.c
index b50f978..bbd0392 100644
--- a/arch/powerpc/sysdev/ipic.c
+++ b/arch/powerpc/sysdev/ipic.c
@@ -32,7 +32,7 @@
 
 static struct ipic * primary_ipic;
 static struct irq_chip ipic_level_irq_chip, ipic_edge_irq_chip;
-static DEFINE_RAW_SPINLOCK(ipic_lock);
+static IPIPE_DEFINE_RAW_SPINLOCK(ipic_lock);
 
 static struct ipic_info ipic_info[] = {
 	[1] = {
@@ -533,19 +533,17 @@ static void ipic_unmask_irq(struct irq_data *d)
 	temp = ipic_read(ipic->regs, ipic_info[src].mask);
 	temp |= (1 << (31 - ipic_info[src].bit));
 	ipic_write(ipic->regs, ipic_info[src].mask, temp);
+	ipipe_unlock_irq(d->irq);
 
 	raw_spin_unlock_irqrestore(&ipic_lock, flags);
 }
 
-static void ipic_mask_irq(struct irq_data *d)
+static void __ipic_mask_irq(struct irq_data *d)
 {
 	struct ipic *ipic = ipic_from_irq(d->irq);
 	unsigned int src = irqd_to_hwirq(d);
-	unsigned long flags;
 	u32 temp;
 
-	raw_spin_lock_irqsave(&ipic_lock, flags);
-
 	temp = ipic_read(ipic->regs, ipic_info[src].mask);
 	temp &= ~(1 << (31 - ipic_info[src].bit));
 	ipic_write(ipic->regs, ipic_info[src].mask, temp);
@@ -553,6 +551,27 @@ static void ipic_mask_irq(struct irq_data *d)
 	/* mb() can't guarantee that masking is finished.  But it does finish
 	 * for nearly all cases. */
 	mb();
+}
+
+static void ipic_mask_irq(struct irq_data *d)
+{
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&ipic_lock, flags);
+
+	ipipe_lock_irq(d->irq);
+	__ipic_mask_irq(d);
+
+	raw_spin_unlock_irqrestore(&ipic_lock, flags);
+}
+
+static void ipic_mask_irq_no_ack(struct irq_data *d)
+{
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&ipic_lock, flags);
+
+	__ipic_mask_irq(d);
 
 	raw_spin_unlock_irqrestore(&ipic_lock, flags);
 }
@@ -659,7 +678,7 @@ static struct irq_chip ipic_level_irq_chip = {
 	.name		= "IPIC",
 	.irq_unmask	= ipic_unmask_irq,
 	.irq_mask	= ipic_mask_irq,
-	.irq_mask_ack	= ipic_mask_irq,
+	.irq_mask_ack	= ipic_mask_irq_no_ack,
 	.irq_set_type	= ipic_set_irq_type,
 };
 
diff --git a/arch/powerpc/sysdev/mpc8xx_pic.c b/arch/powerpc/sysdev/mpc8xx_pic.c
index c4828c0..fb9654a 100644
--- a/arch/powerpc/sysdev/mpc8xx_pic.c
+++ b/arch/powerpc/sysdev/mpc8xx_pic.c
@@ -27,14 +27,24 @@ static inline unsigned long mpc8xx_irqd_to_bit(struct irq_data *d)
 
 static void mpc8xx_unmask_irq(struct irq_data *d)
 {
+	unsigned long flags;
+
+	flags = hard_cond_local_irq_save();
 	mpc8xx_cached_irq_mask |= mpc8xx_irqd_to_bit(d);
 	out_be32(&siu_reg->sc_simask, mpc8xx_cached_irq_mask);
+	ipipe_unlock_irq(d->irq);
+	hard_cond_local_irq_restore(flags);
 }
 
 static void mpc8xx_mask_irq(struct irq_data *d)
 {
+	unsigned long flags;
+
+	flags = hard_cond_local_irq_save();
 	mpc8xx_cached_irq_mask &= ~mpc8xx_irqd_to_bit(d);
 	out_be32(&siu_reg->sc_simask, mpc8xx_cached_irq_mask);
+	ipipe_unlock_irq(d->irq);
+	hard_cond_local_irq_restore(flags);
 }
 
 static void mpc8xx_ack(struct irq_data *d)
@@ -42,10 +52,29 @@ static void mpc8xx_ack(struct irq_data *d)
 	out_be32(&siu_reg->sc_sipend, mpc8xx_irqd_to_bit(d));
 }
 
+#ifdef CONFIG_IPIPE
+
+static void mpc8xx_mask_ack_irq(struct irq_data *d)
+{
+	unsigned long flags;
+
+	flags = hard_cond_local_irq_save();
+	mpc8xx_cached_irq_mask &= ~mpc8xx_irqd_to_bit(d);
+	out_be32(&siu_reg->sc_simask, mpc8xx_cached_irq_mask);
+	out_be32(&siu_reg->sc_sipend, mpc8xx_irqd_to_bit(d));
+	hard_cond_local_irq_restore(flags);
+}
+
+#endif
+
 static void mpc8xx_end_irq(struct irq_data *d)
 {
+	unsigned long flags;
+
+	flags = hard_cond_local_irq_save();
 	mpc8xx_cached_irq_mask |= mpc8xx_irqd_to_bit(d);
 	out_be32(&siu_reg->sc_simask, mpc8xx_cached_irq_mask);
+	hard_cond_local_irq_restore(flags);
 }
 
 static int mpc8xx_set_irq_type(struct irq_data *d, unsigned int flow_type)
@@ -55,7 +84,9 @@ static int mpc8xx_set_irq_type(struct irq_data *d, unsigned int flow_type)
 		unsigned int siel = in_be32(&siu_reg->sc_siel);
 		siel |= mpc8xx_irqd_to_bit(d);
 		out_be32(&siu_reg->sc_siel, siel);
+#ifndef CONFIG_IPIPE
 		__irq_set_handler_locked(d->irq, handle_edge_irq);
+#endif
 	}
 	return 0;
 }
@@ -67,6 +98,9 @@ static struct irq_chip mpc8xx_pic = {
 	.irq_ack = mpc8xx_ack,
 	.irq_eoi = mpc8xx_end_irq,
 	.irq_set_type = mpc8xx_set_irq_type,
+#ifdef CONFIG_IPIPE
+	.mask_ack = mpc8xx_mask_ack_irq,
+#endif
 };
 
 unsigned int mpc8xx_get_irq(void)
diff --git a/arch/powerpc/sysdev/mpic.c b/arch/powerpc/sysdev/mpic.c
index 89cec0e..07b42a1 100644
--- a/arch/powerpc/sysdev/mpic.c
+++ b/arch/powerpc/sysdev/mpic.c
@@ -30,6 +30,7 @@
 #include <linux/slab.h>
 #include <linux/syscore_ops.h>
 #include <linux/ratelimit.h>
+#include <linux/ipipe.h>
 
 #include <asm/ptrace.h>
 #include <asm/signal.h>
@@ -56,7 +57,7 @@ EXPORT_SYMBOL_GPL(mpic_subsys);
 
 static struct mpic *mpics;
 static struct mpic *mpic_primary;
-static DEFINE_RAW_SPINLOCK(mpic_lock);
+static IPIPE_DEFINE_RAW_SPINLOCK(mpic_lock);
 
 #ifdef CONFIG_PPC32	/* XXX for now */
 #ifdef CONFIG_IRQ_ALL_CPUS
@@ -664,7 +665,7 @@ static inline void mpic_eoi(struct mpic *mpic)
  */
 
 
-void mpic_unmask_irq(struct irq_data *d)
+void __mpic_unmask_irq(struct irq_data *d)
 {
 	unsigned int loops = 100000;
 	struct mpic *mpic = mpic_from_irq_data(d);
@@ -685,7 +686,17 @@ void mpic_unmask_irq(struct irq_data *d)
 	} while(mpic_irq_read(src, MPIC_INFO(IRQ_VECTOR_PRI)) & MPIC_VECPRI_MASK);
 }
 
-void mpic_mask_irq(struct irq_data *d)
+void mpic_unmask_irq(struct irq_data *d)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&mpic_lock, flags);
+	ipipe_unlock_irq(d->irq);
+	__mpic_unmask_irq(d);
+	spin_unlock_irqrestore(&mpic_lock, flags);
+}
+
+void __mpic_mask_irq(struct irq_data *d)
 {
 	unsigned int loops = 100000;
 	struct mpic *mpic = mpic_from_irq_data(d);
@@ -707,6 +718,16 @@ void mpic_mask_irq(struct irq_data *d)
 	} while(!(mpic_irq_read(src, MPIC_INFO(IRQ_VECTOR_PRI)) & MPIC_VECPRI_MASK));
 }
 
+void mpic_mask_irq(struct irq_data *d)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&mpic_lock, flags);
+	__mpic_mask_irq(d);
+	ipipe_lock_irq(d->irq);
+	spin_unlock_irqrestore(&mpic_lock, flags);
+}
+
 void mpic_end_irq(struct irq_data *d)
 {
 	struct mpic *mpic = mpic_from_irq_data(d);
@@ -722,6 +743,30 @@ void mpic_end_irq(struct irq_data *d)
 	mpic_eoi(mpic);
 }
 
+#ifdef CONFIG_IPIPE
+
+void mpic_hold_irq(struct irq_data *d)
+{
+	struct mpic *mpic = mpic_from_irq_data(d);
+	unsigned long flags;
+
+	spin_lock_irqsave(&mpic_lock, flags);
+	mpic_eoi(mpic);
+	__mpic_mask_irq(d);
+	spin_unlock_irqrestore(&mpic_lock, flags);
+}
+
+void mpic_release_irq(struct irq_data *d)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&mpic_lock, flags);
+	__mpic_unmask_irq(d);
+	spin_unlock_irqrestore(&mpic_lock, flags);
+}
+
+#endif
+
 #ifdef CONFIG_MPIC_U3_HT_IRQS
 
 static void mpic_unmask_ht_irq(struct irq_data *d)
@@ -729,7 +774,7 @@ static void mpic_unmask_ht_irq(struct irq_data *d)
 	struct mpic *mpic = mpic_from_irq_data(d);
 	unsigned int src = irqd_to_hwirq(d);
 
-	mpic_unmask_irq(d);
+	__mpic_unmask_irq(d);
 
 	if (irqd_is_level_type(d))
 		mpic_ht_end_irq(mpic, src);
@@ -740,7 +785,7 @@ static unsigned int mpic_startup_ht_irq(struct irq_data *d)
 	struct mpic *mpic = mpic_from_irq_data(d);
 	unsigned int src = irqd_to_hwirq(d);
 
-	mpic_unmask_irq(d);
+	__mpic_unmask_irq(d);
 	mpic_startup_ht_interrupt(mpic, src, irqd_is_level_type(d));
 
 	return 0;
@@ -761,7 +806,7 @@ static void mpic_end_ht_irq(struct irq_data *d)
 	unsigned int src = irqd_to_hwirq(d);
 
 #ifdef DEBUG_IRQ
-	DBG("%s: end_irq: %d\n", mpic->name, d->irq);
+	DBG("%s: end_ht_irq: %d\n", mpic->name, d->irq);
 #endif
 	/* We always EOI on end_irq() even for edge interrupts since that
 	 * should only lower the priority, the MPIC should have properly
@@ -780,9 +825,12 @@ static void mpic_unmask_ipi(struct irq_data *d)
 {
 	struct mpic *mpic = mpic_from_ipi(d);
 	unsigned int src = virq_to_hw(d->irq) - mpic->ipi_vecs[0];
+	unsigned long flags;
 
-	DBG("%s: enable_ipi: %d (ipi %d)\n", mpic->name, d->irq, src);
+	DBG("%s: unmask_ipi: %d (ipi %d)\n", mpic->name, d->irq, src);
+	spin_lock_irqsave(&mpic_lock, flags);
 	mpic_ipi_write(src, mpic_ipi_read(src) & ~MPIC_VECPRI_MASK);
+	spin_unlock_irqrestore(&mpic_lock, flags);
 }
 
 static void mpic_mask_ipi(struct irq_data *d)
@@ -871,6 +919,7 @@ int mpic_set_irq_type(struct irq_data *d, unsigned int flow_type)
 	struct mpic *mpic = mpic_from_irq_data(d);
 	unsigned int src = irqd_to_hwirq(d);
 	unsigned int vecpri, vold, vnew;
+	unsigned long flags;
 
 	DBG("mpic: set_irq_type(mpic:@%p,virq:%d,src:0x%x,type:0x%x)\n",
 	    mpic, d->irq, src, flow_type);
@@ -878,6 +927,8 @@ int mpic_set_irq_type(struct irq_data *d, unsigned int flow_type)
 	if (src >= mpic->num_sources)
 		return -EINVAL;
 
+	flags = hard_cond_local_irq_save();
+
 	vold = mpic_irq_read(src, MPIC_INFO(IRQ_VECTOR_PRI));
 
 	/* We don't support "none" type */
@@ -923,6 +974,8 @@ int mpic_set_irq_type(struct irq_data *d, unsigned int flow_type)
 	if (vold != vnew)
 		mpic_irq_write(src, MPIC_INFO(IRQ_VECTOR_PRI), vnew);
 
+	hard_cond_local_irq_restore(flags);
+
 	return IRQ_SET_MASK_OK_NOCOPY;
 }
 
@@ -980,6 +1033,10 @@ static struct irq_chip mpic_irq_chip = {
 	.irq_eoi	= mpic_end_irq,
 	.irq_set_type	= mpic_set_irq_type,
 	.irq_set_wake	= mpic_irq_set_wake,
+#ifdef CONFIG_IPIPE
+	.irq_hold	= mpic_hold_irq,
+	.irq_release	= mpic_release_irq,
+#endif
 };
 
 #ifdef CONFIG_SMP
@@ -1192,7 +1249,7 @@ static void mpic_cascade(unsigned int irq, struct irq_desc *desc)
 
 	virq = mpic_get_one_irq(mpic);
 	if (virq)
-		generic_handle_irq(virq);
+		ipipe_handle_demuxed_irq(virq);
 
 	chip->irq_eoi(&desc->irq_data);
 }
diff --git a/arch/powerpc/sysdev/qe_lib/qe_ic.c b/arch/powerpc/sysdev/qe_lib/qe_ic.c
index b2b87c3..6fb8a63 100644
--- a/arch/powerpc/sysdev/qe_lib/qe_ic.c
+++ b/arch/powerpc/sysdev/qe_lib/qe_ic.c
@@ -32,7 +32,7 @@
 
 #include "qe_ic.h"
 
-static DEFINE_RAW_SPINLOCK(qe_ic_lock);
+static IPIPE_DEFINE_RAW_SPINLOCK(qe_ic_lock);
 
 static struct qe_ic_info qe_ic_info[] = {
 	[1] = {
diff --git a/arch/powerpc/sysdev/tsi108_pci.c b/arch/powerpc/sysdev/tsi108_pci.c
index 188012c..eea016e 100644
--- a/arch/powerpc/sysdev/tsi108_pci.c
+++ b/arch/powerpc/sysdev/tsi108_pci.c
@@ -249,7 +249,9 @@ static void tsi108_pci_int_mask(u_int irq)
 {
 	u_int irp_cfg;
 	int int_line = (irq - IRQ_PCI_INTAD_BASE);
+	unsigned long flags;
 
+	flags = hard_cond_local_irq_save();
 	irp_cfg = tsi108_read_reg(TSI108_PCI_OFFSET + TSI108_PCI_IRP_CFG_CTL);
 	mb();
 	irp_cfg |= (1 << int_line);	/* INTx_DIR = output */
@@ -257,19 +259,23 @@ static void tsi108_pci_int_mask(u_int irq)
 	tsi108_write_reg(TSI108_PCI_OFFSET + TSI108_PCI_IRP_CFG_CTL, irp_cfg);
 	mb();
 	irp_cfg = tsi108_read_reg(TSI108_PCI_OFFSET + TSI108_PCI_IRP_CFG_CTL);
+	hard_cond_local_irq_restore(flags);
 }
 
 static void tsi108_pci_int_unmask(u_int irq)
 {
 	u_int irp_cfg;
 	int int_line = (irq - IRQ_PCI_INTAD_BASE);
+	unsigned long flags;
 
+	flags = hard_cond_local_irq_save();
 	irp_cfg = tsi108_read_reg(TSI108_PCI_OFFSET + TSI108_PCI_IRP_CFG_CTL);
 	mb();
 	irp_cfg &= ~(1 << int_line);
 	irp_cfg |= (3 << (8 + (int_line * 2)));
 	tsi108_write_reg(TSI108_PCI_OFFSET + TSI108_PCI_IRP_CFG_CTL, irp_cfg);
 	mb();
+	hard_cond_local_irq_restore(flags);
 }
 
 static void init_pci_source(void)
@@ -345,6 +351,9 @@ static inline unsigned int get_pci_source(void)
 
 static void tsi108_pci_irq_unmask(struct irq_data *d)
 {
+	unsigned long flags;
+
+	flags = hard_cond_local_irq_save();
 	tsi108_pci_int_unmask(d->irq);
 
 	/* Enable interrupts from PCI block */
@@ -352,6 +361,7 @@ static void tsi108_pci_irq_unmask(struct irq_data *d)
 			 tsi108_read_reg(TSI108_PCI_OFFSET +
 					 TSI108_PCI_IRP_ENABLE) |
 			 TSI108_PCI_IRP_ENABLE_P_INT);
+	hard_cond_local_irq_restore(flags);
 	mb();
 }
 
@@ -434,7 +444,7 @@ void tsi108_irq_cascade(unsigned int irq, struct irq_desc *desc)
 	unsigned int cascade_irq = get_pci_source();
 
 	if (cascade_irq != NO_IRQ)
-		generic_handle_irq(cascade_irq);
+		ipipe_handle_demuxed_irq(cascade_irq);
 
 	chip->irq_eoi(&desc->irq_data);
 }
diff --git a/arch/powerpc/sysdev/uic.c b/arch/powerpc/sysdev/uic.c
index 9203393..8b03381 100644
--- a/arch/powerpc/sysdev/uic.c
+++ b/arch/powerpc/sysdev/uic.c
@@ -46,7 +46,7 @@ struct uic {
 	int index;
 	int dcrbase;
 
-	raw_spinlock_t lock;
+	ipipe_spinlock_t lock;
 
 	/* The remapper for this UIC */
 	struct irq_domain	*irqhost;
@@ -67,6 +67,7 @@ static void uic_unmask_irq(struct irq_data *d)
 	er = mfdcr(uic->dcrbase + UIC_ER);
 	er |= sr;
 	mtdcr(uic->dcrbase + UIC_ER, er);
+	ipipe_unlock_irq(d->irq);
 	raw_spin_unlock_irqrestore(&uic->lock, flags);
 }
 
@@ -78,6 +79,7 @@ static void uic_mask_irq(struct irq_data *d)
 	u32 er;
 
 	raw_spin_lock_irqsave(&uic->lock, flags);
+	ipipe_lock_irq(d->irq);
 	er = mfdcr(uic->dcrbase + UIC_ER);
 	er &= ~(1 << (31 - src));
 	mtdcr(uic->dcrbase + UIC_ER, er);
@@ -204,12 +206,16 @@ void uic_irq_cascade(unsigned int virq, struct irq_desc *desc)
 	int src;
 	int subvirq;
 
+#ifndef CONFIG_IPIPE
 	raw_spin_lock(&desc->lock);
 	if (irqd_is_level_type(idata))
 		chip->irq_mask(idata);
 	else
 		chip->irq_mask_ack(idata);
 	raw_spin_unlock(&desc->lock);
+#else
+	chip->irq_mask_ack(idata);
+#endif
 
 	msr = mfdcr(uic->dcrbase + UIC_MSR);
 	if (!msr) /* spurious interrupt */
@@ -218,15 +224,20 @@ void uic_irq_cascade(unsigned int virq, struct irq_desc *desc)
 	src = 32 - ffs(msr);
 
 	subvirq = irq_linear_revmap(uic->irqhost, src);
-	generic_handle_irq(subvirq);
+	ipipe_handle_demuxed_irq(subvirq);
 
 uic_irq_ret:
+#ifndef CONFIG_IPIPE
 	raw_spin_lock(&desc->lock);
 	if (irqd_is_level_type(idata))
 		chip->irq_ack(idata);
 	if (!irqd_irq_disabled(idata) && chip->irq_unmask)
 		chip->irq_unmask(idata);
 	raw_spin_unlock(&desc->lock);
+#else
+	if (chip->irq_unmask)
+		chip->irq_unmask(idata);
+#endif
 }
 
 static struct uic * __init uic_init_one(struct device_node *node)
diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
index e23f559..4e32013 100644
--- a/arch/powerpc/xmon/xmon.c
+++ b/arch/powerpc/xmon/xmon.c
@@ -1460,7 +1460,7 @@ static void excprint(struct pt_regs *fp)
 	}
 
 	printf("  current = 0x%lx\n", current);
-#ifdef CONFIG_PPC64
+#if defined(CONFIG_PPC64) && !defined(CONFIG_IPIPE)
 	printf("  paca    = 0x%lx\t softe: %d\t irq_happened: 0x%02x\n",
 	       local_paca, local_paca->soft_enabled, local_paca->irq_happened);
 #endif
@@ -2076,8 +2076,10 @@ static void dump_one_paca(int cpu)
 	DUMP(p, stab_rr, "lx");
 	DUMP(p, saved_r1, "lx");
 	DUMP(p, trap_save, "x");
+#ifndef CONFIG_IPIPE
 	DUMP(p, soft_enabled, "x");
 	DUMP(p, irq_happened, "x");
+#endif
 	DUMP(p, io_sync, "x");
 	DUMP(p, irq_work_pending, "x");
 	DUMP(p, nap_state_lost, "x");
diff --git a/drivers/gpio/gpio-mpc8xxx.c b/drivers/gpio/gpio-mpc8xxx.c
index d7d6d72..4007460 100644
--- a/drivers/gpio/gpio-mpc8xxx.c
+++ b/drivers/gpio/gpio-mpc8xxx.c
@@ -18,6 +18,7 @@
 #include <linux/gpio.h>
 #include <linux/slab.h>
 #include <linux/irq.h>
+#include <linux/ipipe.h>
 
 #define MPC8XXX_GPIO_PINS	32
 
@@ -166,8 +167,8 @@ static void mpc8xxx_gpio_irq_cascade(unsigned int irq, struct irq_desc *desc)
 
 	mask = in_be32(mm->regs + GPIO_IER) & in_be32(mm->regs + GPIO_IMR);
 	if (mask)
-		generic_handle_irq(irq_linear_revmap(mpc8xxx_gc->irq,
-						     32 - ffs(mask)));
+		ipipe_handle_demuxed_irq(irq_linear_revmap(mpc8xxx_gc->irq,
+							   32 - ffs(mask)));
 	if (chip->irq_eoi)
 		chip->irq_eoi(&desc->irq_data);
 }
diff --git a/drivers/tty/serial/mpc52xx_uart.c b/drivers/tty/serial/mpc52xx_uart.c
index a5f4e36..d32d84b 100644
--- a/drivers/tty/serial/mpc52xx_uart.c
+++ b/drivers/tty/serial/mpc52xx_uart.c
@@ -1878,6 +1878,40 @@ mpc52xx_uart_of_enumerate(void)
 	}
 }
 
+#if defined(CONFIG_SERIAL_MPC52xx_CONSOLE) && defined(CONFIG_IPIPE_DEBUG)
+
+#include <stdarg.h>
+
+void __ipipe_serial_debug(const char *fmt, ...)
+{
+	struct uart_port *port = &mpc52xx_uart_ports[0];
+        unsigned int count, n;
+        unsigned long flags;
+        char buf[128], *s;
+        va_list ap;
+
+	if (psc_ops == NULL)
+		return;
+
+        va_start(ap, fmt);
+        vsprintf(buf, fmt, ap);
+        va_end(ap);
+        count = strlen(buf);
+
+        flags = hard_local_irq_save();
+
+	/* Write all the chars */
+	for (n = 0, s = buf; n < count; n++, s++) {
+		if (*s == '\n')
+			psc_ops->write_char(port, '\r');
+		psc_ops->write_char(port, *s);
+	}
+
+        hard_local_irq_restore(flags);
+}
+
+#endif
+
 MODULE_DEVICE_TABLE(of, mpc52xx_uart_of_match);
 
 static struct platform_driver mpc52xx_uart_of_driver = {
diff --git a/drivers/cpuidle/Kconfig b/drivers/cpuidle/Kconfig
index c5029c1..2d68014 100644
--- a/drivers/cpuidle/Kconfig
+++ b/drivers/cpuidle/Kconfig
@@ -3,6 +3,7 @@ menu "CPU Idle"
 config CPU_IDLE
 	bool "CPU idle PM support"
 	default y if ACPI || PPC_PSERIES
+	depends on !(ARCH_OMAP4 && IPIPE)
 	select CPU_IDLE_GOV_LADDER if (!NO_HZ && !NO_HZ_IDLE)
 	select CPU_IDLE_GOV_MENU if (NO_HZ || NO_HZ_IDLE)
 	help
diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c
index 74a1767..7cf4766 100644
--- a/drivers/iommu/irq_remapping.c
+++ b/drivers/iommu/irq_remapping.c
@@ -366,12 +366,12 @@ void panic_if_irq_remap(const char *msg)
 
 static void ir_ack_apic_edge(struct irq_data *data)
 {
-	ack_APIC_irq();
+	__ack_APIC_irq();
 }
 
 static void ir_ack_apic_level(struct irq_data *data)
 {
-	ack_APIC_irq();
+	__ack_APIC_irq();
 	eoi_ioapic_irq(data->irq, data->chip_data);
 }
 
diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c
index e5c31ea..8dce8751 100644
--- a/drivers/tty/serial/8250/8250_core.c
+++ b/drivers/tty/serial/8250/8250_core.c
@@ -3480,6 +3480,84 @@ static int serial8250_resume(struct platform_device *dev)
 	return 0;
 }
 
+#if defined(CONFIG_IPIPE_DEBUG) && defined(CONFIG_SERIAL_8250_CONSOLE)
+
+static IPIPE_DEFINE_SPINLOCK(ipipe_8250_lock);
+
+#include <stdarg.h>
+
+static void wait_for_xmitr_nodelay(struct uart_8250_port *up, int bits)
+{
+	unsigned int status, tmout = 10000;
+
+	for (;;) {
+		status = serial_in(up, UART_LSR);
+
+		up->lsr_saved_flags |= status & LSR_SAVE_FLAGS;
+
+		if ((status & bits) == bits)
+			break;
+		if (--tmout == 0)
+			break;
+		cpu_relax();
+	}
+}
+
+static void serial8250_console_putchar_nodelay(struct uart_port *port, int ch)
+{
+	struct uart_8250_port *up =
+		container_of(port, struct uart_8250_port, port);
+
+	wait_for_xmitr_nodelay(up, UART_LSR_THRE);
+	serial_port_out(port, UART_TX, ch);
+}
+
+void __weak __ipipe_serial_debug(const char *fmt, ...)
+{
+        struct uart_8250_port *up = &serial8250_ports[0];
+        unsigned int ier, count;
+        unsigned long flags;
+        char buf[128];
+        va_list ap;
+
+	if (up->port.membase == NULL
+	    && up->port.iobase == 0
+	    && up->port.mapbase == 0)
+		return;
+
+        va_start(ap, fmt);
+        vsprintf(buf, fmt, ap);
+        va_end(ap);
+        count = strlen(buf);
+
+        touch_nmi_watchdog();
+
+        spin_lock_irqsave(&ipipe_8250_lock, flags);
+
+        /*
+         *      First save the IER then disable the interrupts
+        */
+        ier = serial_in(up, UART_IER);
+
+        if (up->capabilities & UART_CAP_UUE)
+                serial_out(up, UART_IER, UART_IER_UUE);
+        else
+                serial_out(up, UART_IER, 0);
+
+        uart_console_write(&up->port, buf, count, serial8250_console_putchar_nodelay);
+
+        /*
+         *      Finally, wait for transmitter to become empty
+         *      and restore the IER
+         */
+        wait_for_xmitr_nodelay(up, BOTH_EMPTY);
+        serial_out(up, UART_IER, ier);
+
+        spin_unlock_irqrestore(&ipipe_8250_lock, flags);
+}
+
+#endif
+
 static struct platform_driver serial8250_isa_driver = {
 	.probe		= serial8250_probe,
 	.remove		= serial8250_remove,
diff --git a/fs/exec.c b/fs/exec.c
index b7a5f46..0582bd6 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -823,6 +823,7 @@ static int exec_mmap(struct mm_struct *mm)
 {
 	struct task_struct *tsk;
 	struct mm_struct *old_mm, *active_mm;
+	unsigned long flags;
 
 	/* Notify parent that we're no longer interested in the old VM */
 	tsk = current;
@@ -846,8 +847,10 @@ static int exec_mmap(struct mm_struct *mm)
 	task_lock(tsk);
 	active_mm = tsk->active_mm;
 	tsk->mm = mm;
+	ipipe_mm_switch_protect(flags);
 	tsk->active_mm = mm;
 	activate_mm(active_mm, mm);
+	ipipe_mm_switch_unprotect(flags);
 	tsk->mm->vmacache_seqnum = 0;
 	vmacache_flush(tsk);
 	task_unlock(tsk);
diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h
index 1973ad2..c3f2321 100644
--- a/include/asm-generic/atomic.h
+++ b/include/asm-generic/atomic.h
@@ -70,9 +70,9 @@ static inline void atomic_##op(int i, atomic_t *v)			\
 {									\
 	unsigned long flags;						\
 									\
-	raw_local_irq_save(flags);					\
+	flags = hard_local_irq_save();					\
 	v->counter = v->counter c_op i;					\
-	raw_local_irq_restore(flags);					\
+	hard_local_irq_restore(flags);					\
 }
 
 #define ATOMIC_OP_RETURN(op, c_op)					\
@@ -81,9 +81,9 @@ static inline int atomic_##op##_return(int i, atomic_t *v)		\
 	unsigned long flags;						\
 	int ret;							\
 									\
-	raw_local_irq_save(flags);					\
+	flags = hard_local_irq_save();					\
 	ret = (v->counter = v->counter c_op i);				\
-	raw_local_irq_restore(flags);					\
+	hard_local_irq_restore(flags);					\
 									\
 	return ret;							\
 }
diff --git a/include/asm-generic/bitops/atomic.h b/include/asm-generic/bitops/atomic.h
index 4967351..ba3d54d 100644
--- a/include/asm-generic/bitops/atomic.h
+++ b/include/asm-generic/bitops/atomic.h
@@ -21,20 +21,20 @@ extern arch_spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] __lock_aligned;
  * this is the substitute */
 #define _atomic_spin_lock_irqsave(l,f) do {	\
 	arch_spinlock_t *s = ATOMIC_HASH(l);	\
-	local_irq_save(f);			\
+	(f) = hard_local_irq_save();		\
 	arch_spin_lock(s);			\
 } while(0)
 
 #define _atomic_spin_unlock_irqrestore(l,f) do {	\
 	arch_spinlock_t *s = ATOMIC_HASH(l);		\
 	arch_spin_unlock(s);				\
-	local_irq_restore(f);				\
+	hard_local_irq_restore(f);			\
 } while(0)
 
 
 #else
-#  define _atomic_spin_lock_irqsave(l,f) do { local_irq_save(f); } while (0)
-#  define _atomic_spin_unlock_irqrestore(l,f) do { local_irq_restore(f); } while (0)
+#  define _atomic_spin_lock_irqsave(l,f) do { (f) = hard_local_irq_save(); } while (0)
+#  define _atomic_spin_unlock_irqrestore(l,f) do { hard_local_irq_restore(f); } while (0)
 #endif
 
 /*
diff --git a/include/asm-generic/cmpxchg-local.h b/include/asm-generic/cmpxchg-local.h
index 70bef78..145e071 100644
--- a/include/asm-generic/cmpxchg-local.h
+++ b/include/asm-generic/cmpxchg-local.h
@@ -22,7 +22,7 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr,
 	if (size == 8 && sizeof(unsigned long) != 8)
 		wrong_size_cmpxchg(ptr);
 
-	raw_local_irq_save(flags);
+	flags = hard_local_irq_save();
 	switch (size) {
 	case 1: prev = *(u8 *)ptr;
 		if (prev == old)
@@ -43,7 +43,7 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr,
 	default:
 		wrong_size_cmpxchg(ptr);
 	}
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 	return prev;
 }
 
@@ -56,11 +56,11 @@ static inline u64 __cmpxchg64_local_generic(volatile void *ptr,
 	u64 prev;
 	unsigned long flags;
 
-	raw_local_irq_save(flags);
+	flags = hard_local_irq_save();
 	prev = *(u64 *)ptr;
 	if (prev == old)
 		*(u64 *)ptr = new;
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 	return prev;
 }
 
diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h
index 4d9f233..f95a36d 100644
--- a/include/asm-generic/percpu.h
+++ b/include/asm-generic/percpu.h
@@ -43,11 +43,33 @@ extern unsigned long __per_cpu_offset[NR_CPUS];
 #define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
 #endif
 
+#ifdef CONFIG_IPIPE
+#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP)
+extern int __ipipe_check_percpu_access(void);
+#define __ipipe_cpu_offset					\
+	({							\
+		WARN_ON_ONCE(__ipipe_check_percpu_access());	\
+		__my_cpu_offset;				\
+	})
+#else
+#define __ipipe_cpu_offset  __my_cpu_offset
+#endif
+#ifndef __ipipe_raw_cpu_ptr
+#define __ipipe_raw_cpu_ptr(ptr)  SHIFT_PERCPU_PTR(ptr, __ipipe_cpu_offset)
+#endif
+#define __ipipe_raw_cpu_read(var) (*__ipipe_raw_cpu_ptr(&(var)))
+#endif /* CONFIG_IPIPE */
+
 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
 extern void setup_per_cpu_areas(void);
 #endif
 
-#endif	/* SMP */
+#else /* !SMP */
+
+#define __ipipe_raw_cpu_ptr(ptr)  VERIFY_PERCPU_PTR(ptr)
+#define __ipipe_raw_cpu_read(var) (*__ipipe_raw_cpu_ptr(&(var)))
+
+#endif	/* !SMP */
 
 #ifndef PER_CPU_BASE_SECTION
 #ifdef CONFIG_SMP
@@ -117,19 +139,19 @@ do {									\
 #define this_cpu_generic_to_op(pcp, val, op)				\
 do {									\
 	unsigned long __flags;						\
-	raw_local_irq_save(__flags);					\
+	__flags = hard_local_irq_save();				\
 	*raw_cpu_ptr(&(pcp)) op val;					\
-	raw_local_irq_restore(__flags);					\
+	hard_local_irq_restore(__flags);				\
 } while (0)
 
 #define this_cpu_generic_add_return(pcp, val)				\
 ({									\
 	typeof(pcp) __ret;						\
 	unsigned long __flags;						\
-	raw_local_irq_save(__flags);					\
+	__flags = hard_local_irq_save();				\
 	raw_cpu_add(pcp, val);						\
 	__ret = raw_cpu_read(pcp);					\
-	raw_local_irq_restore(__flags);					\
+	hard_local_irq_restore(__flags);				\
 	__ret;								\
 })
 
@@ -137,10 +159,10 @@ do {									\
 ({									\
 	typeof(pcp) __ret;						\
 	unsigned long __flags;						\
-	raw_local_irq_save(__flags);					\
+	__flags = hard_local_irq_save();				\
 	__ret = raw_cpu_read(pcp);					\
 	raw_cpu_write(pcp, nval);					\
-	raw_local_irq_restore(__flags);					\
+	hard_local_irq_restore(__flags);				\
 	__ret;								\
 })
 
@@ -148,11 +170,11 @@ do {									\
 ({									\
 	typeof(pcp) __ret;						\
 	unsigned long __flags;						\
-	raw_local_irq_save(__flags);					\
+	__flags = hard_local_irq_save();				\
 	__ret = raw_cpu_read(pcp);					\
 	if (__ret == (oval))						\
 		raw_cpu_write(pcp, nval);				\
-	raw_local_irq_restore(__flags);					\
+	hard_local_irq_restore(__flags);				\
 	__ret;								\
 })
 
@@ -160,10 +182,10 @@ do {									\
 ({									\
 	int __ret;							\
 	unsigned long __flags;						\
-	raw_local_irq_save(__flags);					\
+	__flags = hard_local_irq_save();				\
 	__ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2,		\
 			oval1, oval2, nval1, nval2);			\
-	raw_local_irq_restore(__flags);					\
+	hard_local_irq_restore(__flags);				\
 	__ret;								\
 })
 
diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h
index 1cd3f5d..acb3fc8 100644
--- a/include/asm-generic/preempt.h
+++ b/include/asm-generic/preempt.h
@@ -53,16 +53,19 @@ static __always_inline bool test_preempt_need_resched(void)
 
 static __always_inline void __preempt_count_add(int val)
 {
+	ipipe_preempt_root_only();
 	*preempt_count_ptr() += val;
 }
 
 static __always_inline void __preempt_count_sub(int val)
 {
+	ipipe_preempt_root_only();
 	*preempt_count_ptr() -= val;
 }
 
 static __always_inline bool __preempt_count_dec_and_test(void)
 {
+	ipipe_preempt_root_only();
 	/*
 	 * Because of load-store architectures cannot do per-cpu atomic
 	 * operations; we cannot use PREEMPT_NEED_RESCHED because it might get
diff --git a/include/ipipe/setup.h b/include/ipipe/setup.h
new file mode 100644
index 0000000..c2bc521
--- /dev/null
+++ b/include/ipipe/setup.h
@@ -0,0 +1,10 @@
+#ifndef _IPIPE_SETUP_H
+#define _IPIPE_SETUP_H
+
+/*
+ * Placeholders for setup hooks defined by client domains.
+ */
+
+static inline void __ipipe_early_client_setup(void) { }
+
+#endif /* !_IPIPE_SETUP_H */
diff --git a/include/ipipe/thread_info.h b/include/ipipe/thread_info.h
new file mode 100644
index 0000000..1f6e9c3
--- /dev/null
+++ b/include/ipipe/thread_info.h
@@ -0,0 +1,14 @@
+#ifndef _IPIPE_THREAD_INFO_H
+#define _IPIPE_THREAD_INFO_H
+
+/*
+ * Placeholder for private thread information defined by client
+ * domains.
+ */
+
+struct ipipe_threadinfo {
+};
+
+static inline void __ipipe_init_threadinfo(struct ipipe_threadinfo *p) { }
+
+#endif /* !_IPIPE_THREAD_INFO_H */
diff --git a/include/linux/clockchips.h b/include/linux/clockchips.h
index 2e4cb67..0cd8b80 100644
--- a/include/linux/clockchips.h
+++ b/include/linux/clockchips.h
@@ -123,6 +123,15 @@ struct clock_event_device {
 	const struct cpumask	*cpumask;
 	struct list_head	list;
 	struct module		*owner;
+
+#ifdef CONFIG_IPIPE
+	struct ipipe_timer      *ipipe_timer;
+	unsigned                ipipe_stolen;
+
+#define clockevent_ipipe_stolen(evt) ((evt)->ipipe_stolen)
+#else
+#define clockevent_ipipe_stolen(evt) (0)
+#endif /* !CONFIG_IPIPE */
 } ____cacheline_aligned;
 
 /*
diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h
index abcafaa..a8440e4 100644
--- a/include/linux/clocksource.h
+++ b/include/linux/clocksource.h
@@ -196,6 +196,10 @@ struct clocksource {
 	cycle_t wd_last;
 #endif
 	struct module *owner;
+#ifdef CONFIG_IPIPE_WANT_CLOCKSOURCE
+	cycle_t (*ipipe_read)(struct clocksource *cs);
+#endif /* CONFIG_IPIPE_WANT_CLOCKSOURCE */
+
 } ____cacheline_aligned;
 
 /*
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index 662697b..6a9b6ad 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -108,6 +108,7 @@ enum {
 	FTRACE_OPS_FL_ADDING			= 1 << 9,
 	FTRACE_OPS_FL_REMOVING			= 1 << 10,
 	FTRACE_OPS_FL_MODIFYING			= 1 << 11,
+	FTRACE_OPS_FL_IPIPE_EXCLUSIVE		= 1 << 12,
 };
 
 #ifdef CONFIG_DYNAMIC_FTRACE
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index cba442e..b513a46 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -61,6 +61,7 @@ extern void irq_exit(void);
 
 #define nmi_enter()						\
 	do {							\
+		__ipipe_nmi_enter();				\
 		lockdep_off();					\
 		ftrace_nmi_enter();				\
 		BUG_ON(in_nmi());				\
@@ -77,6 +78,7 @@ extern void irq_exit(void);
 		preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET);	\
 		ftrace_nmi_exit();				\
 		lockdep_on();					\
+		__ipipe_nmi_exit();				\
 	} while (0)
 
 #endif /* LINUX_HARDIRQ_H */
diff --git a/include/linux/i8253.h b/include/linux/i8253.h
index e6bb36a..898a91a 100644
--- a/include/linux/i8253.h
+++ b/include/linux/i8253.h
@@ -12,6 +12,7 @@
 #include <linux/param.h>
 #include <linux/spinlock.h>
 #include <linux/timex.h>
+#include <linux/ipipe_lock.h>
 
 /* i8253A PIT registers */
 #define PIT_MODE	0x43
@@ -20,7 +21,7 @@
 
 #define PIT_LATCH	((PIT_TICK_RATE + HZ/2) / HZ)
 
-extern raw_spinlock_t i8253_lock;
+IPIPE_DECLARE_RAW_SPINLOCK(i8253_lock);
 extern struct clock_event_device i8253_clockevent;
 extern void clockevent_i8253_init(bool oneshot);
 
diff --git a/include/linux/ipipe.h b/include/linux/ipipe.h
new file mode 100644
index 0000000..0a9b5b6
--- /dev/null
+++ b/include/linux/ipipe.h
@@ -0,0 +1,461 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe.h
+ *
+ * Copyright (C) 2002-2014 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_H
+#define __LINUX_IPIPE_H
+
+#include <linux/spinlock.h>
+#include <linux/cache.h>
+#include <linux/percpu.h>
+#include <linux/irq.h>
+#include <linux/thread_info.h>
+#include <linux/ipipe_base.h>
+#include <linux/ipipe_debug.h>
+#include <asm/ptrace.h>
+#include <asm/ipipe.h>
+
+#ifdef CONFIG_IPIPE
+
+#include <linux/ipipe_domain.h>
+
+/* ipipe_set_hooks(..., enables) */
+#define IPIPE_SYSCALL	__IPIPE_SYSCALL_E
+#define IPIPE_TRAP	__IPIPE_TRAP_E
+#define IPIPE_KEVENT	__IPIPE_KEVENT_E
+
+struct ipipe_sysinfo {
+	int sys_nr_cpus;	/* Number of CPUs on board */
+	int sys_hrtimer_irq;	/* hrtimer device IRQ */
+	u64 sys_hrtimer_freq;	/* hrtimer device frequency */
+	u64 sys_hrclock_freq;	/* hrclock device frequency */
+	u64 sys_cpu_freq;	/* CPU frequency (Hz) */
+	struct ipipe_arch_sysinfo arch;
+};
+
+struct ipipe_work_header {
+	size_t size;
+	void (*handler)(struct ipipe_work_header *work);
+};
+
+extern unsigned int __ipipe_printk_virq;
+
+void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned int irq);
+
+void __ipipe_complete_domain_migration(void);
+
+int __ipipe_switch_tail(void);
+
+void __ipipe_share_current(int flags);
+
+void __ipipe_arch_share_current(int flags);
+
+int __ipipe_migrate_head(void);
+
+void __ipipe_reenter_root(void);
+
+int __ipipe_disable_ondemand_mappings(struct task_struct *p);
+
+int __ipipe_pin_vma(struct mm_struct *mm, struct vm_area_struct *vma);
+
+#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH
+
+#define prepare_arch_switch(next)			\
+	do {						\
+		hard_local_irq_enable();		\
+		__ipipe_report_schedule(current, next);	\
+	} while(0)
+
+#ifndef ipipe_get_active_mm
+static inline struct mm_struct *ipipe_get_active_mm(void)
+{
+	return __this_cpu_read(ipipe_percpu.active_mm);
+}
+#define ipipe_get_active_mm ipipe_get_active_mm
+#endif
+
+#else /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */
+
+#define prepare_arch_switch(next)			\
+	do {						\
+		__ipipe_report_schedule(current, next);	\
+		hard_local_irq_disable();		\
+	} while(0)
+
+#ifndef ipipe_get_active_mm
+#define ipipe_get_active_mm()	(current->active_mm)
+#endif
+
+#endif /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */
+
+#ifdef CONFIG_IPIPE_WANT_CLOCKSOURCE
+
+extern unsigned long long __ipipe_cs_freq;
+
+extern struct clocksource *__ipipe_cs;
+
+#endif /* CONFIG_IPIPE_WANT_CLOCKSOURCE */
+
+static inline bool __ipipe_hrclock_ok(void)
+{
+	return __ipipe_hrclock_freq != 0;
+}
+
+static inline void __ipipe_nmi_enter(void)
+{
+	__this_cpu_write(ipipe_percpu.nmi_state, __ipipe_root_status);
+	__set_bit(IPIPE_STALL_FLAG, &__ipipe_root_status);
+	ipipe_save_context_nmi();
+}
+
+static inline void __ipipe_nmi_exit(void)
+{
+	ipipe_restore_context_nmi();
+	if (!test_bit(IPIPE_STALL_FLAG, raw_cpu_ptr(&ipipe_percpu.nmi_state)))
+		__clear_bit(IPIPE_STALL_FLAG, &__ipipe_root_status);
+}
+
+/* KVM-side calls, hw IRQs off. */
+static inline void __ipipe_enter_vm(struct ipipe_vm_notifier *vmf)
+{
+	struct ipipe_percpu_data *p;
+
+	p = raw_cpu_ptr(&ipipe_percpu);
+	p->vm_notifier = vmf;
+	barrier();
+}
+
+static inline void __ipipe_exit_vm(void)
+{
+	struct ipipe_percpu_data *p;
+
+	p = raw_cpu_ptr(&ipipe_percpu);
+	p->vm_notifier = NULL;
+	barrier();
+}
+
+/* Client-side call, hw IRQs off. */
+void __ipipe_notify_vm_preemption(void);
+
+static inline void __ipipe_sync_pipeline(struct ipipe_domain *top)
+{
+	if (__ipipe_current_domain != top) {
+		__ipipe_do_sync_pipeline(top);
+		return;
+	}
+	if (!test_bit(IPIPE_STALL_FLAG, &ipipe_this_cpu_context(top)->status))
+		__ipipe_sync_stage();
+}
+
+void ipipe_register_head(struct ipipe_domain *ipd,
+			 const char *name);
+
+void ipipe_unregister_head(struct ipipe_domain *ipd);
+
+int ipipe_request_irq(struct ipipe_domain *ipd,
+		      unsigned int irq,
+		      ipipe_irq_handler_t handler,
+		      void *cookie,
+		      ipipe_irq_ackfn_t ackfn);
+
+void ipipe_free_irq(struct ipipe_domain *ipd,
+		    unsigned int irq);
+
+void ipipe_raise_irq(unsigned int irq);
+
+void ipipe_set_hooks(struct ipipe_domain *ipd,
+		     int enables);
+
+unsigned int ipipe_alloc_virq(void);
+
+void ipipe_free_virq(unsigned int virq);
+
+static inline void ipipe_post_irq_head(unsigned int irq)
+{
+	__ipipe_set_irq_pending(ipipe_head_domain, irq);
+}
+
+static inline void ipipe_post_irq_root(unsigned int irq)
+{
+	__ipipe_set_irq_pending(&ipipe_root, irq);
+}
+
+static inline void ipipe_stall_head(void)
+{
+	hard_local_irq_disable();
+	__set_bit(IPIPE_STALL_FLAG, &__ipipe_head_status);
+}
+
+static inline unsigned long ipipe_test_and_stall_head(void)
+{
+	hard_local_irq_disable();
+	return __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_head_status);
+}
+
+static inline unsigned long ipipe_test_head(void)
+{
+	unsigned long flags, ret;
+
+	flags = hard_smp_local_irq_save();
+	ret = test_bit(IPIPE_STALL_FLAG, &__ipipe_head_status);
+	hard_smp_local_irq_restore(flags);
+
+	return ret;
+}
+
+void ipipe_unstall_head(void);
+
+void __ipipe_restore_head(unsigned long x);
+
+static inline void ipipe_restore_head(unsigned long x)
+{
+	ipipe_check_irqoff();
+	if ((x ^ test_bit(IPIPE_STALL_FLAG, &__ipipe_head_status)) & 1)
+		__ipipe_restore_head(x);
+}
+
+void __ipipe_post_work_root(struct ipipe_work_header *work);
+
+#define ipipe_post_work_root(p, header)			\
+	do {						\
+		void header_not_at_start(void);		\
+		if (offsetof(typeof(*(p)), header)) {	\
+			header_not_at_start();		\
+		}					\
+		__ipipe_post_work_root(&(p)->header);	\
+	} while (0)
+
+int ipipe_get_sysinfo(struct ipipe_sysinfo *sysinfo);
+
+unsigned long ipipe_critical_enter(void (*syncfn)(void));
+
+void ipipe_critical_exit(unsigned long flags);
+
+void ipipe_prepare_panic(void);
+
+#ifdef CONFIG_SMP
+#ifndef ipipe_smp_p
+#define ipipe_smp_p (1)
+#endif
+void ipipe_set_irq_affinity(unsigned int irq, cpumask_t cpumask);
+void ipipe_send_ipi(unsigned int ipi, cpumask_t cpumask);
+#else  /* !CONFIG_SMP */
+#define ipipe_smp_p (0)
+static inline
+void ipipe_set_irq_affinity(unsigned int irq, cpumask_t cpumask) { }
+static inline void ipipe_send_ipi(unsigned int ipi, cpumask_t cpumask) { }
+static inline void ipipe_disable_smp(void) { }
+#endif	/* CONFIG_SMP */
+
+static inline void ipipe_restore_root_nosync(unsigned long x)
+{
+	unsigned long flags;
+
+	flags = hard_smp_local_irq_save();
+	__ipipe_restore_root_nosync(x);
+	hard_smp_local_irq_restore(flags);
+}
+
+/* Must be called hw IRQs off. */
+static inline void ipipe_lock_irq(unsigned int irq)
+{
+	struct ipipe_domain *ipd = __ipipe_current_domain;
+	if (ipd == ipipe_root_domain)
+		__ipipe_lock_irq(irq);
+}
+
+/* Must be called hw IRQs off. */
+static inline void ipipe_unlock_irq(unsigned int irq)
+{
+	struct ipipe_domain *ipd = __ipipe_current_domain;
+	if (ipd == ipipe_root_domain)
+		__ipipe_unlock_irq(irq);
+}
+
+static inline struct ipipe_threadinfo *ipipe_current_threadinfo(void)
+{
+	return &current_thread_info()->ipipe_data;
+}
+
+#define ipipe_task_threadinfo(p) (&task_thread_info(p)->ipipe_data)
+
+void ipipe_enable_irq(unsigned int irq);
+
+static inline void ipipe_disable_irq(unsigned int irq)
+{
+	struct irq_desc *desc;
+	struct irq_chip *chip;
+
+	desc = irq_to_desc(irq);
+	if (desc == NULL)
+		return;
+
+	chip = irq_desc_get_chip(desc);
+
+	if (WARN_ON_ONCE(chip->irq_disable == NULL && chip->irq_mask == NULL))
+		return;
+
+	if (chip->irq_disable)
+		chip->irq_disable(&desc->irq_data);
+	else
+		chip->irq_mask(&desc->irq_data);
+}
+
+static inline void ipipe_end_irq(unsigned int irq)
+{
+	struct irq_desc *desc = irq_to_desc(irq);
+
+	if (desc)
+		desc->ipipe_end(irq, desc);
+}
+
+static inline int ipipe_chained_irq_p(struct irq_desc *desc)
+{
+	void __ipipe_chained_irq(unsigned irq, struct irq_desc *desc);
+
+	return desc->handle_irq == __ipipe_chained_irq;
+}
+
+static inline void ipipe_handle_demuxed_irq(unsigned int cascade_irq)
+{
+	ipipe_trace_irq_entry(cascade_irq);
+	__ipipe_dispatch_irq(cascade_irq, IPIPE_IRQF_NOSYNC);
+	ipipe_trace_irq_exit(cascade_irq);
+}
+
+static inline void __ipipe_init_threadflags(struct thread_info *ti)
+{
+	ti->ipipe_flags = 0;
+}
+
+static inline
+void ipipe_set_ti_thread_flag(struct thread_info *ti, int flag)
+{
+	set_bit(flag, &ti->ipipe_flags);
+}
+
+static inline
+void ipipe_clear_ti_thread_flag(struct thread_info *ti, int flag)
+{
+	clear_bit(flag, &ti->ipipe_flags);
+}
+
+static inline
+void ipipe_test_and_clear_ti_thread_flag(struct thread_info *ti, int flag)
+{
+	test_and_clear_bit(flag, &ti->ipipe_flags);
+}
+
+static inline
+int ipipe_test_ti_thread_flag(struct thread_info *ti, int flag)
+{
+	return test_bit(flag, &ti->ipipe_flags);
+}
+
+#define ipipe_set_thread_flag(flag) \
+	ipipe_set_ti_thread_flag(current_thread_info(), flag)
+
+#define ipipe_clear_thread_flag(flag) \
+	ipipe_clear_ti_thread_flag(current_thread_info(), flag)
+
+#define ipipe_test_and_clear_thread_flag(flag) \
+	ipipe_test_and_clear_ti_thread_flag(current_thread_info(), flag)
+
+#define ipipe_test_thread_flag(flag) \
+	ipipe_test_ti_thread_flag(current_thread_info(), flag)
+
+#define ipipe_enable_notifier(p)					\
+	ipipe_set_ti_thread_flag(task_thread_info(p), TIP_NOTIFY)
+
+#define ipipe_disable_notifier(p)					\
+	do {								\
+		struct thread_info *ti = task_thread_info(p);		\
+		ipipe_clear_ti_thread_flag(ti, TIP_NOTIFY);		\
+		ipipe_clear_ti_thread_flag(ti, TIP_MAYDAY);		\
+	} while (0)
+
+#define ipipe_notifier_enabled_p(p)					\
+	ipipe_test_ti_thread_flag(task_thread_info(p), TIP_NOTIFY)
+
+#define ipipe_raise_mayday(p)						\
+	do {								\
+		struct thread_info *ti = task_thread_info(p);		\
+		ipipe_check_irqoff();					\
+		if (ipipe_test_ti_thread_flag(ti, TIP_NOTIFY))		\
+			ipipe_set_ti_thread_flag(ti, TIP_MAYDAY);	\
+	} while (0)
+
+extern bool __ipipe_probe_access;
+
+long ipipe_probe_kernel_read(void *dst, void *src, size_t size);
+long ipipe_probe_kernel_write(void *dst, void *src, size_t size);
+
+#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || defined(CONFIG_PROVE_LOCKING) || \
+	defined(CONFIG_PREEMPT_VOLUNTARY) || defined(CONFIG_IPIPE_DEBUG_CONTEXT)
+extern void __ipipe_uaccess_might_fault(void);
+#else
+#define __ipipe_uaccess_might_fault() might_fault()
+#endif
+
+#ifdef CONFIG_IPIPE_TRACE
+void __ipipe_tracer_hrclock_initialized(void);
+#else /* !CONFIG_IPIPE_TRACE */
+#define __ipipe_tracer_hrclock_initialized()	do { } while(0)
+#endif /* !CONFIG_IPIPE_TRACE */
+
+#include <linux/ipipe_compat.h>
+
+#else	/* !CONFIG_IPIPE */
+
+#define __ipipe_root_p		1
+#define ipipe_root_p		1
+
+static inline void __ipipe_init_threadflags(struct thread_info *ti) { }
+
+static inline void __ipipe_complete_domain_migration(void) { }
+
+static inline int __ipipe_switch_tail(void)
+{
+	return 0;
+}
+
+static inline void __ipipe_nmi_enter(void) { }
+
+static inline void __ipipe_nmi_exit(void) { }
+
+#define ipipe_safe_current()	current
+#define ipipe_processor_id()	smp_processor_id()
+
+static inline int ipipe_test_foreign_stack(void)
+{
+	return 0;
+}
+
+static inline void ipipe_lock_irq(unsigned int irq) { }
+
+static inline void ipipe_unlock_irq(unsigned int irq) { }
+
+#define ipipe_probe_kernel_read(d, s, sz)	probe_kernel_read(d, s, sz)
+#define ipipe_probe_kernel_write(d, s, sz)	probe_kernel_write(d, s, sz)
+#define __ipipe_uaccess_might_fault()		might_fault()
+
+#endif	/* !CONFIG_IPIPE */
+
+#endif	/* !__LINUX_IPIPE_H */
diff --git a/include/linux/ipipe_base.h b/include/linux/ipipe_base.h
new file mode 100644
index 0000000..a37358c
--- /dev/null
+++ b/include/linux/ipipe_base.h
@@ -0,0 +1,358 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_base.h
+ *
+ * Copyright (C) 2002-2014 Philippe Gerum.
+ *               2007 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_BASE_H
+#define __LINUX_IPIPE_BASE_H
+
+struct kvm_vcpu;
+struct ipipe_vm_notifier;
+struct irq_desc;
+
+#ifdef CONFIG_IPIPE
+
+#define IPIPE_CORE_APIREV  CONFIG_IPIPE_CORE_APIREV
+
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+void ipipe_root_only(void);
+#else /* !CONFIG_IPIPE_DEBUG_CONTEXT */
+static inline void ipipe_root_only(void) { }
+#endif /* !CONFIG_IPIPE_DEBUG_CONTEXT */
+
+typedef void (*ipipe_irq_handler_t)(unsigned int irq,
+				    void *cookie);
+
+void ipipe_unstall_root(void);
+
+void ipipe_restore_root(unsigned long x);
+
+#include <asm/ipipe_base.h>
+#include <linux/compiler.h>
+#include <linux/linkage.h>
+
+#ifndef IPIPE_NR_ROOT_IRQS
+#define IPIPE_NR_ROOT_IRQS	NR_IRQS
+#endif /* !IPIPE_NR_ROOT_IRQS */
+
+#define __bpl_up(x)		(((x)+(BITS_PER_LONG-1)) & ~(BITS_PER_LONG-1))
+/* Number of virtual IRQs (must be a multiple of BITS_PER_LONG) */
+#define IPIPE_NR_VIRQS		BITS_PER_LONG
+/* First virtual IRQ # (must be aligned on BITS_PER_LONG) */
+#define IPIPE_VIRQ_BASE		__bpl_up(IPIPE_NR_XIRQS)
+/* Total number of IRQ slots */
+#define IPIPE_NR_IRQS		(IPIPE_VIRQ_BASE+IPIPE_NR_VIRQS)
+
+static inline int ipipe_virtual_irq_p(unsigned int irq)
+{
+	return irq >= IPIPE_VIRQ_BASE && irq < IPIPE_NR_IRQS;
+}
+
+#define IPIPE_IRQ_LOMAPSZ	(IPIPE_NR_IRQS / BITS_PER_LONG)
+#if IPIPE_IRQ_LOMAPSZ > BITS_PER_LONG
+/*
+ * We need a 3-level mapping. This allows us to handle up to 32k IRQ
+ * vectors on 32bit machines, 256k on 64bit ones.
+ */
+#define __IPIPE_3LEVEL_IRQMAP	1
+#define IPIPE_IRQ_MDMAPSZ	(__bpl_up(IPIPE_IRQ_LOMAPSZ) / BITS_PER_LONG)
+#else
+/*
+ * 2-level mapping is enough. This allows us to handle up to 1024 IRQ
+ * vectors on 32bit machines, 4096 on 64bit ones.
+ */
+#define __IPIPE_2LEVEL_IRQMAP	1
+#endif
+
+/* Per-cpu pipeline status */
+#define IPIPE_STALL_FLAG	0 /* interrupts (virtually) disabled. */
+#define IPIPE_STALL_MASK	(1L << IPIPE_STALL_FLAG)
+
+/* Interrupt control bits */
+#define IPIPE_HANDLE_FLAG	0
+#define IPIPE_STICKY_FLAG	1
+#define IPIPE_LOCK_FLAG		2
+#define IPIPE_HANDLE_MASK	(1 << IPIPE_HANDLE_FLAG)
+#define IPIPE_STICKY_MASK	(1 << IPIPE_STICKY_FLAG)
+#define IPIPE_LOCK_MASK		(1 << IPIPE_LOCK_FLAG)
+
+struct pt_regs;
+struct ipipe_domain;
+
+struct ipipe_trap_data {
+	int exception;
+	struct pt_regs *regs;
+};
+
+#define IPIPE_KEVT_SCHEDULE	0
+#define IPIPE_KEVT_SIGWAKE	1
+#define IPIPE_KEVT_SETSCHED	2
+#define IPIPE_KEVT_SETAFFINITY	3
+#define IPIPE_KEVT_EXIT		4
+#define IPIPE_KEVT_CLEANUP	5
+#define IPIPE_KEVT_HOSTRT	6
+
+struct ipipe_vm_notifier {
+	void (*handler)(struct ipipe_vm_notifier *nfy);
+};
+
+void __ipipe_init_early(void);
+
+void __ipipe_init(void);
+
+#ifdef CONFIG_PROC_FS
+void __ipipe_init_proc(void);
+#ifdef CONFIG_IPIPE_TRACE
+void __ipipe_init_tracer(void);
+#else /* !CONFIG_IPIPE_TRACE */
+static inline void __ipipe_init_tracer(void) { }
+#endif /* CONFIG_IPIPE_TRACE */
+#else	/* !CONFIG_PROC_FS */
+static inline void __ipipe_init_proc(void) { }
+#endif	/* CONFIG_PROC_FS */
+
+void __ipipe_restore_root_nosync(unsigned long x);
+
+#define IPIPE_IRQF_NOACK    0x1
+#define IPIPE_IRQF_NOSYNC   0x2
+
+void __ipipe_dispatch_irq(unsigned int irq, int flags);
+
+void __ipipe_do_sync_stage(void);
+
+void __ipipe_do_sync_pipeline(struct ipipe_domain *top);
+
+void __ipipe_lock_irq(unsigned int irq);
+
+void __ipipe_unlock_irq(unsigned int irq);
+
+void __ipipe_do_critical_sync(unsigned int irq, void *cookie);
+
+void __ipipe_ack_edge_irq(unsigned int irq, struct irq_desc *desc);
+
+void __ipipe_nop_irq(unsigned int irq, struct irq_desc *desc);
+
+static inline void __ipipe_idle(void)
+{
+	ipipe_unstall_root();
+}
+
+#ifndef __ipipe_sync_check
+#define __ipipe_sync_check	1
+#endif
+
+static inline void __ipipe_sync_stage(void)
+{
+	if (likely(__ipipe_sync_check))
+		__ipipe_do_sync_stage();
+}
+
+#ifndef __ipipe_check_root_resched
+#ifdef CONFIG_PREEMPT
+#define __ipipe_check_root_resched()	\
+	(preempt_count() == 0 && need_resched())
+#else
+#define __ipipe_check_root_resched()	0
+#endif
+#endif
+
+#ifndef __ipipe_run_irqtail
+#define __ipipe_run_irqtail(irq) do { } while(0)
+#endif
+
+void __ipipe_flush_printk(unsigned int irq, void *cookie);
+
+#define __ipipe_get_cpu(flags)	({ (flags) = hard_preempt_disable(); ipipe_processor_id(); })
+#define __ipipe_put_cpu(flags)	hard_preempt_enable(flags)
+
+int __ipipe_notify_syscall(struct pt_regs *regs);
+
+int __ipipe_notify_trap(int exception, struct pt_regs *regs);
+
+int __ipipe_notify_kevent(int event, void *data);
+
+#define __ipipe_report_trap(exception, regs)				\
+	__ipipe_notify_trap(exception, regs)
+
+#define __ipipe_report_sigwake(p)					\
+	do {								\
+		if (ipipe_notifier_enabled_p(p))			\
+			__ipipe_notify_kevent(IPIPE_KEVT_SIGWAKE, p);	\
+	} while (0)
+
+struct ipipe_cpu_migration_data {
+	struct task_struct *task;
+	int dest_cpu;
+};
+
+#define __ipipe_report_setaffinity(__p, __dest_cpu)			\
+	do {								\
+		struct ipipe_cpu_migration_data d = {			\
+			.task = (__p),					\
+			.dest_cpu = (__dest_cpu),			\
+		};							\
+		if (ipipe_notifier_enabled_p(__p))			\
+			__ipipe_notify_kevent(IPIPE_KEVT_SETAFFINITY, &d); \
+	} while (0)
+
+#define __ipipe_report_exit(p)						\
+	do {								\
+		if (ipipe_notifier_enabled_p(p))			\
+			__ipipe_notify_kevent(IPIPE_KEVT_EXIT, p);	\
+	} while (0)
+
+#define __ipipe_report_setsched(p)					\
+	do {								\
+		if (ipipe_notifier_enabled_p(p))			\
+			__ipipe_notify_kevent(IPIPE_KEVT_SETSCHED, p); \
+	} while (0)
+
+#define __ipipe_report_schedule(prev, next)				\
+do {									\
+	if (ipipe_notifier_enabled_p(next) ||				\
+	    ipipe_notifier_enabled_p(prev)) {				\
+		__this_cpu_write(ipipe_percpu.rqlock_owner, prev);	\
+		__ipipe_notify_kevent(IPIPE_KEVT_SCHEDULE, next);	\
+	}								\
+} while (0)
+
+#define __ipipe_report_cleanup(mm)					\
+	__ipipe_notify_kevent(IPIPE_KEVT_CLEANUP, mm)
+
+void __ipipe_notify_vm_preemption(void);
+
+void __ipipe_call_mayday(struct pt_regs *regs);
+
+#define hard_cond_local_irq_enable()		hard_local_irq_enable()
+#define hard_cond_local_irq_disable()		hard_local_irq_disable()
+#define hard_cond_local_irq_save()		hard_local_irq_save()
+#define hard_cond_local_irq_restore(flags)	hard_local_irq_restore(flags)
+
+#ifdef CONFIG_IPIPE_LEGACY
+
+#define IPIPE_FIRST_EVENT	IPIPE_NR_FAULTS
+#define IPIPE_EVENT_SCHEDULE	IPIPE_FIRST_EVENT
+#define IPIPE_EVENT_SIGWAKE	(IPIPE_FIRST_EVENT + 1)
+#define IPIPE_EVENT_SETSCHED	(IPIPE_FIRST_EVENT + 2)
+#define IPIPE_EVENT_SETAFFINITY	(IPIPE_FIRST_EVENT + 3)
+#define IPIPE_EVENT_EXIT	(IPIPE_FIRST_EVENT + 4)
+#define IPIPE_EVENT_CLEANUP	(IPIPE_FIRST_EVENT + 5)
+#define IPIPE_EVENT_HOSTRT	(IPIPE_FIRST_EVENT + 6)
+#define IPIPE_EVENT_SYSCALL	(IPIPE_FIRST_EVENT + 7)
+#define IPIPE_LAST_EVENT	IPIPE_EVENT_SYSCALL
+#define IPIPE_NR_EVENTS		(IPIPE_LAST_EVENT + 1)
+
+typedef int (*ipipe_event_handler_t)(unsigned int event,
+				     struct ipipe_domain *from,
+				     void *data);
+struct ipipe_legacy_context {
+	unsigned int domid;
+	int priority;
+	void *pdd;
+	ipipe_event_handler_t handlers[IPIPE_NR_EVENTS];
+};
+
+#define __ipipe_init_taskinfo(p)			\
+	do {						\
+		memset(p->ptd, 0, sizeof(p->ptd));	\
+	} while (0)
+
+#else /* !CONFIG_IPIPE_LEGACY */
+
+struct ipipe_legacy_context {
+};
+
+static inline void __ipipe_init_taskinfo(struct task_struct *p) { }
+
+#endif /* !CONFIG_IPIPE_LEGACY */
+
+#else /* !CONFIG_IPIPE */
+
+struct task_struct;
+struct mm_struct;
+
+static inline void __ipipe_init_early(void) { }
+
+static inline void __ipipe_init(void) { }
+
+static inline void __ipipe_init_proc(void) { }
+
+static inline void __ipipe_idle(void) { }
+
+static inline void __ipipe_report_sigwake(struct task_struct *p) { }
+
+static inline void __ipipe_report_setaffinity(struct task_struct *p,
+					      int dest_cpu) { }
+
+static inline void __ipipe_report_setsched(struct task_struct *p) { }
+
+static inline void __ipipe_report_exit(struct task_struct *p) { }
+
+static inline void __ipipe_report_cleanup(struct mm_struct *mm) { }
+
+#define __ipipe_report_trap(exception, regs)  0
+
+static inline void __ipipe_init_taskinfo(struct task_struct *p) { }
+
+#define hard_preempt_disable()		({ preempt_disable(); 0; })
+#define hard_preempt_enable(flags)	({ preempt_enable(); (void)(flags); })
+
+#define __ipipe_get_cpu(flags)		({ (void)(flags); get_cpu(); })
+#define __ipipe_put_cpu(flags)		\
+	do {				\
+		(void)(flags);		\
+		put_cpu();		\
+	} while (0)
+
+#define __ipipe_root_tick_p(regs)	1
+
+#define ipipe_handle_demuxed_irq(irq)		generic_handle_irq(irq)
+
+#define __ipipe_serial_debug(fmt, args...)	do { } while (0)
+
+#define __ipipe_enter_vm(vmf)	do { } while (0)
+
+static inline void __ipipe_exit_vm(void) { }
+
+static inline void __ipipe_notify_vm_preemption(void) { }
+
+static inline void ipipe_root_only(void) { }
+
+#endif	/* !CONFIG_IPIPE */
+
+#ifdef CONFIG_IPIPE_WANT_PTE_PINNING
+void __ipipe_pin_mapping_globally(unsigned long start,
+				  unsigned long end);
+#else
+static inline void __ipipe_pin_mapping_globally(unsigned long start,
+						unsigned long end)
+{ }
+#endif
+
+static inline void ipipe_preempt_root_only(void)
+{
+#if defined(CONFIG_IPIPE_DEBUG_CONTEXT) && \
+    defined(CONFIG_IPIPE_LEGACY) && \
+    !defined(CONFIG_IPIPE_HAVE_SAFE_THREAD_INFO)
+	ipipe_root_only();
+#endif
+}
+
+#endif	/* !__LINUX_IPIPE_BASE_H */
diff --git a/include/linux/ipipe_compat.h b/include/linux/ipipe_compat.h
new file mode 100644
index 0000000..6521852
--- /dev/null
+++ b/include/linux/ipipe_compat.h
@@ -0,0 +1,319 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_compat.h
+ *
+ * Copyright (C) 2012 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_COMPAT_H
+#define __LINUX_IPIPE_COMPAT_H
+
+#ifndef __LINUX_IPIPE_H
+#error "Do not include this file directly, use linux/ipipe.h instead"
+#endif
+
+#ifdef CONFIG_IPIPE_LEGACY
+
+#define IPIPE_HEAD_PRIORITY	(-1)
+#define IPIPE_ROOT_PRIO		100
+#define IPIPE_ROOT_ID		0
+#define IPIPE_ROOT_NPTDKEYS	4
+
+/* Legacy pipeline status bit */
+#define IPIPE_NOSTACK_FLAG	1 /* running on foreign stack. */
+#define IPIPE_NOSTACK_MASK	(1L << IPIPE_NOSTACK_FLAG)
+
+/* Legacy interrupt control bits */
+#define IPIPE_DUMMY_FLAG	31
+#define IPIPE_WIRED_FLAG	IPIPE_HANDLE_FLAG
+#define IPIPE_WIRED_MASK	(1 << IPIPE_WIRED_FLAG)
+#define IPIPE_PASS_FLAG		IPIPE_DUMMY_FLAG
+#define IPIPE_PASS_MASK		(1 << IPIPE_PASS_FLAG)
+#define IPIPE_DYNAMIC_FLAG	IPIPE_HANDLE_FLAG
+#define IPIPE_DYNAMIC_MASK	(1 << IPIPE_DYNAMIC_FLAG)
+#define IPIPE_SYSTEM_FLAG	IPIPE_DUMMY_FLAG
+#define IPIPE_SYSTEM_MASK	(1 << IPIPE_SYSTEM_FLAG)
+#define IPIPE_EXCLUSIVE_FLAG	IPIPE_DUMMY_FLAG
+#define IPIPE_EXCLUSIVE_MASK	(1 << IPIPE_EXCLUSIVE_FLAG)
+
+#define IPIPE_NR_CPUS		NR_CPUS
+
+#define IPIPE_EVENT_SELF        0x80000000
+#define IPIPE_EVENT_RETURN	IPIPE_TRAP_MAYDAY
+
+#define TASK_ATOMICSWITCH	TASK_HARDENING
+
+struct ipipe_domain_attr {
+	unsigned int domid;
+	const char *name;
+	int priority;
+	void (*entry) (void);
+	void *pdd;
+};
+
+void ipipe_init_attr(struct ipipe_domain_attr *attr);
+
+int ipipe_register_domain(struct ipipe_domain *ipd,
+			  struct ipipe_domain_attr *attr);
+
+int ipipe_unregister_domain(struct ipipe_domain *ipd);
+
+int ipipe_alloc_ptdkey(void);
+
+int ipipe_free_ptdkey(int key);
+
+int ipipe_set_ptd(int key, void *value);
+
+void *ipipe_get_ptd(int key);
+
+int ipipe_virtualize_irq(struct ipipe_domain *ipd,
+			 unsigned int irq,
+			 ipipe_irq_handler_t handler,
+			 void *cookie,
+			 ipipe_irq_ackfn_t ackfn,
+			 unsigned int modemask);
+
+ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd,
+					unsigned int event,
+					ipipe_event_handler_t handler);
+
+int ipipe_setscheduler_root(struct task_struct *p,
+			    int policy,
+			    int prio);
+
+static inline void ipipe_check_context(struct ipipe_domain *border_ipd)
+{
+	ipipe_root_only();
+}
+
+static inline void ipipe_set_printk_sync(struct ipipe_domain *ipd)
+{
+	ipipe_prepare_panic();
+}
+
+static inline void __ipipe_propagate_irq(unsigned int irq)
+{
+	ipipe_post_irq_root(irq);
+}
+
+static inline void __ipipe_schedule_irq_head(unsigned int irq)
+{
+	ipipe_post_irq_head(irq);
+}
+
+static inline void __ipipe_schedule_irq_root(unsigned int irq)
+{
+	ipipe_post_irq_root(irq);
+}
+
+static inline int ipipe_trigger_irq(unsigned int irq)
+{
+	ipipe_raise_irq(irq);
+	return 1;
+}
+
+static inline void ipipe_stall_pipeline_from(struct ipipe_domain *ipd)
+{
+	if (ipd != ipipe_root_domain)
+		ipipe_stall_head();
+	else
+		ipipe_stall_root();
+}
+
+static inline
+unsigned long ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd)
+{
+	if (ipd != ipipe_root_domain)
+		return ipipe_test_and_stall_head();
+
+	return ipipe_test_and_stall_root();
+}
+
+static inline
+void ipipe_unstall_pipeline_from(struct ipipe_domain *ipd)
+{
+	if (ipd != ipipe_root_domain)
+		ipipe_unstall_head();
+	else
+		ipipe_unstall_root();
+}
+
+static inline
+void ipipe_restore_pipeline_from(struct ipipe_domain *ipd,
+				 unsigned long x)
+{
+	if (ipd != ipipe_root_domain)
+		ipipe_restore_head(x);
+	else
+		ipipe_restore_root(x);
+}
+
+static inline
+unsigned long ipipe_test_pipeline_from(struct ipipe_domain *ipd)
+{
+	return test_bit(IPIPE_STALL_FLAG, &ipipe_this_cpu_context(ipd)->status);
+}
+
+static inline void ipipe_stall_pipeline_head(void)
+{
+	ipipe_stall_head();
+}
+
+static inline unsigned long ipipe_test_and_stall_pipeline_head(void)
+{
+	return ipipe_test_and_stall_head();
+}
+
+static inline void ipipe_unstall_pipeline_head(void)
+{
+	ipipe_unstall_head();
+}
+
+static inline void ipipe_restore_pipeline_head(unsigned long x)
+{
+	ipipe_restore_head(x);
+}
+
+static inline int ipipe_disable_ondemand_mappings(struct task_struct *p)
+{
+	return __ipipe_disable_ondemand_mappings(p);
+}
+
+static inline int ipipe_reenter_root(struct task_struct *prev,
+				     int policy,
+				     int prio)
+{
+	__ipipe_reenter_root();
+	return 0;
+}
+
+static inline void ipipe_root_preempt_notify(void)
+{
+	ipipe_notify_root_preemption();
+}
+
+#define ipipe_return_notify(p)	ipipe_raise_mayday(p)
+
+/*
+ * Keep the following as a macro, so that client code could check for
+ * the support of the invariant pipeline head optimization.
+ */
+#define __ipipe_pipeline_head() ipipe_head_domain
+
+static inline int irqs_disabled_hw(void)
+{
+	return hard_irqs_disabled();
+}
+
+static inline void local_irq_disable_hw(void)
+{
+	hard_local_irq_disable();
+}
+
+static inline void local_irq_enable_hw(void)
+{
+	hard_local_irq_enable();
+}
+
+#define local_irq_save_hw(flags)			\
+	do {						\
+		(flags) = hard_local_irq_save();	\
+	} while (0)
+
+static inline void local_irq_restore_hw(unsigned long flags)
+{
+	hard_local_irq_restore(flags);
+}
+
+#define local_save_flags_hw(flags)			\
+	do {						\
+		(flags) = hard_local_save_flags();	\
+	} while (0)
+
+#define local_irq_save_hw_smp(flags)			\
+	do {						\
+		(flags) = hard_smp_local_irq_save();	\
+	} while (0)
+#define local_irq_restore_hw_smp(flags)   hard_smp_local_irq_restore(flags)
+
+#define local_irq_save_hw_cond(flags)			\
+	do {						\
+		(flags) = hard_cond_local_irq_save();	\
+	} while (0)
+#define local_irq_restore_hw_cond(flags)  hard_cond_local_irq_restore(flags)
+
+static inline void ipipe_set_foreign_stack(struct ipipe_domain *ipd)
+{
+	/* Must be called hw interrupts off. */
+	__set_bit(IPIPE_NOSTACK_FLAG, &ipipe_this_cpu_context(ipd)->status);
+}
+
+static inline void ipipe_clear_foreign_stack(struct ipipe_domain *ipd)
+{
+	/* Must be called hw interrupts off. */
+	__clear_bit(IPIPE_NOSTACK_FLAG, &ipipe_this_cpu_context(ipd)->status);
+}
+
+static inline int ipipe_test_foreign_stack(void)
+{
+	/* Must be called hw interrupts off. */
+	return test_bit(IPIPE_NOSTACK_FLAG, &__ipipe_current_context->status);
+}
+
+#ifndef ipipe_safe_current
+#define ipipe_safe_current()						\
+	({								\
+		struct task_struct *__p__;				\
+		unsigned long __flags__;				\
+		__flags__ = hard_smp_local_irq_save();			\
+		__p__ = ipipe_test_foreign_stack() ? &init_task : current; \
+		hard_smp_local_irq_restore(__flags__);			\
+		__p__;							\
+	})
+#endif
+
+void __ipipe_legacy_init_stage(struct ipipe_domain *ipd);
+
+/*
+ * These values have no real meaning from a versioning POV, however
+ * they are guaranteed to look more recent than any legacy patch
+ * release ever published in the past.
+ */
+#define IPIPE_MAJOR_NUMBER  3
+#define IPIPE_MINOR_NUMBER  0
+#define IPIPE_PATCH_NUMBER  0
+
+#define __IPIPE_FEATURE_REQUEST_TICKDEV		1
+#define __IPIPE_FEATURE_FASTPEND_IRQ		1
+#define __IPIPE_FEATURE_TRACE_EVENT		1
+#define __IPIPE_FEATURE_ENABLE_NOTIFIER		1
+#define __IPIPE_FEATURE_PREPARE_PANIC		1
+#define __IPIPE_FEATURE_SYSINFO_V2		1
+#define __IPIPE_FEATURE_PIC_MUTE		1
+#ifdef CONFIG_IPIPE_HAVE_VM_NOTIFIER
+#define __IPIPE_FEATURE_ROOTPREEMPT_NOTIFIER	1
+#endif
+
+#else  /* !CONFIG_IPIPE_LEGACY */
+
+static inline void __ipipe_legacy_init_stage(struct ipipe_domain *ipd)
+{
+}
+
+#endif /* !CONFIG_IPIPE_LEGACY */
+
+#endif	/* !__LINUX_IPIPE_COMPAT_H */
diff --git a/include/linux/ipipe_debug.h b/include/linux/ipipe_debug.h
new file mode 100644
index 0000000..5d7efef
--- /dev/null
+++ b/include/linux/ipipe_debug.h
@@ -0,0 +1,100 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_debug.h
+ *
+ * Copyright (C) 2012 Philippe Gerum <rpm@xenomai.org>.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_DEBUG_H
+#define __LINUX_IPIPE_DEBUG_H
+
+#include <linux/ipipe_domain.h>
+
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+
+#include <asm/bug.h>
+
+static inline int ipipe_disable_context_check(void)
+{
+	return xchg(raw_cpu_ptr(&ipipe_percpu.context_check), 0);
+}
+
+static inline void ipipe_restore_context_check(int old_state)
+{
+	__this_cpu_write(ipipe_percpu.context_check, old_state);
+}
+
+static inline void ipipe_context_check_off(void)
+{
+	int cpu;
+	for_each_online_cpu(cpu)
+		per_cpu(ipipe_percpu, cpu).context_check = 0;
+}
+
+static inline void ipipe_save_context_nmi(void)
+{
+	int state = ipipe_disable_context_check();
+	__this_cpu_write(ipipe_percpu.context_check_saved, state);
+}
+
+static inline void ipipe_restore_context_nmi(void)
+{
+	ipipe_restore_context_check(__this_cpu_read(ipipe_percpu.context_check_saved));
+}
+
+#else	/* !CONFIG_IPIPE_DEBUG_CONTEXT */
+
+static inline int ipipe_disable_context_check(void)
+{
+	return 0;
+}
+
+static inline void ipipe_restore_context_check(int old_state) { }
+
+static inline void ipipe_context_check_off(void) { }
+
+static inline void ipipe_save_context_nmi(void) { }
+
+static inline void ipipe_restore_context_nmi(void) { }
+
+#endif	/* !CONFIG_IPIPE_DEBUG_CONTEXT */
+
+#ifdef CONFIG_IPIPE_DEBUG
+
+#define ipipe_check_irqoff()					\
+	do {							\
+		if (WARN_ON_ONCE(!hard_irqs_disabled()))	\
+			hard_local_irq_disable();		\
+	} while (0)
+
+#else /* !CONFIG_IPIPE_DEBUG */
+
+static inline void ipipe_check_irqoff(void) { }
+
+#endif /* !CONFIG_IPIPE_DEBUG */
+
+#ifdef CONFIG_IPIPE_DEBUG_INTERNAL
+#define IPIPE_WARN(c)		WARN_ON(c)
+#define IPIPE_WARN_ONCE(c)	WARN_ON_ONCE(c)
+#define IPIPE_BUG_ON(c)		BUG_ON(c)
+#else
+#define IPIPE_WARN(c)		do { (void)(c); } while (0)
+#define IPIPE_WARN_ONCE(c)	do { (void)(c); } while (0)
+#define IPIPE_BUG_ON(c)		do { (void)(c); } while (0)
+#endif
+
+#endif /* !__LINUX_IPIPE_DEBUG_H */
diff --git a/include/linux/ipipe_domain.h b/include/linux/ipipe_domain.h
new file mode 100644
index 0000000..d00c56b
--- /dev/null
+++ b/include/linux/ipipe_domain.h
@@ -0,0 +1,309 @@
+/*   -*- linux-c -*-
+ *   include/linux/ipipe_domain.h
+ *
+ *   Copyright (C) 2007-2012 Philippe Gerum.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ *   USA; either version 2 of the License, or (at your option) any later
+ *   version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the Free Software
+ *   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_DOMAIN_H
+#define __LINUX_IPIPE_DOMAIN_H
+
+#ifdef CONFIG_IPIPE
+
+#include <linux/mutex.h>
+#include <linux/percpu.h>
+#include <asm/ptrace.h>
+
+struct task_struct;
+struct mm_struct;
+struct irq_desc;
+struct ipipe_vm_notifier;
+
+#define __IPIPE_SYSCALL_P  0
+#define __IPIPE_TRAP_P     1
+#define __IPIPE_KEVENT_P   2
+#define __IPIPE_SYSCALL_E (1 << __IPIPE_SYSCALL_P)
+#define __IPIPE_TRAP_E	  (1 << __IPIPE_TRAP_P)
+#define __IPIPE_KEVENT_E  (1 << __IPIPE_KEVENT_P)
+#define __IPIPE_ALL_E	   0x7
+#define __IPIPE_SYSCALL_R (8 << __IPIPE_SYSCALL_P)
+#define __IPIPE_TRAP_R	  (8 << __IPIPE_TRAP_P)
+#define __IPIPE_KEVENT_R  (8 << __IPIPE_KEVENT_P)
+#define __IPIPE_SHIFT_R	   3
+#define __IPIPE_ALL_R	  (__IPIPE_ALL_E << __IPIPE_SHIFT_R)
+
+typedef void (*ipipe_irq_ackfn_t)(unsigned int irq, struct irq_desc *desc);
+
+struct ipipe_domain {
+	int context_offset;
+	struct ipipe_irqdesc {
+		unsigned long control;
+		ipipe_irq_ackfn_t ackfn;
+		ipipe_irq_handler_t handler;
+		void *cookie;
+	} ____cacheline_aligned irqs[IPIPE_NR_IRQS];
+	const char *name;
+	struct mutex mutex;
+	struct ipipe_legacy_context legacy;
+};
+
+static inline void *
+__ipipe_irq_cookie(struct ipipe_domain *ipd, unsigned int irq)
+{
+	return ipd->irqs[irq].cookie;
+}
+
+static inline ipipe_irq_handler_t
+__ipipe_irq_handler(struct ipipe_domain *ipd, unsigned int irq)
+{
+	return ipd->irqs[irq].handler;
+}
+
+extern struct ipipe_domain ipipe_root;
+
+#define ipipe_root_domain (&ipipe_root)
+
+extern struct ipipe_domain *ipipe_head_domain;
+
+struct ipipe_percpu_domain_data {
+	unsigned long status;	/* <= Must be first in struct. */
+	unsigned long irqpend_himap;
+#ifdef __IPIPE_3LEVEL_IRQMAP
+	unsigned long irqpend_mdmap[IPIPE_IRQ_MDMAPSZ];
+#endif
+	unsigned long irqpend_lomap[IPIPE_IRQ_LOMAPSZ];
+	unsigned long irqheld_map[IPIPE_IRQ_LOMAPSZ];
+	unsigned long irqall[IPIPE_NR_IRQS];
+	struct ipipe_domain *domain;
+	int coflags;
+};
+
+struct ipipe_percpu_data {
+	struct ipipe_percpu_domain_data root;
+	struct ipipe_percpu_domain_data head;
+	struct ipipe_percpu_domain_data *curr;
+	struct pt_regs tick_regs;
+	int hrtimer_irq;
+	struct task_struct *task_hijacked;
+	struct task_struct *rqlock_owner;
+	struct ipipe_vm_notifier *vm_notifier;
+	unsigned long nmi_state;
+	struct mm_struct *active_mm;
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+	int context_check;
+	int context_check_saved;
+#endif
+};
+
+/*
+ * CAREFUL: all accessors based on __ipipe_raw_cpu_ptr() you may find
+ * in this file should be used only while hw interrupts are off, to
+ * prevent from CPU migration regardless of the running domain.
+ */
+DECLARE_PER_CPU(struct ipipe_percpu_data, ipipe_percpu);
+
+static inline struct ipipe_percpu_domain_data *
+__context_of(struct ipipe_percpu_data *p, struct ipipe_domain *ipd)
+{
+	return (void *)p + ipd->context_offset;
+}
+
+/**
+ * ipipe_percpu_context - return the address of the pipeline context
+ * data for a domain on a given CPU.
+ *
+ * NOTE: this is the slowest accessor, use it carefully. Prefer
+ * ipipe_this_cpu_context() for requests targeted at the current
+ * CPU. Additionally, if the target domain is known at build time,
+ * consider ipipe_this_cpu_{root, head}_context().
+ */
+static inline struct ipipe_percpu_domain_data *
+ipipe_percpu_context(struct ipipe_domain *ipd, int cpu)
+{
+	return __context_of(&per_cpu(ipipe_percpu, cpu), ipd);
+}
+
+/**
+ * ipipe_this_cpu_context - return the address of the pipeline context
+ * data for a domain on the current CPU. hw IRQs must be off.
+ *
+ * NOTE: this accessor is a bit faster, but since we don't know which
+ * one of "root" or "head" ipd refers to, we still need to compute the
+ * context address from its offset.
+ */
+static inline struct ipipe_percpu_domain_data *
+ipipe_this_cpu_context(struct ipipe_domain *ipd)
+{
+	return __context_of(__ipipe_raw_cpu_ptr(&ipipe_percpu), ipd);
+}
+
+/**
+ * ipipe_this_cpu_root_context - return the address of the pipeline
+ * context data for the root domain on the current CPU. hw IRQs must
+ * be off.
+ *
+ * NOTE: this accessor is recommended when the domain we refer to is
+ * known at build time to be the root one.
+ */
+static inline struct ipipe_percpu_domain_data *
+ipipe_this_cpu_root_context(void)
+{
+	return __ipipe_raw_cpu_ptr(&ipipe_percpu.root);
+}
+
+/**
+ * ipipe_this_cpu_head_context - return the address of the pipeline
+ * context data for the registered head domain on the current CPU. hw
+ * IRQs must be off.
+ *
+ * NOTE: this accessor is recommended when the domain we refer to is
+ * known at build time to be the registered head domain. This address
+ * is always different from the context data of the root domain in
+ * absence of registered head domain. To get the address of the
+ * context data for the domain leading the pipeline at the time of the
+ * call (which may be root in absence of registered head domain), use
+ * ipipe_this_cpu_leading_context() instead.
+ */
+static inline struct ipipe_percpu_domain_data *
+ipipe_this_cpu_head_context(void)
+{
+	return __ipipe_raw_cpu_ptr(&ipipe_percpu.head);
+}
+
+/**
+ * ipipe_this_cpu_leading_context - return the address of the pipeline
+ * context data for the domain leading the pipeline on the current
+ * CPU. hw IRQs must be off.
+ *
+ * NOTE: this accessor is required when either root or a registered
+ * head domain may be the final target of this call, depending on
+ * whether the high priority domain was installed via
+ * ipipe_register_head().
+ */
+static inline struct ipipe_percpu_domain_data *
+ipipe_this_cpu_leading_context(void)
+{
+	return ipipe_this_cpu_context(ipipe_head_domain);
+}
+
+/**
+ * __ipipe_get_current_context() - return the address of the pipeline
+ * context data of the domain running on the current CPU. hw IRQs must
+ * be off.
+ */
+static inline struct ipipe_percpu_domain_data *__ipipe_get_current_context(void)
+{
+	return __ipipe_raw_cpu_read(ipipe_percpu.curr);
+}
+
+#define __ipipe_current_context __ipipe_get_current_context()
+
+/**
+ * __ipipe_set_current_context() - switch the current CPU to the
+ * specified domain context.  hw IRQs must be off.
+ *
+ * NOTE: this is the only way to change the current domain for the
+ * current CPU. Don't bypass.
+ */
+static inline
+void __ipipe_set_current_context(struct ipipe_percpu_domain_data *pd)
+{
+	struct ipipe_percpu_data *p;
+	p = __ipipe_raw_cpu_ptr(&ipipe_percpu);
+	p->curr = pd;
+}
+
+/**
+ * __ipipe_set_current_domain() - switch the current CPU to the
+ * specified domain. This is equivalent to calling
+ * __ipipe_set_current_context() with the context data of that
+ * domain. hw IRQs must be off.
+ */
+static inline void __ipipe_set_current_domain(struct ipipe_domain *ipd)
+{
+	struct ipipe_percpu_data *p;
+	p = __ipipe_raw_cpu_ptr(&ipipe_percpu);
+	p->curr = __context_of(p, ipd);
+}
+
+static inline struct ipipe_percpu_domain_data *ipipe_current_context(void)
+{
+	struct ipipe_percpu_domain_data *pd;
+	unsigned long flags;
+
+	flags = hard_smp_local_irq_save();
+	pd = __ipipe_get_current_context();
+	hard_smp_local_irq_restore(flags);
+
+	return pd;
+}
+
+static inline struct ipipe_domain *__ipipe_get_current_domain(void)
+{
+	return __ipipe_get_current_context()->domain;
+}
+
+#define __ipipe_current_domain	__ipipe_get_current_domain()
+
+/**
+ * __ipipe_get_current_domain() - return the address of the pipeline
+ * domain running on the current CPU. hw IRQs must be off.
+ */
+static inline struct ipipe_domain *ipipe_get_current_domain(void)
+{
+	struct ipipe_domain *ipd;
+	unsigned long flags;
+
+	flags = hard_smp_local_irq_save();
+	ipd = __ipipe_get_current_domain();
+	hard_smp_local_irq_restore(flags);
+
+	return ipd;
+}
+
+#define ipipe_current_domain	ipipe_get_current_domain()
+
+#define __ipipe_root_p	(__ipipe_current_domain == ipipe_root_domain)
+#define ipipe_root_p	(ipipe_current_domain == ipipe_root_domain)
+
+#ifdef CONFIG_SMP
+#define __ipipe_root_status	(ipipe_this_cpu_root_context()->status)
+#else
+extern unsigned long __ipipe_root_status;
+#endif
+
+#define __ipipe_head_status	(ipipe_this_cpu_head_context()->status)
+
+/**
+ * __ipipe_ipending_p() - Whether we have interrupts pending
+ * (i.e. logged) for the given domain context on the current CPU. hw
+ * IRQs must be off.
+ */
+static inline int __ipipe_ipending_p(struct ipipe_percpu_domain_data *pd)
+{
+	return pd->irqpend_himap != 0;
+}
+
+static inline unsigned long
+__ipipe_cpudata_irq_hits(struct ipipe_domain *ipd, int cpu, unsigned int irq)
+{
+	return ipipe_percpu_context(ipd, cpu)->irqall[irq];
+}
+
+#endif /* CONFIG_IPIPE */
+
+#endif	/* !__LINUX_IPIPE_DOMAIN_H */
diff --git a/include/linux/ipipe_lock.h b/include/linux/ipipe_lock.h
new file mode 100644
index 0000000..f71d2f1
--- /dev/null
+++ b/include/linux/ipipe_lock.h
@@ -0,0 +1,260 @@
+/*   -*- linux-c -*-
+ *   include/linux/ipipe_lock.h
+ *
+ *   Copyright (C) 2009 Philippe Gerum.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ *   USA; either version 2 of the License, or (at your option) any later
+ *   version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the Free Software
+ *   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_LOCK_H
+#define __LINUX_IPIPE_LOCK_H
+
+typedef struct {
+	arch_spinlock_t arch_lock;
+} __ipipe_spinlock_t;
+
+#define ipipe_spinlock_p(lock)							\
+	__builtin_types_compatible_p(typeof(lock), __ipipe_spinlock_t *) ||	\
+	__builtin_types_compatible_p(typeof(lock), __ipipe_spinlock_t [])
+
+#define std_spinlock_raw_p(lock)					\
+	__builtin_types_compatible_p(typeof(lock), raw_spinlock_t *) ||	\
+	__builtin_types_compatible_p(typeof(lock), raw_spinlock_t [])
+
+#define std_spinlock_p(lock)						\
+	__builtin_types_compatible_p(typeof(lock), spinlock_t *) ||	\
+	__builtin_types_compatible_p(typeof(lock), spinlock_t [])
+
+#define ipipe_spinlock(lock)	((__ipipe_spinlock_t *)(lock))
+#define std_spinlock_raw(lock)	((raw_spinlock_t *)(lock))
+#define std_spinlock(lock)	((spinlock_t *)(lock))
+
+#define PICK_SPINLOCK_IRQSAVE(lock, flags)				\
+	do {								\
+		if (ipipe_spinlock_p(lock))				\
+			(flags) = __ipipe_spin_lock_irqsave(ipipe_spinlock(lock)); \
+		else if (std_spinlock_raw_p(lock))				\
+			__real_raw_spin_lock_irqsave(std_spinlock_raw(lock), flags); \
+		else if (std_spinlock_p(lock))				\
+			__real_raw_spin_lock_irqsave(&std_spinlock(lock)->rlock, flags); \
+		else __bad_lock_type();					\
+	} while (0)
+
+#define PICK_SPINTRYLOCK_IRQSAVE(lock, flags)				\
+	({								\
+		int __ret__;						\
+		if (ipipe_spinlock_p(lock))				\
+			__ret__ = __ipipe_spin_trylock_irqsave(ipipe_spinlock(lock), &(flags)); \
+		else if (std_spinlock_raw_p(lock))				\
+			__ret__ = __real_raw_spin_trylock_irqsave(std_spinlock_raw(lock), flags); \
+		else if (std_spinlock_p(lock))				\
+			__ret__ = __real_raw_spin_trylock_irqsave(&std_spinlock(lock)->rlock, flags); \
+		else __bad_lock_type();					\
+		__ret__;						\
+	 })
+
+#define PICK_SPINTRYLOCK_IRQ(lock)					\
+	({								\
+		int __ret__;						\
+		if (ipipe_spinlock_p(lock))				\
+			__ret__ = __ipipe_spin_trylock_irq(ipipe_spinlock(lock)); \
+		else if (std_spinlock_raw_p(lock))				\
+			__ret__ = __real_raw_spin_trylock_irq(std_spinlock_raw(lock)); \
+		else if (std_spinlock_p(lock))				\
+			__ret__ = __real_raw_spin_trylock_irq(&std_spinlock(lock)->rlock); \
+		else __bad_lock_type();					\
+		__ret__;						\
+	 })
+
+#define PICK_SPINUNLOCK_IRQRESTORE(lock, flags)				\
+	do {								\
+		if (ipipe_spinlock_p(lock))				\
+			__ipipe_spin_unlock_irqrestore(ipipe_spinlock(lock), flags); \
+		else {							\
+			__ipipe_spin_unlock_debug(flags);		\
+			if (std_spinlock_raw_p(lock))			\
+				__real_raw_spin_unlock_irqrestore(std_spinlock_raw(lock), flags); \
+			else if (std_spinlock_p(lock))			\
+				__real_raw_spin_unlock_irqrestore(&std_spinlock(lock)->rlock, flags); \
+		}							\
+	} while (0)
+
+#define PICK_SPINOP(op, lock)						\
+	({								\
+		if (ipipe_spinlock_p(lock))				\
+			arch_spin##op(&ipipe_spinlock(lock)->arch_lock); \
+		else if (std_spinlock_raw_p(lock))			\
+			__real_raw_spin##op(std_spinlock_raw(lock));	\
+		else if (std_spinlock_p(lock))				\
+			__real_raw_spin##op(&std_spinlock(lock)->rlock); \
+		else __bad_lock_type();					\
+		(void)0;						\
+	})
+
+#define PICK_SPINOP_RET(op, lock, type)					\
+	({								\
+		type __ret__;						\
+		if (ipipe_spinlock_p(lock))				\
+			__ret__ = arch_spin##op(&ipipe_spinlock(lock)->arch_lock); \
+		else if (std_spinlock_raw_p(lock))			\
+			__ret__ = __real_raw_spin##op(std_spinlock_raw(lock)); \
+		else if (std_spinlock_p(lock))				\
+			__ret__ = __real_raw_spin##op(&std_spinlock(lock)->rlock); \
+		else { __ret__ = -1; __bad_lock_type(); }		\
+		__ret__;						\
+	})
+
+#define arch_spin_lock_init(lock)					\
+	do {								\
+		IPIPE_DEFINE_SPINLOCK(__lock__);			\
+		*((ipipe_spinlock_t *)lock) = __lock__;			\
+	} while (0)
+
+#define arch_spin_lock_irq(lock)					\
+	do {								\
+		hard_local_irq_disable();				\
+		arch_spin_lock(lock);					\
+	} while (0)
+
+#define arch_spin_unlock_irq(lock)					\
+	do {								\
+		arch_spin_unlock(lock);					\
+		hard_local_irq_enable();				\
+	} while (0)
+
+typedef struct {
+	arch_rwlock_t arch_lock;
+} __ipipe_rwlock_t;
+
+#define ipipe_rwlock_p(lock)						\
+	__builtin_types_compatible_p(typeof(lock), __ipipe_rwlock_t *)
+
+#define std_rwlock_p(lock)						\
+	__builtin_types_compatible_p(typeof(lock), rwlock_t *)
+
+#define ipipe_rwlock(lock)	((__ipipe_rwlock_t *)(lock))
+#define std_rwlock(lock)	((rwlock_t *)(lock))
+
+#define PICK_RWOP(op, lock)						\
+	do {								\
+		if (ipipe_rwlock_p(lock))				\
+			arch##op(&ipipe_rwlock(lock)->arch_lock);	\
+		else if (std_rwlock_p(lock))				\
+			_raw##op(std_rwlock(lock));			\
+		else __bad_lock_type();					\
+	} while (0)
+
+extern int __bad_lock_type(void);
+
+#ifdef CONFIG_IPIPE
+
+#define ipipe_spinlock_t		__ipipe_spinlock_t
+#define IPIPE_DEFINE_RAW_SPINLOCK(x)	ipipe_spinlock_t x = IPIPE_SPIN_LOCK_UNLOCKED
+#define IPIPE_DECLARE_RAW_SPINLOCK(x)	extern ipipe_spinlock_t x
+#define IPIPE_DEFINE_SPINLOCK(x)	IPIPE_DEFINE_RAW_SPINLOCK(x)
+#define IPIPE_DECLARE_SPINLOCK(x)	IPIPE_DECLARE_RAW_SPINLOCK(x)
+
+#define IPIPE_SPIN_LOCK_UNLOCKED					\
+	(__ipipe_spinlock_t) {	.arch_lock = __ARCH_SPIN_LOCK_UNLOCKED }
+
+#define spin_lock_irqsave_cond(lock, flags) \
+	spin_lock_irqsave(lock, flags)
+
+#define spin_unlock_irqrestore_cond(lock, flags) \
+	spin_unlock_irqrestore(lock, flags)
+
+#define raw_spin_lock_irqsave_cond(lock, flags) \
+	raw_spin_lock_irqsave(lock, flags)
+
+#define raw_spin_unlock_irqrestore_cond(lock, flags) \
+	raw_spin_unlock_irqrestore(lock, flags)
+
+void __ipipe_spin_lock_irq(ipipe_spinlock_t *lock);
+
+int __ipipe_spin_trylock_irq(ipipe_spinlock_t *lock);
+
+void __ipipe_spin_unlock_irq(ipipe_spinlock_t *lock);
+
+unsigned long __ipipe_spin_lock_irqsave(ipipe_spinlock_t *lock);
+
+int __ipipe_spin_trylock_irqsave(ipipe_spinlock_t *lock,
+				 unsigned long *x);
+
+void __ipipe_spin_unlock_irqrestore(ipipe_spinlock_t *lock,
+				    unsigned long x);
+
+void __ipipe_spin_unlock_irqbegin(ipipe_spinlock_t *lock);
+
+void __ipipe_spin_unlock_irqcomplete(unsigned long x);
+
+#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP)
+void __ipipe_spin_unlock_debug(unsigned long flags);
+#else
+#define __ipipe_spin_unlock_debug(flags)  do { } while (0)
+#endif
+
+#define ipipe_rwlock_t			__ipipe_rwlock_t
+#define IPIPE_DEFINE_RWLOCK(x)		ipipe_rwlock_t x = IPIPE_RW_LOCK_UNLOCKED
+#define IPIPE_DECLARE_RWLOCK(x)		extern ipipe_rwlock_t x
+
+#define IPIPE_RW_LOCK_UNLOCKED	\
+	(__ipipe_rwlock_t) { .arch_lock = __ARCH_RW_LOCK_UNLOCKED }
+
+#else /* !CONFIG_IPIPE */
+
+#define ipipe_spinlock_t		spinlock_t
+#define IPIPE_DEFINE_SPINLOCK(x)	DEFINE_SPINLOCK(x)
+#define IPIPE_DECLARE_SPINLOCK(x)	extern spinlock_t x
+#define IPIPE_SPIN_LOCK_UNLOCKED	__SPIN_LOCK_UNLOCKED(unknown)
+#define IPIPE_DEFINE_RAW_SPINLOCK(x)	DEFINE_RAW_SPINLOCK(x)
+#define IPIPE_DECLARE_RAW_SPINLOCK(x)	extern raw_spinlock_t x
+
+#define spin_lock_irqsave_cond(lock, flags)		\
+	do {						\
+		(void)(flags);				\
+		spin_lock(lock);			\
+	} while(0)
+
+#define spin_unlock_irqrestore_cond(lock, flags)	\
+	spin_unlock(lock)
+
+#define raw_spin_lock_irqsave_cond(lock, flags) \
+	do {					\
+		(void)(flags);			\
+		raw_spin_lock(lock);		\
+	} while(0)
+
+#define raw_spin_unlock_irqrestore_cond(lock, flags) \
+	raw_spin_unlock(lock)
+
+#define __ipipe_spin_lock_irq(lock)		do { } while (0)
+#define __ipipe_spin_unlock_irq(lock)		do { } while (0)
+#define __ipipe_spin_lock_irqsave(lock)		0
+#define __ipipe_spin_trylock_irq(lock)		1
+#define __ipipe_spin_trylock_irqsave(lock, x)	({ (void)(x); 1; })
+#define __ipipe_spin_unlock_irqrestore(lock, x)	do { (void)(x); } while (0)
+#define __ipipe_spin_unlock_irqbegin(lock)	do { } while (0)
+#define __ipipe_spin_unlock_irqcomplete(x)	do { (void)(x); } while (0)
+#define __ipipe_spin_unlock_debug(flags)	do { } while (0)
+
+#define ipipe_rwlock_t			rwlock_t
+#define IPIPE_DEFINE_RWLOCK(x)		DEFINE_RWLOCK(x)
+#define IPIPE_DECLARE_RWLOCK(x)		extern rwlock_t x
+#define IPIPE_RW_LOCK_UNLOCKED		RW_LOCK_UNLOCKED
+
+#endif /* !CONFIG_IPIPE */
+
+#endif /* !__LINUX_IPIPE_LOCK_H */
diff --git a/include/linux/ipipe_tickdev.h b/include/linux/ipipe_tickdev.h
new file mode 100644
index 0000000..58a4142
--- /dev/null
+++ b/include/linux/ipipe_tickdev.h
@@ -0,0 +1,145 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_tickdev.h
+ *
+ * Copyright (C) 2007 Philippe Gerum.
+ * Copyright (C) 2012 Gilles Chanteperdrix
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_TICKDEV_H
+#define __LINUX_IPIPE_TICKDEV_H
+
+#include <linux/list.h>
+#include <linux/cpumask.h>
+#include <linux/clockchips.h>
+#include <linux/ipipe_domain.h>
+#include <linux/clocksource.h>
+#include <linux/timekeeper_internal.h>
+
+#ifdef CONFIG_IPIPE
+
+enum clock_event_mode;
+struct clock_event_device;
+
+struct ipipe_hostrt_data {
+	short live;
+	seqcount_t seqcount;
+	time_t wall_time_sec;
+	u32 wall_time_nsec;
+	struct timespec wall_to_monotonic;
+	cycle_t cycle_last;
+	cycle_t mask;
+	u32 mult;
+	u32 shift;
+};
+
+struct ipipe_timer {
+	int irq;
+	void (*request)(struct ipipe_timer *timer, int steal);
+	int (*set)(unsigned long ticks, void *timer);
+	void (*ack)(void);
+	void (*release)(struct ipipe_timer *timer);
+
+	/* Only if registering a timer directly */
+	const char *name;
+	unsigned rating;
+	unsigned long freq;
+	unsigned min_delay_ticks;
+	const struct cpumask *cpumask;
+
+	/* For internal use */
+	void *timer_set;	/* pointer passed to ->set() callback */
+	struct clock_event_device *host_timer;
+	struct list_head link;
+	
+	/* Conversions between clock frequency and timer frequency */
+	unsigned c2t_integ;
+	unsigned c2t_frac;
+
+	/* For clockevent interception */
+	u32 real_mult;
+	u32 real_shift;
+	void (*real_set_mode)(enum clock_event_mode mode,
+			      struct clock_event_device *cdev);
+	int (*real_set_next_event)(unsigned long evt,
+				   struct clock_event_device *cdev);
+};
+
+#define __ipipe_hrtimer_irq __ipipe_raw_cpu_read(ipipe_percpu.hrtimer_irq)
+
+extern unsigned long __ipipe_hrtimer_freq;
+
+/*
+ * Called by clockevents_register_device, to register a piggybacked
+ * ipipe timer, if there is one
+ */
+void ipipe_host_timer_register(struct clock_event_device *clkevt);
+
+/*
+ * Register a standalone ipipe timer
+ */
+void ipipe_timer_register(struct ipipe_timer *timer);
+
+/*
+ * Chooses the best timer for each cpu. Take over its handling.
+ */
+int ipipe_select_timers(const struct cpumask *mask);
+
+/*
+ * Release the per-cpu timers
+ */
+void ipipe_timers_release(void);
+
+/*
+ * Start handling the per-cpu timer irq, and intercepting the linux clockevent
+ * device callbacks.
+ */
+int ipipe_timer_start(void (*tick_handler)(void),
+		      void (*emumode)(enum clock_event_mode mode,
+				      struct clock_event_device *cdev),
+		      int (*emutick)(unsigned long evt,
+				     struct clock_event_device *cdev),
+		      unsigned cpu);
+
+/*
+ * Stop handling a per-cpu timer
+ */
+void ipipe_timer_stop(unsigned cpu);
+
+/*
+ * Program the timer
+ */
+void ipipe_timer_set(unsigned long delay);
+
+const char *ipipe_timer_name(void);
+
+unsigned ipipe_timer_ns2ticks(struct ipipe_timer *timer, unsigned ns);
+
+#else /* !CONFIG_IPIPE */
+
+#define ipipe_host_timer_register(clkevt) do { } while (0)
+
+#endif /* !CONFIG_IPIPE */
+
+#ifdef CONFIG_IPIPE_HAVE_HOSTRT
+void ipipe_update_hostrt(struct timekeeper *tk);
+#else
+static inline void
+ipipe_update_hostrt(struct timekeeper *tk) {}
+#endif
+
+#endif /* __LINUX_IPIPE_TICKDEV_H */
diff --git a/include/linux/ipipe_trace.h b/include/linux/ipipe_trace.h
new file mode 100644
index 0000000..deb0a47
--- /dev/null
+++ b/include/linux/ipipe_trace.h
@@ -0,0 +1,77 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_trace.h
+ *
+ * Copyright (C) 2005 Luotao Fu.
+ *               2005-2007 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef _LINUX_IPIPE_TRACE_H
+#define _LINUX_IPIPE_TRACE_H
+
+#ifdef CONFIG_IPIPE_TRACE
+
+#include <linux/types.h>
+
+#ifndef BROKEN_BUILTIN_RETURN_ADDRESS
+#define __BUILTIN_RETURN_ADDRESS0 ((unsigned long)__builtin_return_address(0))
+#define __BUILTIN_RETURN_ADDRESS1 ((unsigned long)__builtin_return_address(1))
+#endif /* !BUILTIN_RETURN_ADDRESS */
+
+void ipipe_trace_begin(unsigned long v);
+void ipipe_trace_end(unsigned long v);
+void ipipe_trace_freeze(unsigned long v);
+void ipipe_trace_special(unsigned char special_id, unsigned long v);
+void ipipe_trace_pid(pid_t pid, short prio);
+void ipipe_trace_event(unsigned char id, unsigned long delay_tsc);
+int ipipe_trace_max_reset(void);
+int ipipe_trace_frozen_reset(void);
+
+#else /* !CONFIG_IPIPE_TRACE */
+
+#define ipipe_trace_begin(v)			do { (void)(v); } while(0)
+#define ipipe_trace_end(v)			do { (void)(v); } while(0)
+#define ipipe_trace_freeze(v)			do { (void)(v); } while(0)
+#define ipipe_trace_special(id, v)		do { (void)(id); (void)(v); } while(0)
+#define ipipe_trace_pid(pid, prio)		do { (void)(pid); (void)(prio); } while(0)
+#define ipipe_trace_event(id, delay_tsc)	do { (void)(id); (void)(delay_tsc); } while(0)
+#define ipipe_trace_max_reset()			({ 0; })
+#define ipipe_trace_frozen_reset()		({ 0; })
+
+#endif /* !CONFIG_IPIPE_TRACE */
+
+#ifdef CONFIG_IPIPE_TRACE_PANIC
+void ipipe_trace_panic_freeze(void);
+void ipipe_trace_panic_dump(void);
+#else
+static inline void ipipe_trace_panic_freeze(void) { }
+static inline void ipipe_trace_panic_dump(void) { }
+#endif
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+#define ipipe_trace_irq_entry(irq)	ipipe_trace_begin(irq)
+#define ipipe_trace_irq_exit(irq)	ipipe_trace_end(irq)
+#define ipipe_trace_irqsoff()		ipipe_trace_begin(0x80000000UL)
+#define ipipe_trace_irqson()		ipipe_trace_end(0x80000000UL)
+#else
+#define ipipe_trace_irq_entry(irq)	do { (void)(irq);} while(0)
+#define ipipe_trace_irq_exit(irq)	do { (void)(irq);} while(0)
+#define ipipe_trace_irqsoff()		do { } while(0)
+#define ipipe_trace_irqson()		do { } while(0)
+#endif
+
+#endif	/* !__LINUX_IPIPE_TRACE_H */
diff --git a/include/linux/irq.h b/include/linux/irq.h
index 03f48d9..2e14475ad 100644
--- a/include/linux/irq.h
+++ b/include/linux/irq.h
@@ -337,6 +337,11 @@ struct irq_chip {
 
 	void		(*irq_bus_lock)(struct irq_data *data);
 	void		(*irq_bus_sync_unlock)(struct irq_data *data);
+#ifdef CONFIG_IPIPE
+	void		(*irq_move)(struct irq_data *data);
+	void		(*irq_hold)(struct irq_data *data);
+	void		(*irq_release)(struct irq_data *data);
+#endif /* CONFIG_IPIPE */
 
 	void		(*irq_cpu_online)(struct irq_data *data);
 	void		(*irq_cpu_offline)(struct irq_data *data);
@@ -541,7 +546,14 @@ extern int irq_set_irq_type(unsigned int irq, unsigned int type);
 extern int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry);
 extern int irq_set_msi_desc_off(unsigned int irq_base, unsigned int irq_offset,
 				struct msi_desc *entry);
-extern struct irq_data *irq_get_irq_data(unsigned int irq);
+
+static inline __attribute__((const)) struct irq_data *
+irq_get_irq_data(unsigned int irq)
+{
+	struct irq_desc *desc = irq_to_desc(irq);
+
+	return desc ? &desc->irq_data : NULL;
+}
 
 static inline struct irq_chip *irq_get_chip(unsigned int irq)
 {
@@ -714,7 +726,11 @@ struct irq_chip_type {
  * different flow mechanisms (level/edge) for it.
  */
 struct irq_chip_generic {
+#ifdef CONFIG_IPIPE
+	ipipe_spinlock_t	lock;
+#else
 	raw_spinlock_t		lock;
+#endif
 	void __iomem		*reg_base;
 	unsigned int		irq_base;
 	unsigned int		irq_cnt;
@@ -807,18 +823,28 @@ static inline struct irq_chip_type *irq_data_get_chip_type(struct irq_data *d)
 #define IRQ_MSK(n) (u32)((n) < 32 ? ((1 << (n)) - 1) : UINT_MAX)
 
 #ifdef CONFIG_SMP
-static inline void irq_gc_lock(struct irq_chip_generic *gc)
+static inline unsigned long irq_gc_lock(struct irq_chip_generic *gc)
 {
-	raw_spin_lock(&gc->lock);
+	unsigned long flags = 0;
+	raw_spin_lock_irqsave_cond(&gc->lock, flags);
+	return flags;
 }
 
-static inline void irq_gc_unlock(struct irq_chip_generic *gc)
+static inline void 
+irq_gc_unlock(struct irq_chip_generic *gc, unsigned long flags)
 {
-	raw_spin_unlock(&gc->lock);
+	raw_spin_unlock_irqrestore_cond(&gc->lock, flags);
 }
 #else
-static inline void irq_gc_lock(struct irq_chip_generic *gc) { }
-static inline void irq_gc_unlock(struct irq_chip_generic *gc) { }
+static inline unsigned long irq_gc_lock(struct irq_chip_generic *gc) 
+{ 
+	return hard_cond_local_irq_save();
+}
+static inline void 
+irq_gc_unlock(struct irq_chip_generic *gc, unsigned long flags) 
+{ 
+	hard_cond_local_irq_restore(flags);
+}
 #endif
 
 #endif /* _LINUX_IRQ_H */
diff --git a/include/linux/irqchip/arm-gic.h b/include/linux/irqchip/arm-gic.h
index 13eed92..9f882a8 100644
--- a/include/linux/irqchip/arm-gic.h
+++ b/include/linux/irqchip/arm-gic.h
@@ -49,7 +49,11 @@
 #define GICD_INT_EN_CLR_X32		0xffffffff
 #define GICD_INT_EN_SET_SGI		0x0000ffff
 #define GICD_INT_EN_CLR_PPI		0xffff0000
+#ifndef CONFIG_IPIPE
 #define GICD_INT_DEF_PRI		0xa0
+#else
+#define GICD_INT_DEF_PRI		0x10
+#endif
 #define GICD_INT_DEF_PRI_X4		((GICD_INT_DEF_PRI << 24) |\
 					(GICD_INT_DEF_PRI << 16) |\
 					(GICD_INT_DEF_PRI << 8) |\
diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h
index faf433a..8fa0b13 100644
--- a/include/linux/irqdesc.h
+++ b/include/linux/irqdesc.h
@@ -49,6 +49,12 @@ struct pt_regs;
 struct irq_desc {
 	struct irq_data		irq_data;
 	unsigned int __percpu	*kstat_irqs;
+#ifdef CONFIG_IPIPE
+	void			(*ipipe_ack)(unsigned int irq,
+					     struct irq_desc *desc);
+	void			(*ipipe_end)(unsigned int irq,
+					     struct irq_desc *desc);
+#endif /* CONFIG_IPIPE */
 	irq_flow_handler_t	handle_irq;
 #ifdef CONFIG_IRQ_PREFLOW_FASTEOI
 	irq_preflow_handler_t	preflow_handler;
@@ -154,6 +160,10 @@ static inline int irq_has_action(unsigned int irq)
 	return desc->action != NULL;
 }
 
+irq_flow_handler_t
+__fixup_irq_handler(struct irq_desc *desc, irq_flow_handler_t handle,
+		    int is_chained);
+
 /* caller has locked the irq_desc and both params are valid */
 static inline void __irq_set_handler_locked(unsigned int irq,
 					    irq_flow_handler_t handler)
@@ -161,6 +171,7 @@ static inline void __irq_set_handler_locked(unsigned int irq,
 	struct irq_desc *desc;
 
 	desc = irq_to_desc(irq);
+	handler = __fixup_irq_handler(desc, handler, 0);
 	desc->handle_irq = handler;
 }
 
diff --git a/include/linux/irqnr.h b/include/linux/irqnr.h
index fdd5cc1..1310416 100644
--- a/include/linux/irqnr.h
+++ b/include/linux/irqnr.h
@@ -5,7 +5,11 @@
 
 
 extern int nr_irqs;
+#if !defined(CONFIG_IPIPE) || defined(CONFIG_SPARSE_IRQ)
 extern struct irq_desc *irq_to_desc(unsigned int irq);
+#else
+#define irq_to_desc(irq)	(&irq_desc[irq])
+#endif
 unsigned int irq_get_next_irq(unsigned int offset);
 
 # define for_each_irq_desc(irq, desc)					\
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index 3d770f55..0b21f46 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -9,6 +9,7 @@
 #include <linux/compiler.h>
 #include <linux/bitops.h>
 #include <linux/log2.h>
+#include <linux/ipipe_base.h>
 #include <linux/typecheck.h>
 #include <linux/printk.h>
 #include <linux/dynamic_debug.h>
@@ -156,9 +157,12 @@ struct user;
 
 #ifdef CONFIG_PREEMPT_VOLUNTARY
 extern int _cond_resched(void);
-# define might_resched() _cond_resched()
+# define might_resched() do { \
+		ipipe_root_only(); \
+		_cond_resched(); \
+	} while (0)
 #else
-# define might_resched() do { } while (0)
+# define might_resched() ipipe_root_only()
 #endif
 
 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index e4d8f70..22c313e 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -233,6 +233,9 @@ struct kvm_vcpu {
 #ifdef CONFIG_PREEMPT_NOTIFIERS
 	struct preempt_notifier preempt_notifier;
 #endif
+#ifdef CONFIG_IPIPE
+	struct ipipe_vm_notifier ipipe_notifier;
+#endif
 	int cpu;
 	int vcpu_id;
 	int srcu_idx;
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index de83b4e..7fdf00b 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -8,6 +8,7 @@
 
 #include <linux/linkage.h>
 #include <linux/list.h>
+#include <linux/ipipe_base.h>
 
 /*
  * We use the MSB mostly because its available; see <linux/preempt_mask.h> for
@@ -124,7 +125,27 @@ do { \
 
 #endif /* CONFIG_PREEMPT_COUNT */
 
-#ifdef MODULE
+#ifdef CONFIG_IPIPE
+#define hard_preempt_disable()				\
+	({						\
+		unsigned long __flags__;		\
+		__flags__ = hard_local_irq_save();	\
+		if (__ipipe_root_p)			\
+			preempt_disable();		\
+		__flags__;				\
+	})
+
+#define hard_preempt_enable(__flags__)			\
+	do {						\
+		if (__ipipe_root_p) {			\
+			preempt_enable_no_resched();	\
+			hard_local_irq_restore(__flags__);	\
+			preempt_check_resched();	\
+		} else					\
+			hard_local_irq_restore(__flags__);	\
+	} while (0)
+
+#elif defined(MODULE)
 /*
  * Modules have no business playing preemption tricks.
  */
@@ -132,7 +153,7 @@ do { \
 #undef preempt_enable_no_resched
 #undef preempt_enable_no_resched_notrace
 #undef preempt_check_resched
-#endif
+#endif	/* !IPIPE && MODULE */
 
 #define preempt_set_need_resched() \
 do { \
diff --git a/include/linux/rwlock.h b/include/linux/rwlock.h
index bc2994e..5e2da8d 100644
--- a/include/linux/rwlock.h
+++ b/include/linux/rwlock.h
@@ -61,8 +61,8 @@ do {								\
 #define read_trylock(lock)	__cond_lock(lock, _raw_read_trylock(lock))
 #define write_trylock(lock)	__cond_lock(lock, _raw_write_trylock(lock))
 
-#define write_lock(lock)	_raw_write_lock(lock)
-#define read_lock(lock)		_raw_read_lock(lock)
+#define write_lock(lock)	PICK_RWOP(_write_lock, lock)
+#define read_lock(lock)		PICK_RWOP(_read_lock, lock)
 
 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 
@@ -96,8 +96,8 @@ do {								\
 #define read_lock_bh(lock)		_raw_read_lock_bh(lock)
 #define write_lock_irq(lock)		_raw_write_lock_irq(lock)
 #define write_lock_bh(lock)		_raw_write_lock_bh(lock)
-#define read_unlock(lock)		_raw_read_unlock(lock)
-#define write_unlock(lock)		_raw_write_unlock(lock)
+#define read_unlock(lock)		PICK_RWOP(_read_unlock, lock)
+#define write_unlock(lock)		PICK_RWOP(_write_unlock, lock)
 #define read_unlock_irq(lock)		_raw_read_unlock_irq(lock)
 #define write_unlock_irq(lock)		_raw_write_unlock_irq(lock)
 
diff --git a/include/linux/rwlock_api_smp.h b/include/linux/rwlock_api_smp.h
index 5b9b84b..6c8bb4d 100644
--- a/include/linux/rwlock_api_smp.h
+++ b/include/linux/rwlock_api_smp.h
@@ -141,7 +141,9 @@ static inline int __raw_write_trylock(rwlock_t *lock)
  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
  */
-#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
+#if !defined(CONFIG_GENERIC_LOCKBREAK) ||	\
+	defined(CONFIG_DEBUG_LOCK_ALLOC) ||	\
+	defined(CONFIG_IPIPE)
 
 static inline void __raw_read_lock(rwlock_t *lock)
 {
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 5e344bb..6d00c89 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -26,6 +26,7 @@ struct sched_param {
 #include <linux/nodemask.h>
 #include <linux/mm_types.h>
 #include <linux/preempt_mask.h>
+#include <linux/ipipe.h>
 
 #include <asm/page.h>
 #include <asm/ptrace.h>
@@ -214,9 +215,17 @@ print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq);
 #define TASK_WAKEKILL		128
 #define TASK_WAKING		256
 #define TASK_PARKED		512
+#ifdef CONFIG_IPIPE
+#define TASK_HARDENING		1024
+#define TASK_NOWAKEUP		2048
+#define TASK_STATE_MAX		4096
+#define TASK_STATE_TO_CHAR_STR "RSDTtZXxKWPHN"
+#else  /* !CONFIG_IPIPE */
+#define TASK_HARDENING		0
+#define TASK_NOWAKEUP		0
 #define TASK_STATE_MAX		1024
-
-#define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWP"
+#define TASK_STATE_TO_CHAR_STR "RSDTtZXxKWP"
+#endif /* CONFIG_IPIPE */
 
 extern char ___assert_task_state[1 - 2*!!(
 		sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1)];
@@ -332,6 +341,15 @@ extern void trap_init(void);
 extern void update_process_times(int user);
 extern void scheduler_tick(void);
 
+#ifdef CONFIG_IPIPE
+void update_root_process_times(struct pt_regs *regs);
+#else  /* !CONFIG_IPIPE */
+static inline void update_root_process_times(struct pt_regs *regs)
+{
+	update_process_times(user_mode(regs));
+}
+#endif /* CONFIG_IPIPE */
+
 extern void sched_show_task(struct task_struct *p);
 
 #ifdef CONFIG_LOCKUP_DETECTOR
@@ -452,6 +470,9 @@ static inline int get_dumpable(struct mm_struct *mm)
 #define MMF_VM_MERGEABLE	16	/* KSM may merge identical pages */
 #define MMF_VM_HUGEPAGE		17	/* set when VM_HUGEPAGE is set on vma */
 #define MMF_EXE_FILE_CHANGED	18	/* see prctl_set_mm_exe_file() */
+#ifdef CONFIG_IPIPE
+#define MMF_VM_PINNED		31	/* ondemand load up and COW disabled */
+#endif
 
 #define MMF_HAS_UPROBES		19	/* has uprobes */
 #define MMF_RECALC_UPROBES	20	/* MMF_HAS_UPROBES can be wrong */
@@ -1591,6 +1612,9 @@ struct task_struct {
 #endif /* CONFIG_NUMA_BALANCING */
 
 	struct rcu_head rcu;
+#ifdef CONFIG_IPIPE_LEGACY
+	void *ptd[IPIPE_ROOT_NPTDKEYS];
+#endif
 
 	/*
 	 * cache last used pipe for splice
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index 262ba4e..3bd24db 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -89,10 +89,12 @@
 # include <linux/spinlock_up.h>
 #endif
 
+#include <linux/ipipe_lock.h>
+
 #ifdef CONFIG_DEBUG_SPINLOCK
   extern void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
 				   struct lock_class_key *key);
-# define raw_spin_lock_init(lock)				\
+# define __real_raw_spin_lock_init(lock)			\
 do {								\
 	static struct lock_class_key __key;			\
 								\
@@ -100,11 +102,14 @@ do {								\
 } while (0)
 
 #else
-# define raw_spin_lock_init(lock)				\
+# define __real_raw_spin_lock_init(lock)			\
 	do { *(lock) = __RAW_SPIN_LOCK_UNLOCKED(lock); } while (0)
 #endif
+#define raw_spin_lock_init(lock)	PICK_SPINOP(_lock_init, lock)
 
-#define raw_spin_is_locked(lock)	arch_spin_is_locked(&(lock)->raw_lock)
+#define __real_raw_spin_is_locked(lock)				\
+	arch_spin_is_locked(&(lock)->raw_lock)
+#define raw_spin_is_locked(lock)	PICK_SPINOP_RET(_is_locked, lock, int)
 
 #ifdef CONFIG_GENERIC_LOCKBREAK
 #define raw_spin_is_contended(lock) ((lock)->break_lock)
@@ -183,9 +188,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
  * various methods are defined as nops in the case they are not
  * required.
  */
-#define raw_spin_trylock(lock)	__cond_lock(lock, _raw_spin_trylock(lock))
+#define __real_raw_spin_trylock(lock)	__cond_lock(lock, _raw_spin_trylock(lock))
+#define raw_spin_trylock(lock)		PICK_SPINOP_RET(_trylock, lock, int)
 
-#define raw_spin_lock(lock)	_raw_spin_lock(lock)
+#define __real_raw_spin_lock(lock)	_raw_spin_lock(lock)
+#define raw_spin_lock(lock)		PICK_SPINOP(_lock, lock)
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 # define raw_spin_lock_nested(lock, subclass) \
@@ -209,7 +216,7 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
 
 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 
-#define raw_spin_lock_irqsave(lock, flags)			\
+#define __real_raw_spin_lock_irqsave(lock, flags)	\
 	do {						\
 		typecheck(unsigned long, flags);	\
 		flags = _raw_spin_lock_irqsave(lock);	\
@@ -231,7 +238,7 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
 
 #else
 
-#define raw_spin_lock_irqsave(lock, flags)		\
+#define __real_raw_spin_lock_irqsave(lock, flags)	\
 	do {						\
 		typecheck(unsigned long, flags);	\
 		_raw_spin_lock_irqsave(lock, flags);	\
@@ -242,34 +249,46 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
 
 #endif
 
-#define raw_spin_lock_irq(lock)		_raw_spin_lock_irq(lock)
+#define raw_spin_lock_irqsave(lock, flags)  \
+	PICK_SPINLOCK_IRQSAVE(lock, flags)
+
+#define __real_raw_spin_lock_irq(lock)	_raw_spin_lock_irq(lock)
+#define raw_spin_lock_irq(lock)		PICK_SPINOP(_lock_irq, lock)
 #define raw_spin_lock_bh(lock)		_raw_spin_lock_bh(lock)
-#define raw_spin_unlock(lock)		_raw_spin_unlock(lock)
-#define raw_spin_unlock_irq(lock)	_raw_spin_unlock_irq(lock)
+#define __real_raw_spin_unlock(lock)	_raw_spin_unlock(lock)
+#define raw_spin_unlock(lock)		PICK_SPINOP(_unlock, lock)
+#define __real_raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock)
+#define raw_spin_unlock_irq(lock)	PICK_SPINOP(_unlock_irq, lock)
 
-#define raw_spin_unlock_irqrestore(lock, flags)		\
+#define __real_raw_spin_unlock_irqrestore(lock, flags)		\
 	do {							\
 		typecheck(unsigned long, flags);		\
 		_raw_spin_unlock_irqrestore(lock, flags);	\
 	} while (0)
+#define raw_spin_unlock_irqrestore(lock, flags)	\
+	PICK_SPINUNLOCK_IRQRESTORE(lock, flags)
+
 #define raw_spin_unlock_bh(lock)	_raw_spin_unlock_bh(lock)
 
 #define raw_spin_trylock_bh(lock) \
 	__cond_lock(lock, _raw_spin_trylock_bh(lock))
 
-#define raw_spin_trylock_irq(lock) \
+#define __real_raw_spin_trylock_irq(lock) \
 ({ \
 	local_irq_disable(); \
-	raw_spin_trylock(lock) ? \
+	__real_raw_spin_trylock(lock) ? \
 	1 : ({ local_irq_enable(); 0;  }); \
 })
+#define raw_spin_trylock_irq(lock)	PICK_SPINTRYLOCK_IRQ(lock)
 
-#define raw_spin_trylock_irqsave(lock, flags) \
+#define __real_raw_spin_trylock_irqsave(lock, flags) \
 ({ \
 	local_irq_save(flags); \
 	raw_spin_trylock(lock) ? \
 	1 : ({ local_irq_restore(flags); 0; }); \
 })
+#define raw_spin_trylock_irqsave(lock, flags)	\
+	PICK_SPINTRYLOCK_IRQSAVE(lock, flags)
 
 /**
  * raw_spin_can_lock - would raw_spin_trylock() succeed?
@@ -300,24 +319,17 @@ static inline raw_spinlock_t *spinlock_check(spinlock_t *lock)
 
 #define spin_lock_init(_lock)				\
 do {							\
-	spinlock_check(_lock);				\
-	raw_spin_lock_init(&(_lock)->rlock);		\
+	raw_spin_lock_init(_lock);			\
 } while (0)
 
-static inline void spin_lock(spinlock_t *lock)
-{
-	raw_spin_lock(&lock->rlock);
-}
+#define spin_lock(lock)		raw_spin_lock(lock)
 
 static inline void spin_lock_bh(spinlock_t *lock)
 {
 	raw_spin_lock_bh(&lock->rlock);
 }
 
-static inline int spin_trylock(spinlock_t *lock)
-{
-	return raw_spin_trylock(&lock->rlock);
-}
+#define spin_trylock(lock)	raw_spin_trylock(lock)
 
 #define spin_lock_nested(lock, subclass)			\
 do {								\
@@ -329,14 +341,11 @@ do {									\
 	raw_spin_lock_nest_lock(spinlock_check(lock), nest_lock);	\
 } while (0)
 
-static inline void spin_lock_irq(spinlock_t *lock)
-{
-	raw_spin_lock_irq(&lock->rlock);
-}
+#define spin_lock_irq(lock)	raw_spin_lock_irq(lock)
 
 #define spin_lock_irqsave(lock, flags)				\
 do {								\
-	raw_spin_lock_irqsave(spinlock_check(lock), flags);	\
+	raw_spin_lock_irqsave(lock, flags);			\
 } while (0)
 
 #define spin_lock_irqsave_nested(lock, flags, subclass)			\
@@ -344,39 +353,28 @@ do {									\
 	raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \
 } while (0)
 
-static inline void spin_unlock(spinlock_t *lock)
-{
-	raw_spin_unlock(&lock->rlock);
-}
+#define spin_unlock(lock)	raw_spin_unlock(lock)
 
 static inline void spin_unlock_bh(spinlock_t *lock)
 {
 	raw_spin_unlock_bh(&lock->rlock);
 }
 
-static inline void spin_unlock_irq(spinlock_t *lock)
-{
-	raw_spin_unlock_irq(&lock->rlock);
-}
+#define spin_unlock_irq(lock)	raw_spin_unlock_irq(lock)
 
-static inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
-{
-	raw_spin_unlock_irqrestore(&lock->rlock, flags);
-}
+#define spin_unlock_irqrestore(lock, flags)	\
+	raw_spin_unlock_irqrestore(lock, flags)
 
 static inline int spin_trylock_bh(spinlock_t *lock)
 {
 	return raw_spin_trylock_bh(&lock->rlock);
 }
 
-static inline int spin_trylock_irq(spinlock_t *lock)
-{
-	return raw_spin_trylock_irq(&lock->rlock);
-}
+#define spin_trylock_irq(lock)	raw_spin_trylock_irq(lock)
 
 #define spin_trylock_irqsave(lock, flags)			\
 ({								\
-	raw_spin_trylock_irqsave(spinlock_check(lock), flags); \
+	raw_spin_trylock_irqsave(lock, flags);			\
 })
 
 static inline void spin_unlock_wait(spinlock_t *lock)
diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h
index 42dfab8..ebdc2a9 100644
--- a/include/linux/spinlock_api_smp.h
+++ b/include/linux/spinlock_api_smp.h
@@ -99,7 +99,9 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock)
  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
  */
-#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
+#if !defined(CONFIG_GENERIC_LOCKBREAK) ||	\
+	defined(CONFIG_DEBUG_LOCK_ALLOC) ||	\
+	defined(CONFIG_IPIPE)
 
 static inline unsigned long __raw_spin_lock_irqsave(raw_spinlock_t *lock)
 {
@@ -113,7 +115,7 @@ static inline unsigned long __raw_spin_lock_irqsave(raw_spinlock_t *lock)
 	 * do_raw_spin_lock_flags() code, because lockdep assumes
 	 * that interrupts are not re-enabled during lock-acquire:
 	 */
-#ifdef CONFIG_LOCKDEP
+#if defined(CONFIG_LOCKDEP) || defined(CONFIG_IPIPE)
 	LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
 #else
 	do_raw_spin_lock_flags(lock, &flags);
diff --git a/include/linux/spinlock_up.h b/include/linux/spinlock_up.h
index 8b3ac0d..cf14140 100644
--- a/include/linux/spinlock_up.h
+++ b/include/linux/spinlock_up.h
@@ -55,16 +55,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock)
 	lock->slock = 1;
 }
 
-/*
- * Read-write spinlocks. No debug version.
- */
-#define arch_read_lock(lock)		do { barrier(); (void)(lock); } while (0)
-#define arch_write_lock(lock)		do { barrier(); (void)(lock); } while (0)
-#define arch_read_trylock(lock)	({ barrier(); (void)(lock); 1; })
-#define arch_write_trylock(lock)	({ barrier(); (void)(lock); 1; })
-#define arch_read_unlock(lock)		do { barrier(); (void)(lock); } while (0)
-#define arch_write_unlock(lock)	do { barrier(); (void)(lock); } while (0)
-
 #else /* DEBUG_SPINLOCK */
 #define arch_spin_is_locked(lock)	((void)(lock), 0)
 /* for sched/core.c and kernel_lock.c: */
@@ -74,6 +64,13 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock)
 # define arch_spin_trylock(lock)	({ barrier(); (void)(lock); 1; })
 #endif /* DEBUG_SPINLOCK */
 
+#define arch_read_lock(lock)		do { barrier(); (void)(lock); } while (0)
+#define arch_write_lock(lock)		do { barrier(); (void)(lock); } while (0)
+#define arch_read_trylock(lock)	({ barrier(); (void)(lock); 1; })
+#define arch_write_trylock(lock)	({ barrier(); (void)(lock); 1; })
+#define arch_read_unlock(lock)		do { barrier(); (void)(lock); } while (0)
+#define arch_write_unlock(lock)	do { barrier(); (void)(lock); } while (0)
+
 #define arch_spin_is_contended(lock)	(((void)(lock), 0))
 
 #define arch_read_can_lock(lock)	(((void)(lock), 1))
diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h
index 95640dc..2e026a3 100644
--- a/include/linux/timekeeper_internal.h
+++ b/include/linux/timekeeper_internal.h
@@ -112,7 +112,7 @@ extern void update_vsyscall_tz(void);
 #elif defined(CONFIG_GENERIC_TIME_VSYSCALL_OLD)
 
 extern void update_vsyscall_old(struct timespec *ts, struct timespec *wtm,
-				struct clocksource *c, u32 mult,
+				struct clocksource *c, u32 mult, u32 shift,
 				cycle_t cycle_last);
 extern void update_vsyscall_tz(void);
 
diff --git a/include/uapi/asm-generic/mman-common.h b/include/uapi/asm-generic/mman-common.h
index ddc3b36..a9dd00a 100644
--- a/include/uapi/asm-generic/mman-common.h
+++ b/include/uapi/asm-generic/mman-common.h
@@ -19,6 +19,9 @@
 #define MAP_TYPE	0x0f		/* Mask for type of mapping */
 #define MAP_FIXED	0x10		/* Interpret addr exactly */
 #define MAP_ANONYMOUS	0x20		/* don't use a file */
+#ifndef MAP_BRK
+# define MAP_BRK	0
+#endif
 #ifdef CONFIG_MMAP_ALLOW_UNINITIALIZED
 # define MAP_UNINITIALIZED 0x4000000	/* For anonymous mmap, memory could be uninitialized */
 #else
diff --git a/include/uapi/asm-generic/resource.h b/include/uapi/asm-generic/resource.h
index c6d10af..83cd38b 100644
--- a/include/uapi/asm-generic/resource.h
+++ b/include/uapi/asm-generic/resource.h
@@ -57,5 +57,13 @@
 # define RLIM_INFINITY		(~0UL)
 #endif
 
+/*
+ * Limit the stack by to some sane default: root can always
+ * increase this limit if needed..  8MB seems reasonable.
+ */
+#ifndef _STK_LIM
+# define _STK_LIM		(8*1024*1024)
+#endif
+
 
 #endif /* _UAPI_ASM_GENERIC_RESOURCE_H */
diff --git a/include/uapi/linux/resource.h b/include/uapi/linux/resource.h
index 36fb3b5..3b5e2dc6 100644
--- a/include/uapi/linux/resource.h
+++ b/include/uapi/linux/resource.h
@@ -59,12 +59,6 @@ struct rlimit64 {
 #define	PRIO_USER	2
 
 /*
- * Limit the stack by to some sane default: root can always
- * increase this limit if needed..  8MB seems reasonable.
- */
-#define _STK_LIM	(8*1024*1024)
-
-/*
  * GPG2 wants 64kB of mlocked memory, to make sure pass phrases
  * and other sensitive information are never written to disk.
  */
diff --git a/init/Kconfig b/init/Kconfig
index 2081a4d..24b8fc0 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -69,6 +69,7 @@ config COMPILE_TEST
 
 config LOCALVERSION
 	string "Local version - append to kernel release"
+	default "-ipipe"
 	help
 	  Append an extra string to the end of your kernel version.
 	  This will show up when you type uname, for example.
diff --git a/init/main.c b/init/main.c
index 321d0ce..fa9d93e 100644
--- a/init/main.c
+++ b/init/main.c
@@ -518,7 +518,7 @@ asmlinkage __visible void __init start_kernel(void)
 
 	cgroup_init_early();
 
-	local_irq_disable();
+	hard_local_irq_disable();
 	early_boot_irqs_disabled = true;
 
 /*
@@ -558,6 +558,7 @@ asmlinkage __visible void __init start_kernel(void)
 	pidhash_init();
 	vfs_caches_init_early();
 	sort_main_extable();
+	__ipipe_init_early();
 	trap_init();
 	mm_init();
 
@@ -589,6 +590,11 @@ asmlinkage __visible void __init start_kernel(void)
 	softirq_init();
 	timekeeping_init();
 	time_init();
+	/*
+	 * We need to wait for the interrupt and time subsystems to be
+	 * initialized before enabling the pipeline.
+	 */
+	__ipipe_init();
 	sched_clock_postinit();
 	perf_event_init();
 	profile_init();
@@ -878,6 +884,7 @@ static void __init do_basic_setup(void)
 	shmem_init();
 	driver_init();
 	init_irq_proc();
+  	__ipipe_init_proc();
 	do_ctors();
 	usermodehelper_enable();
 	do_initcalls();
diff --git a/kernel/Makefile b/kernel/Makefile
index 17ea6d4..38aa23b 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -71,6 +71,7 @@ obj-$(CONFIG_DETECT_HUNG_TASK) += hung_task.o
 obj-$(CONFIG_LOCKUP_DETECTOR) += watchdog.o
 obj-$(CONFIG_SECCOMP) += seccomp.o
 obj-$(CONFIG_RELAY) += relay.o
+obj-$(CONFIG_IPIPE) += ipipe/
 obj-$(CONFIG_SYSCTL) += utsname_sysctl.o
 obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
 obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c
index 937ecdf..1627e18b 100644
--- a/kernel/context_tracking.c
+++ b/kernel/context_tracking.c
@@ -68,7 +68,7 @@ void context_tracking_user_enter(void)
 	 * helpers are enough to protect RCU uses inside the exception. So
 	 * just return immediately if we detect we are in an IRQ.
 	 */
-	if (in_interrupt())
+	if (ipipe_root_p == 0 || in_interrupt())
 		return;
 
 	/* Kernel threads aren't supposed to go to userspace */
@@ -122,7 +122,7 @@ void context_tracking_user_exit(void)
 {
 	unsigned long flags;
 
-	if (!context_tracking_is_enabled())
+	if (!ipipe_root_p || !context_tracking_is_enabled())
 		return;
 
 	if (in_interrupt())
diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c
index 1adf62b..4f9f836 100644
--- a/kernel/debug/debug_core.c
+++ b/kernel/debug/debug_core.c
@@ -115,8 +115,8 @@ static struct kgdb_bkpt		kgdb_break[KGDB_MAX_BREAKPOINTS] = {
  */
 atomic_t			kgdb_active = ATOMIC_INIT(-1);
 EXPORT_SYMBOL_GPL(kgdb_active);
-static DEFINE_RAW_SPINLOCK(dbg_master_lock);
-static DEFINE_RAW_SPINLOCK(dbg_slave_lock);
+static IPIPE_DEFINE_RAW_SPINLOCK(dbg_master_lock);
+static IPIPE_DEFINE_RAW_SPINLOCK(dbg_slave_lock);
 
 /*
  * We use NR_CPUs not PERCPU, in case kgdb is used to debug early
@@ -166,19 +166,21 @@ int __weak kgdb_arch_set_breakpoint(struct kgdb_bkpt *bpt)
 {
 	int err;
 
-	err = probe_kernel_read(bpt->saved_instr, (char *)bpt->bpt_addr,
-				BREAK_INSTR_SIZE);
+	err = ipipe_probe_kernel_read(bpt->saved_instr, (char *)bpt->bpt_addr,
+				      BREAK_INSTR_SIZE);
 	if (err)
 		return err;
-	err = probe_kernel_write((char *)bpt->bpt_addr,
-				 arch_kgdb_ops.gdb_bpt_instr, BREAK_INSTR_SIZE);
+	err = ipipe_probe_kernel_write((char *)bpt->bpt_addr,
+				       arch_kgdb_ops.gdb_bpt_instr,
+				       BREAK_INSTR_SIZE);
 	return err;
 }
 
 int __weak kgdb_arch_remove_breakpoint(struct kgdb_bkpt *bpt)
 {
-	return probe_kernel_write((char *)bpt->bpt_addr,
-				  (char *)bpt->saved_instr, BREAK_INSTR_SIZE);
+	return ipipe_probe_kernel_write((char *)bpt->bpt_addr,
+					(char *)bpt->saved_instr,
+					BREAK_INSTR_SIZE);
 }
 
 int __weak kgdb_validate_break_address(unsigned long addr)
@@ -458,7 +460,9 @@ static int kgdb_reenter_check(struct kgdb_state *ks)
 static void dbg_touch_watchdogs(void)
 {
 	touch_softlockup_watchdog_sync();
+#ifndef CONFIG_IPIPE
 	clocksource_touch_watchdog();
+#endif
 	rcu_cpu_stall_reset();
 }
 
@@ -488,7 +492,7 @@ acquirelock:
 	 * Interrupts will be restored by the 'trap return' code, except when
 	 * single stepping.
 	 */
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 
 	cpu = ks->cpu;
 	kgdb_info[cpu].debuggerinfo = regs;
@@ -537,7 +541,7 @@ return_normal:
 			smp_mb__before_atomic();
 			atomic_dec(&slaves_in_kgdb);
 			dbg_touch_watchdogs();
-			local_irq_restore(flags);
+			hard_local_irq_restore(flags);
 			return 0;
 		}
 		cpu_relax();
@@ -555,7 +559,7 @@ return_normal:
 		atomic_set(&kgdb_active, -1);
 		raw_spin_unlock(&dbg_master_lock);
 		dbg_touch_watchdogs();
-		local_irq_restore(flags);
+		hard_local_irq_restore(flags);
 
 		goto acquirelock;
 	}
@@ -668,7 +672,7 @@ kgdb_restore:
 	atomic_set(&kgdb_active, -1);
 	raw_spin_unlock(&dbg_master_lock);
 	dbg_touch_watchdogs();
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 
 	return kgdb_info[cpu].ret_state;
 }
@@ -779,9 +783,9 @@ static void kgdb_console_write(struct console *co, const char *s,
 	if (!kgdb_connected || atomic_read(&kgdb_active) != -1 || dbg_kdb_mode)
 		return;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	gdbstub_msg_write(s, count);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 static struct console kgdbcons = {
diff --git a/kernel/debug/gdbstub.c b/kernel/debug/gdbstub.c
index 19d9a57..4e99085 100644
--- a/kernel/debug/gdbstub.c
+++ b/kernel/debug/gdbstub.c
@@ -246,7 +246,7 @@ char *kgdb_mem2hex(char *mem, char *buf, int count)
 	 */
 	tmp = buf + count;
 
-	err = probe_kernel_read(tmp, mem, count);
+	err = ipipe_probe_kernel_read(tmp, mem, count);
 	if (err)
 		return NULL;
 	while (count > 0) {
@@ -282,7 +282,7 @@ int kgdb_hex2mem(char *buf, char *mem, int count)
 		*tmp_raw |= hex_to_bin(*tmp_hex--) << 4;
 	}
 
-	return probe_kernel_write(mem, tmp_raw, count);
+	return ipipe_probe_kernel_write(mem, tmp_raw, count);
 }
 
 /*
@@ -334,7 +334,7 @@ static int kgdb_ebin2mem(char *buf, char *mem, int count)
 		size++;
 	}
 
-	return probe_kernel_write(mem, c, size);
+	return ipipe_probe_kernel_write(mem, c, size);
 }
 
 #if DBG_MAX_REG_NUM > 0
diff --git a/kernel/exit.c b/kernel/exit.c
index 2116aac..b156c25 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -719,6 +719,7 @@ void do_exit(long code)
 	 */
 	smp_mb();
 	raw_spin_unlock_wait(&tsk->pi_lock);
+  	__ipipe_report_exit(tsk);
 
 	if (unlikely(in_atomic()))
 		pr_info("note: %s[%d] exited with preempt_count %d\n",
diff --git a/kernel/fork.c b/kernel/fork.c
index 9b7d746..562489c 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -333,6 +333,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig)
 #endif
 
 	setup_thread_stack(tsk, orig);
+	__ipipe_init_threadflags(ti);
+	__ipipe_init_threadinfo(&ti->ipipe_data);
 	clear_user_return_notifier(tsk);
 	clear_tsk_need_resched(tsk);
 	set_task_stack_end_magic(tsk);
@@ -655,6 +657,7 @@ void mmput(struct mm_struct *mm)
 		exit_aio(mm);
 		ksm_exit(mm);
 		khugepaged_exit(mm); /* must run before exit_mmap */
+ 		__ipipe_report_cleanup(mm);
 		exit_mmap(mm);
 		set_mm_exe_file(mm, NULL);
 		if (!list_empty(&mm->mmlist)) {
@@ -1541,6 +1544,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
 	cgroup_post_fork(p);
 	if (clone_flags & CLONE_THREAD)
 		threadgroup_change_end(current);
+	__ipipe_init_taskinfo(p);
 	perf_event_fork(p);
 
 	trace_task_newtask(p, clone_flags);
diff --git a/kernel/ipipe/Kconfig b/kernel/ipipe/Kconfig
new file mode 100644
index 0000000..218f51da
--- /dev/null
+++ b/kernel/ipipe/Kconfig
@@ -0,0 +1,65 @@
+config IPIPE
+	bool "Interrupt pipeline"
+	default y
+	---help---
+	  Activate this option if you want the interrupt pipeline to be
+	  compiled in.
+
+config IPIPE_LEGACY
+	bool "I-pipe legacy interface"
+	depends on IPIPE
+	---help---
+	  Activate this option if you want to control the interrupt
+	  pipeline via the legacy interface.
+
+config IPIPE_CORE
+	def_bool y if IPIPE
+
+config IPIPE_WANT_CLOCKSOURCE
+       bool
+
+config IPIPE_WANT_PTE_PINNING
+       bool
+
+config IPIPE_CORE_APIREV
+       int
+       depends on IPIPE
+       default 2
+	---help---
+	  The API revision level we implement.
+
+config IPIPE_WANT_APIREV_1
+       bool
+
+config IPIPE_WANT_APIREV_2
+       bool
+
+config IPIPE_TARGET_APIREV
+       int
+       depends on IPIPE
+       default 1 if IPIPE_WANT_APIREV_1
+       default 2 if IPIPE_WANT_APIREV_2
+       default 1 if IPIPE_LEGACY
+       default IPIPE_CORE_APIREV
+	---help---
+	  The API revision level the we want (must be <=
+	  IPIPE_CORE_APIREV).
+
+config IPIPE_HAVE_HOSTRT
+       bool
+
+config IPIPE_HAVE_PIC_MUTE
+       bool
+
+config HAVE_IPIPE_HOSTRT
+       depends on IPIPE_LEGACY
+       bool
+
+config IPIPE_DELAYED_ATOMICSW
+       def_bool y if IPIPE_LEGACY
+
+config IPIPE_HAVE_SAFE_THREAD_INFO
+	bool
+
+config IPIPE_HAVE_VM_NOTIFIER
+	bool
diff --git a/kernel/ipipe/Kconfig.debug b/kernel/ipipe/Kconfig.debug
new file mode 100644
index 0000000..3c6cac8
--- /dev/null
+++ b/kernel/ipipe/Kconfig.debug
@@ -0,0 +1,95 @@
+config IPIPE_DEBUG
+	bool "I-pipe debugging"
+	depends on IPIPE
+
+config IPIPE_DEBUG_CONTEXT
+	bool "Check for illicit cross-domain calls"
+	depends on IPIPE_DEBUG
+	default y
+	---help---
+	  Enable this feature to arm checkpoints in the kernel that
+	  verify the correct invocation context. On entry of critical
+	  Linux services a warning is issued if the caller is not
+	  running over the root domain.
+
+config IPIPE_DEBUG_INTERNAL
+	bool "Enable internal debug checks"
+	depends on IPIPE_DEBUG
+	default y
+	---help---
+	  When this feature is enabled, I-pipe will perform internal
+	  consistency checks of its subsystems, e.g. on per-cpu variable
+	  access.
+
+config IPIPE_TRACE
+	bool "Latency tracing"
+	depends on IPIPE_DEBUG
+	select CONFIG_FTRACE
+	select CONFIG_FUNCTION_TRACER
+	select KALLSYMS
+	select PROC_FS
+	---help---
+	  Activate this option if you want to use per-function tracing of
+	  the kernel. The tracer will collect data via instrumentation
+	  features like the one below or with the help of explicite calls
+	  of ipipe_trace_xxx(). See include/linux/ipipe_trace.h for the
+	  in-kernel tracing API. The collected data and runtime control
+	  is available via /proc/ipipe/trace/*.
+
+if IPIPE_TRACE
+
+config IPIPE_TRACE_ENABLE
+	bool "Enable tracing on boot"
+	default y
+	---help---
+	  Disable this option if you want to arm the tracer after booting
+	  manually ("echo 1 > /proc/ipipe/tracer/enable"). This can reduce
+	  boot time on slow embedded devices due to the tracer overhead.
+
+config IPIPE_TRACE_MCOUNT
+	bool "Instrument function entries"
+	default y
+	select FTRACE
+	select FUNCTION_TRACER
+	---help---
+	  When enabled, records every kernel function entry in the tracer
+	  log. While this slows down the system noticeably, it provides
+	  the highest level of information about the flow of events.
+	  However, it can be switch off in order to record only explicit
+	  I-pipe trace points.
+
+config IPIPE_TRACE_IRQSOFF
+	bool "Trace IRQs-off times"
+	default y
+	---help---
+	  Activate this option if I-pipe shall trace the longest path
+	  with hard-IRQs switched off.
+
+config IPIPE_TRACE_SHIFT
+	int "Depth of trace log (14 => 16Kpoints, 15 => 32Kpoints)"
+	range 10 18
+	default 14
+	---help---
+	  The number of trace points to hold tracing data for each
+	  trace path, as a power of 2.
+
+config IPIPE_TRACE_VMALLOC
+	bool "Use vmalloc'ed trace buffer"
+	default y if EMBEDDED
+	---help---
+	  Instead of reserving static kernel data, the required buffer
+	  is allocated via vmalloc during boot-up when this option is
+	  enabled. This can help to start systems that are low on memory,
+	  but it slightly degrades overall performance. Try this option
+	  when a traced kernel hangs unexpectedly at boot time.
+
+config IPIPE_TRACE_PANIC
+	bool "Enable panic back traces"
+	default y
+	---help---
+	  Provides services to freeze and dump a back trace on panic
+	  situations. This is used on IPIPE_DEBUG_CONTEXT exceptions
+	  as well as ordinary kernel oopses. You can control the number
+	  of printed back trace points via /proc/ipipe/trace.
+
+endif
diff --git a/kernel/ipipe/Makefile b/kernel/ipipe/Makefile
new file mode 100644
index 0000000..c3ffe63
--- /dev/null
+++ b/kernel/ipipe/Makefile
@@ -0,0 +1,3 @@
+obj-$(CONFIG_IPIPE)	+= core.o timer.o
+obj-$(CONFIG_IPIPE_TRACE) += tracer.o
+obj-$(CONFIG_IPIPE_LEGACY) += compat.o
diff --git a/kernel/ipipe/compat.c b/kernel/ipipe/compat.c
new file mode 100644
index 0000000..797a849
--- /dev/null
+++ b/kernel/ipipe/compat.c
@@ -0,0 +1,273 @@
+/* -*- linux-c -*-
+ * linux/kernel/ipipe/compat.c
+ *
+ * Copyright (C) 2012 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * I-pipe legacy interface.
+ */
+#include <linux/version.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/ipipe.h>
+
+static int ptd_key_count;
+
+static unsigned long ptd_key_map;
+
+IPIPE_DECLARE_SPINLOCK(__ipipe_lock);
+
+void ipipe_init_attr(struct ipipe_domain_attr *attr)
+{
+	attr->name = "anon";
+	attr->domid = 1;
+	attr->entry = NULL;
+	attr->priority = IPIPE_ROOT_PRIO;
+	attr->pdd = NULL;
+}
+EXPORT_SYMBOL_GPL(ipipe_init_attr);
+
+int ipipe_register_domain(struct ipipe_domain *ipd,
+			  struct ipipe_domain_attr *attr)
+{
+	struct ipipe_percpu_domain_data *p;
+	unsigned long flags;
+
+	BUG_ON(attr->priority != IPIPE_HEAD_PRIORITY);
+
+	ipipe_register_head(ipd, attr->name);
+	ipd->legacy.domid = attr->domid;
+	ipd->legacy.pdd = attr->pdd;
+	ipd->legacy.priority = INT_MAX;
+
+	if (attr->entry == NULL)
+		return 0;
+
+	flags = hard_smp_local_irq_save();
+	__ipipe_set_current_domain(ipd);
+	hard_smp_local_irq_restore(flags);
+
+	attr->entry();
+
+	flags = hard_local_irq_save();
+	__ipipe_set_current_domain(ipipe_root_domain);
+	p = ipipe_this_cpu_root_context();
+	if (__ipipe_ipending_p(p) &&
+	    !test_bit(IPIPE_STALL_FLAG, &p->status))
+		__ipipe_sync_stage();
+	hard_local_irq_restore(flags);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipipe_register_domain);
+
+int ipipe_unregister_domain(struct ipipe_domain *ipd)
+{
+	ipipe_unregister_head(ipd);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipipe_unregister_domain);
+
+int ipipe_alloc_ptdkey(void)
+{
+	unsigned long flags;
+	int key = -1;
+
+	spin_lock_irqsave(&__ipipe_lock,flags);
+
+	if (ptd_key_count < IPIPE_ROOT_NPTDKEYS) {
+		key = ffz(ptd_key_map);
+		set_bit(key,&ptd_key_map);
+		ptd_key_count++;
+	}
+
+	spin_unlock_irqrestore(&__ipipe_lock,flags);
+
+	return key;
+}
+EXPORT_SYMBOL_GPL(ipipe_alloc_ptdkey);
+
+int ipipe_free_ptdkey(int key)
+{
+	unsigned long flags;
+
+	if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+		return -EINVAL;
+
+	spin_lock_irqsave(&__ipipe_lock,flags);
+
+	if (test_and_clear_bit(key,&ptd_key_map))
+		ptd_key_count--;
+
+	spin_unlock_irqrestore(&__ipipe_lock,flags);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipipe_free_ptdkey);
+
+int ipipe_set_ptd(int key, void *value)
+{
+	if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+		return -EINVAL;
+
+	current->ptd[key] = value;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipipe_set_ptd);
+
+void *ipipe_get_ptd(int key)
+{
+	if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+		return NULL;
+
+	return current->ptd[key];
+}
+EXPORT_SYMBOL_GPL(ipipe_get_ptd);
+
+int ipipe_virtualize_irq(struct ipipe_domain *ipd,
+			 unsigned int irq,
+			 ipipe_irq_handler_t handler,
+			 void *cookie,
+			 ipipe_irq_ackfn_t ackfn,
+			 unsigned int modemask)
+{
+	if (handler == NULL) {
+		ipipe_free_irq(ipd, irq);
+		return 0;
+	}
+
+	return ipipe_request_irq(ipd, irq, handler, cookie, ackfn);
+}
+EXPORT_SYMBOL_GPL(ipipe_virtualize_irq);
+
+static int null_handler(unsigned int event,
+			struct ipipe_domain *from, void *data)
+{
+	/*
+	 * Legacy mode users will trap all events, at worst most
+	 * frequent ones. Therefore it is actually faster to run a
+	 * dummy handler once in a while rather than testing for a
+	 * null handler pointer each time an event is fired.
+	 */
+	return 0;
+}
+
+ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd,
+					unsigned int event,
+					ipipe_event_handler_t handler)
+{
+	ipipe_event_handler_t oldhandler;
+	int n, enables = 0;
+
+	if (event & IPIPE_EVENT_SELF) {
+		event &= ~IPIPE_EVENT_SELF;
+		IPIPE_WARN(event >= IPIPE_NR_FAULTS);
+	}
+
+	if (event >= IPIPE_NR_EVENTS)
+		return NULL;
+
+	/*
+	 * It makes no sense to run a SETSCHED notification handler
+	 * over the head domain, this introduces a useless domain
+	 * switch for doing work which ought to be root specific.
+	 * Unfortunately, some client domains using the legacy
+	 * interface still ask for this, so we silently fix their
+	 * request. This prevents ipipe_set_hooks() from yelling at us
+	 * because of an attempt to enable kernel event notifications
+	 * for the head domain.
+	 */
+	if (event == IPIPE_EVENT_SETSCHED)
+		ipd = ipipe_root_domain;
+
+	oldhandler = ipd->legacy.handlers[event];
+	ipd->legacy.handlers[event] = handler ?: null_handler;
+
+	for (n = 0; n < IPIPE_NR_FAULTS; n++) {
+		if (ipd->legacy.handlers[n] != null_handler) {
+			enables |= __IPIPE_TRAP_E;
+			break;
+		}
+	}
+
+	for (n = IPIPE_FIRST_EVENT; n < IPIPE_LAST_EVENT; n++) {
+		if (ipd->legacy.handlers[n] != null_handler) {
+			enables |= __IPIPE_KEVENT_E;
+			break;
+		}
+	}
+
+	if (ipd->legacy.handlers[IPIPE_EVENT_SYSCALL] != null_handler)
+		enables |= __IPIPE_SYSCALL_E;
+
+	ipipe_set_hooks(ipd, enables);
+
+	return oldhandler == null_handler ? NULL : oldhandler;
+}
+EXPORT_SYMBOL_GPL(ipipe_catch_event);
+
+int ipipe_setscheduler_root(struct task_struct *p, int policy, int prio)
+{
+	struct sched_param param = { .sched_priority = prio };
+	return sched_setscheduler_nocheck(p, policy, &param);
+}
+EXPORT_SYMBOL_GPL(ipipe_setscheduler_root);
+
+int ipipe_syscall_hook(struct ipipe_domain *ipd, struct pt_regs *regs)
+{
+	const int event = IPIPE_EVENT_SYSCALL;
+	return ipipe_current_domain->legacy.handlers[event](event, ipd, regs);
+}
+
+int ipipe_trap_hook(struct ipipe_trap_data *data)
+{
+	struct ipipe_domain *ipd = ipipe_head_domain;
+	struct pt_regs *regs = data->regs;
+	int ex = data->exception;
+
+	return ipd->legacy.handlers[ex](ex, ipd, regs);
+}
+
+int ipipe_kevent_hook(int kevent, void *data)
+{
+	unsigned int event = IPIPE_FIRST_EVENT + kevent;
+	struct ipipe_domain *ipd = ipipe_root_domain;
+
+	return ipd->legacy.handlers[event](event, ipd, data);
+}
+
+void __ipipe_legacy_init_stage(struct ipipe_domain *ipd)
+{
+	int n;
+
+	for (n = 0; n < IPIPE_NR_EVENTS; n++)
+		ipd->legacy.handlers[n] = null_handler;
+
+	if (ipd == &ipipe_root) {
+		ipd->legacy.domid = IPIPE_ROOT_ID;
+		ipd->legacy.priority = IPIPE_ROOT_PRIO;
+	}
+}
+
+notrace asmlinkage int __ipipe_check_root(void) /* hw IRQs off */
+{
+	return __ipipe_root_p;
+}
diff --git a/kernel/ipipe/core.c b/kernel/ipipe/core.c
new file mode 100644
index 0000000..d7c8934
--- /dev/null
+++ b/kernel/ipipe/core.c
@@ -0,0 +1,1916 @@
+/* -*- linux-c -*-
+ * linux/kernel/ipipe/core.c
+ *
+ * Copyright (C) 2002-2012 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-independent I-PIPE core support.
+ */
+#include <linux/version.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/kallsyms.h>
+#include <linux/bitops.h>
+#include <linux/tick.h>
+#include <linux/interrupt.h>
+#include <linux/uaccess.h>
+#ifdef CONFIG_PROC_FS
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#endif	/* CONFIG_PROC_FS */
+#include <linux/ipipe_trace.h>
+#include <linux/ipipe.h>
+#include <ipipe/setup.h>
+
+struct ipipe_domain ipipe_root;
+EXPORT_SYMBOL_GPL(ipipe_root);
+
+struct ipipe_domain *ipipe_head_domain = &ipipe_root;
+EXPORT_SYMBOL_GPL(ipipe_head_domain);
+
+#ifdef CONFIG_SMP
+static __initdata struct ipipe_percpu_domain_data bootup_context = {
+	.status = IPIPE_STALL_MASK,
+	.domain = &ipipe_root,
+};
+#else
+#define bootup_context ipipe_percpu.root
+#endif	/* !CONFIG_SMP */
+
+DEFINE_PER_CPU(struct ipipe_percpu_data, ipipe_percpu) = {
+	.root = {
+		.status = IPIPE_STALL_MASK,
+		.domain = &ipipe_root,
+	},
+	.curr = &bootup_context,
+	.hrtimer_irq = -1,
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+	.context_check = 1,
+#endif
+};
+EXPORT_PER_CPU_SYMBOL(ipipe_percpu);
+
+/* Up to 2k of pending work data per CPU. */
+#define WORKBUF_SIZE 2048
+static DEFINE_PER_CPU_ALIGNED(unsigned char[WORKBUF_SIZE], work_buf);
+static DEFINE_PER_CPU(void *, work_tail);
+static unsigned int __ipipe_work_virq;
+
+static void __ipipe_do_work(unsigned int virq, void *cookie);
+
+#ifdef CONFIG_SMP
+
+#define IPIPE_CRITICAL_TIMEOUT	1000000
+static cpumask_t __ipipe_cpu_sync_map;
+static cpumask_t __ipipe_cpu_lock_map;
+static cpumask_t __ipipe_cpu_pass_map;
+static unsigned long __ipipe_critical_lock;
+static IPIPE_DEFINE_SPINLOCK(__ipipe_cpu_barrier);
+static atomic_t __ipipe_critical_count = ATOMIC_INIT(0);
+static void (*__ipipe_cpu_sync) (void);
+
+#else /* !CONFIG_SMP */
+/*
+ * Create an alias to the unique root status, so that arch-dep code
+ * may get fast access to this percpu variable including from
+ * assembly.  A hard-coded assumption is that root.status appears at
+ * offset #0 of the ipipe_percpu struct.
+ */
+extern unsigned long __ipipe_root_status
+__attribute__((alias(__stringify(ipipe_percpu))));
+EXPORT_SYMBOL(__ipipe_root_status);
+
+#endif /* !CONFIG_SMP */
+
+IPIPE_DEFINE_SPINLOCK(__ipipe_lock);
+
+static unsigned long __ipipe_virtual_irq_map;
+
+#ifdef CONFIG_PRINTK
+unsigned int __ipipe_printk_virq;
+int __ipipe_printk_bypass;
+#endif /* CONFIG_PRINTK */
+
+#ifdef CONFIG_PROC_FS
+
+struct proc_dir_entry *ipipe_proc_root;
+
+static int __ipipe_version_info_show(struct seq_file *p, void *data)
+{
+	seq_printf(p, "%d\n", IPIPE_CORE_RELEASE);
+	return 0;
+}
+
+static int __ipipe_version_info_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, __ipipe_version_info_show, NULL);
+}
+
+static const struct file_operations __ipipe_version_proc_ops = {
+	.open		= __ipipe_version_info_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+static int __ipipe_common_info_show(struct seq_file *p, void *data)
+{
+	struct ipipe_domain *ipd = (struct ipipe_domain *)p->private;
+	char handling, lockbit, virtuality;
+	unsigned long ctlbits;
+	unsigned int irq;
+
+	seq_printf(p, "        +--- Handled\n");
+	seq_printf(p, "        |+-- Locked\n");
+	seq_printf(p, "        ||+- Virtual\n");
+	seq_printf(p, " [IRQ]  |||  Handler\n");
+
+	mutex_lock(&ipd->mutex);
+
+	for (irq = 0; irq < IPIPE_NR_IRQS; irq++) {
+		ctlbits = ipd->irqs[irq].control;
+		/*
+		 * There might be a hole between the last external IRQ
+		 * and the first virtual one; skip it.
+		 */
+		if (irq >= IPIPE_NR_XIRQS && !ipipe_virtual_irq_p(irq))
+			continue;
+
+		if (ipipe_virtual_irq_p(irq)
+		    && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map))
+			/* Non-allocated virtual IRQ; skip it. */
+			continue;
+
+		if (ctlbits & IPIPE_HANDLE_MASK)
+			handling = 'H';
+		else
+			handling = '.';
+
+		if (ctlbits & IPIPE_LOCK_MASK)
+			lockbit = 'L';
+		else
+			lockbit = '.';
+
+		if (ipipe_virtual_irq_p(irq))
+			virtuality = 'V';
+		else
+			virtuality = '.';
+
+		if (ctlbits & IPIPE_HANDLE_MASK)
+			seq_printf(p, " %4u:  %c%c%c  %pf\n",
+				   irq, handling, lockbit, virtuality,
+				   ipd->irqs[irq].handler);
+		else
+			seq_printf(p, " %4u:  %c%c%c\n",
+				   irq, handling, lockbit, virtuality);
+	}
+
+	mutex_unlock(&ipd->mutex);
+
+	return 0;
+}
+
+static int __ipipe_common_info_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, __ipipe_common_info_show, PDE_DATA(inode));
+}
+
+static const struct file_operations __ipipe_info_proc_ops = {
+	.owner		= THIS_MODULE,
+	.open		= __ipipe_common_info_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+void add_domain_proc(struct ipipe_domain *ipd)
+{
+	proc_create_data(ipd->name, 0444, ipipe_proc_root,
+			 &__ipipe_info_proc_ops, ipd);
+}
+
+void remove_domain_proc(struct ipipe_domain *ipd)
+{
+	remove_proc_entry(ipd->name, ipipe_proc_root);
+}
+
+void __init __ipipe_init_proc(void)
+{
+	ipipe_proc_root = proc_mkdir("ipipe", NULL);
+	proc_create("version", 0444, ipipe_proc_root,
+		    &__ipipe_version_proc_ops);
+	add_domain_proc(ipipe_root_domain);
+
+	__ipipe_init_tracer();
+}
+
+#else
+
+static inline void add_domain_proc(struct ipipe_domain *ipd)
+{
+}
+
+static inline void remove_domain_proc(struct ipipe_domain *ipd)
+{
+}
+
+#endif	/* CONFIG_PROC_FS */
+
+static void init_stage(struct ipipe_domain *ipd)
+{
+	memset(&ipd->irqs, 0, sizeof(ipd->irqs));
+	mutex_init(&ipd->mutex);
+	__ipipe_legacy_init_stage(ipd);
+	__ipipe_hook_critical_ipi(ipd);
+}
+
+static inline int root_context_offset(void)
+{
+	void root_context_not_at_start_of_ipipe_percpu(void);
+
+	/* ipipe_percpu.root must be found at offset #0. */
+
+	if (offsetof(struct ipipe_percpu_data, root))
+		root_context_not_at_start_of_ipipe_percpu();
+
+	return 0;
+}
+
+#ifdef CONFIG_SMP
+
+static inline void fixup_percpu_data(void)
+{
+	struct ipipe_percpu_data *p;
+	int cpu;
+
+	/*
+	 * ipipe_percpu.curr cannot be assigned statically to
+	 * &ipipe_percpu.root, due to the dynamic nature of percpu
+	 * data. So we make ipipe_percpu.curr refer to a temporary
+	 * boot up context in static memory, until we can fixup all
+	 * context pointers in this routine, after per-cpu areas have
+	 * been eventually set up. The temporary context data is
+	 * copied to per_cpu(ipipe_percpu, 0).root in the same move.
+	 *
+	 * Obviously, this code must run over the boot CPU, before SMP
+	 * operations start.
+	 */
+	BUG_ON(smp_processor_id() || !irqs_disabled());
+
+	per_cpu(ipipe_percpu, 0).root = bootup_context;
+
+	for_each_possible_cpu(cpu) {
+		p = &per_cpu(ipipe_percpu, cpu);
+		p->curr = &p->root;
+	}
+}
+
+#else /* !CONFIG_SMP */
+
+static inline void fixup_percpu_data(void) { }
+
+#endif /* CONFIG_SMP */
+
+void __init __ipipe_init_early(void)
+{
+	struct ipipe_domain *ipd = &ipipe_root;
+	int cpu;
+
+	fixup_percpu_data();
+
+	/*
+	 * A lightweight registration code for the root domain. We are
+	 * running on the boot CPU, hw interrupts are off, and
+	 * secondary CPUs are still lost in space.
+	 */
+	ipd->name = "Linux";
+	ipd->context_offset = root_context_offset();
+	init_stage(ipd);
+
+	/*
+	 * Do the early init stuff. First we do the per-arch pipeline
+	 * core setup, then we run the per-client setup code. At this
+	 * point, the kernel does not provide much services yet: be
+	 * careful.
+	 */
+	__ipipe_early_core_setup();
+	__ipipe_early_client_setup();
+
+#ifdef CONFIG_PRINTK
+	__ipipe_printk_virq = ipipe_alloc_virq();
+	ipd->irqs[__ipipe_printk_virq].handler = __ipipe_flush_printk;
+	ipd->irqs[__ipipe_printk_virq].cookie = NULL;
+	ipd->irqs[__ipipe_printk_virq].ackfn = NULL;
+	ipd->irqs[__ipipe_printk_virq].control = IPIPE_HANDLE_MASK;
+#endif /* CONFIG_PRINTK */
+
+	__ipipe_work_virq = ipipe_alloc_virq();
+	ipd->irqs[__ipipe_work_virq].handler = __ipipe_do_work;
+	ipd->irqs[__ipipe_work_virq].cookie = NULL;
+	ipd->irqs[__ipipe_work_virq].ackfn = NULL;
+	ipd->irqs[__ipipe_work_virq].control = IPIPE_HANDLE_MASK;
+
+	for_each_possible_cpu(cpu)
+		per_cpu(work_tail, cpu) = per_cpu(work_buf, cpu);
+}
+
+void __init __ipipe_init(void)
+{
+	/* Now we may engage the pipeline. */
+	__ipipe_enable_pipeline();
+
+	pr_info("Interrupt pipeline (release #%d)\n", IPIPE_CORE_RELEASE);
+}
+
+static inline void init_head_stage(struct ipipe_domain *ipd)
+{
+	struct ipipe_percpu_domain_data *p;
+	int cpu;
+
+	/* Must be set first, used in ipipe_percpu_context(). */
+	ipd->context_offset = offsetof(struct ipipe_percpu_data, head);
+
+	for_each_online_cpu(cpu) {
+		p = ipipe_percpu_context(ipd, cpu);
+		memset(p, 0, sizeof(*p));
+		p->domain = ipd;
+	}
+
+	init_stage(ipd);
+}
+
+void ipipe_register_head(struct ipipe_domain *ipd, const char *name)
+{
+	BUG_ON(!ipipe_root_p || ipd == &ipipe_root);
+
+	ipd->name = name;
+	init_head_stage(ipd);
+	barrier();
+	ipipe_head_domain = ipd;
+	add_domain_proc(ipd);
+
+	pr_info("I-pipe: head domain %s registered.\n", name);
+}
+EXPORT_SYMBOL_GPL(ipipe_register_head);
+
+void ipipe_unregister_head(struct ipipe_domain *ipd)
+{
+	BUG_ON(!ipipe_root_p || ipd != ipipe_head_domain);
+
+	ipipe_head_domain = &ipipe_root;
+	smp_mb();
+	mutex_lock(&ipd->mutex);
+	remove_domain_proc(ipd);
+	mutex_unlock(&ipd->mutex);
+
+	pr_info("I-pipe: head domain %s unregistered.\n", ipd->name);
+}
+EXPORT_SYMBOL_GPL(ipipe_unregister_head);
+
+void ipipe_unstall_root(void)
+{
+	struct ipipe_percpu_domain_data *p;
+
+	hard_local_irq_disable();
+
+	/* This helps catching bad usage from assembly call sites. */
+	ipipe_root_only();
+
+	p = ipipe_this_cpu_root_context();
+
+	__clear_bit(IPIPE_STALL_FLAG, &p->status);
+
+	if (unlikely(__ipipe_ipending_p(p)))
+		__ipipe_sync_stage();
+
+	hard_local_irq_enable();
+}
+EXPORT_SYMBOL(ipipe_unstall_root);
+
+void ipipe_restore_root(unsigned long x)
+{
+	ipipe_root_only();
+
+	if (x)
+		ipipe_stall_root();
+	else
+		ipipe_unstall_root();
+}
+EXPORT_SYMBOL(ipipe_restore_root);
+
+void __ipipe_restore_root_nosync(unsigned long x)
+{
+	struct ipipe_percpu_domain_data *p = ipipe_this_cpu_root_context();
+
+	if (raw_irqs_disabled_flags(x)) {
+		__set_bit(IPIPE_STALL_FLAG, &p->status);
+		trace_hardirqs_off();
+	} else {
+		trace_hardirqs_on();
+		__clear_bit(IPIPE_STALL_FLAG, &p->status);
+	}
+}
+EXPORT_SYMBOL_GPL(__ipipe_restore_root_nosync);
+
+void ipipe_unstall_head(void)
+{
+	struct ipipe_percpu_domain_data *p = ipipe_this_cpu_head_context();
+
+	hard_local_irq_disable();
+
+	__clear_bit(IPIPE_STALL_FLAG, &p->status);
+
+	if (unlikely(__ipipe_ipending_p(p)))
+		__ipipe_sync_pipeline(ipipe_head_domain);
+
+	hard_local_irq_enable();
+}
+EXPORT_SYMBOL_GPL(ipipe_unstall_head);
+
+void __ipipe_restore_head(unsigned long x) /* hw interrupt off */
+{
+	struct ipipe_percpu_domain_data *p = ipipe_this_cpu_head_context();
+
+	if (x) {
+#ifdef CONFIG_DEBUG_KERNEL
+		static int warned;
+		if (!warned &&
+		    __test_and_set_bit(IPIPE_STALL_FLAG, &p->status)) {
+			/*
+			 * Already stalled albeit ipipe_restore_head()
+			 * should have detected it? Send a warning once.
+			 */
+			hard_local_irq_enable();
+			warned = 1;
+			pr_warning("I-pipe: ipipe_restore_head() "
+				   "optimization failed.\n");
+			dump_stack();
+			hard_local_irq_disable();
+		}
+#else /* !CONFIG_DEBUG_KERNEL */
+		__set_bit(IPIPE_STALL_FLAG, &p->status);
+#endif /* CONFIG_DEBUG_KERNEL */
+	} else {
+		__clear_bit(IPIPE_STALL_FLAG, &p->status);
+		if (unlikely(__ipipe_ipending_p(p)))
+			__ipipe_sync_pipeline(ipipe_head_domain);
+		hard_local_irq_enable();
+	}
+}
+EXPORT_SYMBOL_GPL(__ipipe_restore_head);
+
+void __ipipe_spin_lock_irq(ipipe_spinlock_t *lock)
+{
+	hard_local_irq_disable();
+	if (ipipe_smp_p)
+		arch_spin_lock(&lock->arch_lock);
+	__set_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status);
+}
+EXPORT_SYMBOL_GPL(__ipipe_spin_lock_irq);
+
+void __ipipe_spin_unlock_irq(ipipe_spinlock_t *lock)
+{
+	if (ipipe_smp_p)
+		arch_spin_unlock(&lock->arch_lock);
+	__clear_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status);
+	hard_local_irq_enable();
+}
+EXPORT_SYMBOL_GPL(__ipipe_spin_unlock_irq);
+
+unsigned long __ipipe_spin_lock_irqsave(ipipe_spinlock_t *lock)
+{
+	unsigned long flags;
+	int s;
+
+	flags = hard_local_irq_save();
+	if (ipipe_smp_p)
+		arch_spin_lock(&lock->arch_lock);
+	s = __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status);
+
+	return arch_mangle_irq_bits(s, flags);
+}
+EXPORT_SYMBOL_GPL(__ipipe_spin_lock_irqsave);
+
+int __ipipe_spin_trylock_irqsave(ipipe_spinlock_t *lock,
+				 unsigned long *x)
+{
+	unsigned long flags;
+	int s;
+
+	flags = hard_local_irq_save();
+	if (ipipe_smp_p && !arch_spin_trylock(&lock->arch_lock)) {
+		hard_local_irq_restore(flags);
+		return 0;
+	}
+	s = __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status);
+	*x = arch_mangle_irq_bits(s, flags);
+
+	return 1;
+}
+EXPORT_SYMBOL_GPL(__ipipe_spin_trylock_irqsave);
+
+void __ipipe_spin_unlock_irqrestore(ipipe_spinlock_t *lock,
+				    unsigned long x)
+{
+	if (ipipe_smp_p)
+		arch_spin_unlock(&lock->arch_lock);
+	if (!arch_demangle_irq_bits(&x))
+		__clear_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status);
+	hard_local_irq_restore(x);
+}
+EXPORT_SYMBOL_GPL(__ipipe_spin_unlock_irqrestore);
+
+int __ipipe_spin_trylock_irq(ipipe_spinlock_t *lock)
+{
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	if (ipipe_smp_p && !arch_spin_trylock(&lock->arch_lock)) {
+		hard_local_irq_restore(flags);
+		return 0;
+	}
+	__set_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status);
+
+	return 1;
+}
+EXPORT_SYMBOL_GPL(__ipipe_spin_trylock_irq);
+
+void __ipipe_spin_unlock_irqbegin(ipipe_spinlock_t *lock)
+{
+	if (ipipe_smp_p)
+		arch_spin_unlock(&lock->arch_lock);
+}
+
+void __ipipe_spin_unlock_irqcomplete(unsigned long x)
+{
+	if (!arch_demangle_irq_bits(&x))
+		__clear_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status);
+	hard_local_irq_restore(x);
+}
+
+#ifdef __IPIPE_3LEVEL_IRQMAP
+
+/* Must be called hw IRQs off. */
+static inline void __ipipe_set_irq_held(struct ipipe_percpu_domain_data *p,
+					unsigned int irq)
+{
+	__set_bit(irq, p->irqheld_map);
+	p->irqall[irq]++;
+}
+
+/* Must be called hw IRQs off. */
+void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned int irq)
+{
+	struct ipipe_percpu_domain_data *p = ipipe_this_cpu_context(ipd);
+	int l0b, l1b;
+
+	IPIPE_WARN_ONCE(!hard_irqs_disabled());
+
+	l0b = irq / (BITS_PER_LONG * BITS_PER_LONG);
+	l1b = irq / BITS_PER_LONG;
+
+	if (likely(!test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) {
+		__set_bit(irq, p->irqpend_lomap);
+		__set_bit(l1b, p->irqpend_mdmap);
+		__set_bit(l0b, &p->irqpend_himap);
+	} else
+		__set_bit(irq, p->irqheld_map);
+
+	p->irqall[irq]++;
+}
+EXPORT_SYMBOL_GPL(__ipipe_set_irq_pending);
+
+/* Must be called hw IRQs off. */
+void __ipipe_lock_irq(unsigned int irq)
+{
+	struct ipipe_domain *ipd = ipipe_root_domain;
+	struct ipipe_percpu_domain_data *p;
+	int l0b, l1b;
+
+	IPIPE_WARN_ONCE(!hard_irqs_disabled());
+
+	/*
+	 * Interrupts requested by a registered head domain cannot be
+	 * locked, since this would make no sense: interrupts are
+	 * globally masked at CPU level when the head domain is
+	 * stalled, so there is no way we could encounter the
+	 * situation IRQ locks are handling.
+	 */
+	if (test_and_set_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
+		return;
+
+	l0b = irq / (BITS_PER_LONG * BITS_PER_LONG);
+	l1b = irq / BITS_PER_LONG;
+
+	p = ipipe_this_cpu_context(ipd);
+	if (__test_and_clear_bit(irq, p->irqpend_lomap)) {
+		__set_bit(irq, p->irqheld_map);
+		if (p->irqpend_lomap[l1b] == 0) {
+			__clear_bit(l1b, p->irqpend_mdmap);
+			if (p->irqpend_mdmap[l0b] == 0)
+				__clear_bit(l0b, &p->irqpend_himap);
+		}
+	}
+}
+EXPORT_SYMBOL_GPL(__ipipe_lock_irq);
+
+/* Must be called hw IRQs off. */
+void __ipipe_unlock_irq(unsigned int irq)
+{
+	struct ipipe_domain *ipd = ipipe_root_domain;
+	struct ipipe_percpu_domain_data *p;
+	int l0b, l1b, cpu;
+
+	IPIPE_WARN_ONCE(!hard_irqs_disabled());
+
+	if (!test_and_clear_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
+		return;
+
+	l0b = irq / (BITS_PER_LONG * BITS_PER_LONG);
+	l1b = irq / BITS_PER_LONG;
+
+	for_each_online_cpu(cpu) {
+		p = ipipe_this_cpu_root_context();
+		if (test_and_clear_bit(irq, p->irqheld_map)) {
+			/* We need atomic ops here: */
+			set_bit(irq, p->irqpend_lomap);
+			set_bit(l1b, p->irqpend_mdmap);
+			set_bit(l0b, &p->irqpend_himap);
+		}
+	}
+}
+EXPORT_SYMBOL_GPL(__ipipe_unlock_irq);
+
+static inline int __ipipe_next_irq(struct ipipe_percpu_domain_data *p)
+{
+	int l0b, l1b, l2b;
+	unsigned long l0m, l1m, l2m;
+	unsigned int irq;
+
+	l0m = p->irqpend_himap;
+	if (unlikely(l0m == 0))
+		return -1;
+
+	l0b = __ipipe_ffnz(l0m);
+	l1m = p->irqpend_mdmap[l0b];
+	if (unlikely(l1m == 0))
+		return -1;
+
+	l1b = __ipipe_ffnz(l1m) + l0b * BITS_PER_LONG;
+	l2m = p->irqpend_lomap[l1b];
+	if (unlikely(l2m == 0))
+		return -1;
+
+	l2b = __ipipe_ffnz(l2m);
+	irq = l1b * BITS_PER_LONG + l2b;
+
+	__clear_bit(irq, p->irqpend_lomap);
+	if (p->irqpend_lomap[l1b] == 0) {
+		__clear_bit(l1b, p->irqpend_mdmap);
+		if (p->irqpend_mdmap[l0b] == 0)
+			__clear_bit(l0b, &p->irqpend_himap);
+	}
+
+	return irq;
+}
+
+#else /* __IPIPE_2LEVEL_IRQMAP */
+
+/* Must be called hw IRQs off. */
+static inline void __ipipe_set_irq_held(struct ipipe_percpu_domain_data *p,
+					unsigned int irq)
+{
+	__set_bit(irq, p->irqheld_map);
+	p->irqall[irq]++;
+}
+
+/* Must be called hw IRQs off. */
+void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned int irq)
+{
+	struct ipipe_percpu_domain_data *p = ipipe_this_cpu_context(ipd);
+	int l0b = irq / BITS_PER_LONG;
+
+	IPIPE_WARN_ONCE(!hard_irqs_disabled());
+
+	if (likely(!test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) {
+		__set_bit(irq, p->irqpend_lomap);
+		__set_bit(l0b, &p->irqpend_himap);
+	} else
+		__set_bit(irq, p->irqheld_map);
+
+	p->irqall[irq]++;
+}
+EXPORT_SYMBOL_GPL(__ipipe_set_irq_pending);
+
+/* Must be called hw IRQs off. */
+void __ipipe_lock_irq(unsigned int irq)
+{
+	struct ipipe_percpu_domain_data *p;
+	int l0b = irq / BITS_PER_LONG;
+
+	IPIPE_WARN_ONCE(!hard_irqs_disabled());
+
+	if (test_and_set_bit(IPIPE_LOCK_FLAG,
+			     &ipipe_root_domain->irqs[irq].control))
+		return;
+
+	p = ipipe_this_cpu_root_context();
+	if (__test_and_clear_bit(irq, p->irqpend_lomap)) {
+		__set_bit(irq, p->irqheld_map);
+		if (p->irqpend_lomap[l0b] == 0)
+			__clear_bit(l0b, &p->irqpend_himap);
+	}
+}
+EXPORT_SYMBOL_GPL(__ipipe_lock_irq);
+
+/* Must be called hw IRQs off. */
+void __ipipe_unlock_irq(unsigned int irq)
+{
+	struct ipipe_domain *ipd = ipipe_root_domain;
+	struct ipipe_percpu_domain_data *p;
+	int l0b = irq / BITS_PER_LONG, cpu;
+
+	IPIPE_WARN_ONCE(!hard_irqs_disabled());
+
+	if (!test_and_clear_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
+		return;
+
+	for_each_online_cpu(cpu) {
+		p = ipipe_percpu_context(ipd, cpu);
+		if (test_and_clear_bit(irq, p->irqheld_map)) {
+			/* We need atomic ops here: */
+			set_bit(irq, p->irqpend_lomap);
+			set_bit(l0b, &p->irqpend_himap);
+		}
+	}
+}
+EXPORT_SYMBOL_GPL(__ipipe_unlock_irq);
+
+static inline int __ipipe_next_irq(struct ipipe_percpu_domain_data *p)
+{
+	unsigned long l0m, l1m;
+	int l0b, l1b;
+
+	l0m = p->irqpend_himap;
+	if (unlikely(l0m == 0))
+		return -1;
+
+	l0b = __ipipe_ffnz(l0m);
+	l1m = p->irqpend_lomap[l0b];
+	if (unlikely(l1m == 0))
+		return -1;
+
+	l1b = __ipipe_ffnz(l1m);
+	__clear_bit(l1b, &p->irqpend_lomap[l0b]);
+	if (p->irqpend_lomap[l0b] == 0)
+		__clear_bit(l0b, &p->irqpend_himap);
+
+	return l0b * BITS_PER_LONG + l1b;
+}
+
+#endif /* __IPIPE_2LEVEL_IRQMAP */
+
+void __ipipe_do_sync_pipeline(struct ipipe_domain *top)
+{
+	struct ipipe_percpu_domain_data *p;
+	struct ipipe_domain *ipd;
+
+	/* We must enter over the root domain. */
+	IPIPE_WARN_ONCE(__ipipe_current_domain != ipipe_root_domain);
+	ipd = top;
+next:
+	p = ipipe_this_cpu_context(ipd);
+	if (test_bit(IPIPE_STALL_FLAG, &p->status))
+		return;
+
+	if (__ipipe_ipending_p(p)) {
+		if (ipd == ipipe_root_domain)
+			__ipipe_sync_stage();
+		else {
+			/* Switching to head. */
+			p->coflags &= ~__IPIPE_ALL_R;
+			__ipipe_set_current_context(p);
+			__ipipe_sync_stage();
+			__ipipe_set_current_domain(ipipe_root_domain);
+		}
+	}
+
+	if (ipd != ipipe_root_domain) {
+		ipd = ipipe_root_domain;
+		goto next;
+	}
+}
+EXPORT_SYMBOL_GPL(__ipipe_do_sync_pipeline);
+
+unsigned int ipipe_alloc_virq(void)
+{
+	unsigned long flags, irq = 0;
+	int ipos;
+
+	spin_lock_irqsave(&__ipipe_lock, flags);
+
+	if (__ipipe_virtual_irq_map != ~0) {
+		ipos = ffz(__ipipe_virtual_irq_map);
+		set_bit(ipos, &__ipipe_virtual_irq_map);
+		irq = ipos + IPIPE_VIRQ_BASE;
+	}
+
+	spin_unlock_irqrestore(&__ipipe_lock, flags);
+
+	return irq;
+}
+EXPORT_SYMBOL_GPL(ipipe_alloc_virq);
+
+void ipipe_free_virq(unsigned int virq)
+{
+	clear_bit(virq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map);
+	smp_mb__after_atomic();
+}
+EXPORT_SYMBOL_GPL(ipipe_free_virq);
+
+int ipipe_request_irq(struct ipipe_domain *ipd,
+		      unsigned int irq,
+		      ipipe_irq_handler_t handler,
+		      void *cookie,
+		      ipipe_irq_ackfn_t ackfn)
+{
+	unsigned long flags;
+	int ret = 0;
+
+#ifndef CONFIG_IPIPE_LEGACY
+	ipipe_root_only();
+#endif /* CONFIG_IPIPE_LEGACY */
+
+	if (handler == NULL ||
+	    (irq >= IPIPE_NR_XIRQS && !ipipe_virtual_irq_p(irq)))
+		return -EINVAL;
+
+	spin_lock_irqsave(&__ipipe_lock, flags);
+
+	if (ipd->irqs[irq].handler) {
+		ret = -EBUSY;
+		goto out;
+	}
+
+	if (ackfn == NULL)
+		ackfn = ipipe_root_domain->irqs[irq].ackfn;
+
+	ipd->irqs[irq].handler = handler;
+	ipd->irqs[irq].cookie = cookie;
+	ipd->irqs[irq].ackfn = ackfn;
+	ipd->irqs[irq].control = IPIPE_HANDLE_MASK;
+
+	if (irq < IPIPE_NR_ROOT_IRQS)
+		__ipipe_enable_irqdesc(ipd, irq);
+out:
+	spin_unlock_irqrestore(&__ipipe_lock, flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(ipipe_request_irq);
+
+void ipipe_free_irq(struct ipipe_domain *ipd,
+		    unsigned int irq)
+{
+	unsigned long flags;
+
+#ifndef CONFIG_IPIPE_LEGACY
+	ipipe_root_only();
+#endif /* CONFIG_IPIPE_LEGACY */
+
+	spin_lock_irqsave(&__ipipe_lock, flags);
+
+	if (ipd->irqs[irq].handler == NULL)
+		goto out;
+
+	ipd->irqs[irq].handler = NULL;
+	ipd->irqs[irq].cookie = NULL;
+	ipd->irqs[irq].ackfn = NULL;
+	ipd->irqs[irq].control = 0;
+
+	if (irq < IPIPE_NR_ROOT_IRQS)
+		__ipipe_disable_irqdesc(ipd, irq);
+out:
+	spin_unlock_irqrestore(&__ipipe_lock, flags);
+}
+EXPORT_SYMBOL_GPL(ipipe_free_irq);
+
+void ipipe_set_hooks(struct ipipe_domain *ipd, int enables)
+{
+	struct ipipe_percpu_domain_data *p;
+	unsigned long flags;
+	int cpu, wait;
+
+	if (ipd == ipipe_root_domain) {
+		IPIPE_WARN(enables & __IPIPE_TRAP_E);
+		enables &= ~__IPIPE_TRAP_E;
+	} else {
+		IPIPE_WARN(enables & __IPIPE_KEVENT_E);
+		enables &= ~__IPIPE_KEVENT_E;
+	}
+
+	flags = ipipe_critical_enter(NULL);
+
+	for_each_online_cpu(cpu) {
+		p = ipipe_percpu_context(ipd, cpu);
+		p->coflags &= ~__IPIPE_ALL_E;
+		p->coflags |= enables;
+	}
+
+	wait = (enables ^ __IPIPE_ALL_E) << __IPIPE_SHIFT_R;
+	if (wait == 0 || !__ipipe_root_p) {
+		ipipe_critical_exit(flags);
+		return;
+	}
+
+	ipipe_this_cpu_context(ipd)->coflags &= ~wait;
+
+	ipipe_critical_exit(flags);
+
+	/*
+	 * In case we cleared some hooks over the root domain, we have
+	 * to wait for any ongoing execution to finish, since our
+	 * caller might subsequently unmap the target domain code.
+	 *
+	 * We synchronize with the relevant __ipipe_notify_*()
+	 * helpers, disabling all hooks before we start waiting for
+	 * completion on all CPUs.
+	 */
+	for_each_online_cpu(cpu) {
+		while (ipipe_percpu_context(ipd, cpu)->coflags & wait)
+			schedule_timeout_interruptible(HZ / 50);
+	}
+}
+EXPORT_SYMBOL_GPL(ipipe_set_hooks);
+
+int __weak ipipe_fastcall_hook(struct pt_regs *regs)
+{
+	return -1;	/* i.e. fall back to slow path. */
+}
+
+int __weak ipipe_syscall_hook(struct ipipe_domain *ipd, struct pt_regs *regs)
+{
+	return 0;
+}
+
+void __ipipe_root_sync(void)
+{
+	struct ipipe_percpu_domain_data *p;
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+
+	p = ipipe_this_cpu_root_context();
+	if (__ipipe_ipending_p(p))
+		__ipipe_sync_stage();
+
+	hard_local_irq_restore(flags);
+}
+
+int __ipipe_notify_syscall(struct pt_regs *regs)
+{
+	struct ipipe_domain *caller_domain, *this_domain, *ipd;
+	struct ipipe_percpu_domain_data *p;
+	unsigned long flags;
+	int ret = 0;
+
+	/*
+	 * We should definitely not pipeline a syscall with IRQs off.
+	 */
+	IPIPE_WARN_ONCE(hard_irqs_disabled());
+
+	flags = hard_local_irq_save();
+	caller_domain = this_domain = __ipipe_current_domain;
+	ipd = ipipe_head_domain;
+next:
+	p = ipipe_this_cpu_context(ipd);
+	if (likely(p->coflags & __IPIPE_SYSCALL_E)) {
+		__ipipe_set_current_context(p);
+		p->coflags |= __IPIPE_SYSCALL_R;
+		hard_local_irq_restore(flags);
+		ret = ipipe_syscall_hook(caller_domain, regs);
+		flags = hard_local_irq_save();
+		p->coflags &= ~__IPIPE_SYSCALL_R;
+		if (__ipipe_current_domain != ipd)
+			/* Account for domain migration. */
+			this_domain = __ipipe_current_domain;
+		else
+			__ipipe_set_current_domain(this_domain);
+	}
+
+	if (this_domain == ipipe_root_domain) {
+		if (ipd != ipipe_root_domain && ret == 0) {
+			ipd = ipipe_root_domain;
+			goto next;
+		}
+		/*
+		 * Careful: we may have migrated from head->root, so p
+		 * would be ipipe_this_cpu_context(head).
+		 */
+		p = ipipe_this_cpu_root_context();
+		if (__ipipe_ipending_p(p))
+			__ipipe_sync_stage();
+ 	} else if (ipipe_test_thread_flag(TIP_MAYDAY))
+		__ipipe_call_mayday(regs);
+
+	hard_local_irq_restore(flags);
+
+	return ret;
+}
+
+int __weak ipipe_trap_hook(struct ipipe_trap_data *data)
+{
+	return 0;
+}
+
+int __ipipe_notify_trap(int exception, struct pt_regs *regs)
+{
+	struct ipipe_percpu_domain_data *p;
+	struct ipipe_trap_data data;
+	unsigned long flags;
+	int ret = 0;
+
+	flags = hard_local_irq_save();
+
+	/*
+	 * We send a notification about all traps raised over a
+	 * registered head domain only.
+	 */
+	if (__ipipe_root_p)
+		goto out;
+
+	p = ipipe_this_cpu_head_context();
+	if (likely(p->coflags & __IPIPE_TRAP_E)) {
+		p->coflags |= __IPIPE_TRAP_R;
+		hard_local_irq_restore(flags);
+		data.exception = exception;
+		data.regs = regs;
+		ret = ipipe_trap_hook(&data);
+		flags = hard_local_irq_save();
+		p->coflags &= ~__IPIPE_TRAP_R;
+	}
+out:
+	hard_local_irq_restore(flags);
+
+	return ret;
+}
+
+int __weak ipipe_kevent_hook(int kevent, void *data)
+{
+	return 0;
+}
+
+int __ipipe_notify_kevent(int kevent, void *data)
+{
+	struct ipipe_percpu_domain_data *p;
+	unsigned long flags;
+	int ret = 0;
+
+	ipipe_root_only();
+
+	flags = hard_local_irq_save();
+
+	p = ipipe_this_cpu_root_context();
+	if (likely(p->coflags & __IPIPE_KEVENT_E)) {
+		p->coflags |= __IPIPE_KEVENT_R;
+		hard_local_irq_restore(flags);
+		ret = ipipe_kevent_hook(kevent, data);
+		flags = hard_local_irq_save();
+		p->coflags &= ~__IPIPE_KEVENT_R;
+	}
+
+	hard_local_irq_restore(flags);
+
+	return ret;
+}
+
+void __weak ipipe_migration_hook(struct task_struct *p)
+{
+}
+
+#ifdef CONFIG_IPIPE_LEGACY
+
+static inline void complete_domain_migration(void) /* hw IRQs off */
+{
+	if (current->state & TASK_HARDENING) {
+		current->state &= ~TASK_HARDENING;
+		ipipe_set_thread_flag(TIP_HEAD);
+	}
+}
+
+#else /* !CONFIG_IPIPE_LEGACY */
+
+static void complete_domain_migration(void) /* hw IRQs off */
+{
+	struct ipipe_percpu_domain_data *p;
+	struct ipipe_percpu_data *pd;
+	struct task_struct *t;
+
+	ipipe_root_only();
+	pd = raw_cpu_ptr(&ipipe_percpu);
+	t = pd->task_hijacked;
+	if (t == NULL)
+		return;
+
+	pd->task_hijacked = NULL;
+	t->state &= ~TASK_HARDENING;
+	if (t->state != TASK_INTERRUPTIBLE)
+		/* Migration aborted (by signal). */
+		return;
+
+	ipipe_set_ti_thread_flag(task_thread_info(t), TIP_HEAD);
+	p = ipipe_this_cpu_head_context();
+	IPIPE_WARN_ONCE(test_bit(IPIPE_STALL_FLAG, &p->status));
+	/*
+	 * hw IRQs are disabled, but the completion hook assumes the
+	 * head domain is logically stalled: fix it up.
+	 */
+	__set_bit(IPIPE_STALL_FLAG, &p->status);
+	ipipe_migration_hook(t);
+	__clear_bit(IPIPE_STALL_FLAG, &p->status);
+	if (__ipipe_ipending_p(p))
+		__ipipe_sync_pipeline(p->domain);
+}
+
+#endif /* !CONFIG_IPIPE_LEGACY */
+
+void __ipipe_complete_domain_migration(void)
+{
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	complete_domain_migration();
+	hard_local_irq_restore(flags);
+}
+EXPORT_SYMBOL_GPL(__ipipe_complete_domain_migration);
+
+int __ipipe_switch_tail(void)
+{
+	int x;
+
+#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH
+	hard_local_irq_disable();
+#endif
+	x = __ipipe_root_p;
+#ifndef CONFIG_IPIPE_LEGACY
+	if (x)
+#endif
+		complete_domain_migration();
+
+#ifndef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH
+	if (x)
+#endif
+		hard_local_irq_enable();
+
+	return !x;
+}
+
+#ifdef CONFIG_IPIPE_HAVE_VM_NOTIFIER
+void __ipipe_notify_vm_preemption(void)
+{
+	struct ipipe_vm_notifier *vmf;
+	struct ipipe_percpu_data *p;
+
+	ipipe_check_irqoff();
+	p = __ipipe_raw_cpu_ptr(&ipipe_percpu);
+	vmf = p->vm_notifier;
+	if (unlikely(vmf))
+		vmf->handler(vmf);
+}
+EXPORT_SYMBOL_GPL(__ipipe_notify_vm_preemption);
+#endif /* CONFIG_IPIPE_HAVE_VM_NOTIFIER */
+
+static void dispatch_irq_head(unsigned int irq) /* hw interrupts off */
+{
+	struct ipipe_percpu_domain_data *p = ipipe_this_cpu_head_context(), *old;
+	struct ipipe_domain *head = p->domain;
+
+	if (unlikely(test_bit(IPIPE_STALL_FLAG, &p->status))) {
+		__ipipe_set_irq_pending(head, irq);
+		return;
+	}
+
+	/* Switch to the head domain if not current. */
+	old = __ipipe_current_context;
+	if (old != p)
+		__ipipe_set_current_context(p);
+
+	p->irqall[irq]++;
+	__set_bit(IPIPE_STALL_FLAG, &p->status);
+	barrier();
+	head->irqs[irq].handler(irq, head->irqs[irq].cookie);
+	__ipipe_run_irqtail(irq);
+	hard_local_irq_disable();
+	p = ipipe_this_cpu_head_context();
+	__clear_bit(IPIPE_STALL_FLAG, &p->status);
+
+	/* Are we still running in the head domain? */
+	if (likely(__ipipe_current_context == p)) {
+		/* Did we enter this code over the head domain? */
+		if (old->domain == head) {
+			/* Yes, do immediate synchronization. */
+			if (__ipipe_ipending_p(p))
+				__ipipe_sync_stage();
+			return;
+		}
+		__ipipe_set_current_context(ipipe_this_cpu_root_context());
+	}
+
+	/*
+	 * We must be running over the root domain, synchronize
+	 * the pipeline for high priority IRQs (slow path).
+	 */
+	__ipipe_do_sync_pipeline(head);
+}
+
+void __ipipe_dispatch_irq(unsigned int irq, int flags) /* hw interrupts off */
+{
+	struct ipipe_domain *ipd;
+	struct irq_desc *desc;
+	unsigned long control;
+	int chained_irq;
+
+	/*
+	 * Survival kit when reading this code:
+	 *
+	 * - we have two main situations, leading to three cases for
+	 *   handling interrupts:
+	 *
+	 *   a) the root domain is alone, no registered head domain
+	 *      => all interrupts go through the interrupt log
+	 *   b) a head domain is registered
+	 *      => head domain IRQs go through the fast dispatcher
+	 *      => root domain IRQs go through the interrupt log
+	 *
+	 * - when no head domain is registered, ipipe_head_domain ==
+	 *   ipipe_root_domain == &ipipe_root.
+	 *
+	 * - the caller tells us whether we should acknowledge this
+	 *   IRQ. Even virtual IRQs may require acknowledge on some
+	 *   platforms (e.g. arm/SMP).
+	 *
+	 * - the caller tells us whether we may try to run the IRQ log
+	 *   syncer. Typically, demuxed IRQs won't be synced
+	 *   immediately.
+	 *
+	 * - multiplex IRQs most likely have a valid acknowledge
+	 *   handler and we may not be called with IPIPE_IRQF_NOACK
+	 *   for them. The ack handler for the multiplex IRQ actually
+	 *   decodes the demuxed interrupts.
+	 */
+
+#ifdef CONFIG_IPIPE_DEBUG
+	if (unlikely(irq >= IPIPE_NR_IRQS) ||
+	    (irq < IPIPE_NR_ROOT_IRQS && irq_to_desc(irq) == NULL)) {
+		pr_err("I-pipe: spurious interrupt %u\n", irq);
+		return;
+	}
+#endif
+	/*
+	 * CAUTION: on some archs, virtual IRQs may have acknowledge
+	 * handlers. Multiplex IRQs should have one too.
+	 */
+	if (unlikely(irq >= IPIPE_NR_ROOT_IRQS)) {
+		desc = NULL;
+		chained_irq = 0;
+	} else {
+		desc = irq_to_desc(irq);
+		chained_irq = desc ? ipipe_chained_irq_p(desc) : 0;
+	}
+	if (flags & IPIPE_IRQF_NOACK)
+		IPIPE_WARN_ONCE(chained_irq);
+	else {
+		ipd = ipipe_head_domain;
+		control = ipd->irqs[irq].control;
+		if ((control & IPIPE_HANDLE_MASK) == 0)
+			ipd = ipipe_root_domain;
+		if (ipd->irqs[irq].ackfn)
+			ipd->irqs[irq].ackfn(irq, desc);
+		if (chained_irq) {
+			if ((flags & IPIPE_IRQF_NOSYNC) == 0)
+				/* Run demuxed IRQ handlers. */
+				goto sync;
+			return;
+		}
+	}
+
+	/*
+	 * Sticky interrupts must be handled early and separately, so
+	 * that we always process them on the current domain.
+	 */
+	ipd = __ipipe_current_domain;
+	control = ipd->irqs[irq].control;
+	if (control & IPIPE_STICKY_MASK)
+		goto log;
+
+	/*
+	 * In case we have no registered head domain
+	 * (i.e. ipipe_head_domain == &ipipe_root), we always go
+	 * through the interrupt log, and leave the dispatching work
+	 * ultimately to __ipipe_sync_pipeline().
+	 */
+	ipd = ipipe_head_domain;
+	control = ipd->irqs[irq].control;
+	if (ipd == ipipe_root_domain)
+		/*
+		 * The root domain must handle all interrupts, so
+		 * testing the HANDLE bit would be pointless.
+		 */
+		goto log;
+
+	if (control & IPIPE_HANDLE_MASK) {
+		if (unlikely(flags & IPIPE_IRQF_NOSYNC))
+			__ipipe_set_irq_pending(ipd, irq);
+		else
+			dispatch_irq_head(irq);
+		return;
+	}
+
+	ipd = ipipe_root_domain;
+log:
+	__ipipe_set_irq_pending(ipd, irq);
+
+	if (flags & IPIPE_IRQF_NOSYNC)
+		return;
+
+	/*
+	 * Optimize if we preempted a registered high priority head
+	 * domain: we don't need to synchronize the pipeline unless
+	 * there is a pending interrupt for it.
+	 */
+	if (!__ipipe_root_p &&
+	    !__ipipe_ipending_p(ipipe_this_cpu_head_context()))
+		return;
+sync:
+	__ipipe_sync_pipeline(ipipe_head_domain);
+}
+
+void ipipe_raise_irq(unsigned int irq)
+{
+	struct ipipe_domain *ipd = ipipe_head_domain;
+	unsigned long flags, control;
+
+	flags = hard_local_irq_save();
+
+	/*
+	 * Fast path: raising a virtual IRQ handled by the head
+	 * domain.
+	 */
+	if (likely(ipipe_virtual_irq_p(irq) && ipd != ipipe_root_domain)) {
+		control = ipd->irqs[irq].control;
+		if (likely(control & IPIPE_HANDLE_MASK)) {
+			dispatch_irq_head(irq);
+			goto out;
+		}
+	}
+
+	/* Emulate regular device IRQ receipt. */
+	__ipipe_dispatch_irq(irq, IPIPE_IRQF_NOACK);
+out:
+	hard_local_irq_restore(flags);
+
+}
+EXPORT_SYMBOL_GPL(ipipe_raise_irq);
+
+#ifdef CONFIG_PREEMPT
+
+void preempt_schedule_irq(void);
+
+void __sched __ipipe_preempt_schedule_irq(void)
+{
+	struct ipipe_percpu_domain_data *p;
+	unsigned long flags;
+
+	BUG_ON(!hard_irqs_disabled());
+	local_irq_save(flags);
+	hard_local_irq_enable();
+	preempt_schedule_irq(); /* Ok, may reschedule now. */
+	hard_local_irq_disable();
+
+	/*
+	 * Flush any pending interrupt that may have been logged after
+	 * preempt_schedule_irq() stalled the root stage before
+	 * returning to us, and now.
+	 */
+	p = ipipe_this_cpu_root_context();
+	if (unlikely(__ipipe_ipending_p(p))) {
+		__preempt_count_add(PREEMPT_ACTIVE);
+		trace_hardirqs_on();
+		__clear_bit(IPIPE_STALL_FLAG, &p->status);
+		__ipipe_sync_stage();
+		__preempt_count_sub(PREEMPT_ACTIVE);
+	}
+
+	__ipipe_restore_root_nosync(flags);
+}
+
+#else /* !CONFIG_PREEMPT */
+
+#define __ipipe_preempt_schedule_irq()	do { } while (0)
+
+#endif	/* !CONFIG_PREEMPT */
+
+#ifdef CONFIG_TRACE_IRQFLAGS
+#define root_stall_after_handler()	local_irq_disable()
+#else
+#define root_stall_after_handler()	do { } while (0)
+#endif
+
+/*
+ * __ipipe_do_sync_stage() -- Flush the pending IRQs for the current
+ * domain (and processor). This routine flushes the interrupt log (see
+ * "Optimistic interrupt protection" from D. Stodolsky et al. for more
+ * on the deferred interrupt scheme). Every interrupt that occurred
+ * while the pipeline was stalled gets played.
+ *
+ * WARNING: CPU migration may occur over this routine.
+ */
+void __ipipe_do_sync_stage(void)
+{
+	struct ipipe_percpu_domain_data *p;
+	struct ipipe_domain *ipd;
+	int irq;
+
+	p = __ipipe_current_context;
+respin:
+	ipd = p->domain;
+
+	__set_bit(IPIPE_STALL_FLAG, &p->status);
+	smp_wmb();
+
+	if (ipd == ipipe_root_domain)
+		trace_hardirqs_off();
+
+	for (;;) {
+		irq = __ipipe_next_irq(p);
+		if (irq < 0)
+			break;
+		/*
+		 * Make sure the compiler does not reorder wrongly, so
+		 * that all updates to maps are done before the
+		 * handler gets called.
+		 */
+		barrier();
+
+		if (test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
+			continue;
+
+		if (ipd != ipipe_head_domain)
+			hard_local_irq_enable();
+
+		if (likely(ipd != ipipe_root_domain)) {
+			ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie);
+			__ipipe_run_irqtail(irq);
+			hard_local_irq_disable();
+		} else if (ipipe_virtual_irq_p(irq)) {
+			irq_enter();
+			ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie);
+			irq_exit();
+			root_stall_after_handler();
+			hard_local_irq_disable();
+			while (__ipipe_check_root_resched())
+				__ipipe_preempt_schedule_irq();
+		} else {
+			ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie);
+			root_stall_after_handler();
+			hard_local_irq_disable();
+		}
+
+		/*
+		 * We may have migrated to a different CPU (1) upon
+		 * return from the handler, or downgraded from the
+		 * head domain to the root one (2), the opposite way
+		 * is NOT allowed though.
+		 *
+		 * (1) reload the current per-cpu context pointer, so
+		 * that we further pull pending interrupts from the
+		 * proper per-cpu log.
+		 *
+		 * (2) check the stall bit to know whether we may
+		 * dispatch any interrupt pending for the root domain,
+		 * and respin the entire dispatch loop if
+		 * so. Otherwise, immediately return to the caller,
+		 * _without_ affecting the stall state for the root
+		 * domain, since we do not own it at this stage.  This
+		 * case is basically reflecting what may happen in
+		 * dispatch_irq_head() for the fast path.
+		 */
+		p = __ipipe_current_context;
+		if (p->domain != ipd) {
+			IPIPE_BUG_ON(ipd == ipipe_root_domain);
+			if (test_bit(IPIPE_STALL_FLAG, &p->status))
+				return;
+			goto respin;
+		}
+	}
+
+	if (ipd == ipipe_root_domain)
+		trace_hardirqs_on();
+
+	__clear_bit(IPIPE_STALL_FLAG, &p->status);
+}
+
+void __ipipe_call_mayday(struct pt_regs *regs)
+{
+	unsigned long flags;
+
+	ipipe_clear_thread_flag(TIP_MAYDAY);
+	flags = hard_local_irq_save();
+	__ipipe_notify_trap(IPIPE_TRAP_MAYDAY, regs);
+	hard_local_irq_restore(flags);
+}
+
+#ifdef CONFIG_SMP
+
+/* Always called with hw interrupts off. */
+void __ipipe_do_critical_sync(unsigned int irq, void *cookie)
+{
+	int cpu = ipipe_processor_id();
+
+	cpu_set(cpu, __ipipe_cpu_sync_map);
+
+	/*
+	 * Now we are in sync with the lock requestor running on
+	 * another CPU. Enter a spinning wait until he releases the
+	 * global lock.
+	 */
+	spin_lock(&__ipipe_cpu_barrier);
+
+	/* Got it. Now get out. */
+
+	/* Call the sync routine if any. */
+	if (__ipipe_cpu_sync)
+		__ipipe_cpu_sync();
+
+	cpu_set(cpu, __ipipe_cpu_pass_map);
+
+	spin_unlock(&__ipipe_cpu_barrier);
+
+	cpu_clear(cpu, __ipipe_cpu_sync_map);
+}
+#endif	/* CONFIG_SMP */
+
+unsigned long ipipe_critical_enter(void (*syncfn)(void))
+{
+	cpumask_t allbutself __maybe_unused, online __maybe_unused;
+	int cpu __maybe_unused, n __maybe_unused;
+	unsigned long flags, loops __maybe_unused;
+
+	flags = hard_local_irq_save();
+
+	if (num_online_cpus() == 1)
+		return flags;
+
+#ifdef CONFIG_SMP
+
+	cpu = ipipe_processor_id();
+	if (!cpu_test_and_set(cpu, __ipipe_cpu_lock_map)) {
+		while (test_and_set_bit(0, &__ipipe_critical_lock)) {
+			n = 0;
+			hard_local_irq_enable();
+
+			do
+				cpu_relax();
+			while (++n < cpu);
+
+			hard_local_irq_disable();
+		}
+restart:
+		online = *cpu_online_mask;
+		spin_lock(&__ipipe_cpu_barrier);
+
+		__ipipe_cpu_sync = syncfn;
+
+		cpus_clear(__ipipe_cpu_pass_map);
+		cpu_set(cpu, __ipipe_cpu_pass_map);
+
+		/*
+		 * Send the sync IPI to all processors but the current
+		 * one.
+		 */
+		cpus_andnot(allbutself, online, __ipipe_cpu_pass_map);
+		ipipe_send_ipi(IPIPE_CRITICAL_IPI, allbutself);
+		loops = IPIPE_CRITICAL_TIMEOUT;
+
+		while (!cpus_equal(__ipipe_cpu_sync_map, allbutself)) {
+			if (--loops > 0) {
+				cpu_relax();
+				continue;
+			}
+			/*
+			 * We ran into a deadlock due to a contended
+			 * rwlock. Cancel this round and retry.
+			 */
+			__ipipe_cpu_sync = NULL;
+
+			spin_unlock(&__ipipe_cpu_barrier);
+			/*
+			 * Ensure all CPUs consumed the IPI to avoid
+			 * running __ipipe_cpu_sync prematurely. This
+			 * usually resolves the deadlock reason too.
+			 */
+			while (!cpus_equal(online, __ipipe_cpu_pass_map))
+				cpu_relax();
+
+			goto restart;
+		}
+	}
+
+	atomic_inc(&__ipipe_critical_count);
+
+#endif	/* CONFIG_SMP */
+
+	return flags;
+}
+EXPORT_SYMBOL_GPL(ipipe_critical_enter);
+
+void ipipe_critical_exit(unsigned long flags)
+{
+	if (num_online_cpus() == 1) {
+		hard_local_irq_restore(flags);
+		return;
+	}
+
+#ifdef CONFIG_SMP
+	if (atomic_dec_and_test(&__ipipe_critical_count)) {
+		spin_unlock(&__ipipe_cpu_barrier);
+		while (!cpus_empty(__ipipe_cpu_sync_map))
+			cpu_relax();
+		cpu_clear(ipipe_processor_id(), __ipipe_cpu_lock_map);
+		clear_bit(0, &__ipipe_critical_lock);
+		smp_mb__after_atomic();
+	}
+#endif /* CONFIG_SMP */
+
+	hard_local_irq_restore(flags);
+}
+EXPORT_SYMBOL_GPL(ipipe_critical_exit);
+
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+
+void ipipe_root_only(void)
+{
+	struct ipipe_domain *this_domain;
+	unsigned long flags;
+
+	flags = hard_smp_local_irq_save();
+
+	this_domain = __ipipe_current_domain;
+	if (likely(this_domain == ipipe_root_domain &&
+		   !test_bit(IPIPE_STALL_FLAG, &__ipipe_head_status))) {
+		hard_smp_local_irq_restore(flags);
+		return;
+	}
+
+	if (!__this_cpu_read(ipipe_percpu.context_check)) {
+		hard_smp_local_irq_restore(flags);
+		return;
+	}
+
+	hard_smp_local_irq_restore(flags);
+
+	ipipe_prepare_panic();
+	ipipe_trace_panic_freeze();
+
+	if (this_domain != ipipe_root_domain)
+		pr_err("I-pipe: Detected illicit call from head domain '%s'\n"
+		       "        into a regular Linux service\n",
+		       this_domain->name);
+	else
+		pr_err("I-pipe: Detected stalled head domain, "
+			"probably caused by a bug.\n"
+			"        A critical section may have been "
+			"left unterminated.\n");
+	dump_stack();
+	ipipe_trace_panic_dump();
+}
+EXPORT_SYMBOL(ipipe_root_only);
+
+#endif /* CONFIG_IPIPE_DEBUG_CONTEXT */
+
+#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP)
+
+int notrace __ipipe_check_percpu_access(void)
+{
+	struct ipipe_percpu_domain_data *p;
+	struct ipipe_domain *this_domain;
+	unsigned long flags;
+	int ret = 0;
+
+	flags = hard_local_irq_save_notrace();
+
+	/*
+	 * Don't use __ipipe_current_domain here, this would recurse
+	 * indefinitely.
+	 */
+	this_domain = raw_cpu_read(ipipe_percpu.curr)->domain;
+
+	/*
+	 * Only the root domain may implement preemptive CPU migration
+	 * of tasks, so anything above in the pipeline should be fine.
+	 */
+	if (this_domain != ipipe_root_domain)
+		goto out;
+
+	if (raw_irqs_disabled_flags(flags))
+		goto out;
+
+	/*
+	 * Last chance: hw interrupts were enabled on entry while
+	 * running over the root domain, but the root stage might be
+	 * currently stalled, in which case preemption would be
+	 * disabled, and no migration could occur.
+	 */
+
+	p = raw_cpu_ptr(&ipipe_percpu.root);
+	if (!preemptible())
+		goto out;
+	/*
+	 * Our caller may end up accessing the wrong per-cpu variable
+	 * instance due to CPU migration; tell it to complain about
+	 * this.
+	 */
+	ret = 1;
+out:
+	hard_local_irq_restore_notrace(flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(__ipipe_check_percpu_access);
+
+void __ipipe_spin_unlock_debug(unsigned long flags)
+{
+	/*
+	 * We catch a nasty issue where spin_unlock_irqrestore() on a
+	 * regular kernel spinlock is about to re-enable hw interrupts
+	 * in a section entered with hw irqs off. This is clearly the
+	 * sign of a massive breakage coming. Usual suspect is a
+	 * regular spinlock which was overlooked, used within a
+	 * section which must run with hw irqs disabled.
+	 */
+	IPIPE_WARN_ONCE(!raw_irqs_disabled_flags(flags) && hard_irqs_disabled());
+}
+EXPORT_SYMBOL(__ipipe_spin_unlock_debug);
+
+#endif /* CONFIG_IPIPE_DEBUG_INTERNAL && CONFIG_SMP */
+
+void ipipe_prepare_panic(void)
+{
+#ifdef CONFIG_PRINTK
+	__ipipe_printk_bypass = 1;
+#endif
+	ipipe_context_check_off();
+}
+EXPORT_SYMBOL_GPL(ipipe_prepare_panic);
+
+static void __ipipe_do_work(unsigned int virq, void *cookie)
+{
+	struct ipipe_work_header *work;
+	unsigned long flags;
+	void *curr, *tail;
+	int cpu;
+
+	/*
+	 * Work is dispatched in enqueuing order. This interrupt
+	 * context can't migrate to another CPU.
+	 */
+	cpu = smp_processor_id();
+	curr = per_cpu(work_buf, cpu);
+
+	for (;;) {
+		flags = hard_local_irq_save();
+		tail = per_cpu(work_tail, cpu);
+		if (curr == tail) {
+			per_cpu(work_tail, cpu) = per_cpu(work_buf, cpu);
+			hard_local_irq_restore(flags);
+			return;
+		}
+		work = curr;
+		curr += work->size;
+		hard_local_irq_restore(flags);
+		work->handler(work);
+	}
+}
+
+void __ipipe_post_work_root(struct ipipe_work_header *work)
+{
+	unsigned long flags;
+	void *tail;
+	int cpu;
+
+	/*
+	 * Subtle: we want to use the head stall/unstall operators,
+	 * not the hard_* routines to protect against races. This way,
+	 * we ensure that a root-based caller will trigger the virq
+	 * handling immediately when unstalling the head stage, as a
+	 * result of calling __ipipe_sync_pipeline() under the hood.
+	 */
+	flags = ipipe_test_and_stall_head();
+	cpu = ipipe_processor_id();
+	tail = per_cpu(work_tail, cpu);
+
+	if (WARN_ON_ONCE((unsigned char *)tail + work->size >=
+			 per_cpu(work_buf, cpu) + WORKBUF_SIZE))
+		goto out;
+
+	/* Work handling is deferred, so data has to be copied. */
+	memcpy(tail, work, work->size);
+	per_cpu(work_tail, cpu) = tail + work->size;
+	ipipe_post_irq_root(__ipipe_work_virq);
+out:
+	ipipe_restore_head(flags);
+}
+EXPORT_SYMBOL_GPL(__ipipe_post_work_root);
+
+void __weak __ipipe_arch_share_current(int flags)
+{
+}
+
+void __ipipe_share_current(int flags)
+{
+	ipipe_root_only();
+
+	__ipipe_arch_share_current(flags);
+}
+EXPORT_SYMBOL_GPL(__ipipe_share_current);
+
+#ifdef CONFIG_KGDB
+bool __ipipe_probe_access;
+
+long ipipe_probe_kernel_read(void *dst, void *src, size_t size)
+{
+	long ret;
+	mm_segment_t old_fs = get_fs();
+
+	set_fs(KERNEL_DS);
+	__ipipe_probe_access = true;
+	barrier();
+	ret = __copy_from_user_inatomic(dst,
+			(__force const void __user *)src, size);
+	barrier();
+	__ipipe_probe_access = false;
+	set_fs(old_fs);
+
+	return ret ? -EFAULT : 0;
+}
+
+long ipipe_probe_kernel_write(void *dst, void *src, size_t size)
+{
+	long ret;
+	mm_segment_t old_fs = get_fs();
+
+	set_fs(KERNEL_DS);
+	__ipipe_probe_access = true;
+	barrier();
+	ret = __copy_to_user_inatomic((__force void __user *)dst, src, size);
+	barrier();
+	__ipipe_probe_access = false;
+	set_fs(old_fs);
+
+	return ret ? -EFAULT : 0;
+}
+#endif /* CONFIG_KGDB */
+
+#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || defined(CONFIG_PROVE_LOCKING) || \
+	defined(CONFIG_PREEMPT_VOLUNTARY) || defined(CONFIG_IPIPE_DEBUG_CONTEXT)
+void __ipipe_uaccess_might_fault(void)
+{
+	struct ipipe_percpu_domain_data *pdd;
+	struct ipipe_domain *ipd;
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	ipd = __ipipe_current_domain;
+	if (ipd == ipipe_root_domain) {
+		hard_local_irq_restore(flags);
+		might_fault();
+		return;
+	}
+
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+	pdd = ipipe_this_cpu_context(ipd);
+	WARN_ON_ONCE(hard_irqs_disabled_flags(flags) 
+		     || test_bit(IPIPE_STALL_FLAG, &pdd->status));
+#else /* !CONFIG_IPIPE_DEBUG_CONTEXT */
+	(void)pdd;
+#endif /* !CONFIG_IPIPE_DEBUG_CONTEXT */
+	hard_local_irq_restore(flags);
+	
+}
+EXPORT_SYMBOL_GPL(__ipipe_uaccess_might_fault);
+#endif
diff --git a/kernel/ipipe/timer.c b/kernel/ipipe/timer.c
new file mode 100644
index 0000000..143f9e6
--- /dev/null
+++ b/kernel/ipipe/timer.c
@@ -0,0 +1,497 @@
+/* -*- linux-c -*-
+ * linux/kernel/ipipe/timer.c
+ *
+ * Copyright (C) 2012 Gilles Chanteperdrix
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * I-pipe timer request interface.
+ */
+#include <linux/ipipe.h>
+#include <linux/percpu.h>
+#include <linux/irqdesc.h>
+#include <linux/cpumask.h>
+#include <linux/spinlock.h>
+#include <linux/ipipe_tickdev.h>
+#include <linux/interrupt.h>
+#include <linux/export.h>
+
+unsigned long __ipipe_hrtimer_freq;
+
+static LIST_HEAD(timers);
+static IPIPE_DEFINE_SPINLOCK(lock);
+
+static DEFINE_PER_CPU(struct ipipe_timer *, percpu_timer);
+
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+/*
+ * Default request method: switch to oneshot mode if supported.
+ */
+static void ipipe_timer_default_request(struct ipipe_timer *timer, int steal)
+{
+	struct clock_event_device *evtdev = timer->host_timer;
+
+	if (!(evtdev->features & CLOCK_EVT_FEAT_ONESHOT))
+		return;
+
+	if (evtdev->mode != CLOCK_EVT_MODE_ONESHOT) {
+		evtdev->set_mode(CLOCK_EVT_MODE_ONESHOT, evtdev);
+		evtdev->set_next_event(timer->freq / HZ, evtdev);
+	}
+}
+
+/*
+ * Default release method: return the timer to the mode it had when
+ * starting.
+ */
+static void ipipe_timer_default_release(struct ipipe_timer *timer)
+{
+	struct clock_event_device *evtdev = timer->host_timer;
+
+	evtdev->set_mode(evtdev->mode, evtdev);
+	if (evtdev->mode == CLOCK_EVT_MODE_ONESHOT)
+		evtdev->set_next_event(timer->freq / HZ, evtdev);
+}
+
+void ipipe_host_timer_register(struct clock_event_device *evtdev)
+{
+	struct ipipe_timer *timer = evtdev->ipipe_timer;
+
+	if (timer == NULL)
+		return;
+
+	if (timer->request == NULL)
+		timer->request = ipipe_timer_default_request;
+
+	/*
+	 * By default, use the same method as linux timer, on ARM at
+	 * least, most set_next_event methods are safe to be called
+	 * from Xenomai domain anyway.
+	 */
+	if (timer->set == NULL) {
+		timer->timer_set = evtdev;
+		timer->set = (typeof(timer->set))evtdev->set_next_event;
+	}
+
+	if (timer->release == NULL)
+		timer->release = ipipe_timer_default_release;
+
+	if (timer->name == NULL)
+		timer->name = evtdev->name;
+
+	if (timer->rating == 0)
+		timer->rating = evtdev->rating;
+
+	timer->freq = (1000000000ULL * evtdev->mult) >> evtdev->shift;
+
+	if (timer->min_delay_ticks == 0)
+		timer->min_delay_ticks =
+			(evtdev->min_delta_ns * evtdev->mult) >> evtdev->shift;
+
+	if (timer->cpumask == NULL)
+		timer->cpumask = evtdev->cpumask;
+
+	timer->host_timer = evtdev;
+
+	ipipe_timer_register(timer);
+}
+#endif /* CONFIG_GENERIC_CLOCKEVENTS */
+
+/*
+ * register a timer: maintain them in a list sorted by rating
+ */
+void ipipe_timer_register(struct ipipe_timer *timer)
+{
+	struct ipipe_timer *t;
+	unsigned long flags;
+
+	if (timer->timer_set == NULL)
+		timer->timer_set = timer;
+
+	if (timer->cpumask == NULL)
+		timer->cpumask = cpumask_of(smp_processor_id());
+
+	spin_lock_irqsave(&lock, flags);
+
+	list_for_each_entry(t, &timers, link) {
+		if (t->rating <= timer->rating) {
+			__list_add(&timer->link, t->link.prev, &t->link);
+			goto done;
+		}
+	}
+	list_add_tail(&timer->link, &timers);
+  done:
+	spin_unlock_irqrestore(&lock, flags);
+}
+
+static void ipipe_timer_request_sync(void)
+{
+	struct ipipe_timer *timer = __ipipe_raw_cpu_read(percpu_timer);
+	struct clock_event_device *evtdev;
+	int steal;
+
+	if (!timer)
+		return;
+
+	evtdev = timer->host_timer;
+
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+	steal = evtdev != NULL && evtdev->mode != CLOCK_EVT_MODE_UNUSED;
+#else /* !CONFIG_GENERIC_CLOCKEVENTS */
+	steal = 1;
+#endif /* !CONFIG_GENERIC_CLOCKEVENTS */
+
+	timer->request(timer, steal);
+}
+
+/* Set up a timer as per-cpu timer for ipipe */
+static void install_pcpu_timer(unsigned cpu, unsigned hrclock_freq,
+			      struct ipipe_timer *t) {
+	unsigned hrtimer_freq;
+	unsigned long long tmp;
+
+	if (__ipipe_hrtimer_freq == 0)
+		__ipipe_hrtimer_freq = t->freq;
+
+	per_cpu(ipipe_percpu.hrtimer_irq, cpu) = t->irq;
+	per_cpu(percpu_timer, cpu) = t;
+
+	hrtimer_freq = t->freq;
+	if (__ipipe_hrclock_freq > UINT_MAX)
+		hrtimer_freq /= 1000;
+
+	t->c2t_integ = hrtimer_freq / hrclock_freq;
+	tmp = (((unsigned long long)
+		(hrtimer_freq % hrclock_freq)) << 32)
+		+ hrclock_freq - 1;
+	do_div(tmp, hrclock_freq);
+	t->c2t_frac = tmp;
+}
+
+static void select_root_only_timer(unsigned cpu, unsigned hrclock_khz,
+				   const struct cpumask *mask,
+				   struct ipipe_timer *t) {
+	unsigned icpu;
+	struct clock_event_device *evtdev;
+
+	/*
+	 * If no ipipe-supported CPU shares an interrupt with the
+	 * timer, we do not need to care about it.
+	 */
+	for_each_cpu(icpu, mask) {
+		if (t->irq == per_cpu(ipipe_percpu.hrtimer_irq, icpu)) {
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+			evtdev = t->host_timer;
+			if (evtdev && evtdev->mode == CLOCK_EVT_MODE_SHUTDOWN)
+				continue;
+#endif /* CONFIG_GENERIC_CLOCKEVENTS */
+			goto found;
+		}
+	}
+
+	return;
+
+found:
+	install_pcpu_timer(cpu, hrclock_khz, t);
+}
+
+/*
+ * Choose per-cpu timers with the highest rating by traversing the
+ * rating-sorted list for each CPU.
+ */
+int ipipe_select_timers(const struct cpumask *mask)
+{
+	unsigned hrclock_freq;
+	unsigned long long tmp;
+	struct ipipe_timer *t;
+	struct clock_event_device *evtdev;
+	unsigned long flags;
+	unsigned cpu;
+	cpumask_t fixup;
+
+	if (!__ipipe_hrclock_ok()) {
+		printk("I-pipe: high-resolution clock not working\n");
+		return -ENODEV;
+	}
+
+	if (__ipipe_hrclock_freq > UINT_MAX) {
+		tmp = __ipipe_hrclock_freq;
+		do_div(tmp, 1000);
+		hrclock_freq = tmp;
+	} else
+		hrclock_freq = __ipipe_hrclock_freq;
+
+	spin_lock_irqsave(&lock, flags);
+
+	/* First, choose timers for the CPUs handled by ipipe */
+	for_each_cpu(cpu, mask) {
+		list_for_each_entry(t, &timers, link) {
+			if (!cpumask_test_cpu(cpu, t->cpumask))
+				continue;
+
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+			evtdev = t->host_timer;
+			if (evtdev && evtdev->mode == CLOCK_EVT_MODE_SHUTDOWN)
+				continue;
+#endif /* CONFIG_GENERIC_CLOCKEVENTS */
+			goto found;
+		}
+
+		printk("I-pipe: could not find timer for cpu #%d\n",
+		       cpu);
+		goto err_remove_all;
+found:
+		install_pcpu_timer(cpu, hrclock_freq, t);
+	}
+
+	/*
+	 * Second, check if we need to fix up any CPUs not supported
+	 * by ipipe (but by Linux) whose interrupt may need to be
+	 * forwarded because they have the same IRQ as an ipipe-enabled
+	 * timer.
+	 */
+	cpumask_andnot(&fixup, cpu_online_mask, mask);
+
+	for_each_cpu(cpu, &fixup) {
+		list_for_each_entry(t, &timers, link) {
+			if (!cpumask_test_cpu(cpu, t->cpumask))
+				continue;
+
+			select_root_only_timer(cpu, hrclock_freq, mask, t);
+		}
+	}
+
+	spin_unlock_irqrestore(&lock, flags);
+
+	flags = ipipe_critical_enter(ipipe_timer_request_sync);
+	ipipe_timer_request_sync();
+	ipipe_critical_exit(flags);
+
+	return 0;
+
+err_remove_all:
+	spin_unlock_irqrestore(&lock, flags);
+
+	for_each_cpu(cpu, mask) {
+		per_cpu(ipipe_percpu.hrtimer_irq, cpu) = -1;
+		per_cpu(percpu_timer, cpu) = NULL;
+	}
+	__ipipe_hrtimer_freq = 0;
+
+	return -ENODEV;
+}
+
+static void ipipe_timer_release_sync(void)
+{
+	struct ipipe_timer *timer = __ipipe_raw_cpu_read(percpu_timer);
+
+	if (timer)
+		timer->release(timer);
+}
+
+void ipipe_timers_release(void)
+{
+	unsigned long flags;
+	unsigned cpu;
+
+	flags = ipipe_critical_enter(ipipe_timer_release_sync);
+	ipipe_timer_release_sync();
+	ipipe_critical_exit(flags);
+
+	for_each_online_cpu(cpu) {
+		per_cpu(ipipe_percpu.hrtimer_irq, cpu) = -1;
+		per_cpu(percpu_timer, cpu) = NULL;
+		__ipipe_hrtimer_freq = 0;
+	}
+}
+
+static void __ipipe_ack_hrtimer_irq(unsigned int irq, struct irq_desc *desc)
+{
+	struct ipipe_timer *timer = __ipipe_raw_cpu_read(percpu_timer);
+
+	if (desc)
+		desc->ipipe_ack(irq, desc);
+	if (timer->ack)
+		timer->ack();
+	if (desc)
+		desc->ipipe_end(irq, desc);
+}
+
+int ipipe_timer_start(void (*tick_handler)(void),
+		      void (*emumode)(enum clock_event_mode mode,
+				      struct clock_event_device *cdev),
+		      int (*emutick)(unsigned long evt,
+				     struct clock_event_device *cdev),
+		      unsigned cpu)
+{
+	struct clock_event_device *evtdev;
+	struct ipipe_timer *timer;
+	struct irq_desc *desc;
+	unsigned long flags;
+	int steal, ret;
+
+	timer = per_cpu(percpu_timer, cpu);
+	evtdev = timer->host_timer;
+
+	flags = ipipe_critical_enter(NULL);
+
+	ret = ipipe_request_irq(ipipe_head_domain, timer->irq,
+				(ipipe_irq_handler_t)tick_handler,
+				NULL, __ipipe_ack_hrtimer_irq);
+	if (ret < 0 && ret != -EBUSY) {
+		ipipe_critical_exit(flags);
+		return ret;
+	}
+
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+	steal = evtdev != NULL && evtdev->mode != CLOCK_EVT_MODE_UNUSED;
+	if (steal && evtdev->ipipe_stolen == 0) {
+		timer->real_mult = evtdev->mult;
+		timer->real_shift = evtdev->shift;
+		timer->real_set_mode = evtdev->set_mode;
+		timer->real_set_next_event = evtdev->set_next_event;
+		evtdev->mult = 1;
+		evtdev->shift = 0;
+		evtdev->max_delta_ns = UINT_MAX;
+		evtdev->set_mode = emumode;
+		evtdev->set_next_event = emutick;
+		evtdev->ipipe_stolen = 1;
+	}
+
+	ret = evtdev ? evtdev->mode : CLOCK_EVT_MODE_UNUSED;
+#else /* CONFIG_GENERIC_CLOCKEVENTS */
+	steal = 1;
+	ret = 0;
+#endif /* CONFIG_GENERIC_CLOCKEVENTS */
+
+	ipipe_critical_exit(flags);
+
+	desc = irq_to_desc(timer->irq);
+	if (desc && irqd_irq_disabled(&desc->irq_data))
+		ipipe_enable_irq(timer->irq);
+
+	return ret;
+}
+
+void ipipe_timer_stop(unsigned cpu)
+{
+	unsigned long __maybe_unused flags;
+	struct clock_event_device *evtdev;
+	struct ipipe_timer *timer;
+	struct irq_desc *desc;
+
+	timer = per_cpu(percpu_timer, cpu);
+	evtdev = timer->host_timer;
+
+	desc = irq_to_desc(timer->irq);
+	if (desc && irqd_irq_disabled(&desc->irq_data))
+		ipipe_disable_irq(timer->irq);
+
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+	if (evtdev) {
+		flags = ipipe_critical_enter(NULL);
+
+		if (evtdev->ipipe_stolen) {
+			evtdev->mult = timer->real_mult;
+			evtdev->shift = timer->real_shift;
+			evtdev->set_mode = timer->real_set_mode;
+			evtdev->set_next_event = timer->real_set_next_event;
+			timer->real_mult = timer->real_shift = 0;
+			timer->real_set_mode = NULL;
+			timer->real_set_next_event = NULL;
+			evtdev->ipipe_stolen = 0;
+		}
+
+		ipipe_critical_exit(flags);
+	}
+#endif /* CONFIG_GENERIC_CLOCKEVENTS */
+
+	ipipe_free_irq(ipipe_head_domain, timer->irq);
+}
+
+void ipipe_timer_set(unsigned long cdelay)
+{
+	unsigned long tdelay;
+	struct ipipe_timer *t;
+
+	t = __ipipe_raw_cpu_read(percpu_timer);
+
+	/*
+	 * Even though some architectures may use a 64 bits delay
+	 * here, we voluntarily limit to 32 bits, 4 billions ticks
+	 * should be enough for now. Would a timer needs more, an
+	 * extra call to the tick handler would simply occur after 4
+	 * billions ticks.
+	 */
+	if (cdelay > UINT_MAX)
+		cdelay = UINT_MAX;
+
+	tdelay = cdelay;
+	if (t->c2t_integ != 1)
+		tdelay *= t->c2t_integ;
+	if (t->c2t_frac)
+		tdelay += ((unsigned long long)cdelay * t->c2t_frac) >> 32;
+
+	if (tdelay < t->min_delay_ticks
+	    || t->set(tdelay, t->timer_set) < 0)
+		ipipe_raise_irq(t->irq);
+}
+EXPORT_SYMBOL_GPL(ipipe_timer_set);
+
+const char *ipipe_timer_name(void)
+{
+	return per_cpu(percpu_timer, 0)->name;
+}
+EXPORT_SYMBOL_GPL(ipipe_timer_name);
+
+unsigned ipipe_timer_ns2ticks(struct ipipe_timer *timer, unsigned ns)
+{
+	unsigned long long tmp;
+	BUG_ON(!timer->freq);
+	tmp = (unsigned long long)ns * timer->freq;
+	do_div(tmp, 1000000000);
+	return tmp;
+}
+
+#ifdef CONFIG_IPIPE_HAVE_HOSTRT
+/*
+ * NOTE: The architecture specific code must only call this function
+ * when a clocksource suitable for CLOCK_HOST_REALTIME is enabled.
+ * The event receiver is responsible for providing proper locking.
+ */
+void ipipe_update_hostrt(struct timekeeper *tk)
+{
+	struct tk_read_base *tkr = &tk->tkr;
+	struct clocksource *clock = tkr->clock;
+	struct ipipe_hostrt_data data;
+	struct timespec xt;
+
+	xt.tv_sec = tk->xtime_sec;
+	xt.tv_nsec = (long)(tkr->xtime_nsec >> tkr->shift);
+	ipipe_root_only();
+	data.live = 1;
+	data.cycle_last = tkr->cycle_last;
+	data.mask = clock->mask;
+	data.mult = tkr->mult;
+	data.shift = tkr->shift;
+	data.wall_time_sec = xt.tv_sec;
+	data.wall_time_nsec = xt.tv_nsec;
+	data.wall_to_monotonic.tv_sec = tk->wall_to_monotonic.tv_sec;
+	data.wall_to_monotonic.tv_nsec = tk->wall_to_monotonic.tv_nsec;
+	__ipipe_notify_kevent(IPIPE_KEVT_HOSTRT, &data);
+}
+
+#endif /* CONFIG_IPIPE_HAVE_HOSTRT */
diff --git a/kernel/ipipe/tracer.c b/kernel/ipipe/tracer.c
new file mode 100644
index 0000000..8388671
--- /dev/null
+++ b/kernel/ipipe/tracer.c
@@ -0,0 +1,1468 @@
+/* -*- linux-c -*-
+ * kernel/ipipe/tracer.c
+ *
+ * Copyright (C) 2005 Luotao Fu.
+ *		 2005-2008 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/kallsyms.h>
+#include <linux/seq_file.h>
+#include <linux/proc_fs.h>
+#include <linux/ctype.h>
+#include <linux/vmalloc.h>
+#include <linux/pid.h>
+#include <linux/vermagic.h>
+#include <linux/sched.h>
+#include <linux/ipipe.h>
+#include <linux/ftrace.h>
+#include <asm/uaccess.h>
+
+#define IPIPE_TRACE_PATHS	    4 /* <!> Do not lower below 3 */
+#define IPIPE_DEFAULT_ACTIVE	    0
+#define IPIPE_DEFAULT_MAX	    1
+#define IPIPE_DEFAULT_FROZEN	    2
+
+#define IPIPE_TRACE_POINTS	    (1 << CONFIG_IPIPE_TRACE_SHIFT)
+#define WRAP_POINT_NO(point)	    ((point) & (IPIPE_TRACE_POINTS-1))
+
+#define IPIPE_DEFAULT_PRE_TRACE	    10
+#define IPIPE_DEFAULT_POST_TRACE    10
+#define IPIPE_DEFAULT_BACK_TRACE    100
+
+#define IPIPE_DELAY_NOTE	    1000  /* in nanoseconds */
+#define IPIPE_DELAY_WARN	    10000 /* in nanoseconds */
+
+#define IPIPE_TFLG_NMI_LOCK	    0x0001
+#define IPIPE_TFLG_NMI_HIT	    0x0002
+#define IPIPE_TFLG_NMI_FREEZE_REQ   0x0004
+
+#define IPIPE_TFLG_HWIRQ_OFF	    0x0100
+#define IPIPE_TFLG_FREEZING	    0x0200
+#define IPIPE_TFLG_CURRDOM_SHIFT    10	 /* bits 10..11: current domain */
+#define IPIPE_TFLG_CURRDOM_MASK	    0x0C00
+#define IPIPE_TFLG_DOMSTATE_SHIFT   12	 /* bits 12..15: domain stalled? */
+#define IPIPE_TFLG_DOMSTATE_BITS    3
+
+#define IPIPE_TFLG_DOMAIN_STALLED(point, n) \
+	(point->flags & (1 << (n + IPIPE_TFLG_DOMSTATE_SHIFT)))
+#define IPIPE_TFLG_CURRENT_DOMAIN(point) \
+	((point->flags & IPIPE_TFLG_CURRDOM_MASK) >> IPIPE_TFLG_CURRDOM_SHIFT)
+
+struct ipipe_trace_point {
+	short type;
+	short flags;
+	unsigned long eip;
+	unsigned long parent_eip;
+	unsigned long v;
+	unsigned long long timestamp;
+};
+
+struct ipipe_trace_path {
+	volatile int flags;
+	int dump_lock; /* separated from flags due to cross-cpu access */
+	int trace_pos; /* next point to fill */
+	int begin, end; /* finalised path begin and end */
+	int post_trace; /* non-zero when in post-trace phase */
+	unsigned long long length; /* max path length in cycles */
+	unsigned long nmi_saved_eip; /* for deferred requests from NMIs */
+	unsigned long nmi_saved_parent_eip;
+	unsigned long nmi_saved_v;
+	struct ipipe_trace_point point[IPIPE_TRACE_POINTS];
+} ____cacheline_aligned_in_smp;
+
+enum ipipe_trace_type
+{
+	IPIPE_TRACE_FUNC = 0,
+	IPIPE_TRACE_BEGIN,
+	IPIPE_TRACE_END,
+	IPIPE_TRACE_FREEZE,
+	IPIPE_TRACE_SPECIAL,
+	IPIPE_TRACE_PID,
+	IPIPE_TRACE_EVENT,
+};
+
+#define IPIPE_TYPE_MASK		    0x0007
+#define IPIPE_TYPE_BITS		    3
+
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
+static DEFINE_PER_CPU(struct ipipe_trace_path *, trace_path);
+#else /* !CONFIG_IPIPE_TRACE_VMALLOC */
+static DEFINE_PER_CPU(struct ipipe_trace_path, trace_path[IPIPE_TRACE_PATHS]) =
+	{ [0 ... IPIPE_TRACE_PATHS-1] = { .begin = -1, .end = -1 } };
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
+
+int ipipe_trace_enable = 0;
+
+static DEFINE_PER_CPU(int, active_path) = { IPIPE_DEFAULT_ACTIVE };
+static DEFINE_PER_CPU(int, max_path) = { IPIPE_DEFAULT_MAX };
+static DEFINE_PER_CPU(int, frozen_path) = { IPIPE_DEFAULT_FROZEN };
+static IPIPE_DEFINE_SPINLOCK(global_path_lock);
+static int pre_trace = IPIPE_DEFAULT_PRE_TRACE;
+static int post_trace = IPIPE_DEFAULT_POST_TRACE;
+static int back_trace = IPIPE_DEFAULT_BACK_TRACE;
+static int verbose_trace = 1;
+static unsigned long trace_overhead;
+
+static unsigned long trigger_begin;
+static unsigned long trigger_end;
+
+static DEFINE_MUTEX(out_mutex);
+static struct ipipe_trace_path *print_path;
+#ifdef CONFIG_IPIPE_TRACE_PANIC
+static struct ipipe_trace_path *panic_path;
+#endif /* CONFIG_IPIPE_TRACE_PANIC */
+static int print_pre_trace;
+static int print_post_trace;
+
+
+static long __ipipe_signed_tsc2us(long long tsc);
+static void
+__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point);
+static void __ipipe_print_symname(struct seq_file *m, unsigned long eip);
+
+static inline void store_states(struct ipipe_domain *ipd,
+				struct ipipe_trace_point *point, int pos)
+{
+	if (test_bit(IPIPE_STALL_FLAG, &ipipe_this_cpu_context(ipd)->status))
+		point->flags |= 1 << (pos + IPIPE_TFLG_DOMSTATE_SHIFT);
+
+	if (ipd == __ipipe_current_domain)
+		point->flags |= pos << IPIPE_TFLG_CURRDOM_SHIFT;
+}
+
+static notrace void
+__ipipe_store_domain_states(struct ipipe_trace_point *point)
+{
+	store_states(ipipe_root_domain, point, 0);
+	if (ipipe_head_domain != ipipe_root_domain)
+		store_states(ipipe_head_domain, point, 1);
+}
+
+static notrace int __ipipe_get_free_trace_path(int old, int cpu)
+{
+	int new_active = old;
+	struct ipipe_trace_path *tp;
+
+	do {
+		if (++new_active == IPIPE_TRACE_PATHS)
+			new_active = 0;
+		tp = &per_cpu(trace_path, cpu)[new_active];
+	} while (new_active == per_cpu(max_path, cpu) ||
+		 new_active == per_cpu(frozen_path, cpu) ||
+		 tp->dump_lock);
+
+	return new_active;
+}
+
+static notrace void
+__ipipe_migrate_pre_trace(struct ipipe_trace_path *new_tp,
+			  struct ipipe_trace_path *old_tp, int old_pos)
+{
+	int i;
+
+	new_tp->trace_pos = pre_trace+1;
+
+	for (i = new_tp->trace_pos; i > 0; i--)
+		memcpy(&new_tp->point[WRAP_POINT_NO(new_tp->trace_pos-i)],
+		       &old_tp->point[WRAP_POINT_NO(old_pos-i)],
+		       sizeof(struct ipipe_trace_point));
+
+	/* mark the end (i.e. the point before point[0]) invalid */
+	new_tp->point[IPIPE_TRACE_POINTS-1].eip = 0;
+}
+
+static notrace struct ipipe_trace_path *
+__ipipe_trace_end(int cpu, struct ipipe_trace_path *tp, int pos)
+{
+	struct ipipe_trace_path *old_tp = tp;
+	long active = per_cpu(active_path, cpu);
+	unsigned long long length;
+
+	/* do we have a new worst case? */
+	length = tp->point[tp->end].timestamp -
+		 tp->point[tp->begin].timestamp;
+	if (length > per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)].length) {
+		/* we need protection here against other cpus trying
+		   to start a proc dump */
+		spin_lock(&global_path_lock);
+
+		/* active path holds new worst case */
+		tp->length = length;
+		per_cpu(max_path, cpu) = active;
+
+		/* find next unused trace path */
+		active = __ipipe_get_free_trace_path(active, cpu);
+
+		spin_unlock(&global_path_lock);
+
+		tp = &per_cpu(trace_path, cpu)[active];
+
+		/* migrate last entries for pre-tracing */
+		__ipipe_migrate_pre_trace(tp, old_tp, pos);
+	}
+
+	return tp;
+}
+
+static notrace struct ipipe_trace_path *
+__ipipe_trace_freeze(int cpu, struct ipipe_trace_path *tp, int pos)
+{
+	struct ipipe_trace_path *old_tp = tp;
+	long active = per_cpu(active_path, cpu);
+	int n;
+
+	/* frozen paths have no core (begin=end) */
+	tp->begin = tp->end;
+
+	/* we need protection here against other cpus trying
+	 * to set their frozen path or to start a proc dump */
+	spin_lock(&global_path_lock);
+
+	per_cpu(frozen_path, cpu) = active;
+
+	/* find next unused trace path */
+	active = __ipipe_get_free_trace_path(active, cpu);
+
+	/* check if this is the first frozen path */
+	for_each_possible_cpu(n) {
+		if (n != cpu &&
+		    per_cpu(trace_path, n)[per_cpu(frozen_path, n)].end >= 0)
+			tp->end = -1;
+	}
+
+	spin_unlock(&global_path_lock);
+
+	tp = &per_cpu(trace_path, cpu)[active];
+
+	/* migrate last entries for pre-tracing */
+	__ipipe_migrate_pre_trace(tp, old_tp, pos);
+
+	return tp;
+}
+
+void notrace
+__ipipe_trace(enum ipipe_trace_type type, unsigned long eip,
+	      unsigned long parent_eip, unsigned long v)
+{
+	struct ipipe_trace_path *tp, *old_tp;
+	int pos, next_pos, begin;
+	struct ipipe_trace_point *point;
+	unsigned long flags;
+	int cpu;
+
+	flags = hard_local_irq_save_notrace();
+
+	cpu = ipipe_processor_id();
+ restart:
+	tp = old_tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
+
+	/* here starts a race window with NMIs - catched below */
+
+	/* check for NMI recursion */
+	if (unlikely(tp->flags & IPIPE_TFLG_NMI_LOCK)) {
+		tp->flags |= IPIPE_TFLG_NMI_HIT;
+
+		/* first freeze request from NMI context? */
+		if ((type == IPIPE_TRACE_FREEZE) &&
+		    !(tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)) {
+			/* save arguments and mark deferred freezing */
+			tp->flags |= IPIPE_TFLG_NMI_FREEZE_REQ;
+			tp->nmi_saved_eip = eip;
+			tp->nmi_saved_parent_eip = parent_eip;
+			tp->nmi_saved_v = v;
+		}
+		return; /* no need for restoring flags inside IRQ */
+	}
+
+	/* clear NMI events and set lock (atomically per cpu) */
+	tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT |
+				   IPIPE_TFLG_NMI_FREEZE_REQ))
+			       | IPIPE_TFLG_NMI_LOCK;
+
+	/* check active_path again - some nasty NMI may have switched
+	 * it meanwhile */
+	if (unlikely(tp !=
+		     &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)])) {
+		/* release lock on wrong path and restart */
+		tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+		/* there is no chance that the NMI got deferred
+		 * => no need to check for pending freeze requests */
+		goto restart;
+	}
+
+	/* get the point buffer */
+	pos = tp->trace_pos;
+	point = &tp->point[pos];
+
+	/* store all trace point data */
+	point->type = type;
+	point->flags = hard_irqs_disabled_flags(flags) ? IPIPE_TFLG_HWIRQ_OFF : 0;
+	point->eip = eip;
+	point->parent_eip = parent_eip;
+	point->v = v;
+	ipipe_read_tsc(point->timestamp);
+
+	__ipipe_store_domain_states(point);
+
+	/* forward to next point buffer */
+	next_pos = WRAP_POINT_NO(pos+1);
+	tp->trace_pos = next_pos;
+
+	/* only mark beginning if we haven't started yet */
+	begin = tp->begin;
+	if (unlikely(type == IPIPE_TRACE_BEGIN) && (begin < 0))
+		tp->begin = pos;
+
+	/* end of critical path, start post-trace if not already started */
+	if (unlikely(type == IPIPE_TRACE_END) &&
+	    (begin >= 0) && !tp->post_trace)
+		tp->post_trace = post_trace + 1;
+
+	/* freeze only if the slot is free and we are not already freezing */
+	if ((unlikely(type == IPIPE_TRACE_FREEZE) ||
+	     (unlikely(eip >= trigger_begin && eip <= trigger_end) &&
+	     type == IPIPE_TRACE_FUNC)) &&
+	    per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)].begin < 0 &&
+	    !(tp->flags & IPIPE_TFLG_FREEZING)) {
+		tp->post_trace = post_trace + 1;
+		tp->flags |= IPIPE_TFLG_FREEZING;
+	}
+
+	/* enforce end of trace in case of overflow */
+	if (unlikely(WRAP_POINT_NO(next_pos + 1) == begin)) {
+		tp->end = pos;
+		goto enforce_end;
+	}
+
+	/* stop tracing this path if we are in post-trace and
+	 *  a) that phase is over now or
+	 *  b) a new TRACE_BEGIN came in but we are not freezing this path */
+	if (unlikely((tp->post_trace > 0) && ((--tp->post_trace == 0) ||
+		     ((type == IPIPE_TRACE_BEGIN) &&
+		      !(tp->flags & IPIPE_TFLG_FREEZING))))) {
+		/* store the path's end (i.e. excluding post-trace) */
+		tp->end = WRAP_POINT_NO(pos - post_trace + tp->post_trace);
+
+ enforce_end:
+		if (tp->flags & IPIPE_TFLG_FREEZING)
+			tp = __ipipe_trace_freeze(cpu, tp, pos);
+		else
+			tp = __ipipe_trace_end(cpu, tp, pos);
+
+		/* reset the active path, maybe already start a new one */
+		tp->begin = (type == IPIPE_TRACE_BEGIN) ?
+			WRAP_POINT_NO(tp->trace_pos - 1) : -1;
+		tp->end = -1;
+		tp->post_trace = 0;
+		tp->flags = 0;
+
+		/* update active_path not earlier to avoid races with NMIs */
+		per_cpu(active_path, cpu) = tp - per_cpu(trace_path, cpu);
+	}
+
+	/* we still have old_tp and point,
+	 * let's reset NMI lock and check for catches */
+	old_tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+	if (unlikely(old_tp->flags & IPIPE_TFLG_NMI_HIT)) {
+		/* well, this late tagging may not immediately be visible for
+		 * other cpus already dumping this path - a minor issue */
+		point->flags |= IPIPE_TFLG_NMI_HIT;
+
+		/* handle deferred freezing from NMI context */
+		if (old_tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)
+			__ipipe_trace(IPIPE_TRACE_FREEZE, old_tp->nmi_saved_eip,
+				      old_tp->nmi_saved_parent_eip,
+				      old_tp->nmi_saved_v);
+	}
+
+	hard_local_irq_restore_notrace(flags);
+}
+
+static unsigned long __ipipe_global_path_lock(void)
+{
+	unsigned long flags;
+	int cpu;
+	struct ipipe_trace_path *tp;
+
+	spin_lock_irqsave(&global_path_lock, flags);
+
+	cpu = ipipe_processor_id();
+ restart:
+	tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
+
+	/* here is small race window with NMIs - catched below */
+
+	/* clear NMI events and set lock (atomically per cpu) */
+	tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT |
+				   IPIPE_TFLG_NMI_FREEZE_REQ))
+			       | IPIPE_TFLG_NMI_LOCK;
+
+	/* check active_path again - some nasty NMI may have switched
+	 * it meanwhile */
+	if (tp != &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)]) {
+		/* release lock on wrong path and restart */
+		tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+		/* there is no chance that the NMI got deferred
+		 * => no need to check for pending freeze requests */
+		goto restart;
+	}
+
+	return flags;
+}
+
+static void __ipipe_global_path_unlock(unsigned long flags)
+{
+	int cpu;
+	struct ipipe_trace_path *tp;
+
+	/* release spinlock first - it's not involved in the NMI issue */
+	__ipipe_spin_unlock_irqbegin(&global_path_lock);
+
+	cpu = ipipe_processor_id();
+	tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
+
+	tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+	/* handle deferred freezing from NMI context */
+	if (tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)
+		__ipipe_trace(IPIPE_TRACE_FREEZE, tp->nmi_saved_eip,
+			      tp->nmi_saved_parent_eip, tp->nmi_saved_v);
+
+	/* See __ipipe_spin_lock_irqsave() and friends. */
+	__ipipe_spin_unlock_irqcomplete(flags);
+}
+
+void notrace asmlinkage
+ipipe_trace_asm(enum ipipe_trace_type type, unsigned long eip,
+		unsigned long parent_eip, unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(type, eip, parent_eip, v);
+}
+
+void notrace ipipe_trace_begin(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_BEGIN, __BUILTIN_RETURN_ADDRESS0,
+		      __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_begin);
+
+void notrace ipipe_trace_end(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_END, __BUILTIN_RETURN_ADDRESS0,
+		      __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_end);
+
+void notrace ipipe_trace_freeze(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_FREEZE, __BUILTIN_RETURN_ADDRESS0,
+		      __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_freeze);
+
+void notrace ipipe_trace_special(unsigned char id, unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_SPECIAL | (id << IPIPE_TYPE_BITS),
+		      __BUILTIN_RETURN_ADDRESS0,
+		      __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_special);
+
+void notrace ipipe_trace_pid(pid_t pid, short prio)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_PID | (prio << IPIPE_TYPE_BITS),
+		      __BUILTIN_RETURN_ADDRESS0,
+		      __BUILTIN_RETURN_ADDRESS1, pid);
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_pid);
+
+void notrace ipipe_trace_event(unsigned char id, unsigned long delay_tsc)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_EVENT | (id << IPIPE_TYPE_BITS),
+		      __BUILTIN_RETURN_ADDRESS0,
+		      __BUILTIN_RETURN_ADDRESS1, delay_tsc);
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_event);
+
+int ipipe_trace_max_reset(void)
+{
+	int cpu;
+	unsigned long flags;
+	struct ipipe_trace_path *path;
+	int ret = 0;
+
+	flags = __ipipe_global_path_lock();
+
+	for_each_possible_cpu(cpu) {
+		path = &per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)];
+
+		if (path->dump_lock) {
+			ret = -EBUSY;
+			break;
+		}
+
+		path->begin	= -1;
+		path->end	= -1;
+		path->trace_pos = 0;
+		path->length	= 0;
+	}
+
+	__ipipe_global_path_unlock(flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_max_reset);
+
+int ipipe_trace_frozen_reset(void)
+{
+	int cpu;
+	unsigned long flags;
+	struct ipipe_trace_path *path;
+	int ret = 0;
+
+	flags = __ipipe_global_path_lock();
+
+	for_each_online_cpu(cpu) {
+		path = &per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)];
+
+		if (path->dump_lock) {
+			ret = -EBUSY;
+			break;
+		}
+
+		path->begin = -1;
+		path->end = -1;
+		path->trace_pos = 0;
+		path->length	= 0;
+	}
+
+	__ipipe_global_path_unlock(flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_frozen_reset);
+
+static void
+__ipipe_get_task_info(char *task_info, struct ipipe_trace_point *point,
+		      int trylock)
+{
+	struct task_struct *task = NULL;
+	char buf[8];
+	int i;
+	int locked = 1;
+
+	if (trylock) {
+		if (!read_trylock(&tasklist_lock))
+			locked = 0;
+	} else
+		read_lock(&tasklist_lock);
+
+	if (locked)
+		task = find_task_by_pid_ns((pid_t)point->v, &init_pid_ns);
+
+	if (task)
+		strncpy(task_info, task->comm, 11);
+	else
+		strcpy(task_info, "-<?>-");
+
+	if (locked)
+		read_unlock(&tasklist_lock);
+
+	for (i = strlen(task_info); i < 11; i++)
+		task_info[i] = ' ';
+
+	sprintf(buf, " %d ", point->type >> IPIPE_TYPE_BITS);
+	strcpy(task_info + (11 - strlen(buf)), buf);
+}
+
+static void
+__ipipe_get_event_date(char *buf,struct ipipe_trace_path *path,
+		       struct ipipe_trace_point *point)
+{
+	long time;
+	int type;
+
+	time = __ipipe_signed_tsc2us(point->timestamp -
+				     path->point[path->begin].timestamp + point->v);
+	type = point->type >> IPIPE_TYPE_BITS;
+
+	if (type == 0)
+		/*
+		 * Event type #0 is predefined, stands for the next
+		 * timer tick.
+		 */
+		sprintf(buf, "tick@%-6ld", time);
+	else
+		sprintf(buf, "%3d@%-7ld", type, time);
+}
+
+#ifdef CONFIG_IPIPE_TRACE_PANIC
+
+void ipipe_trace_panic_freeze(void)
+{
+	unsigned long flags;
+	int cpu;
+
+	if (!ipipe_trace_enable)
+		return;
+
+	ipipe_trace_enable = 0;
+	flags = hard_local_irq_save_notrace();
+
+	cpu = ipipe_processor_id();
+
+	panic_path = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
+
+	hard_local_irq_restore(flags);
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_panic_freeze);
+
+void ipipe_trace_panic_dump(void)
+{
+	int cnt = back_trace;
+	int start, pos;
+	char buf[16];
+
+	if (!panic_path)
+		return;
+
+	ipipe_context_check_off();
+
+	printk("I-pipe tracer log (%d points):\n", cnt);
+
+	start = pos = WRAP_POINT_NO(panic_path->trace_pos-1);
+
+	while (cnt-- > 0) {
+		struct ipipe_trace_point *point = &panic_path->point[pos];
+		long time;
+		char info[16];
+		int i;
+
+		printk(" %c",
+		       (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' ');
+
+		for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--)
+			printk("%c",
+			       (IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ?
+				(IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
+					'#' : '+') :
+				(IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
+					'*' : ' '));
+
+		if (!point->eip)
+			printk("-<invalid>-\n");
+		else {
+			__ipipe_trace_point_type(buf, point);
+			printk("%s", buf);
+
+			switch (point->type & IPIPE_TYPE_MASK) {
+				case IPIPE_TRACE_FUNC:
+					printk("	   ");
+					break;
+
+				case IPIPE_TRACE_PID:
+					__ipipe_get_task_info(info,
+							      point, 1);
+					printk("%s", info);
+					break;
+
+				case IPIPE_TRACE_EVENT:
+					__ipipe_get_event_date(info,
+							       panic_path, point);
+					printk("%s", info);
+					break;
+
+				default:
+					printk("0x%08lx ", point->v);
+			}
+
+			time = __ipipe_signed_tsc2us(point->timestamp -
+				panic_path->point[start].timestamp);
+			printk(" %5ld ", time);
+
+			__ipipe_print_symname(NULL, point->eip);
+			printk(" (");
+			__ipipe_print_symname(NULL, point->parent_eip);
+			printk(")\n");
+		}
+		pos = WRAP_POINT_NO(pos - 1);
+	}
+
+	panic_path = NULL;
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_panic_dump);
+
+#endif /* CONFIG_IPIPE_TRACE_PANIC */
+
+
+/* --- /proc output --- */
+
+static notrace int __ipipe_in_critical_trpath(long point_no)
+{
+	return ((WRAP_POINT_NO(point_no-print_path->begin) <
+		 WRAP_POINT_NO(print_path->end-print_path->begin)) ||
+		((print_path->end == print_path->begin) &&
+		 (WRAP_POINT_NO(point_no-print_path->end) >
+		  print_post_trace)));
+}
+
+static long __ipipe_signed_tsc2us(long long tsc)
+{
+	unsigned long long abs_tsc;
+	long us;
+
+	if (!__ipipe_hrclock_ok())
+		return 0;
+
+	/* ipipe_tsc2us works on unsigned => handle sign separately */
+	abs_tsc = (tsc >= 0) ? tsc : -tsc;
+	us = ipipe_tsc2us(abs_tsc);
+	if (tsc < 0)
+		return -us;
+	else
+		return us;
+}
+
+static void
+__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point)
+{
+	switch (point->type & IPIPE_TYPE_MASK) {
+		case IPIPE_TRACE_FUNC:
+			strcpy(buf, "func    ");
+			break;
+
+		case IPIPE_TRACE_BEGIN:
+			strcpy(buf, "begin   ");
+			break;
+
+		case IPIPE_TRACE_END:
+			strcpy(buf, "end     ");
+			break;
+
+		case IPIPE_TRACE_FREEZE:
+			strcpy(buf, "freeze  ");
+			break;
+
+		case IPIPE_TRACE_SPECIAL:
+			sprintf(buf, "(0x%02x)	",
+				point->type >> IPIPE_TYPE_BITS);
+			break;
+
+		case IPIPE_TRACE_PID:
+			sprintf(buf, "[%5d] ", (pid_t)point->v);
+			break;
+
+		case IPIPE_TRACE_EVENT:
+			sprintf(buf, "event   ");
+			break;
+	}
+}
+
+static void
+__ipipe_print_pathmark(struct seq_file *m, struct ipipe_trace_point *point)
+{
+	char mark = ' ';
+	int point_no = point - print_path->point;
+	int i;
+
+	if (print_path->end == point_no)
+		mark = '<';
+	else if (print_path->begin == point_no)
+		mark = '>';
+	else if (__ipipe_in_critical_trpath(point_no))
+		mark = ':';
+	seq_printf(m, "%c%c", mark,
+		   (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' ');
+
+	if (!verbose_trace)
+		return;
+
+	for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--)
+		seq_printf(m, "%c",
+			(IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ?
+			    (IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
+				'#' : '+') :
+			(IPIPE_TFLG_DOMAIN_STALLED(point, i) ? '*' : ' '));
+}
+
+static void
+__ipipe_print_delay(struct seq_file *m, struct ipipe_trace_point *point)
+{
+	unsigned long delay = 0;
+	int next;
+	char *mark = "	";
+
+	next = WRAP_POINT_NO(point+1 - print_path->point);
+
+	if (next != print_path->trace_pos)
+		delay = ipipe_tsc2ns(print_path->point[next].timestamp -
+				     point->timestamp);
+
+	if (__ipipe_in_critical_trpath(point - print_path->point)) {
+		if (delay > IPIPE_DELAY_WARN)
+			mark = "! ";
+		else if (delay > IPIPE_DELAY_NOTE)
+			mark = "+ ";
+	}
+	seq_puts(m, mark);
+
+	if (verbose_trace)
+		seq_printf(m, "%3lu.%03lu%c ", delay/1000, delay%1000,
+			   (point->flags & IPIPE_TFLG_NMI_HIT) ? 'N' : ' ');
+	else
+		seq_puts(m, " ");
+}
+
+static void __ipipe_print_symname(struct seq_file *m, unsigned long eip)
+{
+	char namebuf[KSYM_NAME_LEN+1];
+	unsigned long size, offset;
+	const char *sym_name;
+	char *modname;
+
+	sym_name = kallsyms_lookup(eip, &size, &offset, &modname, namebuf);
+
+#ifdef CONFIG_IPIPE_TRACE_PANIC
+	if (!m) {
+		/* panic dump */
+		if (sym_name) {
+			printk("%s+0x%lx", sym_name, offset);
+			if (modname)
+				printk(" [%s]", modname);
+		} else
+			printk("<%08lx>", eip);
+	} else
+#endif /* CONFIG_IPIPE_TRACE_PANIC */
+	{
+		if (sym_name) {
+			if (verbose_trace) {
+				seq_printf(m, "%s+0x%lx", sym_name, offset);
+				if (modname)
+					seq_printf(m, " [%s]", modname);
+			} else
+				seq_puts(m, sym_name);
+		} else
+			seq_printf(m, "<%08lx>", eip);
+	}
+}
+
+static void __ipipe_print_headline(struct seq_file *m)
+{
+	const char *name[2];
+
+	seq_printf(m, "Calibrated minimum trace-point overhead: %lu.%03lu "
+		   "us\n\n", trace_overhead/1000, trace_overhead%1000);
+
+	if (verbose_trace) {
+		name[0] = ipipe_root_domain->name;
+		if (ipipe_head_domain != ipipe_root_domain)
+			name[1] = ipipe_head_domain->name;
+		else
+			name[1] = "<unused>";
+
+		seq_printf(m,
+			   " +----- Hard IRQs ('|': locked)\n"
+			   " |+-- %s\n"
+			   " ||+- %s%s\n"
+			   " |||			  +---------- "
+			       "Delay flag ('+': > %d us, '!': > %d us)\n"
+			   " |||			  |	   +- "
+			       "NMI noise ('N')\n"
+			   " |||			  |	   |\n"
+			   "	  Type	  User Val.   Time    Delay  Function "
+			       "(Parent)\n",
+			   name[1], name[0],
+			   " ('*': domain stalled, '+': current, "
+			   "'#': current+stalled)",
+			   IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000);
+	} else
+		seq_printf(m,
+			   " +--------------- Hard IRQs ('|': locked)\n"
+			   " |		   +- Delay flag "
+			       "('+': > %d us, '!': > %d us)\n"
+			   " |		   |\n"
+			   "  Type     Time   Function (Parent)\n",
+			   IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000);
+}
+
+static void *__ipipe_max_prtrace_start(struct seq_file *m, loff_t *pos)
+{
+	loff_t n = *pos;
+
+	mutex_lock(&out_mutex);
+
+	if (!n) {
+		struct ipipe_trace_path *tp;
+		unsigned long length_usecs;
+		int points, cpu;
+		unsigned long flags;
+
+		/* protect against max_path/frozen_path updates while we
+		 * haven't locked our target path, also avoid recursively
+		 * taking global_path_lock from NMI context */
+		flags = __ipipe_global_path_lock();
+
+		/* find the longest of all per-cpu paths */
+		print_path = NULL;
+		for_each_online_cpu(cpu) {
+			tp = &per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)];
+			if ((print_path == NULL) ||
+			    (tp->length > print_path->length)) {
+				print_path = tp;
+				break;
+			}
+		}
+		print_path->dump_lock = 1;
+
+		__ipipe_global_path_unlock(flags);
+
+		if (!__ipipe_hrclock_ok()) {
+			seq_printf(m, "No hrclock available, dumping traces disabled\n");
+			return NULL;
+		}
+
+		/* does this path actually contain data? */
+		if (print_path->end == print_path->begin)
+			return NULL;
+
+		/* number of points inside the critical path */
+		points = WRAP_POINT_NO(print_path->end-print_path->begin+1);
+
+		/* pre- and post-tracing length, post-trace length was frozen
+		   in __ipipe_trace, pre-trace may have to be reduced due to
+		   buffer overrun */
+		print_pre_trace	 = pre_trace;
+		print_post_trace = WRAP_POINT_NO(print_path->trace_pos -
+						 print_path->end - 1);
+		if (points+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1)
+			print_pre_trace = IPIPE_TRACE_POINTS - 1 - points -
+				print_post_trace;
+
+		length_usecs = ipipe_tsc2us(print_path->length);
+		seq_printf(m, "I-pipe worst-case tracing service on %s/ipipe release #%d\n"
+			   "-------------------------------------------------------------\n",
+			UTS_RELEASE, IPIPE_CORE_RELEASE);
+		seq_printf(m, "CPU: %d, Begin: %lld cycles, Trace Points: "
+			"%d (-%d/+%d), Length: %lu us\n",
+			cpu, print_path->point[print_path->begin].timestamp,
+			points, print_pre_trace, print_post_trace, length_usecs);
+		__ipipe_print_headline(m);
+	}
+
+	/* check if we are inside the trace range */
+	if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 +
+			       print_pre_trace + print_post_trace))
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin -
+						print_pre_trace + n)];
+}
+
+static void *__ipipe_prtrace_next(struct seq_file *m, void *p, loff_t *pos)
+{
+	loff_t n = ++*pos;
+
+	/* check if we are inside the trace range with the next entry */
+	if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 +
+			       print_pre_trace + print_post_trace))
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin -
+						print_pre_trace + *pos)];
+}
+
+static void __ipipe_prtrace_stop(struct seq_file *m, void *p)
+{
+	if (print_path)
+		print_path->dump_lock = 0;
+	mutex_unlock(&out_mutex);
+}
+
+static int __ipipe_prtrace_show(struct seq_file *m, void *p)
+{
+	long time;
+	struct ipipe_trace_point *point = p;
+	char buf[16];
+
+	if (!point->eip) {
+		seq_puts(m, "-<invalid>-\n");
+		return 0;
+	}
+
+	__ipipe_print_pathmark(m, point);
+	__ipipe_trace_point_type(buf, point);
+	seq_puts(m, buf);
+	if (verbose_trace)
+		switch (point->type & IPIPE_TYPE_MASK) {
+			case IPIPE_TRACE_FUNC:
+				seq_puts(m, "           ");
+				break;
+
+			case IPIPE_TRACE_PID:
+				__ipipe_get_task_info(buf, point, 0);
+				seq_puts(m, buf);
+				break;
+
+			case IPIPE_TRACE_EVENT:
+				__ipipe_get_event_date(buf, print_path, point);
+				seq_puts(m, buf);
+				break;
+
+			default:
+				seq_printf(m, "0x%08lx ", point->v);
+		}
+
+	time = __ipipe_signed_tsc2us(point->timestamp -
+		print_path->point[print_path->begin].timestamp);
+	seq_printf(m, "%5ld", time);
+
+	__ipipe_print_delay(m, point);
+	__ipipe_print_symname(m, point->eip);
+	seq_puts(m, " (");
+	__ipipe_print_symname(m, point->parent_eip);
+	seq_puts(m, ")\n");
+
+	return 0;
+}
+
+static struct seq_operations __ipipe_max_ptrace_ops = {
+	.start = __ipipe_max_prtrace_start,
+	.next  = __ipipe_prtrace_next,
+	.stop  = __ipipe_prtrace_stop,
+	.show  = __ipipe_prtrace_show
+};
+
+static int __ipipe_max_prtrace_open(struct inode *inode, struct file *file)
+{
+	return seq_open(file, &__ipipe_max_ptrace_ops);
+}
+
+static ssize_t
+__ipipe_max_reset(struct file *file, const char __user *pbuffer,
+		  size_t count, loff_t *data)
+{
+	mutex_lock(&out_mutex);
+	ipipe_trace_max_reset();
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+static const struct file_operations __ipipe_max_prtrace_fops = {
+	.open	    = __ipipe_max_prtrace_open,
+	.read	    = seq_read,
+	.write	    = __ipipe_max_reset,
+	.llseek	    = seq_lseek,
+	.release    = seq_release,
+};
+
+static void *__ipipe_frozen_prtrace_start(struct seq_file *m, loff_t *pos)
+{
+	loff_t n = *pos;
+
+	mutex_lock(&out_mutex);
+
+	if (!n) {
+		struct ipipe_trace_path *tp;
+		int cpu;
+		unsigned long flags;
+
+		/* protect against max_path/frozen_path updates while we
+		 * haven't locked our target path, also avoid recursively
+		 * taking global_path_lock from NMI context */
+		flags = __ipipe_global_path_lock();
+
+		/* find the first of all per-cpu frozen paths */
+		print_path = NULL;
+		for_each_online_cpu(cpu) {
+			tp = &per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)];
+			if (tp->end >= 0) {
+				print_path = tp;
+				break;
+			}
+		}
+		if (print_path)
+			print_path->dump_lock = 1;
+
+		__ipipe_global_path_unlock(flags);
+
+		if (!print_path)
+			return NULL;
+
+		if (!__ipipe_hrclock_ok()) {
+			seq_printf(m, "No hrclock available, dumping traces disabled\n");
+			return NULL;
+		}
+
+		/* back- and post-tracing length, post-trace length was frozen
+		   in __ipipe_trace, back-trace may have to be reduced due to
+		   buffer overrun */
+		print_pre_trace	 = back_trace-1; /* substract freeze point */
+		print_post_trace = WRAP_POINT_NO(print_path->trace_pos -
+						 print_path->end - 1);
+		if (1+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1)
+			print_pre_trace = IPIPE_TRACE_POINTS - 2 -
+				print_post_trace;
+
+		seq_printf(m, "I-pipe frozen back-tracing service on %s/ipipe release #%d\n"
+			      "------------------------------------------------------------\n",
+			   UTS_RELEASE, IPIPE_CORE_RELEASE);
+		seq_printf(m, "CPU: %d, Freeze: %lld cycles, Trace Points: %d (+%d)\n",
+			cpu, print_path->point[print_path->begin].timestamp,
+			print_pre_trace+1, print_post_trace);
+		__ipipe_print_headline(m);
+	}
+
+	/* check if we are inside the trace range */
+	if (n >= print_pre_trace + 1 + print_post_trace)
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin-
+						print_pre_trace+n)];
+}
+
+static struct seq_operations __ipipe_frozen_ptrace_ops = {
+	.start = __ipipe_frozen_prtrace_start,
+	.next  = __ipipe_prtrace_next,
+	.stop  = __ipipe_prtrace_stop,
+	.show  = __ipipe_prtrace_show
+};
+
+static int __ipipe_frozen_prtrace_open(struct inode *inode, struct file *file)
+{
+	return seq_open(file, &__ipipe_frozen_ptrace_ops);
+}
+
+static ssize_t
+__ipipe_frozen_ctrl(struct file *file, const char __user *pbuffer,
+		    size_t count, loff_t *data)
+{
+	char *end, buf[16];
+	int val;
+	int n;
+
+	n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+	if (copy_from_user(buf, pbuffer, n))
+		return -EFAULT;
+
+	buf[n] = '\0';
+	val = simple_strtol(buf, &end, 0);
+
+	if (((*end != '\0') && !isspace(*end)) || (val < 0))
+		return -EINVAL;
+
+	mutex_lock(&out_mutex);
+	ipipe_trace_frozen_reset();
+	if (val > 0)
+		ipipe_trace_freeze(-1);
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+static const struct file_operations __ipipe_frozen_prtrace_fops = {
+	.open	    = __ipipe_frozen_prtrace_open,
+	.read	    = seq_read,
+	.write	    = __ipipe_frozen_ctrl,
+	.llseek	    = seq_lseek,
+	.release    = seq_release,
+};
+
+static int __ipipe_rd_proc_val(struct seq_file *p, void *data)
+{
+	seq_printf(p, "%u\n", *(int *)p->private);
+	return 0;
+}
+
+static ssize_t
+__ipipe_wr_proc_val(struct file *file, const char __user *buffer,
+		    size_t count, loff_t *data)
+{
+	struct seq_file *p = file->private_data;
+	char *end, buf[16];
+	int val;
+	int n;
+
+	n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+	if (copy_from_user(buf, buffer, n))
+		return -EFAULT;
+
+	buf[n] = '\0';
+	val = simple_strtol(buf, &end, 0);
+
+	if (((*end != '\0') && !isspace(*end)) || (val < 0))
+		return -EINVAL;
+
+	mutex_lock(&out_mutex);
+	*(int *)p->private = val;
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+static int __ipipe_rw_proc_val_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, __ipipe_rd_proc_val, PDE_DATA(inode));
+}
+
+static const struct file_operations __ipipe_rw_proc_val_ops = {
+	.open		= __ipipe_rw_proc_val_open,
+	.read		= seq_read,
+	.write		= __ipipe_wr_proc_val,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+static void __init
+__ipipe_create_trace_proc_val(struct proc_dir_entry *trace_dir,
+			      const char *name, int *value_ptr)
+{
+	proc_create_data(name, 0644, trace_dir, &__ipipe_rw_proc_val_ops,
+			 value_ptr);
+}
+
+static int __ipipe_rd_trigger(struct seq_file *p, void *data)
+{
+	char str[KSYM_SYMBOL_LEN];
+
+	if (trigger_begin) {
+		sprint_symbol(str, trigger_begin);
+		seq_printf(p, "%s\n", str);
+	}
+	return 0;
+}
+
+static ssize_t
+__ipipe_wr_trigger(struct file *file, const char __user *buffer,
+		   size_t count, loff_t *data)
+{
+	char buf[KSYM_SYMBOL_LEN];
+	unsigned long begin, end;
+
+	if (count > sizeof(buf) - 1)
+		count = sizeof(buf) - 1;
+	if (copy_from_user(buf, buffer, count))
+		return -EFAULT;
+	buf[count] = 0;
+	if (buf[count-1] == '\n')
+		buf[count-1] = 0;
+
+	begin = kallsyms_lookup_name(buf);
+	if (!begin || !kallsyms_lookup_size_offset(begin, &end, NULL))
+		return -ENOENT;
+	end += begin - 1;
+
+	mutex_lock(&out_mutex);
+	/* invalidate the current range before setting a new one */
+	trigger_end = 0;
+	wmb();
+	ipipe_trace_frozen_reset();
+
+	/* set new range */
+	trigger_begin = begin;
+	wmb();
+	trigger_end = end;
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+static int __ipipe_rw_trigger_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, __ipipe_rd_trigger, NULL);
+}
+
+static const struct file_operations __ipipe_rw_trigger_ops = {
+	.open		= __ipipe_rw_trigger_open,
+	.read		= seq_read,
+	.write		= __ipipe_wr_trigger,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+
+#ifdef CONFIG_IPIPE_TRACE_MCOUNT
+static void notrace
+ipipe_trace_function(unsigned long ip, unsigned long parent_ip,
+		     struct ftrace_ops *op, struct pt_regs *regs)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_FUNC, ip, parent_ip, 0);
+}
+
+static struct ftrace_ops ipipe_trace_ops = {
+	.func = ipipe_trace_function,
+	.flags = FTRACE_OPS_FL_IPIPE_EXCLUSIVE,
+};
+
+static ssize_t __ipipe_wr_enable(struct file *file, const char __user *buffer,
+				 size_t count, loff_t *data)
+{
+	char *end, buf[16];
+	int val;
+	int n;
+
+	n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+	if (copy_from_user(buf, buffer, n))
+		return -EFAULT;
+
+	buf[n] = '\0';
+	val = simple_strtol(buf, &end, 0);
+
+	if (((*end != '\0') && !isspace(*end)) || (val < 0))
+		return -EINVAL;
+
+	mutex_lock(&out_mutex);
+
+	if (ipipe_trace_enable) {
+		if (!val)
+			unregister_ftrace_function(&ipipe_trace_ops);
+	} else if (val)
+		register_ftrace_function(&ipipe_trace_ops);
+
+	ipipe_trace_enable = val;
+
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+static const struct file_operations __ipipe_rw_enable_ops = {
+	.open		= __ipipe_rw_proc_val_open,
+	.read		= seq_read,
+	.write		= __ipipe_wr_enable,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+#endif /* CONFIG_IPIPE_TRACE_MCOUNT */
+
+extern struct proc_dir_entry *ipipe_proc_root;
+
+void __init __ipipe_tracer_hrclock_initialized(void)
+{
+	unsigned long long start, end, min = ULLONG_MAX;
+	int i;
+
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
+	if (!per_cpu(trace_path, 0))
+		return;
+#endif
+	/* Calculate minimum overhead of __ipipe_trace() */
+	hard_local_irq_disable();
+	for (i = 0; i < 100; i++) {
+		ipipe_read_tsc(start);
+		__ipipe_trace(IPIPE_TRACE_FUNC, __BUILTIN_RETURN_ADDRESS0,
+			      __BUILTIN_RETURN_ADDRESS1, 0);
+		ipipe_read_tsc(end);
+
+		end -= start;
+		if (end < min)
+			min = end;
+	}
+	hard_local_irq_enable();
+	trace_overhead = ipipe_tsc2ns(min);
+}
+
+void __init __ipipe_init_tracer(void)
+{
+	struct proc_dir_entry *trace_dir;
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
+	int cpu, path;
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
+
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
+	for_each_possible_cpu(cpu) {
+		struct ipipe_trace_path *tp_buf;
+
+		tp_buf = vmalloc_node(sizeof(struct ipipe_trace_path) *
+				      IPIPE_TRACE_PATHS, cpu_to_node(cpu));
+		if (!tp_buf) {
+			pr_err("I-pipe: "
+			       "insufficient memory for trace buffer.\n");
+			return;
+		}
+		memset(tp_buf, 0,
+		       sizeof(struct ipipe_trace_path) * IPIPE_TRACE_PATHS);
+		for (path = 0; path < IPIPE_TRACE_PATHS; path++) {
+			tp_buf[path].begin = -1;
+			tp_buf[path].end   = -1;
+		}
+		per_cpu(trace_path, cpu) = tp_buf;
+	}
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
+
+	if (__ipipe_hrclock_ok() && !trace_overhead)
+		__ipipe_tracer_hrclock_initialized();
+
+#ifdef CONFIG_IPIPE_TRACE_ENABLE
+	ipipe_trace_enable = 1;
+#ifdef CONFIG_IPIPE_TRACE_MCOUNT
+	ftrace_enabled = 1;
+	register_ftrace_function(&ipipe_trace_ops);
+#endif /* CONFIG_IPIPE_TRACE_MCOUNT */
+#endif /* CONFIG_IPIPE_TRACE_ENABLE */
+
+	trace_dir = proc_mkdir("trace", ipipe_proc_root);
+
+	proc_create("max", 0644, trace_dir, &__ipipe_max_prtrace_fops);
+	proc_create("frozen", 0644, trace_dir, &__ipipe_frozen_prtrace_fops);
+
+	proc_create("trigger", 0644, trace_dir, &__ipipe_rw_trigger_ops);
+
+	__ipipe_create_trace_proc_val(trace_dir, "pre_trace_points",
+				      &pre_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "post_trace_points",
+				      &post_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "back_trace_points",
+				      &back_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "verbose",
+				      &verbose_trace);
+#ifdef CONFIG_IPIPE_TRACE_MCOUNT
+	proc_create_data("enable", 0644, trace_dir, &__ipipe_rw_enable_ops,
+			 &ipipe_trace_enable);
+#else /* !CONFIG_IPIPE_TRACE_MCOUNT */
+	__ipipe_create_trace_proc_val(trace_dir, "enable",
+				      &ipipe_trace_enable);
+#endif /* !CONFIG_IPIPE_TRACE_MCOUNT */
+}
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index e5202f0..524c2e5 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -15,6 +15,7 @@
 #include <linux/module.h>
 #include <linux/interrupt.h>
 #include <linux/kernel_stat.h>
+#include <linux/ipipe.h>
 
 #include <trace/events/irq.h>
 
@@ -143,14 +144,6 @@ int irq_set_chip_data(unsigned int irq, void *data)
 }
 EXPORT_SYMBOL(irq_set_chip_data);
 
-struct irq_data *irq_get_irq_data(unsigned int irq)
-{
-	struct irq_desc *desc = irq_to_desc(irq);
-
-	return desc ? &desc->irq_data : NULL;
-}
-EXPORT_SYMBOL_GPL(irq_get_irq_data);
-
 static void irq_state_clr_disabled(struct irq_desc *desc)
 {
 	irqd_clear(&desc->irq_data, IRQD_IRQ_DISABLED);
@@ -179,8 +172,13 @@ int irq_startup(struct irq_desc *desc, bool resend)
 	desc->depth = 0;
 
 	if (desc->irq_data.chip->irq_startup) {
+		unsigned long flags = hard_cond_local_irq_save();
 		ret = desc->irq_data.chip->irq_startup(&desc->irq_data);
 		irq_state_clr_masked(desc);
+		hard_cond_local_irq_restore(flags);
+#ifdef CONFIG_IPIPE
+		desc->istate &= ~IPIPE_IRQS_NEEDS_STARTUP;
+#endif
 	} else {
 		irq_enable(desc);
 	}
@@ -193,9 +191,12 @@ void irq_shutdown(struct irq_desc *desc)
 {
 	irq_state_set_disabled(desc);
 	desc->depth = 1;
-	if (desc->irq_data.chip->irq_shutdown)
+	if (desc->irq_data.chip->irq_shutdown) {
 		desc->irq_data.chip->irq_shutdown(&desc->irq_data);
-	else if (desc->irq_data.chip->irq_disable)
+#ifdef CONFIG_IPIPE
+		desc->istate |= IPIPE_IRQS_NEEDS_STARTUP;
+#endif
+	} else if (desc->irq_data.chip->irq_disable)
 		desc->irq_data.chip->irq_disable(&desc->irq_data);
 	else
 		desc->irq_data.chip->irq_mask(&desc->irq_data);
@@ -204,12 +205,14 @@ void irq_shutdown(struct irq_desc *desc)
 
 void irq_enable(struct irq_desc *desc)
 {
+	unsigned long flags = hard_cond_local_irq_save();
 	irq_state_clr_disabled(desc);
 	if (desc->irq_data.chip->irq_enable)
 		desc->irq_data.chip->irq_enable(&desc->irq_data);
 	else
 		desc->irq_data.chip->irq_unmask(&desc->irq_data);
 	irq_state_clr_masked(desc);
+	hard_cond_local_irq_restore(flags);
 }
 
 /**
@@ -236,11 +239,13 @@ void irq_disable(struct irq_desc *desc)
 
 void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu)
 {
+	unsigned long flags = hard_cond_local_irq_save();
 	if (desc->irq_data.chip->irq_enable)
 		desc->irq_data.chip->irq_enable(&desc->irq_data);
 	else
 		desc->irq_data.chip->irq_unmask(&desc->irq_data);
 	cpumask_set_cpu(cpu, desc->percpu_enabled);
+	hard_cond_local_irq_restore(flags);
 }
 
 void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu)
@@ -274,9 +279,13 @@ void mask_irq(struct irq_desc *desc)
 
 void unmask_irq(struct irq_desc *desc)
 {
+	unsigned long flags;
+
 	if (desc->irq_data.chip->irq_unmask) {
+		flags = hard_cond_local_irq_save();
 		desc->irq_data.chip->irq_unmask(&desc->irq_data);
 		irq_state_clr_masked(desc);
+		hard_cond_local_irq_restore(flags);
 	}
 }
 
@@ -434,7 +443,9 @@ void
 handle_level_irq(unsigned int irq, struct irq_desc *desc)
 {
 	raw_spin_lock(&desc->lock);
+#ifndef CONFIG_IPIPE
 	mask_ack_irq(desc);
+#endif
 
 	if (!irq_may_run(desc))
 		goto out_unlock;
@@ -470,7 +481,17 @@ static inline void preflow_handler(struct irq_desc *desc)
 static inline void preflow_handler(struct irq_desc *desc) { }
 #endif
 
-static void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip)
+#ifdef CONFIG_IPIPE
+static void cond_release_fasteoi_irq(struct irq_desc *desc,
+				     struct irq_chip *chip)
+{
+	if (chip->irq_release && 
+	    !irqd_irq_disabled(&desc->irq_data) && !desc->threads_oneshot)
+		chip->irq_release(&desc->irq_data);
+}
+#endif /* CONFIG_IPIPE */
+
+static inline void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip)
 {
 	if (!(desc->istate & IRQS_ONESHOT)) {
 		chip->irq_eoi(&desc->irq_data);
@@ -524,14 +545,23 @@ handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc)
 		goto out;
 	}
 
+#ifndef CONFIG_IPIPE
 	if (desc->istate & IRQS_ONESHOT)
 		mask_irq(desc);
+#endif
 
 	preflow_handler(desc);
 	handle_irq_event(desc);
 
+#ifdef CONFIG_IPIPE
+	/*
+	 * IRQCHIP_EOI_IF_HANDLED is ignored as the I-pipe always
+	 * sends EOI.
+	 */
+	cond_release_fasteoi_irq(desc, chip);
+#else  /* !CONFIG_IPIPE */
 	cond_unmask_eoi_irq(desc, chip);
-
+#endif	/* !CONFIG_IPIPE */
 	raw_spin_unlock(&desc->lock);
 	return;
 out:
@@ -583,7 +613,9 @@ handle_edge_irq(unsigned int irq, struct irq_desc *desc)
 	kstat_incr_irqs_this_cpu(irq, desc);
 
 	/* Start handling the irq */
+#ifndef CONFIG_IPIPE
 	desc->irq_data.chip->irq_ack(&desc->irq_data);
+#endif
 
 	do {
 		if (unlikely(!desc->action)) {
@@ -674,6 +706,18 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc)
 
 	kstat_incr_irqs_this_cpu(irq, desc);
 
+#ifdef CONFIG_IPIPE
+	(void)chip;
+
+	handle_irq_event_percpu(desc, desc->action);
+
+	if ((desc->percpu_enabled == NULL ||
+	     cpumask_test_cpu(smp_processor_id(), desc->percpu_enabled)) &&
+	    !irqd_irq_masked(&desc->irq_data) &&
+	    !desc->threads_oneshot &&
+	    desc->ipipe_end)
+		desc->ipipe_end(desc->irq_data.irq, desc);
+#else
 	if (chip->irq_ack)
 		chip->irq_ack(&desc->irq_data);
 
@@ -681,6 +725,7 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc)
 
 	if (chip->irq_eoi)
 		chip->irq_eoi(&desc->irq_data);
+#endif
 }
 
 /**
@@ -704,17 +749,180 @@ void handle_percpu_devid_irq(unsigned int irq, struct irq_desc *desc)
 
 	kstat_incr_irqs_this_cpu(irq, desc);
 
+#ifndef CONFIG_IPIPE
 	if (chip->irq_ack)
 		chip->irq_ack(&desc->irq_data);
+#else
+	(void)chip;
+#endif
 
 	trace_irq_handler_entry(irq, action);
 	res = action->handler(irq, dev_id);
 	trace_irq_handler_exit(irq, action, res);
 
+#ifndef CONFIG_IPIPE
 	if (chip->irq_eoi)
 		chip->irq_eoi(&desc->irq_data);
+#else
+	if ((desc->percpu_enabled == NULL ||
+	     cpumask_test_cpu(smp_processor_id(), desc->percpu_enabled)) &&
+	    !irqd_irq_masked(&desc->irq_data) &&
+	    !desc->threads_oneshot &&
+	    desc->ipipe_end)
+		desc->ipipe_end(desc->irq_data.irq, desc);
+#endif
+}
+
+#ifdef CONFIG_IPIPE
+
+void __ipipe_ack_level_irq(unsigned irq, struct irq_desc *desc)
+{
+	mask_ack_irq(desc);
+}
+
+void __ipipe_end_level_irq(unsigned irq, struct irq_desc *desc)
+{
+	desc->irq_data.chip->irq_unmask(&desc->irq_data);
+}
+
+void __ipipe_ack_fasteoi_irq(unsigned irq, struct irq_desc *desc)
+{
+	desc->irq_data.chip->irq_hold(&desc->irq_data);
+}
+
+void __ipipe_end_fasteoi_irq(unsigned irq, struct irq_desc *desc)
+{
+	if (desc->irq_data.chip->irq_release)
+		desc->irq_data.chip->irq_release(&desc->irq_data);
+}
+
+void __ipipe_ack_edge_irq(unsigned irq, struct irq_desc *desc)
+{
+	desc->irq_data.chip->irq_ack(&desc->irq_data);
+}
+
+void __ipipe_ack_percpu_irq(unsigned irq, struct irq_desc *desc)
+{
+	if (desc->irq_data.chip->irq_ack)
+		desc->irq_data.chip->irq_ack(&desc->irq_data);
+
+	if (desc->irq_data.chip->irq_eoi)
+		desc->irq_data.chip->irq_eoi(&desc->irq_data);
 }
 
+void __ipipe_nop_irq(unsigned irq, struct irq_desc *desc)
+{
+}
+
+void __ipipe_chained_irq(unsigned irq, struct irq_desc *desc)
+{
+	/*
+	 * XXX: Do NOT fold this into __ipipe_nop_irq(), see
+	 * ipipe_chained_irq_p().
+	 */
+}
+
+static void __ipipe_ack_bad_irq(unsigned irq, struct irq_desc *desc)
+{
+	handle_bad_irq(irq, desc);
+	WARN_ON_ONCE(1);
+}
+
+irq_flow_handler_t
+__fixup_irq_handler(struct irq_desc *desc, irq_flow_handler_t handle, int is_chained)
+{
+	if (unlikely(handle == NULL)) {
+		desc->ipipe_ack = __ipipe_ack_bad_irq;
+		desc->ipipe_end = __ipipe_nop_irq;
+	} else {
+		if (is_chained) {
+			desc->ipipe_ack = handle;
+			desc->ipipe_end = __ipipe_nop_irq;
+			handle = __ipipe_chained_irq;
+		} else if (handle == handle_simple_irq) {
+			desc->ipipe_ack = __ipipe_nop_irq;
+			desc->ipipe_end = __ipipe_nop_irq;
+		} else if (handle == handle_level_irq) {
+			desc->ipipe_ack = __ipipe_ack_level_irq;
+			desc->ipipe_end = __ipipe_end_level_irq;
+		} else if (handle == handle_edge_irq) {
+			desc->ipipe_ack = __ipipe_ack_edge_irq;
+			desc->ipipe_end = __ipipe_nop_irq;
+		} else if (handle == handle_fasteoi_irq) {
+			desc->ipipe_ack = __ipipe_ack_fasteoi_irq;
+			desc->ipipe_end = __ipipe_end_fasteoi_irq;
+		} else if (handle == handle_percpu_irq ||
+			   handle == handle_percpu_devid_irq) {
+			if (irq_desc_get_chip(desc) &&
+			    irq_desc_get_chip(desc)->irq_hold) {
+				desc->ipipe_ack = __ipipe_ack_fasteoi_irq;
+				desc->ipipe_end = __ipipe_end_fasteoi_irq;
+			} else {
+				desc->ipipe_ack = __ipipe_ack_percpu_irq;
+				desc->ipipe_end = __ipipe_nop_irq;
+			}
+		} else if (irq_desc_get_chip(desc) == &no_irq_chip) {
+			desc->ipipe_ack = __ipipe_nop_irq;
+			desc->ipipe_end = __ipipe_nop_irq;
+		} else {
+			desc->ipipe_ack = __ipipe_ack_bad_irq;
+			desc->ipipe_end = __ipipe_nop_irq;
+		}
+	}
+
+	/* Suppress intermediate trampoline routine. */
+	ipipe_root_domain->irqs[desc->irq_data.irq].ackfn = desc->ipipe_ack;
+
+	return handle;
+}
+
+void ipipe_enable_irq(unsigned int irq)
+{
+	struct irq_desc *desc;
+	struct irq_chip *chip;
+	unsigned long flags;
+
+	desc = irq_to_desc(irq);
+	if (desc == NULL)
+		return;
+
+	chip = irq_desc_get_chip(desc);
+
+	if (chip->irq_startup && (desc->istate & IPIPE_IRQS_NEEDS_STARTUP)) {
+
+		ipipe_root_only();
+
+		raw_spin_lock_irqsave(&desc->lock, flags);
+		if (desc->istate & IPIPE_IRQS_NEEDS_STARTUP) {
+			desc->istate &= ~IPIPE_IRQS_NEEDS_STARTUP;
+			chip->irq_startup(&desc->irq_data);
+		}
+		raw_spin_unlock_irqrestore(&desc->lock, flags);
+
+		return;
+	}
+
+	if (WARN_ON_ONCE(chip->irq_enable == NULL && chip->irq_unmask == NULL))
+		return;
+
+	if (chip->irq_enable)
+		chip->irq_enable(&desc->irq_data);
+	else
+		chip->irq_unmask(&desc->irq_data);
+}
+EXPORT_SYMBOL_GPL(ipipe_enable_irq);
+
+#else /* !CONFIG_IPIPE */
+
+irq_flow_handler_t
+__fixup_irq_handler(struct irq_desc *desc, irq_flow_handler_t handle, int is_chained)
+{
+	return handle;
+}
+
+#endif /* !CONFIG_IPIPE */
+EXPORT_SYMBOL_GPL(__fixup_irq_handler);
+
 void
 __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
 		  const char *name)
@@ -732,6 +940,8 @@ __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
 			goto out;
 	}
 
+	handle = __fixup_irq_handler(desc, handle, is_chained);
+
 	/* Uninstall? */
 	if (handle == handle_bad_irq) {
 		if (desc->irq_data.chip != &no_irq_chip)
diff --git a/kernel/irq/generic-chip.c b/kernel/irq/generic-chip.c
index cf80e7b..acbf7ae 100644
--- a/kernel/irq/generic-chip.c
+++ b/kernel/irq/generic-chip.c
@@ -36,12 +36,13 @@ void irq_gc_mask_disable_reg(struct irq_data *d)
 {
 	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
 	struct irq_chip_type *ct = irq_data_get_chip_type(d);
+	unsigned long flags;
 	u32 mask = d->mask;
 
-	irq_gc_lock(gc);
+	flags = irq_gc_lock(gc);
 	irq_reg_writel(mask, gc->reg_base + ct->regs.disable);
 	*ct->mask_cache &= ~mask;
-	irq_gc_unlock(gc);
+	irq_gc_unlock(gc, flags);
 }
 
 /**
@@ -55,12 +56,13 @@ void irq_gc_mask_set_bit(struct irq_data *d)
 {
 	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
 	struct irq_chip_type *ct = irq_data_get_chip_type(d);
+	unsigned long flags;
 	u32 mask = d->mask;
 
-	irq_gc_lock(gc);
+	flags = irq_gc_lock(gc);
 	*ct->mask_cache |= mask;
 	irq_reg_writel(*ct->mask_cache, gc->reg_base + ct->regs.mask);
-	irq_gc_unlock(gc);
+	irq_gc_unlock(gc, flags);
 }
 EXPORT_SYMBOL_GPL(irq_gc_mask_set_bit);
 
@@ -75,12 +77,13 @@ void irq_gc_mask_clr_bit(struct irq_data *d)
 {
 	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
 	struct irq_chip_type *ct = irq_data_get_chip_type(d);
+	unsigned long flags;
 	u32 mask = d->mask;
 
-	irq_gc_lock(gc);
+	flags = irq_gc_lock(gc);
 	*ct->mask_cache &= ~mask;
 	irq_reg_writel(*ct->mask_cache, gc->reg_base + ct->regs.mask);
-	irq_gc_unlock(gc);
+	irq_gc_unlock(gc, flags);
 }
 EXPORT_SYMBOL_GPL(irq_gc_mask_clr_bit);
 
@@ -95,12 +98,13 @@ void irq_gc_unmask_enable_reg(struct irq_data *d)
 {
 	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
 	struct irq_chip_type *ct = irq_data_get_chip_type(d);
+	unsigned long flags;
 	u32 mask = d->mask;
 
-	irq_gc_lock(gc);
+	flags = irq_gc_lock(gc);
 	irq_reg_writel(mask, gc->reg_base + ct->regs.enable);
 	*ct->mask_cache |= mask;
-	irq_gc_unlock(gc);
+	irq_gc_unlock(gc, flags);
 }
 
 /**
@@ -111,11 +115,12 @@ void irq_gc_ack_set_bit(struct irq_data *d)
 {
 	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
 	struct irq_chip_type *ct = irq_data_get_chip_type(d);
+	unsigned long flags;
 	u32 mask = d->mask;
 
-	irq_gc_lock(gc);
+	flags = irq_gc_lock(gc);
 	irq_reg_writel(mask, gc->reg_base + ct->regs.ack);
-	irq_gc_unlock(gc);
+	irq_gc_unlock(gc, flags);
 }
 EXPORT_SYMBOL_GPL(irq_gc_ack_set_bit);
 
@@ -127,11 +132,12 @@ void irq_gc_ack_clr_bit(struct irq_data *d)
 {
 	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
 	struct irq_chip_type *ct = irq_data_get_chip_type(d);
+	unsigned long flags;
 	u32 mask = ~d->mask;
 
-	irq_gc_lock(gc);
+	flags = irq_gc_lock(gc);
 	irq_reg_writel(mask, gc->reg_base + ct->regs.ack);
-	irq_gc_unlock(gc);
+	irq_gc_unlock(gc, flags);
 }
 
 /**
@@ -142,12 +148,13 @@ void irq_gc_mask_disable_reg_and_ack(struct irq_data *d)
 {
 	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
 	struct irq_chip_type *ct = irq_data_get_chip_type(d);
+	unsigned long flags;
 	u32 mask = d->mask;
 
-	irq_gc_lock(gc);
+	flags = irq_gc_lock(gc);
 	irq_reg_writel(mask, gc->reg_base + ct->regs.mask);
 	irq_reg_writel(mask, gc->reg_base + ct->regs.ack);
-	irq_gc_unlock(gc);
+	irq_gc_unlock(gc, flags);
 }
 
 /**
@@ -158,11 +165,12 @@ void irq_gc_eoi(struct irq_data *d)
 {
 	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
 	struct irq_chip_type *ct = irq_data_get_chip_type(d);
+	unsigned long flags;
 	u32 mask = d->mask;
 
-	irq_gc_lock(gc);
+	flags = irq_gc_lock(gc);
 	irq_reg_writel(mask, gc->reg_base + ct->regs.eoi);
-	irq_gc_unlock(gc);
+	irq_gc_unlock(gc, flags);
 }
 
 /**
@@ -177,17 +185,18 @@ void irq_gc_eoi(struct irq_data *d)
 int irq_gc_set_wake(struct irq_data *d, unsigned int on)
 {
 	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
+	unsigned long flags;
 	u32 mask = d->mask;
 
 	if (!(mask & gc->wake_enabled))
 		return -EINVAL;
 
-	irq_gc_lock(gc);
+	flags = irq_gc_lock(gc);
 	if (on)
 		gc->wake_active |= mask;
 	else
 		gc->wake_active &= ~mask;
-	irq_gc_unlock(gc);
+	irq_gc_unlock(gc, flags);
 	return 0;
 }
 
diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h
index df553b0..065468d 100644
--- a/kernel/irq/internals.h
+++ b/kernel/irq/internals.h
@@ -54,6 +54,9 @@ enum {
 	IRQS_WAITING		= 0x00000080,
 	IRQS_PENDING		= 0x00000200,
 	IRQS_SUSPENDED		= 0x00000800,
+#ifdef CONFIG_IPIPE
+	IPIPE_IRQS_NEEDS_STARTUP= 0x80000000,
+#endif
 };
 
 #include "debug.h"
diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c
index 99793b9..16cda25 100644
--- a/kernel/irq/irqdesc.c
+++ b/kernel/irq/irqdesc.c
@@ -92,6 +92,9 @@ static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node,
 	for_each_possible_cpu(cpu)
 		*per_cpu_ptr(desc->kstat_irqs, cpu) = 0;
 	desc_smp_init(desc, node);
+#ifdef CONFIG_IPIPE
+	desc->istate |= IPIPE_IRQS_NEEDS_STARTUP;
+#endif
 }
 
 int nr_irqs = NR_IRQS;
@@ -287,11 +290,13 @@ int __init early_irq_init(void)
 	return arch_early_irq_init();
 }
 
+#ifndef CONFIG_IPIPE
 struct irq_desc *irq_to_desc(unsigned int irq)
 {
 	return (irq < NR_IRQS) ? irq_desc + irq : NULL;
 }
 EXPORT_SYMBOL(irq_to_desc);
+#endif /* CONFIG_IPIPE */
 
 static void free_desc(unsigned int irq)
 {
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index 0a9104b..eeec493 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -723,9 +723,14 @@ again:
 
 	desc->threads_oneshot &= ~action->thread_mask;
 
+#ifndef CONFIG_IPIPE
 	if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data) &&
 	    irqd_irq_masked(&desc->irq_data))
 		unmask_threaded_irq(desc);
+#else /* CONFIG_IPIPE */
+	if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data))
+		desc->ipipe_end(desc->irq_data.irq, desc);
+#endif /* CONFIG_IPIPE */
 
 out_unlock:
 	raw_spin_unlock_irq(&desc->lock);
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index 88d0d44..237481e 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -2581,7 +2581,7 @@ __visible void trace_hardirqs_on_caller(unsigned long ip)
 	 * already enabled, yet we find the hardware thinks they are in fact
 	 * enabled.. someone messed up their IRQ state tracing.
 	 */
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !hard_irqs_disabled()))
 		return;
 
 	/*
@@ -2625,7 +2625,7 @@ __visible void trace_hardirqs_off_caller(unsigned long ip)
 	 * So we're supposed to get called after you mask local IRQs, but for
 	 * some reason the hardware doesn't quite think you did a proper job.
 	 */
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !hard_irqs_disabled()))
 		return;
 
 	if (curr->hardirqs_enabled) {
@@ -2661,7 +2661,7 @@ void trace_softirqs_on(unsigned long ip)
 	 * We fancy IRQs being disabled here, see softirq.c, avoids
 	 * funny state and nesting things.
 	 */
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !hard_irqs_disabled()))
 		return;
 
 	if (curr->softirqs_enabled) {
@@ -2700,7 +2700,7 @@ void trace_softirqs_off(unsigned long ip)
 	/*
 	 * We fancy IRQs being disabled here, see softirq.c
 	 */
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !hard_irqs_disabled()))
 		return;
 
 	if (curr->softirqs_enabled) {
diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c
index 4b082b5..67447fc 100644
--- a/kernel/locking/spinlock.c
+++ b/kernel/locking/spinlock.c
@@ -26,7 +26,9 @@
  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
  */
-#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
+#if !defined(CONFIG_GENERIC_LOCKBREAK) ||			\
+	defined(CONFIG_DEBUG_LOCK_ALLOC) ||			\
+	defined(CONFIG_IPIPE)
 /*
  * The __lock_function inlines are taken from
  * include/linux/spinlock_api_smp.h
diff --git a/kernel/module.c b/kernel/module.c
index c353707..79b3790 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -878,7 +878,7 @@ static inline void print_unload_info(struct seq_file *m, struct module *mod)
 	seq_printf(m, " %lu ", module_refcount(mod));
 
 	/* Always include a trailing , so userspace can differentiate
-           between this and the old multi-field proc format. */
+	   between this and the old multi-field proc format. */
 	list_for_each_entry(use, &mod->source_list, source_list) {
 		printed_something = 1;
 		seq_printf(m, "%s,", use->source->name);
@@ -947,7 +947,7 @@ bool try_module_get(struct module *module)
 	bool ret = true;
 
 	if (module) {
-		preempt_disable();
+		unsigned long flags = hard_preempt_disable();
 
 		if (likely(module_is_live(module))) {
 			__this_cpu_inc(module->refptr->incs);
@@ -955,7 +955,7 @@ bool try_module_get(struct module *module)
 		} else
 			ret = false;
 
-		preempt_enable();
+		hard_preempt_enable(flags);
 	}
 	return ret;
 }
@@ -964,12 +964,12 @@ EXPORT_SYMBOL(try_module_get);
 void module_put(struct module *module)
 {
 	if (module) {
-		preempt_disable();
+		unsigned long flags = hard_preempt_disable();
 		smp_wmb(); /* see comment in module_refcount */
 		__this_cpu_inc(module->refptr->decs);
 
 		trace_module_put(module, _RET_IP_);
-		preempt_enable();
+		hard_preempt_enable(flags);
 	}
 }
 EXPORT_SYMBOL(module_put);
@@ -1131,7 +1131,7 @@ static unsigned long maybe_relocated(unsigned long crc,
 static int check_version(Elf_Shdr *sechdrs,
 			 unsigned int versindex,
 			 const char *symname,
-			 struct module *mod, 
+			 struct module *mod,
 			 const unsigned long *crc,
 			 const struct module *crc_owner)
 {
@@ -1200,7 +1200,7 @@ static inline int same_magic(const char *amagic, const char *bmagic,
 static inline int check_version(Elf_Shdr *sechdrs,
 				unsigned int versindex,
 				const char *symname,
-				struct module *mod, 
+				struct module *mod,
 				const unsigned long *crc,
 				const struct module *crc_owner)
 {
@@ -2259,7 +2259,7 @@ static char elf_type(const Elf_Sym *sym, const struct load_info *info)
 }
 
 static bool is_core_symbol(const Elf_Sym *src, const Elf_Shdr *sechdrs,
-                           unsigned int shnum)
+			   unsigned int shnum)
 {
 	const Elf_Shdr *sec;
 
@@ -3024,7 +3024,7 @@ static int do_init_module(struct module *mod)
 		ret = do_one_initcall(mod->init);
 	if (ret < 0) {
 		/* Init routine failed: abort.  Try to protect us from
-                   buggy refcounters. */
+		   buggy refcounters. */
 		mod->state = MODULE_STATE_GOING;
 		synchronize_sched();
 		module_put(mod);
@@ -3176,7 +3176,7 @@ out:
 
 static int unknown_module_param_cb(char *param, char *val, const char *modname)
 {
-	/* Check for magic 'dyndbg' arg */ 
+	/* Check for magic 'dyndbg' arg */
 	int ret = ddebug_dyndbg_module_param_cb(param, val, modname);
 	if (ret != 0)
 		pr_warn("%s: unknown parameter '%s' ignored\n", modname, param);
diff --git a/kernel/panic.c b/kernel/panic.c
index cf80672..e85d686 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -23,6 +23,7 @@
 #include <linux/sysrq.h>
 #include <linux/init.h>
 #include <linux/nmi.h>
+#include <linux/ipipe_trace.h>
 
 #define PANIC_TIMER_STEP 100
 #define PANIC_BLINK_SPD 18
@@ -373,6 +374,8 @@ void oops_enter(void)
 {
 	tracing_off();
 	/* can't trust the integrity of the kernel anymore: */
+	ipipe_trace_panic_freeze();
+	ipipe_disable_context_check();
 	debug_locks_off();
 	do_oops_enter_exit();
 }
diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c
index 1f35a34..4b3828b 100644
--- a/kernel/power/hibernate.c
+++ b/kernel/power/hibernate.c
@@ -286,6 +286,7 @@ static int create_image(int platform_mode)
 		goto Enable_cpus;
 
 	local_irq_disable();
+	hard_cond_local_irq_disable();
 
 	error = syscore_suspend();
 	if (error) {
@@ -439,6 +440,7 @@ static int resume_target_kernel(bool platform_mode)
 		goto Enable_cpus;
 
 	local_irq_disable();
+	hard_cond_local_irq_disable();
 
 	error = syscore_suspend();
 	if (error)
@@ -557,6 +559,7 @@ int hibernation_platform_enter(void)
 		goto Platform_finish;
 
 	local_irq_disable();
+	hard_cond_local_irq_disable();
 	syscore_suspend();
 	if (pm_wakeup_pending()) {
 		error = -EAGAIN;
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index 3b9f01b..343b6c2 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -1804,6 +1804,43 @@ asmlinkage int printk_emit(int facility, int level,
 }
 EXPORT_SYMBOL(printk_emit);
 
+#ifdef CONFIG_IPIPE
+
+extern int __ipipe_printk_bypass;
+
+static IPIPE_DEFINE_SPINLOCK(__ipipe_printk_lock);
+
+static int __ipipe_printk_fill;
+
+static char __ipipe_printk_buf[__LOG_BUF_LEN];
+
+void __ipipe_flush_printk (unsigned virq, void *cookie)
+{
+	char *p = __ipipe_printk_buf;
+	int len, lmax, out = 0;
+	unsigned long flags;
+
+	goto start;
+
+	do {
+		spin_unlock_irqrestore(&__ipipe_printk_lock, flags);
+ start:
+		lmax = __ipipe_printk_fill;
+		while (out < lmax) {
+			len = strlen(p) + 1;
+			printk("%s",p);
+			p += len;
+			out += len;
+		}
+		spin_lock_irqsave(&__ipipe_printk_lock, flags);
+	}
+	while (__ipipe_printk_fill != lmax);
+
+	__ipipe_printk_fill = 0;
+
+	spin_unlock_irqrestore(&__ipipe_printk_lock, flags);
+}
+
 /**
  * printk - print a kernel message
  * @fmt: format string
@@ -1827,6 +1864,59 @@ EXPORT_SYMBOL(printk_emit);
  */
 asmlinkage __visible int printk(const char *fmt, ...)
 {
+	int sprintk = 1, cs = -1;
+	int r, fbytes, oldcount;
+	unsigned long flags;
+	va_list args;
+
+	va_start(args, fmt);
+
+	flags = hard_local_irq_save();
+
+	if (__ipipe_printk_bypass || oops_in_progress)
+		cs = ipipe_disable_context_check();
+	else if (__ipipe_current_domain == ipipe_root_domain) {
+		if (ipipe_head_domain != ipipe_root_domain && 
+		    (raw_irqs_disabled_flags(flags) ||
+		     test_bit(IPIPE_STALL_FLAG, &__ipipe_head_status)))
+			sprintk = 0;
+	} else
+		sprintk = 0;
+
+	hard_local_irq_restore(flags);
+
+	if (sprintk) {
+		r = vprintk(fmt, args);
+		if (cs != -1)
+			ipipe_restore_context_check(cs);
+		goto out;
+	}
+
+	spin_lock_irqsave(&__ipipe_printk_lock, flags);
+
+	oldcount = __ipipe_printk_fill;
+	fbytes = __LOG_BUF_LEN - oldcount;
+	if (fbytes > 1)	{
+		r = vscnprintf(__ipipe_printk_buf + __ipipe_printk_fill,
+			       fbytes, fmt, args) + 1;
+		__ipipe_printk_fill += r;
+	} else
+		r = 0;
+
+	spin_unlock_irqrestore(&__ipipe_printk_lock, flags);
+
+	if (oldcount == 0)
+		ipipe_raise_irq(__ipipe_printk_virq);
+out:
+	va_end(args);
+
+	return r;
+}
+
+#else /* !CONFIG_IPIPE */
+
+asmlinkage int printk(const char *fmt, ...)
+{
 	va_list args;
 	int r;
 
@@ -1844,6 +1934,8 @@ asmlinkage __visible int printk(const char *fmt, ...)
 
 	return r;
 }
+#endif /* CONFIG_IPIPE */
+
 EXPORT_SYMBOL(printk);
 
 #else /* CONFIG_PRINTK */
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 6810e57..fdc4ae3 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1598,7 +1598,9 @@ void scheduler_ipi(void)
 	 * however a fair share of IPIs are still resched only so this would
 	 * somewhat pessimize the simple resched case.
 	 */
+#ifndef IPIPE_ARCH_HAVE_VIRQ_IPI
 	irq_enter();
+#endif
 	sched_ttwu_pending();
 
 	/*
@@ -1608,7 +1610,9 @@ void scheduler_ipi(void)
 		this_rq()->idle_balance = 1;
 		raise_softirq_irqoff(SCHED_SOFTIRQ);
 	}
+#ifndef IPIPE_ARCH_HAVE_VIRQ_IPI
 	irq_exit();
+#endif
 }
 
 static void ttwu_queue_remote(struct task_struct *p, int cpu)
@@ -1699,7 +1703,8 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
 	 */
 	smp_mb__before_spinlock();
 	raw_spin_lock_irqsave(&p->pi_lock, flags);
-	if (!(p->state & state))
+	if (!(p->state & state) ||
+	    (p->state & (TASK_NOWAKEUP|TASK_HARDENING)))
 		goto out;
 
 	success = 1; /* we're going to change ->state */
@@ -2321,6 +2326,8 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev)
 {
 	struct rq *rq = this_rq();
 
+	__ipipe_complete_domain_migration();
+
 	finish_task_switch(rq, prev);
 
 	/*
@@ -2337,7 +2344,7 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev)
  * context_switch - switch to the new MM and the new
  * thread's register state.
  */
-static inline void
+static inline int
 context_switch(struct rq *rq, struct task_struct *prev,
 	       struct task_struct *next)
 {
@@ -2378,12 +2385,18 @@ context_switch(struct rq *rq, struct task_struct *prev,
 	switch_to(prev, next, prev);
 
 	barrier();
+
+	if (unlikely(__ipipe_switch_tail()))
+		return 1;
+
 	/*
 	 * this_rq must be evaluated again because prev may have moved
 	 * CPUs since it called schedule(), thus the 'rq' on its stack
 	 * frame will be invalid.
 	 */
 	finish_task_switch(this_rq(), prev);
+
+	return 0;
 }
 
 /*
@@ -2597,6 +2610,7 @@ notrace unsigned long get_parent_ip(unsigned long addr)
 
 void preempt_count_add(int val)
 {
+	ipipe_preempt_root_only();
 #ifdef CONFIG_DEBUG_PREEMPT
 	/*
 	 * Underflow?
@@ -2625,6 +2639,7 @@ NOKPROBE_SYMBOL(preempt_count_add);
 
 void preempt_count_sub(int val)
 {
+	ipipe_preempt_root_only();
 #ifdef CONFIG_DEBUG_PREEMPT
 	/*
 	 * Underflow?
@@ -2679,6 +2694,7 @@ static noinline void __schedule_bug(struct task_struct *prev)
  */
 static inline void schedule_debug(struct task_struct *prev)
 {
+	ipipe_root_only();
 #ifdef CONFIG_SCHED_STACK_END_CHECK
 	BUG_ON(unlikely(task_stack_end_corrupted(prev)));
 #endif
@@ -2772,7 +2788,7 @@ again:
  *          - return from syscall or exception to user-space
  *          - return from interrupt-handler to user-space
  */
-static void __sched __schedule(void)
+static int __sched __schedule(void)
 {
 	struct task_struct *prev, *next;
 	unsigned long *switch_count;
@@ -2786,6 +2802,10 @@ need_resched:
 	rcu_note_context_switch(cpu);
 	prev = rq->curr;
 
+ 	if (unlikely(prev->state & TASK_HARDENING))
+		/* Pop one disable level -- one still remains. */
+		preempt_enable();
+
 	schedule_debug(prev);
 
 	if (sched_feat(HRTICK))
@@ -2836,7 +2856,8 @@ need_resched:
 		rq->curr = next;
 		++*switch_count;
 
-		context_switch(rq, prev, next); /* unlocks the rq */
+ 		if (context_switch(rq, prev, next)) /* unlocks the rq */
+  			return 1; /* task hijacked by head domain */
 		/*
 		 * The context switch have flipped the stack from under us
 		 * and restored the local variables which were saved when
@@ -2845,14 +2866,18 @@ need_resched:
 		 */
 		cpu = smp_processor_id();
 		rq = cpu_rq(cpu);
-	} else
+	} else {
+  		prev->state &= ~TASK_HARDENING;
 		raw_spin_unlock_irq(&rq->lock);
+	}
 
 	post_schedule(rq);
 
 	sched_preempt_enable_no_resched();
 	if (need_resched())
 		goto need_resched;
+
+	return 0;
 }
 
 static inline void sched_submit_work(struct task_struct *tsk)
@@ -2919,12 +2944,13 @@ asmlinkage __visible void __sched notrace preempt_schedule(void)
 	 * If there is a non-zero preempt_count or interrupts are disabled,
 	 * we do not want to preempt the current task. Just return..
 	 */
-	if (likely(!preemptible()))
+	if (likely(!preemptible() || !ipipe_root_p))
 		return;
 
 	do {
 		__preempt_count_add(PREEMPT_ACTIVE);
-		__schedule();
+		if (__schedule())
+			return;
 		__preempt_count_sub(PREEMPT_ACTIVE);
 
 		/*
@@ -3628,6 +3654,7 @@ change:
 
 	prev_class = p->sched_class;
 	__setscheduler(rq, p, attr, true);
+	__ipipe_report_setsched(p);
 
 	if (running)
 		p->sched_class->set_curr_task(rq);
@@ -4747,10 +4774,13 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
 	do_set_cpus_allowed(p, new_mask);
 
 	/* Can the task run on the task's current CPU? If so, we're done */
-	if (cpumask_test_cpu(task_cpu(p), new_mask))
+	if (cpumask_test_cpu(task_cpu(p), new_mask)) {
+		__ipipe_report_setaffinity(p, task_cpu(p));
 		goto out;
+	}
 
 	dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
+	__ipipe_report_setaffinity(p, dest_cpu);
 	if (task_running(rq, p) || p->state == TASK_WAKING) {
 		struct migration_arg arg = { p, dest_cpu };
 		/* Need help from migration thread: drop lock and wait. */
@@ -8269,3 +8299,42 @@ void dump_cpu_task(int cpu)
 	pr_info("Task dump for CPU %d:\n", cpu);
 	sched_show_task(cpu_curr(cpu));
 }
+
+#ifdef CONFIG_IPIPE
+
+int __ipipe_migrate_head(void)
+{
+	struct task_struct *p = current;
+
+	preempt_disable();
+
+	IPIPE_WARN_ONCE(__this_cpu_read(ipipe_percpu.task_hijacked) != NULL);
+
+	__this_cpu_write(ipipe_percpu.task_hijacked, p);
+	set_current_state(TASK_INTERRUPTIBLE | TASK_HARDENING);
+	sched_submit_work(p);
+	if (likely(__schedule()))
+		return 0;
+
+	if (signal_pending(p))
+		return -ERESTARTSYS;
+
+	BUG();
+}
+EXPORT_SYMBOL_GPL(__ipipe_migrate_head);
+
+void __ipipe_reenter_root(void)
+{
+	struct rq *rq = this_rq();
+	struct task_struct *p;
+
+	p = __this_cpu_read(ipipe_percpu.rqlock_owner);
+	BUG_ON(p == NULL);
+	ipipe_clear_thread_flag(TIP_HEAD);
+	finish_task_switch(rq, p);
+	post_schedule(rq);
+	sched_preempt_enable_no_resched();
+}
+EXPORT_SYMBOL_GPL(__ipipe_reenter_root);
+
+#endif /* CONFIG_IPIPE */
diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c
index 4dae188..992aeaa 100644
--- a/kernel/sched/wait.c
+++ b/kernel/sched/wait.c
@@ -66,6 +66,8 @@ static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
 {
 	wait_queue_t *curr, *next;
 
+	ipipe_root_only();
+
 	list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
 		unsigned flags = curr->flags;
 
diff --git a/kernel/signal.c b/kernel/signal.c
index 8f0876f..04f0b2d 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -690,6 +690,10 @@ int dequeue_signal(struct task_struct *tsk, sigset_t *mask, siginfo_t *info)
 void signal_wake_up_state(struct task_struct *t, unsigned int state)
 {
 	set_tsk_thread_flag(t, TIF_SIGPENDING);
+
+	/* TIF_SIGPENDING must be prior to reporting. */
+	__ipipe_report_sigwake(t);
+
 	/*
 	 * TASK_WAKEKILL also means wake it up in the stopped/traced/killable
 	 * case. We don't check t->state here because there is a race with it
@@ -913,8 +917,11 @@ static inline int wants_signal(int sig, struct task_struct *p)
 		return 0;
 	if (sig == SIGKILL)
 		return 1;
-	if (task_is_stopped_or_traced(p))
+	if (task_is_stopped_or_traced(p)) {
+		if (!signal_pending(p))
+			__ipipe_report_sigwake(p);
 		return 0;
+	}
 	return task_curr(p) || !signal_pending(p);
 }
 
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c
index 5544990..51a14f9 100644
--- a/kernel/time/clockevents.c
+++ b/kernel/time/clockevents.c
@@ -17,6 +17,7 @@
 #include <linux/module.h>
 #include <linux/smp.h>
 #include <linux/device.h>
+#include <linux/ipipe_tickdev.h>
 
 #include "tick-internal.h"
 
@@ -382,6 +383,9 @@ void clockevents_register_device(struct clock_event_device *dev)
 	unsigned long flags;
 
 	BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED);
+
+	ipipe_host_timer_register(dev);
+
 	if (!dev->cpumask) {
 		WARN_ON(num_possible_cpus() > 1);
 		dev->cpumask = cpumask_of(smp_processor_id());
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index 2e949cc..c250d08 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -30,6 +30,7 @@
 #include <linux/sched.h> /* for spin_unlock_irq() using preempt_count() m68k */
 #include <linux/tick.h>
 #include <linux/kthread.h>
+#include <linux/kallsyms.h>
 
 #include "tick-internal.h"
 #include "timekeeping_internal.h"
@@ -253,6 +254,9 @@ static void clocksource_watchdog(unsigned long data)
 	cycle_t csnow, wdnow, delta;
 	int64_t wd_nsec, cs_nsec;
 	int next_cpu, reset_pending;
+#ifdef CONFIG_IPIPE
+	cycle_t wdref;
+#endif
 
 	spin_lock(&watchdog_lock);
 	if (!watchdog_running)
@@ -269,11 +273,24 @@ static void clocksource_watchdog(unsigned long data)
 			continue;
 		}
 
+#ifdef CONFIG_IPIPE
+retry:
+#endif
 		local_irq_disable();
+#ifdef CONFIG_IPIPE
+		wdref = watchdog->read(watchdog);
+#endif
 		csnow = cs->read(cs);
 		wdnow = watchdog->read(watchdog);
 		local_irq_enable();
 
+#ifdef CONFIG_IPIPE
+		wd_nsec = clocksource_cyc2ns((wdnow - wdref) & watchdog->mask,
+					     watchdog->mult, watchdog->shift);
+		if (wd_nsec > WATCHDOG_THRESHOLD)
+			goto retry;
+#endif
+
 		/* Clocksource initialized ? */
 		if (!(cs->flags & CLOCK_SOURCE_WATCHDOG) ||
 		    atomic_read(&watchdog_reset_pending)) {
@@ -708,6 +725,95 @@ static int __init clocksource_done_booting(void)
 }
 fs_initcall(clocksource_done_booting);
 
+#ifdef CONFIG_IPIPE_WANT_CLOCKSOURCE
+unsigned long long __ipipe_cs_freq;
+EXPORT_SYMBOL_GPL(__ipipe_cs_freq);
+
+struct clocksource *__ipipe_cs;
+EXPORT_SYMBOL_GPL(__ipipe_cs);
+
+cycle_t (*__ipipe_cs_read)(struct clocksource *cs);
+cycle_t __ipipe_cs_last_tsc;
+cycle_t __ipipe_cs_mask;
+unsigned __ipipe_cs_lat = 0xffffffff;
+
+static void ipipe_check_clocksource(struct clocksource *cs)
+{
+	cycle_t (*cread)(struct clocksource *cs);
+	cycle_t lat, mask, saved;
+	unsigned long long freq;
+	unsigned long flags;
+	unsigned i;
+
+	if (cs->ipipe_read) {
+		mask = CLOCKSOURCE_MASK(64);
+		cread = cs->ipipe_read;
+	} else {
+		mask = cs->mask;
+		cread = cs->read;
+
+		if ((cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) == 0)
+			return;
+
+		/*
+		 * We only support masks such that cs->mask + 1 is a power of 2,
+		 * 64 bits masks or masks lesser than 32 bits
+		 */
+		if (mask != CLOCKSOURCE_MASK(64)
+		    && ((mask & (mask + 1)) != 0 || mask > 0xffffffff))
+			return;
+	}
+
+	/*
+	 * We prefer a clocksource with a better resolution than 1us
+	 */
+	if (cs->shift <= 34) {
+		freq = 1000000000ULL << cs->shift;
+		do_div(freq, cs->mult);
+	} else {
+		freq = 1000000ULL << cs->shift;
+		do_div(freq, cs->mult);
+		freq *= 1000;
+	}
+	if (freq < 1000000)
+		return;
+
+	/* Measure the clocksource latency */
+	flags = hard_local_irq_save();
+	saved = __ipipe_cs_last_tsc;
+	lat = cread(cs);
+	for (i = 0; i < 10; i++)
+		cread(cs);
+	lat = cread(cs) - lat;
+	__ipipe_cs_last_tsc = saved;
+	hard_local_irq_restore(flags);
+	lat = (lat * cs->mult) >> cs->shift;
+	do_div(lat, i + 1);
+
+	if (!strcmp(cs->name, override_name))
+		goto skip_tests;
+
+	if (lat > __ipipe_cs_lat)
+		return;
+
+	if (__ipipe_cs && !strcmp(__ipipe_cs->name, override_name))
+		return;
+
+  skip_tests:
+	flags = hard_local_irq_save();
+	if (__ipipe_cs_last_tsc == 0) {
+		__ipipe_cs_lat = lat;
+		__ipipe_cs_freq = freq;
+		__ipipe_cs = cs;
+		__ipipe_cs_read = cread;
+		__ipipe_cs_mask = mask;
+	}
+	hard_local_irq_restore(flags);
+}
+#else /* !CONFIG_IPIPE_WANT_CLOCKSOURCE */
+#define ipipe_check_clocksource(cs)	do { }while (0)
+#endif /* !CONFIG_IPIPE_WANT_CLOCKSOURCE */
+
 /*
  * Enqueue the clocksource sorted by rating
  */
@@ -721,6 +827,8 @@ static void clocksource_enqueue(struct clocksource *cs)
 		if (tmp->rating >= cs->rating)
 			entry = &tmp->list;
 	list_add(&cs->list, entry);
+
+	ipipe_check_clocksource(cs);
 }
 
 /**
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
index 7efeedf..94a8c21 100644
--- a/kernel/time/tick-common.c
+++ b/kernel/time/tick-common.c
@@ -88,7 +88,7 @@ static void tick_periodic(int cpu)
 		update_wall_time();
 	}
 
-	update_process_times(user_mode(get_irq_regs()));
+	update_root_process_times(get_irq_regs());
 	profile_tick(CPU_PROFILING);
 }
 
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 8c30ef7..0393d10 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -148,7 +148,7 @@ static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
 			ts->idle_jiffies++;
 	}
 #endif
-	update_process_times(user_mode(regs));
+	update_root_process_times(regs);
 	profile_tick(CPU_PROFILING);
 }
 
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index ec1791f..e724267 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -343,7 +343,7 @@ static inline void update_vsyscall(struct timekeeper *tk)
 	xt = timespec64_to_timespec(tk_xtime(tk));
 	wm = timespec64_to_timespec(tk->wall_to_monotonic);
 	update_vsyscall_old(&xt, &wm, tk->tkr.clock, tk->tkr.mult,
-			    tk->tkr.cycle_last);
+			    tk->tkr.shift, tk->tkr.cycle_last);
 }
 
 static inline void old_vsyscall_fixup(struct timekeeper *tk)
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 3260ffd..76c9b92 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -1391,6 +1391,25 @@ void update_process_times(int user_tick)
 	run_posix_cpu_timers(p);
 }
 
+#ifdef CONFIG_IPIPE
+
+void update_root_process_times(struct pt_regs *regs)
+{
+	int cpu, user_tick = user_mode(regs);
+
+	if (__ipipe_root_tick_p(regs)) {
+		update_process_times(user_tick);
+		return;
+	}
+
+	run_local_timers();
+	cpu = smp_processor_id();
+	rcu_check_callbacks(cpu, user_tick);
+	run_posix_cpu_timers(current);
+}
+
+#endif
+
 /*
  * This function runs timers and the timer-tq in bottom half context.
  */
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index a5da09c..6650799 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -439,6 +439,7 @@ config DYNAMIC_FTRACE
 	bool "enable/disable function tracing dynamically"
 	depends on FUNCTION_TRACER
 	depends on HAVE_DYNAMIC_FTRACE
+	depends on !IPIPE
 	default y
 	help
 	  This option will modify all the calls to function tracing
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index d1eff3d..f8b9472 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -32,6 +32,7 @@
 #include <linux/list.h>
 #include <linux/hash.h>
 #include <linux/rcupdate.h>
+#include <linux/ipipe.h>
 
 #include <trace/events/sched.h>
 
@@ -251,8 +252,17 @@ static inline void update_function_graph_func(void) { }
 
 static void update_ftrace_function(void)
 {
+	struct ftrace_ops *ops;
 	ftrace_func_t func;
 
+	for (ops = ftrace_ops_list;
+	     ops != &ftrace_list_end; ops = ops->next)
+		if (ops->flags & FTRACE_OPS_FL_IPIPE_EXCLUSIVE) {
+			set_function_trace_op = ops;
+			func = ops->func;
+			goto set_pointers;
+		}
+
 	/*
 	 * Prepare the ftrace_ops that the arch callback will use.
 	 * If there's only one ftrace_ops registered, the ftrace_ops_list
@@ -280,6 +290,7 @@ static void update_ftrace_function(void)
 
 	update_function_graph_func();
 
+  set_pointers:
 	/* If there's no change, then do nothing more here */
 	if (ftrace_trace_function == func)
 		return;
@@ -2298,6 +2309,9 @@ void __weak arch_ftrace_update_code(int command)
 
 static void ftrace_run_update_code(int command)
 {
+#ifdef CONFIG_IPIPE
+	unsigned long flags;
+#endif /* CONFIG_IPIPE */
 	int ret;
 
 	ret = ftrace_arch_code_modify_prepare();
@@ -2311,7 +2325,13 @@ static void ftrace_run_update_code(int command)
 	 * is safe. The stop_machine() is the safest, but also
 	 * produces the most overhead.
 	 */
+#ifdef CONFIG_IPIPE
+	flags = ipipe_critical_enter(NULL);
+	__ftrace_modify_code(&command);
+	ipipe_critical_exit(flags);
+#else  /* !CONFIG_IPIPE */
 	arch_ftrace_update_code(command);
+#endif /* !CONFIG_IPIPE */
 
 	ret = ftrace_arch_code_modify_post_process();
 	FTRACE_WARN_ON(ret);
@@ -4621,10 +4641,10 @@ static int ftrace_process_locs(struct module *mod,
 	 * reason to cause large interrupt latencies while we do it.
 	 */
 	if (!mod)
-		local_irq_save(flags);
+		flags = hard_local_irq_save();
 	ftrace_update_code(mod, start_pg);
 	if (!mod)
-		local_irq_restore(flags);
+		hard_local_irq_restore(flags);
 	ret = 0;
  out:
 	mutex_unlock(&ftrace_lock);
@@ -4723,9 +4743,11 @@ void __init ftrace_init(void)
 	unsigned long count, flags;
 	int ret;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save_notrace();
 	ret = ftrace_dyn_arch_init();
-	local_irq_restore(flags);
+	hard_local_irq_restore_notrace(flags);
+
+	/* ftrace_dyn_arch_init places the return code in addr */
 	if (ret)
 		goto failed;
 
@@ -4891,7 +4913,16 @@ __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
 		}
 	} while_for_each_ftrace_op(op);
 out:
-	preempt_enable_notrace();
+#ifdef CONFIG_IPIPE
+	if (hard_irqs_disabled() || !__ipipe_root_p)
+		/*
+		 * Nothing urgent to schedule here. At latest the timer tick
+		 * will pick up whatever the tracing functions kicked off.
+		 */
+		preempt_enable_no_resched_notrace();
+	else
+#endif
+		preempt_enable_notrace();
 	trace_clear_recursion(bit);
 }
 
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 0fc5cfe..33055b3 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -2684,7 +2684,8 @@ static DEFINE_PER_CPU(unsigned int, current_context);
 
 static __always_inline int trace_recursive_lock(void)
 {
-	unsigned int val = __this_cpu_read(current_context);
+	unsigned long flags;
+	unsigned int val;
 	int bit;
 
 	if (in_interrupt()) {
@@ -2697,21 +2698,34 @@ static __always_inline int trace_recursive_lock(void)
 	} else
 		bit = 3;
 
-	if (unlikely(val & (1 << bit)))
+	flags = hard_local_irq_save();
+
+	val = __this_cpu_read(current_context);
+	if (unlikely(val & (1 << bit))) {
+		hard_local_irq_restore(flags);
 		return 1;
+	}
 
 	val |= (1 << bit);
 	__this_cpu_write(current_context, val);
 
+	hard_local_irq_restore(flags);
+
 	return 0;
 }
 
 static __always_inline void trace_recursive_unlock(void)
 {
-	unsigned int val = __this_cpu_read(current_context);
+	unsigned long flags;
+	unsigned int val;
+
+	flags = hard_local_irq_save();
 
+	val = __this_cpu_read(current_context);
 	val &= val & (val - 1);
 	__this_cpu_write(current_context, val);
+
+	hard_local_irq_restore(flags);
 }
 
 #else
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 72c7134..da07f49 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -2092,8 +2092,9 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
 	/* Don't pollute graph traces with trace_vprintk internals */
 	pause_graph_tracing();
 
+	flags = hard_local_irq_save();
+
 	pc = preempt_count();
-	preempt_disable_notrace();
 
 	tbuffer = get_trace_buf();
 	if (!tbuffer) {
@@ -2106,7 +2107,6 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
 	if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
 		goto out;
 
-	local_save_flags(flags);
 	size = sizeof(*entry) + sizeof(u32) * len;
 	buffer = tr->trace_buffer.buffer;
 	event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
@@ -2124,7 +2124,7 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
 	}
 
 out:
-	preempt_enable_notrace();
+	hard_local_irq_restore(flags);
 	unpause_graph_tracing();
 
 	return len;
diff --git a/kernel/trace/trace_clock.c b/kernel/trace/trace_clock.c
index 57b67b1..66334fa 100644
--- a/kernel/trace/trace_clock.c
+++ b/kernel/trace/trace_clock.c
@@ -93,7 +93,7 @@ u64 notrace trace_clock_global(void)
 	int this_cpu;
 	u64 now;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save_notrace();
 
 	this_cpu = raw_smp_processor_id();
 	now = sched_clock_cpu(this_cpu);
@@ -119,7 +119,7 @@ u64 notrace trace_clock_global(void)
 	arch_spin_unlock(&trace_clock_struct.lock);
 
  out:
-	local_irq_restore(flags);
+	hard_local_irq_restore_notrace(flags);
 
 	return now;
 }
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c
index 57f0ec9..80437ac 100644
--- a/kernel/trace/trace_functions.c
+++ b/kernel/trace/trace_functions.c
@@ -171,7 +171,7 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
 	 * Need to use raw, since this must be called before the
 	 * recursive protection is performed.
 	 */
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	cpu = raw_smp_processor_id();
 	data = per_cpu_ptr(tr->trace_buffer.data, cpu);
 	disabled = atomic_inc_return(&data->disabled);
@@ -191,7 +191,7 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
 	}
 
 	atomic_dec(&data->disabled);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 static struct tracer_opt func_opts[] = {
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 2964333..7b15e42 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -336,7 +336,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace)
 	if (ftrace_graph_notrace_addr(trace->func))
 		return 1;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save_notrace();
 	cpu = raw_smp_processor_id();
 	data = per_cpu_ptr(tr->trace_buffer.data, cpu);
 	disabled = atomic_inc_return(&data->disabled);
@@ -348,7 +348,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace)
 	}
 
 	atomic_dec(&data->disabled);
-	local_irq_restore(flags);
+	hard_local_irq_restore_notrace(flags);
 
 	return ret;
 }
@@ -421,7 +421,7 @@ void trace_graph_return(struct ftrace_graph_ret *trace)
 	int cpu;
 	int pc;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save_notrace();
 	cpu = raw_smp_processor_id();
 	data = per_cpu_ptr(tr->trace_buffer.data, cpu);
 	disabled = atomic_inc_return(&data->disabled);
@@ -430,7 +430,7 @@ void trace_graph_return(struct ftrace_graph_ret *trace)
 		__trace_graph_return(tr, trace, flags, pc);
 	}
 	atomic_dec(&data->disabled);
-	local_irq_restore(flags);
+	hard_local_irq_restore_notrace(flags);
 }
 
 void set_graph_array(struct trace_array *tr)
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 4e35a5d..f40e14c 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -336,6 +336,7 @@ config MAGIC_SYSRQ
 	  keys are documented in <file:Documentation/sysrq.txt>. Don't say Y
 	  unless you really know what this hack does.
 
+
 config MAGIC_SYSRQ_DEFAULT_ENABLE
 	hex "Enable magic SysRq key functions by default"
 	depends on MAGIC_SYSRQ
@@ -345,6 +346,8 @@ config MAGIC_SYSRQ_DEFAULT_ENABLE
 	  This may be set to 1 or 0 to enable or disable them all, or
 	  to a bitmask as described in Documentation/sysrq.txt.
 
+source "kernel/ipipe/Kconfig.debug"
+
 config DEBUG_KERNEL
 	bool "Kernel debugging"
 	help
@@ -617,7 +620,7 @@ config HAVE_DEBUG_STACKOVERFLOW
 
 config DEBUG_STACKOVERFLOW
 	bool "Check for stack overflows"
-	depends on DEBUG_KERNEL && HAVE_DEBUG_STACKOVERFLOW
+	depends on DEBUG_KERNEL && HAVE_DEBUG_STACKOVERFLOW && !IPIPE_LEGACY
 	---help---
 	  Say Y here if you want to check for overflows of kernel, IRQ
 	  and exception stacks (if your archicture uses them). This
@@ -1152,7 +1155,7 @@ menu "RCU Debugging"
 
 config PROVE_RCU
 	bool "RCU debugging: prove RCU correctness"
-	depends on PROVE_LOCKING
+	depends on PROVE_LOCKING && !IPIPE
 	default n
 	help
 	 This feature enables lockdep extensions that check for correct
diff --git a/lib/atomic64.c b/lib/atomic64.c
index 1298c05e..6ef2528 100644
--- a/lib/atomic64.c
+++ b/lib/atomic64.c
@@ -29,15 +29,15 @@
  * Ensure each lock is in a separate cacheline.
  */
 static union {
-	raw_spinlock_t lock;
+	ipipe_spinlock_t lock;
 	char pad[L1_CACHE_BYTES];
 } atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = {
 	[0 ... (NR_LOCKS - 1)] = {
-		.lock =  __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock),
+		.lock =  IPIPE_SPIN_LOCK_UNLOCKED,
 	},
 };
 
-static inline raw_spinlock_t *lock_addr(const atomic64_t *v)
+static inline ipipe_spinlock_t *lock_addr(const atomic64_t *v)
 {
 	unsigned long addr = (unsigned long) v;
 
@@ -49,7 +49,7 @@ static inline raw_spinlock_t *lock_addr(const atomic64_t *v)
 long long atomic64_read(const atomic64_t *v)
 {
 	unsigned long flags;
-	raw_spinlock_t *lock = lock_addr(v);
+	ipipe_spinlock_t *lock = lock_addr(v);
 	long long val;
 
 	raw_spin_lock_irqsave(lock, flags);
@@ -62,7 +62,7 @@ EXPORT_SYMBOL(atomic64_read);
 void atomic64_set(atomic64_t *v, long long i)
 {
 	unsigned long flags;
-	raw_spinlock_t *lock = lock_addr(v);
+	ipipe_spinlock_t *lock = lock_addr(v);
 
 	raw_spin_lock_irqsave(lock, flags);
 	v->counter = i;
@@ -74,7 +74,7 @@ EXPORT_SYMBOL(atomic64_set);
 void atomic64_##op(long long a, atomic64_t *v)				\
 {									\
 	unsigned long flags;						\
-	raw_spinlock_t *lock = lock_addr(v);				\
+	ipipe_spinlock_t *lock = lock_addr(v);				\
 									\
 	raw_spin_lock_irqsave(lock, flags);				\
 	v->counter c_op a;						\
@@ -86,7 +86,7 @@ EXPORT_SYMBOL(atomic64_##op);
 long long atomic64_##op##_return(long long a, atomic64_t *v)		\
 {									\
 	unsigned long flags;						\
-	raw_spinlock_t *lock = lock_addr(v);				\
+	ipipe_spinlock_t *lock = lock_addr(v);				\
 	long long val;							\
 									\
 	raw_spin_lock_irqsave(lock, flags);				\
@@ -110,7 +110,7 @@ ATOMIC64_OPS(sub, -=)
 long long atomic64_dec_if_positive(atomic64_t *v)
 {
 	unsigned long flags;
-	raw_spinlock_t *lock = lock_addr(v);
+	ipipe_spinlock_t *lock = lock_addr(v);
 	long long val;
 
 	raw_spin_lock_irqsave(lock, flags);
@@ -125,7 +125,7 @@ EXPORT_SYMBOL(atomic64_dec_if_positive);
 long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n)
 {
 	unsigned long flags;
-	raw_spinlock_t *lock = lock_addr(v);
+	ipipe_spinlock_t *lock = lock_addr(v);
 	long long val;
 
 	raw_spin_lock_irqsave(lock, flags);
@@ -140,7 +140,7 @@ EXPORT_SYMBOL(atomic64_cmpxchg);
 long long atomic64_xchg(atomic64_t *v, long long new)
 {
 	unsigned long flags;
-	raw_spinlock_t *lock = lock_addr(v);
+	ipipe_spinlock_t *lock = lock_addr(v);
 	long long val;
 
 	raw_spin_lock_irqsave(lock, flags);
@@ -154,7 +154,7 @@ EXPORT_SYMBOL(atomic64_xchg);
 int atomic64_add_unless(atomic64_t *v, long long a, long long u)
 {
 	unsigned long flags;
-	raw_spinlock_t *lock = lock_addr(v);
+	ipipe_spinlock_t *lock = lock_addr(v);
 	int ret = 0;
 
 	raw_spin_lock_irqsave(lock, flags);
diff --git a/lib/bust_spinlocks.c b/lib/bust_spinlocks.c
index f8e0e53..02175aa3 100644
--- a/lib/bust_spinlocks.c
+++ b/lib/bust_spinlocks.c
@@ -14,6 +14,7 @@
 #include <linux/wait.h>
 #include <linux/vt_kern.h>
 #include <linux/console.h>
+#include <linux/ipipe_trace.h>
 
 
 void __attribute__((weak)) bust_spinlocks(int yes)
@@ -25,6 +26,7 @@ void __attribute__((weak)) bust_spinlocks(int yes)
 		unblank_screen();
 #endif
 		console_unblank();
+  		ipipe_trace_panic_dump();
 		if (--oops_in_progress == 0)
 			wake_up_klogd();
 	}
diff --git a/lib/ioremap.c b/lib/ioremap.c
index 0c9216c..00a9a30 100644
--- a/lib/ioremap.c
+++ b/lib/ioremap.c
@@ -10,6 +10,7 @@
 #include <linux/sched.h>
 #include <linux/io.h>
 #include <linux/export.h>
+#include <linux/hardirq.h>
 #include <asm/cacheflush.h>
 #include <asm/pgtable.h>
 
@@ -87,7 +88,12 @@ int ioremap_page_range(unsigned long addr,
 			break;
 	} while (pgd++, addr = next, addr != end);
 
-	flush_cache_vmap(start, end);
+	/* APEI may invoke this for temporarily remapping pages in interrupt
+	 * context - nothing we can and need to propagate globally. */
+	if (!in_interrupt()) {
+		__ipipe_pin_mapping_globally(start, end);
+		flush_cache_vmap(start, end);
+	}
 
 	return err;
 }
diff --git a/lib/smp_processor_id.c b/lib/smp_processor_id.c
index 1afec32..f7c1a2a 100644
--- a/lib/smp_processor_id.c
+++ b/lib/smp_processor_id.c
@@ -12,6 +12,12 @@ notrace static unsigned int check_preemption_disabled(const char *what1,
 {
 	int this_cpu = raw_smp_processor_id();
 
+	if (hard_irqs_disabled())
+		goto out;
+
+	if (!ipipe_root_p)
+		goto out;
+
 	if (likely(preempt_count()))
 		goto out;
 
diff --git a/mm/memory.c b/mm/memory.c
index 90fb265..8a1fd79 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -793,6 +793,34 @@ out:
 	return pfn_to_page(pfn);
 }
 
+static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
+{
+	debug_dma_assert_idle(src);
+
+	/*
+	 * If the source page was a PFN mapping, we don't have
+	 * a "struct page" for it. We do a best-effort copy by
+	 * just copying from the original user address. If that
+	 * fails, we just zero-fill it. Live with it.
+	 */
+	if (unlikely(!src)) {
+		void *kaddr = kmap_atomic(dst);
+		void __user *uaddr = (void __user *)(va & PAGE_MASK);
+
+		/*
+		 * This really shouldn't fail, because the page is there
+		 * in the page tables. But it might just be unreadable,
+		 * in which case we just give up and fill the result with
+		 * zeroes.
+		 */
+		if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
+			clear_page(kaddr);
+		kunmap_atomic(kaddr);
+		flush_dcache_page(dst);
+	} else
+		copy_user_highpage(dst, src, va, vma);
+}
+
 /*
  * copy one vm_area from one task to the other. Assumes the page tables
  * already present in the new task to be cleared in the whole range
@@ -801,8 +829,8 @@ out:
 
 static inline unsigned long
 copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
-		pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
-		unsigned long addr, int *rss)
+	     pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
+	     unsigned long addr, int *rss, struct page *uncow_page)
 {
 	unsigned long vm_flags = vma->vm_flags;
 	pte_t pte = *src_pte;
@@ -856,6 +884,21 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 	 * in the parent and the child
 	 */
 	if (is_cow_mapping(vm_flags)) {
+#ifdef CONFIG_IPIPE
+		if (uncow_page) {
+			struct page *old_page = vm_normal_page(vma, addr, pte);
+			cow_user_page(uncow_page, old_page, addr, vma);
+			pte = mk_pte(uncow_page, vma->vm_page_prot);
+
+			if (vm_flags & VM_SHARED)
+				pte = pte_mkclean(pte);
+			pte = pte_mkold(pte);
+
+			page_add_new_anon_rmap(uncow_page, vma, addr);
+			rss[!!PageAnon(uncow_page)]++;
+			goto out_set_pte;
+		}
+#endif /* CONFIG_IPIPE */
 		ptep_set_wrprotect(src_mm, addr, src_pte);
 		pte = pte_wrprotect(pte);
 	}
@@ -893,13 +936,27 @@ static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 	int progress = 0;
 	int rss[NR_MM_COUNTERS];
 	swp_entry_t entry = (swp_entry_t){0};
-
+	struct page *uncow_page = NULL;
+#ifdef CONFIG_IPIPE
+	int do_cow_break = 0;
 again:
+ 	if (do_cow_break) {
+ 		uncow_page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
+		if (uncow_page == NULL)
+ 			return -ENOMEM;
+		do_cow_break = 0;
+	}
+#else
+again:
+#endif
 	init_rss_vec(rss);
 
 	dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
-	if (!dst_pte)
+	if (!dst_pte) {
+		if (uncow_page)
+			page_cache_release(uncow_page);
 		return -ENOMEM;
+	}
 	src_pte = pte_offset_map(src_pmd, addr);
 	src_ptl = pte_lockptr(src_mm, src_pmd);
 	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
@@ -922,8 +979,25 @@ again:
 			progress++;
 			continue;
 		}
+#ifdef CONFIG_IPIPE
+		if (likely(uncow_page == NULL) && likely(pte_present(*src_pte))) {
+			if (is_cow_mapping(vma->vm_flags) &&
+			    test_bit(MMF_VM_PINNED, &src_mm->flags) &&
+			    ((vma->vm_flags|src_mm->def_flags) & VM_LOCKED)) {
+				arch_leave_lazy_mmu_mode();
+				spin_unlock(src_ptl);
+				pte_unmap(src_pte);
+				add_mm_rss_vec(dst_mm, rss);
+				pte_unmap_unlock(dst_pte, dst_ptl);
+				cond_resched();
+				do_cow_break = 1;
+				goto again;
+			}
+		}
+#endif
 		entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte,
-							vma, addr, rss);
+					 vma, addr, rss, uncow_page);
+		uncow_page = NULL;
 		if (entry.val)
 			break;
 		progress += 8;
@@ -1945,34 +2019,6 @@ static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd,
 	return same;
 }
 
-static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
-{
-	debug_dma_assert_idle(src);
-
-	/*
-	 * If the source page was a PFN mapping, we don't have
-	 * a "struct page" for it. We do a best-effort copy by
-	 * just copying from the original user address. If that
-	 * fails, we just zero-fill it. Live with it.
-	 */
-	if (unlikely(!src)) {
-		void *kaddr = kmap_atomic(dst);
-		void __user *uaddr = (void __user *)(va & PAGE_MASK);
-
-		/*
-		 * This really shouldn't fail, because the page is there
-		 * in the page tables. But it might just be unreadable,
-		 * in which case we just give up and fill the result with
-		 * zeroes.
-		 */
-		if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
-			clear_page(kaddr);
-		kunmap_atomic(kaddr);
-		flush_dcache_page(dst);
-	} else
-		copy_user_highpage(dst, src, va, vma);
-}
-
 /*
  * Notify the address space that the page is about to become writable so that
  * it can prohibit this or wait for the page to get into an appropriate state.
@@ -3791,6 +3837,41 @@ void copy_user_huge_page(struct page *dst, struct page *src,
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
 
+#ifdef CONFIG_IPIPE
+
+int __ipipe_disable_ondemand_mappings(struct task_struct *tsk)
+{
+	struct vm_area_struct *vma;
+	struct mm_struct *mm;
+	int result = 0;
+
+	mm = get_task_mm(tsk);
+	if (!mm)
+		return -EPERM;
+
+	down_write(&mm->mmap_sem);
+	if (test_bit(MMF_VM_PINNED, &mm->flags))
+		goto done_mm;
+
+	for (vma = mm->mmap; vma; vma = vma->vm_next) {
+		if (is_cow_mapping(vma->vm_flags) &&
+		    (vma->vm_flags & VM_WRITE)) {
+			result = __ipipe_pin_vma(mm, vma);
+			if (result < 0)
+				goto done_mm;
+		}
+	}
+	set_bit(MMF_VM_PINNED, &mm->flags);
+
+  done_mm:
+	up_write(&mm->mmap_sem);
+	mmput(mm);
+	return result;
+}
+EXPORT_SYMBOL_GPL(__ipipe_disable_ondemand_mappings);
+
+#endif /* CONFIG_IPIPE */
+
 #if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS
 
 static struct kmem_cache *page_ptl_cachep;
diff --git a/mm/mlock.c b/mm/mlock.c
index 73cf098..6928e67 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -871,3 +871,28 @@ void user_shm_unlock(size_t size, struct user_struct *user)
 	spin_unlock(&shmlock_user_lock);
 	free_uid(user);
 }
+
+#ifdef CONFIG_IPIPE
+int __ipipe_pin_vma(struct mm_struct *mm, struct vm_area_struct *vma)
+{
+	int ret, write, len;
+
+	if (vma->vm_flags & (VM_IO | VM_PFNMAP))
+		return 0;
+
+	if (!((vma->vm_flags & VM_DONTEXPAND) ||
+	    is_vm_hugetlb_page(vma) || vma == get_gate_vma(mm))) {
+		ret = __mlock_vma_pages_range(vma, vma->vm_start, vma->vm_end,
+					      NULL);
+		return ret < 0 ? ret : 0;
+	}
+
+	write = (vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE;
+	len = DIV_ROUND_UP(vma->vm_end, PAGE_SIZE) - vma->vm_start/PAGE_SIZE;
+	ret = get_user_pages(current, mm, vma->vm_start,
+			     len, write, 0, NULL, NULL);
+	if (ret < 0)
+		return ret;
+	return ret == len ? 0 : -EFAULT;
+}
+#endif
diff --git a/mm/mmap.c b/mm/mmap.c
index f88b4f9..e9b401a 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -49,6 +49,10 @@
 
 #include "internal.h"
 
+#ifndef MMAP_BRK
+#define MMAP_BRK 0
+#endif
+
 #ifndef arch_mmap_check
 #define arch_mmap_check(addr, len, flags)	(0)
 #endif
@@ -2658,7 +2662,7 @@ static unsigned long do_brk(unsigned long addr, unsigned long len)
 
 	flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
 
-	error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
+	error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED | MAP_BRK);
 	if (error & ~PAGE_MASK)
 		return error;
 
diff --git a/mm/mmu_context.c b/mm/mmu_context.c
index f802c2d..b320432 100644
--- a/mm/mmu_context.c
+++ b/mm/mmu_context.c
@@ -21,15 +21,18 @@ void use_mm(struct mm_struct *mm)
 {
 	struct mm_struct *active_mm;
 	struct task_struct *tsk = current;
+	unsigned long flags;
 
 	task_lock(tsk);
 	active_mm = tsk->active_mm;
+ 	ipipe_mm_switch_protect(flags);
 	if (active_mm != mm) {
 		atomic_inc(&mm->mm_count);
 		tsk->active_mm = mm;
 	}
 	tsk->mm = mm;
-	switch_mm(active_mm, mm, tsk);
+	__switch_mm(active_mm, mm, tsk);
+ 	ipipe_mm_switch_unprotect(flags);
 	task_unlock(tsk);
 #ifdef finish_arch_post_lock_switch
 	finish_arch_post_lock_switch();
diff --git a/mm/mprotect.c b/mm/mprotect.c
index ace9345..6b56407 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -64,7 +64,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
 	struct mm_struct *mm = vma->vm_mm;
 	pte_t *pte, oldpte;
 	spinlock_t *ptl;
-	unsigned long pages = 0;
+	unsigned long pages = 0, flags;
 
 	pte = lock_pte_protection(vma, pmd, addr, prot_numa, &ptl);
 	if (!pte)
@@ -78,6 +78,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
 			bool updated = false;
 
 			if (!prot_numa) {
+				flags = hard_local_irq_save();
 				ptent = ptep_modify_prot_start(mm, addr, pte);
 				if (pte_numa(ptent))
 					ptent = pte_mknonnuma(ptent);
@@ -91,6 +92,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
 				     !(vma->vm_flags & VM_SOFTDIRTY)))
 					ptent = pte_mkwrite(ptent);
 				ptep_modify_prot_commit(mm, addr, pte, ptent);
+				hard_local_irq_restore(flags);
 				updated = true;
 			} else {
 				struct page *page;
@@ -247,6 +249,12 @@ unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
 		pages = hugetlb_change_protection(vma, start, end, newprot);
 	else
 		pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa);
+#ifdef CONFIG_IPIPE
+	if (test_bit(MMF_VM_PINNED, &vma->vm_mm->flags) &&
+	    ((vma->vm_flags | vma->vm_mm->def_flags) & VM_LOCKED) &&
+	    (vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)))
+		__ipipe_pin_vma(vma->vm_mm, vma);
+#endif
 
 	return pages;
 }
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 90520af..dadc22d 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -193,6 +193,8 @@ static int vmap_page_range_noflush(unsigned long start, unsigned long end,
 			return err;
 	} while (pgd++, addr = next, addr != end);
 
+ 	__ipipe_pin_mapping_globally(start, end);
+ 
 	return nr;
 }
 
