diff --git a/arch/blackfin/Kconfig b/arch/blackfin/Kconfig
index abe5a9e..ff2f9f7 100644
--- a/arch/blackfin/Kconfig
+++ b/arch/blackfin/Kconfig
@@ -72,6 +72,8 @@ source "kernel/Kconfig.preempt"
 
 source "kernel/Kconfig.freezer"
 
+source "kernel/ipipe/Kconfig"
+
 menu "Blackfin Processor Options"
 
 comment "Processor and Board Settings"
diff --git a/arch/blackfin/include/asm/ipipe.h b/arch/blackfin/include/asm/ipipe.h
index 17b5e92..3a0c506 100644
--- a/arch/blackfin/include/asm/ipipe.h
+++ b/arch/blackfin/include/asm/ipipe.h
@@ -28,7 +28,7 @@
 #include <linux/list.h>
 #include <linux/threads.h>
 #include <linux/irq.h>
-#include <linux/ipipe_percpu.h>
+#include <linux/ipipe_domain.h>
 #include <asm/ptrace.h>
 #include <asm/irq.h>
 #include <asm/bitops.h>
@@ -36,10 +36,7 @@
 #include <asm/traps.h>
 #include <asm/bitsperlong.h>
 
-#define IPIPE_ARCH_STRING     "1.16-01"
-#define IPIPE_MAJOR_NUMBER    1
-#define IPIPE_MINOR_NUMBER    16
-#define IPIPE_PATCH_NUMBER    1
+#define IPIPE_CORE_RELEASE	2
 
 #ifdef CONFIG_SMP
 #error "I-pipe/blackfin: SMP not implemented"
@@ -47,28 +44,9 @@
 #define ipipe_processor_id()	0
 #endif	/* CONFIG_SMP */
 
-#define prepare_arch_switch(next)		\
-do {						\
-	ipipe_schedule_notify(current, next);	\
-	hard_local_irq_disable();			\
-} while (0)
-
-#define task_hijacked(p)						\
-	({								\
-		int __x__ = __ipipe_root_domain_p;			\
-		if (__x__)						\
-			hard_local_irq_enable();			\
-		!__x__;							\
-	})
-
 struct ipipe_domain;
 
-struct ipipe_sysinfo {
-	int sys_nr_cpus;	/* Number of CPUs on board */
-	int sys_hrtimer_irq;	/* hrtimer device IRQ */
-	u64 sys_hrtimer_freq;	/* hrtimer device frequency */
-	u64 sys_hrclock_freq;	/* hrclock device frequency */
-	u64 sys_cpu_freq;	/* CPU frequency (Hz) */
+struct ipipe_arch_sysinfo {
 };
 
 #define ipipe_read_tsc(t)					\
@@ -92,49 +70,24 @@ struct ipipe_sysinfo {
 
 /* Private interface -- Internal use only */
 
-#define __ipipe_check_platform()	do { } while (0)
-
-#define __ipipe_init_platform()		do { } while (0)
+#define __ipipe_early_core_setup()	do { } while (0)
 
 extern atomic_t __ipipe_irq_lvdepth[IVG15 + 1];
 
 extern unsigned long __ipipe_irq_lvmask;
 
-extern struct ipipe_domain ipipe_root;
-
 /* enable/disable_irqdesc _must_ be used in pairs. */
 
 void __ipipe_enable_irqdesc(struct ipipe_domain *ipd,
-			    unsigned irq);
+			    unsigned int irq);
 
 void __ipipe_disable_irqdesc(struct ipipe_domain *ipd,
-			     unsigned irq);
-
-#define __ipipe_enable_irq(irq)						\
-	do {								\
-		struct irq_desc *desc = irq_to_desc(irq);		\
-		struct irq_chip *chip = get_irq_desc_chip(desc);	\
-		chip->irq_unmask(&desc->irq_data);			\
-	} while (0)
-
-#define __ipipe_disable_irq(irq)					\
-	do {								\
-		struct irq_desc *desc = irq_to_desc(irq);		\
-		struct irq_chip *chip = get_irq_desc_chip(desc);	\
-		chip->irq_mask(&desc->irq_data);			\
-	} while (0)
-
-static inline int __ipipe_check_tickdev(const char *devname)
-{
-	return 1;
-}
+			     unsigned int irq);
 
 void __ipipe_enable_pipeline(void);
 
 #define __ipipe_hook_critical_ipi(ipd) do { } while (0)
 
-void ___ipipe_sync_pipeline(void);
-
 void __ipipe_handle_irq(unsigned irq, struct pt_regs *regs);
 
 int __ipipe_get_irq_priority(unsigned int irq);
@@ -143,37 +96,12 @@ void __ipipe_serial_debug(const char *fmt, ...);
 
 asmlinkage void __ipipe_call_irqtail(unsigned long addr);
 
-DECLARE_PER_CPU(struct pt_regs, __ipipe_tick_regs);
-
 extern unsigned long __ipipe_core_clock;
 
 extern unsigned long __ipipe_freq_scale;
 
 extern unsigned long __ipipe_irq_tail_hook;
 
-static inline unsigned long __ipipe_ffnz(unsigned long ul)
-{
-	return ffs(ul) - 1;
-}
-
-#define __ipipe_do_root_xirq(ipd, irq)					\
-	((ipd)->irqs[irq].handler(irq, &__raw_get_cpu_var(__ipipe_tick_regs)))
-
-#define __ipipe_run_irqtail(irq)  /* Must be a macro */			\
-	do {								\
-		unsigned long __pending;				\
-		CSYNC();						\
-		__pending = bfin_read_IPEND();				\
-		if (__pending & 0x8000) {				\
-			__pending &= ~0x8010;				\
-			if (__pending && (__pending & (__pending - 1)) == 0) \
-				__ipipe_call_irqtail(__ipipe_irq_tail_hook); \
-		}							\
-	} while (0)
-
-#define __ipipe_syscall_watched_p(p, sc)	\
-	(ipipe_notifier_enabled_p(p) || (unsigned long)sc >= NR_syscalls)
-
 #ifdef CONFIG_BF561
 #define bfin_write_TIMER_DISABLE(val)	bfin_write_TMRS8_DISABLE(val)
 #define bfin_write_TIMER_ENABLE(val)	bfin_write_TMRS8_ENABLE(val)
@@ -188,11 +116,11 @@ static inline unsigned long __ipipe_ffnz(unsigned long ul)
 
 #define __ipipe_root_tick_p(regs)	((regs->ipend & 0x10) != 0)
 
-#else /* !CONFIG_IPIPE */
+static inline void ipipe_mute_pic(void) { }
 
-#define task_hijacked(p)		0
-#define ipipe_trap_notify(t, r)  	0
-#define __ipipe_root_tick_p(regs)	1
+static inline void ipipe_unmute_pic(void) { }
+
+static inline void ipipe_notify_root_preemption(void) { }
 
 #endif /* !CONFIG_IPIPE */
 
@@ -204,6 +132,4 @@ static inline unsigned long __ipipe_ffnz(unsigned long ul)
 #define IRQ_PRIOTMR		CONFIG_IRQ_TIMER0
 #endif
 
-#define ipipe_update_tick_evtdev(evtdev)	do { } while (0)
-
 #endif	/* !__ASM_BLACKFIN_IPIPE_H */
diff --git a/arch/blackfin/include/asm/ipipe_base.h b/arch/blackfin/include/asm/ipipe_base.h
index 84a4ffd..922bdac 100644
--- a/arch/blackfin/include/asm/ipipe_base.h
+++ b/arch/blackfin/include/asm/ipipe_base.h
@@ -33,42 +33,51 @@
 #define IPIPE_SYNCDEFER_FLAG	15
 #define IPIPE_SYNCDEFER_MASK	(1L << IPIPE_SYNCDEFER_MASK)
 
- /* Blackfin traps -- i.e. exception vector numbers */
-#define IPIPE_NR_FAULTS		52 /* We leave a gap after VEC_ILL_RES. */
-/* Pseudo-vectors used for kernel events */
-#define IPIPE_FIRST_EVENT	IPIPE_NR_FAULTS
-#define IPIPE_EVENT_SYSCALL	(IPIPE_FIRST_EVENT)
-#define IPIPE_EVENT_SCHEDULE	(IPIPE_FIRST_EVENT + 1)
-#define IPIPE_EVENT_SIGWAKE	(IPIPE_FIRST_EVENT + 2)
-#define IPIPE_EVENT_SETSCHED	(IPIPE_FIRST_EVENT + 3)
-#define IPIPE_EVENT_INIT	(IPIPE_FIRST_EVENT + 4)
-#define IPIPE_EVENT_EXIT	(IPIPE_FIRST_EVENT + 5)
-#define IPIPE_EVENT_CLEANUP	(IPIPE_FIRST_EVENT + 6)
-#define IPIPE_EVENT_RETURN	(IPIPE_FIRST_EVENT + 7)
-#define IPIPE_LAST_EVENT	IPIPE_EVENT_RETURN
-#define IPIPE_NR_EVENTS		(IPIPE_LAST_EVENT + 1)
-
-#define IPIPE_TIMER_IRQ		IRQ_CORETMR
-
-#define __IPIPE_FEATURE_SYSINFO_V2	1
+/*
+ * Blackfin traps -- i.e. exception vector numbers, we leave a gap
+ * after VEC_ILL_RES.
+ */
+#define IPIPE_TRAP_MAYDAY	52	/* Internal recovery trap */
+#define IPIPE_NR_FAULTS		53
 
 #ifndef __ASSEMBLY__
 
-extern unsigned long __ipipe_root_status; /* Alias to ipipe_root_cpudom_var(status) */
+extern unsigned long __ipipe_root_status;
 
-void __ipipe_stall_root(void);
+void ipipe_stall_root(void);
 
-unsigned long __ipipe_test_and_stall_root(void);
+unsigned long ipipe_test_and_stall_root(void);
 
-unsigned long __ipipe_test_root(void);
+unsigned long ipipe_test_root(void);
 
 void __ipipe_lock_root(void);
 
 void __ipipe_unlock_root(void);
 
-#endif /* !__ASSEMBLY__ */
+int __ipipe_do_sync_check(void);
+#define __ipipe_sync_check __ipipe_do_sync_check()
+
+static inline unsigned long __ipipe_ffnz(unsigned long ul)
+{
+	return ffs(ul) - 1;
+}
+
+#define __ipipe_run_irqtail(irq)  /* Must be a macro */			\
+	do {								\
+		unsigned long __pending;				\
+		CSYNC();						\
+		__pending = bfin_read_IPEND();				\
+		if (__pending & 0x8000) {				\
+			__pending &= ~0x8010;				\
+			if (__pending && (__pending & (__pending - 1)) == 0) \
+				__ipipe_call_irqtail(__ipipe_irq_tail_hook); \
+		}							\
+	} while (0)
+
+#define __ipipe_syscall_watched_p(p, sc)	\
+	(ipipe_notifier_enabled_p(p) || (unsigned long)sc >= NR_syscalls)
 
-#define __IPIPE_FEATURE_SYSINFO_V2	1
+#endif /* !__ASSEMBLY__ */
 
 #endif /* CONFIG_IPIPE */
 
diff --git a/arch/blackfin/include/asm/irqflags.h b/arch/blackfin/include/asm/irqflags.h
index 43eb474..59648f4 100644
--- a/arch/blackfin/include/asm/irqflags.h
+++ b/arch/blackfin/include/asm/irqflags.h
@@ -95,41 +95,30 @@ static inline notrace void __hard_local_irq_restore(unsigned long flags)
  * we redeclare the required bits we cannot pick from
  * <asm/ipipe_base.h> to prevent circular dependencies.
  */
-void __ipipe_stall_root(void);
-void __ipipe_unstall_root(void);
-unsigned long __ipipe_test_root(void);
-unsigned long __ipipe_test_and_stall_root(void);
-void __ipipe_restore_root(unsigned long flags);
-
-#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
-struct ipipe_domain;
-extern struct ipipe_domain ipipe_root;
-void ipipe_check_context(struct ipipe_domain *ipd);
-#define __check_irqop_context(ipd)  ipipe_check_context(&ipipe_root)
-#else /* !CONFIG_IPIPE_DEBUG_CONTEXT */
-#define __check_irqop_context(ipd)  do { } while (0)
-#endif /* !CONFIG_IPIPE_DEBUG_CONTEXT */
+void ipipe_stall_root(void);
+void ipipe_unstall_root(void);
+unsigned long ipipe_test_root(void);
+unsigned long ipipe_test_and_stall_root(void);
+void ipipe_restore_root(unsigned long flags);
 
 /*
  * Interrupt pipe interface to linux/irqflags.h.
  */
 static inline notrace void arch_local_irq_disable(void)
 {
-	__check_irqop_context();
-	__ipipe_stall_root();
+	ipipe_stall_root();
 	barrier();
 }
 
 static inline notrace void arch_local_irq_enable(void)
 {
 	barrier();
-	__check_irqop_context();
-	__ipipe_unstall_root();
+	ipipe_unstall_root();
 }
 
 static inline notrace unsigned long arch_local_save_flags(void)
 {
-	return __ipipe_test_root() ? bfin_no_irqs : bfin_irq_flags;
+	return ipipe_test_root() ? bfin_no_irqs : bfin_irq_flags;
 }
 
 static inline notrace int arch_irqs_disabled_flags(unsigned long flags)
@@ -141,8 +130,7 @@ static inline notrace unsigned long arch_local_irq_save(void)
 {
 	unsigned long flags;
 
-	__check_irqop_context();
-	flags = __ipipe_test_and_stall_root() ? bfin_no_irqs : bfin_irq_flags;
+	flags = ipipe_test_and_stall_root() ? bfin_no_irqs : bfin_irq_flags;
 	barrier();
 
 	return flags;
@@ -150,8 +138,7 @@ static inline notrace unsigned long arch_local_irq_save(void)
 
 static inline notrace void arch_local_irq_restore(unsigned long flags)
 {
-	__check_irqop_context();
-	__ipipe_restore_root(flags == bfin_no_irqs);
+	ipipe_restore_root(flags == bfin_no_irqs);
 }
 
 static inline notrace unsigned long arch_mangle_irq_bits(int virt, unsigned long real)
@@ -215,8 +202,28 @@ static inline notrace void hard_local_irq_restore(unsigned long flags)
 # define hard_local_irq_restore(flags)	__hard_local_irq_restore(flags)
 #endif /* !CONFIG_IPIPE_TRACE_IRQSOFF */
 
-#define hard_local_irq_save_cond()		hard_local_irq_save()
-#define hard_local_irq_restore_cond(flags)	hard_local_irq_restore(flags)
+#define hard_cond_local_irq_save()		hard_local_irq_save()
+#define hard_cond_local_irq_restore(flags)	hard_local_irq_restore(flags)
+
+static inline notrace unsigned long hard_local_irq_save_notrace(void)
+{
+	return __hard_local_irq_save();
+}
+
+static inline notrace void hard_local_irq_restore_notrace(unsigned long flags)
+{
+	return __hard_local_irq_restore(flags);
+}
+
+static inline notrace void hard_local_irq_disable_notrace(void)
+{
+	return __hard_local_irq_disable();
+}
+
+static inline notrace void hard_local_irq_enable_notrace(void)
+{
+	return __hard_local_irq_enable();
+}
 
 #else /* !CONFIG_IPIPE */
 
@@ -224,7 +231,7 @@ static inline notrace void hard_local_irq_restore(unsigned long flags)
  * Direct interface to linux/irqflags.h.
  */
 #define arch_local_save_flags()		hard_local_save_flags()
-#define arch_local_irq_save(flags)	__hard_local_irq_save()
+#define arch_local_irq_save()		__hard_local_irq_save()
 #define arch_local_irq_restore(flags)	__hard_local_irq_restore(flags)
 #define arch_local_irq_enable()		__hard_local_irq_enable()
 #define arch_local_irq_disable()	__hard_local_irq_disable()
@@ -238,48 +245,17 @@ static inline notrace void hard_local_irq_restore(unsigned long flags)
 #define hard_local_irq_restore(flags)	__hard_local_irq_restore(flags)
 #define hard_local_irq_enable()		__hard_local_irq_enable()
 #define hard_local_irq_disable()	__hard_local_irq_disable()
-#define hard_local_irq_save_cond()		hard_local_save_flags()
-#define hard_local_irq_restore_cond(flags)	do { (void)(flags); } while (0)
+#define hard_cond_local_irq_save()		hard_local_save_flags()
+#define hard_cond_local_irq_restore(flags)	do { (void)(flags); } while (0)
 
 #endif /* !CONFIG_IPIPE */
 
-#ifdef CONFIG_SMP
-#define hard_local_irq_save_smp()		hard_local_irq_save()
-#define hard_local_irq_restore_smp(flags)	hard_local_irq_restore(flags)
+#if defined(CONFIG_SMP) && defined(CONFIG_IPIPE)
+#define hard_smp_local_irq_save()		hard_local_irq_save()
+#define hard_smp_local_irq_restore(flags)	hard_local_irq_restore(flags)
 #else
-#define hard_local_irq_save_smp()		hard_local_save_flags()
-#define hard_local_irq_restore_smp(flags)	do { (void)(flags); } while (0)
+#define hard_smp_local_irq_save()		hard_local_save_flags()
+#define hard_smp_local_irq_restore(flags)	do { (void)(flags); } while (0)
 #endif
 
-/*
- * Remap the arch-neutral IRQ state manipulation macros to the
- * blackfin-specific hard_local_irq_* API.
- */
-#define local_irq_save_hw(flags)			\
-	do {						\
-		(flags) = hard_local_irq_save();	\
-	} while (0)
-#define local_irq_restore_hw(flags)		\
-	do {					\
-		hard_local_irq_restore(flags);	\
-	} while (0)
-#define local_irq_disable_hw()			\
-	do {					\
-		hard_local_irq_disable();	\
-	} while (0)
-#define local_irq_enable_hw()			\
-	do {					\
-		hard_local_irq_enable();	\
-	} while (0)
-#define local_irq_save_hw_notrace(flags)		\
-	do {						\
-		(flags) = __hard_local_irq_save();	\
-	} while (0)
-#define local_irq_restore_hw_notrace(flags)		\
-	do {						\
-		__hard_local_irq_restore(flags);	\
-	} while (0)
-
-#define irqs_disabled_hw()	hard_irqs_disabled()
-
 #endif
diff --git a/arch/blackfin/include/asm/mmu_context.h b/arch/blackfin/include/asm/mmu_context.h
index 3828c70..e23c30b 100644
--- a/arch/blackfin/include/asm/mmu_context.h
+++ b/arch/blackfin/include/asm/mmu_context.h
@@ -97,8 +97,11 @@ static inline void __switch_mm(struct mm_struct *prev_mm, struct mm_struct *next
 }
 
 #ifdef CONFIG_IPIPE
-#define lock_mm_switch(flags)	flags = hard_local_irq_save_cond()
-#define unlock_mm_switch(flags)	hard_local_irq_restore_cond(flags)
+#define lock_mm_switch(flags)				\
+	do {						\
+		flags = hard_cond_local_irq_save();	\
+	} while (0)
+#define unlock_mm_switch(flags)	hard_cond_local_irq_restore(flags)
 #else
 #define lock_mm_switch(flags)	do { (void)(flags); } while (0)
 #define unlock_mm_switch(flags)	do { (void)(flags); } while (0)
@@ -205,9 +208,9 @@ static inline void destroy_context(struct mm_struct *mm)
 }
 
 #define ipipe_mm_switch_protect(flags)		\
-	flags = hard_local_irq_save_cond()
+	flags = hard_cond_local_irq_save()
 
 #define ipipe_mm_switch_unprotect(flags)	\
-	hard_local_irq_restore_cond(flags)
+	hard_cond_local_irq_restore(flags)
 
 #endif
diff --git a/arch/blackfin/include/asm/thread_info.h b/arch/blackfin/include/asm/thread_info.h
index 02560fd..b2184aa 100644
--- a/arch/blackfin/include/asm/thread_info.h
+++ b/arch/blackfin/include/asm/thread_info.h
@@ -28,6 +28,14 @@
 
 #ifndef __ASSEMBLY__
 
+#ifdef CONFIG_IPIPE
+#include <ipipe/thread_info.h>
+#else
+struct ipipe_threadinfo {
+};
+static inline void __ipipe_init_threadinfo(struct ipipe_threadinfo *p) { }
+#endif
+
 typedef unsigned long mm_segment_t;
 
 /*
@@ -43,6 +51,7 @@ struct thread_info {
 	int preempt_count;	/* 0 => preemptable, <0 => BUG */
 	mm_segment_t addr_limit;	/* address limit */
 	struct restart_block restart_block;
+	struct ipipe_threadinfo ipipe_data;
 #ifndef CONFIG_SMP
 	struct l1_scratch_task_info l1_task_info;
 #endif
diff --git a/arch/blackfin/include/asm/time.h b/arch/blackfin/include/asm/time.h
index 9ca7db8..636728e 100644
--- a/arch/blackfin/include/asm/time.h
+++ b/arch/blackfin/include/asm/time.h
@@ -43,4 +43,12 @@ extern void bfin_coretmr_init(void);
 extern void bfin_coretmr_clockevent_init(void);
 #endif
 
+#if !defined(CONFIG_GENERIC_CLOCKEVENTS) || defined(CONFIG_TICKSOURCE_GPTMR0)
+#ifdef CONFIG_IPIPE
+extern void bfin_ipipe_coretmr_register(void)
+#else /* !CONFIG_IPIPE */
+#define bfin_ipipe_coretmr_register() do { } while (0)
+#endif /* !CONFIG_IPIPE */
+#endif
+
 #endif
diff --git a/arch/blackfin/kernel/asm-offsets.c b/arch/blackfin/kernel/asm-offsets.c
index 17e3546..cab0ce2 100644
--- a/arch/blackfin/kernel/asm-offsets.c
+++ b/arch/blackfin/kernel/asm-offsets.c
@@ -153,5 +153,10 @@ int main(void)
 	DEFINE(SIZEOF_CORELOCK, sizeof(struct corelock_slot));
 #endif
 
+#ifdef CONFIG_IPIPE
+	DEFINE(IPIPE_CURRENT_DOMAIN, offsetof(struct ipipe_percpu_data, curr));
+	DEFINE(IPIPE_DOMAIN_DESC, offsetof(struct ipipe_percpu_domain_data, domain));
+#endif
+
 	return 0;
 }
diff --git a/arch/blackfin/kernel/bfin_gpio.c b/arch/blackfin/kernel/bfin_gpio.c
index 02796b8..1c2a80b 100644
--- a/arch/blackfin/kernel/bfin_gpio.c
+++ b/arch/blackfin/kernel/bfin_gpio.c
@@ -395,6 +395,12 @@ static int portmux_group_check(unsigned short per)
 * MODIFICATION HISTORY :
 **************************************************************/
 
+#ifdef CONFIG_IPIPE
+#define IPIPE_GPIO_ACCESS  1
+#else
+#define IPIPE_GPIO_ACCESS  0
+#endif
+
 /* Set a specific bit */
 
 #define SET_GPIO(name) \
@@ -422,7 +428,7 @@ SET_GPIO(both)  /* set_gpio_both() */
 void set_gpio_ ## name(unsigned gpio, unsigned short arg) \
 { \
 	unsigned long flags; \
-	if (ANOMALY_05000311 || ANOMALY_05000323) \
+	if (ANOMALY_05000311 || ANOMALY_05000323 || IPIPE_GPIO_ACCESS) \
 		flags = hard_local_irq_save(); \
 	if (arg) \
 		gpio_array[gpio_bank(gpio)]->name ## _set = gpio_bit(gpio); \
@@ -430,6 +436,8 @@ void set_gpio_ ## name(unsigned gpio, unsigned short arg) \
 		gpio_array[gpio_bank(gpio)]->name ## _clear = gpio_bit(gpio); \
 	if (ANOMALY_05000311 || ANOMALY_05000323) { \
 		AWA_DUMMY_READ(name); \
+	} \
+	if (ANOMALY_05000311 || ANOMALY_05000323 || IPIPE_GPIO_ACCESS) { \
 		hard_local_irq_restore(flags); \
 	} \
 } \
@@ -442,11 +450,13 @@ SET_GPIO_SC(data)
 void set_gpio_toggle(unsigned gpio)
 {
 	unsigned long flags;
-	if (ANOMALY_05000311 || ANOMALY_05000323)
+	if (ANOMALY_05000311 || ANOMALY_05000323 || IPIPE_GPIO_ACCESS)
 		flags = hard_local_irq_save();
 	gpio_array[gpio_bank(gpio)]->toggle = gpio_bit(gpio);
 	if (ANOMALY_05000311 || ANOMALY_05000323) {
 		AWA_DUMMY_READ(toggle);
+	}
+	if (ANOMALY_05000311 || ANOMALY_05000323 || IPIPE_GPIO_ACCESS) {
 		hard_local_irq_restore(flags);
 	}
 }
@@ -459,11 +469,13 @@ EXPORT_SYMBOL(set_gpio_toggle);
 void set_gpiop_ ## name(unsigned gpio, unsigned short arg) \
 { \
 	unsigned long flags; \
-	if (ANOMALY_05000311 || ANOMALY_05000323) \
+	if (ANOMALY_05000311 || ANOMALY_05000323 || IPIPE_GPIO_ACCESS) \
 		flags = hard_local_irq_save(); \
 	gpio_array[gpio_bank(gpio)]->name = arg; \
 	if (ANOMALY_05000311 || ANOMALY_05000323) { \
 		AWA_DUMMY_READ(name); \
+	} \
+	if (ANOMALY_05000311 || ANOMALY_05000323 || IPIPE_GPIO_ACCESS) { \
 		hard_local_irq_restore(flags); \
 	} \
 } \
diff --git a/arch/blackfin/kernel/ipipe.c b/arch/blackfin/kernel/ipipe.c
index dbe1122..5efcb54 100644
--- a/arch/blackfin/kernel/ipipe.c
+++ b/arch/blackfin/kernel/ipipe.c
@@ -33,27 +33,30 @@
 #include <linux/io.h>
 #include <asm/system.h>
 #include <linux/atomic.h>
+#include <linux/ipipe_tickdev.h>
 #include <asm/irq_handler.h>
-
-DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs);
+#include <asm/blackfin.h>
+#include <asm/time.h>
 
 asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs);
 
+static void __ipipe_do_IRQ(unsigned int irq, void *cookie);
+
 static void __ipipe_no_irqtail(void);
 
-unsigned long __ipipe_irq_tail_hook = (unsigned long)&__ipipe_no_irqtail;
-EXPORT_SYMBOL(__ipipe_irq_tail_hook);
+unsigned long __ipipe_irq_tail_hook = (unsigned long)__ipipe_no_irqtail;
+EXPORT_SYMBOL_GPL(__ipipe_irq_tail_hook);
 
 unsigned long __ipipe_core_clock;
-EXPORT_SYMBOL(__ipipe_core_clock);
+EXPORT_SYMBOL_GPL(__ipipe_core_clock);
 
 unsigned long __ipipe_freq_scale;
-EXPORT_SYMBOL(__ipipe_freq_scale);
+EXPORT_SYMBOL_GPL(__ipipe_freq_scale);
 
 atomic_t __ipipe_irq_lvdepth[IVG15 + 1];
 
 unsigned long __ipipe_irq_lvmask = bfin_no_irqs;
-EXPORT_SYMBOL(__ipipe_irq_lvmask);
+EXPORT_SYMBOL_GPL(__ipipe_irq_lvmask);
 
 static void __ipipe_ack_irq(unsigned irq, struct irq_desc *desc)
 {
@@ -72,95 +75,24 @@ void __ipipe_enable_pipeline(void)
 	__ipipe_freq_scale = 1000000000UL / __ipipe_core_clock;
 
 	for (irq = 0; irq < NR_IRQS; ++irq)
-		ipipe_virtualize_irq(ipipe_root_domain,
-				     irq,
-				     (ipipe_irq_handler_t)&asm_do_IRQ,
-				     NULL,
-				     &__ipipe_ack_irq,
-				     IPIPE_HANDLE_MASK | IPIPE_PASS_MASK);
+		ipipe_request_irq(ipipe_root_domain, irq,
+				  __ipipe_do_IRQ, NULL,
+				  __ipipe_ack_irq);
 }
 
-/*
- * __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic
- * interrupt protection log is maintained here for each domain. Hw
- * interrupts are masked on entry.
- */
-void __ipipe_handle_irq(unsigned irq, struct pt_regs *regs)
+void __ipipe_handle_irq(unsigned int irq, struct pt_regs *regs) /* hw IRQs off */
 {
-	struct ipipe_percpu_domain_data *p = ipipe_root_cpudom_ptr();
-	struct ipipe_domain *this_domain, *next_domain;
-	struct list_head *head, *pos;
-	struct ipipe_irqdesc *idesc;
-	int m_ack, s = -1;
+	struct ipipe_percpu_domain_data *p = ipipe_this_cpu_root_context();
+	int flags, s = -1;
 
-	/*
-	 * Software-triggered IRQs do not need any ack.  The contents
-	 * of the register frame should only be used when processing
-	 * the timer interrupt, but not for handling any other
-	 * interrupt.
-	 */
-	m_ack = (regs == NULL || irq == IRQ_SYSTMR || irq == IRQ_CORETMR);
-	this_domain = __ipipe_current_domain;
-	idesc = &this_domain->irqs[irq];
-
-	if (unlikely(test_bit(IPIPE_STICKY_FLAG, &idesc->control)))
-		head = &this_domain->p_link;
-	else {
-		head = __ipipe_pipeline.next;
-		next_domain = list_entry(head, struct ipipe_domain, p_link);
-		idesc = &next_domain->irqs[irq];
-		if (likely(test_bit(IPIPE_WIRED_FLAG, &idesc->control))) {
-			if (!m_ack && idesc->acknowledge != NULL)
-				idesc->acknowledge(irq, irq_to_desc(irq));
-			if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status))
-				s = __test_and_set_bit(IPIPE_STALL_FLAG,
-						       &p->status);
-			__ipipe_dispatch_wired(next_domain, irq);
-			goto out;
-		}
-	}
-
-	/* Ack the interrupt. */
-
-	pos = head;
-	while (pos != &__ipipe_pipeline) {
-		next_domain = list_entry(pos, struct ipipe_domain, p_link);
-		idesc = &next_domain->irqs[irq];
-		if (test_bit(IPIPE_HANDLE_FLAG, &idesc->control)) {
-			__ipipe_set_irq_pending(next_domain, irq);
-			if (!m_ack && idesc->acknowledge != NULL) {
-				idesc->acknowledge(irq, irq_to_desc(irq));
-				m_ack = 1;
-			}
-		}
-		if (!test_bit(IPIPE_PASS_FLAG, &idesc->control))
-			break;
-		pos = next_domain->p_link.next;
-	}
-
-	/*
-	 * Now walk the pipeline, yielding control to the highest
-	 * priority domain that has pending interrupt(s) or
-	 * immediately to the current domain if the interrupt has been
-	 * marked as 'sticky'. This search does not go beyond the
-	 * current domain in the pipeline. We also enforce the
-	 * additional root stage lock (blackfin-specific).
-	 */
 	if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status))
 		s = __test_and_set_bit(IPIPE_STALL_FLAG, &p->status);
 
-	/*
-	 * If the interrupt preempted the head domain, then do not
-	 * even try to walk the pipeline, unless an interrupt is
-	 * pending for it.
-	 */
-	if (test_bit(IPIPE_AHEAD_FLAG, &this_domain->flags) &&
-	    !__ipipe_ipending_p(ipipe_head_cpudom_ptr()))
-		goto out;
+	flags = (regs && irq != IRQ_SYSTMR && irq != IRQ_CORETMR) ?
+		0 : IPIPE_IRQF_NOACK;
+	__ipipe_dispatch_irq(irq, flags);
 
-	__ipipe_walk_pipeline(head);
-out:
-	if (!s)
+	if (s == 0)
 		__clear_bit(IPIPE_STALL_FLAG, &p->status);
 }
 
@@ -170,21 +102,19 @@ void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
 	int prio = __ipipe_get_irq_priority(irq);
 
 	desc->depth = 0;
-	if (ipd != &ipipe_root &&
+	if (ipd != ipipe_root_domain &&
 	    atomic_inc_return(&__ipipe_irq_lvdepth[prio]) == 1)
 		__set_bit(prio, &__ipipe_irq_lvmask);
 }
-EXPORT_SYMBOL(__ipipe_enable_irqdesc);
 
 void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
 {
 	int prio = __ipipe_get_irq_priority(irq);
 
-	if (ipd != &ipipe_root &&
+	if (ipd != ipipe_root_domain &&
 	    atomic_dec_and_test(&__ipipe_irq_lvdepth[prio]))
 		__clear_bit(prio, &__ipipe_irq_lvmask);
 }
-EXPORT_SYMBOL(__ipipe_disable_irqdesc);
 
 asmlinkage int __ipipe_syscall_root(struct pt_regs *regs)
 {
@@ -192,7 +122,7 @@ asmlinkage int __ipipe_syscall_root(struct pt_regs *regs)
 	void (*hook)(void);
 	int ret;
 
-	WARN_ON_ONCE(irqs_disabled_hw());
+	WARN_ON_ONCE(hard_irqs_disabled());
 
 	/*
 	 * We need to run the IRQ tail hook each time we intercept a
@@ -211,11 +141,10 @@ asmlinkage int __ipipe_syscall_root(struct pt_regs *regs)
 	 * tail work has to be performed (for handling signals etc).
 	 */
 
-	if (!__ipipe_syscall_watched_p(current, regs->orig_p0) ||
-	    !__ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL))
+	if (!__ipipe_syscall_watched_p(current, regs->orig_p0))
 		return 0;
 
-	ret = __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL, regs);
+	ret = __ipipe_notify_syscall(regs);
 
 	hard_local_irq_disable();
 
@@ -223,17 +152,17 @@ asmlinkage int __ipipe_syscall_root(struct pt_regs *regs)
 	 * This is the end of the syscall path, so we may
 	 * safely assume a valid Linux task stack here.
 	 */
-	if (current->ipipe_flags & PF_EVTRET) {
-		current->ipipe_flags &= ~PF_EVTRET;
-		__ipipe_dispatch_event(IPIPE_EVENT_RETURN, regs);
+	if (current->ipipe.flags & PF_MAYDAY) {
+		current->ipipe.flags &= ~PF_MAYDAY;
+		__ipipe_notify_trap(IPIPE_TRAP_MAYDAY, regs);
 	}
 
-	if (!__ipipe_root_domain_p)
+	if (!__ipipe_root_p)
 		ret = -1;
 	else {
-		p = ipipe_root_cpudom_ptr();
+		p = ipipe_this_cpu_root_context();
 		if (__ipipe_ipending_p(p))
-			__ipipe_sync_pipeline();
+			__ipipe_sync_stage();
 	}
 
 	hard_local_irq_enable();
@@ -241,43 +170,43 @@ asmlinkage int __ipipe_syscall_root(struct pt_regs *regs)
 	return -ret;
 }
 
+static void __ipipe_do_IRQ(unsigned int irq, void *cookie)
+{
+	struct pt_regs *regs = __this_cpu_ptr(&ipipe_percpu.tick_regs);
+	asm_do_IRQ(irq, regs);
+}
+
 static void __ipipe_no_irqtail(void)
 {
 }
 
+int __ipipe_do_sync_check(void)
+{
+	return !(ipipe_root_p &&
+		 test_bit(IPIPE_SYNCDEFER_FLAG, &__ipipe_root_status));
+}
+
 int ipipe_get_sysinfo(struct ipipe_sysinfo *info)
 {
 	info->sys_nr_cpus = num_online_cpus();
 	info->sys_cpu_freq = ipipe_cpu_freq();
-	info->sys_hrtimer_irq = IPIPE_TIMER_IRQ;
-	info->sys_hrtimer_freq = __ipipe_core_clock;
+	info->sys_hrtimer_irq = per_cpu(ipipe_percpu.hrtimer_irq, 0);
+	info->sys_hrtimer_freq = __ipipe_hrtimer_freq;
 	info->sys_hrclock_freq = __ipipe_core_clock;
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(ipipe_get_sysinfo);
 
-/*
- * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline
- * just like if it has been actually received from a hw source. Also
- * works for virtual interrupts.
- */
-int ipipe_trigger_irq(unsigned irq)
+void ipipe_raise_irq(unsigned int irq)
 {
 	unsigned long flags;
 
-#ifdef CONFIG_IPIPE_DEBUG
-	if (irq >= IPIPE_NR_IRQS ||
-	    (ipipe_virtual_irq_p(irq)
-	     && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)))
-		return -EINVAL;
-#endif
-
 	flags = hard_local_irq_save();
 	__ipipe_handle_irq(irq, NULL);
 	hard_local_irq_restore(flags);
-
-	return 1;
 }
+EXPORT_SYMBOL_GPL(ipipe_raise_irq);
 
 asmlinkage void __ipipe_sync_root(void)
 {
@@ -294,23 +223,14 @@ asmlinkage void __ipipe_sync_root(void)
 
 	clear_thread_flag(TIF_IRQ_SYNC);
 
-	p = ipipe_root_cpudom_ptr();
+	p = ipipe_this_cpu_root_context();
 	if (__ipipe_ipending_p(p))
-		__ipipe_sync_pipeline();
+		__ipipe_sync_stage();
 
 	hard_local_irq_restore(flags);
 }
 
-void ___ipipe_sync_pipeline(void)
-{
-	if (__ipipe_root_domain_p &&
-	    test_bit(IPIPE_SYNCDEFER_FLAG, &ipipe_root_cpudom_var(status)))
-		return;
-
-	__ipipe_sync_stage();
-}
-
-void __ipipe_disable_root_irqs_hw(void)
+asmlinkage unsigned long __ipipe_hard_save_root_irqs(void)
 {
 	/*
 	 * This code is called by the ins{bwl} routines (see
@@ -321,12 +241,15 @@ void __ipipe_disable_root_irqs_hw(void)
 	 * the real-time domain.
 	 */
 	bfin_sti(__ipipe_irq_lvmask);
-	__set_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
+	return __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_root_status) ?
+		bfin_no_irqs : bfin_irq_flags;
 }
 
-void __ipipe_enable_root_irqs_hw(void)
+asmlinkage void __ipipe_hard_restore_root_irqs(unsigned long flags)
 {
-	__clear_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
+	if (flags != bfin_no_irqs)
+		__clear_bit(IPIPE_STALL_FLAG, &__ipipe_root_status);
+
 	bfin_sti(bfin_irq_flags);
 }
 
@@ -335,64 +258,122 @@ void __ipipe_enable_root_irqs_hw(void)
  * manipulation routines, but let's prepare for SMP support in the
  * same move, preventing CPU migration as required.
  */
-void __ipipe_stall_root(void)
+void ipipe_stall_root(void)
 {
 	unsigned long *p, flags;
 
-	flags = hard_local_irq_save();
+	ipipe_root_only();
+	flags = hard_smp_local_irq_save();
 	p = &__ipipe_root_status;
 	__set_bit(IPIPE_STALL_FLAG, p);
-	hard_local_irq_restore(flags);
+	hard_smp_local_irq_restore(flags);
 }
-EXPORT_SYMBOL(__ipipe_stall_root);
+EXPORT_SYMBOL_GPL(ipipe_stall_root);
 
-unsigned long __ipipe_test_and_stall_root(void)
+unsigned long ipipe_test_and_stall_root(void)
 {
 	unsigned long *p, flags;
 	int x;
 
-	flags = hard_local_irq_save();
+	ipipe_root_only();
+	flags = hard_smp_local_irq_save();
 	p = &__ipipe_root_status;
 	x = __test_and_set_bit(IPIPE_STALL_FLAG, p);
-	hard_local_irq_restore(flags);
+	hard_smp_local_irq_restore(flags);
 
 	return x;
 }
-EXPORT_SYMBOL(__ipipe_test_and_stall_root);
+EXPORT_SYMBOL_GPL(ipipe_test_and_stall_root);
 
-unsigned long __ipipe_test_root(void)
+unsigned long ipipe_test_root(void)
 {
 	const unsigned long *p;
 	unsigned long flags;
 	int x;
 
-	flags = hard_local_irq_save_smp();
+	ipipe_root_only();
+	flags = hard_smp_local_irq_save();
 	p = &__ipipe_root_status;
 	x = test_bit(IPIPE_STALL_FLAG, p);
-	hard_local_irq_restore_smp(flags);
+	hard_smp_local_irq_restore(flags);
 
 	return x;
 }
-EXPORT_SYMBOL(__ipipe_test_root);
+EXPORT_SYMBOL_GPL(ipipe_test_root);
 
 void __ipipe_lock_root(void)
 {
 	unsigned long *p, flags;
 
-	flags = hard_local_irq_save();
+	flags = hard_smp_local_irq_save();
 	p = &__ipipe_root_status;
 	__set_bit(IPIPE_SYNCDEFER_FLAG, p);
-	hard_local_irq_restore(flags);
+	hard_smp_local_irq_restore(flags);
 }
-EXPORT_SYMBOL(__ipipe_lock_root);
+EXPORT_SYMBOL_GPL(__ipipe_lock_root);
 
 void __ipipe_unlock_root(void)
 {
 	unsigned long *p, flags;
 
-	flags = hard_local_irq_save();
+	flags = hard_smp_local_irq_save();
 	p = &__ipipe_root_status;
 	__clear_bit(IPIPE_SYNCDEFER_FLAG, p);
-	hard_local_irq_restore(flags);
+	hard_smp_local_irq_restore(flags);
+}
+EXPORT_SYMBOL_GPL(__ipipe_unlock_root);
+
+#if !defined(CONFIG_GENERIC_CLOCKEVENTS) || defined(CONFIG_TICKSOURCE_GPTMR0)
+static void icoretmr_request(struct ipipe_timer *timer, int steal)
+{
+	bfin_write_TCNTL(TMPWR);
+	CSYNC();
+	bfin_write_TSCALE(TIME_SCALE - 1);
+	bfin_write_TPERIOD(0);
+	bfin_write_TCOUNT(0);
+	CSYNC();
+}
+
+static int icoretmr_set(unsigned long evt, void *timer)
+{
+	bfin_write_TCNTL(TMPWR);
+	CSYNC();
+	bfin_write_TCOUNT(evt - 1);
+	CSYNC();
+	bfin_write_TCNTL(TMPWR | TMREN);
+
+	return 0;
+}
+
+#ifdef CONFIG_DO_IRQ_L1
+__attribute__((l1_text))
+#endif
+static void icoretmr_ack(void)
+{
+	bfin_write_TIMER_STATUS(1); /* Latch TIMIL0 */
+}
+
+static void icoretmr_release(struct ipipe_timer *timer)
+{
+	/* Power down the core timer */
+	bfin_write_TCNTL(0);
+}
+
+static struct ipipe_timer icoretmr = {
+	.irq			= IRQ_CORETMR,
+	.request		= icoretmr_request,
+	.set			= icoretmr_set,
+	.ack			= icoretmr_ack,
+	.release		= icoretmr_release,
+
+	.name			= "bfin_coretmr",
+	.rating			= 100,
+	.min_delay_ticks	= 2,
+};
+
+void bfin_ipipe_coretmr_register(void)
+{
+	icoretmr.freq = get_cclk() / TIME_SCALE;
+	ipipe_timer_register(&icoretmr);
 }
-EXPORT_SYMBOL(__ipipe_unlock_root);
+#endif /* !CLOCKEVENTS || GPTMR0 */
diff --git a/arch/blackfin/kernel/process.c b/arch/blackfin/kernel/process.c
index 6a80a9e..2e19b95 100644
--- a/arch/blackfin/kernel/process.c
+++ b/arch/blackfin/kernel/process.c
@@ -61,9 +61,7 @@ void cpu_idle(void)__attribute__((l1_text));
  */
 static void default_idle(void)
 {
-#ifdef CONFIG_IPIPE
-	ipipe_suspend_domain();
-#endif
+	__ipipe_idle();
 	hard_local_irq_disable();
 	if (!need_resched())
 		idle_with_irq_disabled();
diff --git a/arch/blackfin/kernel/time-ts.c b/arch/blackfin/kernel/time-ts.c
index 1bcf3a3..20dd021 100644
--- a/arch/blackfin/kernel/time-ts.c
+++ b/arch/blackfin/kernel/time-ts.c
@@ -17,6 +17,7 @@
 #include <linux/clocksource.h>
 #include <linux/clockchips.h>
 #include <linux/cpufreq.h>
+#include <linux/ipipe_tickdev.h>
 
 #include <asm/blackfin.h>
 #include <asm/time.h>
@@ -120,7 +121,7 @@ notrace unsigned long long sched_clock(void)
 
 #if defined(CONFIG_TICKSOURCE_GPTMR0)
 static int bfin_gptmr0_set_next_event(unsigned long cycles,
-                                     struct clock_event_device *evt)
+				     struct clock_event_device *evt)
 {
 	disable_gptimers(TIMER0bit);
 
@@ -214,6 +215,7 @@ static void __init bfin_gptmr0_clockevent_init(struct clock_event_device *evt)
 	evt->cpumask = cpumask_of(0);
 
 	clockevents_register_device(evt);
+	bfin_ipipe_coretmr_register();
 }
 #endif /* CONFIG_TICKSOURCE_GPTMR0 */
 
@@ -221,6 +223,21 @@ static void __init bfin_gptmr0_clockevent_init(struct clock_event_device *evt)
 /* per-cpu local core timer */
 static DEFINE_PER_CPU(struct clock_event_device, coretmr_events);
 
+#ifdef CONFIG_CORE_TIMER_IRQ_L1
+__attribute__((l1_text))
+#endif
+static void bfin_coretmr_ack(void)
+{
+	bfin_write_TIMER_STATUS(1); /* Latch TIMIL0 */
+}
+
+#ifdef CONFIG_IPIPE
+struct ipipe_timer bfin_coretmr_itimer = {
+	.irq = IRQ_CORETMR,
+	.ack = &bfin_coretmr_ack,
+};
+#endif /* CONFIG_IPIPE */
+
 static int bfin_coretmr_set_next_event(unsigned long cycles,
 				struct clock_event_device *evt)
 {
@@ -287,6 +304,8 @@ irqreturn_t bfin_coretmr_interrupt(int irq, void *dev_id)
 	struct clock_event_device *evt = &per_cpu(coretmr_events, cpu);
 
 	smp_mb();
+	if (clockevent_ipipe_stolen(evt) == 0)
+		bfin_coretmr_ack();
 	evt->event_handler(evt);
 
 	touch_nmi_watchdog();
@@ -320,6 +339,9 @@ void bfin_coretmr_clockevent_init(void)
 	evt->min_delta_ns = clockevent_delta2ns(100, evt);
 
 	evt->cpumask = cpumask_of(cpu);
+#ifdef CONFIG_IPIPE
+	evt->ipipe_timer = &bfin_coretmr_itimer;
+#endif /* CONFIG_IPIPE */
 
 	clockevents_register_device(evt);
 }
diff --git a/arch/blackfin/kernel/time.c b/arch/blackfin/kernel/time.c
index 2310b24..f0004ab 100644
--- a/arch/blackfin/kernel/time.c
+++ b/arch/blackfin/kernel/time.c
@@ -43,6 +43,7 @@ void __init setup_system_timer0(void)
 	set_gptimer_pwidth(TIMER0_id, 1);
 	SSYNC();
 	enable_gptimers(TIMER0bit);
+	bfin_ipipe_coretmr_register();
 }
 #else
 void __init setup_core_timer(void)
diff --git a/arch/blackfin/kernel/traps.c b/arch/blackfin/kernel/traps.c
index 655f25d..5cc238c30 100644
--- a/arch/blackfin/kernel/traps.c
+++ b/arch/blackfin/kernel/traps.c
@@ -468,10 +468,7 @@ asmlinkage notrace void trap_c(struct pt_regs *fp)
 		}
 	}
 
-#ifdef CONFIG_IPIPE
-	if (!ipipe_trap_notify(fp->seqstat & 0x3f, fp))
-#endif
-	{
+	if (__ipipe_report_trap(fp->seqstat & 0x3f, fp) == 0) {
 		info.si_signo = sig;
 		info.si_errno = 0;
 		switch (trapnr) {
diff --git a/arch/blackfin/lib/ins.S b/arch/blackfin/lib/ins.S
index 79cacce..28fb153 100644
--- a/arch/blackfin/lib/ins.S
+++ b/arch/blackfin/lib/ins.S
@@ -11,29 +11,60 @@
 
 .align 2
 
+/*
+ * Reads on the Blackfin are speculative. In Blackfin terms, this means they
+ * can be interrupted at any time (even after they have been issued on to the
+ * external bus), and re-issued after the interrupt occurs.
+ *
+ * If a FIFO is sitting on the end of the read, it will see two reads,
+ * when the core only sees one. The FIFO receives the read which is cancelled,
+ * and not delivered to the core.
+ *
+ * To solve this, interrupts are turned off before reads occur to I/O space.
+ * There are 3 versions of all these functions
+ *  - turns interrupts off every read (higher overhead, but lower latency)
+ *  - turns interrupts off every loop (low overhead, but longer latency)
+ *  - DMA version, which do not suffer from this issue. DMA versions have
+ *      different name (prefixed by dma_ ), and are located in
+ *      ../kernel/bfin_dma_5xx.c
+ * Using the dma related functions are recommended for transferring large
+ * buffers in/out of FIFOs.
+ * When the interrupt pipeline is enabled, a low overhead
+ * version is always used, which still preserves low latency for the
+ * real-time domain.
+ */
+
 #ifdef CONFIG_IPIPE
-# define DO_CLI \
-	[--sp] = rets; \
-	[--sp] = (P5:0); \
-	sp += -12; \
-	call ___ipipe_disable_root_irqs_hw; \
-	sp += 12; \
-	(P5:0) = [sp++];
-# define CLI_INNER_NOP
-#else
+
+#define COMMON_INS(func, ops) \
+ENTRY(_ins##func) \
+	P0 = R0;	/* P0 = port */ \
+	P1 = R1;	/* P1 = address */ \
+	P2 = R2;	/* P2 = count */ \
+	[--SP] = RETS; \
+	[--SP] = (P5:0); \
+	SP += -12; \
+	CALL ___ipipe_hard_save_root_irqs; \
+	SP += 12; \
+	R3 = R0;\
+	(P5:0) = [SP++];\
+	SSYNC; \
+	LSETUP(1f, 2f) LC0 = P2; \
+1:	ops; \
+2:	nop; \
+	R0 = R3;\
+	SP += -12; \
+	CALL ___ipipe_hard_restore_root_irqs; \
+	SP += 12; \
+	RETS = [SP++];\
+	RTS; \
+ENDPROC(_ins##func)
+
+#else /* !CONFIG_IPIPE */
+
 # define DO_CLI cli R3;
 # define CLI_INNER_NOP nop; nop; nop;
-#endif
-
-#ifdef CONFIG_IPIPE
-# define DO_STI \
-	sp += -12; \
-	call ___ipipe_enable_root_irqs_hw; \
-	sp += 12; \
-2:	rets = [sp++];
-#else
 # define DO_STI 2: sti R3;
-#endif
 
 #ifdef CONFIG_BFIN_INS_LOWOVERHEAD
 # define CLI_OUTER DO_CLI;
@@ -51,26 +82,6 @@
 # define STI_INNER DO_STI;
 #endif
 
-/*
- * Reads on the Blackfin are speculative. In Blackfin terms, this means they
- * can be interrupted at any time (even after they have been issued on to the
- * external bus), and re-issued after the interrupt occurs.
- *
- * If a FIFO is sitting on the end of the read, it will see two reads,
- * when the core only sees one. The FIFO receives the read which is cancelled,
- * and not delivered to the core.
- *
- * To solve this, interrupts are turned off before reads occur to I/O space.
- * There are 3 versions of all these functions
- *  - turns interrupts off every read (higher overhead, but lower latency)
- *  - turns interrupts off every loop (low overhead, but longer latency)
- *  - DMA version, which do not suffer from this issue. DMA versions have
- *      different name (prefixed by dma_ ), and are located in
- *      ../kernel/bfin_dma_5xx.c
- * Using the dma related functions are recommended for transferring large
- * buffers in/out of FIFOs.
- */
-
 #define COMMON_INS(func, ops) \
 ENTRY(_ins##func) \
 	P0 = R0;	/* P0 = port */ \
@@ -88,6 +99,8 @@ ENTRY(_ins##func) \
 	RTS; \
 ENDPROC(_ins##func)
 
+#endif /* !CONFIG_IPIPE */
+
 COMMON_INS(l, \
 	R0 = [P0]; \
 	[P1++] = R0; \
diff --git a/arch/blackfin/mach-common/dpmc.c b/arch/blackfin/mach-common/dpmc.c
index f5685a4..d343b6f 100644
--- a/arch/blackfin/mach-common/dpmc.c
+++ b/arch/blackfin/mach-common/dpmc.c
@@ -65,7 +65,7 @@ static void bfin_idle_this_cpu(void *info)
 	unsigned long iwr0, iwr1, iwr2;
 	unsigned int cpu = smp_processor_id();
 
-	local_irq_save_hw(flags);
+	flags = hard_local_irq_save();
 	bfin_iwr_set_sup0(&iwr0, &iwr1, &iwr2);
 
 	platform_clear_ipi(cpu, IRQ_SUPPLE_0);
@@ -73,7 +73,7 @@ static void bfin_idle_this_cpu(void *info)
 	asm("IDLE;");
 	bfin_iwr_restore(iwr0, iwr1, iwr2);
 
-	local_irq_restore_hw(flags);
+	hard_local_irq_restore(flags);
 }
 
 static void bfin_idle_cpu(void)
diff --git a/arch/blackfin/mach-common/entry.S b/arch/blackfin/mach-common/entry.S
index e413729..4e4eb63 100644
--- a/arch/blackfin/mach-common/entry.S
+++ b/arch/blackfin/mach-common/entry.S
@@ -824,15 +824,6 @@ _new_old_task:
 ENDPROC(_resume)
 
 ENTRY(_ret_from_exception)
-#ifdef CONFIG_IPIPE
-	p2.l = _ipipe_percpu_domain;
-	p2.h = _ipipe_percpu_domain;
-	r0.l = _ipipe_root;
-	r0.h = _ipipe_root;
-	r2 = [p2];
-	cc = r0 == r2;
-	if !cc jump 4f;  /* not on behalf of the root domain, get out */
-#endif /* CONFIG_IPIPE */
 	p2.l = lo(IPEND);
 	p2.h = hi(IPEND);
 
@@ -860,6 +851,16 @@ ENTRY(_ret_from_exception)
 	r0 = r0 & r1;
 	cc = r0 == 0;
 	if !cc jump 5f;
+#ifdef CONFIG_IPIPE
+	p2.l = _ipipe_percpu;
+	p2.h = _ipipe_percpu;
+	p2 = [p2+IPIPE_CURRENT_DOMAIN];
+	r2 = [p2+IPIPE_DOMAIN_DESC];
+	r0.l = _ipipe_root;
+	r0.h = _ipipe_root;
+	cc = r0 == r2;
+	if !cc jump 5f;  /* not on behalf of the root domain, get out */
+#endif /* CONFIG_IPIPE */
 
 	/* Set the stack for the current process */
 	r7 = sp;
diff --git a/arch/blackfin/mach-common/ints-priority.c b/arch/blackfin/mach-common/ints-priority.c
index 332dace..02e42d9 100644
--- a/arch/blackfin/mach-common/ints-priority.c
+++ b/arch/blackfin/mach-common/ints-priority.c
@@ -16,9 +16,7 @@
 #include <linux/seq_file.h>
 #include <linux/irq.h>
 #include <linux/sched.h>
-#ifdef CONFIG_IPIPE
 #include <linux/ipipe.h>
-#endif
 #include <asm/traps.h>
 #include <asm/blackfin.h>
 #include <asm/gpio.h>
@@ -297,14 +295,7 @@ static struct irq_chip bfin_internal_irqchip = {
 
 void bfin_handle_irq(unsigned irq)
 {
-#ifdef CONFIG_IPIPE
-	struct pt_regs regs;    /* Contents not used. */
-	ipipe_trace_irq_entry(irq);
-	__ipipe_handle_irq(irq, &regs);
-	ipipe_trace_irq_exit(irq);
-#else /* !CONFIG_IPIPE */
-	generic_handle_irq(irq);
-#endif  /* !CONFIG_IPIPE */
+	ipipe_handle_demuxed_irq(irq);
 }
 
 #if defined(CONFIG_BFIN_MAC) || defined(CONFIG_BFIN_MAC_MODULE)
@@ -1162,7 +1153,7 @@ void do_irq(int vec, struct pt_regs *fp)
 
 #ifdef CONFIG_IPIPE
 
-int __ipipe_get_irq_priority(unsigned irq)
+int __ipipe_get_irq_priority(unsigned int irq)
 {
 	int ient, prio;
 
@@ -1189,27 +1180,25 @@ __attribute__((l1_text))
 #endif
 asmlinkage int __ipipe_grab_irq(int vec, struct pt_regs *regs)
 {
-	struct ipipe_percpu_domain_data *p = ipipe_root_cpudom_ptr();
+	struct ipipe_percpu_domain_data *p = ipipe_this_cpu_root_context();
+	struct ipipe_percpu_data *q = __ipipe_this_cpu_ptr(&ipipe_percpu);
 	struct ipipe_domain *this_domain = __ipipe_current_domain;
-	struct ivgx *ivg_stop = ivg7_13[vec-IVG7].istop;
-	struct ivgx *ivg = ivg7_13[vec-IVG7].ifirst;
+	struct pt_regs *tick_regs;
 	int irq, s = 0;
 
 	irq = vec_to_irq(vec);
 	if (irq == -1)
 		return 0;
 
-	if (irq == IRQ_SYSTMR) {
-#if !defined(CONFIG_GENERIC_CLOCKEVENTS) || defined(CONFIG_TICKSOURCE_GPTMR0)
-		bfin_write_TIMER_STATUS(1); /* Latch TIMIL0 */
-#endif
+	if (irq == q->hrtimer_irq || q->hrtimer_irq == -1) {
 		/* This is basically what we need from the register frame. */
-		__raw_get_cpu_var(__ipipe_tick_regs).ipend = regs->ipend;
-		__raw_get_cpu_var(__ipipe_tick_regs).pc = regs->pc;
+		tick_regs = &q->tick_regs;
+		tick_regs->ipend = regs->ipend;
+		tick_regs->pc = regs->pc;
 		if (this_domain != ipipe_root_domain)
-			__raw_get_cpu_var(__ipipe_tick_regs).ipend &= ~0x10;
+			tick_regs->ipend &= ~0x10;
 		else
-			__raw_get_cpu_var(__ipipe_tick_regs).ipend |= 0x10;
+			tick_regs->ipend |= 0x10;
 	}
 
 	/*
@@ -1238,7 +1227,7 @@ asmlinkage int __ipipe_grab_irq(int vec, struct pt_regs *regs)
 
 	if (user_mode(regs) &&
 	    !ipipe_test_foreign_stack() &&
-	    (current->ipipe_flags & PF_EVTRET) != 0) {
+	    (current->ipipe.flags & PF_MAYDAY) != 0) {
 		/*
 		 * Testing for user_regs() does NOT fully eliminate
 		 * foreign stack contexts, because of the forged
@@ -1250,13 +1239,13 @@ asmlinkage int __ipipe_grab_irq(int vec, struct pt_regs *regs)
 		 * which case user_mode() is now true, and the event
 		 * gets dispatched spuriously.
 		 */
-		current->ipipe_flags &= ~PF_EVTRET;
-		__ipipe_dispatch_event(IPIPE_EVENT_RETURN, regs);
+		current->ipipe.flags &= ~PF_MAYDAY;
+		__ipipe_notify_trap(IPIPE_TRAP_MAYDAY, regs);
 	}
 
 	if (this_domain == ipipe_root_domain) {
 		set_thread_flag(TIF_IRQ_SYNC);
-		if (!s) {
+		if (s == 0) {
 			__clear_bit(IPIPE_SYNCDEFER_FLAG, &p->status);
 			return !test_bit(IPIPE_STALL_FLAG, &p->status);
 		}
diff --git a/drivers/tty/serial/bfin_uart.c b/drivers/tty/serial/bfin_uart.c
index 66afb98..e840c5b 100644
--- a/drivers/tty/serial/bfin_uart.c
+++ b/drivers/tty/serial/bfin_uart.c
@@ -1127,6 +1127,54 @@ static void bfin_serial_console_putchar(struct uart_port *port, int ch)
 	UART_PUT_CHAR(uart, ch);
 }
 
+#ifdef CONFIG_IPIPE
+
+#include <stdarg.h>
+
+void __ipipe_serial_debug(const char *fmt, ...)
+{
+	struct bfin_serial_port *uart = bfin_serial_ports[0];
+	unsigned short status, tmp;
+	int flags, i, count;
+	char buf[128];
+	va_list ap;
+
+	if (uart == NULL || port_membase(uart) == NULL)
+		return;
+
+	va_start(ap, fmt);
+	vsprintf(buf, fmt, ap);
+	va_end(ap);
+	count = strlen(buf);
+
+	flags = hard_local_irq_save();
+
+	for (i = 0; i < count; i++) {
+		do {
+			status = UART_GET_LSR(uart);
+		} while (!(status & THRE));
+
+#ifndef CONFIG_BF54x
+		tmp = UART_GET_LCR(uart);
+		tmp &= ~DLAB;
+		UART_PUT_LCR(uart, tmp);
+#endif
+
+		UART_PUT_CHAR(uart, buf[i]);
+		if (buf[i] == '\n') {
+			do {
+				status = UART_GET_LSR(uart);
+			} while(!(status & THRE));
+			UART_PUT_CHAR(uart, '\r');
+		}
+	}
+
+	hard_local_irq_restore(flags);
+}
+EXPORT_SYMBOL_GPL(__ipipe_serial_debug);
+
+#endif /* CONFIG_IPIPE */
+
 #endif /* defined (CONFIG_SERIAL_BFIN_CONSOLE) ||
 		 defined (CONFIG_EARLY_PRINTK) */
 
diff --git a/drivers/tty/serial/8250.c b/drivers/tty/serial/8250.c
index 70585b6..43e2861 100644
--- a/drivers/tty/serial/8250.c
+++ b/drivers/tty/serial/8250.c
@@ -3135,6 +3135,51 @@ static int serial8250_resume(struct platform_device *dev)
 	return 0;
 }
 
+#if defined(CONFIG_IPIPE_DEBUG) && defined(CONFIG_SERIAL_8250_CONSOLE)
+
+#include <stdarg.h>
+
+void __weak __ipipe_serial_debug(const char *fmt, ...)
+{
+        struct uart_8250_port *up = &serial8250_ports[0];
+        unsigned int ier, count;
+        unsigned long flags;
+        char buf[128];
+        va_list ap;
+
+        va_start(ap, fmt);
+        vsprintf(buf, fmt, ap);
+        va_end(ap);
+        count = strlen(buf);
+
+        touch_nmi_watchdog();
+
+        flags = hard_local_irq_save();
+
+        /*
+         *      First save the IER then disable the interrupts
+        */
+        ier = serial_in(up, UART_IER);
+
+        if (up->capabilities & UART_CAP_UUE)
+                serial_out(up, UART_IER, UART_IER_UUE);
+        else
+                serial_out(up, UART_IER, 0);
+
+        uart_console_write(&up->port, buf, count, serial8250_console_putchar);
+
+        /*
+         *      Finally, wait for transmitter to become empty
+         *      and restore the IER
+         */
+        wait_for_xmitr(up, BOTH_EMPTY);
+        serial_out(up, UART_IER, ier);
+
+        hard_local_irq_restore(flags);
+}
+
+#endif
+
 static struct platform_driver serial8250_isa_driver = {
 	.probe		= serial8250_probe,
 	.remove		= __devexit_p(serial8250_remove),
diff --git a/fs/exec.c b/fs/exec.c
index 160cd2f..fd1ce88 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -816,6 +816,7 @@ static int exec_mmap(struct mm_struct *mm)
 {
 	struct task_struct *tsk;
 	struct mm_struct * old_mm, *active_mm;
+	unsigned long flags;
 
 	/* Notify parent that we're no longer interested in the old VM */
 	tsk = current;
@@ -839,8 +840,10 @@ static int exec_mmap(struct mm_struct *mm)
 	task_lock(tsk);
 	active_mm = tsk->active_mm;
 	tsk->mm = mm;
+	ipipe_mm_switch_protect(flags);
 	tsk->active_mm = mm;
 	activate_mm(active_mm, mm);
+	ipipe_mm_switch_unprotect(flags);
 	task_unlock(tsk);
 	arch_pick_mmap_layout(mm);
 	if (old_mm) {
diff --git a/fs/proc/array.c b/fs/proc/array.c
index 3a1dafd..c0b7439 100644
--- a/fs/proc/array.c
+++ b/fs/proc/array.c
@@ -142,6 +142,10 @@ static const char * const task_state_array[] = {
 	"x (dead)",		/*  64 */
 	"K (wakekill)",		/* 128 */
 	"W (waking)",		/* 256 */
+#ifdef CONFIG_IPIPE
+	"A (atomic switch)",	/* 512 */
+	"N (wakeup disabled)",	/* 1024 */
+#endif
 };
 
 static inline const char *get_task_state(struct task_struct *tsk)
diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h
index e37963c..2d7b7f6 100644
--- a/include/asm-generic/atomic.h
+++ b/include/asm-generic/atomic.h
@@ -67,11 +67,11 @@ static inline int atomic_add_return(int i, atomic_t *v)
 	unsigned long flags;
 	int temp;
 
-	raw_local_irq_save(flags); /* Don't trace it in an irqsoff handler */
+	flags = hard_local_irq_save(); /* Don't trace it in an irqsoff handler */
 	temp = v->counter;
 	temp += i;
 	v->counter = temp;
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 
 	return temp;
 }
@@ -90,11 +90,11 @@ static inline int atomic_sub_return(int i, atomic_t *v)
 	unsigned long flags;
 	int temp;
 
-	raw_local_irq_save(flags); /* Don't trace it in an irqsoff handler */
+	flags = hard_local_irq_save();
 	temp = v->counter;
 	temp -= i;
 	v->counter = temp;
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 
 	return temp;
 }
@@ -163,9 +163,9 @@ static inline void atomic_clear_mask(unsigned long mask, atomic_t *v)
 	unsigned long flags;
 
 	mask = ~mask;
-	raw_local_irq_save(flags); /* Don't trace it in a irqsoff handler */
+	flags = hard_local_irq_save();
 	v->counter &= mask;
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 #endif
 
diff --git a/include/asm-generic/bitops/atomic.h b/include/asm-generic/bitops/atomic.h
index ecc44a8..72ceef0 100644
--- a/include/asm-generic/bitops/atomic.h
+++ b/include/asm-generic/bitops/atomic.h
@@ -21,20 +21,20 @@ extern arch_spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] __lock_aligned;
  * this is the substitute */
 #define _atomic_spin_lock_irqsave(l,f) do {	\
 	arch_spinlock_t *s = ATOMIC_HASH(l);	\
-	local_irq_save(f);			\
+	(f) = hard_local_irq_save();		\
 	arch_spin_lock(s);			\
 } while(0)
 
 #define _atomic_spin_unlock_irqrestore(l,f) do {	\
 	arch_spinlock_t *s = ATOMIC_HASH(l);		\
 	arch_spin_unlock(s);				\
-	local_irq_restore(f);				\
+	hard_local_irq_restore(f);			\
 } while(0)
 
 
 #else
-#  define _atomic_spin_lock_irqsave(l,f) do { local_irq_save(f); } while (0)
-#  define _atomic_spin_unlock_irqrestore(l,f) do { local_irq_restore(f); } while (0)
+#  define _atomic_spin_lock_irqsave(l,f) do { (f) = hard_local_irq_save(); } while (0)
+#  define _atomic_spin_unlock_irqrestore(l,f) do { hard_local_irq_restore(f); } while (0)
 #endif
 
 /*
diff --git a/include/asm-generic/cmpxchg-local.h b/include/asm-generic/cmpxchg-local.h
index 2533fdd..6559e2d 100644
--- a/include/asm-generic/cmpxchg-local.h
+++ b/include/asm-generic/cmpxchg-local.h
@@ -21,7 +21,7 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr,
 	if (size == 8 && sizeof(unsigned long) != 8)
 		wrong_size_cmpxchg(ptr);
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	switch (size) {
 	case 1: prev = *(u8 *)ptr;
 		if (prev == old)
@@ -42,7 +42,7 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr,
 	default:
 		wrong_size_cmpxchg(ptr);
 	}
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 	return prev;
 }
 
@@ -55,11 +55,11 @@ static inline u64 __cmpxchg64_local_generic(volatile void *ptr,
 	u64 prev;
 	unsigned long flags;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	prev = *(u64 *)ptr;
 	if (prev == old)
 		*(u64 *)ptr = new;
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 	return prev;
 }
 
diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h
index d17784e..d8e2912 100644
--- a/include/asm-generic/percpu.h
+++ b/include/asm-generic/percpu.h
@@ -67,6 +67,22 @@ extern unsigned long __per_cpu_offset[NR_CPUS];
 
 #define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
 #define __raw_get_cpu_var(var) (*__this_cpu_ptr(&(var)))
+#ifdef CONFIG_IPIPE
+#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP)
+extern int __ipipe_check_percpu_access(void);
+#define __ipipe_cpu_offset					\
+	({							\
+		WARN_ON_ONCE(__ipipe_check_percpu_access());	\
+		__my_cpu_offset;				\
+	})
+#else
+#define __ipipe_cpu_offset  __my_cpu_offset
+#endif
+#ifndef __ipipe_this_cpu_ptr
+#define __ipipe_this_cpu_ptr(ptr)  SHIFT_PERCPU_PTR(ptr, __ipipe_cpu_offset)
+#endif
+#define __ipipe_this_cpu_read(var) (*__ipipe_this_cpu_ptr(&(var)))
+#endif /* CONFIG_IPIPE */
 
 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
 extern void setup_per_cpu_areas(void);
@@ -82,6 +98,8 @@ extern void setup_per_cpu_areas(void);
 #define per_cpu(var, cpu)	(*((void)(cpu), VERIFY_PERCPU_PTR(&(var))))
 #define __get_cpu_var(var)	(*VERIFY_PERCPU_PTR(&(var)))
 #define __raw_get_cpu_var(var)	(*VERIFY_PERCPU_PTR(&(var)))
+#define __ipipe_this_cpu_ptr(ptr)  VERIFY_PERCPU_PTR(ptr)
+#define __ipipe_this_cpu_read(var) (*__ipipe_this_cpu_ptr(&(var)))
 #define this_cpu_ptr(ptr)	per_cpu_ptr(ptr, 0)
 #define __this_cpu_ptr(ptr)	this_cpu_ptr(ptr)
 
diff --git a/include/asm-generic/resource.h b/include/asm-generic/resource.h
index 61fa862..d36ceb9 100644
--- a/include/asm-generic/resource.h
+++ b/include/asm-generic/resource.h
@@ -58,6 +58,14 @@
 #endif
 
 /*
+ * Limit the stack by to some sane default: root can always
+ * increase this limit if needed..  8MB seems reasonable.
+ */
+#ifndef _STK_LIM
+# define _STK_LIM		(8*1024*1024)
+#endif
+
+/*
  * RLIMIT_STACK default maximum - some architectures override it:
  */
 #ifndef _STK_LIM_MAX
diff --git a/include/ipipe/setup.h b/include/ipipe/setup.h
new file mode 100644
index 0000000..c2bc521
--- /dev/null
+++ b/include/ipipe/setup.h
@@ -0,0 +1,10 @@
+#ifndef _IPIPE_SETUP_H
+#define _IPIPE_SETUP_H
+
+/*
+ * Placeholders for setup hooks defined by client domains.
+ */
+
+static inline void __ipipe_early_client_setup(void) { }
+
+#endif /* !_IPIPE_SETUP_H */
diff --git a/include/ipipe/thread_info.h b/include/ipipe/thread_info.h
new file mode 100644
index 0000000..1f6e9c3
--- /dev/null
+++ b/include/ipipe/thread_info.h
@@ -0,0 +1,14 @@
+#ifndef _IPIPE_THREAD_INFO_H
+#define _IPIPE_THREAD_INFO_H
+
+/*
+ * Placeholder for private thread information defined by client
+ * domains.
+ */
+
+struct ipipe_threadinfo {
+};
+
+static inline void __ipipe_init_threadinfo(struct ipipe_threadinfo *p) { }
+
+#endif /* !_IPIPE_THREAD_INFO_H */
diff --git a/include/linux/clockchips.h b/include/linux/clockchips.h
index 81e803e..254a759 100644
--- a/include/linux/clockchips.h
+++ b/include/linux/clockchips.h
@@ -105,6 +105,15 @@ struct clock_event_device {
 	int			irq;
 	const struct cpumask	*cpumask;
 	struct list_head	list;
+
+#ifdef CONFIG_IPIPE
+	struct ipipe_timer      *ipipe_timer;
+	unsigned                ipipe_stolen;
+
+#define clockevent_ipipe_stolen(evt) ((evt)->ipipe_stolen)
+#else
+#define clockevent_ipipe_stolen(evt) (0)
+#endif /* !CONFIG_IPIPE */
 } ____cacheline_aligned;
 
 /*
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index f743883..9596eed 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -210,6 +210,7 @@ extern void irq_exit(void);
 
 #define nmi_enter()						\
 	do {							\
+		__ipipe_nmi_enter();				\
 		ftrace_nmi_enter();				\
 		BUG_ON(in_nmi());				\
 		add_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET);	\
@@ -226,6 +227,7 @@ extern void irq_exit(void);
 		BUG_ON(!in_nmi());				\
 		sub_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET);	\
 		ftrace_nmi_exit();				\
+		__ipipe_nmi_exit();				\
 	} while (0)
 
 #endif /* LINUX_HARDIRQ_H */
diff --git a/include/linux/i8253.h b/include/linux/i8253.h
index e6bb36a..898a91a 100644
--- a/include/linux/i8253.h
+++ b/include/linux/i8253.h
@@ -12,6 +12,7 @@
 #include <linux/param.h>
 #include <linux/spinlock.h>
 #include <linux/timex.h>
+#include <linux/ipipe_lock.h>
 
 /* i8253A PIT registers */
 #define PIT_MODE	0x43
@@ -20,7 +21,7 @@
 
 #define PIT_LATCH	((PIT_TICK_RATE + HZ/2) / HZ)
 
-extern raw_spinlock_t i8253_lock;
+IPIPE_DECLARE_RAW_SPINLOCK(i8253_lock);
 extern struct clock_event_device i8253_clockevent;
 extern void clockevent_i8253_init(bool oneshot);
 
diff --git a/include/linux/ipipe.h b/include/linux/ipipe.h
new file mode 100644
index 0000000..f282637
--- /dev/null
+++ b/include/linux/ipipe.h
@@ -0,0 +1,399 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe.h
+ *
+ * Copyright (C) 2002-2007 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_H
+#define __LINUX_IPIPE_H
+
+#include <linux/spinlock.h>
+#include <linux/cache.h>
+#include <linux/percpu.h>
+#include <linux/irq.h>
+#include <linux/thread_info.h>
+#include <linux/ipipe_base.h>
+#include <linux/ipipe_debug.h>
+#include <asm/ptrace.h>
+#include <asm/ipipe.h>
+
+#ifdef CONFIG_IPIPE
+
+#include <linux/ipipe_domain.h>
+
+/* ipipe_set_hooks(..., enables) */
+#define IPIPE_SYSCALL	__IPIPE_SYSCALL_E
+#define IPIPE_TRAP	__IPIPE_TRAP_E
+#define IPIPE_KEVENT	__IPIPE_KEVENT_E
+
+struct ipipe_sysinfo {
+	int sys_nr_cpus;	/* Number of CPUs on board */
+	int sys_hrtimer_irq;	/* hrtimer device IRQ */
+	u64 sys_hrtimer_freq;	/* hrtimer device frequency */
+	u64 sys_hrclock_freq;	/* hrclock device frequency */
+	u64 sys_cpu_freq;	/* CPU frequency (Hz) */
+	struct ipipe_arch_sysinfo arch;
+};
+
+struct ipipe_work_header {
+	size_t size;
+	void (*handler)(struct ipipe_work_header *work);
+};
+
+extern unsigned int __ipipe_printk_virq;
+
+void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned int irq);
+
+void __ipipe_complete_domain_migration(void);
+
+int __ipipe_switch_tail(void);
+
+int __ipipe_migrate_head(void);
+
+void __ipipe_reenter_root(void);
+
+int __ipipe_disable_ondemand_mappings(struct task_struct *p);
+int __ipipe_pin_vma(struct mm_struct *mm, struct vm_area_struct *vma);
+
+#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH
+
+#define prepare_arch_switch(next)			\
+	do {						\
+		hard_local_irq_enable();		\
+		__ipipe_report_schedule(current, next);	\
+	} while(0)
+
+static inline struct mm_struct *ipipe_get_active_mm(void)
+{
+	return __this_cpu_read(ipipe_percpu.active_mm);
+}
+
+#else /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */
+
+#define prepare_arch_switch(next)			\
+	do {						\
+		__ipipe_report_schedule(current, next);	\
+		hard_local_irq_disable();		\
+	} while(0)
+
+#define ipipe_get_active_mm()  (current->active_mm)
+
+#endif /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */
+
+static inline void __ipipe_nmi_enter(void)
+{
+	__this_cpu_write(ipipe_percpu.nmi_state, __ipipe_root_status);
+	__set_bit(IPIPE_STALL_FLAG, &__ipipe_root_status);
+	ipipe_save_context_nmi();
+}
+
+static inline void __ipipe_nmi_exit(void)
+{
+	ipipe_restore_context_nmi();
+	if (!test_bit(IPIPE_STALL_FLAG, __this_cpu_ptr(&ipipe_percpu.nmi_state)))
+		__clear_bit(IPIPE_STALL_FLAG, &__ipipe_root_status);
+}
+
+/* KVM-side calls, hw IRQs off. */
+static inline void __ipipe_enter_vm(struct ipipe_vm_notifier *vmf)
+{
+	struct ipipe_percpu_data *p;
+
+	p = __ipipe_this_cpu_ptr(&ipipe_percpu);
+	p->vm_notifier = vmf;
+	barrier();
+}
+
+static inline void __ipipe_exit_vm(void)
+{
+	struct ipipe_percpu_data *p;
+
+	p = __ipipe_this_cpu_ptr(&ipipe_percpu);
+	p->vm_notifier = NULL;
+	barrier();
+}
+
+/* Client-side call, hw IRQs off. */
+void __ipipe_notify_vm_preemption(void);
+
+static inline void __ipipe_sync_pipeline(struct ipipe_domain *top)
+{
+	if (__ipipe_current_domain != top) {
+		__ipipe_do_sync_pipeline(top);
+		return;
+	}
+	if (!test_bit(IPIPE_STALL_FLAG, &ipipe_this_cpu_context(top)->status))
+		__ipipe_sync_stage();
+}
+
+void ipipe_register_head(struct ipipe_domain *ipd,
+			 const char *name);
+
+void ipipe_unregister_head(struct ipipe_domain *ipd);
+
+int ipipe_request_irq(struct ipipe_domain *ipd,
+		      unsigned int irq,
+		      ipipe_irq_handler_t handler,
+		      void *cookie,
+		      ipipe_irq_ackfn_t ackfn);
+
+void ipipe_free_irq(struct ipipe_domain *ipd,
+		    unsigned int irq);
+
+void ipipe_raise_irq(unsigned int irq);
+
+void ipipe_set_hooks(struct ipipe_domain *ipd,
+		     int enables);
+
+unsigned int ipipe_alloc_virq(void);
+
+void ipipe_free_virq(unsigned int virq);
+
+static inline void ipipe_post_irq_head(unsigned int irq)
+{
+	__ipipe_set_irq_pending(ipipe_head_domain, irq);
+}
+
+static inline void ipipe_post_irq_root(unsigned int irq)
+{
+	__ipipe_set_irq_pending(&ipipe_root, irq);
+}
+
+static inline void ipipe_stall_head(void)
+{
+	hard_local_irq_disable();
+	__set_bit(IPIPE_STALL_FLAG, &__ipipe_head_status);
+}
+
+static inline unsigned long ipipe_test_and_stall_head(void)
+{
+	hard_local_irq_disable();
+	return __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_head_status);
+}
+
+static inline unsigned long ipipe_test_head(void)
+{
+	unsigned long flags, ret;
+
+	flags = hard_smp_local_irq_save();
+	ret = test_bit(IPIPE_STALL_FLAG, &__ipipe_head_status);
+	hard_smp_local_irq_restore(flags);
+
+	return ret;
+}
+
+void ipipe_unstall_head(void);
+
+void __ipipe_restore_head(unsigned long x);
+
+static inline void ipipe_restore_head(unsigned long x)
+{
+	ipipe_check_irqoff();
+	if ((x ^ test_bit(IPIPE_STALL_FLAG, &__ipipe_head_status)) & 1)
+		__ipipe_restore_head(x);
+}
+
+void __ipipe_post_work_root(struct ipipe_work_header *work);
+
+#define ipipe_post_work_root(p, header)			\
+	do {						\
+		void header_not_at_start(void);		\
+		if (offsetof(typeof(*(p)), header)) {	\
+			header_not_at_start();		\
+		}					\
+		__ipipe_post_work_root(&(p)->header);	\
+	} while (0)
+
+int ipipe_get_sysinfo(struct ipipe_sysinfo *sysinfo);
+
+unsigned long ipipe_critical_enter(void (*syncfn)(void));
+
+void ipipe_critical_exit(unsigned long flags);
+
+void ipipe_prepare_panic(void);
+
+static inline void ipipe_set_foreign_stack(struct ipipe_domain *ipd)
+{
+	/* Must be called hw interrupts off. */
+	__set_bit(IPIPE_NOSTACK_FLAG, &ipipe_this_cpu_context(ipd)->status);
+}
+
+static inline void ipipe_clear_foreign_stack(struct ipipe_domain *ipd)
+{
+	/* Must be called hw interrupts off. */
+	__clear_bit(IPIPE_NOSTACK_FLAG, &ipipe_this_cpu_context(ipd)->status);
+}
+
+static inline int ipipe_test_foreign_stack(void)
+{
+	/* Must be called hw interrupts off. */
+	return test_bit(IPIPE_NOSTACK_FLAG, &__ipipe_current_context->status);
+}
+
+#ifndef ipipe_safe_current
+#define ipipe_safe_current()						\
+	({								\
+		struct task_struct *__p__;				\
+		unsigned long __flags__;				\
+		__flags__ = hard_smp_local_irq_save();			\
+		__p__ = ipipe_test_foreign_stack() ? &init_task : current; \
+		hard_smp_local_irq_restore(__flags__);			\
+		__p__;							\
+	})
+#endif
+
+#ifdef CONFIG_SMP
+void ipipe_set_irq_affinity(unsigned int irq, cpumask_t cpumask);
+void ipipe_send_ipi(unsigned int ipi, cpumask_t cpumask);
+#else  /* !CONFIG_SMP */
+static inline
+void ipipe_set_irq_affinity(unsigned int irq, cpumask_t cpumask) { }
+static inline void ipipe_send_ipi(unsigned int ipi, cpumask_t cpumask) { }
+#endif	/* CONFIG_SMP */
+
+static inline void ipipe_restore_root_nosync(unsigned long x)
+{
+	unsigned long flags;
+
+	flags = hard_smp_local_irq_save();
+	__ipipe_restore_root_nosync(x);
+	hard_smp_local_irq_restore(flags);
+}
+
+/* Must be called hw IRQs off. */
+static inline void ipipe_lock_irq(unsigned int irq)
+{
+	struct ipipe_domain *ipd = __ipipe_current_domain;
+	if (ipd == ipipe_root_domain)
+		__ipipe_lock_irq(irq);
+}
+
+/* Must be called hw IRQs off. */
+static inline void ipipe_unlock_irq(unsigned int irq)
+{
+	struct ipipe_domain *ipd = __ipipe_current_domain;
+	if (ipd == ipipe_root_domain)
+		__ipipe_unlock_irq(irq);
+}
+
+static inline struct ipipe_threadinfo *ipipe_current_threadinfo(void)
+{
+	return &current_thread_info()->ipipe_data;
+}
+
+#define ipipe_task_threadinfo(p) (&task_thread_info(p)->ipipe_data)
+
+static inline void ipipe_enable_irq(unsigned int irq)
+{
+	struct irq_desc *desc = irq_to_desc(irq);
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+
+	if (WARN_ON_ONCE(chip->irq_unmask == NULL))
+		return;
+
+	chip->irq_unmask(&desc->irq_data);
+}
+
+static inline void ipipe_disable_irq(unsigned int irq)
+{
+	struct irq_desc *desc = irq_to_desc(irq);
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+
+	if (WARN_ON_ONCE(chip->irq_mask == NULL))
+		return;
+
+	chip->irq_mask(&desc->irq_data);
+}
+
+static inline void ipipe_end_irq(unsigned int irq)
+{
+	struct irq_desc *desc = irq_to_desc(irq);
+
+	desc->ipipe_end(irq, desc);
+}
+
+static inline int ipipe_chained_irq_p(unsigned int irq)
+{
+	void __ipipe_chained_irq(unsigned irq, struct irq_desc *desc);
+	struct irq_desc *desc = irq_to_desc(irq);
+
+	return desc->handle_irq == __ipipe_chained_irq;
+}
+
+static inline void ipipe_handle_demuxed_irq(unsigned int cascade_irq)
+{
+	ipipe_trace_irq_entry(cascade_irq);
+	__ipipe_dispatch_irq(cascade_irq, IPIPE_IRQF_NOSYNC);
+	ipipe_trace_irq_exit(cascade_irq);
+}
+
+#define ipipe_enable_notifier(p)			\
+	do {						\
+		barrier();				\
+		(p)->ipipe.flags |= PF_EVNOTIFY;	\
+	} while (0)
+
+#define ipipe_disable_notifier(p)				\
+	do {							\
+		barrier();					\
+		(p)->ipipe.flags &= ~(PF_EVNOTIFY|PF_MAYDAY);	\
+	} while (0)
+
+#define ipipe_notifier_enabled_p(p)			\
+	(((p)->ipipe.flags) & PF_EVNOTIFY)
+
+#define ipipe_raise_mayday(p)				\
+	do {						\
+		ipipe_check_irqoff();			\
+		if (ipipe_notifier_enabled_p(p))	\
+			(p)->ipipe.flags |= PF_MAYDAY;	\
+	} while (0)
+
+#include <linux/ipipe_compat.h>
+
+#else	/* !CONFIG_IPIPE */
+
+#define __ipipe_root_p		1
+#define ipipe_root_p		1
+
+static inline void __ipipe_complete_domain_migration(void) { }
+
+static inline int __ipipe_switch_tail(void)
+{
+	return 0;
+}
+
+static inline void __ipipe_nmi_enter(void) { }
+
+static inline void __ipipe_nmi_exit(void) { }
+
+#define ipipe_safe_current()	current
+#define ipipe_processor_id()	smp_processor_id()
+
+static inline int ipipe_test_foreign_stack(void)
+{
+	return 0;
+}
+
+static inline void ipipe_lock_irq(unsigned int irq) { }
+
+static inline void ipipe_unlock_irq(unsigned int irq) { }
+
+#endif	/* !CONFIG_IPIPE */
+
+#endif	/* !__LINUX_IPIPE_H */
diff --git a/include/linux/ipipe_base.h b/include/linux/ipipe_base.h
new file mode 100644
index 0000000..2a434cb
--- /dev/null
+++ b/include/linux/ipipe_base.h
@@ -0,0 +1,361 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_base.h
+ *
+ * Copyright (C) 2002-2012 Philippe Gerum.
+ *               2007 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_BASE_H
+#define __LINUX_IPIPE_BASE_H
+
+struct kvm_vcpu;
+struct ipipe_vm_notifier;
+struct irq_desc;
+
+#ifdef CONFIG_IPIPE
+
+#define IPIPE_CORE_APIREV  1
+
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+void ipipe_root_only(void);
+#else /* !CONFIG_IPIPE_DEBUG_CONTEXT */
+static inline void ipipe_root_only(void) { }
+#endif /* !CONFIG_IPIPE_DEBUG_CONTEXT */
+
+typedef void (*ipipe_irq_handler_t)(unsigned int irq,
+				    void *cookie);
+
+void ipipe_unstall_root(void);
+
+void ipipe_restore_root(unsigned long x);
+
+#include <asm/ipipe_base.h>
+#include <linux/compiler.h>
+
+#define __bpl_up(x)		(((x)+(BITS_PER_LONG-1)) & ~(BITS_PER_LONG-1))
+/* Number of virtual IRQs (must be a multiple of BITS_PER_LONG) */
+#define IPIPE_NR_VIRQS		BITS_PER_LONG
+/* First virtual IRQ # (must be aligned on BITS_PER_LONG) */
+#define IPIPE_VIRQ_BASE		__bpl_up(IPIPE_NR_XIRQS)
+/* Total number of IRQ slots */
+#define IPIPE_NR_IRQS		(IPIPE_VIRQ_BASE+IPIPE_NR_VIRQS)
+
+static inline int ipipe_virtual_irq_p(unsigned int irq)
+{
+	return irq >= IPIPE_VIRQ_BASE && irq < IPIPE_NR_IRQS;
+}
+
+#define IPIPE_IRQ_LOMAPSZ	(IPIPE_NR_IRQS / BITS_PER_LONG)
+#if IPIPE_IRQ_LOMAPSZ > BITS_PER_LONG
+/*
+ * We need a 3-level mapping. This allows us to handle up to 32k IRQ
+ * vectors on 32bit machines, 256k on 64bit ones.
+ */
+#define __IPIPE_3LEVEL_IRQMAP	1
+#define IPIPE_IRQ_MDMAPSZ	(__bpl_up(IPIPE_IRQ_LOMAPSZ) / BITS_PER_LONG)
+#else
+/*
+ * 2-level mapping is enough. This allows us to handle up to 1024 IRQ
+ * vectors on 32bit machines, 4096 on 64bit ones.
+ */
+#define __IPIPE_2LEVEL_IRQMAP	1
+#endif
+
+/* Per-cpu pipeline status */
+#define IPIPE_STALL_FLAG	0 /* interrupts (virtually) disabled. */
+#define IPIPE_NOSTACK_FLAG	1 /* running on foreign stack. */
+#define IPIPE_STALL_MASK	(1L << IPIPE_STALL_FLAG)
+#define IPIPE_NOSTACK_MASK	(1L << IPIPE_NOSTACK_FLAG)
+
+/* Interrupt control bits */
+#define IPIPE_HANDLE_FLAG	0
+#define IPIPE_STICKY_FLAG	1
+#define IPIPE_LOCK_FLAG		2
+#define IPIPE_HANDLE_MASK	(1 << IPIPE_HANDLE_FLAG)
+#define IPIPE_STICKY_MASK	(1 << IPIPE_STICKY_FLAG)
+#define IPIPE_LOCK_MASK		(1 << IPIPE_LOCK_FLAG)
+
+struct pt_regs;
+struct ipipe_domain;
+
+struct ipipe_trap_data {
+	int exception;
+	struct pt_regs *regs;
+};
+
+#define IPIPE_KEVT_SCHEDULE	0
+#define IPIPE_KEVT_SIGWAKE	1
+#define IPIPE_KEVT_SETSCHED	2
+#define IPIPE_KEVT_EXIT		3
+#define IPIPE_KEVT_CLEANUP	4
+#define IPIPE_KEVT_HOSTRT	5
+
+struct ipipe_vm_notifier {
+	void (*handler)(struct ipipe_vm_notifier *nfy);
+};
+
+void __ipipe_init_early(void);
+
+void __ipipe_init(void);
+
+#ifdef CONFIG_PROC_FS
+void __ipipe_init_proc(void);
+#ifdef CONFIG_IPIPE_TRACE
+void __ipipe_init_tracer(void);
+#else /* !CONFIG_IPIPE_TRACE */
+static inline void __ipipe_init_tracer(void) { }
+#endif /* CONFIG_IPIPE_TRACE */
+#else	/* !CONFIG_PROC_FS */
+static inline void __ipipe_init_proc(void) { }
+#endif	/* CONFIG_PROC_FS */
+
+void __ipipe_restore_root_nosync(unsigned long x);
+
+#define IPIPE_IRQF_NOACK    0x1
+#define IPIPE_IRQF_NOSYNC   0x2
+
+void __ipipe_dispatch_irq(unsigned int irq, int flags);
+
+void __ipipe_do_sync_stage(void);
+
+void __ipipe_do_sync_pipeline(struct ipipe_domain *top);
+
+void __ipipe_lock_irq(unsigned int irq);
+
+void __ipipe_unlock_irq(unsigned int irq);
+
+void __ipipe_dispatch_irq_fast(unsigned int irq);
+
+void __ipipe_do_critical_sync(unsigned int irq, void *cookie);
+
+void __ipipe_ack_edge_irq(unsigned int irq, struct irq_desc *desc);
+
+void __ipipe_nop_irq(unsigned int irq, struct irq_desc *desc);
+
+static inline void __ipipe_idle(void)
+{
+	ipipe_unstall_root();
+}
+
+#ifndef __ipipe_sync_check
+#define __ipipe_sync_check	1
+#endif
+
+static inline void __ipipe_sync_stage(void)
+{
+	if (likely(__ipipe_sync_check))
+		__ipipe_do_sync_stage();
+}
+
+#ifndef __ipipe_check_root_resched
+#ifdef CONFIG_PREEMPT
+#define __ipipe_check_root_resched()	\
+	(preempt_count() == 0 && need_resched())
+#else
+#define __ipipe_check_root_resched()	0
+#endif
+#endif
+
+#ifndef __ipipe_run_irqtail
+#define __ipipe_run_irqtail(irq) do { } while(0)
+#endif
+
+void __ipipe_flush_printk(unsigned int irq, void *cookie);
+
+void __ipipe_pin_range_globally(unsigned long start,
+				unsigned long end);
+
+#define hard_preempt_disable()				\
+	({						\
+		unsigned long __flags__;		\
+		__flags__ = hard_local_irq_save();	\
+		if (__ipipe_root_p)			\
+			preempt_disable();		\
+		__flags__;				\
+	})
+
+#define hard_preempt_enable(flags)			\
+	do {						\
+		if (__ipipe_root_p) {			\
+			preempt_enable_no_resched();	\
+			hard_local_irq_restore(flags);	\
+			preempt_check_resched();	\
+		} else					\
+			hard_local_irq_restore(flags);	\
+	} while (0)
+
+#define __ipipe_get_cpu(flags)	({ (flags) = hard_preempt_disable(); ipipe_processor_id(); })
+#define __ipipe_put_cpu(flags)	hard_preempt_enable(flags)
+
+int __ipipe_notify_syscall(struct pt_regs *regs);
+
+int __ipipe_notify_trap(int exception, struct pt_regs *regs);
+
+int __ipipe_notify_kevent(int event, void *data);
+
+#define __ipipe_report_trap(exception, regs)				\
+	__ipipe_notify_trap(exception, regs)
+
+#define __ipipe_report_sigwake(p)					\
+	do {								\
+		if (ipipe_notifier_enabled_p(p))			\
+			__ipipe_notify_kevent(IPIPE_KEVT_SIGWAKE, p);	\
+	} while (0)
+
+#define __ipipe_report_exit(p)						\
+	do {								\
+		if (ipipe_notifier_enabled_p(p))			\
+			__ipipe_notify_kevent(IPIPE_KEVT_EXIT, p);	\
+	} while (0)
+
+#define __ipipe_report_setsched(p)					\
+	do {								\
+		if (ipipe_notifier_enabled_p(p))			\
+			__ipipe_notify_kevent(IPIPE_KEVT_SETSCHED, p); \
+	} while (0)
+
+#define __ipipe_report_schedule(prev, next)				\
+do {									\
+	if ((ipipe_notifier_enabled_p(next) ||				\
+	     ipipe_notifier_enabled_p(prev))) {				\
+		__this_cpu_write(ipipe_percpu.rqlock_owner, prev);	\
+		__ipipe_notify_kevent(IPIPE_KEVT_SCHEDULE, next);	\
+	}								\
+} while (0)
+
+#define __ipipe_report_cleanup(mm)					\
+	__ipipe_notify_kevent(IPIPE_KEVT_CLEANUP, mm)
+
+void __ipipe_notify_vm_preemption(void);
+
+#define hard_cond_local_irq_enable()		hard_local_irq_enable()
+#define hard_cond_local_irq_disable()		hard_local_irq_disable()
+#define hard_cond_local_irq_save()		hard_local_irq_save()
+#define hard_cond_local_irq_restore(flags)	hard_local_irq_restore(flags)
+
+struct ipipe_task_info {
+	unsigned long flags;
+};
+
+#ifdef CONFIG_IPIPE_LEGACY
+
+#define IPIPE_FIRST_EVENT	IPIPE_NR_FAULTS
+#define IPIPE_EVENT_SCHEDULE	IPIPE_FIRST_EVENT
+#define IPIPE_EVENT_SIGWAKE	(IPIPE_FIRST_EVENT + 1)
+#define IPIPE_EVENT_SETSCHED	(IPIPE_FIRST_EVENT + 2)
+#define IPIPE_EVENT_EXIT	(IPIPE_FIRST_EVENT + 3)
+#define IPIPE_EVENT_CLEANUP	(IPIPE_FIRST_EVENT + 4)
+#define IPIPE_EVENT_HOSTRT	(IPIPE_FIRST_EVENT + 5)
+#define IPIPE_EVENT_SYSCALL	(IPIPE_FIRST_EVENT + 6)
+#define IPIPE_LAST_EVENT	IPIPE_EVENT_SYSCALL
+#define IPIPE_NR_EVENTS		(IPIPE_LAST_EVENT + 1)
+
+typedef int (*ipipe_event_handler_t)(unsigned int event,
+				     struct ipipe_domain *from,
+				     void *data);
+struct ipipe_legacy_context {
+	unsigned int domid;
+	int priority;
+	void *pdd;
+	ipipe_event_handler_t handlers[IPIPE_NR_EVENTS];
+};
+
+#define __ipipe_init_taskinfo(p)			\
+	do {						\
+		__ipipe_clear_taskflags(p);		\
+		memset(p->ptd, 0, sizeof(p->ptd));	\
+	} while (0)
+
+#else /* !CONFIG_IPIPE_LEGACY */
+
+struct ipipe_legacy_context {
+};
+
+#define __ipipe_init_taskinfo(p)			\
+	do {						\
+		__ipipe_clear_taskflags(p);		\
+	} while (0)
+
+#endif /* !CONFIG_IPIPE_LEGACY */
+
+#define __ipipe_clear_taskflags(p)	\
+	do {				\
+		(p)->ipipe.flags = 0;	\
+	} while (0)
+
+#else /* !CONFIG_IPIPE */
+
+struct task_struct;
+struct mm_struct;
+
+struct ipipe_task_info {
+};
+
+static inline void __ipipe_init_early(void) { }
+
+static inline void __ipipe_init(void) { }
+
+static inline void __ipipe_init_proc(void) { }
+
+static inline void __ipipe_idle(void) { }
+
+static inline void __ipipe_report_sigwake(struct task_struct *p) { }
+
+static inline void __ipipe_report_setsched(struct task_struct *p) { }
+
+static inline void __ipipe_report_exit(struct task_struct *p) { }
+
+static inline void __ipipe_report_cleanup(struct mm_struct *mm) { }
+
+#define __ipipe_report_trap(exception, regs)  0
+
+static inline void __ipipe_init_taskinfo(struct task_struct *p) { }
+
+static inline void __ipipe_clear_taskflags(struct task_struct *p) { }
+
+static inline void __ipipe_pin_range_globally(unsigned long start,
+					      unsigned long end)
+{ }
+
+#define hard_preempt_disable()		({ preempt_disable(); 0; })
+#define hard_preempt_enable(flags)	({ preempt_enable(); (void)(flags); })
+
+#define __ipipe_get_cpu(flags)		({ (void)(flags); get_cpu(); })
+#define __ipipe_put_cpu(flags)		\
+	do {				\
+		(void)(flags);		\
+		put_cpu();		\
+	} while (0)
+
+#define __ipipe_root_tick_p(regs)	1
+
+#define ipipe_handle_demuxed_irq(irq)		generic_handle_irq(irq)
+
+#define __ipipe_serial_debug(fmt, args...)	do { } while (0)
+
+#define __ipipe_enter_vm(vmf)	do { } while (0)
+
+static inline void __ipipe_exit_vm(void) { }
+
+static inline void __ipipe_notify_vm_preemption(void) { }
+
+static inline void ipipe_root_only(void) { }
+
+#endif	/* !CONFIG_IPIPE */
+
+#endif	/* !__LINUX_IPIPE_BASE_H */
diff --git a/include/linux/ipipe_compat.h b/include/linux/ipipe_compat.h
new file mode 100644
index 0000000..fab8f45
--- /dev/null
+++ b/include/linux/ipipe_compat.h
@@ -0,0 +1,284 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_compat.h
+ *
+ * Copyright (C) 2012 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_COMPAT_H
+#define __LINUX_IPIPE_COMPAT_H
+
+#ifndef __LINUX_IPIPE_H
+#error "Do not include this file directly, use linux/ipipe.h instead"
+#endif
+
+#ifdef CONFIG_IPIPE_LEGACY
+
+#define IPIPE_HEAD_PRIORITY	(-1)
+#define IPIPE_ROOT_PRIO		100
+#define IPIPE_ROOT_ID		0
+#define IPIPE_ROOT_NPTDKEYS	4
+
+#define IPIPE_DUMMY_FLAG	31
+#define IPIPE_WIRED_FLAG	IPIPE_HANDLE_FLAG
+#define IPIPE_WIRED_MASK	(1 << IPIPE_WIRED_FLAG)
+#define IPIPE_PASS_FLAG		IPIPE_DUMMY_FLAG
+#define IPIPE_PASS_MASK		(1 << IPIPE_PASS_FLAG)
+#define IPIPE_DYNAMIC_FLAG	IPIPE_HANDLE_FLAG
+#define IPIPE_DYNAMIC_MASK	(1 << IPIPE_DYNAMIC_FLAG)
+#define IPIPE_SYSTEM_FLAG	IPIPE_DUMMY_FLAG
+#define IPIPE_SYSTEM_MASK	(1 << IPIPE_SYSTEM_FLAG)
+#define IPIPE_EXCLUSIVE_FLAG	IPIPE_DUMMY_FLAG
+#define IPIPE_EXCLUSIVE_MASK	(1 << IPIPE_EXCLUSIVE_FLAG)
+
+#define IPIPE_NR_CPUS		NR_CPUS
+
+#define IPIPE_EVENT_SELF        0x80000000
+#define IPIPE_EVENT_RETURN	IPIPE_TRAP_MAYDAY
+
+#define TASK_ATOMICSWITCH	TASK_HARDENING
+
+struct ipipe_domain_attr {
+	unsigned int domid;
+	const char *name;
+	int priority;
+	void (*entry) (void);
+	void *pdd;
+};
+
+void ipipe_init_attr(struct ipipe_domain_attr *attr);
+
+int ipipe_register_domain(struct ipipe_domain *ipd,
+			  struct ipipe_domain_attr *attr);
+
+int ipipe_unregister_domain(struct ipipe_domain *ipd);
+
+int ipipe_alloc_ptdkey(void);
+
+int ipipe_free_ptdkey(int key);
+
+int ipipe_set_ptd(int key, void *value);
+
+void *ipipe_get_ptd(int key);
+
+int ipipe_virtualize_irq(struct ipipe_domain *ipd,
+			 unsigned int irq,
+			 ipipe_irq_handler_t handler,
+			 void *cookie,
+			 ipipe_irq_ackfn_t ackfn,
+			 unsigned int modemask);
+
+ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd,
+					unsigned int event,
+					ipipe_event_handler_t handler);
+
+int ipipe_setscheduler_root(struct task_struct *p,
+			    int policy,
+			    int prio);
+
+static inline void ipipe_check_context(struct ipipe_domain *border_ipd)
+{
+	ipipe_root_only();
+}
+
+static inline void ipipe_set_printk_sync(struct ipipe_domain *ipd)
+{
+	ipipe_prepare_panic();
+}
+
+static inline void __ipipe_propagate_irq(unsigned int irq)
+{
+	ipipe_post_irq_root(irq);
+}
+
+static inline void __ipipe_schedule_irq_head(unsigned int irq)
+{
+	ipipe_post_irq_head(irq);
+}
+
+static inline void __ipipe_schedule_irq_root(unsigned int irq)
+{
+	ipipe_post_irq_root(irq);
+}
+
+static inline int ipipe_trigger_irq(unsigned int irq)
+{
+	ipipe_raise_irq(irq);
+	return 1;
+}
+
+static inline void ipipe_stall_pipeline_from(struct ipipe_domain *ipd)
+{
+	if (ipd != ipipe_root_domain)
+		ipipe_stall_head();
+	else
+		ipipe_stall_root();
+}
+
+static inline
+unsigned long ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd)
+{
+	if (ipd != ipipe_root_domain)
+		return ipipe_test_and_stall_head();
+
+	return ipipe_test_and_stall_root();
+}
+
+static inline
+void ipipe_unstall_pipeline_from(struct ipipe_domain *ipd)
+{
+	if (ipd != ipipe_root_domain)
+		ipipe_unstall_head();
+	else
+		ipipe_unstall_root();
+}
+
+static inline
+void ipipe_restore_pipeline_from(struct ipipe_domain *ipd,
+				 unsigned long x)
+{
+	if (ipd != ipipe_root_domain)
+		ipipe_restore_head(x);
+	else
+		ipipe_restore_root(x);
+}
+
+static inline
+unsigned long ipipe_test_pipeline_from(struct ipipe_domain *ipd)
+{
+	return test_bit(IPIPE_STALL_FLAG, &ipipe_this_cpu_context(ipd)->status);
+}
+
+static inline void ipipe_stall_pipeline_head(void)
+{
+	ipipe_stall_head();
+}
+
+static inline unsigned long ipipe_test_and_stall_pipeline_head(void)
+{
+	return ipipe_test_and_stall_head();
+}
+
+static inline void ipipe_unstall_pipeline_head(void)
+{
+	ipipe_unstall_head();
+}
+
+static inline void ipipe_restore_pipeline_head(unsigned long x)
+{
+	ipipe_restore_head(x);
+}
+
+static inline int ipipe_disable_ondemand_mappings(struct task_struct *p)
+{
+	return __ipipe_disable_ondemand_mappings(p);
+}
+
+static inline int ipipe_reenter_root(struct task_struct *prev,
+				     int policy,
+				     int prio)
+{
+	__ipipe_reenter_root();
+	return 0;
+}
+
+static inline void ipipe_root_preempt_notify(void)
+{
+	ipipe_notify_root_preemption();
+}
+
+#define ipipe_return_notify(p)	ipipe_raise_mayday(p)
+
+/*
+ * Keep the following as a macro, so that client code could check for
+ * the support of the invariant pipeline head optimization.
+ */
+#define __ipipe_pipeline_head() ipipe_head_domain
+
+static inline int irqs_disabled_hw(void)
+{
+	return hard_irqs_disabled();
+}
+
+static inline void local_irq_disable_hw(void)
+{
+	hard_local_irq_disable();
+}
+
+static inline void local_irq_enable_hw(void)
+{
+	hard_local_irq_enable();
+}
+
+#define local_irq_save_hw(flags)			\
+	do {						\
+		(flags) = hard_local_irq_save();	\
+	} while (0)
+
+static inline void local_irq_restore_hw(unsigned long flags)
+{
+	hard_local_irq_restore(flags);
+}
+
+#define local_save_flags_hw(flags)			\
+	do {						\
+		(flags) = hard_local_save_flags();	\
+	} while (0)
+
+#define local_irq_save_hw_smp(flags)			\
+	do {						\
+		(flags) = hard_smp_local_irq_save();	\
+	} while (0)
+#define local_irq_restore_hw_smp(flags)   hard_smp_local_irq_restore(flags)
+
+#define local_irq_save_hw_cond(flags)			\
+	do {						\
+		(flags) = hard_cond_local_irq_save();	\
+	} while (0)
+#define local_irq_restore_hw_cond(flags)  hard_cond_local_irq_restore(flags)
+
+void __ipipe_legacy_init_stage(struct ipipe_domain *ipd);
+
+/*
+ * These values have no real meaning from a versioning POV, however
+ * they are guaranteed to look more recent than any legacy patch
+ * release ever published in the past.
+ */
+#define IPIPE_MAJOR_NUMBER  3
+#define IPIPE_MINOR_NUMBER  0
+#define IPIPE_PATCH_NUMBER  0
+
+#define __IPIPE_FEATURE_REQUEST_TICKDEV		1
+#define __IPIPE_FEATURE_FASTPEND_IRQ		1
+#define __IPIPE_FEATURE_TRACE_EVENT		1
+#define __IPIPE_FEATURE_ENABLE_NOTIFIER		1
+#define __IPIPE_FEATURE_PREPARE_PANIC		1
+#define __IPIPE_FEATURE_SYSINFO_V2		1
+#define __IPIPE_FEATURE_PIC_MUTE		1
+#ifdef CONFIG_IPIPE_HAVE_VM_NOTIFIER
+#define __IPIPE_FEATURE_ROOTPREEMPT_NOTIFIER	1
+#endif
+
+#else  /* !CONFIG_IPIPE_LEGACY */
+
+static inline void __ipipe_legacy_init_stage(struct ipipe_domain *ipd)
+{
+}
+
+#endif /* !CONFIG_IPIPE_LEGACY */
+
+#endif	/* !__LINUX_IPIPE_COMPAT_H */
diff --git a/include/linux/ipipe_debug.h b/include/linux/ipipe_debug.h
new file mode 100644
index 0000000..9b5a4a3
--- /dev/null
+++ b/include/linux/ipipe_debug.h
@@ -0,0 +1,98 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_debug.h
+ *
+ * Copyright (C) 2012 Philippe Gerum <rpm@xenomai.org>.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_DEBUG_H
+#define __LINUX_IPIPE_DEBUG_H
+
+#include <linux/ipipe_domain.h>
+
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+
+#include <asm/bug.h>
+
+static inline int ipipe_disable_context_check(void)
+{
+	return xchg(__this_cpu_ptr(&ipipe_percpu.context_check), 0);
+}
+
+static inline void ipipe_restore_context_check(int old_state)
+{
+	__this_cpu_write(ipipe_percpu.context_check, old_state);
+}
+
+static inline void ipipe_context_check_off(void)
+{
+	int cpu;
+	for_each_online_cpu(cpu)
+		per_cpu(ipipe_percpu, cpu).context_check = 0;
+}
+
+static inline void ipipe_save_context_nmi(void)
+{
+	int state = ipipe_disable_context_check();
+	__this_cpu_write(ipipe_percpu.context_check_saved, state);
+}
+
+static inline void ipipe_restore_context_nmi(void)
+{
+	ipipe_restore_context_check(__this_cpu_read(ipipe_percpu.context_check_saved));
+}
+
+#else	/* !CONFIG_IPIPE_DEBUG_CONTEXT */
+
+static inline int ipipe_disable_context_check(void)
+{
+	return 0;
+}
+
+static inline void ipipe_restore_context_check(int old_state) { }
+
+static inline void ipipe_context_check_off(void) { }
+
+static inline void ipipe_save_context_nmi(void) { }
+
+static inline void ipipe_restore_context_nmi(void) { }
+
+#endif	/* !CONFIG_IPIPE_DEBUG_CONTEXT */
+
+#ifdef CONFIG_IPIPE_DEBUG_INTERNAL
+#define IPIPE_WARN(c)		WARN_ON(c)
+#define IPIPE_WARN_ONCE(c)	WARN_ON_ONCE(c)
+#else
+#define IPIPE_WARN(c)		do { (void)(c); } while (0)
+#define IPIPE_WARN_ONCE(c)	do { (void)(c); } while (0)
+#endif
+
+#ifdef CONFIG_IPIPE_DEBUG
+
+static inline void ipipe_check_irqoff(void)
+{
+	if (WARN_ON_ONCE(!hard_irqs_disabled()))
+		hard_local_irq_disable();
+}
+
+#else /* !CONFIG_IPIPE_DEBUG */
+
+static inline void ipipe_check_irqoff(void) { }
+
+#endif /* !CONFIG_IPIPE_DEBUG */
+
+#endif /* !__LINUX_IPIPE_DEBUG_H */
diff --git a/include/linux/ipipe_domain.h b/include/linux/ipipe_domain.h
new file mode 100644
index 0000000..3b0f8be
--- /dev/null
+++ b/include/linux/ipipe_domain.h
@@ -0,0 +1,311 @@
+/*   -*- linux-c -*-
+ *   include/linux/ipipe_domain.h
+ *
+ *   Copyright (C) 2007-2012 Philippe Gerum.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ *   USA; either version 2 of the License, or (at your option) any later
+ *   version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the Free Software
+ *   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_DOMAIN_H
+#define __LINUX_IPIPE_DOMAIN_H
+
+#ifdef CONFIG_IPIPE
+
+#include <linux/mutex.h>
+#include <asm/percpu.h>
+#include <asm/ptrace.h>
+
+struct task_struct;
+struct mm_struct;
+struct irq_desc;
+struct ipipe_vm_notifier;
+
+#define __IPIPE_SYSCALL_P  0
+#define __IPIPE_TRAP_P     1
+#define __IPIPE_KEVENT_P   2
+#define __IPIPE_SYSCALL_E (1 << __IPIPE_SYSCALL_P)
+#define __IPIPE_TRAP_E	  (1 << __IPIPE_TRAP_P)
+#define __IPIPE_KEVENT_E  (1 << __IPIPE_KEVENT_P)
+#define __IPIPE_ALL_E	   0x7
+#define __IPIPE_SYSCALL_R (8 << __IPIPE_SYSCALL_P)
+#define __IPIPE_TRAP_R	  (8 << __IPIPE_TRAP_P)
+#define __IPIPE_KEVENT_R  (8 << __IPIPE_KEVENT_P)
+#define __IPIPE_SHIFT_R	   3
+#define __IPIPE_ALL_R	  (__IPIPE_ALL_E << __IPIPE_SHIFT_R)
+
+typedef void (*ipipe_irq_ackfn_t)(unsigned int irq, struct irq_desc *desc);
+
+struct ipipe_domain {
+	int context_offset;
+	struct ipipe_irqdesc {
+		unsigned long control;
+		ipipe_irq_ackfn_t ackfn;
+		ipipe_irq_handler_t handler;
+		void *cookie;
+	} ____cacheline_aligned irqs[IPIPE_NR_IRQS];
+	const char *name;
+	struct mutex mutex;
+	struct ipipe_legacy_context legacy;
+};
+
+static inline void *
+__ipipe_irq_cookie(struct ipipe_domain *ipd, unsigned int irq)
+{
+	return ipd->irqs[irq].cookie;
+}
+
+static inline ipipe_irq_handler_t
+__ipipe_irq_handler(struct ipipe_domain *ipd, unsigned int irq)
+{
+	return ipd->irqs[irq].handler;
+}
+
+extern struct ipipe_domain ipipe_root;
+
+#define ipipe_root_domain (&ipipe_root)
+
+extern struct ipipe_domain *ipipe_head_domain;
+
+struct ipipe_percpu_domain_data {
+	unsigned long status;	/* <= Must be first in struct. */
+	unsigned long irqpend_himap;
+#ifdef __IPIPE_3LEVEL_IRQMAP
+	unsigned long irqpend_mdmap[IPIPE_IRQ_MDMAPSZ];
+#endif
+	unsigned long irqpend_lomap[IPIPE_IRQ_LOMAPSZ];
+	unsigned long irqheld_map[IPIPE_IRQ_LOMAPSZ];
+	unsigned long irqall[IPIPE_NR_IRQS];
+	struct ipipe_domain *domain;
+	int coflags;
+};
+
+struct ipipe_percpu_data {
+	struct ipipe_percpu_domain_data root;
+	struct ipipe_percpu_domain_data head;
+	struct ipipe_percpu_domain_data *curr;
+	struct pt_regs tick_regs;
+	int hrtimer_irq;
+	struct task_struct *task_hijacked;
+	struct task_struct *rqlock_owner;
+	struct ipipe_vm_notifier *vm_notifier;
+	unsigned long nmi_state;
+#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH
+	struct mm_struct *active_mm;
+#endif
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+	int context_check;
+	int context_check_saved;
+#endif
+};
+
+/*
+ * CAREFUL: all accessors based on __ipipe_this_cpu_ptr() you may find
+ * in this file should be used only while hw interrupts are off, to
+ * prevent from CPU migration regardless of the running domain.
+ */
+DECLARE_PER_CPU(struct ipipe_percpu_data, ipipe_percpu);
+
+static inline struct ipipe_percpu_domain_data *
+__context_of(struct ipipe_percpu_data *p, struct ipipe_domain *ipd)
+{
+	return (void *)p + ipd->context_offset;
+}
+
+/**
+ * ipipe_percpu_context - return the address of the pipeline context
+ * data for a domain on a given CPU.
+ *
+ * NOTE: this is the slowest accessor, use it carefully. Prefer
+ * ipipe_this_cpu_context() for requests targeted at the current
+ * CPU. Additionally, if the target domain is known at build time,
+ * consider ipipe_this_cpu_{root, head}_context().
+ */
+static inline struct ipipe_percpu_domain_data *
+ipipe_percpu_context(struct ipipe_domain *ipd, int cpu)
+{
+	return __context_of(&per_cpu(ipipe_percpu, cpu), ipd);
+}
+
+/**
+ * ipipe_this_cpu_context - return the address of the pipeline context
+ * data for a domain on the current CPU. hw IRQs must be off.
+ *
+ * NOTE: this accessor is a bit faster, but since we don't know which
+ * one of "root" or "head" ipd refers to, we still need to compute the
+ * context address from its offset.
+ */
+static inline struct ipipe_percpu_domain_data *
+ipipe_this_cpu_context(struct ipipe_domain *ipd)
+{
+	return __context_of(__ipipe_this_cpu_ptr(&ipipe_percpu), ipd);
+}
+
+/**
+ * ipipe_this_cpu_root_context - return the address of the pipeline
+ * context data for the root domain on the current CPU. hw IRQs must
+ * be off.
+ *
+ * NOTE: this accessor is recommended when the domain we refer to is
+ * known at build time to be the root one.
+ */
+static inline struct ipipe_percpu_domain_data *
+ipipe_this_cpu_root_context(void)
+{
+	return __ipipe_this_cpu_ptr(&ipipe_percpu.root);
+}
+
+/**
+ * ipipe_this_cpu_head_context - return the address of the pipeline
+ * context data for the registered head domain on the current CPU. hw
+ * IRQs must be off.
+ *
+ * NOTE: this accessor is recommended when the domain we refer to is
+ * known at build time to be the registered head domain. This address
+ * is always different from the context data of the root domain in
+ * absence of registered head domain. To get the address of the
+ * context data for the domain leading the pipeline at the time of the
+ * call (which may be root in absence of registered head domain), use
+ * ipipe_this_cpu_leading_context() instead.
+ */
+static inline struct ipipe_percpu_domain_data *
+ipipe_this_cpu_head_context(void)
+{
+	return __ipipe_this_cpu_ptr(&ipipe_percpu.head);
+}
+
+/**
+ * ipipe_this_cpu_leading_context - return the address of the pipeline
+ * context data for the domain leading the pipeline on the current
+ * CPU. hw IRQs must be off.
+ *
+ * NOTE: this accessor is required when either root or a registered
+ * head domain may be the final target of this call, depending on
+ * whether the high priority domain was installed via
+ * ipipe_register_head().
+ */
+static inline struct ipipe_percpu_domain_data *
+ipipe_this_cpu_leading_context(void)
+{
+	return ipipe_this_cpu_context(ipipe_head_domain);
+}
+
+/**
+ * __ipipe_get_current_context() - return the address of the pipeline
+ * context data of the domain running on the current CPU. hw IRQs must
+ * be off.
+ */
+static inline struct ipipe_percpu_domain_data *__ipipe_get_current_context(void)
+{
+	return __ipipe_this_cpu_read(ipipe_percpu.curr);
+}
+
+#define __ipipe_current_context __ipipe_get_current_context()
+
+/**
+ * __ipipe_set_current_context() - switch the current CPU to the
+ * specified domain context.  hw IRQs must be off.
+ *
+ * NOTE: this is the only way to change the current domain for the
+ * current CPU. Don't bypass.
+ */
+static inline
+void __ipipe_set_current_context(struct ipipe_percpu_domain_data *pd)
+{
+	struct ipipe_percpu_data *p;
+	p = __ipipe_this_cpu_ptr(&ipipe_percpu);
+	p->curr = pd;
+}
+
+/**
+ * __ipipe_set_current_domain() - switch the current CPU to the
+ * specified domain. This is equivalent to calling
+ * __ipipe_set_current_context() with the context data of that
+ * domain. hw IRQs must be off.
+ */
+static inline void __ipipe_set_current_domain(struct ipipe_domain *ipd)
+{
+	struct ipipe_percpu_data *p;
+	p = __ipipe_this_cpu_ptr(&ipipe_percpu);
+	p->curr = __context_of(p, ipd);
+}
+
+static inline struct ipipe_percpu_domain_data *ipipe_current_context(void)
+{
+	struct ipipe_percpu_domain_data *pd;
+	unsigned long flags;
+
+	flags = hard_smp_local_irq_save();
+	pd = __ipipe_get_current_context();
+	hard_smp_local_irq_restore(flags);
+
+	return pd;
+}
+
+static inline struct ipipe_domain *__ipipe_get_current_domain(void)
+{
+	return __ipipe_get_current_context()->domain;
+}
+
+#define __ipipe_current_domain	__ipipe_get_current_domain()
+
+/**
+ * __ipipe_get_current_domain() - return the address of the pipeline
+ * domain running on the current CPU. hw IRQs must be off.
+ */
+static inline struct ipipe_domain *ipipe_get_current_domain(void)
+{
+	struct ipipe_domain *ipd;
+	unsigned long flags;
+
+	flags = hard_smp_local_irq_save();
+	ipd = __ipipe_get_current_domain();
+	hard_smp_local_irq_restore(flags);
+
+	return ipd;
+}
+
+#define ipipe_current_domain	ipipe_get_current_domain()
+
+#define __ipipe_root_p	(__ipipe_current_domain == ipipe_root_domain)
+#define ipipe_root_p	(ipipe_current_domain == ipipe_root_domain)
+
+#ifdef CONFIG_SMP
+#define __ipipe_root_status	(ipipe_this_cpu_root_context()->status)
+#else
+extern unsigned long __ipipe_root_status;
+#endif
+
+#define __ipipe_head_status	(ipipe_this_cpu_head_context()->status)
+
+/**
+ * __ipipe_ipending_p() - Whether we have interrupts pending
+ * (i.e. logged) for the given domain context on the current CPU. hw
+ * IRQs must be off.
+ */
+static inline int __ipipe_ipending_p(struct ipipe_percpu_domain_data *pd)
+{
+	return pd->irqpend_himap != 0;
+}
+
+static inline unsigned long
+__ipipe_cpudata_irq_hits(struct ipipe_domain *ipd, int cpu, unsigned int irq)
+{
+	return ipipe_percpu_context(ipd, cpu)->irqall[irq];
+}
+
+#endif /* CONFIG_IPIPE */
+
+#endif	/* !__LINUX_IPIPE_DOMAIN_H */
diff --git a/include/linux/ipipe_lock.h b/include/linux/ipipe_lock.h
new file mode 100644
index 0000000..5b398ab
--- /dev/null
+++ b/include/linux/ipipe_lock.h
@@ -0,0 +1,241 @@
+/*   -*- linux-c -*-
+ *   include/linux/ipipe_lock.h
+ *
+ *   Copyright (C) 2009 Philippe Gerum.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ *   USA; either version 2 of the License, or (at your option) any later
+ *   version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the Free Software
+ *   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_LOCK_H
+#define __LINUX_IPIPE_LOCK_H
+
+typedef struct {
+	arch_spinlock_t arch_lock;
+} __ipipe_spinlock_t;
+
+#define ipipe_spinlock_p(lock)						\
+	__builtin_types_compatible_p(typeof(lock), __ipipe_spinlock_t *)
+
+#define std_spinlock_raw_p(lock)					\
+	__builtin_types_compatible_p(typeof(lock), raw_spinlock_t *)
+
+#define std_spinlock_p(lock)						\
+	__builtin_types_compatible_p(typeof(lock), spinlock_t *)
+
+#define ipipe_spinlock(lock)	((__ipipe_spinlock_t *)(lock))
+#define std_spinlock_raw(lock)	((raw_spinlock_t *)(lock))
+#define std_spinlock(lock)	((spinlock_t *)(lock))
+
+#define PICK_SPINLOCK_IRQSAVE(lock, flags)				\
+	do {								\
+		if (ipipe_spinlock_p(lock))				\
+			(flags) = __ipipe_spin_lock_irqsave(ipipe_spinlock(lock)); \
+		else if (std_spinlock_raw_p(lock))				\
+			__real_raw_spin_lock_irqsave(std_spinlock_raw(lock), flags); \
+		else if (std_spinlock_p(lock))				\
+			__real_raw_spin_lock_irqsave(&std_spinlock(lock)->rlock, flags); \
+		else __bad_lock_type();					\
+	} while (0)
+
+#define PICK_SPINTRYLOCK_IRQSAVE(lock, flags)				\
+	({								\
+		int __ret__;						\
+		if (ipipe_spinlock_p(lock))				\
+			__ret__ = __ipipe_spin_trylock_irqsave(ipipe_spinlock(lock), &(flags)); \
+		else if (std_spinlock_raw_p(lock))				\
+			__ret__ = __real_raw_spin_trylock_irqsave(std_spinlock_raw(lock), flags); \
+		else if (std_spinlock_p(lock))				\
+			__ret__ = __real_raw_spin_trylock_irqsave(&std_spinlock(lock)->rlock, flags); \
+		else __bad_lock_type();					\
+		__ret__;						\
+	 })
+
+#define PICK_SPINTRYLOCK_IRQ(lock)					\
+	({								\
+		int __ret__;						\
+		if (ipipe_spinlock_p(lock))				\
+			__ret__ = __ipipe_spin_trylock_irq(ipipe_spinlock(lock)); \
+		else if (std_spinlock_raw_p(lock))				\
+			__ret__ = __real_raw_spin_trylock_irq(std_spinlock_raw(lock)); \
+		else if (std_spinlock_p(lock))				\
+			__ret__ = __real_raw_spin_trylock_irq(&std_spinlock(lock)->rlock); \
+		else __bad_lock_type();					\
+		__ret__;						\
+	 })
+
+#define PICK_SPINUNLOCK_IRQRESTORE(lock, flags)				\
+	do {								\
+		if (ipipe_spinlock_p(lock))				\
+			__ipipe_spin_unlock_irqrestore(ipipe_spinlock(lock), flags); \
+		else {							\
+			__ipipe_spin_unlock_debug(flags);		\
+			if (std_spinlock_raw_p(lock))			\
+				__real_raw_spin_unlock_irqrestore(std_spinlock_raw(lock), flags); \
+			else if (std_spinlock_p(lock))			\
+				__real_raw_spin_unlock_irqrestore(&std_spinlock(lock)->rlock, flags); \
+		}							\
+	} while (0)
+
+#define PICK_SPINOP(op, lock)						\
+	do {								\
+		if (ipipe_spinlock_p(lock))				\
+			arch_spin##op(&ipipe_spinlock(lock)->arch_lock); \
+		else if (std_spinlock_raw_p(lock))			\
+			__real_raw_spin##op(std_spinlock_raw(lock));	\
+		else if (std_spinlock_p(lock))				\
+			__real_raw_spin##op(&std_spinlock(lock)->rlock); \
+		else __bad_lock_type();					\
+	} while (0)
+
+#define PICK_SPINOP_RET(op, lock, type)					\
+	({								\
+		type __ret__;						\
+		if (ipipe_spinlock_p(lock))				\
+			__ret__ = arch_spin##op(&ipipe_spinlock(lock)->arch_lock); \
+		else if (std_spinlock_raw_p(lock))			\
+			__ret__ = __real_raw_spin##op(std_spinlock_raw(lock)); \
+		else if (std_spinlock_p(lock))				\
+			__ret__ = __real_raw_spin##op(&std_spinlock(lock)->rlock); \
+		else { __ret__ = -1; __bad_lock_type(); }		\
+		__ret__;						\
+	})
+
+#define arch_spin_lock_init(lock)					\
+	do {								\
+		IPIPE_DEFINE_SPINLOCK(__lock__);			\
+		*((ipipe_spinlock_t *)lock) = __lock__;			\
+	} while (0)
+
+#define arch_spin_lock_irq(lock)					\
+	do {								\
+		hard_local_irq_disable();				\
+		arch_spin_lock(lock);					\
+	} while (0)
+
+#define arch_spin_unlock_irq(lock)					\
+	do {								\
+		arch_spin_unlock(lock);					\
+		hard_local_irq_enable();				\
+	} while (0)
+
+typedef struct {
+	arch_rwlock_t arch_lock;
+} __ipipe_rwlock_t;
+
+#define ipipe_rwlock_p(lock)						\
+	__builtin_types_compatible_p(typeof(lock), __ipipe_rwlock_t *)
+
+#define std_rwlock_p(lock)						\
+	__builtin_types_compatible_p(typeof(lock), rwlock_t *)
+
+#define ipipe_rwlock(lock)	((__ipipe_rwlock_t *)(lock))
+#define std_rwlock(lock)	((rwlock_t *)(lock))
+
+#define PICK_RWOP(op, lock)						\
+	do {								\
+		if (ipipe_rwlock_p(lock))				\
+			arch##op(&ipipe_rwlock(lock)->arch_lock);	\
+		else if (std_rwlock_p(lock))				\
+			_raw##op(std_rwlock(lock));			\
+		else __bad_lock_type();					\
+	} while (0)
+
+extern int __bad_lock_type(void);
+
+#ifdef CONFIG_IPIPE
+
+#define ipipe_spinlock_t		__ipipe_spinlock_t
+#define IPIPE_DEFINE_RAW_SPINLOCK(x)	ipipe_spinlock_t x = IPIPE_SPIN_LOCK_UNLOCKED
+#define IPIPE_DECLARE_RAW_SPINLOCK(x)	extern ipipe_spinlock_t x
+#define IPIPE_DEFINE_SPINLOCK(x)	IPIPE_DEFINE_RAW_SPINLOCK(x)
+#define IPIPE_DECLARE_SPINLOCK(x)	IPIPE_DECLARE_RAW_SPINLOCK(x)
+
+#define IPIPE_SPIN_LOCK_UNLOCKED					\
+	(__ipipe_spinlock_t) {	.arch_lock = __ARCH_SPIN_LOCK_UNLOCKED }
+
+#define spin_lock_irqsave_cond(lock, flags) \
+	spin_lock_irqsave(lock, flags)
+
+#define spin_unlock_irqrestore_cond(lock, flags) \
+	spin_unlock_irqrestore(lock, flags)
+
+void __ipipe_spin_lock_irq(ipipe_spinlock_t *lock);
+
+int __ipipe_spin_trylock_irq(ipipe_spinlock_t *lock);
+
+void __ipipe_spin_unlock_irq(ipipe_spinlock_t *lock);
+
+unsigned long __ipipe_spin_lock_irqsave(ipipe_spinlock_t *lock);
+
+int __ipipe_spin_trylock_irqsave(ipipe_spinlock_t *lock,
+				 unsigned long *x);
+
+void __ipipe_spin_unlock_irqrestore(ipipe_spinlock_t *lock,
+				    unsigned long x);
+
+void __ipipe_spin_unlock_irqbegin(ipipe_spinlock_t *lock);
+
+void __ipipe_spin_unlock_irqcomplete(unsigned long x);
+
+#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP)
+void __ipipe_spin_unlock_debug(unsigned long flags);
+#else
+#define __ipipe_spin_unlock_debug(flags)  do { } while (0)
+#endif
+
+#define ipipe_rwlock_t			__ipipe_rwlock_t
+#define IPIPE_DEFINE_RWLOCK(x)		ipipe_rwlock_t x = IPIPE_RW_LOCK_UNLOCKED
+#define IPIPE_DECLARE_RWLOCK(x)		extern ipipe_rwlock_t x
+
+#define IPIPE_RW_LOCK_UNLOCKED	\
+	(__ipipe_rwlock_t) { .arch_lock = __ARCH_RW_LOCK_UNLOCKED }
+
+#else /* !CONFIG_IPIPE */
+
+#define ipipe_spinlock_t		spinlock_t
+#define IPIPE_DEFINE_SPINLOCK(x)	DEFINE_SPINLOCK(x)
+#define IPIPE_DECLARE_SPINLOCK(x)	extern spinlock_t x
+#define IPIPE_SPIN_LOCK_UNLOCKED	__SPIN_LOCK_UNLOCKED(unknown)
+#define IPIPE_DEFINE_RAW_SPINLOCK(x)	DEFINE_RAW_SPINLOCK(x)
+#define IPIPE_DECLARE_RAW_SPINLOCK(x)	extern raw_spinlock_t x
+
+#define spin_lock_irqsave_cond(lock, flags)		\
+	do {						\
+		(void)(flags);				\
+		spin_lock(lock);			\
+	} while(0)
+
+#define spin_unlock_irqrestore_cond(lock, flags)	\
+	spin_unlock(lock)
+
+#define __ipipe_spin_lock_irq(lock)		do { } while (0)
+#define __ipipe_spin_unlock_irq(lock)		do { } while (0)
+#define __ipipe_spin_lock_irqsave(lock)		0
+#define __ipipe_spin_trylock_irq(lock)		1
+#define __ipipe_spin_trylock_irqsave(lock, x)	({ (void)(x); 1; })
+#define __ipipe_spin_unlock_irqrestore(lock, x)	do { (void)(x); } while (0)
+#define __ipipe_spin_unlock_irqbegin(lock)	do { } while (0)
+#define __ipipe_spin_unlock_irqcomplete(x)	do { (void)(x); } while (0)
+#define __ipipe_spin_unlock_debug(flags)	do { } while (0)
+
+#define ipipe_rwlock_t			rwlock_t
+#define IPIPE_DEFINE_RWLOCK(x)		DEFINE_RWLOCK(x)
+#define IPIPE_DECLARE_RWLOCK(x)		extern rwlock_t x
+#define IPIPE_RW_LOCK_UNLOCKED		RW_LOCK_UNLOCKED
+
+#endif /* !CONFIG_IPIPE */
+
+#endif /* !__LINUX_IPIPE_LOCK_H */
diff --git a/include/linux/ipipe_tickdev.h b/include/linux/ipipe_tickdev.h
new file mode 100644
index 0000000..16e96f1
--- /dev/null
+++ b/include/linux/ipipe_tickdev.h
@@ -0,0 +1,140 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_tickdev.h
+ *
+ * Copyright (C) 2007 Philippe Gerum.
+ * Copyright (C) 2012 Gilles Chanteperdrix
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_TICKDEV_H
+#define __LINUX_IPIPE_TICKDEV_H
+
+#include <linux/list.h>
+#include <linux/cpumask.h>
+#include <linux/clockchips.h>
+#include <linux/ipipe_domain.h>
+#include <linux/clocksource.h>
+
+#ifdef CONFIG_IPIPE
+
+enum clock_event_mode;
+struct clock_event_device;
+
+struct ipipe_hostrt_data {
+	short live;
+	seqcount_t seqcount;
+	time_t wall_time_sec;
+	u32 wall_time_nsec;
+	struct timespec wall_to_monotonic;
+	cycle_t cycle_last;
+	cycle_t mask;
+	u32 mult;
+	u32 shift;
+};
+
+struct ipipe_timer {
+	int irq;
+	void (*request)(struct ipipe_timer *timer, int steal);
+	int (*set)(unsigned long ticks, void *timer);
+	void (*ack)(void);
+	void (*release)(struct ipipe_timer *timer);
+
+	/* Only if registering a timer directly */
+	const char *name;
+	unsigned rating;
+	unsigned long freq;
+	unsigned min_delay_ticks;
+	const struct cpumask *cpumask;
+
+	/* For internal use */
+	void *timer_set;	/* pointer passed to ->set() callback */
+	struct clock_event_device *host_timer;
+	struct list_head holder;
+	u32 real_mult;
+	u32 real_shift;
+	void (*real_set_mode)(enum clock_event_mode mode,
+			      struct clock_event_device *cdev);
+	int (*real_set_next_event)(unsigned long evt,
+				   struct clock_event_device *cdev);
+};
+
+#define __ipipe_hrtimer_irq __ipipe_this_cpu_read(ipipe_percpu.hrtimer_irq)
+
+extern unsigned long __ipipe_hrtimer_freq;
+
+/*
+ * Called by clockevents_register_device, to register a piggybacked
+ * ipipe timer, if there is one
+ */
+void ipipe_host_timer_register(struct clock_event_device *clkevt);
+
+/*
+ * Register a standalone ipipe timer
+ */
+void ipipe_timer_register(struct ipipe_timer *timer);
+
+/*
+ * Chooses the best timer for each cpu. Take over its handling.
+ */
+int ipipe_timers_request(void);
+
+/*
+ * Release the per-cpu timers
+ */
+void ipipe_timers_release(void);
+
+/*
+ * Start handling the per-cpu timer irq, and intercepting the linux clockevent
+ * device callbacks.
+ */
+int ipipe_timer_start(void (*tick_handler)(void),
+		      void (*emumode)(enum clock_event_mode mode,
+				      struct clock_event_device *cdev),
+		      int (*emutick)(unsigned long evt,
+				     struct clock_event_device *cdev),
+		      unsigned cpu);
+
+/*
+ * Stop handling a per-cpu timer
+ */
+void ipipe_timer_stop(unsigned cpu);
+
+/*
+ * Program the timer
+ */
+void ipipe_timer_set(unsigned long delay);
+
+const char *ipipe_timer_name(void);
+
+unsigned ipipe_timer_ns2ticks(struct ipipe_timer *timer, unsigned ns);
+
+#else /* !CONFIG_IPIPE */
+
+#define ipipe_host_timer_register(clkevt) do { } while (0)
+
+#endif /* !CONFIG_IPIPE */
+
+#ifdef CONFIG_IPIPE_HAVE_HOSTRT
+void ipipe_update_hostrt(struct timespec *wall_time, struct timespec *wtm,
+			 struct clocksource *clock, u32 mult);
+#else
+static inline void
+ipipe_update_hostrt(struct timespec *wall_time, struct timespec *wtm,
+		    struct clocksource *clock, u32 mult) {}
+#endif
+
+#endif /* __LINUX_IPIPE_TICKDEV_H */
diff --git a/include/linux/ipipe_trace.h b/include/linux/ipipe_trace.h
new file mode 100644
index 0000000..deb0a47
--- /dev/null
+++ b/include/linux/ipipe_trace.h
@@ -0,0 +1,77 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_trace.h
+ *
+ * Copyright (C) 2005 Luotao Fu.
+ *               2005-2007 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef _LINUX_IPIPE_TRACE_H
+#define _LINUX_IPIPE_TRACE_H
+
+#ifdef CONFIG_IPIPE_TRACE
+
+#include <linux/types.h>
+
+#ifndef BROKEN_BUILTIN_RETURN_ADDRESS
+#define __BUILTIN_RETURN_ADDRESS0 ((unsigned long)__builtin_return_address(0))
+#define __BUILTIN_RETURN_ADDRESS1 ((unsigned long)__builtin_return_address(1))
+#endif /* !BUILTIN_RETURN_ADDRESS */
+
+void ipipe_trace_begin(unsigned long v);
+void ipipe_trace_end(unsigned long v);
+void ipipe_trace_freeze(unsigned long v);
+void ipipe_trace_special(unsigned char special_id, unsigned long v);
+void ipipe_trace_pid(pid_t pid, short prio);
+void ipipe_trace_event(unsigned char id, unsigned long delay_tsc);
+int ipipe_trace_max_reset(void);
+int ipipe_trace_frozen_reset(void);
+
+#else /* !CONFIG_IPIPE_TRACE */
+
+#define ipipe_trace_begin(v)			do { (void)(v); } while(0)
+#define ipipe_trace_end(v)			do { (void)(v); } while(0)
+#define ipipe_trace_freeze(v)			do { (void)(v); } while(0)
+#define ipipe_trace_special(id, v)		do { (void)(id); (void)(v); } while(0)
+#define ipipe_trace_pid(pid, prio)		do { (void)(pid); (void)(prio); } while(0)
+#define ipipe_trace_event(id, delay_tsc)	do { (void)(id); (void)(delay_tsc); } while(0)
+#define ipipe_trace_max_reset()			({ 0; })
+#define ipipe_trace_frozen_reset()		({ 0; })
+
+#endif /* !CONFIG_IPIPE_TRACE */
+
+#ifdef CONFIG_IPIPE_TRACE_PANIC
+void ipipe_trace_panic_freeze(void);
+void ipipe_trace_panic_dump(void);
+#else
+static inline void ipipe_trace_panic_freeze(void) { }
+static inline void ipipe_trace_panic_dump(void) { }
+#endif
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+#define ipipe_trace_irq_entry(irq)	ipipe_trace_begin(irq)
+#define ipipe_trace_irq_exit(irq)	ipipe_trace_end(irq)
+#define ipipe_trace_irqsoff()		ipipe_trace_begin(0x80000000UL)
+#define ipipe_trace_irqson()		ipipe_trace_end(0x80000000UL)
+#else
+#define ipipe_trace_irq_entry(irq)	do { (void)(irq);} while(0)
+#define ipipe_trace_irq_exit(irq)	do { (void)(irq);} while(0)
+#define ipipe_trace_irqsoff()		do { } while(0)
+#define ipipe_trace_irqson()		do { } while(0)
+#endif
+
+#endif	/* !__LINUX_IPIPE_TRACE_H */
diff --git a/include/linux/irq.h b/include/linux/irq.h
index bff29c5..855aae0 100644
--- a/include/linux/irq.h
+++ b/include/linux/irq.h
@@ -312,6 +312,11 @@ struct irq_chip {
 
 	void		(*irq_bus_lock)(struct irq_data *data);
 	void		(*irq_bus_sync_unlock)(struct irq_data *data);
+#ifdef CONFIG_IPIPE
+	void		(*irq_move)(struct irq_data *data);
+	void		(*irq_hold)(struct irq_data *data);
+	void		(*irq_release)(struct irq_data *data);
+#endif /* CONFIG_IPIPE */
 
 	void		(*irq_cpu_online)(struct irq_data *data);
 	void		(*irq_cpu_offline)(struct irq_data *data);
@@ -518,7 +523,7 @@ extern int irq_set_handler_data(unsigned int irq, void *data);
 extern int irq_set_chip_data(unsigned int irq, void *data);
 extern int irq_set_irq_type(unsigned int irq, unsigned int type);
 extern int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry);
-extern struct irq_data *irq_get_irq_data(unsigned int irq);
+extern struct irq_data *irq_get_irq_data(unsigned int irq) __attribute__((const));
 
 static inline struct irq_chip *irq_get_chip(unsigned int irq)
 {
diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h
index f1e2527..427c639 100644
--- a/include/linux/irqdesc.h
+++ b/include/linux/irqdesc.h
@@ -41,6 +41,12 @@ struct irq_desc {
 	struct irq_data		irq_data;
 	struct timer_rand_state *timer_rand_state;
 	unsigned int __percpu	*kstat_irqs;
+#ifdef CONFIG_IPIPE
+	void			(*ipipe_ack)(unsigned int irq,
+					     struct irq_desc *desc);
+	void			(*ipipe_end)(unsigned int irq,
+					     struct irq_desc *desc);
+#endif /* CONFIG_IPIPE */
 	irq_flow_handler_t	handle_irq;
 #ifdef CONFIG_IRQ_PREFLOW_FASTEOI
 	irq_preflow_handler_t	preflow_handler;
@@ -123,6 +129,10 @@ static inline int irq_has_action(unsigned int irq)
 	return desc->action != NULL;
 }
 
+irq_flow_handler_t
+__fixup_irq_handler(struct irq_desc *desc, irq_flow_handler_t handle,
+		    int is_chained);
+
 /* caller has locked the irq_desc and both params are valid */
 static inline void __irq_set_handler_locked(unsigned int irq,
 					    irq_flow_handler_t handler)
@@ -130,6 +140,7 @@ static inline void __irq_set_handler_locked(unsigned int irq,
 	struct irq_desc *desc;
 
 	desc = irq_to_desc(irq);
+	handler = __fixup_irq_handler(desc, handler, 0);
 	desc->handle_irq = handler;
 }
 
diff --git a/include/linux/irqnr.h b/include/linux/irqnr.h
index 3bc4dca..fb1f848 100644
--- a/include/linux/irqnr.h
+++ b/include/linux/irqnr.h
@@ -24,7 +24,11 @@
 #else /* CONFIG_GENERIC_HARDIRQS */
 
 extern int nr_irqs;
+#if !defined(CONFIG_IPIPE) || defined(CONFIG_SPARSE_IRQ)
 extern struct irq_desc *irq_to_desc(unsigned int irq);
+#else
+#define irq_to_desc(irq)	(&irq_desc[irq])
+#endif
 unsigned int irq_get_next_irq(unsigned int offset);
 
 # define for_each_irq_desc(irq, desc)					\
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index a70783d..0bf9f95 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -16,6 +16,7 @@
 #include <linux/compiler.h>
 #include <linux/bitops.h>
 #include <linux/log2.h>
+#include <linux/ipipe_base.h>
 #include <linux/typecheck.h>
 #include <linux/printk.h>
 #include <linux/dynamic_debug.h>
@@ -137,9 +138,12 @@ struct user;
 
 #ifdef CONFIG_PREEMPT_VOLUNTARY
 extern int _cond_resched(void);
-# define might_resched() _cond_resched()
+# define might_resched() do { \
+		ipipe_root_only(); \
+		_cond_resched(); \
+	} while (0)
 #else
-# define might_resched() do { } while (0)
+# define might_resched() ipipe_root_only()
 #endif
 
 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 6136821..94043c2 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -116,6 +116,9 @@ struct kvm_vcpu {
 #ifdef CONFIG_PREEMPT_NOTIFIERS
 	struct preempt_notifier preempt_notifier;
 #endif
+#ifdef CONFIG_IPIPE
+	struct ipipe_vm_notifier ipipe_notifier;
+#endif
 	int cpu;
 	int vcpu_id;
 	int srcu_idx;
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index 58969b2..d38c9e8 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -9,13 +9,20 @@
 #include <linux/thread_info.h>
 #include <linux/linkage.h>
 #include <linux/list.h>
+#include <linux/ipipe_base.h>
 
 #if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER)
   extern void add_preempt_count(int val);
   extern void sub_preempt_count(int val);
 #else
-# define add_preempt_count(val)	do { preempt_count() += (val); } while (0)
-# define sub_preempt_count(val)	do { preempt_count() -= (val); } while (0)
+# define add_preempt_count(val)	do {		\
+    ipipe_root_only();				\
+    preempt_count() += (val);			\
+  } while (0)
+# define sub_preempt_count(val)	do {		\
+    ipipe_root_only();				\
+    preempt_count() -= (val);			\
+  } while (0)
 #endif
 
 #define inc_preempt_count() add_preempt_count(1)
diff --git a/include/linux/resource.h b/include/linux/resource.h
index d01c96c..9e4eb7c 100644
--- a/include/linux/resource.h
+++ b/include/linux/resource.h
@@ -59,12 +59,6 @@ struct rlimit64 {
 #define	PRIO_USER	2
 
 /*
- * Limit the stack by to some sane default: root can always
- * increase this limit if needed..  8MB seems reasonable.
- */
-#define _STK_LIM	(8*1024*1024)
-
-/*
  * GPG2 wants 64kB of mlocked memory, to make sure pass phrases
  * and other sensitive information are never written to disk.
  */
diff --git a/include/linux/rwlock.h b/include/linux/rwlock.h
index bc2994e..5e2da8d 100644
--- a/include/linux/rwlock.h
+++ b/include/linux/rwlock.h
@@ -61,8 +61,8 @@ do {								\
 #define read_trylock(lock)	__cond_lock(lock, _raw_read_trylock(lock))
 #define write_trylock(lock)	__cond_lock(lock, _raw_write_trylock(lock))
 
-#define write_lock(lock)	_raw_write_lock(lock)
-#define read_lock(lock)		_raw_read_lock(lock)
+#define write_lock(lock)	PICK_RWOP(_write_lock, lock)
+#define read_lock(lock)		PICK_RWOP(_read_lock, lock)
 
 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 
@@ -96,8 +96,8 @@ do {								\
 #define read_lock_bh(lock)		_raw_read_lock_bh(lock)
 #define write_lock_irq(lock)		_raw_write_lock_irq(lock)
 #define write_lock_bh(lock)		_raw_write_lock_bh(lock)
-#define read_unlock(lock)		_raw_read_unlock(lock)
-#define write_unlock(lock)		_raw_write_unlock(lock)
+#define read_unlock(lock)		PICK_RWOP(_read_unlock, lock)
+#define write_unlock(lock)		PICK_RWOP(_write_unlock, lock)
 #define read_unlock_irq(lock)		_raw_read_unlock_irq(lock)
 #define write_unlock_irq(lock)		_raw_write_unlock_irq(lock)
 
diff --git a/include/linux/rwlock_api_smp.h b/include/linux/rwlock_api_smp.h
index 9c9f049..62c894150 100644
--- a/include/linux/rwlock_api_smp.h
+++ b/include/linux/rwlock_api_smp.h
@@ -141,7 +141,9 @@ static inline int __raw_write_trylock(rwlock_t *lock)
  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
  */
-#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
+#if !defined(CONFIG_GENERIC_LOCKBREAK) ||	\
+	defined(CONFIG_DEBUG_LOCK_ALLOC) ||	\
+	defined(CONFIG_IPIPE)
 
 static inline void __raw_read_lock(rwlock_t *lock)
 {
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 1c4f3e9..3ce8e58 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -62,6 +62,7 @@ struct sched_param {
 #include <linux/errno.h>
 #include <linux/nodemask.h>
 #include <linux/mm_types.h>
+#include <linux/ipipe.h>
 
 #include <asm/system.h>
 #include <asm/page.h>
@@ -192,9 +193,17 @@ print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
 #define TASK_DEAD		64
 #define TASK_WAKEKILL		128
 #define TASK_WAKING		256
+#ifdef CONFIG_IPIPE
+#define TASK_HARDENING		512
+#define TASK_NOWAKEUP		1024
+#define TASK_STATE_MAX		2048
+#define TASK_STATE_TO_CHAR_STR "RSDTtZXxKWHN"
+#else  /* !CONFIG_IPIPE */
+#define TASK_HARDENING		0
+#define TASK_NOWAKEUP		0
 #define TASK_STATE_MAX		512
-
 #define TASK_STATE_TO_CHAR_STR "RSDTtZXxKW"
+#endif /* CONFIG_IPIPE */
 
 extern char ___assert_task_state[1 - 2*!!(
 		sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1)];
@@ -305,6 +314,15 @@ extern void trap_init(void);
 extern void update_process_times(int user);
 extern void scheduler_tick(void);
 
+#ifdef CONFIG_IPIPE
+void update_root_process_times(struct pt_regs *regs);
+#else  /* !CONFIG_IPIPE */
+static inline void update_root_process_times(struct pt_regs *regs)
+{
+	update_process_times(user_mode(regs));
+}
+#endif /* CONFIG_IPIPE */
+
 extern void sched_show_task(struct task_struct *p);
 
 #ifdef CONFIG_LOCKUP_DETECTOR
@@ -435,6 +453,9 @@ extern int get_dumpable(struct mm_struct *mm);
 					/* leave room for more dump flags */
 #define MMF_VM_MERGEABLE	16	/* KSM may merge identical pages */
 #define MMF_VM_HUGEPAGE		17	/* set when VM_HUGEPAGE is set on vma */
+#ifdef CONFIG_IPIPE
+#define MMF_VM_PINNED		31	/* ondemand load up and COW disabled */
+#endif
 
 #define MMF_INIT_MASK		(MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK)
 
@@ -1310,9 +1331,9 @@ struct task_struct {
 	unsigned long stack_canary;
 #endif
 
-	/* 
+	/*
 	 * pointers to (original) parent process, youngest child, younger sibling,
-	 * older sibling, respectively.  (p->father can be replaced with 
+	 * older sibling, respectively.  (p->father can be replaced with
 	 * p->real_parent->pid)
 	 */
 	struct task_struct *real_parent; /* real parent process */
@@ -1510,6 +1531,10 @@ struct task_struct {
 	short pref_node_fork;
 #endif
 	struct rcu_head rcu;
+	struct ipipe_task_info ipipe;
+#ifdef CONFIG_IPIPE_LEGACY
+	void *ptd[IPIPE_ROOT_NPTDKEYS];
+#endif
 
 	/*
 	 * cache last used pipe for splice
@@ -1790,6 +1815,10 @@ extern void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *
 #define PF_FREEZER_SKIP	0x40000000	/* Freezer should not count it as freezable */
 #define PF_FREEZER_NOSIG 0x80000000	/* Freezer won't send signals to it */
 
+/* p->ipipe.flags */
+#define PF_MAYDAY	0x1	/* MAYDAY call is pending */
+#define PF_EVNOTIFY	0x2	/* Notify head domain about kernel events */
+
 /*
  * Only the _current_ task can read/write to tsk->flags, but other
  * tasks can access tsk->flags in readonly mode for example
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index 7df6c17..bc961fe 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -89,10 +89,12 @@
 # include <linux/spinlock_up.h>
 #endif
 
+#include <linux/ipipe_lock.h>
+
 #ifdef CONFIG_DEBUG_SPINLOCK
   extern void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
 				   struct lock_class_key *key);
-# define raw_spin_lock_init(lock)				\
+# define __real_raw_spin_lock_init(lock)			\
 do {								\
 	static struct lock_class_key __key;			\
 								\
@@ -100,9 +102,10 @@ do {								\
 } while (0)
 
 #else
-# define raw_spin_lock_init(lock)				\
+# define __real_raw_spin_lock_init(lock)			\
 	do { *(lock) = __RAW_SPIN_LOCK_UNLOCKED(lock); } while (0)
 #endif
+#define raw_spin_lock_init(lock)	PICK_SPINOP(_lock_init, lock)
 
 #define raw_spin_is_locked(lock)	arch_spin_is_locked(&(lock)->raw_lock)
 
@@ -165,9 +168,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
  * various methods are defined as nops in the case they are not
  * required.
  */
-#define raw_spin_trylock(lock)	__cond_lock(lock, _raw_spin_trylock(lock))
+#define __real_raw_spin_trylock(lock)	__cond_lock(lock, _raw_spin_trylock(lock))
+#define raw_spin_trylock(lock)		PICK_SPINOP_RET(_trylock, lock, int)
 
-#define raw_spin_lock(lock)	_raw_spin_lock(lock)
+#define __real_raw_spin_lock(lock)	_raw_spin_lock(lock)
+#define raw_spin_lock(lock)		PICK_SPINOP(_lock, lock)
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 # define raw_spin_lock_nested(lock, subclass) \
@@ -185,7 +190,7 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
 
 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 
-#define raw_spin_lock_irqsave(lock, flags)			\
+#define __real_raw_spin_lock_irqsave(lock, flags)	\
 	do {						\
 		typecheck(unsigned long, flags);	\
 		flags = _raw_spin_lock_irqsave(lock);	\
@@ -207,7 +212,7 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
 
 #else
 
-#define raw_spin_lock_irqsave(lock, flags)		\
+#define __real_raw_spin_lock_irqsave(lock, flags)	\
 	do {						\
 		typecheck(unsigned long, flags);	\
 		_raw_spin_lock_irqsave(lock, flags);	\
@@ -218,34 +223,46 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
 
 #endif
 
-#define raw_spin_lock_irq(lock)		_raw_spin_lock_irq(lock)
+#define raw_spin_lock_irqsave(lock, flags)  \
+	PICK_SPINLOCK_IRQSAVE(lock, flags)
+
+#define __real_raw_spin_lock_irq(lock)	_raw_spin_lock_irq(lock)
+#define raw_spin_lock_irq(lock)		PICK_SPINOP(_lock_irq, lock)
 #define raw_spin_lock_bh(lock)		_raw_spin_lock_bh(lock)
-#define raw_spin_unlock(lock)		_raw_spin_unlock(lock)
-#define raw_spin_unlock_irq(lock)	_raw_spin_unlock_irq(lock)
+#define __real_raw_spin_unlock(lock)	_raw_spin_unlock(lock)
+#define raw_spin_unlock(lock)		PICK_SPINOP(_unlock, lock)
+#define __real_raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock)
+#define raw_spin_unlock_irq(lock)	PICK_SPINOP(_unlock_irq, lock)
 
-#define raw_spin_unlock_irqrestore(lock, flags)		\
+#define __real_raw_spin_unlock_irqrestore(lock, flags)		\
 	do {							\
 		typecheck(unsigned long, flags);		\
 		_raw_spin_unlock_irqrestore(lock, flags);	\
 	} while (0)
+#define raw_spin_unlock_irqrestore(lock, flags)	\
+	PICK_SPINUNLOCK_IRQRESTORE(lock, flags)
+
 #define raw_spin_unlock_bh(lock)	_raw_spin_unlock_bh(lock)
 
 #define raw_spin_trylock_bh(lock) \
 	__cond_lock(lock, _raw_spin_trylock_bh(lock))
 
-#define raw_spin_trylock_irq(lock) \
+#define __real_raw_spin_trylock_irq(lock) \
 ({ \
 	local_irq_disable(); \
-	raw_spin_trylock(lock) ? \
+	__real_raw_spin_trylock(lock) ? \
 	1 : ({ local_irq_enable(); 0;  }); \
 })
+#define raw_spin_trylock_irq(lock)	PICK_SPINTRYLOCK_IRQ(lock)
 
-#define raw_spin_trylock_irqsave(lock, flags) \
+#define __real_raw_spin_trylock_irqsave(lock, flags) \
 ({ \
 	local_irq_save(flags); \
 	raw_spin_trylock(lock) ? \
 	1 : ({ local_irq_restore(flags); 0; }); \
 })
+#define raw_spin_trylock_irqsave(lock, flags)	\
+	PICK_SPINTRYLOCK_IRQSAVE(lock, flags)
 
 /**
  * raw_spin_can_lock - would raw_spin_trylock() succeed?
@@ -276,24 +293,17 @@ static inline raw_spinlock_t *spinlock_check(spinlock_t *lock)
 
 #define spin_lock_init(_lock)				\
 do {							\
-	spinlock_check(_lock);				\
-	raw_spin_lock_init(&(_lock)->rlock);		\
+	raw_spin_lock_init(_lock);			\
 } while (0)
 
-static inline void spin_lock(spinlock_t *lock)
-{
-	raw_spin_lock(&lock->rlock);
-}
+#define spin_lock(lock)		raw_spin_lock(lock)
 
 static inline void spin_lock_bh(spinlock_t *lock)
 {
 	raw_spin_lock_bh(&lock->rlock);
 }
 
-static inline int spin_trylock(spinlock_t *lock)
-{
-	return raw_spin_trylock(&lock->rlock);
-}
+#define spin_trylock(lock)	raw_spin_trylock(lock)
 
 #define spin_lock_nested(lock, subclass)			\
 do {								\
@@ -305,14 +315,11 @@ do {									\
 	raw_spin_lock_nest_lock(spinlock_check(lock), nest_lock);	\
 } while (0)
 
-static inline void spin_lock_irq(spinlock_t *lock)
-{
-	raw_spin_lock_irq(&lock->rlock);
-}
+#define spin_lock_irq(lock)	raw_spin_lock_irq(lock)
 
 #define spin_lock_irqsave(lock, flags)				\
 do {								\
-	raw_spin_lock_irqsave(spinlock_check(lock), flags);	\
+	raw_spin_lock_irqsave(lock, flags);			\
 } while (0)
 
 #define spin_lock_irqsave_nested(lock, flags, subclass)			\
@@ -320,39 +327,28 @@ do {									\
 	raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \
 } while (0)
 
-static inline void spin_unlock(spinlock_t *lock)
-{
-	raw_spin_unlock(&lock->rlock);
-}
+#define spin_unlock(lock)	raw_spin_unlock(lock)
 
 static inline void spin_unlock_bh(spinlock_t *lock)
 {
 	raw_spin_unlock_bh(&lock->rlock);
 }
 
-static inline void spin_unlock_irq(spinlock_t *lock)
-{
-	raw_spin_unlock_irq(&lock->rlock);
-}
+#define spin_unlock_irq(lock)	raw_spin_unlock_irq(lock)
 
-static inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
-{
-	raw_spin_unlock_irqrestore(&lock->rlock, flags);
-}
+#define spin_unlock_irqrestore(lock, flags)	\
+	raw_spin_unlock_irqrestore(lock, flags)
 
 static inline int spin_trylock_bh(spinlock_t *lock)
 {
 	return raw_spin_trylock_bh(&lock->rlock);
 }
 
-static inline int spin_trylock_irq(spinlock_t *lock)
-{
-	return raw_spin_trylock_irq(&lock->rlock);
-}
+#define spin_trylock_irq(lock)	raw_spin_trylock_irq(lock)
 
 #define spin_trylock_irqsave(lock, flags)			\
 ({								\
-	raw_spin_trylock_irqsave(spinlock_check(lock), flags); \
+	raw_spin_trylock_irqsave(lock, flags);			\
 })
 
 static inline void spin_unlock_wait(spinlock_t *lock)
diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h
index e253ccd..378e01e 100644
--- a/include/linux/spinlock_api_smp.h
+++ b/include/linux/spinlock_api_smp.h
@@ -99,7 +99,9 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock)
  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
  */
-#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
+#if !defined(CONFIG_GENERIC_LOCKBREAK) ||	\
+	defined(CONFIG_DEBUG_LOCK_ALLOC) ||	\
+	defined(CONFIG_IPIPE)
 
 static inline unsigned long __raw_spin_lock_irqsave(raw_spinlock_t *lock)
 {
@@ -113,7 +115,7 @@ static inline unsigned long __raw_spin_lock_irqsave(raw_spinlock_t *lock)
 	 * do_raw_spin_lock_flags() code, because lockdep assumes
 	 * that interrupts are not re-enabled during lock-acquire:
 	 */
-#ifdef CONFIG_LOCKDEP
+#if defined(CONFIG_LOCKDEP) || defined(CONFIG_IPIPE)
 	LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
 #else
 	do_raw_spin_lock_flags(lock, &flags);
diff --git a/include/linux/spinlock_up.h b/include/linux/spinlock_up.h
index a26e2fb..71348f2 100644
--- a/include/linux/spinlock_up.h
+++ b/include/linux/spinlock_up.h
@@ -51,13 +51,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock)
 /*
  * Read-write spinlocks. No debug version.
  */
-#define arch_read_lock(lock)		do { (void)(lock); } while (0)
-#define arch_write_lock(lock)		do { (void)(lock); } while (0)
-#define arch_read_trylock(lock)	({ (void)(lock); 1; })
-#define arch_write_trylock(lock)	({ (void)(lock); 1; })
-#define arch_read_unlock(lock)		do { (void)(lock); } while (0)
-#define arch_write_unlock(lock)	do { (void)(lock); } while (0)
-
 #else /* DEBUG_SPINLOCK */
 #define arch_spin_is_locked(lock)	((void)(lock), 0)
 /* for sched.c and kernel_lock.c: */
@@ -67,6 +60,13 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock)
 # define arch_spin_trylock(lock)	({ (void)(lock); 1; })
 #endif /* DEBUG_SPINLOCK */
 
+#define arch_read_lock(lock)		do { (void)(lock); } while (0)
+#define arch_write_lock(lock)		do { (void)(lock); } while (0)
+#define arch_read_trylock(lock)		({ (void)(lock); 1; })
+#define arch_write_trylock(lock)	({ (void)(lock); 1; })
+#define arch_read_unlock(lock)		do { (void)(lock); } while (0)
+#define arch_write_unlock(lock)		do { (void)(lock); } while (0)
+
 #define arch_spin_is_contended(lock)	(((void)(lock), 0))
 
 #define arch_read_can_lock(lock)	(((void)(lock), 1))
diff --git a/include/linux/time.h b/include/linux/time.h
index b306178..5f07f57 100644
--- a/include/linux/time.h
+++ b/include/linux/time.h
@@ -127,6 +127,7 @@ struct timespec get_monotonic_coarse(void);
 void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim,
 				struct timespec *wtom, struct timespec *sleep);
 void timekeeping_inject_sleeptime(struct timespec *delta);
+struct timespec get_wall_to_monotonic(void);
 
 #define CURRENT_TIME		(current_kernel_time())
 #define CURRENT_TIME_SEC	((struct timespec) { get_seconds(), 0 })
diff --git a/init/Kconfig b/init/Kconfig
index 43298f9..a9eac04 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -87,6 +87,7 @@ config CROSS_COMPILE
 
 config LOCALVERSION
 	string "Local version - append to kernel release"
+	default "-ipipe"
 	help
 	  Append an extra string to the end of your kernel version.
 	  This will show up when you type uname, for example.
diff --git a/init/main.c b/init/main.c
index cb08fea2..599e1c7 100644
--- a/init/main.c
+++ b/init/main.c
@@ -485,7 +485,7 @@ asmlinkage void __init start_kernel(void)
 
 	cgroup_init_early();
 
-	local_irq_disable();
+	hard_local_irq_disable();
 	early_boot_irqs_disabled = true;
 
 /*
@@ -523,6 +523,7 @@ asmlinkage void __init start_kernel(void)
 	pidhash_init();
 	vfs_caches_init_early();
 	sort_main_extable();
+	__ipipe_init_early();
 	trap_init();
 	mm_init();
 
@@ -555,6 +556,11 @@ asmlinkage void __init start_kernel(void)
 	softirq_init();
 	timekeeping_init();
 	time_init();
+	/*
+	 * We need to wait for the interrupt and time subsystems to be
+	 * initialized before enabling the pipeline.
+	 */
+	__ipipe_init();
 	profile_init();
 	call_function_init();
 	if (!irqs_disabled())
@@ -729,6 +735,7 @@ static void __init do_basic_setup(void)
 	shmem_init();
 	driver_init();
 	init_irq_proc();
+  	__ipipe_init_proc();
 	do_ctors();
 	usermodehelper_enable();
 	do_initcalls();
diff --git a/kernel/Makefile b/kernel/Makefile
index e898c5b..22098e5 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -86,6 +86,7 @@ obj-$(CONFIG_TREE_RCU_TRACE) += rcutree_trace.o
 obj-$(CONFIG_TINY_RCU) += rcutiny.o
 obj-$(CONFIG_TINY_PREEMPT_RCU) += rcutiny.o
 obj-$(CONFIG_RELAY) += relay.o
+obj-$(CONFIG_IPIPE) += ipipe/
 obj-$(CONFIG_SYSCTL) += utsname_sysctl.o
 obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
 obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
diff --git a/kernel/exit.c b/kernel/exit.c
index 5a8a66e..5cd3999 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -958,6 +958,7 @@ NORET_TYPE void do_exit(long code)
 		acct_process();
 	trace_sched_process_exit(tsk);
 
+  	__ipipe_report_exit(tsk);
 	exit_sem(tsk);
 	exit_shm(tsk);
 	exit_files(tsk);
diff --git a/kernel/fork.c b/kernel/fork.c
index 79ee71f..4869a5f 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -276,6 +276,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig)
 	tsk->stack = ti;
 
 	setup_thread_stack(tsk, orig);
+	__ipipe_init_threadinfo(&ti->ipipe_data);
 	clear_user_return_notifier(tsk);
 	clear_tsk_need_resched(tsk);
 	stackend = end_of_stack(tsk);
@@ -556,6 +557,7 @@ void mmput(struct mm_struct *mm)
 		ksm_exit(mm);
 		khugepaged_exit(mm); /* must run before exit_mmap */
 		exit_mmap(mm);
+ 		__ipipe_report_cleanup(mm);
 		set_mm_exe_file(mm, NULL);
 		if (!list_empty(&mm->mmlist)) {
 			spin_lock(&mmlist_lock);
@@ -997,6 +999,7 @@ static void copy_flags(unsigned long clone_flags, struct task_struct *p)
 	new_flags |= PF_FORKNOEXEC;
 	new_flags |= PF_STARTING;
 	p->flags = new_flags;
+	__ipipe_clear_taskflags(p);
 	clear_freeze_flag(p);
 }
 
@@ -1379,6 +1382,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
 	cgroup_post_fork(p);
 	if (clone_flags & CLONE_THREAD)
 		threadgroup_fork_read_unlock(current);
+	__ipipe_init_taskinfo(p);
 	perf_event_fork(p);
 	return p;
 
diff --git a/kernel/ipipe/Kconfig b/kernel/ipipe/Kconfig
new file mode 100644
index 0000000..3e31ff8
--- /dev/null
+++ b/kernel/ipipe/Kconfig
@@ -0,0 +1,30 @@
+config IPIPE
+	bool "Interrupt pipeline"
+	default y
+	---help---
+	  Activate this option if you want the interrupt pipeline to be
+	  compiled in.
+
+config IPIPE_CORE
+	def_bool y
+
+config IPIPE_LEGACY
+	bool "I-pipe legacy interface"
+	depends on IPIPE
+	default n
+	---help---
+	  Activate this option if you want to control the interrupt
+	  pipeline via the legacy interface.
+
+config IPIPE_HAVE_HOSTRT
+       bool
+
+config IPIPE_HAVE_PIC_MUTE
+       bool
+
+config HAVE_IPIPE_HOSTRT
+       depends on IPIPE_LEGACY
+       bool
+
+config IPIPE_DELAYED_ATOMICSW
+       def_bool y if IPIPE_LEGACY
diff --git a/kernel/ipipe/Kconfig.debug b/kernel/ipipe/Kconfig.debug
new file mode 100644
index 0000000..ba87335
--- /dev/null
+++ b/kernel/ipipe/Kconfig.debug
@@ -0,0 +1,95 @@
+config IPIPE_DEBUG
+	bool "I-pipe debugging"
+	depends on IPIPE
+
+config IPIPE_DEBUG_CONTEXT
+	bool "Check for illicit cross-domain calls"
+	depends on IPIPE_DEBUG
+	default y
+	---help---
+	  Enable this feature to arm checkpoints in the kernel that
+	  verify the correct invocation context. On entry of critical
+	  Linux services a warning is issued if the caller is not
+	  running over the root domain.
+
+config IPIPE_DEBUG_INTERNAL
+	bool "Enable internal debug checks"
+	depends on IPIPE_DEBUG
+	default y
+	---help---
+	  When this feature is enabled, I-pipe will perform internal
+	  consistency checks of its subsystems, e.g. on per-cpu variable
+	  access.
+
+config IPIPE_TRACE
+	bool "Latency tracing"
+	depends on IPIPE_DEBUG
+	select ARCH_WANT_FRAME_POINTERS
+	select FRAME_POINTER
+	select KALLSYMS
+	select PROC_FS
+	---help---
+	  Activate this option if you want to use per-function tracing of
+	  the kernel. The tracer will collect data via instrumentation
+	  features like the one below or with the help of explicite calls
+	  of ipipe_trace_xxx(). See include/linux/ipipe_trace.h for the
+	  in-kernel tracing API. The collected data and runtime control
+	  is available via /proc/ipipe/trace/*.
+
+if IPIPE_TRACE
+
+config IPIPE_TRACE_ENABLE
+	bool "Enable tracing on boot"
+	default y
+	---help---
+	  Disable this option if you want to arm the tracer after booting
+	  manually ("echo 1 > /proc/ipipe/tracer/enable"). This can reduce
+	  boot time on slow embedded devices due to the tracer overhead.
+
+config IPIPE_TRACE_MCOUNT
+	bool "Instrument function entries"
+	default y
+	select FTRACE
+	select FUNCTION_TRACER
+	---help---
+	  When enabled, records every kernel function entry in the tracer
+	  log. While this slows down the system noticeably, it provides
+	  the highest level of information about the flow of events.
+	  However, it can be switch off in order to record only explicit
+	  I-pipe trace points.
+
+config IPIPE_TRACE_IRQSOFF
+	bool "Trace IRQs-off times"
+	default y
+	---help---
+	  Activate this option if I-pipe shall trace the longest path
+	  with hard-IRQs switched off.
+
+config IPIPE_TRACE_SHIFT
+	int "Depth of trace log (14 => 16Kpoints, 15 => 32Kpoints)"
+	range 10 18
+	default 14
+	---help---
+	  The number of trace points to hold tracing data for each
+	  trace path, as a power of 2.
+
+config IPIPE_TRACE_VMALLOC
+	bool "Use vmalloc'ed trace buffer"
+	default y if EMBEDDED
+	---help---
+	  Instead of reserving static kernel data, the required buffer
+	  is allocated via vmalloc during boot-up when this option is
+	  enabled. This can help to start systems that are low on memory,
+	  but it slightly degrades overall performance. Try this option
+	  when a traced kernel hangs unexpectedly at boot time.
+
+config IPIPE_TRACE_PANIC
+	bool "Enable panic back traces"
+	default y
+	---help---
+	  Provides services to freeze and dump a back trace on panic
+	  situations. This is used on IPIPE_DEBUG_CONTEXT exceptions
+	  as well as ordinary kernel oopses. You can control the number
+	  of printed back trace points via /proc/ipipe/trace.
+
+endif
diff --git a/kernel/ipipe/Makefile b/kernel/ipipe/Makefile
new file mode 100644
index 0000000..c3ffe63
--- /dev/null
+++ b/kernel/ipipe/Makefile
@@ -0,0 +1,3 @@
+obj-$(CONFIG_IPIPE)	+= core.o timer.o
+obj-$(CONFIG_IPIPE_TRACE) += tracer.o
+obj-$(CONFIG_IPIPE_LEGACY) += compat.o
diff --git a/kernel/ipipe/compat.c b/kernel/ipipe/compat.c
new file mode 100644
index 0000000..1147bf4
--- /dev/null
+++ b/kernel/ipipe/compat.c
@@ -0,0 +1,268 @@
+/* -*- linux-c -*-
+ * linux/kernel/ipipe/compat.c
+ *
+ * Copyright (C) 2012 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * I-pipe legacy interface.
+ */
+#include <linux/version.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/ipipe.h>
+
+static int ptd_key_count;
+
+static unsigned long ptd_key_map;
+
+IPIPE_DECLARE_SPINLOCK(__ipipe_lock);
+
+void ipipe_init_attr(struct ipipe_domain_attr *attr)
+{
+	attr->name = "anon";
+	attr->domid = 1;
+	attr->entry = NULL;
+	attr->priority = IPIPE_ROOT_PRIO;
+	attr->pdd = NULL;
+}
+EXPORT_SYMBOL_GPL(ipipe_init_attr);
+
+int ipipe_register_domain(struct ipipe_domain *ipd,
+			  struct ipipe_domain_attr *attr)
+{
+	struct ipipe_percpu_domain_data *p;
+	unsigned long flags;
+
+	BUG_ON(attr->priority != IPIPE_HEAD_PRIORITY);
+
+	ipipe_register_head(ipd, attr->name);
+	ipd->legacy.domid = attr->domid;
+	ipd->legacy.pdd = attr->pdd;
+	ipd->legacy.priority = INT_MAX;
+
+	if (attr->entry == NULL)
+		return 0;
+
+	flags = hard_smp_local_irq_save();
+	__ipipe_set_current_domain(ipd);
+	hard_smp_local_irq_restore(flags);
+
+	attr->entry();
+
+	flags = hard_local_irq_save();
+	__ipipe_set_current_domain(ipipe_root_domain);
+	p = ipipe_this_cpu_root_context();
+	if (__ipipe_ipending_p(p) &&
+	    !test_bit(IPIPE_STALL_FLAG, &p->status))
+		__ipipe_sync_stage();
+	hard_local_irq_restore(flags);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipipe_register_domain);
+
+int ipipe_unregister_domain(struct ipipe_domain *ipd)
+{
+	ipipe_unregister_head(ipd);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipipe_unregister_domain);
+
+int ipipe_alloc_ptdkey(void)
+{
+	unsigned long flags;
+	int key = -1;
+
+	spin_lock_irqsave(&__ipipe_lock,flags);
+
+	if (ptd_key_count < IPIPE_ROOT_NPTDKEYS) {
+		key = ffz(ptd_key_map);
+		set_bit(key,&ptd_key_map);
+		ptd_key_count++;
+	}
+
+	spin_unlock_irqrestore(&__ipipe_lock,flags);
+
+	return key;
+}
+EXPORT_SYMBOL_GPL(ipipe_alloc_ptdkey);
+
+int ipipe_free_ptdkey(int key)
+{
+	unsigned long flags;
+
+	if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+		return -EINVAL;
+
+	spin_lock_irqsave(&__ipipe_lock,flags);
+
+	if (test_and_clear_bit(key,&ptd_key_map))
+		ptd_key_count--;
+
+	spin_unlock_irqrestore(&__ipipe_lock,flags);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipipe_free_ptdkey);
+
+int ipipe_set_ptd(int key, void *value)
+{
+	if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+		return -EINVAL;
+
+	current->ptd[key] = value;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipipe_set_ptd);
+
+void *ipipe_get_ptd(int key)
+{
+	if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+		return NULL;
+
+	return current->ptd[key];
+}
+EXPORT_SYMBOL_GPL(ipipe_get_ptd);
+
+int ipipe_virtualize_irq(struct ipipe_domain *ipd,
+			 unsigned int irq,
+			 ipipe_irq_handler_t handler,
+			 void *cookie,
+			 ipipe_irq_ackfn_t ackfn,
+			 unsigned int modemask)
+{
+	if (handler == NULL) {
+		ipipe_free_irq(ipd, irq);
+		return 0;
+	}
+
+	return ipipe_request_irq(ipd, irq, handler, cookie, ackfn);
+}
+EXPORT_SYMBOL_GPL(ipipe_virtualize_irq);
+
+static int null_handler(unsigned int event,
+			struct ipipe_domain *from, void *data)
+{
+	/*
+	 * Legacy mode users will trap all events, at worst most
+	 * frequent ones. Therefore it is actually faster to run a
+	 * dummy handler once in a while rather than testing for a
+	 * null handler pointer each time an event is fired.
+	 */
+	return 0;
+}
+
+ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd,
+					unsigned int event,
+					ipipe_event_handler_t handler)
+{
+	ipipe_event_handler_t oldhandler;
+	int n, enables = 0;
+
+	if (event & IPIPE_EVENT_SELF) {
+		event &= ~IPIPE_EVENT_SELF;
+		IPIPE_WARN(event >= IPIPE_NR_FAULTS);
+	}
+
+	if (event >= IPIPE_NR_EVENTS)
+		return NULL;
+
+	/*
+	 * It makes no sense to run a SETSCHED notification handler
+	 * over the head domain, this introduces a useless domain
+	 * switch for doing work which ought to be root specific.
+	 * Unfortunately, some client domains using the legacy
+	 * interface still ask for this, so we silently fix their
+	 * request. This prevents ipipe_set_hooks() from yelling at us
+	 * because of an attempt to enable kernel event notifications
+	 * for the head domain.
+	 */
+	if (event == IPIPE_EVENT_SETSCHED)
+		ipd = ipipe_root_domain;
+
+	oldhandler = ipd->legacy.handlers[event];
+	ipd->legacy.handlers[event] = handler ?: null_handler;
+
+	for (n = 0; n < IPIPE_NR_FAULTS; n++) {
+		if (ipd->legacy.handlers[n] != null_handler) {
+			enables |= __IPIPE_TRAP_E;
+			break;
+		}
+	}
+
+	for (n = IPIPE_FIRST_EVENT; n < IPIPE_LAST_EVENT; n++) {
+		if (ipd->legacy.handlers[n] != null_handler) {
+			enables |= __IPIPE_KEVENT_E;
+			break;
+		}
+	}
+
+	if (ipd->legacy.handlers[IPIPE_EVENT_SYSCALL] != null_handler)
+		enables |= __IPIPE_SYSCALL_E;
+
+	ipipe_set_hooks(ipd, enables);
+
+	return oldhandler == null_handler ? NULL : oldhandler;
+}
+EXPORT_SYMBOL_GPL(ipipe_catch_event);
+
+int ipipe_setscheduler_root(struct task_struct *p, int policy, int prio)
+{
+	struct sched_param param = { .sched_priority = prio };
+	return sched_setscheduler_nocheck(p, policy, &param);
+}
+EXPORT_SYMBOL_GPL(ipipe_setscheduler_root);
+
+int ipipe_syscall_hook(struct ipipe_domain *ipd, struct pt_regs *regs)
+{
+	const int event = IPIPE_EVENT_SYSCALL;
+	return ipipe_current_domain->legacy.handlers[event](event, ipd, regs);
+}
+
+int ipipe_trap_hook(struct ipipe_trap_data *data)
+{
+	struct ipipe_domain *ipd = ipipe_head_domain;
+	struct pt_regs *regs = data->regs;
+	int ex = data->exception;
+
+	return ipd->legacy.handlers[ex](ex, ipd, regs);
+}
+
+int ipipe_kevent_hook(int kevent, void *data)
+{
+	unsigned int event = IPIPE_FIRST_EVENT + kevent;
+	struct ipipe_domain *ipd = ipipe_root_domain;
+
+	return ipd->legacy.handlers[event](event, ipd, data);
+}
+
+void __ipipe_legacy_init_stage(struct ipipe_domain *ipd)
+{
+	int n;
+
+	for (n = 0; n < IPIPE_NR_EVENTS; n++)
+		ipd->legacy.handlers[n] = null_handler;
+
+	if (ipd == &ipipe_root) {
+		ipd->legacy.domid = IPIPE_ROOT_ID;
+		ipd->legacy.priority = IPIPE_ROOT_PRIO;
+	}
+}
diff --git a/kernel/ipipe/core.c b/kernel/ipipe/core.c
new file mode 100644
index 0000000..1f32d4a
--- /dev/null
+++ b/kernel/ipipe/core.c
@@ -0,0 +1,1706 @@
+/* -*- linux-c -*-
+ * linux/kernel/ipipe/core.c
+ *
+ * Copyright (C) 2002-2012 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-independent I-PIPE core support.
+ */
+#include <linux/version.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/kallsyms.h>
+#include <linux/bitops.h>
+#include <linux/tick.h>
+#include <linux/interrupt.h>
+#ifdef CONFIG_PROC_FS
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#endif	/* CONFIG_PROC_FS */
+#include <linux/ipipe_trace.h>
+#include <linux/ipipe.h>
+#include <ipipe/setup.h>
+
+struct ipipe_domain ipipe_root;
+EXPORT_SYMBOL_GPL(ipipe_root);
+
+struct ipipe_domain *ipipe_head_domain = &ipipe_root;
+EXPORT_SYMBOL_GPL(ipipe_head_domain);
+
+#ifdef CONFIG_SMP
+static __initdata struct ipipe_percpu_domain_data bootup_context = {
+	.status = IPIPE_STALL_MASK,
+	.domain = &ipipe_root,
+};
+#else
+#define bootup_context ipipe_percpu.root
+#endif	/* !CONFIG_SMP */
+
+DEFINE_PER_CPU(struct ipipe_percpu_data, ipipe_percpu) = {
+	.root = {
+		.status = IPIPE_STALL_MASK,
+		.domain = &ipipe_root,
+	},
+	.curr = &bootup_context,
+	.hrtimer_irq = -1,
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+	.context_check = 1,
+#endif
+};
+EXPORT_PER_CPU_SYMBOL_GPL(ipipe_percpu);
+
+/* Up to 2k of pending work data per CPU. */
+#define WORKBUF_SIZE 2048
+static DEFINE_PER_CPU_ALIGNED(unsigned char[WORKBUF_SIZE], work_buf);
+static DEFINE_PER_CPU(void *, work_tail);
+static unsigned int __ipipe_work_virq;
+
+static void __ipipe_do_work(unsigned int virq, void *cookie);
+
+#ifdef CONFIG_SMP
+
+#define IPIPE_CRITICAL_TIMEOUT	1000000
+static cpumask_t __ipipe_cpu_sync_map;
+static cpumask_t __ipipe_cpu_lock_map;
+static cpumask_t __ipipe_cpu_pass_map;
+static unsigned long __ipipe_critical_lock;
+static IPIPE_DEFINE_SPINLOCK(__ipipe_cpu_barrier);
+static atomic_t __ipipe_critical_count = ATOMIC_INIT(0);
+static void (*__ipipe_cpu_sync) (void);
+
+#else /* !CONFIG_SMP */
+/*
+ * Create an alias to the unique root status, so that arch-dep code
+ * may get fast access to this percpu variable including from
+ * assembly.  A hard-coded assumption is that root.status appears at
+ * offset #0 of the ipipe_percpu struct.
+ */
+extern unsigned long __ipipe_root_status
+__attribute__((alias(__stringify(ipipe_percpu))));
+EXPORT_SYMBOL_GPL(__ipipe_root_status);
+
+#endif /* !CONFIG_SMP */
+
+IPIPE_DEFINE_SPINLOCK(__ipipe_lock);
+
+static unsigned long __ipipe_virtual_irq_map;
+
+#ifdef CONFIG_PRINTK
+unsigned int __ipipe_printk_virq;
+int __ipipe_printk_bypass;
+#endif /* CONFIG_PRINTK */
+
+#ifdef CONFIG_PROC_FS
+
+struct proc_dir_entry *ipipe_proc_root;
+
+static int __ipipe_version_info_proc(char *page,
+				     char **start,
+				     off_t off, int count, int *eof, void *data)
+{
+	int len = sprintf(page, "%d\n", IPIPE_CORE_RELEASE);
+
+	len -= off;
+
+	if (len <= off + count)
+		*eof = 1;
+
+	*start = page + off;
+
+	if(len > count)
+		len = count;
+
+	if(len < 0)
+		len = 0;
+
+	return len;
+}
+
+static int __ipipe_common_info_show(struct seq_file *p, void *data)
+{
+	struct ipipe_domain *ipd = (struct ipipe_domain *)p->private;
+	char handling, lockbit, virtuality;
+	unsigned long ctlbits;
+	unsigned int irq;
+
+	seq_printf(p, "       +--- Handled\n");
+	seq_printf(p, "       |+-- Locked\n");
+	seq_printf(p, "       ||+- Virtual\n");
+	seq_printf(p, "[IRQ]  |||\n");
+
+	mutex_lock(&ipd->mutex);
+
+	for (irq = 0; irq < IPIPE_NR_IRQS; irq++) {
+		ctlbits = ipd->irqs[irq].control;
+		/*
+		 * There might be a hole between the last external IRQ
+		 * and the first virtual one; skip it.
+		 */
+		if (irq >= IPIPE_NR_XIRQS && !ipipe_virtual_irq_p(irq))
+			continue;
+
+		if (ipipe_virtual_irq_p(irq)
+		    && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map))
+			/* Non-allocated virtual IRQ; skip it. */
+			continue;
+
+		if (ctlbits & IPIPE_HANDLE_MASK)
+			handling = 'H';
+		else
+			handling = '.';
+
+		if (ctlbits & IPIPE_LOCK_MASK)
+			lockbit = 'L';
+		else
+			lockbit = '.';
+
+		if (ipipe_virtual_irq_p(irq))
+			virtuality = 'V';
+		else
+			virtuality = '.';
+
+		seq_printf(p, " %3u:  %c%c%c\n",
+			     irq, handling, lockbit, virtuality);
+	}
+
+	mutex_unlock(&ipd->mutex);
+
+	return 0;
+}
+
+static int __ipipe_common_info_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, __ipipe_common_info_show, PROC_I(inode)->pde->data);
+}
+
+static struct file_operations __ipipe_info_proc_ops = {
+	.owner		= THIS_MODULE,
+	.open		= __ipipe_common_info_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+void add_domain_proc(struct ipipe_domain *ipd)
+{
+	struct proc_dir_entry *e = create_proc_entry(ipd->name, 0444, ipipe_proc_root);
+	if (e) {
+		e->proc_fops = &__ipipe_info_proc_ops;
+		e->data = (void*) ipd;
+	}
+}
+
+void remove_domain_proc(struct ipipe_domain *ipd)
+{
+	remove_proc_entry(ipd->name,ipipe_proc_root);
+}
+
+void __init __ipipe_init_proc(void)
+{
+	ipipe_proc_root = create_proc_entry("ipipe",S_IFDIR, 0);
+	create_proc_read_entry("version",0444,ipipe_proc_root,&__ipipe_version_info_proc,NULL);
+	add_domain_proc(ipipe_root_domain);
+
+	__ipipe_init_tracer();
+}
+
+#else
+
+static inline void add_domain_proc(struct ipipe_domain *ipd)
+{
+}
+
+static inline void remove_domain_proc(struct ipipe_domain *ipd)
+{
+}
+
+#endif	/* CONFIG_PROC_FS */
+
+static void init_stage(struct ipipe_domain *ipd)
+{
+	memset(&ipd->irqs, 0, sizeof(ipd->irqs));
+	mutex_init(&ipd->mutex);
+	__ipipe_legacy_init_stage(ipd);
+	__ipipe_hook_critical_ipi(ipd);
+}
+
+static inline int root_context_offset(void)
+{
+	void root_context_not_at_start_of_ipipe_percpu(void);
+
+	/* ipipe_percpu.root must be found at offset #0. */
+
+	if (offsetof(struct ipipe_percpu_data, root))
+		root_context_not_at_start_of_ipipe_percpu();
+
+	return 0;
+}
+
+#ifdef CONFIG_SMP
+
+static inline void fixup_percpu_data(void)
+{
+	struct ipipe_percpu_data *p;
+	int cpu;
+
+	/*
+	 * ipipe_percpu.curr cannot be assigned statically to
+	 * &ipipe_percpu.root, due to the dynamic nature of percpu
+	 * data. So we make ipipe_percpu.curr refer to a temporary
+	 * boot up context in static memory, until we can fixup all
+	 * context pointers in this routine, after per-cpu areas have
+	 * been eventually set up. The temporary context data is
+	 * copied to per_cpu(ipipe_percpu, 0).root in the same move.
+	 *
+	 * Obviously, this code must run over the boot CPU, before SMP
+	 * operations start.
+	 */
+	BUG_ON(smp_processor_id() || !irqs_disabled());
+
+	per_cpu(ipipe_percpu, 0).root = bootup_context;
+
+	for_each_possible_cpu(cpu) {
+		p = &per_cpu(ipipe_percpu, cpu);
+		p->curr = &p->root;
+	}
+}
+
+#else /* !CONFIG_SMP */
+
+static inline void fixup_percpu_data(void) { }
+
+#endif /* CONFIG_SMP */
+
+void __init __ipipe_init_early(void)
+{
+	struct ipipe_domain *ipd = &ipipe_root;
+	int cpu;
+
+	fixup_percpu_data();
+
+	/*
+	 * A lightweight registration code for the root domain. We are
+	 * running on the boot CPU, hw interrupts are off, and
+	 * secondary CPUs are still lost in space.
+	 */
+	ipd->name = "Linux";
+	ipd->context_offset = root_context_offset();
+	init_stage(ipd);
+
+	/*
+	 * Do the early init stuff. First we do the per-arch pipeline
+	 * core setup, then we run the per-client setup code. At this
+	 * point, the kernel does not provide much services yet: be
+	 * careful.
+	 */
+	__ipipe_early_core_setup();
+	__ipipe_early_client_setup();
+
+#ifdef CONFIG_PRINTK
+	__ipipe_printk_virq = ipipe_alloc_virq();
+	ipd->irqs[__ipipe_printk_virq].handler = __ipipe_flush_printk;
+	ipd->irqs[__ipipe_printk_virq].cookie = NULL;
+	ipd->irqs[__ipipe_printk_virq].ackfn = NULL;
+	ipd->irqs[__ipipe_printk_virq].control = IPIPE_HANDLE_MASK;
+#endif /* CONFIG_PRINTK */
+
+	__ipipe_work_virq = ipipe_alloc_virq();
+	ipd->irqs[__ipipe_work_virq].handler = __ipipe_do_work;
+	ipd->irqs[__ipipe_work_virq].cookie = NULL;
+	ipd->irqs[__ipipe_work_virq].ackfn = NULL;
+	ipd->irqs[__ipipe_work_virq].control = IPIPE_HANDLE_MASK;
+
+	for_each_possible_cpu(cpu)
+		per_cpu(work_tail, cpu) = per_cpu(work_buf, cpu);
+}
+
+void __init __ipipe_init(void)
+{
+	/* Now we may engage the pipeline. */
+	__ipipe_enable_pipeline();
+
+	printk(KERN_INFO "Interrupt pipeline (release #%d)\n",
+	       IPIPE_CORE_RELEASE);
+}
+
+static inline void init_head_stage(struct ipipe_domain *ipd)
+{
+	struct ipipe_percpu_domain_data *p;
+	int cpu;
+
+	/* Must be set first, used in ipipe_percpu_context(). */
+	ipd->context_offset = offsetof(struct ipipe_percpu_data, head);
+
+	for_each_online_cpu(cpu) {
+		p = ipipe_percpu_context(ipd, cpu);
+		memset(p, 0, sizeof(*p));
+		p->domain = ipd;
+	}
+
+	init_stage(ipd);
+}
+
+void ipipe_register_head(struct ipipe_domain *ipd, const char *name)
+{
+	BUG_ON(!ipipe_root_p || ipd == &ipipe_root);
+
+	ipd->name = name;
+	init_head_stage(ipd);
+	barrier();
+	ipipe_head_domain = ipd;
+	add_domain_proc(ipd);
+
+	printk(KERN_INFO "I-pipe: head domain %s registered.\n", name);
+}
+EXPORT_SYMBOL_GPL(ipipe_register_head);
+
+void ipipe_unregister_head(struct ipipe_domain *ipd)
+{
+	BUG_ON(!ipipe_root_p || ipd != ipipe_head_domain);
+
+	ipipe_head_domain = &ipipe_root;
+	smp_mb();
+	mutex_lock(&ipd->mutex);
+	remove_domain_proc(ipd);
+	mutex_unlock(&ipd->mutex);
+
+	printk(KERN_INFO "I-pipe: head domain %s unregistered.\n", ipd->name);
+}
+EXPORT_SYMBOL_GPL(ipipe_unregister_head);
+
+void ipipe_unstall_root(void)
+{
+	struct ipipe_percpu_domain_data *p;
+
+	hard_local_irq_disable();
+
+	/* This helps catching bad usage from assembly call sites. */
+	ipipe_root_only();
+
+	p = ipipe_this_cpu_root_context();
+
+	__clear_bit(IPIPE_STALL_FLAG, &p->status);
+
+	if (unlikely(__ipipe_ipending_p(p)))
+		__ipipe_sync_stage();
+
+	hard_local_irq_enable();
+}
+EXPORT_SYMBOL_GPL(ipipe_unstall_root);
+
+void ipipe_restore_root(unsigned long x)
+{
+	ipipe_root_only();
+
+	if (x)
+		ipipe_stall_root();
+	else
+		ipipe_unstall_root();
+}
+EXPORT_SYMBOL_GPL(ipipe_restore_root);
+
+void __ipipe_restore_root_nosync(unsigned long x)
+{
+	struct ipipe_percpu_domain_data *p = ipipe_this_cpu_root_context();
+
+	if (raw_irqs_disabled_flags(x)) {
+		__set_bit(IPIPE_STALL_FLAG, &p->status);
+		trace_hardirqs_off();
+	} else {
+		trace_hardirqs_on();
+		__clear_bit(IPIPE_STALL_FLAG, &p->status);
+	}
+}
+EXPORT_SYMBOL_GPL(__ipipe_restore_root_nosync);
+
+void ipipe_unstall_head(void)
+{
+	struct ipipe_percpu_domain_data *p = ipipe_this_cpu_head_context();
+
+	hard_local_irq_disable();
+
+	__clear_bit(IPIPE_STALL_FLAG, &p->status);
+
+	if (unlikely(__ipipe_ipending_p(p)))
+		__ipipe_sync_pipeline(ipipe_head_domain);
+
+	hard_local_irq_enable();
+}
+EXPORT_SYMBOL_GPL(ipipe_unstall_head);
+
+void __ipipe_restore_head(unsigned long x) /* hw interrupt off */
+{
+	struct ipipe_percpu_domain_data *p = ipipe_this_cpu_head_context();
+
+	if (x) {
+#ifdef CONFIG_DEBUG_KERNEL
+		static int warned;
+		if (!warned &&
+		    __test_and_set_bit(IPIPE_STALL_FLAG, &p->status)) {
+			/*
+			 * Already stalled albeit ipipe_restore_head()
+			 * should have detected it? Send a warning once.
+			 */
+			hard_local_irq_enable();
+			warned = 1;
+			printk(KERN_WARNING
+				   "I-pipe: ipipe_restore_head() optimization failed.\n");
+			dump_stack();
+			hard_local_irq_disable();
+		}
+#else /* !CONFIG_DEBUG_KERNEL */
+		__set_bit(IPIPE_STALL_FLAG, &p->status);
+#endif /* CONFIG_DEBUG_KERNEL */
+	} else {
+		__clear_bit(IPIPE_STALL_FLAG, &p->status);
+		if (unlikely(__ipipe_ipending_p(p)))
+			__ipipe_sync_pipeline(ipipe_head_domain);
+		hard_local_irq_enable();
+	}
+}
+EXPORT_SYMBOL_GPL(__ipipe_restore_head);
+
+void __ipipe_spin_lock_irq(ipipe_spinlock_t *lock)
+{
+	hard_local_irq_disable();
+	arch_spin_lock(&lock->arch_lock);
+	__set_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status);
+}
+EXPORT_SYMBOL_GPL(__ipipe_spin_lock_irq);
+
+void __ipipe_spin_unlock_irq(ipipe_spinlock_t *lock)
+{
+	arch_spin_unlock(&lock->arch_lock);
+	__clear_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status);
+	hard_local_irq_enable();
+}
+EXPORT_SYMBOL_GPL(__ipipe_spin_unlock_irq);
+
+unsigned long __ipipe_spin_lock_irqsave(ipipe_spinlock_t *lock)
+{
+	unsigned long flags;
+	int s;
+
+	flags = hard_local_irq_save();
+	arch_spin_lock(&lock->arch_lock);
+	s = __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status);
+
+	return arch_mangle_irq_bits(s, flags);
+}
+EXPORT_SYMBOL_GPL(__ipipe_spin_lock_irqsave);
+
+int __ipipe_spin_trylock_irqsave(ipipe_spinlock_t *lock,
+				 unsigned long *x)
+{
+	unsigned long flags;
+	int s;
+
+	flags = hard_local_irq_save();
+	if (!arch_spin_trylock(&lock->arch_lock)) {
+		hard_local_irq_restore(flags);
+		return 0;
+	}
+	s = __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status);
+	*x = arch_mangle_irq_bits(s, flags);
+
+	return 1;
+}
+EXPORT_SYMBOL_GPL(__ipipe_spin_trylock_irqsave);
+
+void __ipipe_spin_unlock_irqrestore(ipipe_spinlock_t *lock,
+				    unsigned long x)
+{
+	arch_spin_unlock(&lock->arch_lock);
+	if (!arch_demangle_irq_bits(&x))
+		__clear_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status);
+	hard_local_irq_restore(x);
+}
+EXPORT_SYMBOL_GPL(__ipipe_spin_unlock_irqrestore);
+
+int __ipipe_spin_trylock_irq(ipipe_spinlock_t *lock)
+{
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	if (!arch_spin_trylock(&lock->arch_lock)) {
+		hard_local_irq_restore(flags);
+		return 0;
+	}
+	__set_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status);
+
+	return 1;
+}
+EXPORT_SYMBOL_GPL(__ipipe_spin_trylock_irq);
+
+void __ipipe_spin_unlock_irqbegin(ipipe_spinlock_t *lock)
+{
+	arch_spin_unlock(&lock->arch_lock);
+}
+
+void __ipipe_spin_unlock_irqcomplete(unsigned long x)
+{
+	if (!arch_demangle_irq_bits(&x))
+		__clear_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status);
+	hard_local_irq_restore(x);
+}
+
+#ifdef __IPIPE_3LEVEL_IRQMAP
+
+/* Must be called hw IRQs off. */
+static inline void __ipipe_set_irq_held(struct ipipe_percpu_domain_data *p,
+					unsigned int irq)
+{
+	__set_bit(irq, p->irqheld_map);
+	p->irqall[irq]++;
+}
+
+/* Must be called hw IRQs off. */
+void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned int irq)
+{
+	struct ipipe_percpu_domain_data *p = ipipe_this_cpu_context(ipd);
+	int l0b, l1b;
+
+	IPIPE_WARN_ONCE(!hard_irqs_disabled());
+
+	l0b = irq / (BITS_PER_LONG * BITS_PER_LONG);
+	l1b = irq / BITS_PER_LONG;
+
+	if (likely(!test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) {
+		__set_bit(irq, p->irqpend_lomap);
+		__set_bit(l1b, p->irqpend_mdmap);
+		__set_bit(l0b, &p->irqpend_himap);
+	} else
+		__set_bit(irq, p->irqheld_map);
+
+	p->irqall[irq]++;
+}
+EXPORT_SYMBOL_GPL(__ipipe_set_irq_pending);
+
+/* Must be called hw IRQs off. */
+void __ipipe_lock_irq(unsigned int irq)
+{
+	struct ipipe_domain *ipd = ipipe_root_domain;
+	struct ipipe_percpu_domain_data *p;
+	int l0b, l1b;
+
+	IPIPE_WARN_ONCE(!hard_irqs_disabled());
+
+	/*
+	 * Interrupts requested by a registered head domain cannot be
+	 * locked, since this would make no sense: interrupts are
+	 * globally masked at CPU level when the head domain is
+	 * stalled, so there is no way we could encounter the
+	 * situation IRQ locks are handling.
+	 */
+	if (test_and_set_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
+		return;
+
+	l0b = irq / (BITS_PER_LONG * BITS_PER_LONG);
+	l1b = irq / BITS_PER_LONG;
+
+	p = ipipe_this_cpu_context(ipd);
+	if (__test_and_clear_bit(irq, p->irqpend_lomap)) {
+		__set_bit(irq, p->irqheld_map);
+		if (p->irqpend_lomap[l1b] == 0) {
+			__clear_bit(l1b, p->irqpend_mdmap);
+			if (p->irqpend_mdmap[l0b] == 0)
+				__clear_bit(l0b, &p->irqpend_himap);
+		}
+	}
+}
+EXPORT_SYMBOL_GPL(__ipipe_lock_irq);
+
+/* Must be called hw IRQs off. */
+void __ipipe_unlock_irq(unsigned int irq)
+{
+	struct ipipe_domain *ipd = ipipe_root_domain;
+	struct ipipe_percpu_domain_data *p;
+	int l0b, l1b, cpu;
+
+	IPIPE_WARN_ONCE(!hard_irqs_disabled());
+
+	if (!test_and_clear_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
+		return;
+
+	l0b = irq / (BITS_PER_LONG * BITS_PER_LONG);
+	l1b = irq / BITS_PER_LONG;
+
+	for_each_online_cpu(cpu) {
+		p = ipipe_this_cpu_root_context();
+		if (test_and_clear_bit(irq, p->irqheld_map)) {
+			/* We need atomic ops here: */
+			set_bit(irq, p->irqpend_lomap);
+			set_bit(l1b, p->irqpend_mdmap);
+			set_bit(l0b, &p->irqpend_himap);
+		}
+	}
+}
+EXPORT_SYMBOL_GPL(__ipipe_unlock_irq);
+
+static inline int __ipipe_next_irq(struct ipipe_percpu_domain_data *p)
+{
+	int l0b, l1b, l2b;
+	unsigned long l0m, l1m, l2m;
+	unsigned int irq;
+
+	l0m = p->irqpend_himap;
+	if (unlikely(l0m == 0))
+		return -1;
+
+	l0b = __ipipe_ffnz(l0m);
+	l1m = p->irqpend_mdmap[l0b];
+	if (unlikely(l1m == 0))
+		return -1;
+
+	l1b = __ipipe_ffnz(l1m) + l0b * BITS_PER_LONG;
+	l2m = p->irqpend_lomap[l1b];
+	if (unlikely(l2m == 0))
+		return -1;
+
+	l2b = __ipipe_ffnz(l2m);
+	irq = l1b * BITS_PER_LONG + l2b;
+
+	__clear_bit(irq, p->irqpend_lomap);
+	if (p->irqpend_lomap[l1b] == 0) {
+		__clear_bit(l1b, p->irqpend_mdmap);
+		if (p->irqpend_mdmap[l0b] == 0)
+			__clear_bit(l0b, &p->irqpend_himap);
+	}
+
+	return irq;
+}
+
+#else /* __IPIPE_2LEVEL_IRQMAP */
+
+/* Must be called hw IRQs off. */
+static inline void __ipipe_set_irq_held(struct ipipe_percpu_domain_data *p,
+					unsigned int irq)
+{
+	__set_bit(irq, p->irqheld_map);
+	p->irqall[irq]++;
+}
+
+/* Must be called hw IRQs off. */
+void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned int irq)
+{
+	struct ipipe_percpu_domain_data *p = ipipe_this_cpu_context(ipd);
+	int l0b = irq / BITS_PER_LONG;
+
+	IPIPE_WARN_ONCE(!hard_irqs_disabled());
+
+	if (likely(!test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) {
+		__set_bit(irq, p->irqpend_lomap);
+		__set_bit(l0b, &p->irqpend_himap);
+	} else
+		__set_bit(irq, p->irqheld_map);
+
+	p->irqall[irq]++;
+}
+EXPORT_SYMBOL_GPL(__ipipe_set_irq_pending);
+
+/* Must be called hw IRQs off. */
+void __ipipe_lock_irq(unsigned int irq)
+{
+	struct ipipe_percpu_domain_data *p;
+	int l0b = irq / BITS_PER_LONG;
+
+	IPIPE_WARN_ONCE(!hard_irqs_disabled());
+
+	if (test_and_set_bit(IPIPE_LOCK_FLAG,
+			     &ipipe_root_domain->irqs[irq].control))
+		return;
+
+	p = ipipe_this_cpu_root_context();
+	if (__test_and_clear_bit(irq, p->irqpend_lomap)) {
+		__set_bit(irq, p->irqheld_map);
+		if (p->irqpend_lomap[l0b] == 0)
+			__clear_bit(l0b, &p->irqpend_himap);
+	}
+}
+EXPORT_SYMBOL_GPL(__ipipe_lock_irq);
+
+/* Must be called hw IRQs off. */
+void __ipipe_unlock_irq(unsigned int irq)
+{
+	struct ipipe_domain *ipd = ipipe_root_domain;
+	struct ipipe_percpu_domain_data *p;
+	int l0b = irq / BITS_PER_LONG, cpu;
+
+	IPIPE_WARN_ONCE(!hard_irqs_disabled());
+
+	if (!test_and_clear_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
+		return;
+
+	for_each_online_cpu(cpu) {
+		p = ipipe_percpu_context(ipd, cpu);
+		if (test_and_clear_bit(irq, p->irqheld_map)) {
+			/* We need atomic ops here: */
+			set_bit(irq, p->irqpend_lomap);
+			set_bit(l0b, &p->irqpend_himap);
+		}
+	}
+}
+EXPORT_SYMBOL_GPL(__ipipe_unlock_irq);
+
+static inline int __ipipe_next_irq(struct ipipe_percpu_domain_data *p)
+{
+	unsigned long l0m, l1m;
+	int l0b, l1b;
+
+	l0m = p->irqpend_himap;
+	if (unlikely(l0m == 0))
+		return -1;
+
+	l0b = __ipipe_ffnz(l0m);
+	l1m = p->irqpend_lomap[l0b];
+	if (unlikely(l1m == 0))
+		return -1;
+
+	l1b = __ipipe_ffnz(l1m);
+	__clear_bit(l1b, &p->irqpend_lomap[l0b]);
+	if (p->irqpend_lomap[l0b] == 0)
+		__clear_bit(l0b, &p->irqpend_himap);
+
+	return l0b * BITS_PER_LONG + l1b;
+}
+
+#endif /* __IPIPE_2LEVEL_IRQMAP */
+
+void __ipipe_do_sync_pipeline(struct ipipe_domain *top)
+{
+	struct ipipe_percpu_domain_data *p;
+	struct ipipe_domain *ipd;
+
+	/* We must enter over the root domain. */
+	IPIPE_WARN_ONCE(__ipipe_current_domain != ipipe_root_domain);
+	ipd = top;
+next:
+	p = ipipe_this_cpu_context(ipd);
+	if (test_bit(IPIPE_STALL_FLAG, &p->status))
+		return;
+
+	if (__ipipe_ipending_p(p)) {
+		if (ipd == ipipe_root_domain)
+			__ipipe_sync_stage();
+		else {
+			/* Switching to head. */
+			p->coflags &= ~__IPIPE_ALL_R;
+			__ipipe_set_current_context(p);
+			__ipipe_sync_stage();
+			__ipipe_set_current_domain(ipipe_root_domain);
+		}
+	}
+
+	if (ipd != ipipe_root_domain) {
+		ipd = ipipe_root_domain;
+		goto next;
+	}
+}
+EXPORT_SYMBOL_GPL(__ipipe_do_sync_pipeline);
+
+unsigned int ipipe_alloc_virq(void)
+{
+	unsigned long flags, irq = 0;
+	int ipos;
+
+	spin_lock_irqsave(&__ipipe_lock, flags);
+
+	if (__ipipe_virtual_irq_map != ~0) {
+		ipos = ffz(__ipipe_virtual_irq_map);
+		set_bit(ipos, &__ipipe_virtual_irq_map);
+		irq = ipos + IPIPE_VIRQ_BASE;
+	}
+
+	spin_unlock_irqrestore(&__ipipe_lock, flags);
+
+	return irq;
+}
+EXPORT_SYMBOL_GPL(ipipe_alloc_virq);
+
+void ipipe_free_virq(unsigned int virq)
+{
+	clear_bit(virq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map);
+	smp_mb__after_clear_bit();
+}
+EXPORT_SYMBOL_GPL(ipipe_free_virq);
+
+int ipipe_request_irq(struct ipipe_domain *ipd,
+		      unsigned int irq,
+		      ipipe_irq_handler_t handler,
+		      void *cookie,
+		      ipipe_irq_ackfn_t ackfn)
+{
+	unsigned long flags;
+	int ret = 0;
+
+	ipipe_root_only();
+
+	if (handler == NULL ||
+	    (irq >= IPIPE_NR_XIRQS && !ipipe_virtual_irq_p(irq)))
+		return -EINVAL;
+
+	spin_lock_irqsave(&__ipipe_lock, flags);
+
+	if (ipd->irqs[irq].handler) {
+		ret = -EBUSY;
+		goto out;
+	}
+
+	if (ackfn == NULL)
+		ackfn = ipipe_root_domain->irqs[irq].ackfn;
+
+	ipd->irqs[irq].handler = handler;
+	ipd->irqs[irq].cookie = cookie;
+	ipd->irqs[irq].ackfn = ackfn;
+	ipd->irqs[irq].control = IPIPE_HANDLE_MASK;
+
+	if (irq < NR_IRQS)
+		__ipipe_enable_irqdesc(ipd, irq);
+out:
+	spin_unlock_irqrestore(&__ipipe_lock, flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(ipipe_request_irq);
+
+void ipipe_free_irq(struct ipipe_domain *ipd,
+		    unsigned int irq)
+{
+	unsigned long flags;
+
+	ipipe_root_only();
+
+	spin_lock_irqsave(&__ipipe_lock, flags);
+
+	if (ipd->irqs[irq].handler == NULL)
+		goto out;
+
+	ipd->irqs[irq].handler = NULL;
+	ipd->irqs[irq].cookie = NULL;
+	ipd->irqs[irq].ackfn = NULL;
+	ipd->irqs[irq].control = 0;
+
+	if (irq < NR_IRQS)
+		__ipipe_disable_irqdesc(ipd, irq);
+out:
+	spin_unlock_irqrestore(&__ipipe_lock, flags);
+}
+EXPORT_SYMBOL_GPL(ipipe_free_irq);
+
+void ipipe_set_hooks(struct ipipe_domain *ipd, int enables)
+{
+	struct ipipe_percpu_domain_data *p;
+	unsigned long flags;
+	int cpu, wait;
+
+	if (ipd == ipipe_root_domain) {
+		IPIPE_WARN(enables & __IPIPE_TRAP_E);
+		enables &= ~__IPIPE_TRAP_E;
+	} else {
+		IPIPE_WARN(enables & __IPIPE_KEVENT_E);
+		enables &= ~__IPIPE_KEVENT_E;
+	}
+
+	flags = ipipe_critical_enter(NULL);
+
+	for_each_online_cpu(cpu) {
+		p = ipipe_percpu_context(ipd, cpu);
+		p->coflags &= ~__IPIPE_ALL_E;
+		p->coflags |= enables;
+	}
+
+	wait = (enables ^ __IPIPE_ALL_E) << __IPIPE_SHIFT_R;
+	if (wait == 0 || !__ipipe_root_p) {
+		ipipe_critical_exit(flags);
+		return;
+	}
+
+	ipipe_this_cpu_context(ipd)->coflags &= ~wait;
+
+	ipipe_critical_exit(flags);
+
+	/*
+	 * In case we cleared some hooks over the root domain, we have
+	 * to wait for any ongoing execution to finish, since our
+	 * caller might subsequently unmap the target domain code.
+	 *
+	 * We synchronize with the relevant __ipipe_notify_*()
+	 * helpers, disabling all hooks before we start waiting for
+	 * completion on all CPUs.
+	 */
+	for_each_online_cpu(cpu) {
+		while (ipipe_percpu_context(ipd, cpu)->coflags & wait)
+			schedule_timeout_interruptible(HZ / 50);
+	}
+}
+EXPORT_SYMBOL_GPL(ipipe_set_hooks);
+
+int __weak ipipe_syscall_hook(struct ipipe_domain *ipd, struct pt_regs *regs)
+{
+	return 0;
+}
+
+int __ipipe_notify_syscall(struct pt_regs *regs)
+{
+	struct ipipe_domain *caller_domain, *this_domain, *ipd;
+	struct ipipe_percpu_domain_data *p;
+	unsigned long flags;
+	int ret = 0;
+
+	flags = hard_local_irq_save();
+	caller_domain = this_domain = __ipipe_current_domain;
+	ipd = ipipe_head_domain;
+next:
+	p = ipipe_this_cpu_context(ipd);
+	if (likely(p->coflags & __IPIPE_SYSCALL_E)) {
+		__ipipe_set_current_context(p);
+		p->coflags |= __IPIPE_SYSCALL_R;
+		hard_local_irq_restore(flags);
+		ret = ipipe_syscall_hook(caller_domain, regs);
+		flags = hard_local_irq_save();
+		p->coflags &= ~__IPIPE_SYSCALL_R;
+		if (__ipipe_current_domain != ipd)
+			/* Account for domain migration. */
+			this_domain = __ipipe_current_domain;
+		else
+			__ipipe_set_current_domain(this_domain);
+	}
+
+	if (this_domain == ipipe_root_domain &&
+	    ipd != ipipe_root_domain && ret == 0) {
+		ipd = ipipe_root_domain;
+		goto next;
+	}
+
+	hard_local_irq_restore(flags);
+
+	return ret;
+}
+
+int __weak ipipe_trap_hook(struct ipipe_trap_data *data)
+{
+	return 0;
+}
+
+int __ipipe_notify_trap(int exception, struct pt_regs *regs)
+{
+	struct ipipe_percpu_domain_data *p;
+	struct ipipe_trap_data data;
+	unsigned long flags;
+	int ret = 0;
+
+	flags = hard_local_irq_save();
+
+	/*
+	 * We send a notification about all traps raised over a
+	 * registered head domain only.
+	 */
+	if (__ipipe_root_p)
+		goto out;
+
+	p = ipipe_this_cpu_head_context();
+	if (likely(p->coflags & __IPIPE_TRAP_E)) {
+		p->coflags |= __IPIPE_TRAP_R;
+		hard_local_irq_restore(flags);
+		data.exception = exception;
+		data.regs = regs;
+		ret = ipipe_trap_hook(&data);
+		flags = hard_local_irq_save();
+		p->coflags &= ~__IPIPE_TRAP_R;
+	}
+out:
+	hard_local_irq_restore(flags);
+
+	return ret;
+}
+
+int __weak ipipe_kevent_hook(int kevent, void *data)
+{
+	return 0;
+}
+
+int __ipipe_notify_kevent(int kevent, void *data)
+{
+	struct ipipe_percpu_domain_data *p;
+	unsigned long flags;
+	int ret = 0;
+
+	ipipe_root_only();
+
+	flags = hard_local_irq_save();
+
+	p = ipipe_this_cpu_root_context();
+	if (likely(p->coflags & __IPIPE_KEVENT_E)) {
+		p->coflags |= __IPIPE_KEVENT_R;
+		hard_local_irq_restore(flags);
+		ret = ipipe_kevent_hook(kevent, data);
+		flags = hard_local_irq_save();
+		p->coflags &= ~__IPIPE_KEVENT_R;
+	}
+
+	hard_local_irq_restore(flags);
+
+	return ret;
+}
+
+void __weak ipipe_migration_hook(struct task_struct *p)
+{
+}
+
+static void complete_domain_migration(void) /* hw IRQs off */
+{
+	struct ipipe_percpu_data *pd;
+	struct task_struct *p;
+
+	pd = __this_cpu_ptr(&ipipe_percpu);
+	p = pd->task_hijacked;
+	if (p) {
+		p->state &= ~TASK_HARDENING;
+		pd->task_hijacked = NULL;
+		ipipe_migration_hook(p);
+	}
+}
+
+void __ipipe_complete_domain_migration(void)
+{
+	unsigned long flags;
+
+	ipipe_root_only();
+	flags = hard_local_irq_save();
+	complete_domain_migration();
+	hard_local_irq_restore(flags);
+}
+EXPORT_SYMBOL_GPL(__ipipe_complete_domain_migration);
+
+int __ipipe_switch_tail(void)
+{
+	int x;
+
+#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH
+	hard_local_irq_disable();
+#endif
+	x = __ipipe_root_p;
+
+#ifdef CONFIG_IPIPE_LEGACY
+	current->state &= ~TASK_HARDENING;
+#else
+	complete_domain_migration();
+#endif	/* !CONFIG_IPIPE_LEGACY */
+
+#ifndef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH
+	if (x)
+#endif
+		hard_local_irq_enable();
+
+	return !x;
+}
+
+void __ipipe_notify_vm_preemption(void)
+{
+	struct ipipe_vm_notifier *vmf;
+	struct ipipe_percpu_data *p;
+
+	ipipe_check_irqoff();
+	p = __ipipe_this_cpu_ptr(&ipipe_percpu);
+	vmf = p->vm_notifier;
+	if (unlikely(vmf))
+		vmf->handler(vmf);
+}
+EXPORT_SYMBOL_GPL(__ipipe_notify_vm_preemption);
+
+void __ipipe_dispatch_irq(unsigned int irq, int flags) /* hw interrupts off */
+{
+	struct ipipe_domain *ipd;
+	struct irq_desc *desc;
+	unsigned long control;
+
+	/*
+	 * Survival kit when reading this code:
+	 *
+	 * - we have two main situations, leading to three cases for
+	 *   handling interrupts:
+	 *
+	 *   a) the root domain is alone, no registered head domain
+	 *      => all interrupts are delivered via the fast dispatcher.
+	 *   b) a head domain is registered
+	 *      => head domain IRQs go through the fast dispatcher
+	 *      => root domain IRQs go through the interrupt log
+	 *
+	 * - when no head domain is registered, ipipe_head_domain ==
+	 *   ipipe_root_domain == &ipipe_root.
+	 *
+	 * - the caller tells us whether we should acknowledge this
+	 *   IRQ. Even virtual IRQs may require acknowledge on some
+	 *   platforms (e.g. arm/SMP).
+	 *
+	 * - the caller tells us whether we may try to run the IRQ log
+	 *   syncer. Typically, demuxed IRQs won't be synced
+	 *   immediately.
+	 *
+	 * - multiplex IRQs most likely have a valid acknowledge
+	 *   handler and we may not be called with IPIPE_IRQF_NOACK
+	 *   for them. The ack handler for the multiplex IRQ actually
+	 *   decodes the demuxed interrupts.
+	 */
+
+#ifdef CONFIG_IPIPE_DEBUG
+	if (unlikely(irq >= IPIPE_NR_IRQS) ||
+	    (irq < NR_IRQS && irq_to_desc(irq) == NULL)) {
+		printk(KERN_ERR "I-pipe: spurious interrupt %u\n", irq);
+		return;
+	}
+#endif
+	/*
+	 * CAUTION: on some archs, virtual IRQs may have acknowledge
+	 * handlers. Multiplex IRQs should have one too.
+	 */
+	desc = irq >= NR_IRQS ? NULL : irq_to_desc(irq);
+	if (flags & IPIPE_IRQF_NOACK)
+		IPIPE_WARN_ONCE(desc && ipipe_chained_irq_p(irq));
+	else {
+		ipd = ipipe_head_domain;
+		control = ipd->irqs[irq].control;
+		if ((control & IPIPE_HANDLE_MASK) == 0)
+			ipd = ipipe_root_domain;
+		if (ipd->irqs[irq].ackfn)
+			ipd->irqs[irq].ackfn(irq, desc);
+		if (desc && ipipe_chained_irq_p(irq)) {
+			if ((flags & IPIPE_IRQF_NOSYNC) == 0)
+				/* Run demuxed IRQ handlers. */
+				goto sync;
+			return;
+		}
+	}
+
+	/*
+	 * Sticky interrupts must be handled early and separately, so
+	 * that we always process them on the current domain.
+	 */
+	ipd = __ipipe_current_domain;
+	control = ipd->irqs[irq].control;
+	if (control & IPIPE_STICKY_MASK)
+		goto log;
+
+	/*
+	 * In case we have no registered head domain
+	 * (i.e. ipipe_head_domain == &ipipe_root), we allow
+	 * interrupts to go through the fast dispatcher, since we
+	 * don't care for additional latency induced by interrupt
+	 * disabling at CPU level. Otherwise, we must go through the
+	 * interrupt log, and leave the dispatching work ultimately to
+	 * __ipipe_sync_pipeline().
+	 */
+	ipd = ipipe_head_domain;
+	control = ipd->irqs[irq].control;
+	if (control & IPIPE_HANDLE_MASK) {
+		if (unlikely(flags & IPIPE_IRQF_NOSYNC))
+			__ipipe_set_irq_pending(ipd, irq);
+		else
+			__ipipe_dispatch_irq_fast(irq);
+		return;
+	}
+
+	/*
+	 * The root domain must handle all interrupts, so testing the
+	 * HANDLE bit for it would be pointless.
+	 */
+	ipd = ipipe_root_domain;
+log:
+	__ipipe_set_irq_pending(ipd, irq);
+
+	if (flags & IPIPE_IRQF_NOSYNC)
+		return;
+
+	/*
+	 * Optimize if we preempted a registered high priority head
+	 * domain: we don't need to synchronize the pipeline unless
+	 * there is a pending interrupt for it.
+	 */
+	if (!__ipipe_root_p &&
+	    !__ipipe_ipending_p(ipipe_this_cpu_head_context()))
+		return;
+sync:
+	__ipipe_sync_pipeline(ipipe_head_domain);
+}
+
+#ifdef CONFIG_TRACE_IRQFLAGS
+#define root_stall_after_handler()	local_irq_disable()
+#else
+#define root_stall_after_handler()	do { } while (0)
+#endif
+
+void __ipipe_dispatch_irq_fast(unsigned int irq) /* hw interrupts off */
+{
+	struct ipipe_percpu_domain_data *p = ipipe_this_cpu_leading_context(), *old;
+	struct ipipe_domain *head = p->domain;
+
+	if (unlikely(test_bit(IPIPE_STALL_FLAG, &p->status))) {
+		__ipipe_set_irq_pending(head, irq);
+		return;
+	}
+
+	old = __ipipe_current_context;
+	/* Switch to the head domain. */
+	__ipipe_set_current_context(p);
+
+	p->irqall[irq]++;
+	__set_bit(IPIPE_STALL_FLAG, &p->status);
+	barrier();
+
+	if (likely(head != ipipe_root_domain)) {
+		head->irqs[irq].handler(irq, head->irqs[irq].cookie);
+		__ipipe_run_irqtail(irq);
+	} else {
+		if (ipipe_virtual_irq_p(irq)) {
+			irq_enter();
+			head->irqs[irq].handler(irq, head->irqs[irq].cookie);
+			irq_exit();
+		} else
+			head->irqs[irq].handler(irq, head->irqs[irq].cookie);
+
+		root_stall_after_handler();
+	}
+
+	hard_local_irq_disable();
+	__clear_bit(IPIPE_STALL_FLAG, &p->status);
+
+	if (__ipipe_current_context == p) {
+		__ipipe_set_current_context(old);
+		if (old == p) {
+			if (__ipipe_ipending_p(p))
+				__ipipe_sync_stage();
+			return;
+		}
+	}
+
+	/*
+	 * We must be running over the root domain, synchronize
+	 * the pipeline for high priority IRQs.
+	 */
+	__ipipe_do_sync_pipeline(head);
+}
+
+#ifdef CONFIG_PREEMPT
+
+asmlinkage void preempt_schedule_irq(void);
+
+asmlinkage void __sched __ipipe_preempt_schedule_irq(void)
+{
+	struct ipipe_percpu_domain_data *p;
+	unsigned long flags;
+
+	BUG_ON(!hard_irqs_disabled());
+	local_irq_save(flags);
+	hard_local_irq_enable();
+	preempt_schedule_irq(); /* Ok, may reschedule now. */
+	hard_local_irq_disable();
+
+	/*
+	 * Flush any pending interrupt that may have been logged after
+	 * preempt_schedule_irq() stalled the root stage before
+	 * returning to us, and now.
+	 */
+	p = ipipe_this_cpu_root_context();
+	if (unlikely(__ipipe_ipending_p(p))) {
+		add_preempt_count(PREEMPT_ACTIVE);
+		trace_hardirqs_on();
+		__clear_bit(IPIPE_STALL_FLAG, &p->status);
+		__ipipe_sync_stage();
+		sub_preempt_count(PREEMPT_ACTIVE);
+	}
+
+	__ipipe_restore_root_nosync(flags);
+}
+
+#else /* !CONFIG_PREEMPT */
+
+#define __ipipe_preempt_schedule_irq()	do { } while (0)
+
+#endif	/* !CONFIG_PREEMPT */
+
+/*
+ * __ipipe_do_sync_stage() -- Flush the pending IRQs for the current
+ * domain (and processor). This routine flushes the interrupt log (see
+ * "Optimistic interrupt protection" from D. Stodolsky et al. for more
+ * on the deferred interrupt scheme). Every interrupt that occurred
+ * while the pipeline was stalled gets played.
+ *
+ * WARNING: CPU migration may occur over this routine.
+ */
+void __ipipe_do_sync_stage(void)
+{
+	struct ipipe_percpu_domain_data *p;
+	struct ipipe_domain *ipd;
+	int irq;
+
+	p = __ipipe_current_context;
+	ipd = p->domain;
+
+	__set_bit(IPIPE_STALL_FLAG, &p->status);
+	smp_wmb();
+
+	if (ipd == ipipe_root_domain)
+		trace_hardirqs_off();
+
+	for (;;) {
+		irq = __ipipe_next_irq(p);
+		if (irq < 0)
+			break;
+		/*
+		 * Make sure the compiler does not reorder wrongly, so
+		 * that all updates to maps are done before the
+		 * handler gets called.
+		 */
+		barrier();
+
+		if (test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
+			continue;
+
+		if (ipd != ipipe_head_domain)
+			hard_local_irq_enable();
+
+		if (likely(ipd != ipipe_root_domain)) {
+			ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie);
+			__ipipe_run_irqtail(irq);
+			hard_local_irq_disable();
+		} else if (ipipe_virtual_irq_p(irq)) {
+			irq_enter();
+			ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie);
+			irq_exit();
+			root_stall_after_handler();
+			hard_local_irq_disable();
+			while (__ipipe_check_root_resched())
+				__ipipe_preempt_schedule_irq();
+		} else {
+			ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie);
+			root_stall_after_handler();
+			hard_local_irq_disable();
+		}
+
+		p = __ipipe_current_context;
+	}
+
+	if (ipd == ipipe_root_domain)
+		trace_hardirqs_on();
+
+	__clear_bit(IPIPE_STALL_FLAG, &p->status);
+}
+
+#ifdef CONFIG_SMP
+
+/* Always called with hw interrupts off. */
+void __ipipe_do_critical_sync(unsigned int irq, void *cookie)
+{
+	int cpu = ipipe_processor_id();
+
+	cpu_set(cpu, __ipipe_cpu_sync_map);
+
+	/*
+	 * Now we are in sync with the lock requestor running on
+	 * another CPU. Enter a spinning wait until he releases the
+	 * global lock.
+	 */
+	spin_lock(&__ipipe_cpu_barrier);
+
+	/* Got it. Now get out. */
+
+	/* Call the sync routine if any. */
+	if (__ipipe_cpu_sync)
+		__ipipe_cpu_sync();
+
+	cpu_set(cpu, __ipipe_cpu_pass_map);
+
+	spin_unlock(&__ipipe_cpu_barrier);
+
+	cpu_clear(cpu, __ipipe_cpu_sync_map);
+}
+
+#endif	/* CONFIG_SMP */
+
+unsigned long ipipe_critical_enter(void (*syncfn)(void))
+{
+	int cpu __maybe_unused, n __maybe_unused;
+	unsigned long flags, loops __maybe_unused;
+	cpumask_t allbutself __maybe_unused;
+
+	flags = hard_local_irq_save();
+
+	if (num_online_cpus() == 1)
+		return flags;
+
+#ifdef CONFIG_SMP
+
+	cpu = ipipe_processor_id();
+	if (!cpu_test_and_set(cpu, __ipipe_cpu_lock_map)) {
+		while (test_and_set_bit(0, &__ipipe_critical_lock)) {
+			n = 0;
+			hard_local_irq_enable();
+
+			do
+				cpu_relax();
+			while (++n < cpu);
+
+			hard_local_irq_disable();
+		}
+restart:
+		spin_lock(&__ipipe_cpu_barrier);
+
+		__ipipe_cpu_sync = syncfn;
+
+		cpus_clear(__ipipe_cpu_pass_map);
+		cpu_set(cpu, __ipipe_cpu_pass_map);
+
+		/*
+		 * Send the sync IPI to all processors but the current
+		 * one.
+		 */
+		cpus_andnot(allbutself, cpu_online_map, __ipipe_cpu_pass_map);
+		ipipe_send_ipi(IPIPE_CRITICAL_IPI, allbutself);
+		loops = IPIPE_CRITICAL_TIMEOUT;
+
+		while (!cpus_equal(__ipipe_cpu_sync_map, allbutself)) {
+			if (--loops > 0) {
+				cpu_relax();
+				continue;
+			}
+			/*
+			 * We ran into a deadlock due to a contended
+			 * rwlock. Cancel this round and retry.
+			 */
+			__ipipe_cpu_sync = NULL;
+
+			spin_unlock(&__ipipe_cpu_barrier);
+			/*
+			 * Ensure all CPUs consumed the IPI to avoid
+			 * running __ipipe_cpu_sync prematurely. This
+			 * usually resolves the deadlock reason too.
+			 */
+			while (!cpus_equal(cpu_online_map, __ipipe_cpu_pass_map))
+				cpu_relax();
+
+			goto restart;
+		}
+	}
+
+	atomic_inc(&__ipipe_critical_count);
+
+#endif	/* CONFIG_SMP */
+
+	return flags;
+}
+EXPORT_SYMBOL_GPL(ipipe_critical_enter);
+
+void ipipe_critical_exit(unsigned long flags)
+{
+	if (num_online_cpus() == 1) {
+		hard_local_irq_restore(flags);
+		return;
+	}
+
+#ifdef CONFIG_SMP
+	if (atomic_dec_and_test(&__ipipe_critical_count)) {
+		spin_unlock(&__ipipe_cpu_barrier);
+		while (!cpus_empty(__ipipe_cpu_sync_map))
+			cpu_relax();
+		cpu_clear(ipipe_processor_id(), __ipipe_cpu_lock_map);
+		clear_bit(0, &__ipipe_critical_lock);
+		smp_mb__after_clear_bit();
+	}
+#endif /* CONFIG_SMP */
+
+	hard_local_irq_restore(flags);
+}
+EXPORT_SYMBOL_GPL(ipipe_critical_exit);
+
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+
+void ipipe_root_only(void)
+{
+	struct ipipe_domain *this_domain;
+	unsigned long flags;
+
+	flags = hard_smp_local_irq_save();
+
+	this_domain = __ipipe_current_domain;
+	if (likely(this_domain == ipipe_root_domain &&
+		   !test_bit(IPIPE_STALL_FLAG, &__ipipe_head_status))) {
+		hard_smp_local_irq_restore(flags);
+		return;
+	}
+
+	if (!__this_cpu_read(ipipe_percpu.context_check)) {
+		hard_smp_local_irq_restore(flags);
+		return;
+	}
+
+	hard_smp_local_irq_restore(flags);
+
+	ipipe_prepare_panic();
+	ipipe_trace_panic_freeze();
+
+	if (this_domain != ipipe_root_domain)
+		printk(KERN_ERR
+		       "I-pipe: Detected illicit call from head domain '%s'\n"
+		       KERN_ERR "        into a regular Linux service\n",
+		       this_domain->name);
+	else
+		printk(KERN_ERR "I-pipe: Detected stalled head domain, "
+				"probably caused by a bug.\n"
+				"        A critical section may have been "
+				"left unterminated.\n");
+	dump_stack();
+	ipipe_trace_panic_dump();
+}
+EXPORT_SYMBOL_GPL(ipipe_root_only);
+
+#endif /* CONFIG_IPIPE_DEBUG_CONTEXT */
+
+#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP)
+
+int notrace __ipipe_check_percpu_access(void)
+{
+	struct ipipe_percpu_domain_data *p;
+	struct ipipe_domain *this_domain;
+	unsigned long flags;
+	int ret = 0;
+
+	flags = hard_local_irq_save_notrace();
+
+	/*
+	 * Don't use __ipipe_current_domain here, this would recurse
+	 * indefinitely.
+	 */
+	this_domain = __this_cpu_read(ipipe_percpu.curr)->domain;
+
+	/*
+	 * Only the root domain may implement preemptive CPU migration
+	 * of tasks, so anything above in the pipeline should be fine.
+	 */
+	if (this_domain != ipipe_root_domain)
+		goto out;
+
+	if (raw_irqs_disabled_flags(flags))
+		goto out;
+
+	/*
+	 * Last chance: hw interrupts were enabled on entry while
+	 * running over the root domain, but the root stage might be
+	 * currently stalled, in which case preemption would be
+	 * disabled, and no migration could occur.
+	 */
+	if (this_domain == ipipe_root_domain) {
+		p = ipipe_this_cpu_root_context();
+		if (test_bit(IPIPE_STALL_FLAG, &p->status))
+			goto out;
+	}
+	/*
+	 * Our caller may end up accessing the wrong per-cpu variable
+	 * instance due to CPU migration; tell it to complain about
+	 * this.
+	 */
+	ret = 1;
+out:
+	hard_local_irq_restore_notrace(flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(__ipipe_check_percpu_access);
+
+void __ipipe_spin_unlock_debug(unsigned long flags)
+{
+	/*
+	 * We catch a nasty issue where spin_unlock_irqrestore() on a
+	 * regular kernel spinlock is about to re-enable hw interrupts
+	 * in a section entered with hw irqs off. This is clearly the
+	 * sign of a massive breakage coming. Usual suspect is a
+	 * regular spinlock which was overlooked, used within a
+	 * section which must run with hw irqs disabled.
+	 */
+	WARN_ON_ONCE(!raw_irqs_disabled_flags(flags) && hard_irqs_disabled());
+}
+EXPORT_SYMBOL_GPL(__ipipe_spin_unlock_debug);
+
+#endif /* CONFIG_IPIPE_DEBUG_INTERNAL && CONFIG_SMP */
+
+void ipipe_prepare_panic(void)
+{
+#ifdef CONFIG_PRINTK
+	__ipipe_printk_bypass = 1;
+#endif
+	ipipe_context_check_off();
+}
+EXPORT_SYMBOL_GPL(ipipe_prepare_panic);
+
+static void __ipipe_do_work(unsigned int virq, void *cookie)
+{
+	struct ipipe_work_header *work;
+	unsigned long flags;
+	void *curr, *tail;
+	int cpu;
+
+	/*
+	 * Work is dispatched in enqueuing order. This interrupt
+	 * context can't migrate to another CPU.
+	 */
+	cpu = smp_processor_id();
+	curr = per_cpu(work_buf, cpu);
+
+	for (;;) {
+		flags = hard_local_irq_save();
+		tail = per_cpu(work_tail, cpu);
+		if (curr == tail) {
+			per_cpu(work_tail, cpu) = per_cpu(work_buf, cpu);
+			hard_local_irq_restore(flags);
+			return;
+		}
+		work = curr;
+		curr += work->size;
+		hard_local_irq_restore(flags);
+		work->handler(work);
+	}
+}
+
+void __ipipe_post_work_root(struct ipipe_work_header *work)
+{
+	unsigned long flags;
+	void *tail;
+	int cpu;
+
+	/*
+	 * Subtle: we want to use the head stall/unstall operators,
+	 * not the hard_* routines to protect against races. This way,
+	 * we ensure that a root-based caller will trigger the virq
+	 * handling immediately when unstalling the head stage, as a
+	 * result of calling __ipipe_sync_pipeline() under the hood.
+	 */
+	flags = ipipe_test_and_stall_head();
+	cpu = ipipe_processor_id();
+	tail = per_cpu(work_tail, cpu);
+
+	if (WARN_ON_ONCE((unsigned char *)tail + work->size >=
+			 per_cpu(work_buf, cpu) + WORKBUF_SIZE))
+		goto out;
+
+	/* Work handling is deferred, so data has to be copied. */
+	memcpy(tail, work, work->size);
+	per_cpu(work_tail, cpu) = tail + work->size;
+	ipipe_post_irq_root(__ipipe_work_virq);
+out:
+	ipipe_restore_head(flags);
+}
+EXPORT_SYMBOL_GPL(__ipipe_post_work_root);
diff --git a/kernel/ipipe/timer.c b/kernel/ipipe/timer.c
new file mode 100644
index 0000000..a2b8672
--- /dev/null
+++ b/kernel/ipipe/timer.c
@@ -0,0 +1,383 @@
+/* -*- linux-c -*-
+ * linux/kernel/ipipe/timer.c
+ *
+ * Copyright (C) 2012 Gilles Chanteperdrix
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * I-pipe timer request interface.
+ */
+#include <linux/ipipe.h>
+#include <linux/percpu.h>
+#include <linux/irqdesc.h>
+#include <linux/cpumask.h>
+#include <linux/spinlock.h>
+#include <linux/ipipe_tickdev.h>
+#include <linux/interrupt.h>
+#include <linux/export.h>
+
+unsigned long __ipipe_hrtimer_freq;
+
+static LIST_HEAD(timers);
+static IPIPE_DEFINE_SPINLOCK(lock);
+
+static DEFINE_PER_CPU(struct ipipe_timer *, percpu_timer);
+
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+/*
+ * Default request method: switch to oneshot mode if supported.
+ */
+static void ipipe_timer_default_request(struct ipipe_timer *timer, int steal)
+{
+	struct clock_event_device *evtdev = timer->host_timer;
+
+	if (!(evtdev->features & CLOCK_EVT_FEAT_ONESHOT))
+		return;
+
+	if (evtdev->mode != CLOCK_EVT_MODE_ONESHOT) {
+		evtdev->set_mode(CLOCK_EVT_MODE_ONESHOT, evtdev);
+		evtdev->set_next_event(timer->freq / HZ, evtdev);
+	}
+}
+
+/*
+ * Default release method: return the timer to the mode it had when
+ * starting.
+ */
+static void ipipe_timer_default_release(struct ipipe_timer *timer)
+{
+	struct clock_event_device *evtdev = timer->host_timer;
+
+	evtdev->set_mode(evtdev->mode, evtdev);
+	if (evtdev->mode == CLOCK_EVT_MODE_ONESHOT)
+		evtdev->set_next_event(timer->freq / HZ, evtdev);
+}
+
+void ipipe_host_timer_register(struct clock_event_device *evtdev)
+{
+	struct ipipe_timer *timer = evtdev->ipipe_timer;
+
+	if (timer == NULL)
+		return;
+
+	if (timer->request == NULL)
+		timer->request = ipipe_timer_default_request;
+
+	/*
+	 * By default, use the same method as linux timer, on ARM at
+	 * least, most set_next_event methods are safe to be called
+	 * from Xenomai domain anyway.
+	 */
+	if (timer->set == NULL) {
+		timer->timer_set = evtdev;
+		timer->set = (typeof(timer->set))evtdev->set_next_event;
+	}
+
+	if (timer->release == NULL)
+		timer->release = ipipe_timer_default_release;
+
+	if (timer->name == NULL)
+		timer->name = evtdev->name;
+
+	if (timer->rating == 0)
+		timer->rating = evtdev->rating;
+
+	timer->freq = (1000000000ULL * evtdev->mult) >> evtdev->shift;
+
+	if (timer->min_delay_ticks == 0)
+		timer->min_delay_ticks =
+			(evtdev->min_delta_ns * evtdev->mult) >> evtdev->shift;
+
+	if (timer->cpumask == NULL)
+		timer->cpumask = evtdev->cpumask;
+
+	timer->host_timer = evtdev;
+
+	ipipe_timer_register(timer);
+}
+#endif /* CONFIG_GENERIC_CLOCKEVENTS */
+
+/*
+ * register a timer: maintain them in a list sorted by rating
+ */
+void ipipe_timer_register(struct ipipe_timer *timer)
+{
+	struct ipipe_timer *t;
+	unsigned long flags;
+
+	if (timer->timer_set == NULL)
+		timer->timer_set = timer;
+
+	if (timer->cpumask == NULL)
+		timer->cpumask = cpumask_of(smp_processor_id());
+
+	spin_lock_irqsave(&lock, flags);
+
+	list_for_each_entry(t, &timers, holder) {
+		if (t->rating <= timer->rating) {
+			__list_add(&timer->holder, t->holder.prev, &t->holder);
+			goto done;
+		}
+	}
+	list_add_tail(&timer->holder, &timers);
+  done:
+	spin_unlock_irqrestore(&lock, flags);
+}
+
+static void ipipe_timer_request_sync(void)
+{
+	struct ipipe_timer *timer = __ipipe_this_cpu_read(percpu_timer);
+	struct clock_event_device *evtdev;
+	int steal;
+
+	evtdev = timer->host_timer;
+
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+	steal = evtdev != NULL && evtdev->mode != CLOCK_EVT_MODE_UNUSED;
+#else /* !CONFIG_GENERIC_CLOCKEVENTS */
+	steal = 1;
+#endif /* !CONFIG_GENERIC_CLOCKEVENTS */
+
+	timer->request(timer, steal);
+}
+
+/*
+ * choose per-cpu timer: we walk the list, and find the timer with the
+ * highest rating.
+ */
+int ipipe_timers_request(void)
+{
+	struct clock_event_device *evtdev;
+	struct ipipe_timer *t;
+	unsigned long flags;
+	unsigned cpu;
+
+	spin_lock_irqsave(&lock, flags);
+	for_each_online_cpu(cpu) {
+		list_for_each_entry(t, &timers, holder) {
+			if (!cpumask_test_cpu(cpu, t->cpumask))
+				continue;
+
+			evtdev = t->host_timer;
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+			if (!evtdev
+			    || evtdev->mode != CLOCK_EVT_MODE_SHUTDOWN)
+#endif /* CONFIG_GENERIC_CLOCKEVENTS */
+				goto found;
+		}
+		printk("I-pipe: could not find timer for cpu #%d\n", cpu);
+		goto err_remove_all;
+
+	  found:
+		/* Sanity check: check that timers on all cpu have the
+		   same frequency, xenomai relies on that. */
+		if (__ipipe_hrtimer_freq == 0)
+			__ipipe_hrtimer_freq = t->freq;
+		else if (__ipipe_hrtimer_freq != t->freq) {
+			printk(KERN_WARNING
+			       "I-pipe: timer on cpu #%d has wrong frequency\n",
+			       cpu);
+			goto err_remove_all;
+		}
+		per_cpu(ipipe_percpu.hrtimer_irq, cpu) = t->irq;
+		per_cpu(percpu_timer, cpu) = t;
+	}
+	spin_unlock_irqrestore(&lock, flags);
+
+	flags = ipipe_critical_enter(ipipe_timer_request_sync);
+	ipipe_timer_request_sync();
+	ipipe_critical_exit(flags);
+
+	return 0;
+
+err_remove_all:
+	spin_unlock_irqrestore(&lock, flags);
+
+	for_each_online_cpu(cpu) {
+		per_cpu(ipipe_percpu.hrtimer_irq, cpu) = -1;
+		per_cpu(percpu_timer, cpu) = NULL;
+	}
+	__ipipe_hrtimer_freq = 0;
+
+	return -ENODEV;
+}
+
+static void ipipe_timer_release_sync(void)
+{
+	struct ipipe_timer *timer = __ipipe_this_cpu_read(percpu_timer);
+
+	timer->release(timer);
+}
+
+void ipipe_timers_release(void)
+{
+	unsigned long flags;
+	unsigned cpu;
+
+	flags = ipipe_critical_enter(ipipe_timer_release_sync);
+	ipipe_timer_release_sync();
+	ipipe_critical_exit(flags);
+
+	for_each_online_cpu(cpu) {
+		per_cpu(ipipe_percpu.hrtimer_irq, cpu) = -1;
+		per_cpu(percpu_timer, cpu) = NULL;
+		__ipipe_hrtimer_freq = 0;
+	}
+}
+
+static void __ipipe_ack_hrtimer_irq(unsigned int irq, struct irq_desc *desc)
+{
+	struct ipipe_timer *timer = __ipipe_this_cpu_read(percpu_timer);
+
+	if (desc)
+		desc->ipipe_ack(irq, desc);
+	if (timer->ack)
+		timer->ack();
+	if (desc)
+		desc->ipipe_end(irq, desc);
+}
+
+int ipipe_timer_start(void (*tick_handler)(void),
+		      void (*emumode)(enum clock_event_mode mode,
+				      struct clock_event_device *cdev),
+		      int (*emutick)(unsigned long evt,
+				     struct clock_event_device *cdev),
+		      unsigned cpu)
+{
+	struct clock_event_device *evtdev;
+	struct ipipe_timer *timer;
+	unsigned long flags;
+	int steal, rc;
+
+	timer = per_cpu(percpu_timer, cpu);
+	evtdev = timer->host_timer;
+
+	flags = ipipe_critical_enter(NULL);
+
+	if (cpu == 0 || timer->irq != per_cpu(ipipe_percpu.hrtimer_irq, 0)) {
+		rc = ipipe_request_irq(ipipe_head_domain, timer->irq,
+				       (ipipe_irq_handler_t)tick_handler,
+				       NULL, __ipipe_ack_hrtimer_irq);
+		if (rc < 0)
+			goto done;
+	}
+
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+	steal = evtdev != NULL && evtdev->mode != CLOCK_EVT_MODE_UNUSED;
+
+	if (steal && evtdev->ipipe_stolen == 0) {
+		timer->real_mult = evtdev->mult;
+		timer->real_shift = evtdev->shift;
+		timer->real_set_mode = evtdev->set_mode;
+		timer->real_set_next_event = evtdev->set_next_event;
+		evtdev->mult = 1;
+		evtdev->shift = 0;
+		evtdev->set_mode = emumode;
+		evtdev->set_next_event = emutick;
+		evtdev->ipipe_stolen = 1;
+	}
+
+	rc = evtdev ? evtdev->mode : CLOCK_EVT_MODE_UNUSED;
+#else /* CONFIG_GENERIC_CLOCKEVENTS */
+	steal = 1;
+	rc = 0;
+#endif /* CONFIG_GENERIC_CLOCKEVENTS */
+
+  done:
+	ipipe_critical_exit(flags);
+
+	return rc;
+}
+
+void ipipe_timer_stop(unsigned cpu)
+{
+	struct clock_event_device *evtdev;
+	struct ipipe_timer *timer;
+	unsigned long flags;
+
+	timer = per_cpu(percpu_timer, cpu);
+	evtdev = timer->host_timer;
+
+	flags = ipipe_critical_enter(NULL);
+
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+	if (evtdev && evtdev->ipipe_stolen) {
+		evtdev->mult = timer->real_mult;
+		evtdev->shift = timer->real_shift;
+		evtdev->set_mode = timer->real_set_mode;
+		timer->real_mult = timer->real_shift = 0;
+		timer->real_set_mode = NULL;
+		timer->real_set_next_event = NULL;
+		evtdev->ipipe_stolen = 0;
+	}
+#endif /* CONFIG_GENERIC_CLOCKEVENTS */
+
+	ipipe_free_irq(ipipe_head_domain, timer->irq);
+
+	ipipe_critical_exit(flags);
+}
+
+void ipipe_timer_set(unsigned long delay)
+{
+	struct ipipe_timer *timer;
+
+	timer = __ipipe_this_cpu_read(percpu_timer);
+
+	if (delay < timer->min_delay_ticks
+	    || timer->set(delay, timer->timer_set) < 0)
+		ipipe_raise_irq(timer->irq);
+}
+EXPORT_SYMBOL_GPL(ipipe_timer_set);
+
+const char *ipipe_timer_name(void)
+{
+	return per_cpu(percpu_timer, 0)->name;
+}
+EXPORT_SYMBOL_GPL(ipipe_timer_name);
+
+unsigned ipipe_timer_ns2ticks(struct ipipe_timer *timer, unsigned ns)
+{
+	unsigned long long tmp;
+	BUG_ON(!timer->freq);
+	tmp = (unsigned long long)ns * timer->freq;
+	do_div(tmp, 1000000000);
+	return tmp;
+}
+
+#ifdef CONFIG_IPIPE_HAVE_HOSTRT
+/*
+ * NOTE: The architecture specific code must only call this function
+ * when a clocksource suitable for CLOCK_HOST_REALTIME is enabled.
+ * The event receiver is responsible for providing proper locking.
+ */
+void ipipe_update_hostrt(struct timespec *wall_time, struct timespec *wtm,
+			 struct clocksource *clock, u32 mult)
+{
+	struct ipipe_hostrt_data data;
+
+	ipipe_root_only();
+	data.live = 1;
+	data.cycle_last = clock->cycle_last;
+	data.mask = clock->mask;
+	data.mult = mult;
+	data.shift = clock->shift;
+	data.wall_time_sec = wall_time->tv_sec;
+	data.wall_time_nsec = wall_time->tv_nsec;
+	data.wall_to_monotonic = *wtm;
+	__ipipe_notify_kevent(IPIPE_KEVT_HOSTRT, &data);
+}
+
+#endif /* CONFIG_IPIPE_HAVE_HOSTRT */
diff --git a/kernel/ipipe/tracer.c b/kernel/ipipe/tracer.c
new file mode 100644
index 0000000..8820ac6
--- /dev/null
+++ b/kernel/ipipe/tracer.c
@@ -0,0 +1,1432 @@
+/* -*- linux-c -*-
+ * kernel/ipipe/tracer.c
+ *
+ * Copyright (C) 2005 Luotao Fu.
+ *               2005-2008 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/kallsyms.h>
+#include <linux/seq_file.h>
+#include <linux/proc_fs.h>
+#include <linux/ctype.h>
+#include <linux/vmalloc.h>
+#include <linux/pid.h>
+#include <linux/vermagic.h>
+#include <linux/sched.h>
+#include <linux/ipipe.h>
+#include <linux/ftrace.h>
+#include <asm/uaccess.h>
+
+#define IPIPE_TRACE_PATHS           4 /* <!> Do not lower below 3 */
+#define IPIPE_DEFAULT_ACTIVE        0
+#define IPIPE_DEFAULT_MAX           1
+#define IPIPE_DEFAULT_FROZEN        2
+
+#define IPIPE_TRACE_POINTS          (1 << CONFIG_IPIPE_TRACE_SHIFT)
+#define WRAP_POINT_NO(point)        ((point) & (IPIPE_TRACE_POINTS-1))
+
+#define IPIPE_DEFAULT_PRE_TRACE     10
+#define IPIPE_DEFAULT_POST_TRACE    10
+#define IPIPE_DEFAULT_BACK_TRACE    100
+
+#define IPIPE_DELAY_NOTE            1000  /* in nanoseconds */
+#define IPIPE_DELAY_WARN            10000 /* in nanoseconds */
+
+#define IPIPE_TFLG_NMI_LOCK         0x0001
+#define IPIPE_TFLG_NMI_HIT          0x0002
+#define IPIPE_TFLG_NMI_FREEZE_REQ   0x0004
+
+#define IPIPE_TFLG_HWIRQ_OFF        0x0100
+#define IPIPE_TFLG_FREEZING         0x0200
+#define IPIPE_TFLG_CURRDOM_SHIFT    10   /* bits 10..11: current domain */
+#define IPIPE_TFLG_CURRDOM_MASK     0x0C00
+#define IPIPE_TFLG_DOMSTATE_SHIFT   12   /* bits 12..15: domain stalled? */
+#define IPIPE_TFLG_DOMSTATE_BITS    3
+
+#define IPIPE_TFLG_DOMAIN_STALLED(point, n) \
+	(point->flags & (1 << (n + IPIPE_TFLG_DOMSTATE_SHIFT)))
+#define IPIPE_TFLG_CURRENT_DOMAIN(point) \
+	((point->flags & IPIPE_TFLG_CURRDOM_MASK) >> IPIPE_TFLG_CURRDOM_SHIFT)
+
+struct ipipe_trace_point {
+	short type;
+	short flags;
+	unsigned long eip;
+	unsigned long parent_eip;
+	unsigned long v;
+	unsigned long long timestamp;
+};
+
+struct ipipe_trace_path {
+	volatile int flags;
+	int dump_lock; /* separated from flags due to cross-cpu access */
+	int trace_pos; /* next point to fill */
+	int begin, end; /* finalised path begin and end */
+	int post_trace; /* non-zero when in post-trace phase */
+	unsigned long long length; /* max path length in cycles */
+	unsigned long nmi_saved_eip; /* for deferred requests from NMIs */
+	unsigned long nmi_saved_parent_eip;
+	unsigned long nmi_saved_v;
+	struct ipipe_trace_point point[IPIPE_TRACE_POINTS];
+} ____cacheline_aligned_in_smp;
+
+enum ipipe_trace_type
+{
+	IPIPE_TRACE_FUNC = 0,
+	IPIPE_TRACE_BEGIN,
+	IPIPE_TRACE_END,
+	IPIPE_TRACE_FREEZE,
+	IPIPE_TRACE_SPECIAL,
+	IPIPE_TRACE_PID,
+	IPIPE_TRACE_EVENT,
+};
+
+#define IPIPE_TYPE_MASK             0x0007
+#define IPIPE_TYPE_BITS             3
+
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
+static DEFINE_PER_CPU(struct ipipe_trace_path *, trace_path);
+#else /* !CONFIG_IPIPE_TRACE_VMALLOC */
+static DEFINE_PER_CPU(struct ipipe_trace_path, trace_path[IPIPE_TRACE_PATHS]) =
+	{ [0 ... IPIPE_TRACE_PATHS-1] = { .begin = -1, .end = -1 } };
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
+
+int ipipe_trace_enable = 0;
+
+static DEFINE_PER_CPU(int, active_path) = { IPIPE_DEFAULT_ACTIVE };
+static DEFINE_PER_CPU(int, max_path) = { IPIPE_DEFAULT_MAX };
+static DEFINE_PER_CPU(int, frozen_path) = { IPIPE_DEFAULT_FROZEN };
+static IPIPE_DEFINE_SPINLOCK(global_path_lock);
+static int pre_trace = IPIPE_DEFAULT_PRE_TRACE;
+static int post_trace = IPIPE_DEFAULT_POST_TRACE;
+static int back_trace = IPIPE_DEFAULT_BACK_TRACE;
+static int verbose_trace = 1;
+static unsigned long trace_overhead;
+
+static unsigned long trigger_begin;
+static unsigned long trigger_end;
+
+static DEFINE_MUTEX(out_mutex);
+static struct ipipe_trace_path *print_path;
+#ifdef CONFIG_IPIPE_TRACE_PANIC
+static struct ipipe_trace_path *panic_path;
+#endif /* CONFIG_IPIPE_TRACE_PANIC */
+static int print_pre_trace;
+static int print_post_trace;
+
+
+static long __ipipe_signed_tsc2us(long long tsc);
+static void
+__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point);
+static void __ipipe_print_symname(struct seq_file *m, unsigned long eip);
+
+static inline void store_states(struct ipipe_domain *ipd,
+				struct ipipe_trace_point *point, int pos)
+{
+	if (test_bit(IPIPE_STALL_FLAG, &ipipe_this_cpu_context(ipd)->status))
+		point->flags |= 1 << (pos + IPIPE_TFLG_DOMSTATE_SHIFT);
+
+	if (ipd == __ipipe_current_domain)
+		point->flags |= pos << IPIPE_TFLG_CURRDOM_SHIFT;
+}
+
+static notrace void
+__ipipe_store_domain_states(struct ipipe_trace_point *point)
+{
+	store_states(ipipe_root_domain, point, 0);
+	if (ipipe_head_domain != ipipe_root_domain)
+		store_states(ipipe_head_domain, point, 1);
+}
+
+static notrace int __ipipe_get_free_trace_path(int old, int cpu)
+{
+	int new_active = old;
+	struct ipipe_trace_path *tp;
+
+	do {
+		if (++new_active == IPIPE_TRACE_PATHS)
+			new_active = 0;
+		tp = &per_cpu(trace_path, cpu)[new_active];
+	} while (new_active == per_cpu(max_path, cpu) ||
+	         new_active == per_cpu(frozen_path, cpu) ||
+	         tp->dump_lock);
+
+	return new_active;
+}
+
+static notrace void
+__ipipe_migrate_pre_trace(struct ipipe_trace_path *new_tp,
+                          struct ipipe_trace_path *old_tp, int old_pos)
+{
+	int i;
+
+	new_tp->trace_pos = pre_trace+1;
+
+	for (i = new_tp->trace_pos; i > 0; i--)
+		memcpy(&new_tp->point[WRAP_POINT_NO(new_tp->trace_pos-i)],
+		       &old_tp->point[WRAP_POINT_NO(old_pos-i)],
+		       sizeof(struct ipipe_trace_point));
+
+	/* mark the end (i.e. the point before point[0]) invalid */
+	new_tp->point[IPIPE_TRACE_POINTS-1].eip = 0;
+}
+
+static notrace struct ipipe_trace_path *
+__ipipe_trace_end(int cpu, struct ipipe_trace_path *tp, int pos)
+{
+	struct ipipe_trace_path *old_tp = tp;
+	long active = per_cpu(active_path, cpu);
+	unsigned long long length;
+
+	/* do we have a new worst case? */
+	length = tp->point[tp->end].timestamp -
+	         tp->point[tp->begin].timestamp;
+	if (length > per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)].length) {
+		/* we need protection here against other cpus trying
+		   to start a proc dump */
+		spin_lock(&global_path_lock);
+
+		/* active path holds new worst case */
+		tp->length = length;
+		per_cpu(max_path, cpu) = active;
+
+		/* find next unused trace path */
+		active = __ipipe_get_free_trace_path(active, cpu);
+
+		spin_unlock(&global_path_lock);
+
+		tp = &per_cpu(trace_path, cpu)[active];
+
+		/* migrate last entries for pre-tracing */
+		__ipipe_migrate_pre_trace(tp, old_tp, pos);
+	}
+
+	return tp;
+}
+
+static notrace struct ipipe_trace_path *
+__ipipe_trace_freeze(int cpu, struct ipipe_trace_path *tp, int pos)
+{
+	struct ipipe_trace_path *old_tp = tp;
+	long active = per_cpu(active_path, cpu);
+	int n;
+
+	/* frozen paths have no core (begin=end) */
+	tp->begin = tp->end;
+
+	/* we need protection here against other cpus trying
+	 * to set their frozen path or to start a proc dump */
+	spin_lock(&global_path_lock);
+
+	per_cpu(frozen_path, cpu) = active;
+
+	/* find next unused trace path */
+	active = __ipipe_get_free_trace_path(active, cpu);
+
+	/* check if this is the first frozen path */
+	for_each_possible_cpu(n) {
+		if (n != cpu &&
+		    per_cpu(trace_path, n)[per_cpu(frozen_path, n)].end >= 0)
+			tp->end = -1;
+	}
+
+	spin_unlock(&global_path_lock);
+
+	tp = &per_cpu(trace_path, cpu)[active];
+
+	/* migrate last entries for pre-tracing */
+	__ipipe_migrate_pre_trace(tp, old_tp, pos);
+
+	return tp;
+}
+
+void notrace
+__ipipe_trace(enum ipipe_trace_type type, unsigned long eip,
+              unsigned long parent_eip, unsigned long v)
+{
+	struct ipipe_trace_path *tp, *old_tp;
+	int pos, next_pos, begin;
+	struct ipipe_trace_point *point;
+	unsigned long flags;
+	int cpu;
+
+	flags = hard_local_irq_save_notrace();
+
+	cpu = ipipe_processor_id();
+ restart:
+	tp = old_tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
+
+	/* here starts a race window with NMIs - catched below */
+
+	/* check for NMI recursion */
+	if (unlikely(tp->flags & IPIPE_TFLG_NMI_LOCK)) {
+		tp->flags |= IPIPE_TFLG_NMI_HIT;
+
+		/* first freeze request from NMI context? */
+		if ((type == IPIPE_TRACE_FREEZE) &&
+		    !(tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)) {
+			/* save arguments and mark deferred freezing */
+			tp->flags |= IPIPE_TFLG_NMI_FREEZE_REQ;
+			tp->nmi_saved_eip = eip;
+			tp->nmi_saved_parent_eip = parent_eip;
+			tp->nmi_saved_v = v;
+		}
+		return; /* no need for restoring flags inside IRQ */
+	}
+
+	/* clear NMI events and set lock (atomically per cpu) */
+	tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT |
+	                           IPIPE_TFLG_NMI_FREEZE_REQ))
+	                       | IPIPE_TFLG_NMI_LOCK;
+
+	/* check active_path again - some nasty NMI may have switched
+	 * it meanwhile */
+	if (unlikely(tp !=
+		     &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)])) {
+		/* release lock on wrong path and restart */
+		tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+		/* there is no chance that the NMI got deferred
+		 * => no need to check for pending freeze requests */
+		goto restart;
+	}
+
+	/* get the point buffer */
+	pos = tp->trace_pos;
+	point = &tp->point[pos];
+
+	/* store all trace point data */
+	point->type = type;
+	point->flags = raw_irqs_disabled_flags(flags) ? IPIPE_TFLG_HWIRQ_OFF : 0;
+	point->eip = eip;
+	point->parent_eip = parent_eip;
+	point->v = v;
+	ipipe_read_tsc(point->timestamp);
+
+	__ipipe_store_domain_states(point);
+
+	/* forward to next point buffer */
+	next_pos = WRAP_POINT_NO(pos+1);
+	tp->trace_pos = next_pos;
+
+	/* only mark beginning if we haven't started yet */
+	begin = tp->begin;
+	if (unlikely(type == IPIPE_TRACE_BEGIN) && (begin < 0))
+		tp->begin = pos;
+
+	/* end of critical path, start post-trace if not already started */
+	if (unlikely(type == IPIPE_TRACE_END) &&
+	    (begin >= 0) && !tp->post_trace)
+		tp->post_trace = post_trace + 1;
+
+	/* freeze only if the slot is free and we are not already freezing */
+	if ((unlikely(type == IPIPE_TRACE_FREEZE) ||
+	     (unlikely(eip >= trigger_begin && eip <= trigger_end) &&
+	     type == IPIPE_TRACE_FUNC)) &&
+	    per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)].begin < 0 &&
+	    !(tp->flags & IPIPE_TFLG_FREEZING)) {
+		tp->post_trace = post_trace + 1;
+		tp->flags |= IPIPE_TFLG_FREEZING;
+	}
+
+	/* enforce end of trace in case of overflow */
+	if (unlikely(WRAP_POINT_NO(next_pos + 1) == begin)) {
+		tp->end = pos;
+		goto enforce_end;
+	}
+
+	/* stop tracing this path if we are in post-trace and
+	 *  a) that phase is over now or
+	 *  b) a new TRACE_BEGIN came in but we are not freezing this path */
+	if (unlikely((tp->post_trace > 0) && ((--tp->post_trace == 0) ||
+	             ((type == IPIPE_TRACE_BEGIN) &&
+	              !(tp->flags & IPIPE_TFLG_FREEZING))))) {
+		/* store the path's end (i.e. excluding post-trace) */
+		tp->end = WRAP_POINT_NO(pos - post_trace + tp->post_trace);
+
+ enforce_end:
+		if (tp->flags & IPIPE_TFLG_FREEZING)
+			tp = __ipipe_trace_freeze(cpu, tp, pos);
+		else
+			tp = __ipipe_trace_end(cpu, tp, pos);
+
+		/* reset the active path, maybe already start a new one */
+		tp->begin = (type == IPIPE_TRACE_BEGIN) ?
+			WRAP_POINT_NO(tp->trace_pos - 1) : -1;
+		tp->end = -1;
+		tp->post_trace = 0;
+		tp->flags = 0;
+
+		/* update active_path not earlier to avoid races with NMIs */
+		per_cpu(active_path, cpu) = tp - per_cpu(trace_path, cpu);
+	}
+
+	/* we still have old_tp and point,
+	 * let's reset NMI lock and check for catches */
+	old_tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+	if (unlikely(old_tp->flags & IPIPE_TFLG_NMI_HIT)) {
+		/* well, this late tagging may not immediately be visible for
+		 * other cpus already dumping this path - a minor issue */
+		point->flags |= IPIPE_TFLG_NMI_HIT;
+
+		/* handle deferred freezing from NMI context */
+		if (old_tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)
+			__ipipe_trace(IPIPE_TRACE_FREEZE, old_tp->nmi_saved_eip,
+			              old_tp->nmi_saved_parent_eip,
+			              old_tp->nmi_saved_v);
+	}
+
+	hard_local_irq_restore_notrace(flags);
+}
+
+static unsigned long __ipipe_global_path_lock(void)
+{
+	unsigned long flags;
+	int cpu;
+	struct ipipe_trace_path *tp;
+
+	spin_lock_irqsave(&global_path_lock, flags);
+
+	cpu = ipipe_processor_id();
+ restart:
+	tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
+
+	/* here is small race window with NMIs - catched below */
+
+	/* clear NMI events and set lock (atomically per cpu) */
+	tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT |
+	                           IPIPE_TFLG_NMI_FREEZE_REQ))
+	                       | IPIPE_TFLG_NMI_LOCK;
+
+	/* check active_path again - some nasty NMI may have switched
+	 * it meanwhile */
+	if (tp != &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)]) {
+		/* release lock on wrong path and restart */
+		tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+		/* there is no chance that the NMI got deferred
+		 * => no need to check for pending freeze requests */
+		goto restart;
+	}
+
+	return flags;
+}
+
+static void __ipipe_global_path_unlock(unsigned long flags)
+{
+	int cpu;
+	struct ipipe_trace_path *tp;
+
+	/* release spinlock first - it's not involved in the NMI issue */
+	__ipipe_spin_unlock_irqbegin(&global_path_lock);
+
+	cpu = ipipe_processor_id();
+	tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
+
+	tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+	/* handle deferred freezing from NMI context */
+	if (tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)
+		__ipipe_trace(IPIPE_TRACE_FREEZE, tp->nmi_saved_eip,
+		              tp->nmi_saved_parent_eip, tp->nmi_saved_v);
+
+	/* See __ipipe_spin_lock_irqsave() and friends. */
+	__ipipe_spin_unlock_irqcomplete(flags);
+}
+
+void notrace ipipe_trace_begin(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_BEGIN, __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_begin);
+
+void notrace ipipe_trace_end(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_END, __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_end);
+
+void notrace ipipe_trace_freeze(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_FREEZE, __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_freeze);
+
+void notrace ipipe_trace_special(unsigned char id, unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_SPECIAL | (id << IPIPE_TYPE_BITS),
+	              __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_special);
+
+void notrace ipipe_trace_pid(pid_t pid, short prio)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_PID | (prio << IPIPE_TYPE_BITS),
+	              __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, pid);
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_pid);
+
+void notrace ipipe_trace_event(unsigned char id, unsigned long delay_tsc)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_EVENT | (id << IPIPE_TYPE_BITS),
+	              __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, delay_tsc);
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_event);
+
+int ipipe_trace_max_reset(void)
+{
+	int cpu;
+	unsigned long flags;
+	struct ipipe_trace_path *path;
+	int ret = 0;
+
+	flags = __ipipe_global_path_lock();
+
+	for_each_possible_cpu(cpu) {
+		path = &per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)];
+
+		if (path->dump_lock) {
+			ret = -EBUSY;
+			break;
+		}
+
+		path->begin     = -1;
+		path->end       = -1;
+		path->trace_pos = 0;
+		path->length    = 0;
+	}
+
+	__ipipe_global_path_unlock(flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_max_reset);
+
+int ipipe_trace_frozen_reset(void)
+{
+	int cpu;
+	unsigned long flags;
+	struct ipipe_trace_path *path;
+	int ret = 0;
+
+	flags = __ipipe_global_path_lock();
+
+	for_each_online_cpu(cpu) {
+		path = &per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)];
+
+		if (path->dump_lock) {
+			ret = -EBUSY;
+			break;
+		}
+
+		path->begin = -1;
+		path->end = -1;
+		path->trace_pos = 0;
+		path->length    = 0;
+	}
+
+	__ipipe_global_path_unlock(flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_frozen_reset);
+
+static void
+__ipipe_get_task_info(char *task_info, struct ipipe_trace_point *point,
+                      int trylock)
+{
+	struct task_struct *task = NULL;
+	char buf[8];
+	int i;
+	int locked = 1;
+
+	if (trylock) {
+		if (!read_trylock(&tasklist_lock))
+			locked = 0;
+	} else
+		read_lock(&tasklist_lock);
+
+	if (locked)
+		task = find_task_by_pid_ns((pid_t)point->v, &init_pid_ns);
+
+	if (task)
+		strncpy(task_info, task->comm, 11);
+	else
+		strcpy(task_info, "-<?>-");
+
+	if (locked)
+		read_unlock(&tasklist_lock);
+
+	for (i = strlen(task_info); i < 11; i++)
+		task_info[i] = ' ';
+
+	sprintf(buf, " %d ", point->type >> IPIPE_TYPE_BITS);
+	strcpy(task_info + (11 - strlen(buf)), buf);
+}
+
+static void
+__ipipe_get_event_date(char *buf,struct ipipe_trace_path *path,
+		       struct ipipe_trace_point *point)
+{
+	long time;
+	int type;
+
+	time = __ipipe_signed_tsc2us(point->timestamp -
+				     path->point[path->begin].timestamp + point->v);
+	type = point->type >> IPIPE_TYPE_BITS;
+
+	if (type == 0)
+		/*
+		 * Event type #0 is predefined, stands for the next
+		 * timer tick.
+		 */
+		sprintf(buf, "tick@%-6ld", time);
+	else
+		sprintf(buf, "%3d@%-7ld", type, time);
+}
+
+#ifdef CONFIG_IPIPE_TRACE_PANIC
+
+void ipipe_trace_panic_freeze(void)
+{
+	unsigned long flags;
+	int cpu;
+
+	if (!ipipe_trace_enable)
+		return;
+
+	ipipe_trace_enable = 0;
+	flags = hard_local_irq_save_notrace();
+
+	cpu = ipipe_processor_id();
+
+	panic_path = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
+
+	hard_local_irq_restore(flags);
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_panic_freeze);
+
+void ipipe_trace_panic_dump(void)
+{
+	int cnt = back_trace;
+	int start, pos;
+	char buf[16];
+
+	if (!panic_path)
+		return;
+
+	ipipe_context_check_off();
+
+	printk("I-pipe tracer log (%d points):\n", cnt);
+
+	start = pos = WRAP_POINT_NO(panic_path->trace_pos-1);
+
+	while (cnt-- > 0) {
+		struct ipipe_trace_point *point = &panic_path->point[pos];
+		long time;
+		char info[16];
+		int i;
+
+		printk(" %c",
+		       (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' ');
+
+		for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--)
+			printk("%c",
+			       (IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ?
+				(IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
+					'#' : '+') :
+				(IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
+					'*' : ' '));
+
+		if (!point->eip)
+			printk("-<invalid>-\n");
+		else {
+			__ipipe_trace_point_type(buf, point);
+			printk("%s", buf);
+
+			switch (point->type & IPIPE_TYPE_MASK) {
+				case IPIPE_TRACE_FUNC:
+					printk("           ");
+					break;
+
+				case IPIPE_TRACE_PID:
+					__ipipe_get_task_info(info,
+							      point, 1);
+					printk("%s", info);
+					break;
+
+				case IPIPE_TRACE_EVENT:
+					__ipipe_get_event_date(info,
+							       panic_path, point);
+					printk("%s", info);
+					break;
+
+				default:
+					printk("0x%08lx ", point->v);
+			}
+
+			time = __ipipe_signed_tsc2us(point->timestamp -
+				panic_path->point[start].timestamp);
+			printk(" %5ld ", time);
+
+			__ipipe_print_symname(NULL, point->eip);
+			printk(" (");
+			__ipipe_print_symname(NULL, point->parent_eip);
+			printk(")\n");
+		}
+		pos = WRAP_POINT_NO(pos - 1);
+	}
+
+	panic_path = NULL;
+}
+EXPORT_SYMBOL_GPL(ipipe_trace_panic_dump);
+
+#endif /* CONFIG_IPIPE_TRACE_PANIC */
+
+
+/* --- /proc output --- */
+
+static notrace int __ipipe_in_critical_trpath(long point_no)
+{
+	return ((WRAP_POINT_NO(point_no-print_path->begin) <
+	         WRAP_POINT_NO(print_path->end-print_path->begin)) ||
+	        ((print_path->end == print_path->begin) &&
+	         (WRAP_POINT_NO(point_no-print_path->end) >
+	          print_post_trace)));
+}
+
+static long __ipipe_signed_tsc2us(long long tsc)
+{
+        unsigned long long abs_tsc;
+        long us;
+
+	/* ipipe_tsc2us works on unsigned => handle sign separately */
+        abs_tsc = (tsc >= 0) ? tsc : -tsc;
+        us = ipipe_tsc2us(abs_tsc);
+        if (tsc < 0)
+                return -us;
+        else
+                return us;
+}
+
+static void
+__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point)
+{
+	switch (point->type & IPIPE_TYPE_MASK) {
+		case IPIPE_TRACE_FUNC:
+			strcpy(buf, "func    ");
+			break;
+
+		case IPIPE_TRACE_BEGIN:
+			strcpy(buf, "begin   ");
+			break;
+
+		case IPIPE_TRACE_END:
+			strcpy(buf, "end     ");
+			break;
+
+		case IPIPE_TRACE_FREEZE:
+			strcpy(buf, "freeze  ");
+			break;
+
+		case IPIPE_TRACE_SPECIAL:
+			sprintf(buf, "(0x%02x)  ",
+				point->type >> IPIPE_TYPE_BITS);
+			break;
+
+		case IPIPE_TRACE_PID:
+			sprintf(buf, "[%5d] ", (pid_t)point->v);
+			break;
+
+		case IPIPE_TRACE_EVENT:
+			sprintf(buf, "event   ");
+			break;
+	}
+}
+
+static void
+__ipipe_print_pathmark(struct seq_file *m, struct ipipe_trace_point *point)
+{
+	char mark = ' ';
+	int point_no = point - print_path->point;
+	int i;
+
+	if (print_path->end == point_no)
+		mark = '<';
+	else if (print_path->begin == point_no)
+		mark = '>';
+	else if (__ipipe_in_critical_trpath(point_no))
+		mark = ':';
+	seq_printf(m, "%c%c", mark,
+	           (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' ');
+
+	if (!verbose_trace)
+		return;
+
+	for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--)
+		seq_printf(m, "%c",
+			(IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ?
+			    (IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
+				'#' : '+') :
+			(IPIPE_TFLG_DOMAIN_STALLED(point, i) ? '*' : ' '));
+}
+
+static void
+__ipipe_print_delay(struct seq_file *m, struct ipipe_trace_point *point)
+{
+	unsigned long delay = 0;
+	int next;
+	char *mark = "  ";
+
+	next = WRAP_POINT_NO(point+1 - print_path->point);
+
+	if (next != print_path->trace_pos)
+		delay = ipipe_tsc2ns(print_path->point[next].timestamp -
+		                     point->timestamp);
+
+	if (__ipipe_in_critical_trpath(point - print_path->point)) {
+		if (delay > IPIPE_DELAY_WARN)
+			mark = "! ";
+		else if (delay > IPIPE_DELAY_NOTE)
+			mark = "+ ";
+	}
+	seq_puts(m, mark);
+
+	if (verbose_trace)
+		seq_printf(m, "%3lu.%03lu%c ", delay/1000, delay%1000,
+		           (point->flags & IPIPE_TFLG_NMI_HIT) ? 'N' : ' ');
+	else
+		seq_puts(m, " ");
+}
+
+static void __ipipe_print_symname(struct seq_file *m, unsigned long eip)
+{
+	char namebuf[KSYM_NAME_LEN+1];
+	unsigned long size, offset;
+	const char *sym_name;
+	char *modname;
+
+	sym_name = kallsyms_lookup(eip, &size, &offset, &modname, namebuf);
+
+#ifdef CONFIG_IPIPE_TRACE_PANIC
+	if (!m) {
+		/* panic dump */
+		if (sym_name) {
+			printk("%s+0x%lx", sym_name, offset);
+			if (modname)
+				printk(" [%s]", modname);
+		}
+	} else
+#endif /* CONFIG_IPIPE_TRACE_PANIC */
+	{
+		if (sym_name) {
+			if (verbose_trace) {
+				seq_printf(m, "%s+0x%lx", sym_name, offset);
+				if (modname)
+					seq_printf(m, " [%s]", modname);
+			} else
+				seq_puts(m, sym_name);
+		} else
+			seq_printf(m, "<%08lx>", eip);
+	}
+}
+
+static void __ipipe_print_headline(struct seq_file *m)
+{
+	const char *name[2];
+
+	seq_printf(m, "Calibrated minimum trace-point overhead: %lu.%03lu "
+		   "us\n\n", trace_overhead/1000, trace_overhead%1000);
+
+	if (verbose_trace) {
+		name[0] = ipipe_root_domain->name;
+		if (ipipe_head_domain != ipipe_root_domain)
+			name[1] = ipipe_head_domain->name;
+		else
+			name[1] = "<unused>";
+
+		seq_printf(m,
+		           " +----- Hard IRQs ('|': locked)\n"
+		           " |+-- %s\n"
+		           " ||+- %s%s\n"
+		           " |||                          +---------- "
+		               "Delay flag ('+': > %d us, '!': > %d us)\n"
+		           " |||                          |        +- "
+		               "NMI noise ('N')\n"
+		           " |||                          |        |\n"
+		           "      Type    User Val.   Time    Delay  Function "
+		               "(Parent)\n",
+		           name[1], name[0],
+		           " ('*': domain stalled, '+': current, "
+			   "'#': current+stalled)",
+		           IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000);
+	} else
+		seq_printf(m,
+		           " +--------------- Hard IRQs ('|': locked)\n"
+		           " |             +- Delay flag "
+		               "('+': > %d us, '!': > %d us)\n"
+		           " |             |\n"
+		           "  Type     Time   Function (Parent)\n",
+		           IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000);
+}
+
+static void *__ipipe_max_prtrace_start(struct seq_file *m, loff_t *pos)
+{
+	loff_t n = *pos;
+
+	mutex_lock(&out_mutex);
+
+	if (!n) {
+		struct ipipe_trace_path *tp;
+		unsigned long length_usecs;
+		int points, cpu;
+		unsigned long flags;
+
+		/* protect against max_path/frozen_path updates while we
+		 * haven't locked our target path, also avoid recursively
+		 * taking global_path_lock from NMI context */
+		flags = __ipipe_global_path_lock();
+
+		/* find the longest of all per-cpu paths */
+		print_path = NULL;
+		for_each_online_cpu(cpu) {
+			tp = &per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)];
+			if ((print_path == NULL) ||
+			    (tp->length > print_path->length)) {
+				print_path = tp;
+				break;
+			}
+		}
+		print_path->dump_lock = 1;
+
+		__ipipe_global_path_unlock(flags);
+
+		/* does this path actually contain data? */
+		if (print_path->end == print_path->begin)
+			return NULL;
+
+		/* number of points inside the critical path */
+		points = WRAP_POINT_NO(print_path->end-print_path->begin+1);
+
+		/* pre- and post-tracing length, post-trace length was frozen
+		   in __ipipe_trace, pre-trace may have to be reduced due to
+		   buffer overrun */
+		print_pre_trace  = pre_trace;
+		print_post_trace = WRAP_POINT_NO(print_path->trace_pos -
+		                                 print_path->end - 1);
+		if (points+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1)
+			print_pre_trace = IPIPE_TRACE_POINTS - 1 - points -
+				print_post_trace;
+
+		length_usecs = ipipe_tsc2us(print_path->length);
+		seq_printf(m, "I-pipe worst-case tracing service on %s/ipipe release #%d\n"
+			   "-------------------------------------------------------------\n",
+			UTS_RELEASE, IPIPE_CORE_RELEASE);
+		seq_printf(m, "CPU: %d, Begin: %lld cycles, Trace Points: "
+			"%d (-%d/+%d), Length: %lu us\n",
+			cpu, print_path->point[print_path->begin].timestamp,
+			points, print_pre_trace, print_post_trace, length_usecs);
+		__ipipe_print_headline(m);
+	}
+
+	/* check if we are inside the trace range */
+	if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 +
+	                       print_pre_trace + print_post_trace))
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin -
+	                                        print_pre_trace + n)];
+}
+
+static void *__ipipe_prtrace_next(struct seq_file *m, void *p, loff_t *pos)
+{
+	loff_t n = ++*pos;
+
+	/* check if we are inside the trace range with the next entry */
+	if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 +
+	                       print_pre_trace + print_post_trace))
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin -
+	                                        print_pre_trace + *pos)];
+}
+
+static void __ipipe_prtrace_stop(struct seq_file *m, void *p)
+{
+	if (print_path)
+		print_path->dump_lock = 0;
+	mutex_unlock(&out_mutex);
+}
+
+static int __ipipe_prtrace_show(struct seq_file *m, void *p)
+{
+	long time;
+	struct ipipe_trace_point *point = p;
+	char buf[16];
+
+	if (!point->eip) {
+		seq_puts(m, "-<invalid>-\n");
+		return 0;
+	}
+
+	__ipipe_print_pathmark(m, point);
+	__ipipe_trace_point_type(buf, point);
+	seq_puts(m, buf);
+	if (verbose_trace)
+		switch (point->type & IPIPE_TYPE_MASK) {
+			case IPIPE_TRACE_FUNC:
+				seq_puts(m, "           ");
+				break;
+
+			case IPIPE_TRACE_PID:
+				__ipipe_get_task_info(buf, point, 0);
+				seq_puts(m, buf);
+				break;
+
+			case IPIPE_TRACE_EVENT:
+				__ipipe_get_event_date(buf, print_path, point);
+				seq_puts(m, buf);
+				break;
+
+			default:
+				seq_printf(m, "0x%08lx ", point->v);
+		}
+
+	time = __ipipe_signed_tsc2us(point->timestamp -
+		print_path->point[print_path->begin].timestamp);
+	seq_printf(m, "%5ld", time);
+
+	__ipipe_print_delay(m, point);
+	__ipipe_print_symname(m, point->eip);
+	seq_puts(m, " (");
+	__ipipe_print_symname(m, point->parent_eip);
+	seq_puts(m, ")\n");
+
+	return 0;
+}
+
+static struct seq_operations __ipipe_max_ptrace_ops = {
+	.start = __ipipe_max_prtrace_start,
+	.next  = __ipipe_prtrace_next,
+	.stop  = __ipipe_prtrace_stop,
+	.show  = __ipipe_prtrace_show
+};
+
+static int __ipipe_max_prtrace_open(struct inode *inode, struct file *file)
+{
+	return seq_open(file, &__ipipe_max_ptrace_ops);
+}
+
+static ssize_t
+__ipipe_max_reset(struct file *file, const char __user *pbuffer,
+                  size_t count, loff_t *data)
+{
+	mutex_lock(&out_mutex);
+	ipipe_trace_max_reset();
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+struct file_operations __ipipe_max_prtrace_fops = {
+	.open       = __ipipe_max_prtrace_open,
+	.read       = seq_read,
+	.write      = __ipipe_max_reset,
+	.llseek     = seq_lseek,
+	.release    = seq_release,
+};
+
+static void *__ipipe_frozen_prtrace_start(struct seq_file *m, loff_t *pos)
+{
+	loff_t n = *pos;
+
+	mutex_lock(&out_mutex);
+
+	if (!n) {
+		struct ipipe_trace_path *tp;
+		int cpu;
+		unsigned long flags;
+
+		/* protect against max_path/frozen_path updates while we
+		 * haven't locked our target path, also avoid recursively
+		 * taking global_path_lock from NMI context */
+		flags = __ipipe_global_path_lock();
+
+		/* find the first of all per-cpu frozen paths */
+		print_path = NULL;
+		for_each_online_cpu(cpu) {
+			tp = &per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)];
+			if (tp->end >= 0) {
+				print_path = tp;
+				break;
+			}
+		}
+		if (print_path)
+			print_path->dump_lock = 1;
+
+		__ipipe_global_path_unlock(flags);
+
+		if (!print_path)
+			return NULL;
+
+		/* back- and post-tracing length, post-trace length was frozen
+		   in __ipipe_trace, back-trace may have to be reduced due to
+		   buffer overrun */
+		print_pre_trace  = back_trace-1; /* substract freeze point */
+		print_post_trace = WRAP_POINT_NO(print_path->trace_pos -
+		                                 print_path->end - 1);
+		if (1+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1)
+			print_pre_trace = IPIPE_TRACE_POINTS - 2 -
+				print_post_trace;
+
+		seq_printf(m, "I-pipe frozen back-tracing service on %s/ipipe release #%d\n"
+			      "------------------------------------------------------------\n",
+			   UTS_RELEASE, IPIPE_CORE_RELEASE);
+		seq_printf(m, "CPU: %d, Freeze: %lld cycles, Trace Points: %d (+%d)\n",
+			cpu, print_path->point[print_path->begin].timestamp,
+			print_pre_trace+1, print_post_trace);
+		__ipipe_print_headline(m);
+	}
+
+	/* check if we are inside the trace range */
+	if (n >= print_pre_trace + 1 + print_post_trace)
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin-
+	                                        print_pre_trace+n)];
+}
+
+static struct seq_operations __ipipe_frozen_ptrace_ops = {
+	.start = __ipipe_frozen_prtrace_start,
+	.next  = __ipipe_prtrace_next,
+	.stop  = __ipipe_prtrace_stop,
+	.show  = __ipipe_prtrace_show
+};
+
+static int __ipipe_frozen_prtrace_open(struct inode *inode, struct file *file)
+{
+	return seq_open(file, &__ipipe_frozen_ptrace_ops);
+}
+
+static ssize_t
+__ipipe_frozen_ctrl(struct file *file, const char __user *pbuffer,
+                    size_t count, loff_t *data)
+{
+	char *end, buf[16];
+	int val;
+	int n;
+
+	n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+	if (copy_from_user(buf, pbuffer, n))
+		return -EFAULT;
+
+	buf[n] = '\0';
+	val = simple_strtol(buf, &end, 0);
+
+	if (((*end != '\0') && !isspace(*end)) || (val < 0))
+		return -EINVAL;
+
+	mutex_lock(&out_mutex);
+	ipipe_trace_frozen_reset();
+	if (val > 0)
+		ipipe_trace_freeze(-1);
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+struct file_operations __ipipe_frozen_prtrace_fops = {
+	.open       = __ipipe_frozen_prtrace_open,
+	.read       = seq_read,
+	.write      = __ipipe_frozen_ctrl,
+	.llseek     = seq_lseek,
+	.release    = seq_release,
+};
+
+static int __ipipe_rd_proc_val(char *page, char **start, off_t off,
+                               int count, int *eof, void *data)
+{
+	int len;
+
+	len = sprintf(page, "%u\n", *(int *)data);
+	len -= off;
+	if (len <= off + count)
+		*eof = 1;
+	*start = page + off;
+	if (len > count)
+		len = count;
+	if (len < 0)
+		len = 0;
+
+	return len;
+}
+
+static int __ipipe_wr_proc_val(struct file *file, const char __user *buffer,
+                               unsigned long count, void *data)
+{
+	char *end, buf[16];
+	int val;
+	int n;
+
+	n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+	if (copy_from_user(buf, buffer, n))
+		return -EFAULT;
+
+	buf[n] = '\0';
+	val = simple_strtol(buf, &end, 0);
+
+	if (((*end != '\0') && !isspace(*end)) || (val < 0))
+		return -EINVAL;
+
+	mutex_lock(&out_mutex);
+	*(int *)data = val;
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+static int __ipipe_rd_trigger(char *page, char **start, off_t off, int count,
+			      int *eof, void *data)
+{
+	int len;
+
+	if (!trigger_begin)
+		return 0;
+
+	len = sprint_symbol(page, trigger_begin);
+	page[len++] = '\n';
+
+	len -= off;
+	if (len <= off + count)
+		*eof = 1;
+	*start = page + off;
+	if (len > count)
+		len = count;
+	if (len < 0)
+		len = 0;
+
+	return len;
+}
+
+static int __ipipe_wr_trigger(struct file *file, const char __user *buffer,
+			      unsigned long count, void *data)
+{
+	char buf[KSYM_SYMBOL_LEN];
+	unsigned long begin, end;
+
+	if (count > sizeof(buf) - 1)
+		count = sizeof(buf) - 1;
+	if (copy_from_user(buf, buffer, count))
+		return -EFAULT;
+	buf[count] = 0;
+	if (buf[count-1] == '\n')
+		buf[count-1] = 0;
+
+	begin = kallsyms_lookup_name(buf);
+	if (!begin || !kallsyms_lookup_size_offset(begin, &end, NULL))
+		return -ENOENT;
+	end += begin - 1;
+
+	mutex_lock(&out_mutex);
+	/* invalidate the current range before setting a new one */
+	trigger_end = 0;
+	wmb();
+	ipipe_trace_frozen_reset();
+
+	/* set new range */
+	trigger_begin = begin;
+	wmb();
+	trigger_end = end;
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+#ifdef CONFIG_IPIPE_TRACE_MCOUNT
+static void notrace
+ipipe_trace_function(unsigned long ip, unsigned long parent_ip)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_FUNC, ip, parent_ip, 0);
+}
+
+static struct ftrace_ops ipipe_trace_ops = {
+	.func = ipipe_trace_function
+};
+
+static int __ipipe_wr_enable(struct file *file, const char __user *buffer,
+			     unsigned long count, void *data)
+{
+	char *end, buf[16];
+	int val;
+	int n;
+
+	n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+	if (copy_from_user(buf, buffer, n))
+		return -EFAULT;
+
+	buf[n] = '\0';
+	val = simple_strtol(buf, &end, 0);
+
+	if (((*end != '\0') && !isspace(*end)) || (val < 0))
+		return -EINVAL;
+
+	mutex_lock(&out_mutex);
+
+	if (ipipe_trace_enable) {
+		if (!val)
+			unregister_ftrace_function(&ipipe_trace_ops);
+	} else if (val)
+		register_ftrace_function(&ipipe_trace_ops);
+
+	ipipe_trace_enable = val;
+
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+#endif /* CONFIG_IPIPE_TRACE_MCOUNT */
+
+extern struct proc_dir_entry *ipipe_proc_root;
+
+static struct proc_dir_entry * __init
+__ipipe_create_trace_proc_val(struct proc_dir_entry *trace_dir,
+                              const char *name, int *value_ptr)
+{
+	struct proc_dir_entry *entry;
+
+	entry = create_proc_entry(name, 0644, trace_dir);
+	if (entry) {
+		entry->data = value_ptr;
+		entry->read_proc = __ipipe_rd_proc_val;
+		entry->write_proc = __ipipe_wr_proc_val;
+	}
+	return entry;
+}
+
+void __init __ipipe_init_tracer(void)
+{
+	struct proc_dir_entry *trace_dir;
+	struct proc_dir_entry *entry;
+	unsigned long long start, end, min = ULLONG_MAX;
+	int i;
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
+	int cpu, path;
+
+	for_each_possible_cpu(cpu) {
+		struct ipipe_trace_path *tp_buf;
+
+		tp_buf = vmalloc_node(sizeof(struct ipipe_trace_path) *
+				      IPIPE_TRACE_PATHS, cpu_to_node(cpu));
+		if (!tp_buf) {
+			printk(KERN_ERR "I-pipe: "
+			       "insufficient memory for trace buffer.\n");
+			return;
+		}
+		memset(tp_buf, 0,
+		       sizeof(struct ipipe_trace_path) * IPIPE_TRACE_PATHS);
+		for (path = 0; path < IPIPE_TRACE_PATHS; path++) {
+			tp_buf[path].begin = -1;
+			tp_buf[path].end   = -1;
+		}
+		per_cpu(trace_path, cpu) = tp_buf;
+	}
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
+
+	/* Calculate minimum overhead of __ipipe_trace() */
+	hard_local_irq_disable();
+	for (i = 0; i < 100; i++) {
+		ipipe_read_tsc(start);
+		__ipipe_trace(IPIPE_TRACE_FUNC, __BUILTIN_RETURN_ADDRESS0,
+			      __BUILTIN_RETURN_ADDRESS1, 0);
+		ipipe_read_tsc(end);
+
+		end -= start;
+		if (end < min)
+			min = end;
+	}
+	hard_local_irq_enable();
+	trace_overhead = ipipe_tsc2ns(min);
+
+#ifdef CONFIG_IPIPE_TRACE_ENABLE
+	ipipe_trace_enable = 1;
+#ifdef CONFIG_IPIPE_TRACE_MCOUNT
+	ftrace_enabled = 1;
+	register_ftrace_function(&ipipe_trace_ops);
+#endif /* CONFIG_IPIPE_TRACE_MCOUNT */
+#endif /* CONFIG_IPIPE_TRACE_ENABLE */
+
+	trace_dir = create_proc_entry("trace", S_IFDIR, ipipe_proc_root);
+
+	entry = create_proc_entry("max", 0644, trace_dir);
+	if (entry)
+		entry->proc_fops = &__ipipe_max_prtrace_fops;
+
+	entry = create_proc_entry("frozen", 0644, trace_dir);
+	if (entry)
+		entry->proc_fops = &__ipipe_frozen_prtrace_fops;
+
+	entry = create_proc_entry("trigger", 0644, trace_dir);
+	if (entry) {
+		entry->read_proc = __ipipe_rd_trigger;
+		entry->write_proc = __ipipe_wr_trigger;
+	}
+
+	__ipipe_create_trace_proc_val(trace_dir, "pre_trace_points",
+	                              &pre_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "post_trace_points",
+	                              &post_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "back_trace_points",
+	                              &back_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "verbose",
+	                              &verbose_trace);
+	entry = __ipipe_create_trace_proc_val(trace_dir, "enable",
+					      &ipipe_trace_enable);
+#ifdef CONFIG_IPIPE_TRACE_MCOUNT
+	if (entry)
+		entry->write_proc = __ipipe_wr_enable;
+#endif /* CONFIG_IPIPE_TRACE_MCOUNT */
+}
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index 25784d6..ff74a48 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -15,6 +15,7 @@
 #include <linux/module.h>
 #include <linux/interrupt.h>
 #include <linux/kernel_stat.h>
+#include <linux/ipipe.h>
 
 #include "internals.h"
 
@@ -246,9 +247,13 @@ void mask_irq(struct irq_desc *desc)
 
 void unmask_irq(struct irq_desc *desc)
 {
+	unsigned long flags;
+
 	if (desc->irq_data.chip->irq_unmask) {
+		flags = hard_cond_local_irq_save();
 		desc->irq_data.chip->irq_unmask(&desc->irq_data);
 		irq_state_clr_masked(desc);
+		hard_cond_local_irq_restore(flags);
 	}
 }
 
@@ -364,7 +369,9 @@ void
 handle_level_irq(unsigned int irq, struct irq_desc *desc)
 {
 	raw_spin_lock(&desc->lock);
+#ifndef CONFIG_IPIPE
 	mask_ack_irq(desc);
+#endif
 
 	if (unlikely(irqd_irq_inprogress(&desc->irq_data)))
 		if (!irq_check_poll(desc))
@@ -431,17 +438,26 @@ handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc)
 		goto out;
 	}
 
+#ifndef CONFIG_IPIPE
 	if (desc->istate & IRQS_ONESHOT)
 		mask_irq(desc);
+#endif
 
 	preflow_handler(desc);
 	handle_irq_event(desc);
 
+#ifdef CONFIG_IPIPE
+	/* XXX: IRQCHIP_EOI_IF_HANDLED is ignored. */
+	if (desc->irq_data.chip->irq_release)
+		desc->irq_data.chip->irq_release(&desc->irq_data);
+out_eoi:
+#else  /* !CONFIG_IPIPE */
 	if (desc->istate & IRQS_ONESHOT)
 		cond_unmask_irq(desc);
 
 out_eoi:
 	desc->irq_data.chip->irq_eoi(&desc->irq_data);
+#endif	/* !CONFIG_IPIPE */
 out_unlock:
 	raw_spin_unlock(&desc->lock);
 	return;
@@ -489,7 +505,9 @@ handle_edge_irq(unsigned int irq, struct irq_desc *desc)
 	kstat_incr_irqs_this_cpu(irq, desc);
 
 	/* Start handling the irq */
+#ifndef CONFIG_IPIPE
 	desc->irq_data.chip->irq_ack(&desc->irq_data);
+#endif
 
 	do {
 		if (unlikely(!desc->action)) {
@@ -576,6 +594,12 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc)
 
 	kstat_incr_irqs_this_cpu(irq, desc);
 
+#ifdef CONFIG_IPIPE
+	handle_irq_event_percpu(desc, desc->action);
+
+	if (chip->irq_eoi && !irqd_irq_masked(&desc->irq_data))
+		chip->irq_unmask(&desc->irq_data);
+#else
 	if (chip->irq_ack)
 		chip->irq_ack(&desc->irq_data);
 
@@ -583,6 +607,7 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc)
 
 	if (chip->irq_eoi)
 		chip->irq_eoi(&desc->irq_data);
+#endif
 }
 
 /**
@@ -606,17 +631,133 @@ void handle_percpu_devid_irq(unsigned int irq, struct irq_desc *desc)
 
 	kstat_incr_irqs_this_cpu(irq, desc);
 
+#ifndef CONFIG_IPIPE
 	if (chip->irq_ack)
 		chip->irq_ack(&desc->irq_data);
+#else
+	(void)chip;
+#endif
 
 	trace_irq_handler_entry(irq, action);
 	res = action->handler(irq, dev_id);
 	trace_irq_handler_exit(irq, action, res);
 
+#ifndef CONFIG_IPIPE
 	if (chip->irq_eoi)
 		chip->irq_eoi(&desc->irq_data);
+#else
+	if (chip->irq_eoi && !irqd_irq_masked(&desc->irq_data))
+		chip->irq_unmask(&desc->irq_data);
+#endif
 }
 
+#ifdef CONFIG_IPIPE
+
+void __ipipe_ack_level_irq(unsigned irq, struct irq_desc *desc)
+{
+	mask_ack_irq(desc);
+}
+
+void __ipipe_end_level_irq(unsigned irq, struct irq_desc *desc)
+{
+	desc->irq_data.chip->irq_unmask(&desc->irq_data);
+}
+
+void __ipipe_ack_fasteoi_irq(unsigned irq, struct irq_desc *desc)
+{
+	desc->irq_data.chip->irq_hold(&desc->irq_data);
+}
+
+void __ipipe_end_fasteoi_irq(unsigned irq, struct irq_desc *desc)
+{
+	if (desc->irq_data.chip->irq_release)
+		desc->irq_data.chip->irq_release(&desc->irq_data);
+}
+
+void __ipipe_ack_edge_irq(unsigned irq, struct irq_desc *desc)
+{
+	desc->irq_data.chip->irq_ack(&desc->irq_data);
+}
+
+void __ipipe_ack_percpu_irq(unsigned irq, struct irq_desc *desc)
+{
+	if (desc->irq_data.chip->irq_ack)
+		desc->irq_data.chip->irq_ack(&desc->irq_data);
+
+	if (desc->irq_data.chip->irq_eoi)
+		desc->irq_data.chip->irq_eoi(&desc->irq_data);
+}
+
+void __ipipe_nop_irq(unsigned irq, struct irq_desc *desc)
+{
+}
+
+void __ipipe_chained_irq(unsigned irq, struct irq_desc *desc)
+{
+	/*
+	 * XXX: Do NOT fold this into __ipipe_nop_irq(), see
+	 * ipipe_chained_irq_p().
+	 */
+}
+
+static void __ipipe_ack_bad_irq(unsigned irq, struct irq_desc *desc)
+{
+	handle_bad_irq(irq, desc);
+	WARN_ON_ONCE(1);
+}
+
+irq_flow_handler_t
+__fixup_irq_handler(struct irq_desc *desc, irq_flow_handler_t handle, int is_chained)
+{
+	if (unlikely(handle == NULL)) {
+		desc->ipipe_ack = __ipipe_ack_bad_irq;
+		desc->ipipe_end = __ipipe_nop_irq;
+	} else {
+		if (is_chained) {
+			desc->ipipe_ack = handle;
+			desc->ipipe_end = __ipipe_nop_irq;
+			handle = __ipipe_chained_irq;
+		} else if (handle == handle_simple_irq) {
+			desc->ipipe_ack = __ipipe_nop_irq;
+			desc->ipipe_end = __ipipe_nop_irq;
+		} else if (handle == handle_level_irq) {
+			desc->ipipe_ack = __ipipe_ack_level_irq;
+			desc->ipipe_end = __ipipe_end_level_irq;
+		} else if (handle == handle_edge_irq) {
+			desc->ipipe_ack = __ipipe_ack_edge_irq;
+			desc->ipipe_end = __ipipe_nop_irq;
+		} else if (handle == handle_fasteoi_irq) {
+			desc->ipipe_ack = __ipipe_ack_fasteoi_irq;
+			desc->ipipe_end = __ipipe_end_fasteoi_irq;
+		} else if (handle == handle_percpu_irq ||
+			   handle == handle_percpu_devid_irq) {
+			desc->ipipe_ack = __ipipe_ack_percpu_irq;
+			desc->ipipe_end = __ipipe_nop_irq;
+		} else if (irq_desc_get_chip(desc) == &no_irq_chip) {
+			desc->ipipe_ack = __ipipe_nop_irq;
+			desc->ipipe_end = __ipipe_nop_irq;
+		} else {
+			desc->ipipe_ack = __ipipe_ack_bad_irq;
+			desc->ipipe_end = __ipipe_nop_irq;
+		}
+	}
+
+	/* Suppress intermediate trampoline routine. */
+	ipipe_root_domain->irqs[desc->irq_data.irq].ackfn = desc->ipipe_ack;
+
+	return handle;
+}
+
+#else /* !CONFIG_IPIPE */
+
+irq_flow_handler_t
+__fixup_irq_handler(struct irq_desc *desc, irq_flow_handler_t handle, int is_chained)
+{
+	return handle;
+}
+
+#endif /* !CONFIG_IPIPE */
+
 void
 __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
 		  const char *name)
@@ -634,6 +775,8 @@ __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
 			goto out;
 	}
 
+	handle = __fixup_irq_handler(desc, handle, is_chained);
+
 	/* Uninstall? */
 	if (handle == handle_bad_irq) {
 		if (desc->irq_data.chip != &no_irq_chip)
diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c
index d86e254..4a3b808 100644
--- a/kernel/irq/irqdesc.c
+++ b/kernel/irq/irqdesc.c
@@ -269,10 +269,12 @@ int __init early_irq_init(void)
 	return arch_early_irq_init();
 }
 
+#ifndef CONFIG_IPIPE
 struct irq_desc *irq_to_desc(unsigned int irq)
 {
 	return (irq < NR_IRQS) ? irq_desc + irq : NULL;
 }
+#endif /* CONFIG_IPIPE */
 
 static void free_desc(unsigned int irq)
 {
diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index b2e08c9..65ad2a1 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -2576,7 +2576,7 @@ void trace_hardirqs_on_caller(unsigned long ip)
 	 * already enabled, yet we find the hardware thinks they are in fact
 	 * enabled.. someone messed up their IRQ state tracing.
 	 */
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !hard_irqs_disabled()))
 		return;
 
 	/*
@@ -2620,7 +2620,7 @@ void trace_hardirqs_off_caller(unsigned long ip)
 	 * So we're supposed to get called after you mask local IRQs, but for
 	 * some reason the hardware doesn't quite think you did a proper job.
 	 */
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !hard_irqs_disabled()))
 		return;
 
 	if (curr->hardirqs_enabled) {
@@ -2656,7 +2656,7 @@ void trace_softirqs_on(unsigned long ip)
 	 * We fancy IRQs being disabled here, see softirq.c, avoids
 	 * funny state and nesting things.
 	 */
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !hard_irqs_disabled()))
 		return;
 
 	if (curr->softirqs_enabled) {
@@ -2695,7 +2695,7 @@ void trace_softirqs_off(unsigned long ip)
 	/*
 	 * We fancy IRQs being disabled here, see softirq.c
 	 */
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !hard_irqs_disabled()))
 		return;
 
 	if (curr->softirqs_enabled) {
diff --git a/kernel/panic.c b/kernel/panic.c
index 3458469..c996333 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -23,6 +23,7 @@
 #include <linux/init.h>
 #include <linux/nmi.h>
 #include <linux/dmi.h>
+#include <linux/ipipe_trace.h>
 
 #define PANIC_TIMER_STEP 100
 #define PANIC_BLINK_SPD 18
@@ -332,6 +333,8 @@ void oops_enter(void)
 {
 	tracing_off();
 	/* can't trust the integrity of the kernel anymore: */
+	ipipe_trace_panic_freeze();
+	ipipe_disable_context_check();
 	debug_locks_off();
 	do_oops_enter_exit();
 }
diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c
index 7c0d578..a6e3f0d 100644
--- a/kernel/power/hibernate.c
+++ b/kernel/power/hibernate.c
@@ -283,6 +283,7 @@ static int create_image(int platform_mode)
 		goto Enable_cpus;
 
 	local_irq_disable();
+	hard_cond_local_irq_disable();
 
 	error = syscore_suspend();
 	if (error) {
@@ -437,6 +438,7 @@ static int resume_target_kernel(bool platform_mode)
 		goto Enable_cpus;
 
 	local_irq_disable();
+	hard_cond_local_irq_disable();
 
 	error = syscore_suspend();
 	if (error)
@@ -549,6 +551,7 @@ int hibernation_platform_enter(void)
 		goto Platform_finish;
 
 	local_irq_disable();
+	hard_cond_local_irq_disable();
 	syscore_suspend();
 	if (pm_wakeup_pending()) {
 		error = -EAGAIN;
diff --git a/kernel/printk.c b/kernel/printk.c
index 7982a0a..03c145c 100644
--- a/kernel/printk.c
+++ b/kernel/printk.c
@@ -713,6 +713,43 @@ static int have_callable_console(void)
 	return 0;
 }
 
+#ifdef CONFIG_IPIPE
+
+extern int __ipipe_printk_bypass;
+
+static IPIPE_DEFINE_SPINLOCK(__ipipe_printk_lock);
+
+static int __ipipe_printk_fill;
+
+static char __ipipe_printk_buf[__LOG_BUF_LEN];
+
+void __ipipe_flush_printk (unsigned virq, void *cookie)
+{
+	char *p = __ipipe_printk_buf;
+	int len, lmax, out = 0;
+	unsigned long flags;
+
+	goto start;
+
+	do {
+		spin_unlock_irqrestore(&__ipipe_printk_lock, flags);
+ start:
+		lmax = __ipipe_printk_fill;
+		while (out < lmax) {
+			len = strlen(p) + 1;
+			printk("%s",p);
+			p += len;
+			out += len;
+		}
+		spin_lock_irqsave(&__ipipe_printk_lock, flags);
+	}
+	while (__ipipe_printk_fill != lmax);
+
+	__ipipe_printk_fill = 0;
+
+	spin_unlock_irqrestore(&__ipipe_printk_lock, flags);
+}
+
 /**
  * printk - print a kernel message
  * @fmt: format string
@@ -737,6 +774,59 @@ static int have_callable_console(void)
 
 asmlinkage int printk(const char *fmt, ...)
 {
+	int sprintk = 1, cs = -1;
+	int r, fbytes, oldcount;
+	unsigned long flags;
+	va_list args;
+
+	va_start(args, fmt);
+
+	flags = hard_local_irq_save();
+
+	if (__ipipe_printk_bypass || oops_in_progress)
+		cs = ipipe_disable_context_check();
+	else if (__ipipe_current_domain == ipipe_root_domain) {
+		if (ipipe_head_domain != ipipe_root_domain && 
+		    (raw_irqs_disabled_flags(flags) ||
+		     test_bit(IPIPE_STALL_FLAG, &__ipipe_head_status)))
+			sprintk = 0;
+	} else
+		sprintk = 0;
+
+	hard_local_irq_restore(flags);
+
+	if (sprintk) {
+		r = vprintk(fmt, args);
+		if (cs != -1)
+			ipipe_restore_context_check(cs);
+		goto out;
+	}
+
+	spin_lock_irqsave(&__ipipe_printk_lock, flags);
+
+	oldcount = __ipipe_printk_fill;
+	fbytes = __LOG_BUF_LEN - oldcount;
+	if (fbytes > 1)	{
+		r = vscnprintf(__ipipe_printk_buf + __ipipe_printk_fill,
+			       fbytes, fmt, args) + 1;
+		__ipipe_printk_fill += r;
+	} else
+		r = 0;
+
+	spin_unlock_irqrestore(&__ipipe_printk_lock, flags);
+
+	if (oldcount == 0)
+		ipipe_raise_irq(__ipipe_printk_virq);
+out:
+	va_end(args);
+
+	return r;
+}
+
+#else /* !CONFIG_IPIPE */
+
+asmlinkage int printk(const char *fmt, ...)
+{
 	va_list args;
 	int r;
 
@@ -755,6 +845,8 @@ asmlinkage int printk(const char *fmt, ...)
 	return r;
 }
 
+#endif /* CONFIG_IPIPE */
+
 /* cpu currently holding logbuf_lock */
 static volatile unsigned int printk_cpu = UINT_MAX;
 
@@ -951,7 +1043,7 @@ asmlinkage int vprintk(const char *fmt, va_list args)
 	 * Try to acquire and then immediately release the
 	 * console semaphore. The release will do all the
 	 * actual magic (print out buffers, wake up klogd,
-	 * etc). 
+	 * etc).
 	 *
 	 * The console_trylock_for_printk() function
 	 * will release 'logbuf_lock' regardless of whether it
@@ -1568,7 +1660,7 @@ EXPORT_SYMBOL(register_console);
 
 int unregister_console(struct console *console)
 {
-        struct console *a, *b;
+	struct console *a, *b;
 	int res = 1;
 
 #ifdef CONFIG_A11Y_BRAILLE_CONSOLE
diff --git a/kernel/sched.c b/kernel/sched.c
index 576a27f..e61169c 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -2825,7 +2825,8 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
 
 	smp_wmb();
 	raw_spin_lock_irqsave(&p->pi_lock, flags);
-	if (!(p->state & state))
+	if (!(p->state & state) ||
+	    (p->state & (TASK_NOWAKEUP|TASK_HARDENING)))
 		goto out;
 
 	success = 1; /* we're going to change ->state */
@@ -3263,6 +3264,8 @@ asmlinkage void schedule_tail(struct task_struct *prev)
 {
 	struct rq *rq = this_rq();
 
+	__ipipe_complete_domain_migration();
+
 	finish_task_switch(rq, prev);
 
 	/*
@@ -3283,7 +3286,7 @@ asmlinkage void schedule_tail(struct task_struct *prev)
  * context_switch - switch to the new MM and the new
  * thread's register state.
  */
-static inline void
+static inline int
 context_switch(struct rq *rq, struct task_struct *prev,
 	       struct task_struct *next)
 {
@@ -3325,12 +3328,18 @@ context_switch(struct rq *rq, struct task_struct *prev,
 	switch_to(prev, next, prev);
 
 	barrier();
+
+	if (unlikely(__ipipe_switch_tail()))
+		return 1;
+
 	/*
 	 * this_rq must be evaluated again because prev may have moved
 	 * CPUs since it called schedule(), thus the 'rq' on its stack
 	 * frame will be invalid.
 	 */
 	finish_task_switch(this_rq(), prev);
+
+	return 0;
 }
 
 /*
@@ -4269,6 +4278,7 @@ notrace unsigned long get_parent_ip(unsigned long addr)
 
 void __kprobes add_preempt_count(int val)
 {
+ 	ipipe_root_only();
 #ifdef CONFIG_DEBUG_PREEMPT
 	/*
 	 * Underflow?
@@ -4291,6 +4301,7 @@ EXPORT_SYMBOL(add_preempt_count);
 
 void __kprobes sub_preempt_count(int val)
 {
+ 	ipipe_root_only();
 #ifdef CONFIG_DEBUG_PREEMPT
 	/*
 	 * Underflow?
@@ -4339,6 +4350,7 @@ static noinline void __schedule_bug(struct task_struct *prev)
  */
 static inline void schedule_debug(struct task_struct *prev)
 {
+	ipipe_root_only();
 	/*
 	 * Test if we are atomic. Since do_exit() needs to call into
 	 * schedule() atomically, we ignore that path for now.
@@ -4391,7 +4403,7 @@ pick_next_task(struct rq *rq)
 /*
  * __schedule() is the main scheduler function.
  */
-static void __sched __schedule(void)
+static int __sched __schedule(void)
 {
 	struct task_struct *prev, *next;
 	unsigned long *switch_count;
@@ -4405,6 +4417,10 @@ need_resched:
 	rcu_note_context_switch(cpu);
 	prev = rq->curr;
 
+ 	if (unlikely(prev->state & TASK_HARDENING))
+		/* Pop one disable level -- one still remains. */
+		preempt_enable();
+
 	schedule_debug(prev);
 
 	if (sched_feat(HRTICK))
@@ -4451,7 +4467,8 @@ need_resched:
 		rq->curr = next;
 		++*switch_count;
 
-		context_switch(rq, prev, next); /* unlocks the rq */
+ 		if (context_switch(rq, prev, next)) /* unlocks the rq */
+  			return 1; /* task hijacked by higher domain */
 		/*
 		 * The context switch have flipped the stack from under us
 		 * and restored the local variables which were saved when
@@ -4460,14 +4477,18 @@ need_resched:
 		 */
 		cpu = smp_processor_id();
 		rq = cpu_rq(cpu);
-	} else
+	} else {
+  		prev->state &= ~TASK_HARDENING;
 		raw_spin_unlock_irq(&rq->lock);
+	}
 
 	post_schedule(rq);
 
 	preempt_enable_no_resched();
 	if (need_resched())
 		goto need_resched;
+
+	return 0;
 }
 
 static inline void sched_submit_work(struct task_struct *tsk)
@@ -4555,7 +4576,8 @@ asmlinkage void __sched notrace preempt_schedule(void)
 
 	do {
 		add_preempt_count_notrace(PREEMPT_ACTIVE);
-		__schedule();
+		if (__schedule())
+			return;
 		sub_preempt_count_notrace(PREEMPT_ACTIVE);
 
 		/*
@@ -5382,6 +5404,7 @@ recheck:
 	oldprio = p->prio;
 	prev_class = p->sched_class;
 	__setscheduler(rq, p, policy, param->sched_priority);
+  	__ipipe_report_setsched(p);
 
 	if (running)
 		p->sched_class->set_curr_task(rq);
@@ -6106,6 +6129,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
 
 	/* Set the preempt count _outside_ the spinlocks! */
 	task_thread_info(idle)->preempt_count = 0;
+	ipipe_root_only();
 
 	/*
 	 * The idle tasks have their own, simple scheduling class:
@@ -9789,3 +9813,38 @@ struct cgroup_subsys cpuacct_subsys = {
 	.subsys_id = cpuacct_subsys_id,
 };
 #endif	/* CONFIG_CGROUP_CPUACCT */
+
+#ifdef CONFIG_IPIPE
+
+int __ipipe_migrate_head(void)
+{
+	struct task_struct *p = current;
+
+	preempt_disable();
+
+	__this_cpu_write(ipipe_percpu.task_hijacked, p);
+	set_current_state(TASK_INTERRUPTIBLE | TASK_HARDENING);
+	sched_submit_work(p);
+	if (likely(__schedule()))
+		return 0;
+
+	if (signal_pending(p))
+		return -ERESTARTSYS;
+
+	BUG();
+}
+EXPORT_SYMBOL_GPL(__ipipe_migrate_head);
+
+void __ipipe_reenter_root(void)
+{
+	struct rq *rq = this_rq();
+	struct task_struct *p;
+
+	p = __this_cpu_read(ipipe_percpu.rqlock_owner);
+	finish_task_switch(rq, p);
+	post_schedule(rq);
+	preempt_enable_no_resched();
+}
+EXPORT_SYMBOL_GPL(__ipipe_reenter_root);
+
+#endif /* CONFIG_IPIPE */
diff --git a/kernel/signal.c b/kernel/signal.c
index 08e0b97..370cf50 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -681,6 +681,8 @@ void signal_wake_up(struct task_struct *t, int resume)
 	unsigned int mask;
 
 	set_tsk_thread_flag(t, TIF_SIGPENDING);
+	/* TIF_SIGPENDING must be prior to reporting. */
+	__ipipe_report_sigwake(t);
 
 	/*
 	 * For SIGKILL, we want to wake it up in the stopped/traced/killable
diff --git a/kernel/spinlock.c b/kernel/spinlock.c
index 84c7d96..5901c36 100644
--- a/kernel/spinlock.c
+++ b/kernel/spinlock.c
@@ -26,7 +26,9 @@
  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
  */
-#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
+#if !defined(CONFIG_GENERIC_LOCKBREAK) ||			\
+	defined(CONFIG_DEBUG_LOCK_ALLOC) ||			\
+	defined(CONFIG_IPIPE)
 /*
  * The __lock_function inlines are taken from
  * include/linux/spinlock_api_smp.h
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c
index 1ecd6ba..c2250bb 100644
--- a/kernel/time/clockevents.c
+++ b/kernel/time/clockevents.c
@@ -18,6 +18,7 @@
 #include <linux/notifier.h>
 #include <linux/smp.h>
 #include <linux/sysdev.h>
+#include <linux/ipipe_tickdev.h>
 
 #include "tick-internal.h"
 
@@ -283,6 +284,9 @@ void clockevents_register_device(struct clock_event_device *dev)
 	unsigned long flags;
 
 	BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED);
+
+	ipipe_host_timer_register(dev);
+
 	if (!dev->cpumask) {
 		WARN_ON(num_possible_cpus() > 1);
 		dev->cpumask = cpumask_of(smp_processor_id());
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
index da6c9ec..0c4be00 100644
--- a/kernel/time/tick-common.c
+++ b/kernel/time/tick-common.c
@@ -72,7 +72,7 @@ static void tick_periodic(int cpu)
 		write_sequnlock(&xtime_lock);
 	}
 
-	update_process_times(user_mode(get_irq_regs()));
+	update_root_process_times(get_irq_regs());
 	profile_tick(CPU_PROFILING);
 }
 
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index c923640..de83930 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -620,7 +620,7 @@ static void tick_nohz_handler(struct clock_event_device *dev)
 		ts->idle_jiffies++;
 	}
 
-	update_process_times(user_mode(regs));
+	update_root_process_times(regs);
 	profile_tick(CPU_PROFILING);
 
 	while (tick_nohz_reprogram(ts, now)) {
@@ -777,7 +777,7 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
 			touch_softlockup_watchdog();
 			ts->idle_jiffies++;
 		}
-		update_process_times(user_mode(regs));
+		update_root_process_times(regs);
 		profile_tick(CPU_PROFILING);
 	}
 
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 2378413..110c265 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -1243,3 +1243,9 @@ void xtime_update(unsigned long ticks)
 	do_timer(ticks);
 	write_sequnlock(&xtime_lock);
 }
+
+struct timespec get_wall_to_monotonic(void)
+{
+	/* xtime_lock must be held. */
+	return wall_to_monotonic;
+}
diff --git a/kernel/timer.c b/kernel/timer.c
index 9c3c62b..4576f56 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -1301,6 +1301,25 @@ void update_process_times(int user_tick)
 	run_posix_cpu_timers(p);
 }
 
+#ifdef CONFIG_IPIPE
+
+void update_root_process_times(struct pt_regs *regs)
+{
+	int cpu, user_tick = user_mode(regs);
+
+	if (__ipipe_root_tick_p(regs)) {
+		update_process_times(user_tick);
+		return;
+	}
+
+	run_local_timers();
+	cpu = smp_processor_id();
+	rcu_check_callbacks(cpu, user_tick);
+	run_posix_cpu_timers(current);
+}
+
+#endif
+
 /*
  * This function runs timers and the timer-tq in bottom half context.
  */
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index cd31345..ea6218f 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -389,6 +389,7 @@ config DYNAMIC_FTRACE
 	bool "enable/disable ftrace tracepoints dynamically"
 	depends on FUNCTION_TRACER
 	depends on HAVE_DYNAMIC_FTRACE
+	depends on !IPIPE_TRACE_MCOUNT
 	default y
 	help
           This option will modify all the calls to ftrace dynamically
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 25b4f4d..dbdc667 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -30,6 +30,7 @@
 #include <linux/list.h>
 #include <linux/hash.h>
 #include <linux/rcupdate.h>
+#include <linux/ipipe.h>
 
 #include <trace/events/sched.h>
 
@@ -1653,6 +1654,9 @@ static int __ftrace_modify_code(void *data)
 
 static void ftrace_run_update_code(int command)
 {
+#ifdef CONFIG_IPIPE
+	unsigned long flags;
+#endif /* CONFIG_IPIPE */
 	int ret;
 
 	ret = ftrace_arch_code_modify_prepare();
@@ -1660,7 +1664,13 @@ static void ftrace_run_update_code(int command)
 	if (ret)
 		return;
 
+#ifdef CONFIG_IPIPE
+	flags = ipipe_critical_enter(NULL);
+	__ftrace_modify_code(&command);
+	ipipe_critical_exit(flags);
+#else  /* !CONFIG_IPIPE */
 	stop_machine(__ftrace_modify_code, &command, NULL);
+#endif /* !CONFIG_IPIPE */
 
 	ret = ftrace_arch_code_modify_post_process();
 	FTRACE_WARN_ON(ret);
@@ -3403,6 +3413,11 @@ static int ftrace_process_locs(struct module *mod,
 		ftrace_record_ip(addr);
 	}
 
+#ifdef CONFIG_IPIPE
+	flags = hard_local_irq_save_notrace();
+	ftrace_update_code(mod);
+	hard_local_irq_restore_notrace(flags);
+#else
 	/*
 	 * We only need to disable interrupts on start up
 	 * because we are modifying code that an interrupt
@@ -3416,6 +3431,7 @@ static int ftrace_process_locs(struct module *mod,
 	ftrace_update_code(mod);
 	if (!mod)
 		local_irq_restore(flags);
+#endif
 	mutex_unlock(&ftrace_lock);
 
 	return 0;
@@ -3496,9 +3512,9 @@ void __init ftrace_init(void)
 	/* Keep the ftrace pointer to the stub */
 	addr = (unsigned long)ftrace_stub;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save_notrace();
 	ftrace_dyn_arch_init(&addr);
-	local_irq_restore(flags);
+	hard_local_irq_restore_notrace(flags);
 
 	/* ftrace_dyn_arch_init places the return code in addr */
 	if (addr)
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 82928f5..374c1bb 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -143,6 +143,8 @@ config DEBUG_SECTION_MISMATCH
 	  - Enable verbose reporting from modpost in order to help resolve
 	    the section mismatches that are reported.
 
+source "kernel/ipipe/Kconfig.debug"
+
 config DEBUG_KERNEL
 	bool "Kernel debugging"
 	help
diff --git a/lib/bust_spinlocks.c b/lib/bust_spinlocks.c
index 9681d54..2dba50c 100644
--- a/lib/bust_spinlocks.c
+++ b/lib/bust_spinlocks.c
@@ -13,6 +13,7 @@
 #include <linux/wait.h>
 #include <linux/vt_kern.h>
 #include <linux/console.h>
+#include <linux/ipipe_trace.h>
 
 
 void __attribute__((weak)) bust_spinlocks(int yes)
@@ -24,6 +25,7 @@ void __attribute__((weak)) bust_spinlocks(int yes)
 		unblank_screen();
 #endif
 		console_unblank();
+  		ipipe_trace_panic_dump();
 		if (--oops_in_progress == 0)
 			wake_up_klogd();
 	}
diff --git a/lib/ioremap.c b/lib/ioremap.c
index da4e2ad..729c995 100644
--- a/lib/ioremap.c
+++ b/lib/ioremap.c
@@ -86,8 +86,8 @@ int ioremap_page_range(unsigned long addr,
 		if (err)
 			break;
 	} while (pgd++, addr = next, addr != end);
-
-	flush_cache_vmap(start, end);
+	__ipipe_pin_range_globally(start, end);
+ 	flush_cache_vmap(start, end);
 
 	return err;
 }
diff --git a/lib/smp_processor_id.c b/lib/smp_processor_id.c
index 503f087..2d1958f 100644
--- a/lib/smp_processor_id.c
+++ b/lib/smp_processor_id.c
@@ -12,10 +12,13 @@ notrace unsigned int debug_smp_processor_id(void)
 	unsigned long preempt_count = preempt_count();
 	int this_cpu = raw_smp_processor_id();
 
+	if (!ipipe_root_p)
+		goto out;
+
 	if (likely(preempt_count))
 		goto out;
 
-	if (irqs_disabled())
+	if (irqs_disabled() || hard_irqs_disabled())
 		goto out;
 
 	/*
diff --git a/mm/memory.c b/mm/memory.c
index 1b1ca17..5f98bfb 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -845,6 +845,32 @@ out:
 	return pfn_to_page(pfn);
 }
 
+static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
+{
+	/*
+	 * If the source page was a PFN mapping, we don't have
+	 * a "struct page" for it. We do a best-effort copy by
+	 * just copying from the original user address. If that
+	 * fails, we just zero-fill it. Live with it.
+	 */
+	if (unlikely(!src)) {
+		void *kaddr = kmap_atomic(dst, KM_USER0);
+		void __user *uaddr = (void __user *)(va & PAGE_MASK);
+
+		/*
+		 * This really shouldn't fail, because the page is there
+		 * in the page tables. But it might just be unreadable,
+		 * in which case we just give up and fill the result with
+		 * zeroes.
+		 */
+		if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
+			clear_page(kaddr);
+		kunmap_atomic(kaddr, KM_USER0);
+		flush_dcache_page(dst);
+	} else
+		copy_user_highpage(dst, src, va, vma);
+}
+
 /*
  * copy one vm_area from one task to the other. Assumes the page tables
  * already present in the new task to be cleared in the whole range
@@ -853,8 +879,8 @@ out:
 
 static inline unsigned long
 copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
-		pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
-		unsigned long addr, int *rss)
+	     pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
+	     unsigned long addr, int *rss, struct page *uncow_page)
 {
 	unsigned long vm_flags = vma->vm_flags;
 	pte_t pte = *src_pte;
@@ -897,6 +923,21 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 	 * in the parent and the child
 	 */
 	if (is_cow_mapping(vm_flags)) {
+#ifdef CONFIG_IPIPE
+		if (uncow_page) {
+			struct page *old_page = vm_normal_page(vma, addr, pte);
+			cow_user_page(uncow_page, old_page, addr, vma);
+			pte = mk_pte(uncow_page, vma->vm_page_prot);
+
+			if (vm_flags & VM_SHARED)
+				pte = pte_mkclean(pte);
+			pte = pte_mkold(pte);
+
+			page_add_new_anon_rmap(uncow_page, vma, addr);
+			rss[!!PageAnon(uncow_page)]++;
+			goto out_set_pte;
+		}
+#endif /* CONFIG_IPIPE */
 		ptep_set_wrprotect(src_mm, addr, src_pte);
 		pte = pte_wrprotect(pte);
 	}
@@ -934,13 +975,27 @@ int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 	int progress = 0;
 	int rss[NR_MM_COUNTERS];
 	swp_entry_t entry = (swp_entry_t){0};
-
+	struct page *uncow_page = NULL;
+#ifdef CONFIG_IPIPE
+	int do_cow_break = 0;
+again:
+ 	if (do_cow_break) {
+ 		uncow_page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
+		if (uncow_page == NULL)
+ 			return -ENOMEM;
+		do_cow_break = 0;
+	}
+#else
 again:
+#endif
 	init_rss_vec(rss);
 
 	dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
-	if (!dst_pte)
+	if (!dst_pte) {
+		if (uncow_page)
+			page_cache_release(uncow_page);
 		return -ENOMEM;
+	}
 	src_pte = pte_offset_map(src_pmd, addr);
 	src_ptl = pte_lockptr(src_mm, src_pmd);
 	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
@@ -963,8 +1018,25 @@ again:
 			progress++;
 			continue;
 		}
+#ifdef CONFIG_IPIPE
+		if (likely(uncow_page == NULL) && likely(pte_present(*src_pte))) {
+			if (is_cow_mapping(vma->vm_flags) &&
+			    test_bit(MMF_VM_PINNED, &src_mm->flags) &&
+			    ((vma->vm_flags|src_mm->def_flags) & VM_LOCKED)) {
+				arch_leave_lazy_mmu_mode();
+				spin_unlock(src_ptl);
+				pte_unmap(src_pte);
+				add_mm_rss_vec(dst_mm, rss);
+				pte_unmap_unlock(dst_pte, dst_ptl);
+				cond_resched();
+				do_cow_break = 1;
+				goto again;
+			}
+		}
+#endif
 		entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte,
-							vma, addr, rss);
+					 vma, addr, rss, uncow_page);
+		uncow_page = NULL;
 		if (entry.val)
 			break;
 		progress += 8;
@@ -2427,32 +2499,6 @@ static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd,
 	return same;
 }
 
-static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
-{
-	/*
-	 * If the source page was a PFN mapping, we don't have
-	 * a "struct page" for it. We do a best-effort copy by
-	 * just copying from the original user address. If that
-	 * fails, we just zero-fill it. Live with it.
-	 */
-	if (unlikely(!src)) {
-		void *kaddr = kmap_atomic(dst, KM_USER0);
-		void __user *uaddr = (void __user *)(va & PAGE_MASK);
-
-		/*
-		 * This really shouldn't fail, because the page is there
-		 * in the page tables. But it might just be unreadable,
-		 * in which case we just give up and fill the result with
-		 * zeroes.
-		 */
-		if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
-			clear_page(kaddr);
-		kunmap_atomic(kaddr, KM_USER0);
-		flush_dcache_page(dst);
-	} else
-		copy_user_highpage(dst, src, va, vma);
-}
-
 /*
  * This routine handles present pages, when users try to write
  * to a shared page. It is done by copying the page to a new address
@@ -3992,3 +4038,117 @@ void copy_user_huge_page(struct page *dst, struct page *src,
 	}
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
+
+#ifdef CONFIG_IPIPE
+
+static inline int ipipe_pin_pte_range(struct mm_struct *mm, pmd_t *pmd,
+				      struct vm_area_struct *vma,
+				      unsigned long addr, unsigned long end)
+{
+	spinlock_t *ptl;
+	pte_t *pte;
+
+	do {
+		pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
+		if (!pte)
+			continue;
+
+		if (!pte_present(*pte) || pte_write(*pte)) {
+			pte_unmap_unlock(pte, ptl);
+			continue;
+		}
+
+		if (do_wp_page(mm, vma, addr, pte, pmd, ptl, *pte) == VM_FAULT_OOM)
+			return -ENOMEM;
+	} while (addr += PAGE_SIZE, addr != end);
+	return 0;
+}
+
+static inline int ipipe_pin_pmd_range(struct mm_struct *mm, pud_t *pud,
+				      struct vm_area_struct *vma,
+				      unsigned long addr, unsigned long end)
+{
+	unsigned long next;
+	pmd_t *pmd;
+
+	pmd = pmd_offset(pud, addr);
+	do {
+		next = pmd_addr_end(addr, end);
+		if (pmd_none_or_clear_bad(pmd))
+			continue;
+		if (ipipe_pin_pte_range(mm, pmd, vma, addr, next))
+			return -ENOMEM;
+	} while (pmd++, addr = next, addr != end);
+	return 0;
+}
+
+static inline int ipipe_pin_pud_range(struct mm_struct *mm, pgd_t *pgd,
+				      struct vm_area_struct *vma,
+				      unsigned long addr, unsigned long end)
+{
+	unsigned long next;
+	pud_t *pud;
+
+	pud = pud_offset(pgd, addr);
+	do {
+		next = pud_addr_end(addr, end);
+		if (pud_none_or_clear_bad(pud))
+			continue;
+		if (ipipe_pin_pmd_range(mm, pud, vma, addr, next))
+			return -ENOMEM;
+	} while (pud++, addr = next, addr != end);
+	return 0;
+}
+
+int __ipipe_pin_vma(struct mm_struct *mm, struct vm_area_struct *vma)
+{
+	unsigned long addr, next, end;
+	pgd_t *pgd;
+
+	addr = vma->vm_start;
+	end = vma->vm_end;
+
+	pgd = pgd_offset(mm, addr);
+	do {
+		next = pgd_addr_end(addr, end);
+		if (pgd_none_or_clear_bad(pgd))
+			continue;
+		if (ipipe_pin_pud_range(mm, pgd, vma, addr, next))
+			return -ENOMEM;
+	} while (pgd++, addr = next, addr != end);
+
+	return 0;
+}
+
+int __ipipe_disable_ondemand_mappings(struct task_struct *tsk)
+{
+	struct vm_area_struct *vma;
+	struct mm_struct *mm;
+	int result = 0;
+
+	mm = get_task_mm(tsk);
+	if (!mm)
+		return -EPERM;
+
+	down_write(&mm->mmap_sem);
+	if (test_bit(MMF_VM_PINNED, &mm->flags))
+		goto done_mm;
+
+	for (vma = mm->mmap; vma; vma = vma->vm_next) {
+		if (is_cow_mapping(vma->vm_flags) &&
+		    (vma->vm_flags & VM_WRITE)) {
+			result = __ipipe_pin_vma(mm, vma);
+			if (result < 0)
+				goto done_mm;
+		}
+	}
+	set_bit(MMF_VM_PINNED, &mm->flags);
+
+  done_mm:
+	up_write(&mm->mmap_sem);
+	mmput(mm);
+	return result;
+}
+EXPORT_SYMBOL_GPL(__ipipe_disable_ondemand_mappings);
+
+#endif
diff --git a/mm/mmu_context.c b/mm/mmu_context.c
index cf332bc..1178028 100644
--- a/mm/mmu_context.c
+++ b/mm/mmu_context.c
@@ -24,15 +24,18 @@ void use_mm(struct mm_struct *mm)
 {
 	struct mm_struct *active_mm;
 	struct task_struct *tsk = current;
+	unsigned long flags;
 
 	task_lock(tsk);
 	active_mm = tsk->active_mm;
+ 	ipipe_mm_switch_protect(flags);
 	if (active_mm != mm) {
 		atomic_inc(&mm->mm_count);
 		tsk->active_mm = mm;
 	}
 	tsk->mm = mm;
-	switch_mm(active_mm, mm, tsk);
+	__switch_mm(active_mm, mm, tsk);
+ 	ipipe_mm_switch_unprotect(flags);
 	task_unlock(tsk);
 
 	if (active_mm != mm)
diff --git a/mm/mprotect.c b/mm/mprotect.c
index 5a688a2..9981d19 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -147,6 +147,7 @@ mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
 {
 	struct mm_struct *mm = vma->vm_mm;
 	unsigned long oldflags = vma->vm_flags;
+	unsigned long protflags;
 	long nrpages = (end - start) >> PAGE_SHIFT;
 	unsigned long charged = 0;
 	pgoff_t pgoff;
@@ -205,8 +206,17 @@ success:
 	 * held in write mode.
 	 */
 	vma->vm_flags = newflags;
+	protflags = newflags;
+#ifdef CONFIG_IPIPE
+	/*
+	 * Enforce non-COW vm_page_prot by faking VM_SHARED on locked regions.
+	 */
+	if (test_bit(MMF_VM_PINNED, &mm->flags) &&
+	    ((vma->vm_flags | mm->def_flags) & VM_LOCKED))
+		protflags |= VM_SHARED;
+#endif
 	vma->vm_page_prot = pgprot_modify(vma->vm_page_prot,
-					  vm_get_page_prot(newflags));
+					  vm_get_page_prot(protflags));
 
 	if (vma_wants_writenotify(vma)) {
 		vma->vm_page_prot = vm_get_page_prot(newflags & ~VM_SHARED);
@@ -214,6 +224,24 @@ success:
 	}
 
 	mmu_notifier_invalidate_range_start(mm, start, end);
+#ifdef CONFIG_IPIPE
+	/*
+	 * Privatize potential COW pages
+	 */
+	if (test_bit(MMF_VM_PINNED, &mm->flags) &&
+	    (((vma->vm_flags | mm->def_flags) & (VM_LOCKED | VM_WRITE)) ==
+	     (VM_LOCKED | VM_WRITE))) {
+		error = __ipipe_pin_vma(mm, vma);
+		if (error)
+			/*
+			 * OOM. Just revert the fake VM_SHARED so that the
+			 * zero page cannot be overwritten.
+			 */
+			vma->vm_page_prot =
+				pgprot_modify(vma->vm_page_prot,
+					      vm_get_page_prot(newflags));
+	}
+#endif
 	if (is_vm_hugetlb_page(vma))
 		hugetlb_change_protection(vma, start, end, vma->vm_page_prot);
 	else
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index eeba3bb..7b2ae6d 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -171,6 +171,8 @@ static int vmap_page_range_noflush(unsigned long start, unsigned long end,
 			return err;
 	} while (pgd++, addr = next, addr != end);
 
+ 	__ipipe_pin_range_globally(start, end);
+ 
 	return nr;
 }
 
