diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 166efa2..1a6f2c4 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -51,6 +51,9 @@ config HAVE_SCHED_CLOCK
 config GENERIC_GPIO
 	bool
 
+config GENERIC_TIME_VSYSCALL
+	bool
+
 config ARCH_USES_GETTIMEOFFSET
 	bool
 	default n
@@ -1031,6 +1034,14 @@ config PLAT_VERSATILE
 config ARM_TIMER_SP804
 	bool
 
+if IPIPE
+config IPIPE_ARM_KUSER_TSC
+       bool
+       select GENERIC_TIME_VSYSCALL
+       select HAVE_IPIPE_HOSTRT
+       default y if ARCH_AT91 || ARM_TIMER_SP804 || ARCH_MXC || ARCH_OMAP || PLAT_PXA || PLAT_S3C24XX || ARCH_SA1100
+endif
+
 source arch/arm/mm/Kconfig
 
 config IWMMXT
@@ -1373,6 +1384,8 @@ config LOCAL_TIMERS
 	  accounting to be spread across the timer interval, preventing a
 	  "thundering herd" at every timer tick.
 
+source kernel/ipipe/Kconfig
+
 source kernel/Kconfig.preempt
 
 config HZ
diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S
index 7193884..1b97c91 100644
--- a/arch/arm/boot/compressed/head.S
+++ b/arch/arm/boot/compressed/head.S
@@ -1087,6 +1087,15 @@ memdump:	mov	r12, r0
 		mov	pc, r10
 #endif
 
+#ifdef CONFIG_IPIPE_TRACE_MCOUNT
+		.text
+		.align 0
+		.type mcount %function
+		.global mcount
+mcount:
+		mov pc, lr	@ just return
+#endif
+
 		.ltorg
 reloc_end:
 
diff --git a/arch/arm/common/gic.c b/arch/arm/common/gic.c
index 2243772..f42d628 100644
--- a/arch/arm/common/gic.c
+++ b/arch/arm/common/gic.c
@@ -28,12 +28,14 @@
 #include <linux/smp.h>
 #include <linux/cpumask.h>
 #include <linux/io.h>
+#include <linux/spinlock.h>
 
 #include <asm/irq.h>
 #include <asm/mach/irq.h>
 #include <asm/hardware/gic.h>
+#include <asm/ipipe.h>
 
-static DEFINE_SPINLOCK(irq_controller_lock);
+static IPIPE_DEFINE_SPINLOCK(irq_controller_lock);
 
 /* Address of GIC 0 CPU interface */
 void __iomem *gic_cpu_base_addr __read_mostly;
@@ -73,27 +75,42 @@ static inline unsigned int gic_irq(struct irq_data *d)
  */
 static void gic_ack_irq(struct irq_data *d)
 {
-	spin_lock(&irq_controller_lock);
-	writel(gic_irq(d), gic_cpu_base(d) + GIC_CPU_EOI);
-	spin_unlock(&irq_controller_lock);
+	unsigned long flags;
+
+	spin_lock_irqsave_cond(&irq_controller_lock, flags);
+	writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_EOI);
+	spin_unlock_irqrestore_cond(&irq_controller_lock, flags);
 }
 
 static void gic_mask_irq(struct irq_data *d)
 {
 	u32 mask = 1 << (d->irq % 32);
+	unsigned long flags;
 
-	spin_lock(&irq_controller_lock);
-	writel(mask, gic_dist_base(d) + GIC_DIST_ENABLE_CLEAR + (gic_irq(d) / 32) * 4);
-	spin_unlock(&irq_controller_lock);
+	spin_lock_irqsave_cond(&irq_controller_lock, flags);
+	writel_relaxed(mask, gic_dist_base(d) + GIC_DIST_ENABLE_CLEAR + (gic_irq(d) / 32) * 4);
+	spin_unlock_irqrestore_cond(&irq_controller_lock, flags);
+}
+
+static void gic_mask_ack_irq(struct irq_data *d)
+{
+	u32 mask = 1 << (d->irq % 32);
+	unsigned long flags;
+
+	spin_lock_irqsave_cond(&irq_controller_lock, flags);
+	writel_relaxed(mask, gic_dist_base(d) + GIC_DIST_ENABLE_CLEAR + (gic_irq(d) / 32) * 4);
+	writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_EOI);
+	spin_unlock_irqrestore_cond(&irq_controller_lock, flags);
 }
 
 static void gic_unmask_irq(struct irq_data *d)
 {
 	u32 mask = 1 << (d->irq % 32);
+	unsigned long flags;
 
-	spin_lock(&irq_controller_lock);
-	writel(mask, gic_dist_base(d) + GIC_DIST_ENABLE_SET + (gic_irq(d) / 32) * 4);
-	spin_unlock(&irq_controller_lock);
+	spin_lock_irqsave_cond(&irq_controller_lock, flags);
+	writel_relaxed(mask, gic_dist_base(d) + GIC_DIST_ENABLE_SET + (gic_irq(d) / 32) * 4);
+	spin_unlock_irqrestore_cond(&irq_controller_lock, flags);
 }
 
 static int gic_set_type(struct irq_data *d, unsigned int type)
@@ -105,6 +122,7 @@ static int gic_set_type(struct irq_data *d, unsigned int type)
 	u32 confmask = 0x2 << ((gicirq % 16) * 2);
 	u32 confoff = (gicirq / 16) * 4;
 	bool enabled = false;
+	unsigned long flags;
 	u32 val;
 
 	/* Interrupt configuration for SGIs can't be changed */
@@ -114,9 +132,9 @@ static int gic_set_type(struct irq_data *d, unsigned int type)
 	if (type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING)
 		return -EINVAL;
 
-	spin_lock(&irq_controller_lock);
+	spin_lock_irqsave_cond(&irq_controller_lock, flags);
 
-	val = readl(base + GIC_DIST_CONFIG + confoff);
+	val = readl_relaxed(base + GIC_DIST_CONFIG + confoff);
 	if (type == IRQ_TYPE_LEVEL_HIGH)
 		val &= ~confmask;
 	else if (type == IRQ_TYPE_EDGE_RISING)
@@ -126,17 +144,17 @@ static int gic_set_type(struct irq_data *d, unsigned int type)
 	 * As recommended by the spec, disable the interrupt before changing
 	 * the configuration
 	 */
-	if (readl(base + GIC_DIST_ENABLE_SET + enableoff) & enablemask) {
-		writel(enablemask, base + GIC_DIST_ENABLE_CLEAR + enableoff);
+	if (readl_relaxed(base + GIC_DIST_ENABLE_SET + enableoff) & enablemask) {
+		writel_relaxed(enablemask, base + GIC_DIST_ENABLE_CLEAR + enableoff);
 		enabled = true;
 	}
 
-	writel(val, base + GIC_DIST_CONFIG + confoff);
+	writel_relaxed(val, base + GIC_DIST_CONFIG + confoff);
 
 	if (enabled)
-		writel(enablemask, base + GIC_DIST_ENABLE_SET + enableoff);
+		writel_relaxed(enablemask, base + GIC_DIST_ENABLE_SET + enableoff);
 
-	spin_unlock(&irq_controller_lock);
+	spin_unlock_irqrestore_cond(&irq_controller_lock, flags);
 
 	return 0;
 }
@@ -150,18 +168,19 @@ gic_set_cpu(struct irq_data *d, const struct cpumask *mask_val, bool force)
 	unsigned int cpu = cpumask_first(mask_val);
 	u32 val;
 	struct irq_desc *desc;
+	unsigned long flags;
 
-	spin_lock(&irq_controller_lock);
+	spin_lock_irqsave_cond(&irq_controller_lock, flags);
 	desc = irq_to_desc(d->irq);
 	if (desc == NULL) {
-		spin_unlock(&irq_controller_lock);
+		spin_unlock_irqrestore_cond(&irq_controller_lock, flags);
 		return -EINVAL;
 	}
 	d->node = cpu;
-	val = readl(reg) & ~(0xff << shift);
+	val = readl_relaxed(reg) & ~(0xff << shift);
 	val |= 1 << (cpu + shift);
-	writel(val, reg);
-	spin_unlock(&irq_controller_lock);
+	writel_relaxed(val, reg);
+	spin_unlock_irqrestore_cond(&irq_controller_lock, flags);
 
 	return 0;
 }
@@ -172,14 +191,14 @@ static void gic_handle_cascade_irq(unsigned int irq, struct irq_desc *desc)
 	struct gic_chip_data *chip_data = get_irq_data(irq);
 	struct irq_chip *chip = get_irq_chip(irq);
 	unsigned int cascade_irq, gic_irq;
-	unsigned long status;
+	unsigned long status, flags;
 
 	/* primary controller ack'ing */
 	chip->irq_ack(&desc->irq_data);
 
-	spin_lock(&irq_controller_lock);
-	status = readl(chip_data->cpu_base + GIC_CPU_INTACK);
-	spin_unlock(&irq_controller_lock);
+	spin_lock_irqsave_cond(&irq_controller_lock, flags);
+	status = readl_relaxed(chip_data->cpu_base + GIC_CPU_INTACK);
+	spin_unlock_irqrestore_cond(&irq_controller_lock, flags);
 
 	gic_irq = (status & 0x3ff);
 	if (gic_irq == 1023)
@@ -189,7 +208,7 @@ static void gic_handle_cascade_irq(unsigned int irq, struct irq_desc *desc)
 	if (unlikely(gic_irq < 32 || gic_irq > 1020 || cascade_irq >= NR_IRQS))
 		do_bad_IRQ(cascade_irq, desc);
 	else
-		generic_handle_irq(cascade_irq);
+		ipipe_handle_chained_irq(cascade_irq);
 
  out:
 	/* primary controller unmasking */
@@ -200,6 +219,9 @@ static struct irq_chip gic_chip = {
 	.name			= "GIC",
 	.irq_ack		= gic_ack_irq,
 	.irq_mask		= gic_mask_irq,
+#ifdef CONFIG_IPIPE
+	.irq_mask_ack		= gic_mask_ack_irq,
+#endif
 	.irq_unmask		= gic_unmask_irq,
 	.irq_set_type		= gic_set_type,
 #ifdef CONFIG_SMP
@@ -216,6 +238,38 @@ void __init gic_cascade_irq(unsigned int gic_nr, unsigned int irq)
 	set_irq_chained_handler(irq, gic_handle_cascade_irq);
 }
 
+#if defined(CONFIG_IPIPE) && defined(__IPIPE_FEATURE_PIC_MUTE)
+DECLARE_PER_CPU(__ipipe_irqbits_t, __ipipe_muted_irqs);
+
+void gic_mute(void)
+{
+	writel_relaxed(0x90, gic_data[0].cpu_base + GIC_CPU_PRIMASK);
+}
+
+void gic_unmute(void)
+{
+	writel_relaxed(0xf0, gic_data[0].cpu_base + GIC_CPU_PRIMASK);
+}
+
+void gic_set_irq_prio(int irq, int hi)
+{
+	void __iomem *dist_base;
+	unsigned gic_irqs;
+
+	if (irq < 32) /* The IPIs always are high priority */
+		return;
+
+	dist_base = gic_data[0].dist_base;
+	gic_irqs = readl_relaxed(dist_base + GIC_DIST_CTR) & 0x1f;
+	gic_irqs = (gic_irqs + 1) * 32;
+	if (gic_irqs > 1020)
+		gic_irqs = 1020;
+	if (irq >= gic_irqs)
+		return;
+
+	writeb_relaxed(hi ? 0x10 : 0xa0, dist_base + GIC_DIST_PRI + irq);
+}
+#endif /* __IPIPE_FEATURE_PIC_MUTE */
 static void __init gic_dist_init(struct gic_chip_data *gic,
 	unsigned int irq_start)
 {
@@ -226,13 +280,13 @@ static void __init gic_dist_init(struct gic_chip_data *gic,
 	cpumask |= cpumask << 8;
 	cpumask |= cpumask << 16;
 
-	writel(0, base + GIC_DIST_CTRL);
+	writel_relaxed(0, base + GIC_DIST_CTRL);
 
 	/*
 	 * Find out how many interrupts are supported.
 	 * The GIC only supports up to 1020 interrupt sources.
 	 */
-	gic_irqs = readl(base + GIC_DIST_CTR) & 0x1f;
+	gic_irqs = readl_relaxed(base + GIC_DIST_CTR) & 0x1f;
 	gic_irqs = (gic_irqs + 1) * 32;
 	if (gic_irqs > 1020)
 		gic_irqs = 1020;
@@ -241,26 +295,26 @@ static void __init gic_dist_init(struct gic_chip_data *gic,
 	 * Set all global interrupts to be level triggered, active low.
 	 */
 	for (i = 32; i < gic_irqs; i += 16)
-		writel(0, base + GIC_DIST_CONFIG + i * 4 / 16);
+		writel_relaxed(0, base + GIC_DIST_CONFIG + i * 4 / 16);
 
 	/*
 	 * Set all global interrupts to this CPU only.
 	 */
 	for (i = 32; i < gic_irqs; i += 4)
-		writel(cpumask, base + GIC_DIST_TARGET + i * 4 / 4);
+		writel_relaxed(cpumask, base + GIC_DIST_TARGET + i * 4 / 4);
 
 	/*
 	 * Set priority on all global interrupts.
 	 */
 	for (i = 32; i < gic_irqs; i += 4)
-		writel(0xa0a0a0a0, base + GIC_DIST_PRI + i * 4 / 4);
+		writel_relaxed(0xa0a0a0a0, base + GIC_DIST_PRI + i * 4 / 4);
 
 	/*
 	 * Disable all interrupts.  Leave the PPI and SGIs alone
 	 * as these enables are banked registers.
 	 */
 	for (i = 32; i < gic_irqs; i += 32)
-		writel(0xffffffff, base + GIC_DIST_ENABLE_CLEAR + i * 4 / 32);
+		writel_relaxed(0xffffffff, base + GIC_DIST_ENABLE_CLEAR + i * 4 / 32);
 
 	/*
 	 * Limit number of interrupts registered to the platform maximum
@@ -279,7 +333,7 @@ static void __init gic_dist_init(struct gic_chip_data *gic,
 		set_irq_flags(i, IRQF_VALID | IRQF_PROBE);
 	}
 
-	writel(1, base + GIC_DIST_CTRL);
+	writel_relaxed(1, base + GIC_DIST_CTRL);
 }
 
 static void __cpuinit gic_cpu_init(struct gic_chip_data *gic)
@@ -292,17 +346,21 @@ static void __cpuinit gic_cpu_init(struct gic_chip_data *gic)
 	 * Deal with the banked PPI and SGI interrupts - disable all
 	 * PPI interrupts, ensure all SGI interrupts are enabled.
 	 */
-	writel(0xffff0000, dist_base + GIC_DIST_ENABLE_CLEAR);
-	writel(0x0000ffff, dist_base + GIC_DIST_ENABLE_SET);
+	writel_relaxed(0xffff0000, dist_base + GIC_DIST_ENABLE_CLEAR);
+	writel_relaxed(0x0000ffff, dist_base + GIC_DIST_ENABLE_SET);
 
 	/*
 	 * Set priority on PPI and SGI interrupts
 	 */
 	for (i = 0; i < 32; i += 4)
-		writel(0xa0a0a0a0, dist_base + GIC_DIST_PRI + i * 4 / 4);
-
-	writel(0xf0, base + GIC_CPU_PRIMASK);
-	writel(1, base + GIC_CPU_CTRL);
+#if !defined(CONFIG_IPIPE) || !defined(__IPIPE_FEATURE_PIC_MUTE)
+		writel_relaxed(0xa0a0a0a0, dist_base + GIC_DIST_PRI + i * 4 / 4);
+#else /* IPIPE && FEATURE_PIC_MUTE */
+		writel_relaxed(0x10101010, dist_base + GIC_DIST_PRI + i * 4 / 4);
+#endif /* IPIPE && FEATURE_PIC_MUTE */
+
+	writel_relaxed(0xf0, base + GIC_CPU_PRIMASK);
+	writel_relaxed(1, base + GIC_CPU_CTRL);
 }
 
 void __init gic_init(unsigned int gic_nr, unsigned int irq_start,
@@ -347,6 +405,6 @@ void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
 	unsigned long map = *cpus_addr(*mask);
 
 	/* this always happens on GIC0 */
-	writel(map << 16 | irq, gic_data[0].dist_base + GIC_DIST_SOFTINT);
+	writel_relaxed(map << 16 | irq, gic_data[0].dist_base + GIC_DIST_SOFTINT);
 }
 #endif
diff --git a/arch/arm/common/it8152.c b/arch/arm/common/it8152.c
index fcddd48..fef7b1e 100644
--- a/arch/arm/common/it8152.c
+++ b/arch/arm/common/it8152.c
@@ -25,6 +25,7 @@
 #include <linux/ioport.h>
 #include <linux/irq.h>
 #include <linux/io.h>
+#include <linux/ipipe.h>
 
 #include <asm/mach/pci.h>
 #include <asm/hardware/it8152.h>
@@ -123,21 +124,21 @@ void it8152_irq_demux(unsigned int irq, struct irq_desc *desc)
 	       bits_pd &= ((1 << IT8152_PD_IRQ_COUNT) - 1);
 	       while (bits_pd) {
 		       i = __ffs(bits_pd);
-		       generic_handle_irq(IT8152_PD_IRQ(i));
+		       ipipe_handle_chained_irq(IT8152_PD_IRQ(i));
 		       bits_pd &= ~(1 << i);
 	       }
 
 	       bits_lp &= ((1 << IT8152_LP_IRQ_COUNT) - 1);
 	       while (bits_lp) {
 		       i = __ffs(bits_lp);
-		       generic_handle_irq(IT8152_LP_IRQ(i));
+		       ipipe_handle_chained_irq(IT8152_LP_IRQ(i));
 		       bits_lp &= ~(1 << i);
 	       }
 
 	       bits_ld &= ((1 << IT8152_LD_IRQ_COUNT) - 1);
 	       while (bits_ld) {
 		       i = __ffs(bits_ld);
-		       generic_handle_irq(IT8152_LD_IRQ(i));
+		       ipipe_handle_chained_irq(IT8152_LD_IRQ(i));
 		       bits_ld &= ~(1 << i);
 	       }
        }
diff --git a/arch/arm/common/timer-sp.c b/arch/arm/common/timer-sp.c
index 6ef3342..c46ab1f 100644
--- a/arch/arm/common/timer-sp.c
+++ b/arch/arm/common/timer-sp.c
@@ -23,6 +23,8 @@
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/io.h>
+#include <linux/module.h>
+#include <linux/ipipe.h>
 
 #include <asm/hardware/arm_timer.h>
 
@@ -34,6 +36,26 @@
 
 static void __iomem *clksrc_base;
 
+#ifdef CONFIG_IPIPE
+int __ipipe_mach_timerint;
+EXPORT_SYMBOL(__ipipe_mach_timerint);
+
+int __ipipe_mach_timerstolen;
+EXPORT_SYMBOL(__ipipe_mach_timerstolen);
+
+unsigned int __ipipe_mach_ticks_per_jiffy;
+EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy);
+
+static struct __ipipe_tscinfo tsc_info = {
+	.type = IPIPE_TSC_TYPE_FREERUNNING_COUNTDOWN,
+	.u = {
+		{
+			.mask = 0xffffffff,
+		},
+	},
+};
+#endif /* CONFIG_IPIPE */
+
 static cycle_t sp804_read(struct clocksource *cs)
 {
 	return ~readl(clksrc_base + TIMER_VALUE);
@@ -47,7 +69,7 @@ static struct clocksource clocksource_sp804 = {
 	.flags		= CLOCK_SOURCE_IS_CONTINUOUS,
 };
 
-void __init sp804_clocksource_init(void __iomem *base)
+void __init sp804_clocksource_init(void __iomem *base, unsigned long phys)
 {
 	struct clocksource *cs = &clocksource_sp804;
 
@@ -60,6 +82,13 @@ void __init sp804_clocksource_init(void __iomem *base)
 	writel(TIMER_CTRL_32BIT | TIMER_CTRL_ENABLE | TIMER_CTRL_PERIODIC,
 		clksrc_base + TIMER_CTRL);
 
+#ifdef CONFIG_IPIPE
+	tsc_info.freq = TIMER_FREQ_KHZ * 1000;
+	tsc_info.counter_vaddr = (unsigned long)base + TIMER_VALUE;
+	tsc_info.u.counter_paddr = phys + TIMER_VALUE;
+	__ipipe_tsc_register(&tsc_info);
+#endif
+
 	clocksource_register_khz(cs, TIMER_FREQ_KHZ);
 }
 
@@ -73,8 +102,12 @@ static irqreturn_t sp804_timer_interrupt(int irq, void *dev_id)
 {
 	struct clock_event_device *evt = dev_id;
 
+#ifndef CONFIG_IPIPE
 	/* clear the interrupt */
 	writel(1, clkevt_base + TIMER_INTCLR);
+#else /* CONFIG_IPIPE */
+	__ipipe_tsc_update();
+#endif /* CONFIG_IPIPE */
 
 	evt->event_handler(evt);
 
@@ -147,6 +180,51 @@ void __init sp804_clockevents_init(void __iomem *base, unsigned int timer_irq)
 	evt->max_delta_ns = clockevent_delta2ns(0xffffffff, evt);
 	evt->min_delta_ns = clockevent_delta2ns(0xf, evt);
 
+#ifdef CONFIG_IPIPE
+	__ipipe_mach_timerint = timer_irq;
+	__ipipe_mach_ticks_per_jiffy = (TIMER_FREQ_KHZ * 1000 + HZ / 2) / HZ;
+#endif /* CONFIG_IPIPE */
+
 	setup_irq(timer_irq, &sp804_timer_irq);
 	clockevents_register_device(evt);
 }
+
+#ifdef CONFIG_IPIPE
+int __ipipe_check_tickdev(const char *devname)
+{
+	/* Keep compatibility with old patches */
+	return !strcmp(devname, "TIMER1");
+}
+
+void __ipipe_mach_acktimer(void)
+{
+	writel(1, clkevt_base + TIMER_INTCLR);
+}
+
+/*
+ * Reprogram the timer
+ */
+
+void __ipipe_mach_set_dec(unsigned long delay)
+{
+	if (delay > 0xf)
+		sp804_set_next_event(delay, NULL);
+	else
+		ipipe_trigger_irq(__ipipe_mach_timerint);
+}
+EXPORT_SYMBOL(__ipipe_mach_set_dec);
+
+void __ipipe_mach_release_timer(void)
+{
+	struct clock_event_device *ckdev = &sp804_clockevent;
+	ckdev->set_mode(ckdev->mode, ckdev);
+	if (ckdev->mode == CLOCK_EVT_MODE_ONESHOT)
+		ckdev->set_next_event(__ipipe_mach_ticks_per_jiffy, ckdev);
+}
+EXPORT_SYMBOL(__ipipe_mach_release_timer);
+
+unsigned long __ipipe_mach_get_dec(void)
+{
+	return ~readl(clkevt_base + TIMER_VALUE);
+}
+#endif /* CONFIG_IPIPE */
diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h
index bc2d2d7..780942a 100644
--- a/arch/arm/include/asm/assembler.h
+++ b/arch/arm/include/asm/assembler.h
@@ -82,6 +82,18 @@
 	.macro	enable_irq_notrace
 	cpsie	i
 	.endm
+
+	.macro  disable_irq_cond
+#ifdef CONFIG_IPIPE
+	cpsid	i
+#endif /* CONFIG_IPIPE */
+	.endm
+
+	.macro  enable_irq_cond
+#ifdef CONFIG_IPIPE
+	cpsie	i
+#endif /* CONFIG_IPIPE */
+	.endm
 #else
 	.macro	disable_irq_notrace
 	msr	cpsr_c, #PSR_I_BIT | SVC_MODE
@@ -90,6 +102,18 @@
 	.macro	enable_irq_notrace
 	msr	cpsr_c, #SVC_MODE
 	.endm
+
+	.macro	disable_irq_cond
+#ifdef CONFIG_IPIPE
+	msr	cpsr_c, #PSR_I_BIT | SVC_MODE
+#endif /* CONFIG_IPIPE */
+	.endm
+
+	.macro	enable_irq_cond
+#ifdef CONFIG_IPIPE
+	msr	cpsr_c, #SVC_MODE
+#endif /* CONFIG_IPIPE */
+	.endm
 #endif
 
 	.macro asm_trace_hardirqs_off
diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h
index 7e79503..e584db9 100644
--- a/arch/arm/include/asm/atomic.h
+++ b/arch/arm/include/asm/atomic.h
@@ -158,10 +158,10 @@ static inline int atomic_add_return(int i, atomic_t *v)
 	unsigned long flags;
 	int val;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	val = v->counter;
 	v->counter = val += i;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return val;
 }
@@ -172,10 +172,10 @@ static inline int atomic_sub_return(int i, atomic_t *v)
 	unsigned long flags;
 	int val;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	val = v->counter;
 	v->counter = val -= i;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return val;
 }
@@ -186,11 +186,11 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
 	int ret;
 	unsigned long flags;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	ret = v->counter;
 	if (likely(ret == old))
 		v->counter = new;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return ret;
 }
@@ -199,9 +199,9 @@ static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
 {
 	unsigned long flags;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	*addr &= ~mask;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 }
 
 #endif /* __LINUX_ARM_ARCH__ */
diff --git a/arch/arm/include/asm/bitops.h b/arch/arm/include/asm/bitops.h
index 7b1bb2b..40da67b 100644
--- a/arch/arm/include/asm/bitops.h
+++ b/arch/arm/include/asm/bitops.h
@@ -41,9 +41,9 @@ static inline void ____atomic_set_bit(unsigned int bit, volatile unsigned long *
 
 	p += bit >> 5;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	*p |= mask;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 }
 
 static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long *p)
@@ -53,9 +53,9 @@ static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long
 
 	p += bit >> 5;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	*p &= ~mask;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 }
 
 static inline void ____atomic_change_bit(unsigned int bit, volatile unsigned long *p)
@@ -65,9 +65,9 @@ static inline void ____atomic_change_bit(unsigned int bit, volatile unsigned lon
 
 	p += bit >> 5;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	*p ^= mask;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 }
 
 static inline int
@@ -79,10 +79,10 @@ ____atomic_test_and_set_bit(unsigned int bit, volatile unsigned long *p)
 
 	p += bit >> 5;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	res = *p;
 	*p = res | mask;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return (res & mask) != 0;
 }
@@ -96,10 +96,10 @@ ____atomic_test_and_clear_bit(unsigned int bit, volatile unsigned long *p)
 
 	p += bit >> 5;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	res = *p;
 	*p = res & ~mask;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return (res & mask) != 0;
 }
@@ -113,10 +113,10 @@ ____atomic_test_and_change_bit(unsigned int bit, volatile unsigned long *p)
 
 	p += bit >> 5;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	res = *p;
 	*p = res ^ mask;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return (res & mask) != 0;
 }
diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h
index 3acd8fa..fc3bb73 100644
--- a/arch/arm/include/asm/cacheflush.h
+++ b/arch/arm/include/asm/cacheflush.h
@@ -16,6 +16,7 @@
 #include <asm/shmparam.h>
 #include <asm/cachetype.h>
 #include <asm/outercache.h>
+#include <asm/fcse.h>
 
 #define CACHE_COLOUR(vaddr)	((vaddr & (SHMLBA - 1)) >> PAGE_SHIFT)
 
@@ -286,6 +287,27 @@ extern void dmac_flush_range(const void *, const void *);
 
 #endif
 
+#ifdef CONFIG_ARM_FCSE
+#define FCSE_CACHE_MASK (~(L1_CACHE_BYTES - 1))
+#define FCSE_CACHE_ALIGN(addr) (((addr) + ~FCSE_CACHE_MASK) & FCSE_CACHE_MASK)
+
+static inline void
+fcse_flush_cache_user_range(struct vm_area_struct *vma,
+			    unsigned long start, unsigned long end)
+{
+	if (cache_is_vivt()
+	    && fcse_mm_in_cache(vma->vm_mm)) {
+		start = fcse_va_to_mva(vma->vm_mm, start & FCSE_CACHE_MASK);
+		end = fcse_va_to_mva(vma->vm_mm, FCSE_CACHE_ALIGN(end));
+		__cpuc_flush_user_range(start, end, vma->vm_flags);
+	}
+}
+#undef FCSE_CACHE_MASK
+#undef FCSE_CACHE_ALIGN
+#else /* ! CONFIG_ARM_FCSE */
+#define fcse_flush_cache_user_range(vma, start, end) do { } while (0)
+#endif /* ! CONFIG_ARM_FCSE */
+
 /*
  * Copy user data from/to a page which is mapped into a different
  * processes address space.  Really, we want to allow our "user
@@ -293,9 +315,10 @@ extern void dmac_flush_range(const void *, const void *);
  */
 extern void copy_to_user_page(struct vm_area_struct *, struct page *,
 	unsigned long, void *, const void *, unsigned long);
-#define copy_from_user_page(vma, page, vaddr, dst, src, len) \
-	do {							\
-		memcpy(dst, src, len);				\
+#define copy_from_user_page(vma, page, vaddr, dst, src, len)		\
+	do {								\
+		fcse_flush_cache_user_range(vma, vaddr, vaddr + len);	\
+		memcpy(dst, src, len);					\
 	} while (0)
 
 /*
@@ -336,23 +359,29 @@ static inline void __flush_icache_all(void)
 
 static inline void vivt_flush_cache_mm(struct mm_struct *mm)
 {
-	if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm)))
+	if (fcse_mm_in_cache(mm)) {
+		unsigned seq = fcse_flush_all_start();
 		__cpuc_flush_user_all();
+		fcse_flush_all_done(seq, 1);
+	}
 }
 
 static inline void
 vivt_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
 {
-	if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm)))
-		__cpuc_flush_user_range(start & PAGE_MASK, PAGE_ALIGN(end),
-					vma->vm_flags);
+	if (fcse_mm_in_cache(vma->vm_mm)) {
+		start = fcse_va_to_mva(vma->vm_mm, start & PAGE_MASK);
+		end = fcse_va_to_mva(vma->vm_mm, PAGE_ALIGN(end));
+		__cpuc_flush_user_range(start, end, vma->vm_flags);
+	}
 }
 
 static inline void
 vivt_flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn)
 {
-	if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) {
-		unsigned long addr = user_addr & PAGE_MASK;
+	if (fcse_mm_in_cache(vma->vm_mm)) {
+		unsigned long addr;
+		addr = fcse_va_to_mva(vma->vm_mm, user_addr) & PAGE_MASK;
 		__cpuc_flush_user_range(addr, addr + PAGE_SIZE, vma->vm_flags);
 	}
 }
@@ -377,14 +406,22 @@ extern void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr
  * Harvard caches are synchronised for the user space address range.
  * This is used for the ARM private sys_cacheflush system call.
  */
-#define flush_cache_user_range(vma,start,end) \
-	__cpuc_coherent_user_range((start) & PAGE_MASK, PAGE_ALIGN(end))
+#define flush_cache_user_range(vma, start, end)				\
+	({								\
+		struct mm_struct *_mm = (vma)->vm_mm;			\
+		unsigned long _start, _end;				\
+		_start = fcse_va_to_mva(_mm, start) & PAGE_MASK;	\
+		_end = PAGE_ALIGN(fcse_va_to_mva(_mm, end));		\
+		__cpuc_coherent_user_range(_start, _end);		\
+	})
 
 /*
  * Perform necessary cache operations to ensure that data previously
  * stored within this range of addresses can be executed by the CPU.
  */
-#define flush_icache_range(s,e)		__cpuc_coherent_kern_range(s,e)
+#define flush_icache_range(s,e)						\
+	__cpuc_coherent_kern_range(fcse_va_to_mva(current->mm, (s)),	\
+				   fcse_va_to_mva(current->mm, (e)))
 
 /*
  * Perform necessary cache operations to ensure that the TLB will
@@ -425,7 +462,8 @@ static inline void flush_anon_page(struct vm_area_struct *vma,
 	extern void __flush_anon_page(struct vm_area_struct *vma,
 				struct page *, unsigned long);
 	if (PageAnon(page))
-		__flush_anon_page(vma, page, vmaddr);
+		__flush_anon_page(vma, page,
+				  fcse_va_to_mva(vma->vm_mm, vmaddr));
 }
 
 #define ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE
@@ -456,9 +494,11 @@ static inline void flush_kernel_dcache_page(struct page *page)
  */
 static inline void flush_cache_vmap(unsigned long start, unsigned long end)
 {
-	if (!cache_is_vipt_nonaliasing())
+	if (!cache_is_vipt_nonaliasing()) {
+		unsigned seq = fcse_flush_all_start();
 		flush_cache_all();
-	else
+		fcse_flush_all_done(seq, 1);
+	} else
 		/*
 		 * set_pte_at() called from vmap_pte_range() does not
 		 * have a DSB after cleaning the cache line.
@@ -468,8 +508,11 @@ static inline void flush_cache_vmap(unsigned long start, unsigned long end)
 
 static inline void flush_cache_vunmap(unsigned long start, unsigned long end)
 {
-	if (!cache_is_vipt_nonaliasing())
+	if (!cache_is_vipt_nonaliasing()) {
+		unsigned seq = fcse_flush_all_start();
 		flush_cache_all();
+		fcse_flush_all_done(seq, 1);
+	}
 }
 
 #endif
diff --git a/arch/arm/include/asm/cpu-multi32.h b/arch/arm/include/asm/cpu-multi32.h
index e2b5b0b..af6d49c 100644
--- a/arch/arm/include/asm/cpu-multi32.h
+++ b/arch/arm/include/asm/cpu-multi32.h
@@ -52,7 +52,13 @@ extern struct processor {
 	/*
 	 * Set the page table
 	 */
+#ifndef CONFIG_ARM_FCSE_BEST_EFFORT
 	void (*switch_mm)(unsigned long pgd_phys, struct mm_struct *mm);
+#else /* !CONFIG_ARM_FCSE_BEST_EFFORT */
+	void (*switch_mm)(unsigned long pgd_phys,
+			  struct mm_struct *mm, unsigned flush);
+#endif /* !CONFIG_ARM_FCSE_BEST_EFFORT */
+
 	/*
 	 * Set a possibly extended PTE.  Non-extended PTEs should
 	 * ignore 'ext'.
@@ -66,4 +72,8 @@ extern struct processor {
 #define cpu_do_idle()			processor._do_idle()
 #define cpu_dcache_clean_area(addr,sz)	processor.dcache_clean_area(addr,sz)
 #define cpu_set_pte_ext(ptep,pte,ext)	processor.set_pte_ext(ptep,pte,ext)
+#ifndef CONFIG_ARM_FCSE_BEST_EFFORT
 #define cpu_do_switch_mm(pgd,mm)	processor.switch_mm(pgd,mm)
+#else /* !CONFIG_ARM_FCSE_BEST_EFFORT */
+#define cpu_do_switch_mm(pgd,mm,flush)	processor.switch_mm(pgd,mm,flush)
+#endif /* !CONFIG_ARM_FCSE_BEST_EFFORT */
diff --git a/arch/arm/include/asm/cpu-single.h b/arch/arm/include/asm/cpu-single.h
index f073a6d..b6b6314 100644
--- a/arch/arm/include/asm/cpu-single.h
+++ b/arch/arm/include/asm/cpu-single.h
@@ -39,6 +39,11 @@ extern void cpu_proc_init(void);
 extern void cpu_proc_fin(void);
 extern int cpu_do_idle(void);
 extern void cpu_dcache_clean_area(void *, int);
+#ifndef CONFIG_ARM_FCSE_BEST_EFFORT
 extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm);
+#else /* !CONFIG_ARM_FCSE_BEST_EFFORT */
+extern void cpu_do_switch_mm(unsigned long pgd_phys,
+			     struct mm_struct *mm, unsigned flush);
+#endif /* !CONFIG_ARM_FCSE_BEST_EFFORT */
 extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext);
 extern void cpu_reset(unsigned long addr) __attribute__((noreturn));
diff --git a/arch/arm/include/asm/entry-macro-multi.S b/arch/arm/include/asm/entry-macro-multi.S
index ec0bbf7..538ae7b 100644
--- a/arch/arm/include/asm/entry-macro-multi.S
+++ b/arch/arm/include/asm/entry-macro-multi.S
@@ -9,7 +9,11 @@
 	@ routine called with r0 = irq number, r1 = struct pt_regs *
 	@
 	adrne	lr, BSYM(1b)
+#ifdef CONFIG_IPIPE
+	bne	__ipipe_grab_irq
+#else
 	bne	asm_do_IRQ
+#endif
 
 #ifdef CONFIG_SMP
 	/*
@@ -22,15 +26,23 @@
 	ALT_UP_B(9997f)
 	movne	r1, sp
 	adrne	lr, BSYM(1b)
+#ifdef CONFIG_IPIPE
+	bne	__ipipe_grab_ipi
+#else
 	bne	do_IPI
+#endif
 
 #ifdef CONFIG_LOCAL_TIMERS
 	test_for_ltirq r0, r6, r5, lr
 	movne	r0, sp
 	adrne	lr, BSYM(1b)
+#ifdef CONFIG_IPIPE
+	bne	__ipipe_grab_localtimer
+#else
 	bne	do_local_timer
 #endif
 #endif
+#endif
 9997:
 	.endm
 
diff --git a/arch/arm/include/asm/fcse.h b/arch/arm/include/asm/fcse.h
new file mode 100644
index 0000000..3a0bf32
--- /dev/null
+++ b/arch/arm/include/asm/fcse.h
@@ -0,0 +1,181 @@
+/*
+ * arch/arm/include/asm/fcse.h
+ *
+ * Helper header for using the ARM Fast Context Switch Extension with
+ * processors supporting it, lifted from the Fast Address Space
+ * Switching (FASS) patch for ARM Linux.
+ *
+ * Copyright (C) 2001, 2002 Adam Wiggins <awiggins@cse.unsw.edu.au>
+ * Copyright (C) 2007 Sebastian Smolorz <ssm@emlix.com>
+ * Copyright (C) 2008 Richard Cochran
+ * Copyright (C) 2009-2011 Gilles Chanteperdrix <gch@xenomai.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef __ASM_ARM_FCSE_H
+#define __ASM_ARM_FCSE_H
+
+#ifdef CONFIG_ARM_FCSE
+
+#include <linux/mm_types.h>	/* For struct mm_struct */
+#include <linux/sched.h>
+#include <linux/hardirq.h>
+
+#include <asm/bitops.h>
+#include <asm/cachetype.h>
+
+#define FCSE_PID_SHIFT 25
+
+/* Size of PID relocation area */
+#define FCSE_PID_TASK_SIZE (1UL << FCSE_PID_SHIFT)
+
+/* Mask to get rid of PID from relocated address */
+#define FCSE_PID_MASK (FCSE_PID_TASK_SIZE - 1)
+
+#define FCSE_PID_INVALID (~0 << FCSE_PID_SHIFT)
+extern unsigned long fcse_pids_cache_dirty[];
+
+#ifdef CONFIG_ARM_FCSE_DEBUG
+#define FCSE_BUG_ON(expr) BUG_ON(expr)
+#else /* !CONFIG_ARM_FCSE_DEBUG */
+#define FCSE_BUG_ON(expr) do { } while(0)
+#endif /* !CONFIG_ARM_FCSE_DEBUG */
+
+#if defined(CONFIG_ARM_FCSE_DYNPID) && defined(CONFIG_PREEMPT)
+#define fcse_check_context(mm)					\
+	FCSE_BUG_ON(!in_atomic()				\
+		    && (mm)->context.fcse.active		\
+		    && atomic_read(&(mm)->mm_users)		\
+		    && !(mm)->core_state			\
+		    && !rwsem_is_locked(&(mm)->mmap_sem)	\
+		    && !irqs_disabled_hw());
+#else /* !CONFIG_ARM_FCSE_DYNPID */
+#define fcse_check_context(mm) do { (void)(mm); } while(0)
+#endif /* !CONFIG_ARM_FCSE_DYNPID */
+
+int fcse_pid_alloc(struct mm_struct *mm);
+void fcse_pid_free(struct mm_struct *mm);
+unsigned fcse_flush_all_start(void);
+void fcse_flush_all_done(unsigned seq, unsigned dirty);
+unsigned long
+fcse_check_mmap_inner(struct mm_struct *mm, unsigned long start_addr,
+		      unsigned long addr, unsigned long len, unsigned long fl);
+
+/* Sets the CPU's PID Register */
+static inline void fcse_pid_set(unsigned long pid)
+{
+	__asm__ __volatile__ ("mcr p15, 0, %0, c13, c0, 0"
+			      : /* */: "r" (pid) : "cc", "memory");
+}
+
+static inline unsigned long
+fcse_va_to_mva(struct mm_struct *mm, unsigned long va)
+{
+	if (cache_is_vivt() && va < FCSE_PID_TASK_SIZE) {
+		fcse_check_context(mm);
+		return mm->context.fcse.pid | va;
+	}
+	return va;
+}
+
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+struct fcse_user {
+	struct mm_struct *mm;
+	unsigned count;
+};
+extern struct fcse_user fcse_pids_user[];
+int fcse_switch_mm_inner(struct mm_struct *prev, struct mm_struct *next);
+void fcse_pid_reference(unsigned pid);
+
+static inline int fcse_switch_mm(struct mm_struct *prev, struct mm_struct *next)
+{
+	if (!cache_is_vivt())
+		return 0;
+
+	return fcse_switch_mm_inner(prev, next);
+}
+
+static inline int fcse_mm_in_cache(struct mm_struct *mm)
+{
+	unsigned fcse_pid = mm->context.fcse.pid >> FCSE_PID_SHIFT;
+	int res;
+	fcse_check_context(mm);
+	res = test_bit(fcse_pid, fcse_pids_cache_dirty)
+		&& fcse_pids_user[fcse_pid].mm == mm;
+	return res;
+}
+
+static inline unsigned long
+fcse_check_mmap_addr(struct mm_struct *mm, unsigned long start_addr,
+		     unsigned long addr, unsigned long len, unsigned long fl)
+{
+       const unsigned long stack_base = ALIGN(mm->start_stack, PAGE_SIZE)
+	       - current->signal->rlim[RLIMIT_STACK].rlim_cur;
+
+       if (addr + len <= stack_base)
+	       return addr;
+
+       return fcse_check_mmap_inner(mm, start_addr, addr, len, fl);
+}
+
+#else /* CONFIG_ARM_FCSE_GUARANTEED */
+static inline int
+fcse_switch_mm(struct mm_struct *prev, struct mm_struct *next)
+{
+	unsigned fcse_pid;
+
+	if (!cache_is_vivt())
+		return 0;
+
+	fcse_pid = next->context.fcse.pid >> FCSE_PID_SHIFT;
+	set_bit(fcse_pid, fcse_pids_cache_dirty);
+	fcse_pid_set(next->context.fcse.pid);
+	return 0;
+}
+
+static inline int fcse_mm_in_cache(struct mm_struct *mm)
+{
+	unsigned fcse_pid = mm->context.fcse.pid >> FCSE_PID_SHIFT;
+	return test_bit(fcse_pid, fcse_pids_cache_dirty);
+}
+
+static inline unsigned long
+fcse_check_mmap_addr(struct mm_struct *mm, unsigned long start_addr,
+		     unsigned long addr, unsigned long len, unsigned long fl)
+{
+       if (addr + len <= FCSE_TASK_SIZE)
+	       return addr;
+
+       return fcse_check_mmap_inner(mm, start_addr, addr, len, fl);
+}
+#endif /* CONFIG_ARM_FCSE_GUARANTEED */
+
+static inline void fcse_mark_dirty(struct mm_struct *mm)
+{
+	if (cache_is_vivt()) {
+		set_bit(mm->context.fcse.pid >> FCSE_PID_SHIFT,
+			fcse_pids_cache_dirty);
+		FCSE_BUG_ON(!fcse_mm_in_cache(mm));
+	}
+}
+
+#else /* ! CONFIG_ARM_FCSE */
+#define fcse_switch_mm(prev, next) 1
+#define fcse_va_to_mva(mm, x) ({ (void)(mm); (x); })
+#define fcse_mark_dirty(mm) do { (void)(mm); } while(0)
+#define fcse_flush_all_start() (0)
+#define fcse_flush_all_done(seq, dirty) do { (void)(seq); } while (0)
+#define fcse_mm_in_cache(mm) \
+		(cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm)))
+#endif /* ! CONFIG_ARM_FCSE */
+
+#ifdef CONFIG_ARM_FCSE_MESSAGES
+void fcse_notify_segv(struct mm_struct *mm,
+		      unsigned long addr, struct pt_regs *regs);
+#else /* !FCSE_MESSAGES */
+#define fcse_notify_segv(mm, addr, regs) do { } while(0)
+#endif /* !FCSE_MESSAGES */
+
+#endif /* __ASM_ARM_FCSE_H */
diff --git a/arch/arm/include/asm/hardware/timer-sp.h b/arch/arm/include/asm/hardware/timer-sp.h
index 21e75e3..2cae6e8 100644
--- a/arch/arm/include/asm/hardware/timer-sp.h
+++ b/arch/arm/include/asm/hardware/timer-sp.h
@@ -1,2 +1,2 @@
-void sp804_clocksource_init(void __iomem *);
+void sp804_clocksource_init(void __iomem *, unsigned long);
 void sp804_clockevents_init(void __iomem *, unsigned int);
diff --git a/arch/arm/include/asm/ipipe.h b/arch/arm/include/asm/ipipe.h
new file mode 100644
index 0000000..b96e3e2
--- /dev/null
+++ b/arch/arm/include/asm/ipipe.h
@@ -0,0 +1,340 @@
+/* -*- linux-c -*-
+ * arch/arm/include/asm/ipipe.h
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ * Copyright (C) 2005 Stelian Pop.
+ * Copyright (C) 2006-2008 Gilles Chanteperdrix.
+ * Copyright (C) 2010 Philippe Gerum (SMP port).
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __ARM_IPIPE_H
+#define __ARM_IPIPE_H
+
+#ifdef CONFIG_IPIPE
+
+#include <linux/ipipe_percpu.h>
+#include <linux/ipipe_trace.h>
+
+#define IPIPE_ARCH_STRING	"1.18-03"
+#define IPIPE_MAJOR_NUMBER	1
+#define IPIPE_MINOR_NUMBER	18
+#define IPIPE_PATCH_NUMBER	3
+
+#ifdef CONFIG_SMP
+#define ipipe_processor_id()	hard_smp_processor_id()
+#else /* !CONFIG_SMP */
+#define ipipe_processor_id()	0
+#endif	/* CONFIG_SMP */
+
+#define smp_processor_id_hw() ipipe_processor_id()
+
+#if defined(CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH) || defined(CONFIG_SMP)
+
+#define prepare_arch_switch(next)			\
+	do {						\
+		local_irq_enable_hw();			\
+		ipipe_schedule_notify(current, next);	\
+	} while(0)
+
+#define task_hijacked(p)				\
+	({						\
+		int __x__ = ipipe_root_domain_p;	\
+		!__x__;					\
+	})
+
+#define ipipe_mm_switch_protect(flags)		\
+	do {					\
+		(void)(flags);			\
+	} while(0)
+
+#define ipipe_mm_switch_unprotect(flags)	\
+	do {					\
+		(void)(flags);			\
+	} while(0)
+
+#else /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH && !SMP */
+
+#define prepare_arch_switch(next)			\
+	do {                                            \
+		ipipe_schedule_notify(current ,next);   \
+		local_irq_disable_hw();                 \
+	} while(0)
+
+#define task_hijacked(p)				\
+	({						\
+		int __x__ = __ipipe_root_domain_p;	\
+		if (__x__) local_irq_enable_hw();	\
+		!__x__;					\
+	})
+
+#define ipipe_mm_switch_protect(flags) \
+	local_irq_save_hw_cond(flags)
+
+#define ipipe_mm_switch_unprotect(flags) \
+	local_irq_restore_hw_cond(flags)
+
+#endif /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH && !SMP */
+
+extern unsigned long arm_return_addr(int level);
+
+#define BROKEN_BUILTIN_RETURN_ADDRESS
+#define __BUILTIN_RETURN_ADDRESS0 arm_return_addr(0)
+#define __BUILTIN_RETURN_ADDRESS1 arm_return_addr(1)
+
+struct ipipe_domain;
+
+#define IPIPE_TSC_TYPE_NONE	   		0
+#define IPIPE_TSC_TYPE_FREERUNNING 		1
+#define IPIPE_TSC_TYPE_DECREMENTER 		2
+#define IPIPE_TSC_TYPE_FREERUNNING_COUNTDOWN	3
+
+/* tscinfo, exported to user-space */
+struct __ipipe_tscinfo {
+	unsigned type;
+	unsigned freq;
+	unsigned long counter_vaddr;
+	union {
+		struct {
+			unsigned long counter_paddr;
+			unsigned long long mask;
+		};
+		struct {
+			unsigned *counter; /* Hw counter physical address */
+			unsigned long long mask; /* Significant bits in the hw counter. */
+			unsigned long long *tsc; /* 64 bits tsc value. */
+		} fr;
+		struct {
+			unsigned *counter; /* Hw counter physical address */
+			unsigned long long mask; /* Significant bits in the hw counter. */
+			unsigned *last_cnt; /* Counter value when updating
+						tsc value. */
+			unsigned long long *tsc; /* 64 bits tsc value. */
+		} dec;
+	} u;
+};
+
+struct ipipe_sysinfo {
+       int sys_nr_cpus;        /* Number of CPUs on board */
+       int sys_hrtimer_irq;    /* hrtimer device IRQ */
+       u64 sys_hrtimer_freq;   /* hrtimer device frequency */
+       u64 sys_hrclock_freq;   /* hrclock device frequency */
+       u64 sys_cpu_freq;       /* CPU frequency (Hz) */
+       struct __ipipe_tscinfo arch_tsc; /* exported data for u.s. tsc */
+};
+
+DECLARE_PER_CPU(struct mm_struct *,ipipe_active_mm);
+/* arch specific stuff */
+extern char __ipipe_tsc_area[];
+extern int __ipipe_mach_timerstolen;
+extern unsigned int __ipipe_mach_ticks_per_jiffy;
+extern void __ipipe_mach_acktimer(void);
+extern void __ipipe_mach_set_dec(unsigned long);
+extern void __ipipe_mach_release_timer(void);
+extern unsigned long __ipipe_mach_get_dec(void);
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info);
+int __ipipe_check_tickdev(const char *devname);
+
+#ifdef CONFIG_IPIPE_ARM_KUSER_TSC
+unsigned long long __ipipe_tsc_get(void) __attribute__((long_call));
+void __ipipe_tsc_register(struct __ipipe_tscinfo *info);
+void __ipipe_tsc_update(void);
+extern unsigned long __ipipe_kuser_tsc_freq;
+#define __ipipe_mach_hrclock_freq __ipipe_kuser_tsc_freq
+#else /* ! generic tsc */
+unsigned long long __ipipe_mach_get_tsc(void);
+#define __ipipe_tsc_get() __ipipe_mach_get_tsc()
+#endif /* ! generic tsc */
+
+#ifndef __ipipe_cpu_freq
+#define __ipipe_cpu_freq __ipipe_mach_hrclock_freq
+#endif
+#ifndef __ipipe_mach_hrtimer_freq
+#define __ipipe_mach_hrtimer_freq (HZ * __ipipe_mach_ticks_per_jiffy)
+#endif
+#ifndef __ipipe_mach_hrclock_freq
+#define	__ipipe_mach_hrclock_freq __ipipe_mach_hrtimer_freq
+#endif
+/*
+ * hrtimer IRQ advertised to domains. Defaults to the contents of the
+ * __ipipe_mach_timerint variable found in legacy platform supports.
+ * May be overridden on SMP platforms with distinct per-CPU timer
+ * interrupts.
+ */
+#ifndef __ipipe_mach_hrtimer_irq
+extern int __ipipe_mach_timerint;
+#define __ipipe_mach_hrtimer_irq	__ipipe_mach_timerint
+#endif
+#ifndef __ipipe_mach_ext_hrtimer
+/*
+ * hrtimer external IRQ number for the given CPU. Same as the internal
+ * hrtimer IRQ number by default.
+ */
+#define __ipipe_mach_ext_hrtimer(cpu)				\
+	({							\
+		(void)(cpu);					\
+		__ipipe_mach_hrtimer_irq;			\
+	})
+#define __ipipe_mach_ext_hrtimer_p(irq)				\
+	((irq) == __ipipe_mach_hrtimer_irq)
+#endif
+#ifndef __ipipe_mach_localtimer
+/*
+ * Some SMP platforms may use different IRQ numbers depending on the
+ * CPU, remapping them to a single virq to advertise a common local
+ * timer interrupt to domains. The macro below provides the hrtimer
+ * IRQ number after remapping, if any. By default, there is no
+ * remapping.
+ */
+#define __ipipe_mach_localtimer(ext_irq)  (ext_irq)
+#endif
+#ifndef __ipipe_mach_doirq
+#define __ipipe_mach_doirq(irq)		__ipipe_do_IRQ
+#endif
+#ifndef __ipipe_mach_ackirq
+#define __ipipe_mach_ackirq(irq)			\
+	({						\
+		__ipipe_mach_ext_hrtimer_p(irq)		\
+			? __ipipe_ack_timerirq		\
+			: __ipipe_ack_irq;		\
+	})
+#endif
+#ifndef __ipipe_mach_hrtimer_debug
+#define __ipipe_mach_hrtimer_debug(irq)	do { } while (0)
+#endif
+
+#define ipipe_read_tsc(t)	do { t = __ipipe_tsc_get(); } while (0)
+#define __ipipe_read_timebase()	__ipipe_tsc_get()
+
+#define ipipe_tsc2ns(t) \
+({ \
+	unsigned long long delta = (t)*1000; \
+	do_div(delta, __ipipe_cpu_freq / 1000000 + 1); \
+	(unsigned long)delta; \
+})
+#define ipipe_tsc2us(t) \
+({ \
+	unsigned long long delta = (t); \
+	do_div(delta, __ipipe_cpu_freq / 1000000 + 1); \
+	(unsigned long)delta; \
+})
+
+/* Private interface -- Internal use only */
+
+#define __ipipe_check_platform()	do { } while(0)
+#define __ipipe_enable_irq(irq)		irq_to_desc(irq)->chip->enable(irq)
+#define __ipipe_disable_irq(irq)	irq_to_desc(irq)->chip->disable(irq)
+
+#ifdef CONFIG_SMP
+void __ipipe_init_platform(void);
+void __ipipe_hook_critical_ipi(struct ipipe_domain *ipd);
+void __ipipe_root_ipi(unsigned int irq, void *cookie);
+void __ipipe_root_localtimer(unsigned int irq, void *cookie);
+void __ipipe_send_vnmi(void (*fn)(void *), cpumask_t cpumask, void *arg);
+void __ipipe_do_vnmi(unsigned int irq, void *cookie);
+#else /* !CONFIG_SMP */
+#define __ipipe_init_platform()		do { } while(0)
+#define __ipipe_hook_critical_ipi(ipd)	do { } while(0)
+#endif /* !CONFIG_SMP */
+#ifndef __ipipe_mach_init_platform
+#define __ipipe_mach_init_platform()	do { } while(0)
+#endif
+
+void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq);
+
+#ifndef __IPIPE_FEATURE_PIC_MUTE
+#define __ipipe_disable_irqdesc(ipd, irq) do { } while (0)
+#else /* __IPIPE_FEATURE_PIC_MUTE */
+
+typedef unsigned long
+__ipipe_irqbits_t[(NR_IRQS + BITS_PER_LONG - 1) / BITS_PER_LONG];
+extern __ipipe_irqbits_t __ipipe_irqbits;
+
+void __ipipe_mach_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq);
+
+void __ipipe_mach_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq);
+
+void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq);
+
+void ipipe_mute_pic(void);
+
+void ipipe_unmute_pic(void);
+#endif /* __IPIPE_FEATURE_PIC_MUTE */
+
+void __ipipe_enable_pipeline(void);
+
+void __ipipe_do_critical_sync(unsigned irq, void *cookie);
+
+DECLARE_PER_CPU(struct pt_regs, __ipipe_tick_regs);
+
+void __ipipe_grab_irq(int irq, struct pt_regs *regs);
+
+void __ipipe_exit_irq(struct pt_regs *regs);
+
+void __ipipe_handle_irq(int irq, struct pt_regs *regs);
+
+static inline void ipipe_handle_chained_irq(unsigned int irq)
+{
+	struct pt_regs regs;    /* dummy */
+
+	ipipe_trace_irq_entry(irq);
+	__ipipe_handle_irq(irq, &regs);
+	ipipe_trace_irq_exit(irq);
+}
+
+#define ipipe_update_tick_evtdev(evtdev) do { } while (0)
+
+static inline unsigned long __ipipe_ffnz(unsigned long ul)
+{
+	return ffs(ul) - 1;
+}
+
+#define __ipipe_syscall_watched_p(p, sc)				\
+	(ipipe_notifier_enabled_p(p) || (unsigned long)sc >= __ARM_NR_BASE + 64)
+
+#define __ipipe_root_tick_p(regs) (!arch_irqs_disabled_flags(regs->ARM_cpsr))
+
+#else /* !CONFIG_IPIPE */
+
+#define task_hijacked(p)		0
+
+#define ipipe_update_tick_evtdev(evtdev)	do { } while (0)
+
+#define smp_processor_id_hw()		smp_processor_id()
+
+#define ipipe_handle_chained_irq(irq)   generic_handle_irq(irq)
+
+#define ipipe_mm_switch_protect(flags) \
+	do {					\
+		(void) (flags);			\
+	} while (0)
+
+#define ipipe_mm_switch_unprotect(flags)	\
+	do {					\
+		(void) (flags);			\
+	} while (0)
+
+#endif /* CONFIG_IPIPE */
+
+#if defined (CONFIG_IPIPE_DEBUG) &&		\
+	(defined(CONFIG_DEBUG_LL) || defined(CONFIG_SERIAL_8250_CONSOLE))
+void __ipipe_serial_debug(const char *fmt, ...);
+#else
+#define __ipipe_serial_debug(fmt, args...)	do { } while (0)
+#endif
+
+#endif	/* !__ARM_IPIPE_H */
diff --git a/arch/arm/include/asm/ipipe_base.h b/arch/arm/include/asm/ipipe_base.h
new file mode 100644
index 0000000..e06ed29
--- /dev/null
+++ b/arch/arm/include/asm/ipipe_base.h
@@ -0,0 +1,150 @@
+/* -*- linux-c -*-
+ * arch/arm/include/asm/ipipe_base.h
+ *
+ * Copyright (C) 2007 Gilles Chanteperdrix.
+ * Copyright (C) 2010 Philippe Gerum (SMP port).
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __ARM_IPIPE_BASE_H
+#define __ARM_IPIPE_BASE_H
+
+#include <linux/threads.h>
+#include <linux/types.h>
+#include <asm/ptrace.h>
+#include <asm/irq.h>
+#include <mach/irqs.h>		/* For __IPIPE_FEATURE_PIC_MUTE */
+
+#define IPIPE_NR_XIRQS		NR_IRQS
+
+/* ARM traps */
+#define IPIPE_TRAP_ACCESS	 0	/* Data or instruction access exception */
+#define IPIPE_TRAP_SECTION	 1	/* Section fault */
+#define IPIPE_TRAP_DABT		 2	/* Generic data abort */
+#define IPIPE_TRAP_UNKNOWN	 3	/* Unknown exception */
+#define IPIPE_TRAP_BREAK	 4	/* Instruction breakpoint */
+#define IPIPE_TRAP_FPU		 5	/* Floating point exception */
+#define IPIPE_TRAP_VFP		 6	/* VFP floating point exception */
+#define IPIPE_TRAP_UNDEFINSTR	 7	/* Undefined instruction */
+#define IPIPE_TRAP_ALIGNMENT	 8	/* Unaligned access exception */
+#define IPIPE_NR_FAULTS		 9
+
+/* Pseudo-vectors used for kernel events */
+#define IPIPE_FIRST_EVENT	IPIPE_NR_FAULTS
+#define IPIPE_EVENT_SYSCALL	(IPIPE_FIRST_EVENT)
+#define IPIPE_EVENT_SCHEDULE	(IPIPE_FIRST_EVENT + 1)
+#define IPIPE_EVENT_SIGWAKE	(IPIPE_FIRST_EVENT + 2)
+#define IPIPE_EVENT_SETSCHED	(IPIPE_FIRST_EVENT + 3)
+#define IPIPE_EVENT_INIT	(IPIPE_FIRST_EVENT + 4)
+#define IPIPE_EVENT_EXIT	(IPIPE_FIRST_EVENT + 5)
+#define IPIPE_EVENT_CLEANUP	(IPIPE_FIRST_EVENT + 6)
+#define IPIPE_EVENT_RETURN	(IPIPE_FIRST_EVENT + 7)
+#define IPIPE_EVENT_HOSTRT	(IPIPE_FIRST_EVENT + 8)
+#define IPIPE_LAST_EVENT	IPIPE_EVENT_HOSTRT
+#define IPIPE_NR_EVENTS		(IPIPE_LAST_EVENT + 1)
+
+#ifndef __ASSEMBLY__
+
+#include <asm/irqflags.h>
+
+#ifdef CONFIG_SMP
+
+#define IPIPE_FIRST_IPI		IPIPE_VIRQ_BASE
+
+#define IPIPE_CRITICAL_IPI	IPIPE_FIRST_IPI
+#define IPIPE_SERVICE_IPI0	(IPIPE_CRITICAL_IPI + 1)
+#define IPIPE_SERVICE_IPI1	(IPIPE_CRITICAL_IPI + 2)
+#define IPIPE_SERVICE_IPI2	(IPIPE_CRITICAL_IPI + 3)
+#define IPIPE_SERVICE_IPI3	(IPIPE_CRITICAL_IPI + 4)
+#define IPIPE_SERVICE_VNMI	(IPIPE_CRITICAL_IPI + 5)
+
+#define IPIPE_LAST_IPI		IPIPE_SERVICE_VNMI
+
+#define hard_smp_processor_id()						\
+	({								\
+		unsigned int cpunum;					\
+		__asm__ __volatile__ ("\n"				\
+			"1:	mrc p15, 0, %0, c0, c0, 5\n"		\
+			"	.pushsection \".alt.smp.init\", \"a\"\n"\
+			"	.long	1b\n"				\
+			"	mov	%0, #0\n"			\
+			"	.popsection"				\
+				      : "=r" (cpunum));			\
+		cpunum &= 0x0F;						\
+	})
+
+#define ipipe_ipi_p(ipi)	\
+	((ipi) >= IPIPE_SERVICE_IPI0 && (ipi) <= IPIPE_SERVICE_IPI3)
+
+void __ipipe_stall_root(void);
+
+unsigned long __ipipe_test_and_stall_root(void);
+
+unsigned long __ipipe_test_root(void);
+
+#else /* !CONFIG_SMP */
+
+#if __GNUC__ >= 4
+/* Alias to ipipe_root_cpudom_var(status) */
+extern unsigned long __ipipe_root_status;
+#else
+extern unsigned long *const __ipipe_root_status_addr;
+#define __ipipe_root_status	(*__ipipe_root_status_addr)
+#endif
+
+static inline void __ipipe_stall_root(void)
+{
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
+	__ipipe_root_status |= 1;
+	local_irq_restore_hw(flags);
+}
+
+static inline unsigned __ipipe_test_root(void)
+{
+	return __ipipe_root_status & 1;
+}
+
+static inline unsigned __ipipe_test_and_stall_root(void)
+{
+	unsigned long flags, res;
+
+	local_irq_save_hw(flags);
+	res = __ipipe_root_status;
+	__ipipe_root_status = res | 1;
+	local_irq_restore_hw(flags);
+
+	return res & 1;
+}
+
+#endif	/* CONFIG_SMP */
+
+#define __IPIPE_FEATURE_PREEMPTIBLE_SWITCH	1
+#define __IPIPE_FEATURE_SYSINFO_V2		1
+
+#ifdef CONFIG_VFP
+#define __IPIPE_FEATURE_VFP_SAFE		1
+#endif
+
+#ifdef CONFIG_IPIPE_ARM_KUSER_TSC
+#define __IPIPE_FEATURE_KUSER_TSC		1
+#endif
+
+#endif /* !__ASSEMBLY__ */
+
+#endif /* __ARM_IPIPE_BASE_H */
diff --git a/arch/arm/include/asm/irqflags.h b/arch/arm/include/asm/irqflags.h
index 1e6cca5..c83b76b 100644
--- a/arch/arm/include/asm/irqflags.h
+++ b/arch/arm/include/asm/irqflags.h
@@ -10,90 +10,71 @@
  */
 #if __LINUX_ARM_ARCH__ >= 6
 
-static inline unsigned long arch_local_irq_save(void)
-{
-	unsigned long flags;
-
-	asm volatile(
-		"	mrs	%0, cpsr	@ arch_local_irq_save\n"
-		"	cpsid	i"
-		: "=r" (flags) : : "memory", "cc");
-	return flags;
-}
-
-static inline void arch_local_irq_enable(void)
-{
-	asm volatile(
-		"	cpsie i			@ arch_local_irq_enable"
-		:
-		:
-		: "memory", "cc");
-}
+#define local_irq_save_hw_notrace(x)					\
+	({							\
+	__asm__ __volatile__(					\
+	"mrs	%0, cpsr		@ local_irq_save_hw\n"	\
+	"cpsid	i"						\
+	: "=r" (x) : : "memory", "cc");				\
+	})
 
-static inline void arch_local_irq_disable(void)
-{
-	asm volatile(
-		"	cpsid i			@ arch_local_irq_disable"
-		:
-		:
-		: "memory", "cc");
-}
+#define local_irq_enable_hw_notrace()  __asm__("cpsie i	@ __sti" : : : "memory", "cc")
+#define local_irq_disable_hw_notrace() __asm__("cpsid i	@ __cli" : : : "memory", "cc")
+#define local_fiq_enable_hw_notrace()  __asm__("cpsie f	@ __stf" : : : "memory", "cc")
+#define local_fiq_disable_hw_notrace() __asm__("cpsid f	@ __clf" : : : "memory", "cc")
 
-#define local_fiq_enable()  __asm__("cpsie f	@ __stf" : : : "memory", "cc")
-#define local_fiq_disable() __asm__("cpsid f	@ __clf" : : : "memory", "cc")
 #else
 
 /*
  * Save the current interrupt enable state & disable IRQs
  */
-static inline unsigned long arch_local_irq_save(void)
-{
-	unsigned long flags, temp;
-
-	asm volatile(
-		"	mrs	%0, cpsr	@ arch_local_irq_save\n"
-		"	orr	%1, %0, #128\n"
-		"	msr	cpsr_c, %1"
-		: "=r" (flags), "=r" (temp)
-		:
-		: "memory", "cc");
-	return flags;
-}
+#define local_irq_save_hw_notrace(x)					\
+	({							\
+		unsigned long temp;				\
+		(void) (&temp == &x);				\
+	__asm__ __volatile__(					\
+	"mrs	%0, cpsr		@ local_irq_save_hw\n"	\
+"	orr	%1, %0, #128\n"					\
+"	msr	cpsr_c, %1"					\
+	: "=r" (x), "=r" (temp)					\
+	:							\
+	: "memory", "cc");					\
+	})
 
 /*
  * Enable IRQs
  */
-static inline void arch_local_irq_enable(void)
-{
-	unsigned long temp;
-	asm volatile(
-		"	mrs	%0, cpsr	@ arch_local_irq_enable\n"
-		"	bic	%0, %0, #128\n"
-		"	msr	cpsr_c, %0"
-		: "=r" (temp)
-		:
-		: "memory", "cc");
-}
+#define local_irq_enable_hw_notrace()				\
+	({							\
+		unsigned long temp;				\
+	__asm__ __volatile__(					\
+	"mrs	%0, cpsr		@ local_irq_enable_hw\n"	\
+"	bic	%0, %0, #128\n"					\
+"	msr	cpsr_c, %0"					\
+	: "=r" (temp)						\
+	:							\
+	: "memory", "cc");					\
+	})
 
 /*
  * Disable IRQs
  */
-static inline void arch_local_irq_disable(void)
-{
-	unsigned long temp;
-	asm volatile(
-		"	mrs	%0, cpsr	@ arch_local_irq_disable\n"
-		"	orr	%0, %0, #128\n"
-		"	msr	cpsr_c, %0"
-		: "=r" (temp)
-		:
-		: "memory", "cc");
-}
+#define local_irq_disable_hw_notrace()				\
+	({							\
+		unsigned long temp;				\
+	__asm__ __volatile__(					\
+	"mrs	%0, cpsr		@ local_irq_disable_hw\n"	\
+"	orr	%0, %0, #128\n"					\
+"	msr	cpsr_c, %0"					\
+	: "=r" (temp)						\
+	:							\
+	: "memory", "cc");					\
+	})
 
 /*
  * Enable FIQs
  */
-#define local_fiq_enable()					\
+#define local_fiq_enable_hw_notrace()				\
 	({							\
 		unsigned long temp;				\
 	__asm__ __volatile__(					\
@@ -108,7 +89,7 @@ static inline void arch_local_irq_disable(void)
 /*
  * Disable FIQs
  */
-#define local_fiq_disable()					\
+#define local_fiq_disable_hw_notrace()				\
 	({							\
 		unsigned long temp;				\
 	__asm__ __volatile__(					\
@@ -125,31 +106,130 @@ static inline void arch_local_irq_disable(void)
 /*
  * Save the current interrupt enable state.
  */
-static inline unsigned long arch_local_save_flags(void)
-{
-	unsigned long flags;
-	asm volatile(
-		"	mrs	%0, cpsr	@ local_save_flags"
-		: "=r" (flags) : : "memory", "cc");
-	return flags;
-}
+#define local_save_flags_hw(x)					\
+	({							\
+	__asm__ __volatile__(					\
+	"mrs	%0, cpsr		@ local_save_flags_hw"	\
+	: "=r" (x) : : "memory", "cc");				\
+	})
 
 /*
  * restore saved IRQ & FIQ state
  */
-static inline void arch_local_irq_restore(unsigned long flags)
+#define local_irq_restore_hw_notrace(x)				\
+	__asm__ __volatile__(					\
+	"msr	cpsr_c, %0		@ local_irq_restore_hw\n"	\
+	:							\
+	: "r" (x)						\
+	: "memory", "cc")
+
+#define arch_irqs_disabled_flags(_flags)	\
+({					\
+	(int)((_flags) & PSR_I_BIT);	\
+})
+
+#define irqs_disabled_hw()			\
+({						\
+	unsigned long _flags;			\
+	local_save_flags_hw(_flags);		\
+	arch_irqs_disabled_flags(_flags);		\
+})
+
+static inline unsigned long arch_mangle_irq_bits(int virt, unsigned long real)
 {
-	asm volatile(
-		"	msr	cpsr_c, %0	@ local_irq_restore"
-		:
-		: "r" (flags)
-		: "memory", "cc");
+	/* Merge virtual and real interrupt mask bits into a single
+	   32bit word. */
+	return (real & ~(1L << 8)) | ((virt != 0) << 8);
 }
 
-static inline int arch_irqs_disabled_flags(unsigned long flags)
+static inline int arch_demangle_irq_bits(unsigned long *x)
 {
-	return flags & PSR_I_BIT;
+	int virt = (*x & (1 << 8)) != 0;
+	*x &= ~(1L << 8);
+	return virt;
 }
 
+#ifdef CONFIG_IPIPE
+
+void __ipipe_unstall_root(void);
+void __ipipe_restore_root(unsigned long flags);
+
+#define arch_local_save_flags() (__ipipe_test_root() << 7)
+#define arch_local_irq_save() (__ipipe_test_and_stall_root() << 7)
+#define arch_local_irq_restore(flags) (__ipipe_restore_root(flags & (1 << 7)))
+#define arch_local_irq_disable() __ipipe_stall_root()
+#define arch_local_irq_enable() __ipipe_unstall_root()
+#define local_fiq_enable() __ipipe_unstall_root()
+#define local_fiq_disable() __ipipe_stall_root()
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+
+#include <linux/ipipe_trace.h>
+
+#define local_irq_disable_hw() do { \
+	if (!irqs_disabled_hw()) { \
+		local_irq_disable_hw_notrace(); \
+		ipipe_trace_begin(0x80000000); \
+	} \
+} while (0)
+#define local_irq_enable_hw() do { \
+	if (irqs_disabled_hw()) { \
+		ipipe_trace_end(0x80000000); \
+		local_irq_enable_hw_notrace(); \
+	} \
+} while (0)
+#define local_irq_save_hw(x) do { \
+	local_save_flags_hw(x); \
+	if (!arch_irqs_disable_flags(x)) { \
+		local_irq_disable_hw_notrace(); \
+		ipipe_trace_begin(0x80000001); \
+	} \
+} while (0)
+#define local_irq_restore_hw(x) do { \
+	if (!arch_irqs_disable_flags(x)) \
+		ipipe_trace_end(0x80000001); \
+	local_irq_restore_hw_notrace(x); \
+} while (0)
+
+#else /* !CONFIG_IPIPE_TRACE_IRQSOFF */
+
+#define local_irq_save_hw(flags)	local_irq_save_hw_notrace(flags)
+#define local_irq_enable_hw()		local_irq_enable_hw_notrace()
+#define local_irq_disable_hw()		local_irq_disable_hw_notrace()
+#define local_fiq_enable_hw()		local_fiq_enable_hw_notrace()
+#define local_fiq_disable_hw()		local_fiq_disable_hw_notrace()
+#define local_irq_restore_hw(flags)	local_irq_restore_hw_notrace(flags)
+
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+
+#else /* !CONFIG_IPIPE */
+
+#define arch_local_save_flags()			\
+	({					\
+		unsigned long _flags;		\
+		local_save_flags_hw(_flags);	\
+		(_flags);			\
+	})
+#define arch_local_irq_save()			\
+	({					\
+		unsigned long _flags;		\
+		local_irq_save_hw(_flags);	\
+		(_flags);			\
+	})
+#define arch_local_irq_restore(flags) local_irq_restore_hw_notrace(flags)
+#define arch_local_irq_disable() local_irq_disable_hw_notrace()
+#define arch_local_irq_enable() local_irq_enable_hw_notrace()
+#define local_fiq_enable() local_fiq_enable_hw_notrace()
+#define local_fiq_disable() local_fiq_disable_hw_notrace()
+
+#define local_irq_save_hw(flags)	local_irq_save_hw_notrace(flags)
+#define local_irq_enable_hw()		local_irq_enable_hw_notrace()
+#define local_irq_disable_hw()		local_irq_disable_hw_notrace()
+#define local_fiq_enable_hw()		local_fiq_enable_hw_notrace()
+#define local_fiq_disable_hw()		local_fiq_disable_hw_notrace()
+#define local_irq_restore_hw(flags)	local_irq_restore_hw_notrace(flags)
+
+#endif /* CONFIG_IPIPE */
+
 #endif
 #endif
diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h
index d0ee74b..7455059 100644
--- a/arch/arm/include/asm/memory.h
+++ b/arch/arm/include/asm/memory.h
@@ -33,7 +33,12 @@
  */
 #define PAGE_OFFSET		UL(CONFIG_PAGE_OFFSET)
 #define TASK_SIZE		(UL(CONFIG_PAGE_OFFSET) - UL(0x01000000))
+#ifndef CONFIG_ARM_FCSE
 #define TASK_UNMAPPED_BASE	(UL(CONFIG_PAGE_OFFSET) / 3)
+#else /* CONFIG_ARM_FCSE */
+#define TASK_UNMAPPED_BASE	UL(0x00800000)
+#define FCSE_TASK_SIZE		UL(0x02000000)
+#endif /* CONFIG_ARM_FCSE */
 
 /*
  * The maximum size of a 26-bit user space task.
diff --git a/arch/arm/include/asm/mmu.h b/arch/arm/include/asm/mmu.h
index b4ffe9d..c0465e5 100644
--- a/arch/arm/include/asm/mmu.h
+++ b/arch/arm/include/asm/mmu.h
@@ -6,8 +6,19 @@
 typedef struct {
 #ifdef CONFIG_CPU_HAS_ASID
 	unsigned int id;
-	spinlock_t id_lock;
+	ipipe_spinlock_t id_lock;
 #endif
+#ifdef CONFIG_ARM_FCSE
+	struct {
+		unsigned long pid;
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+		unsigned shared_dirty_pages;
+		unsigned active : 1;
+		unsigned large : 1;
+		unsigned high_pages;
+#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */
+	} fcse;
+#endif /* CONFIG_ARM_FCSE */
 	unsigned int kvm_seq;
 } mm_context_t;
 
@@ -16,7 +27,7 @@ typedef struct {
 
 /* init_mm.context.id_lock should be initialized. */
 #define INIT_MM_CONTEXT(name)                                                 \
-	.context.id_lock    = __SPIN_LOCK_UNLOCKED(name.context.id_lock),
+	.context.id_lock    = IPIPE_SPIN_LOCK_UNLOCKED,
 #else
 #define ASID(mm)	(0)
 #endif
diff --git a/arch/arm/include/asm/mmu_context.h b/arch/arm/include/asm/mmu_context.h
index 71605d9..b1c5dc7 100644
--- a/arch/arm/include/asm/mmu_context.h
+++ b/arch/arm/include/asm/mmu_context.h
@@ -18,6 +18,7 @@
 #include <asm/cacheflush.h>
 #include <asm/cachetype.h>
 #include <asm/proc-fns.h>
+#include <asm/fcse.h>
 
 void __check_kvm_seq(struct mm_struct *mm);
 
@@ -78,11 +79,58 @@ static inline void check_context(struct mm_struct *mm)
 #endif
 }
 
-#define init_new_context(tsk,mm)	0
+static inline int
+init_new_context(struct task_struct *tsk, struct mm_struct *mm)
+{
+#ifdef CONFIG_ARM_FCSE
+	int fcse_pid;
+
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+	if (!mm->context.fcse.large) {
+		fcse_pid = fcse_pid_alloc(mm);
+		mm->context.fcse.pid = fcse_pid << FCSE_PID_SHIFT;
+	} else {
+		/* We are normally forking a process vith a virtual address
+		   space larger than 32 MB, so its pid should be 0. */
+		FCSE_BUG_ON(mm->context.fcse.pid);
+		fcse_pid_reference(0);
+	}
+	/* If we are forking, set_pte_at will restore the correct high pages
+	   count, and shared writable pages are write-protected again. */
+	mm->context.fcse.shared_dirty_pages = 0;
+	mm->context.fcse.high_pages = 0;
+	mm->context.fcse.active = 0;
+#else /* CONFIG_ARM_FCSE_GUARANTEED */
+	fcse_pid = fcse_pid_alloc(mm);
+	if (fcse_pid < 0) {
+		/*
+		 * Set mm pid to FCSE_PID_INVALID, as even when
+		 * init_new_context fails, destroy_context is called.
+		 */
+		mm->context.fcse.pid = FCSE_PID_INVALID;
+		return fcse_pid;
+	}
+	mm->context.fcse.pid = fcse_pid << FCSE_PID_SHIFT;
+#endif /* CONFIG_ARM_FCSE_GUARANTEED */
+	FCSE_BUG_ON(fcse_mm_in_cache(mm));
+#endif /* CONFIG_ARM_FCSE */
+
+	return 0;
+}
 
 #endif
 
-#define destroy_context(mm)		do { } while(0)
+static inline void destroy_context(struct mm_struct *mm)
+{
+#ifdef CONFIG_ARM_FCSE
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+	FCSE_BUG_ON(mm->context.fcse.shared_dirty_pages);
+	FCSE_BUG_ON(mm->context.fcse.high_pages);
+#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */
+	if (mm->context.fcse.pid != FCSE_PID_INVALID)
+		fcse_pid_free(mm);
+#endif /* CONFIG_ARM_FCSE */
+}
 
 /*
  * This is called when "tsk" is about to enter lazy TLB mode.
@@ -105,11 +153,11 @@ enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
  * actually changed.
  */
 static inline void
-switch_mm(struct mm_struct *prev, struct mm_struct *next,
+__switch_mm(struct mm_struct *prev, struct mm_struct *next,
 	  struct task_struct *tsk)
 {
 #ifdef CONFIG_MMU
-	unsigned int cpu = smp_processor_id();
+	unsigned int cpu = ipipe_processor_id();
 
 #ifdef CONFIG_SMP
 	/* check for possible thread migration */
@@ -120,18 +168,80 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next,
 	if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next)) || prev != next) {
 #ifdef CONFIG_SMP
 		struct mm_struct **crt_mm = &per_cpu(current_mm, cpu);
-		*crt_mm = next;
 #endif
-		check_context(next);
-		cpu_switch_mm(next->pgd, next);
-		if (cache_is_vivt())
+#if defined(CONFIG_IPIPE)
+		if (ipipe_root_domain_p) {
+			/* mark mm state as undefined. */
+			per_cpu(ipipe_active_mm, cpu) = NULL;
+			barrier();
+#ifdef CONFIG_SMP
+			*crt_mm = next;
+#endif
+			check_context(next);
+			cpu_switch_mm(next->pgd, next,
+				      fcse_switch_mm(prev, next));
+			barrier();
+			per_cpu(ipipe_active_mm, cpu) = next;
+			while (test_and_clear_thread_flag(TIF_MMSWITCH_INT)) {
+				/* mark mm state as undefined. */
+				per_cpu(ipipe_active_mm, cpu) = NULL;
+#ifdef CONFIG_SMP
+				*crt_mm = next;
+#endif
+				barrier();
+				check_context(next);
+				cpu_switch_mm(next->pgd, next,
+					      fcse_switch_mm(NULL, next));
+				barrier();
+				per_cpu(ipipe_active_mm, cpu) = next;
+			}
+		} else
+#endif /* CONFIG_IPIPE */
+		{
+#ifdef CONFIG_SMP
+			*crt_mm = next;
+#endif
+			check_context(next);
+			cpu_switch_mm(next->pgd, next,
+				      fcse_switch_mm(prev, next));
+		}
+#if defined(CONFIG_IPIPE) && defined(CONFIG_ARM_FCSE)
+		if (tsk)
+			set_tsk_thread_flag(tsk, TIF_SWITCHED);
+#endif /* CONFIG_IPIPE && CONFIG_ARM_FCSE */
+		if (cache_is_vivt() && prev)
 			cpumask_clear_cpu(cpu, mm_cpumask(prev));
-	}
+	} else
+		fcse_mark_dirty(next);
 #endif
 }
 
+static inline void
+switch_mm(struct mm_struct *prev, struct mm_struct *next,
+	  struct task_struct *tsk)
+{
+#if !defined(CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH) && !defined(CONFIG_SMP)
+	unsigned long flags;
+	local_irq_save_hw(flags);
+#endif /* !(CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH && SMP) */
+	__switch_mm(prev, next, tsk);
+#if !defined(CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH) && !defined(CONFIG_SMP)
+	local_irq_restore_hw(flags);
+#endif /* !(CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH && SMP) */
+}
+
 #define deactivate_mm(tsk,mm)	do { } while (0)
+
+#ifndef CONFIG_ARM_FCSE_BEST_EFFORT
 #define activate_mm(prev,next)	switch_mm(prev, next, NULL)
+#else
+#define activate_mm(prev,next)						\
+	({								\
+	switch_mm(prev, next, NULL);					\
+	next->context.fcse.active = 1;					\
+	FCSE_BUG_ON(current->mm == next && !fcse_mm_in_cache(next));	\
+	})
+#endif
 
 /*
  * We are inserting a "fake" vma for the user-accessible vector page so
diff --git a/arch/arm/include/asm/percpu.h b/arch/arm/include/asm/percpu.h
index b4e32d8..175dd38 100644
--- a/arch/arm/include/asm/percpu.h
+++ b/arch/arm/include/asm/percpu.h
@@ -1,6 +1,10 @@
 #ifndef __ARM_PERCPU
 #define __ARM_PERCPU
 
+#if defined(CONFIG_IPIPE) && defined(CONFIG_SMP)
+#define __my_cpu_offset		per_cpu_offset(hard_smp_processor_id())
+#endif
+
 #include <asm-generic/percpu.h>
 
 #endif
diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h
index ebcb643..aa40a89 100644
--- a/arch/arm/include/asm/pgtable.h
+++ b/arch/arm/include/asm/pgtable.h
@@ -117,6 +117,8 @@
 #define LIBRARY_TEXT_START	0x0c000000
 
 #ifndef __ASSEMBLY__
+#include <asm/fcse.h>
+
 extern void __pte_error(const char *file, int line, pte_t);
 extern void __pmd_error(const char *file, int line, pmd_t);
 extern void __pgd_error(const char *file, int line, pgd_t);
@@ -273,6 +275,40 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
 #define __S111  __PAGE_SHARED_EXEC
 
 #ifndef __ASSEMBLY__
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+#define fcse_account_page_removal(mm, addr, val) do {		\
+	struct mm_struct *_mm = (mm);				\
+	unsigned long _addr = (addr);				\
+	unsigned long _val = (val);				\
+	if (pte_present(_val) && ((_val) & L_PTE_SHARED))	\
+		--_mm->context.fcse.shared_dirty_pages;		\
+	if (pte_present(_val) && _addr < TASK_SIZE) {		\
+		if (_addr >= FCSE_TASK_SIZE)			\
+			--_mm->context.fcse.high_pages;		\
+	}							\
+} while (0)
+
+#define fcse_account_page_addition(mm, addr, val) ({			\
+	struct mm_struct *_mm = (mm);					\
+	unsigned long _addr = (addr);					\
+	unsigned long _val = (val);					\
+	if (pte_present(_val) && (_val & L_PTE_SHARED)) {		\
+		if ((_val & (PTE_CACHEABLE | L_PTE_RDONLY | L_PTE_DIRTY)) \
+		    != (PTE_CACHEABLE | L_PTE_DIRTY))			\
+			_val &= ~L_PTE_SHARED;                          \
+		else                                                    \
+			++_mm->context.fcse.shared_dirty_pages;         \
+	}                                                               \
+	if (pte_present(_val)						\
+	    && _addr < TASK_SIZE && _addr >= FCSE_TASK_SIZE)		\
+		++_mm->context.fcse.high_pages;				\
+	_val;								\
+})
+#else /* CONFIG_ARM_FCSE_GUARANTEED || !CONFIG_ARM_FCSE */
+#define fcse_account_page_removal(mm, addr, val) do { } while (0)
+#define fcse_account_page_addition(mm, addr, val) (val)
+#endif /* CONFIG_ARM_FCSE_GUARANTEED || !CONFIG_ARM_FCSE */
+
 /*
  * ZERO_PAGE is a global shared page that is always zero: used
  * for zero-mapped memory areas etc..
@@ -280,16 +316,19 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
 extern struct page *empty_zero_page;
 #define ZERO_PAGE(vaddr)	(empty_zero_page)
 
-
 extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
 
 /* to find an entry in a page-table-directory */
 #define pgd_index(addr)		((addr) >> PGDIR_SHIFT)
 
-#define pgd_offset(mm, addr)	((mm)->pgd + pgd_index(addr))
+#define pgd_offset(mm, addr)						\
+	({								\
+		struct mm_struct *_mm = (mm);				\
+		(_mm->pgd + pgd_index(fcse_va_to_mva(_mm, (addr))));	\
+	})
 
 /* to find an entry in a kernel page-table-directory */
-#define pgd_offset_k(addr)	pgd_offset(&init_mm, addr)
+#define pgd_offset_k(addr)	(init_mm.pgd + pgd_index(addr))
 
 /*
  * The "pgd_xxx()" functions here are trivial for a folded two-level
@@ -334,7 +373,6 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd)
 /* we don't need complex calculations here as the pmd is folded into the pgd */
 #define pmd_addr_end(addr,end)	(end)
 
-
 #ifndef CONFIG_HIGHPTE
 #define __pte_map(pmd)		pmd_page_vaddr(*(pmd))
 #define __pte_unmap(pte)	do { } while (0)
@@ -357,7 +395,10 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd)
 #define mk_pte(page,prot)	pfn_pte(page_to_pfn(page), prot)
 
 #define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,pte,ext)
-#define pte_clear(mm,addr,ptep)	set_pte_ext(ptep, __pte(0), 0)
+#define pte_clear(mm,addr,ptep)	do {				\
+	fcse_account_page_removal(mm, addr, pte_val(*ptep));	\
+	set_pte_ext(ptep, __pte(0), 0);				\
+} while (0)
 
 #if __LINUX_ARM_ARCH__ < 6
 static inline void __sync_icache_dcache(pte_t pteval)
@@ -367,9 +408,28 @@ static inline void __sync_icache_dcache(pte_t pteval)
 extern void __sync_icache_dcache(pte_t pteval);
 #endif
 
+/*
+ * The following only work if pte_present() is true.
+ * Undefined behaviour if not..
+ */
+#define pte_present(pte)	(pte_val(pte) & L_PTE_PRESENT)
+#define pte_write(pte)		(!(pte_val(pte) & L_PTE_RDONLY))
+#define pte_dirty(pte)		(pte_val(pte) & L_PTE_DIRTY)
+#define pte_young(pte)		(pte_val(pte) & L_PTE_YOUNG)
+#define pte_exec(pte)		(!(pte_val(pte) & L_PTE_XN))
+#define pte_special(pte)	(0)
+
+#define pte_present_user(pte) \
+	((pte_val(pte) & (L_PTE_PRESENT | L_PTE_USER)) == \
+	 (L_PTE_PRESENT | L_PTE_USER))
+
 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
-			      pte_t *ptep, pte_t pteval)
+				pte_t *ptep, pte_t pteval)
 {
+	fcse_account_page_removal(mm, addr, pte_val(*ptep));
+	pte_val(pteval) =
+		fcse_account_page_addition(mm, addr, pte_val(pteval));
+
 	if (addr >= TASK_SIZE)
 		set_pte_ext(ptep, pteval, 0);
 	else {
@@ -379,16 +439,6 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
 }
 
 #define pte_none(pte)		(!pte_val(pte))
-#define pte_present(pte)	(pte_val(pte) & L_PTE_PRESENT)
-#define pte_write(pte)		(!(pte_val(pte) & L_PTE_RDONLY))
-#define pte_dirty(pte)		(pte_val(pte) & L_PTE_DIRTY)
-#define pte_young(pte)		(pte_val(pte) & L_PTE_YOUNG)
-#define pte_exec(pte)		(!(pte_val(pte) & L_PTE_XN))
-#define pte_special(pte)	(0)
-
-#define pte_present_user(pte) \
-	((pte_val(pte) & (L_PTE_PRESENT | L_PTE_USER)) == \
-	 (L_PTE_PRESENT | L_PTE_USER))
 
 #define PTE_BIT_FUNC(fn,op) \
 static inline pte_t pte_##fn(pte_t pte) { pte_val(pte) op; return pte; }
diff --git a/arch/arm/include/asm/proc-fns.h b/arch/arm/include/asm/proc-fns.h
index 8fdae9b..c662e55 100644
--- a/arch/arm/include/asm/proc-fns.h
+++ b/arch/arm/include/asm/proc-fns.h
@@ -261,12 +261,21 @@
 
 #ifdef CONFIG_MMU
 
-#define cpu_switch_mm(pgd,mm) cpu_do_switch_mm(virt_to_phys(pgd),mm)
+#ifndef CONFIG_ARM_FCSE_BEST_EFFORT
+#define cpu_switch_mm(pgd,mm,fcse_switch)			\
+	({							\
+		(void)(fcse_switch);				\
+		cpu_do_switch_mm(virt_to_phys(pgd), (mm));	\
+	})
+#else /* CONFIG_ARM_FCSE_BEST_EFFORT */
+#define cpu_switch_mm(pgd,mm,fcse_switch)	\
+	cpu_do_switch_mm(virt_to_phys(pgd), (mm), (fcse_switch))
+#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */
 
 #define cpu_get_pgd()	\
 	({						\
 		unsigned long pg;			\
-		__asm__("mrc	p15, 0, %0, c2, c0, 0"	\
+		__asm__ __volatile__ ("mrc	p15, 0, %0, c2, c0, 0"	\
 			 : "=r" (pg) : : "cc");		\
 		pg &= ~0x3fff;				\
 		(pgd_t *)phys_to_virt(pg);		\
diff --git a/arch/arm/include/asm/processor.h b/arch/arm/include/asm/processor.h
index 67357ba..c71c8f4 100644
--- a/arch/arm/include/asm/processor.h
+++ b/arch/arm/include/asm/processor.h
@@ -24,9 +24,14 @@
 #include <asm/types.h>
 
 #ifdef __KERNEL__
+#ifndef CONFIG_ARM_FCSE
 #define STACK_TOP	((current->personality & ADDR_LIMIT_32BIT) ? \
 			 TASK_SIZE : TASK_SIZE_26)
 #define STACK_TOP_MAX	TASK_SIZE
+#else /* CONFIG_ARM_FCSE */
+#define STACK_TOP	FCSE_TASK_SIZE
+#define STACK_TOP_MAX	FCSE_TASK_SIZE
+#endif /* CONFIG_ARM_FCSE */
 #endif
 
 union debug_insn {
diff --git a/arch/arm/include/asm/resource.h b/arch/arm/include/asm/resource.h
index 734b581..6579eec 100644
--- a/arch/arm/include/asm/resource.h
+++ b/arch/arm/include/asm/resource.h
@@ -1,6 +1,16 @@
 #ifndef _ARM_RESOURCE_H
 #define _ARM_RESOURCE_H
 
+/*
+ * When FCSE is enabled, reduce the default stack size to 1MB, and maximum
+ * to 16MB, the address space is only 32MB.
+ */
+#ifdef CONFIG_ARM_FCSE
+#define _STK_LIM		(1024*1024)
+
+#define _STK_LIM_MAX		(16*1024*1024)
+#endif /* CONFIG_ARM_FCSE */
+
 #include <asm-generic/resource.h>
 
 #endif
diff --git a/arch/arm/include/asm/smp_twd.h b/arch/arm/include/asm/smp_twd.h
index fed9981..9206812 100644
--- a/arch/arm/include/asm/smp_twd.h
+++ b/arch/arm/include/asm/smp_twd.h
@@ -22,7 +22,65 @@ struct clock_event_device;
 
 extern void __iomem *twd_base;
 
+#ifdef CONFIG_IPIPE
+
+#define __ipipe_mach_hrtimer_irq 29
+
+#define __ipipe_mach_ipi_p(irq) ((irq) < 16)
+
+#define __ipipe_mach_ext_hrtimer(cpu)		\
+	({					\
+		(void)(cpu);			\
+		__ipipe_mach_hrtimer_irq;	\
+	})
+
+#define __ipipe_mach_localtimer(irq) (irq)
+
+#define __ipipe_twd_localtimer_p(irq) ((irq) == __ipipe_mach_hrtimer_irq)
+
+#define __ipipe_mach_doirq(irq)					\
+	({							\
+		__ipipe_mach_ipi_p(irq)				\
+			? __ipipe_root_ipi			\
+			: __ipipe_twd_localtimer_p(irq)		\
+			? __ipipe_root_localtimer		\
+			: __ipipe_do_IRQ;			\
+	})
+
+#define __ipipe_mach_ackirq(irq)					\
+	({								\
+		__ipipe_mach_ipi_p(irq)					\
+			? NULL						\
+			: __ipipe_twd_localtimer_p(irq)			\
+			? __ipipe_ack_localtimer			\
+			: __ipipe_ack_irq;				\
+	})
+
+#define __ipipe_mach_relay_ipi(ipi, thiscpu)	\
+	({					\
+		(void)(thiscpu);		\
+		__ipipe_handle_irq(ipi, NULL);	\
+	})
+
+struct irq_desc;
+
+void __ipipe_ack_localtimer(unsigned int irq, struct irq_desc *desc);
+
+#ifdef CONFIG_IPIPE_DEBUG_INTERNAL
+void twd_hrtimer_debug(unsigned int irq);
+#define __ipipe_mach_hrtimer_debug(irq)	twd_hrtimer_debug(irq)
+#else
+#define __ipipe_mach_hrtimer_debug(irq)	do { } while (0)
+#endif
+
+extern unsigned __ipipe_twd_hrtimer_freq;
+#define __ipipe_mach_hrtimer_freq	__ipipe_twd_hrtimer_freq
+
+/* Global timer setup, the global timer is a convenient 64 bits
+ * high-resoluion counter, part of the cortex a9 mpcore */
+void gt_setup(unsigned long base_paddr, unsigned bits);
+#endif /* CONFIG_IPIPE */
+
 int twd_timer_ack(void);
 void twd_timer_setup(struct clock_event_device *);
-
 #endif
diff --git a/arch/arm/include/asm/system.h b/arch/arm/include/asm/system.h
index 97f6d60..2efcabb 100644
--- a/arch/arm/include/asm/system.h
+++ b/arch/arm/include/asm/system.h
@@ -238,10 +238,19 @@ static inline void set_copro_access(unsigned int val)
  */
 extern struct task_struct *__switch_to(struct task_struct *, struct thread_info *, struct thread_info *);
 
+#if defined(CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH) || defined(CONFIG_SMP)
 #define switch_to(prev,next,last)					\
 do {									\
+	local_irq_disable_hw_cond();					\
 	last = __switch_to(prev,task_thread_info(prev), task_thread_info(next));	\
+	local_irq_enable_hw_cond();					\
 } while (0)
+#else /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH && !SMP */
+#define switch_to(prev,next,last)					\
+do {									\
+	last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \
+} while (0)
+#endif /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH && !SMP */
 
 #if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110)
 /*
@@ -302,17 +311,17 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size
 #error SMP is not supported on this platform
 #endif
 	case 1:
-		raw_local_irq_save(flags);
+		local_irq_save_hw(flags);
 		ret = *(volatile unsigned char *)ptr;
 		*(volatile unsigned char *)ptr = x;
-		raw_local_irq_restore(flags);
+		local_irq_restore_hw(flags);
 		break;
 
 	case 4:
-		raw_local_irq_save(flags);
+		local_irq_save_hw(flags);
 		ret = *(volatile unsigned long *)ptr;
 		*(volatile unsigned long *)ptr = x;
-		raw_local_irq_restore(flags);
+		local_irq_restore_hw(flags);
 		break;
 #else
 	case 1:
diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h
index 7b5cc8d..0f4f2b8 100644
--- a/arch/arm/include/asm/thread_info.h
+++ b/arch/arm/include/asm/thread_info.h
@@ -145,6 +145,12 @@ extern void vfp_flush_hwstate(struct thread_info *);
 #define TIF_FREEZE		19
 #define TIF_RESTORE_SIGMASK	20
 #define TIF_SECCOMP		21
+#ifdef CONFIG_IPIPE
+#define TIF_MMSWITCH_INT	22
+#ifdef CONFIG_ARM_FCSE
+#define TIF_SWITCHED		23
+#endif /* CONFIG_ARM_FCSE */
+#endif /* CONFIG_IPIPE */
 
 #define _TIF_SIGPENDING		(1 << TIF_SIGPENDING)
 #define _TIF_NEED_RESCHED	(1 << TIF_NEED_RESCHED)
@@ -155,6 +161,12 @@ extern void vfp_flush_hwstate(struct thread_info *);
 #define _TIF_FREEZE		(1 << TIF_FREEZE)
 #define _TIF_RESTORE_SIGMASK	(1 << TIF_RESTORE_SIGMASK)
 #define _TIF_SECCOMP		(1 << TIF_SECCOMP)
+#ifdef CONFIG_IPIPE
+#define _TIF_MMSWITCH_INT	(1 << TIF_MMSWITCH_INT)
+#ifdef CONFIG_ARM_FCSE
+#define _TIF_SWITCHED		(1 << TIF_SWITCHED)
+#endif /* CONFIG_ARM_FCSE */
+#endif /* CONFIG_IPIPE */
 
 /*
  * Change these and you break ASM code in entry-common.S
diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h
index d2005de..f84cd57 100644
--- a/arch/arm/include/asm/tlbflush.h
+++ b/arch/arm/include/asm/tlbflush.h
@@ -218,6 +218,7 @@
 #ifndef __ASSEMBLY__
 
 #include <linux/sched.h>
+#include <asm/fcse.h>
 
 struct cpu_tlb_fns {
 	void (*flush_user_range)(unsigned long, unsigned long, struct vm_area_struct *);
@@ -408,7 +409,8 @@ local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
 	const int zero = 0;
 	const unsigned int __tlb_flag = __cpu_tlb_flags;
 
-	uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm);
+	uaddr = (fcse_va_to_mva(vma->vm_mm, uaddr) & PAGE_MASK)
+		| ASID(vma->vm_mm);
 
 	if (tlb_flag(TLB_WB))
 		dsb();
@@ -545,7 +547,15 @@ static inline void clean_pmd_entry(pmd_t *pmd)
 /*
  * Convert calls to our calling convention.
  */
-#define local_flush_tlb_range(vma,start,end)	__cpu_flush_user_tlb_range(start,end,vma)
+#define local_flush_tlb_range(vma, start, end)			\
+	({							\
+		struct mm_struct *_mm = (vma)->vm_mm;		\
+		unsigned long _start, _end;			\
+		_start = fcse_va_to_mva(_mm, start);		\
+		_end = fcse_va_to_mva(_mm, end);		\
+		__cpu_flush_user_tlb_range(_start, _end, vma);	\
+	})
+
 #define local_flush_tlb_kernel_range(s,e)	__cpu_flush_kern_tlb_range(s,e)
 
 #ifndef CONFIG_SMP
diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile
index 185ee82..82a583f 100644
--- a/arch/arm/kernel/Makefile
+++ b/arch/arm/kernel/Makefile
@@ -23,7 +23,7 @@ obj-$(CONFIG_LEDS)		+= leds.o
 obj-$(CONFIG_OC_ETM)		+= etm.o
 
 obj-$(CONFIG_ISA_DMA_API)	+= dma.o
-obj-$(CONFIG_ARCH_ACORN)	+= ecard.o 
+obj-$(CONFIG_ARCH_ACORN)	+= ecard.o
 obj-$(CONFIG_FIQ)		+= fiq.o
 obj-$(CONFIG_MODULES)		+= armksyms.o module.o
 obj-$(CONFIG_ARTHUR)		+= arthur.o
@@ -67,5 +67,7 @@ endif
 head-y			:= head$(MMUEXT).o
 obj-$(CONFIG_DEBUG_LL)	+= debug.o
 obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
+obj-$(CONFIG_IPIPE)	+= ipipe.o
+obj-$(CONFIG_IPIPE_ARM_KUSER_TSC) += ipipe_tsc.o ipipe_tsc_asm.o
 
 extra-y := $(head-y) init_task.o vmlinux.lds
diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S
index 2b46fea..9d84974 100644
--- a/arch/arm/kernel/entry-armv.S
+++ b/arch/arm/kernel/entry-armv.S
@@ -4,6 +4,7 @@
  *  Copyright (C) 1996,1997,1998 Russell King.
  *  ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk)
  *  nommu support by Hyok S. Choi (hyok.choi@samsung.com)
+ *  Copyright (C) 2005 Stelian Pop.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -41,6 +42,25 @@
 #endif
 	arch_irq_handler_default
 9997:
+#ifdef CONFIG_IPIPE
+#if !defined(CONFIG_SMP)
+	ldr	r0, =ipipe_percpu_domain
+	ldr	r1, =ipipe_root
+	ldr	r0, [r0]
+	cmp	r0, r1
+#if __GNUC__ >= 4
+	ldreq	r0, =__ipipe_root_status
+#else /* gcc < 4 */
+	ldreq 	r0, =__ipipe_root_status_addr
+	ldreq	r0, [r0]
+#endif /* gcc < 4 */
+	ldreq	r0, [r0]
+	tsteq	r0, #1
+#else /* CONFIG_SMP */
+	bl	__ipipe_check_root_interruptible
+	cmp	r0, #1
+#endif /* CONFIG_SMP */
+#endif /* CONFIG_IPIPE */
 	.endm
 
 #ifdef CONFIG_KPROBES
@@ -61,6 +81,21 @@
 	mov	r1, #\reason
 	.endm
 
+	.macro fcse_dabt_mva_to_va
+#ifdef	CONFIG_ARM_FCSE
+	@
+	@ If FCSE is enabled the data abort fault address must be
+	@ converted from MVA to VA. This must happen before the irqs are
+	@ enabled if PREEMPT is enabled, as context switches may
+	@ change the FCSE pid.
+	@
+	mrc     p15, 0, r2, c13, c0, 0
+	eor	r2, r2, r0
+	tst	r2, #0xfe000000
+	moveq   r0, r2
+#endif
+	.endm
+
 __pabt_invalid:
 	inv_entry BAD_PREFETCH
 	b	common_invalid
@@ -174,6 +209,7 @@ __dabt_svc:
 #else
 	bl	CPU_DABORT_HANDLER
 #endif
+	fcse_dabt_mva_to_va
 
 	@
 	@ set desired IRQ state, then call main handler
@@ -205,14 +241,25 @@ __irq_svc:
 #endif
 #ifdef CONFIG_PREEMPT
 	get_thread_info tsk
+#ifndef CONFIG_IPIPE
 	ldr	r8, [tsk, #TI_PREEMPT]		@ get preempt count
 	add	r7, r8, #1			@ increment it
 	str	r7, [tsk, #TI_PREEMPT]
 #endif
+#endif
 
 	irq_handler
+#ifdef CONFIG_IPIPE
+	ldrne	r4, [sp, #S_PSR]		@ irqs are already disabled
+	bne	__ipipe_fast_svc_irq_exit
+#endif
+
 #ifdef CONFIG_PREEMPT
+#ifndef	CONFIG_IPIPE
 	str	r8, [tsk, #TI_PREEMPT]		@ restore preempt count
+#else
+	ldr	r8, [tsk, #TI_PREEMPT]
+#endif
 	ldr	r0, [tsk, #TI_FLAGS]		@ get flags
 	teq	r8, #0				@ if preempt count != 0
 	movne	r0, #0				@ force flags to 0
@@ -224,6 +271,9 @@ __irq_svc:
 	tst	r4, #PSR_I_BIT
 	bleq	trace_hardirqs_on
 #endif
+#ifdef CONFIG_IPIPE
+__ipipe_fast_svc_irq_exit:
+#endif
 	svc_exit r4				@ return from exception
  UNWIND(.fnend		)
 ENDPROC(__irq_svc)
@@ -233,12 +283,16 @@ ENDPROC(__irq_svc)
 #ifdef CONFIG_PREEMPT
 svc_preempt:
 	mov	r8, lr
+#ifndef CONFIG_IPIPE
 1:	bl	preempt_schedule_irq		@ irq en/disable is done inside
+#else /* CONFIG_IPIPE */
+1:	bl	__ipipe_preempt_schedule_irq	@ irq en/disable is done inside
+#endif /* CONFIG_IPIPE */
 	ldr	r0, [tsk, #TI_FLAGS]		@ get new tasks TI_FLAGS
 	tst	r0, #_TIF_NEED_RESCHED
 	moveq	pc, r8				@ go again
 	b	1b
-#endif
+#endif /* CONFIG_PREEMPT */
 
 	.align	5
 __und_svc:
@@ -251,6 +305,15 @@ __und_svc:
 	svc_entry
 #endif
 
+#ifdef CONFIG_IPIPE
+	mov	r4, r2
+	mov	r0, #7				@ r0 = IPIPE_TRAP_UNDEFINSTR
+	mov	r1, sp				@ r1 = &regs
+	bl	__ipipe_dispatch_event		@ branch to trap handler
+	cmp	r0, #0
+	bne	1f
+	mov	r2, r4
+#endif /* CONFIG_IPIPE */
 	@
 	@ call emulation code, which returns using r9 if it has emulated
 	@ the instruction, or the more conventional lr if we are to treat
@@ -417,6 +480,7 @@ __dabt_usr:
 #else
 	bl	CPU_DABORT_HANDLER
 #endif
+	fcse_dabt_mva_to_va
 
 	@
 	@ IRQs on, then call the main handler
@@ -436,13 +500,21 @@ __irq_usr:
 
 	get_thread_info tsk
 #ifdef CONFIG_PREEMPT
+#ifndef CONFIG_IPIPE
 	ldr	r8, [tsk, #TI_PREEMPT]		@ get preempt count
 	add	r7, r8, #1			@ increment it
 	str	r7, [tsk, #TI_PREEMPT]
 #endif
+#endif
 
 	irq_handler
+#ifdef CONFIG_IPIPE
+	beq	__ipipe_usr_irq_continue
+	slow_restore_user_regs 		@ Fast exit path over non-root domains
+__ipipe_usr_irq_continue:
+#endif
 #ifdef CONFIG_PREEMPT
+#ifndef CONFIG_IPIPE
 	ldr	r0, [tsk, #TI_PREEMPT]
 	str	r8, [tsk, #TI_PREEMPT]
 	teq	r0, r7
@@ -450,6 +522,7 @@ __irq_usr:
  THUMB(	movne	r0, #0		)
  THUMB(	strne	r0, [r0]	)
 #endif
+#endif
 
 	mov	why, #0
 	b	ret_to_user
@@ -462,6 +535,17 @@ ENDPROC(__irq_usr)
 __und_usr:
 	usr_entry
 
+#ifdef CONFIG_IPIPE
+	mov	r0, #7				@ r0 = IPIPE_TRAP_UNDEFINSTR
+	mov	r1, sp				@ r1 = &regs
+	mov	r4, r2
+	mov	r5, r3
+	bl	__ipipe_dispatch_event		@ branch to trap handler
+	cmp	r0, #0
+	bne	ret_from_exception
+	mov	r2, r4
+	mov	r3, r5
+#endif /* CONFIG_IPIPE */
 	@
 	@ fall through to the emulation code, which returns using r9 if
 	@ it has emulated the instruction, or the more conventional lr
@@ -698,6 +782,21 @@ __pabt_usr:
 ENTRY(ret_from_exception)
  UNWIND(.fnstart	)
  UNWIND(.cantunwind	)
+#ifdef CONFIG_IPIPE
+#if !defined(CONFIG_SMP)
+	ldr	r0, =ipipe_percpu_domain
+	ldr	r1, =ipipe_root
+	ldr	r0, [r0]
+	cmp	r0, r1
+#else /* CONFIG_SMP */
+	bl     __ipipe_check_root
+	cmp     r0, #1
+#endif /* CONFIG_SMP */
+	beq     1f
+	disable_irq
+	slow_restore_user_regs          @ Fast exit path over non-root domains
+1:
+#endif /* CONFIG_IPIPE */
 	get_thread_info tsk
 	mov	why, #0
 	b	ret_to_user
@@ -735,7 +834,11 @@ ENTRY(__switch_to)
 	add	r4, r2, #TI_CPU_SAVE
 	ldr	r0, =thread_notify_head
 	mov	r1, #THREAD_NOTIFY_SWITCH
+#ifndef CONFIG_IPIPE
 	bl	atomic_notifier_call_chain
+#else /* CONFIG_IPIPE */
+	bl	__ipipe_switch_to_notifier_call_chain
+#endif /* CONFIG_IPIPE */
 #if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP)
 	str	r7, [r8]
 #endif
@@ -789,6 +892,50 @@ ENDPROC(__switch_to)
 #endif
 	.endm
 
+#ifdef CONFIG_IPIPE
+/*
+	I-pipe tsc area, here we store data shared with user-space for
+	tsc-emulation. If CONFIG_IPIPE_ARM_KUSER_TSC is enabled
+	__ipipe_kuser_get_tsc will be overwritten with the real TSC
+	emulation code.
+*/
+	.globl	__ipipe_tsc_area
+	.equ	__ipipe_tsc_area, CONFIG_VECTORS_BASE + 0x1000 + __ipipe_tsc_area_start - __kuser_helper_end
+
+#ifdef CONFIG_IPIPE_ARM_KUSER_TSC
+	.globl  __ipipe_tsc_addr
+	.equ	__ipipe_tsc_addr, CONFIG_VECTORS_BASE + 0x1000 + .LCcntr_addr - __kuser_helper_end
+
+	.globl	__ipipe_tsc_get
+	.equ	__ipipe_tsc_get, CONFIG_VECTORS_BASE + 0x1000 + __ipipe_kuser_get_tsc - __kuser_helper_end
+#endif
+
+	.align 5
+	.globl  __ipipe_tsc_area_start
+__ipipe_tsc_area_start:
+	.rep  3
+	.word 0
+	.endr
+
+#ifdef CONFIG_IPIPE_ARM_KUSER_TSC
+	.rep  4
+	.word 0
+	.endr
+.LCcntr_addr:
+	.word 0
+
+	.align 5
+__ipipe_kuser_get_tsc:
+	nop
+	mov	r0, #0
+	mov	r1, #0
+	usr_ret	lr
+	.rep 20
+	.word 0
+	.endr
+#endif
+#endif
+
 	.align	5
 	.globl	__kuser_helper_start
 __kuser_helper_start:
@@ -822,7 +969,7 @@ __kuser_helper_start:
  *
  * #define __kernel_dmb() \
  *         asm volatile ( "mov r0, #0xffff0fff; mov lr, pc; sub pc, r0, #95" \
- *	        : : : "r0", "lr","cc" )
+ *		: : : "r0", "lr","cc" )
  */
 
 __kuser_memory_barrier:				@ 0xffff0fa0
@@ -987,7 +1134,7 @@ kuser_cmpxchg_fixup:
  * #define __kernel_get_tls() \
  *	({ register unsigned int __val asm("r0"); \
  *         asm( "mov r0, #0xffff0fff; mov lr, pc; sub pc, r0, #31" \
- *	        : "=r" (__val) : : "lr","cc" ); \
+ *		: "=r" (__val) : : "lr","cc" ); \
  *	   __val; })
  */
 
diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S
index 1e7b04a..d0c86b7 100644
--- a/arch/arm/kernel/entry-common.S
+++ b/arch/arm/kernel/entry-common.S
@@ -2,6 +2,7 @@
  *  linux/arch/arm/kernel/entry-common.S
  *
  *  Copyright (C) 2000 Russell King
+ *  Copyright (C) 2005 Stelian Pop.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -37,6 +38,13 @@ ret_fast_syscall:
 	arch_ret_to_user r1, lr
 
 	restore_user_regs fast = 1, offset = S_OFF
+
+#ifdef CONFIG_IPIPE
+__ipipe_fast_exit_syscall:
+	disable_irq				@ disable interrupts
+	slow_restore_user_regs
+#endif /* CONFIG_IPIPE */
+
  UNWIND(.fnend		)
 
 /*
@@ -49,6 +57,7 @@ work_pending:
 	bne	work_resched
 	tst	r1, #_TIF_SIGPENDING|_TIF_NOTIFY_RESUME
 	beq	no_work_pending
+	enable_irq_cond
 	mov	r0, sp				@ 'regs'
 	mov	r2, why				@ 'syscall'
 	tst	r1, #_TIF_SIGPENDING		@ delivering a signal?
@@ -57,6 +66,7 @@ work_pending:
 	b	ret_slow_syscall		@ Check work again
 
 work_resched:
+	enable_irq_cond
 	bl	schedule
 /*
  * "slow" syscall return path.  "why" tells us if this was a real syscall.
@@ -71,16 +81,14 @@ no_work_pending:
 #if defined(CONFIG_IRQSOFF_TRACER)
 	asm_trace_hardirqs_on
 #endif
-	/* perform architecture specific actions before user return */
-	arch_ret_to_user r1, lr
-
-	restore_user_regs fast = 0, offset = 0
+	slow_restore_user_regs
 ENDPROC(ret_to_user)
 
 /*
  * This is how we return from a fork.
  */
 ENTRY(ret_from_fork)
+	enable_irq_cond
 	bl	schedule_tail
 	get_thread_info tsk
 	ldr	r1, [tsk, #TI_FLAGS]		@ check for syscall tracing
@@ -324,8 +332,8 @@ ENDPROC(ftrace_stub)
  *-----------------------------------------------------------------------------
  */
 
-	/* If we're optimising for StrongARM the resulting code won't 
-	   run on an ARM7 and we can save a couple of instructions.  
+	/* If we're optimising for StrongARM the resulting code won't
+	   run on an ARM7 and we can save a couple of instructions.
 								--pb */
 #ifdef CONFIG_CPU_ARM710
 #define A710(code...) code
@@ -410,7 +418,9 @@ ENTRY(vector_swi)
 #endif
 	enable_irq
 
+#ifndef	CONFIG_IPIPE
 	get_thread_info tsk
+#endif /* !CONFIG_IPIPE */
 	adr	tbl, sys_call_table		@ load syscall table pointer
 
 #if defined(CONFIG_OABI_COMPAT)
@@ -428,6 +438,16 @@ ENTRY(vector_swi)
 	eor	scno, scno, #__NR_SYSCALL_BASE	@ check OS number
 #endif
 
+#ifdef CONFIG_IPIPE
+	mov	r1, sp
+	mov	r0, scno
+	bl	__ipipe_syscall_root
+	cmp	r0, #0
+	bgt	__ipipe_fast_exit_syscall
+	get_thread_info tsk
+	blt	ret_slow_syscall
+	ldm	sp, { r0 - r3 }
+#endif /* CONFIG_IPIPE */
 	ldr	r10, [tsk, #TI_FLAGS]		@ check for syscall tracing
 	stmdb	sp!, {r4, r5}			@ push fifth and sixth args
 
@@ -435,7 +455,7 @@ ENTRY(vector_swi)
 	tst	r10, #_TIF_SECCOMP
 	beq	1f
 	mov	r0, scno
-	bl	__secure_computing	
+	bl	__secure_computing
 	add	r0, sp, #S_R0 + S_OFF		@ pointer to regs
 	ldmia	r0, {r0 - r3}			@ have to reload r0 - r3
 1:
@@ -452,7 +472,7 @@ ENTRY(vector_swi)
 2:	mov	why, #0				@ no longer a real syscall
 	cmp	scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE)
 	eor	r0, scno, #__NR_SYSCALL_BASE	@ put OS number back
-	bcs	arm_syscall	
+	bcs	arm_syscall
 	b	sys_ni_syscall			@ not private func
 ENDPROC(vector_swi)
 
@@ -488,6 +508,9 @@ __sys_trace_return:
 __cr_alignment:
 	.word	cr_alignment
 #endif
+#ifdef CONFIG_IPIPE
+	.word	__ipipe_syscall_root
+#endif
 	.ltorg
 
 /*
@@ -643,3 +666,28 @@ ENTRY(sys_oabi_call_table)
 
 #endif
 
+
+#ifdef CONFIG_FRAME_POINTER
+
+	.text
+	.align 0
+	.type arm_return_addr %function
+	.global arm_return_addr
+
+arm_return_addr:
+	mov	ip, r0
+	mov	r0, fp
+3:
+	cmp	r0, #0
+	beq	1f		@ frame list hit end, bail
+	cmp	ip, #0
+	beq	2f		@ reached desired frame
+	ldr	r0, [r0, #-12]  @ else continue, get next fp
+	sub	ip, ip, #1
+	b	3b
+2:
+	ldr	r0, [r0, #-4]   @ get target return address
+1:
+	mov	pc, lr
+
+#endif
diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S
index ae94649..98af3e1 100644
--- a/arch/arm/kernel/entry-header.S
+++ b/arch/arm/kernel/entry-header.S
@@ -22,7 +22,7 @@
 @
 #define S_OFF		8
 
-/* 
+/*
  * The SWI code relies on the fact that R0 is at the bottom of the stack
  * (due to slow/fast restore user regs).
  */
@@ -184,6 +184,12 @@
 #endif
 	.endm
 
+	.macro slow_restore_user_regs
+	/* perform architecture specific actions before user return */
+	arch_ret_to_user r1, lr
+	restore_user_regs fast = 0, offset = 0
+       .endm
+
 /*
  * These are the registers used in the syscall handler, and allow us to
  * have in theory up to 7 arguments to a function - r0 to r6.
diff --git a/arch/arm/kernel/ipipe.c b/arch/arm/kernel/ipipe.c
new file mode 100644
index 0000000..a06bf98
--- /dev/null
+++ b/arch/arm/kernel/ipipe.c
@@ -0,0 +1,675 @@
+/* -*- linux-c -*-
+ * linux/arch/arm/kernel/ipipe.c
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ * Copyright (C) 2004 Wolfgang Grandegger (Adeos/arm port over 2.4).
+ * Copyright (C) 2005 Heikki Lindholm (PowerPC 970 fixes).
+ * Copyright (C) 2005 Stelian Pop.
+ * Copyright (C) 2006-2008 Gilles Chanteperdrix.
+ * Copyright (C) 2010 Philippe Gerum (SMP port).
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-dependent I-PIPE support for ARM.
+ */
+
+#include <linux/kernel.h>
+#include <linux/smp.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/bitops.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/kallsyms.h>
+#include <linux/kprobes.h>
+#include <linux/ipipe_trace.h>
+#include <linux/irq.h>
+#include <linux/irqnr.h>
+#include <asm/system.h>
+#include <asm/atomic.h>
+#include <asm/hardirq.h>
+#include <asm/io.h>
+#include <asm/unistd.h>
+#include <asm/mach/irq.h>
+#include <asm/mmu_context.h>
+
+/* Next tick date (timebase value). */
+DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs);
+DEFINE_PER_CPU(struct mm_struct *,ipipe_active_mm);
+EXPORT_PER_CPU_SYMBOL(ipipe_active_mm);
+#ifdef __IPIPE_FEATURE_PIC_MUTE
+__ipipe_irqbits_t __ipipe_irqbits;
+IPIPE_DEFINE_SPINLOCK(__ipipe_irqbits_lock);
+#endif /* __IPIPE_FEATURE_PIC_MUTE */
+
+asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs);
+
+static void __ipipe_do_IRQ(unsigned irq, void *cookie);
+
+#ifdef CONFIG_SMP
+
+struct __ipipe_vnmidata {
+	void (*fn)(void *);
+	void *arg;
+	cpumask_t cpumask;
+};
+
+static struct __ipipe_vnmislot {
+	ipipe_spinlock_t lock;
+	struct __ipipe_vnmidata *data;
+	ipipe_rwlock_t data_lock;
+} __ipipe_vnmi __cacheline_aligned_in_smp = {
+	.lock		= IPIPE_SPIN_LOCK_UNLOCKED,
+	.data		= NULL,
+	.data_lock	= IPIPE_RW_LOCK_UNLOCKED,
+};
+
+void __ipipe_init_platform(void)
+{
+	unsigned int virq, _virq;
+
+	for (virq = IPIPE_FIRST_IPI; virq <= IPIPE_LAST_IPI; virq++) {
+		_virq = ipipe_alloc_virq();
+		if (virq != _virq)
+			panic("I-pipe: cannot reserve virq #%d (got #%d)\n",
+			      virq, _virq);
+	}
+
+	__ipipe_mach_init_platform();
+}
+
+void __ipipe_stall_root(void)
+{
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
+	set_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
+	local_irq_restore_hw(flags);
+}
+EXPORT_SYMBOL(__ipipe_stall_root);
+
+unsigned long __ipipe_test_and_stall_root(void)
+{
+	unsigned long flags;
+	int x;
+
+	local_irq_save_hw(flags);
+	x = test_and_set_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
+	local_irq_restore_hw(flags);
+
+	return x;
+}
+EXPORT_SYMBOL(__ipipe_test_and_stall_root);
+
+unsigned long __ipipe_test_root(void)
+{
+	unsigned long flags;
+	int x;
+
+	local_irq_save_hw(flags);
+	x = test_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
+	local_irq_restore_hw(flags);
+
+	return x;
+}
+EXPORT_SYMBOL(__ipipe_test_root);
+
+void __ipipe_do_vnmi(unsigned int irq, void *cookie)
+{
+	int cpu = ipipe_processor_id();
+	struct __ipipe_vnmidata *data;
+
+	read_lock(&__ipipe_vnmi.data_lock);
+
+	data = __ipipe_vnmi.data;
+	if (likely(data && cpumask_test_cpu(cpu, &data->cpumask))) {
+		data->fn(data->arg);
+		cpu_clear(cpu, data->cpumask);
+	}
+
+	read_unlock(&__ipipe_vnmi.data_lock);
+}
+
+static inline void
+hook_internal_ipi(struct ipipe_domain *ipd, int virq,
+		  void (*handler)(unsigned int irq, void *cookie))
+{
+	ipd->irqs[virq].acknowledge = NULL;
+	ipd->irqs[virq].handler = handler;
+	ipd->irqs[virq].cookie = NULL;
+	/* Immediately handle in the current domain but *never* pass */
+	ipd->irqs[virq].control =
+		IPIPE_HANDLE_MASK|IPIPE_STICKY_MASK|IPIPE_SYSTEM_MASK;
+}
+
+void __ipipe_hook_critical_ipi(struct ipipe_domain *ipd)
+{
+	hook_internal_ipi(ipd, IPIPE_CRITICAL_IPI, __ipipe_do_critical_sync);
+	hook_internal_ipi(ipd, IPIPE_SERVICE_VNMI, __ipipe_do_vnmi);
+}
+
+cpumask_t __ipipe_set_irq_affinity(unsigned irq, cpumask_t cpumask)
+{
+	cpumask_t oldmask;
+
+	if (irq_to_desc(irq)->chip->set_affinity == NULL)
+		return CPU_MASK_NONE;
+
+	if (cpus_empty(cpumask))
+		return CPU_MASK_NONE; /* Return mask value -- no change. */
+
+	cpus_and(cpumask, cpumask, cpu_online_map);
+	if (cpus_empty(cpumask))
+		return CPU_MASK_NONE;	/* Error -- bad mask value or non-routable IRQ. */
+
+	cpumask_copy(&oldmask, irq_to_desc(irq)->affinity);
+	irq_to_desc(irq)->chip->set_affinity(irq, &cpumask);
+
+	return oldmask;
+}
+
+void __ipipe_send_vnmi(void (*fn)(void *), cpumask_t cpumask, void *arg)
+{
+	struct __ipipe_vnmidata data;
+	unsigned long flags;
+	int cpu;
+
+	data.fn = fn;
+	data.arg = arg;
+	data.cpumask = cpumask;
+
+	while (!spin_trylock_irqsave(&__ipipe_vnmi.lock, flags)) {
+		if (irqs_disabled_hw())
+			__ipipe_do_vnmi(IPIPE_SERVICE_VNMI, NULL);
+		cpu_relax();
+	}
+
+	cpu = ipipe_processor_id();
+	cpu_clear(cpu, data.cpumask);
+	if (cpus_empty(data.cpumask)) {
+		spin_unlock_irqrestore(&__ipipe_vnmi.lock, flags);
+		return;
+	}
+
+	write_lock(&__ipipe_vnmi.data_lock);
+	__ipipe_vnmi.data = &data;
+	write_unlock(&__ipipe_vnmi.data_lock);
+
+	__ipipe_send_ipi(IPIPE_SERVICE_VNMI, data.cpumask);
+	while (!cpus_empty(data.cpumask))
+		cpu_relax();
+
+	write_lock(&__ipipe_vnmi.data_lock);
+	__ipipe_vnmi.data = NULL;
+	write_unlock(&__ipipe_vnmi.data_lock);
+
+	spin_unlock_irqrestore(&__ipipe_vnmi.lock, flags);
+}
+EXPORT_SYMBOL_GPL(__ipipe_send_vnmi);
+
+static void __ipipe_relay_ext_hrtimer(unsigned int irq, void *cookie)
+{
+	int cpu = ipipe_processor_id();
+	/*
+	 * We are running over the root domain here, so hw IRQs are
+	 * ON.  Therefore, we have to use the atomic form of
+	 * __ipipe_schedule_irq_root(). Don't mess with that, or you
+	 * may lose interrupt notifications.
+	 */
+	ipipe_schedule_irq_root(__ipipe_mach_ext_hrtimer(cpu));
+}
+
+#endif	/* CONFIG_SMP */
+
+/*
+ * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline
+ * just like if it has been actually received from a hw source. Also
+ * works for virtual interrupts.
+ */
+int ipipe_trigger_irq(unsigned irq)
+{
+	unsigned long flags;
+
+#ifdef CONFIG_IPIPE_DEBUG
+	if (irq >= IPIPE_NR_IRQS)
+		return -EINVAL;
+	if (ipipe_virtual_irq_p(irq)) {
+		if (!test_bit(irq - IPIPE_VIRQ_BASE,
+			      &__ipipe_virtual_irq_map))
+			return -EINVAL;
+	} else if (irq_to_desc(irq) == NULL)
+		return -EINVAL;
+#endif
+	local_irq_save_hw(flags);
+	__ipipe_handle_irq(irq, NULL);
+	local_irq_restore_hw(flags);
+
+	return 1;
+}
+
+int ipipe_get_sysinfo(struct ipipe_sysinfo *info)
+{
+	info->sys_nr_cpus = num_online_cpus();
+	info->sys_cpu_freq = __ipipe_cpu_freq;
+	info->sys_hrtimer_irq = __ipipe_mach_hrtimer_irq;
+	info->sys_hrtimer_freq = __ipipe_mach_hrtimer_freq;
+	info->sys_hrclock_freq = __ipipe_mach_hrclock_freq;
+	__ipipe_mach_get_tscinfo(&info->arch_tsc);
+
+	return 0;
+}
+
+static void __ipipe_ack_irq(unsigned irq, struct irq_desc *desc)
+{
+	desc->ipipe_ack(irq, desc);
+}
+
+static void __maybe_unused __ipipe_ack_timerirq(unsigned int irq, struct irq_desc *desc)
+{
+	desc->ipipe_ack(irq, desc);
+	__ipipe_mach_acktimer();
+	desc->ipipe_end(irq, desc);
+}
+
+void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
+{
+	unsigned long flags;
+	irq_to_desc(irq)->status &= ~IRQ_DISABLED;
+
+#ifdef __IPIPE_FEATURE_PIC_MUTE
+	spin_lock_irqsave(&__ipipe_irqbits_lock, flags);
+	if (ipd == &ipipe_root)
+		__ipipe_irqbits[irq / BITS_PER_LONG]
+			|= (1 << (irq % BITS_PER_LONG));
+	else
+		__ipipe_irqbits[irq / BITS_PER_LONG]
+			&= ~(1 << (irq % BITS_PER_LONG));
+	spin_unlock_irqrestore(&__ipipe_irqbits_lock, flags);
+
+	__ipipe_mach_enable_irqdesc(ipd, irq);
+#else
+	(void) flags;
+#endif /* __IPIPE_FEATURE_PIC_MUTE */
+}
+EXPORT_SYMBOL(__ipipe_enable_irqdesc);
+
+#ifdef __IPIPE_FEATURE_PIC_MUTE
+void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
+{
+	unsigned long flags;
+
+	__ipipe_mach_disable_irqdesc(ipd, irq);
+
+	spin_lock_irqsave(&__ipipe_irqbits_lock, flags);
+	if (ipd == &ipipe_root || !ipipe_root.irqs[irq].handler)
+		__ipipe_irqbits[irq / BITS_PER_LONG]
+			&= ~(1 << (irq % BITS_PER_LONG));
+	spin_unlock_irqrestore(&__ipipe_irqbits_lock, flags);
+
+}
+EXPORT_SYMBOL(__ipipe_disable_irqdesc);
+EXPORT_SYMBOL(ipipe_mute_pic);
+EXPORT_SYMBOL(ipipe_unmute_pic);
+#endif /* __IPIPE_FEATURE_PIC_MUTE */
+
+/*
+ * __ipipe_enable_pipeline() -- We are running on the boot CPU, hw
+ * interrupts are off, and secondary CPUs are still lost in space.
+ */
+void __ipipe_enable_pipeline(void)
+{
+	unsigned long flags;
+	unsigned int irq;
+
+#ifdef CONFIG_CPU_ARM926T
+	/*
+	 * We do not want "wfi" to be called in arm926ejs based
+	 * processor, as this causes Linux to disable the I-cache
+	 * when idle.
+	 */
+	disable_hlt();
+#endif
+	flags = ipipe_critical_enter(NULL);
+
+	/* First, virtualize all interrupts from the root domain. */
+
+	for (irq = 0; irq < NR_IRQS; irq++)
+		ipipe_virtualize_irq(ipipe_root_domain,
+				     irq,
+				     (ipipe_irq_handler_t)__ipipe_mach_doirq(irq),
+				     NULL, __ipipe_mach_ackirq(irq),
+				     IPIPE_HANDLE_MASK | IPIPE_PASS_MASK);
+#ifdef CONFIG_SMP
+	/*
+	 * If the external hrtimer IRQ is remapped internally to a
+	 * common localtimer IRQ number, we have to relay the virtual
+	 * localtimer interrupt reaching the root stage to the proper
+	 * external IRQ handlers defined by the kernel.
+	 *
+	 * NOTE: we don't make any assumption regarding whether the
+	 * virtual localtimer interrupt should be acked or not, so we
+	 * set the ack handler as the platform-specific
+	 * __ipipe_mach_ackirq() tells us, like for any other
+	 * interrupt.
+	 */
+	if (__ipipe_mach_ext_hrtimer(0) != __ipipe_mach_hrtimer_irq)
+		ipipe_virtualize_irq(ipipe_root_domain,
+				     __ipipe_mach_hrtimer_irq,
+				     __ipipe_relay_ext_hrtimer,
+				     NULL, __ipipe_mach_ackirq(__ipipe_mach_hrtimer_irq),
+				     IPIPE_HANDLE_MASK);
+#endif
+
+	ipipe_critical_exit(flags);
+}
+
+#ifdef CONFIG_SMP
+asmlinkage int __ipipe_check_root(void)
+{
+	return ipipe_root_domain_p;
+}
+
+asmlinkage int __ipipe_check_root_interruptible(void)
+{
+	return __ipipe_root_domain_p && !irqs_disabled();
+}
+#endif
+
+__kprobes int
+__ipipe_switch_to_notifier_call_chain(struct atomic_notifier_head *nh,
+				      unsigned long val, void *v)
+{
+	unsigned long flags;
+	int ret;
+
+	local_irq_save(flags);
+	ret = atomic_notifier_call_chain(nh, val, v);
+	__local_irq_restore_nosync(flags);
+
+	return ret;
+}
+
+asmlinkage int __ipipe_syscall_root(unsigned long scno, struct pt_regs *regs)
+{
+	struct ipipe_percpu_domain_data *p;
+	unsigned long orig_r7;
+	int ret = 0;
+
+	WARN_ON_ONCE(irqs_disabled_hw());
+
+	/*
+	 * We use r7 to pass the syscall number to the other domains.
+	 */
+	orig_r7 = regs->ARM_r7;
+	regs->ARM_r7 = __NR_SYSCALL_BASE + scno;
+
+	/*
+	 * This routine either returns:
+	 * 0 -- if the syscall is to be passed to Linux;
+	 * >0 -- if the syscall should not be passed to Linux, and no
+	 * tail work should be performed;
+	 * <0 -- if the syscall should not be passed to Linux but the
+	 * tail work has to be performed (for handling signals etc).
+	 */
+
+	if (!__ipipe_syscall_watched_p(current, regs->ARM_r7) ||
+	    !__ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL))
+		goto out;
+
+	ret = __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL, regs);
+
+	local_irq_disable_hw();
+
+	/*
+	 * This is the end of the syscall path, so we may
+	 * safely assume a valid Linux task stack here.
+	 */
+	if (current->ipipe_flags & PF_EVTRET) {
+		current->ipipe_flags &= ~PF_EVTRET;
+		__ipipe_dispatch_event(IPIPE_EVENT_RETURN, regs);
+	}
+
+	if (!__ipipe_root_domain_p)
+		ret = -1;
+	else {
+		p = ipipe_root_cpudom_ptr();
+		if (__ipipe_ipending_p(p))
+			__ipipe_sync_pipeline();
+	}
+
+	local_irq_enable_hw();
+out:
+	regs->ARM_r7 = orig_r7;
+
+	return -ret;
+}
+
+/*
+ * __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic
+ * interrupt protection log is maintained here for each domain. Hw
+ * interrupts are off on entry.
+ */
+void __ipipe_handle_irq(int irq, struct pt_regs *regs)
+{
+	struct ipipe_domain *this_domain, *next_domain;
+	struct list_head *head, *pos;
+	struct irq_desc *desc;
+	int m_ack;
+
+	/* Software-triggered IRQs do not need any ack. */
+	m_ack = (regs == NULL);
+
+#ifdef CONFIG_IPIPE_DEBUG
+	if (unlikely(irq >= IPIPE_NR_IRQS) ||
+	    (!ipipe_virtual_irq_p(irq) && irq_to_desc(irq) == NULL)) {
+		printk(KERN_ERR "I-pipe: spurious interrupt %d\n", irq);
+		return;
+	}
+#endif
+	this_domain = ipipe_current_domain;
+	if (unlikely(test_bit(IPIPE_STICKY_FLAG, &this_domain->irqs[irq].control)))
+		head = &this_domain->p_link;
+	else {
+		head = __ipipe_pipeline.next;
+		next_domain = list_entry(head, struct ipipe_domain, p_link);
+		if (likely(test_bit(IPIPE_WIRED_FLAG, &next_domain->irqs[irq].control))) {
+			if (!m_ack && next_domain->irqs[irq].acknowledge) {
+				desc = ipipe_virtual_irq_p(irq) ? NULL : irq_to_desc(irq);
+				next_domain->irqs[irq].acknowledge(irq, desc);
+			}
+			__ipipe_dispatch_wired(next_domain, irq);
+			return;
+		}
+	}
+
+	/* Ack the interrupt. */
+	pos = head;
+	while (pos != &__ipipe_pipeline) {
+		next_domain = list_entry(pos, struct ipipe_domain, p_link);
+		prefetch(next_domain);
+		/*
+		 * For each domain handling the incoming IRQ, mark it as
+		 * pending in its log.
+		 */
+		if (test_bit(IPIPE_HANDLE_FLAG, &next_domain->irqs[irq].control)) {
+			/*
+			 * Domains that handle this IRQ are polled for
+			 * acknowledging it by decreasing priority order. The
+			 * interrupt must be made pending _first_ in the
+			 * domain's status flags before the PIC is unlocked.
+			 */
+			__ipipe_set_irq_pending(next_domain, irq);
+
+			if (!m_ack && next_domain->irqs[irq].acknowledge) {
+				desc = ipipe_virtual_irq_p(irq) ? NULL : irq_to_desc(irq);
+				next_domain->irqs[irq].acknowledge(irq, desc);
+				m_ack = 1;
+			}
+		}
+
+		/*
+		 * If the domain does not want the IRQ to be passed down the
+		 * interrupt pipe, exit the loop now.
+		 */
+		if (!test_bit(IPIPE_PASS_FLAG, &next_domain->irqs[irq].control))
+			break;
+
+		pos = next_domain->p_link.next;
+	}
+
+	/*
+	 * If the interrupt preempted the head domain, then do not
+	 * even try to walk the pipeline, unless an interrupt is
+	 * pending for it.
+	 */
+	if (test_bit(IPIPE_AHEAD_FLAG, &this_domain->flags) &&
+	    !__ipipe_ipending_p(ipipe_head_cpudom_ptr()))
+		return;
+
+	/*
+	 * Now walk the pipeline, yielding control to the highest
+	 * priority domain that has pending interrupt(s) or
+	 * immediately to the current domain if the interrupt has been
+	 * marked as 'sticky'. This search does not go beyond the
+	 * current domain in the pipeline.
+	 */
+	__ipipe_walk_pipeline(head);
+}
+
+void __ipipe_exit_irq(struct pt_regs *regs)
+{
+	if (user_mode(regs) &&
+	    (current->ipipe_flags & PF_EVTRET) != 0) {
+		/*
+		 * Testing for user_regs() eliminates foreign stack
+		 * contexts, including from careless domains which did
+		 * not set the foreign stack bit (foreign stacks are
+		 * always kernel-based).
+		 */
+		current->ipipe_flags &= ~PF_EVTRET;
+		__ipipe_dispatch_event(IPIPE_EVENT_RETURN, regs);
+	}
+}
+
+asmlinkage void __ipipe_grab_irq(int irq, struct pt_regs *regs)
+{
+	int cpu = ipipe_processor_id();
+#ifdef irq_finish
+	/* AT91 specific workaround */
+	irq_finish(irq);
+#endif /* irq_finish */
+	/*
+	 * Some SMP platforms may remap different per-CPU local timer
+	 * interrupts onto a single (usually virtual) IRQ, so that the
+	 * I-pipe advertises the same local timer IRQ number to the
+	 * client domains, regardless of the CPU. So we first check
+	 * whether we received an external hrtimer IRQ for the current
+	 * CPU, then remap it to its local timer number if so - most
+	 * platforms won't change this value though.
+	 */
+	if (likely(__ipipe_mach_ext_hrtimer(cpu) == irq)) {
+		__ipipe_mach_hrtimer_debug(irq);
+		irq = __ipipe_mach_localtimer(irq);
+		/*
+		 * Given our deferred dispatching model for regular IRQs, we
+		 * only record CPU regs for the last timer interrupt, so that
+		 * the timer handler charges CPU times properly. It is assumed
+		 * that other interrupt handlers don't actually care for such
+		 * information.
+		 */
+		__raw_get_cpu_var(__ipipe_tick_regs).ARM_cpsr =
+			(ipipe_root_domain_p
+			 ? regs->ARM_cpsr
+			 : regs->ARM_cpsr | PSR_I_BIT);
+		__raw_get_cpu_var(__ipipe_tick_regs).ARM_pc = regs->ARM_pc;
+	}
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+	ipipe_trace_begin(regs->ARM_ORIG_r0);
+#endif
+
+	__ipipe_handle_irq(irq, regs);
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+	ipipe_trace_end(regs->ARM_ORIG_r0);
+#endif
+	__ipipe_exit_irq(regs);
+}
+
+static void __ipipe_do_IRQ(unsigned irq, void *cookie)
+{
+	asm_do_IRQ(irq, &__raw_get_cpu_var(__ipipe_tick_regs));
+}
+
+asmlinkage int __ipipe_dispatch_trap(int trap, struct pt_regs *regs)
+{
+	return ipipe_trap_notify(trap, regs);
+}
+
+#if defined(CONFIG_DEBUG_LL) && defined(CONFIG_IPIPE_DEBUG)
+
+void printascii(const char *s);
+
+static IPIPE_DEFINE_SPINLOCK(serial_debug_lock);
+
+void __ipipe_serial_debug(const char *fmt, ...)
+{
+	unsigned long flags;
+	char buf[128];
+	va_list ap;
+	int n;
+
+	va_start(ap, fmt);
+	n = vsnprintf(buf, sizeof(buf) - 2, fmt, ap);
+	va_end(ap);
+
+	if (n > 0 && buf[n - 1] == '\n') {
+		buf[n] = '\r';
+		buf[n+1] = '\0';
+	}
+
+	spin_lock_irqsave(&serial_debug_lock, flags);
+	printascii(buf);
+	spin_unlock_irqrestore(&serial_debug_lock, flags);
+}
+
+#ifndef CONFIG_SERIAL_8250_CONSOLE
+EXPORT_SYMBOL_GPL(__ipipe_serial_debug);
+#endif
+
+#endif
+
+EXPORT_SYMBOL_GPL(show_stack);
+EXPORT_SYMBOL_GPL(init_mm);
+#ifndef MULTI_CPU
+EXPORT_SYMBOL_GPL(cpu_do_switch_mm);
+#endif
+EXPORT_SYMBOL_GPL(__check_kvm_seq);
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
+EXPORT_SYMBOL_GPL(tasklist_lock);
+#endif /* CONFIG_SMP || CONFIG_DEBUG_SPINLOCK */
+
+#ifdef CONFIG_SPARSE_IRQ
+EXPORT_SYMBOL_GPL(irq_to_desc);
+#else
+EXPORT_SYMBOL_GPL(irq_desc);
+#endif
+
+#ifdef CONFIG_CPU_HAS_ASID
+EXPORT_SYMBOL_GPL(__new_context);
+EXPORT_SYMBOL_GPL(cpu_last_asid);
+#endif /* CONFIG_CPU_HAS_ASID */
diff --git a/arch/arm/kernel/ipipe_tsc.c b/arch/arm/kernel/ipipe_tsc.c
new file mode 100644
index 0000000..c6a69f0
--- /dev/null
+++ b/arch/arm/kernel/ipipe_tsc.c
@@ -0,0 +1,136 @@
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/clocksource.h>
+#include <linux/sched.h>
+#include <linux/ipipe_tickdev.h>
+
+#include <linux/ipipe.h>
+
+#include <asm/cacheflush.h>
+
+typedef unsigned long long __ipipe_tsc_t(void);
+
+extern __ipipe_tsc_t __ipipe_freerunning_64,
+	__ipipe_freerunning_32,
+	__ipipe_freerunning_16,
+	__ipipe_freerunning_countdown,
+	__ipipe_decrementer_16;
+extern unsigned long __ipipe_tsc_addr;
+
+static struct __ipipe_tscinfo tsc_info;
+
+static struct clocksource clksrc = {
+	.name = "ipipe_tsc",
+	.rating = 0x7fffffff,
+	.read = (typeof(clksrc.read))__ipipe_tsc_get,
+	.mask = CLOCKSOURCE_MASK(64),
+	.flags = CLOCK_SOURCE_IS_CONTINUOUS,
+};
+
+struct ipipe_tsc_value_t {
+	unsigned long long last_tsc;
+	unsigned last_cnt;
+};
+
+unsigned long __ipipe_kuser_tsc_freq;
+
+struct ipipe_tsc_value_t *const ipipe_tsc_value =
+	(struct ipipe_tsc_value_t *)&__ipipe_tsc_area[0];
+
+void __ipipe_tsc_register(struct __ipipe_tscinfo *info)
+{
+	__ipipe_tsc_t *implem;
+	unsigned long flags;
+
+	switch(info->type) {
+	case IPIPE_TSC_TYPE_FREERUNNING:
+		switch(info->u.mask) {
+		case 0xffff:
+			implem = &__ipipe_freerunning_16;
+			break;
+		case 0xffffffff:
+			implem = &__ipipe_freerunning_32;
+			break;
+		case 0xffffffffffffffffULL:
+			implem = &__ipipe_freerunning_64;
+			break;
+		default:
+			goto unimplemented;
+		}
+		break;
+
+	case IPIPE_TSC_TYPE_DECREMENTER:
+		if (info->u.mask != 0xffff)
+			goto unimplemented;
+		implem = &__ipipe_decrementer_16;
+		break;
+
+	case IPIPE_TSC_TYPE_FREERUNNING_COUNTDOWN:
+		if (info->u.mask != 0xffffffff)
+			goto unimplemented;
+		implem = &__ipipe_freerunning_countdown;
+		break;
+
+	default:
+	unimplemented:
+		printk("I-pipel: Unimplemented tsc configuration, "
+		       "type: %d, mask: 0x%08Lx\n", info->type, info->u.mask);
+		BUG();
+	}
+
+	tsc_info = *info;
+	__ipipe_tsc_addr = tsc_info.counter_vaddr;
+	if (tsc_info.type == IPIPE_TSC_TYPE_DECREMENTER) {
+		tsc_info.u.dec.last_cnt = &ipipe_tsc_value->last_cnt;
+		tsc_info.u.dec.tsc = &ipipe_tsc_value->last_tsc;
+	} else
+		tsc_info.u.fr.tsc = &ipipe_tsc_value->last_tsc;
+
+	local_irq_save_hw(flags);
+	memcpy(__ipipe_tsc_area + 0x20, implem, 0x60);
+	flush_icache_range((unsigned long)(__ipipe_tsc_area),
+			   (unsigned long)(__ipipe_tsc_area + 0x80));
+	local_irq_restore_hw(flags);
+
+	clksrc.shift = fls(tsc_info.freq) - 1;
+	clksrc.mult = clocksource_hz2mult(tsc_info.freq, clksrc.shift);
+	printk(KERN_INFO "I-pipe, %u.%03u MHz clocksource\n",
+	       tsc_info.freq / 1000000, (tsc_info.freq % 1000000) / 1000);
+	clocksource_register(&clksrc);
+
+	__ipipe_kuser_tsc_freq = tsc_info.freq;
+}
+
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info)
+{
+	*info = tsc_info;
+}
+
+void __ipipe_tsc_update(void)
+{
+	if (tsc_info.type == IPIPE_TSC_TYPE_DECREMENTER) {
+		unsigned cnt = *(unsigned *)tsc_info.counter_vaddr;
+		int offset = ipipe_tsc_value->last_cnt - cnt;
+		if (offset < 0)
+			offset += tsc_info.u.dec.mask + 1;
+		ipipe_tsc_value->last_tsc += offset;
+		ipipe_tsc_value->last_cnt = cnt;
+		return;
+	}
+
+	/* Update last_tsc, in order to remain compatible with legacy
+	   user-space 32 bits free-running counter implementation */
+	ipipe_tsc_value->last_tsc = __ipipe_tsc_get() - 1;
+}
+EXPORT_SYMBOL(__ipipe_tsc_get);
+
+void update_vsyscall(struct timespec *wall_time, struct timespec *wtm,
+		     struct clocksource *clock, u32 mult)
+{
+	if (clock == &clksrc)
+		ipipe_update_hostrt(wall_time, clock);
+}
+
+void update_vsyscall_tz(void)
+{
+}
diff --git a/arch/arm/kernel/ipipe_tsc_asm.S b/arch/arm/kernel/ipipe_tsc_asm.S
new file mode 100644
index 0000000..ae7ea7d
--- /dev/null
+++ b/arch/arm/kernel/ipipe_tsc_asm.S
@@ -0,0 +1,205 @@
+#include <asm/assembler.h>
+#include <asm/asm-offsets.h>
+#include <asm/glue.h>
+
+	.macro	usr_ret, reg
+#ifdef CONFIG_ARM_THUMB
+	bx	\reg
+#else
+	mov	pc, \reg
+#endif
+	.endm
+
+	.macro	usr_reteq, reg
+#ifdef CONFIG_ARM_THUMB
+	bxeq	\reg
+#else
+	moveq	 pc, \reg
+#endif
+	.endm
+
+	.macro	myldrd, rd1, rd2, rtmp, label
+#if __LINUX_ARM_ARCH__ < 5
+	adr	\rtmp, \label
+	ldm	\rtmp, { \rd1, \rd2 }
+#else
+	ldrd	\rd1, \label
+#endif
+	.endm
+
+/*
+	We use the same mechanism as Linux user helpers to store
+	variables and functions related to TSC emulation, so that they
+	can also be used in user-space.
+
+	The function ipipe_tsc_register will copy the proper
+	implemntation to the vectors page. We repeat the data area so
+	that the PC relative operations are computed correctly.
+*/
+
+	.align 5
+	.rep	7
+	.word	0
+	.endr
+.LCfr64_cntr_addr:
+	.word 	0
+
+	.align 5
+	.globl 	__ipipe_freerunning_64
+__ipipe_freerunning_64:
+	ldr	r0, .LCfr64_cntr_addr
+/* User-space entry-point: r0 is the hardware counter virtual address */
+	mov 	r2, r0
+#ifndef CONFIG_CPU_BIG_ENDIAN
+/* Little endian */
+	ldr 	r1, [r2, #4]
+1:	ldr	r0, [r2]
+	ldr	r3, [r2, #4]
+	cmp	r3, r1
+	usr_reteq lr
+	mov	r1, r3
+	b	1b
+#else /* Big endian */
+	ldr 	r0, [r2]
+1:	ldr	r1, [r2, #4]
+	ldr	r3, [r2]
+	cmp	r3, r0
+	usr_reteq lr
+	mov	r0, r3
+	b	1b
+#endif /* Big endian */
+
+	.align 5
+.LCfr32_last_tsc:
+	.rep	7
+	.word	0
+	.endr
+.LCfr32_cntr_addr:
+	.word 	0
+
+	.align 5
+	.globl 	__ipipe_freerunning_32
+__ipipe_freerunning_32:
+	ldr	r0, .LCfr32_cntr_addr
+/* User-space entry-point: r0 is the hardware counter virtual address */
+	myldrd	r2, r3, r1, .LCfr32_last_tsc
+#ifndef CONFIG_CPU_BIG_ENDIAN
+/* Little endian */
+	ldr	r0, [r0]
+	cmp	r2, r0
+	adc	r1, r3, #0
+#else /* Big endian */
+	ldr	r1, [r0]
+	cmp	r3, r1
+	adc	r0, r2, #0
+#endif /* Big endian */
+	usr_ret lr
+
+	.align 5
+.LCfrcd32_last_tsc:
+	.rep	7
+	.word	0
+	.endr
+.LCfrcd32_cntr_addr:
+	.word 	0
+
+	.align 5
+	.globl __ipipe_freerunning_countdown
+__ipipe_freerunning_countdown:
+	ldr	r0, .LCfrcd32_cntr_addr
+/* User-space entry-point: r0 is the hardware counter virtual address */
+	myldrd	r2, r3, r1, .LCfrcd32_last_tsc
+#ifndef CONFIG_CPU_BIG_ENDIAN
+/* Little endian */
+	ldr	r0, [r0]
+	mvn	r0, r0
+	cmp	r2, r0
+	adc	r1, r3, #0
+#else /* Big endian */
+	ldr	r1, [r0]
+	mvn	r1, r1
+	cmp	r3, r1
+	adc	r0, r2, #0
+#endif /* Big endian */
+	usr_ret lr
+
+	.align 5
+.LCfr16_last_tsc:
+	.rep	7
+	.word	0
+	.endr
+.LCfr16_cntr_addr:
+	.word 	0
+
+	.align 5
+	.globl __ipipe_freerunning_16
+__ipipe_freerunning_16:
+	ldr	r0, .LCfr16_cntr_addr
+/* User-space entry-point: r0 is the hardware counter virtual address */
+1:	myldrd	r2, r3, r1, .LCfr16_last_tsc
+	ldr	ip, [r0]
+#ifndef CONFIG_CPU_BIG_ENDIAN
+/* Little endian */
+	ldr	r1, .LCfr16_last_tsc
+	cmp	r1, r2
+	bne	1b
+	mov	r1, r2, lsr #16
+	orr	r0, ip, r1, lsl #16
+	cmp	r2, r0
+	addhis	r0, r0, #0x10000
+	adc	r1, r3, #0
+#else /* Big endian */
+	ldr	r1, .LCfr16_last_tsc + 4
+	cmp	r1, r3
+	bne	1b
+	mov	r1, r3, lsr #16
+	orr	r1, ip, r1, lsl #16
+	cmp	r3, r0
+	addhis	r1, r1, #0x10000
+	adc	r0, r2, #0
+#endif /* Big endian */
+	usr_ret lr
+
+	.align 5
+.LCdec16_last_tsc:
+	.rep	2
+	.word	0
+	.endr
+.LCdec16_last_cnt:
+	.rep	5
+	.word	0
+	.endr
+.LCdec16_cntr_addr:
+	.word 	0
+
+	.align 5
+	.globl __ipipe_decrementer_16
+__ipipe_decrementer_16:
+	ldr	r0, .LCdec16_cntr_addr
+/* User-space entry-point: r0 is the hardware counter virtual address */
+#ifndef CONFIG_CPU_BIG_ENDIAN
+/* Little endian */
+1:	ldr	r1, .LCdec16_last_tsc
+	ldr	ip, [r0]
+	ldr	r2, .LCdec16_last_cnt
+	subs 	ip, r2, ip
+	addcc	ip, ip, #0x10000
+	myldrd	r2, r3, r3, .LCdec16_last_tsc
+	cmp	r1, r2
+	bne	1b
+	adds	r0, ip, r2
+	adc	r1, r3, #0
+#else /* Big endian */
+/* Little endian */
+1:	ldr	r1, .LCdec16_last_tsc + 4
+	ldr	ip, [r0]
+	ldr	r2, .LCdec16_last_cnt
+	subs 	ip, r2, ip
+	addcc	ip, ip, #0x10000
+	myldrd	r2, r3, r3, .LCdec16_last_tsc
+	cmp	r1, r3
+	bne	1b
+	adds	r1, ip, r3
+	adc	r0, r2, #0
+#endif /* Big endian */
+	usr_ret	lr
diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c
index 28536e3..6477009 100644
--- a/arch/arm/kernel/irq.c
+++ b/arch/arm/kernel/irq.c
@@ -135,8 +135,10 @@ asm_do_IRQ(unsigned int irq, struct pt_regs *regs)
 		generic_handle_irq(irq);
 	}
 
+#ifndef CONFIG_IPIPE
 	/* AT91 specific workaround */
 	irq_finish(irq);
+#endif /* !CONFIG_IPIPE */
 
 	irq_exit();
 	set_irq_regs(old_regs);
diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c
index 94bbedb..0726d94 100644
--- a/arch/arm/kernel/process.c
+++ b/arch/arm/kernel/process.c
@@ -155,6 +155,34 @@ void cpu_idle_wait(void)
 	smp_call_function(do_nothing, NULL, 1);
 }
 EXPORT_SYMBOL_GPL(cpu_idle_wait);
+#ifdef CONFIG_IPIPE
+static void __ipipe_halt_root(void)
+{
+	struct ipipe_percpu_domain_data *p;
+
+	/* Emulate idle entry sequence over the root domain. */
+
+	local_irq_disable_hw();
+
+	p = ipipe_root_cpudom_ptr();
+
+	trace_hardirqs_on();
+	clear_bit(IPIPE_STALL_FLAG, &p->status);
+
+	if (unlikely(__ipipe_ipending_p(p))) {
+		__ipipe_sync_pipeline();
+		local_irq_enable_hw();
+	} else {
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+		ipipe_trace_end(0x8000000E);
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+		local_irq_enable_hw();
+		arch_idle();
+	}
+}
+#else /* !CONFIG_IPIPE */
+#define __ipipe_halt_root() arch_idle()
+#endif /* !CONFIG_IPIPE */
 
 /*
  * This is our default idle handler.  We need to disable
@@ -163,7 +191,7 @@ EXPORT_SYMBOL_GPL(cpu_idle_wait);
 static void default_idle(void)
 {
 	if (!need_resched())
-		arch_idle();
+		__ipipe_halt_root();
 	local_irq_enable();
 }
 
diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c
index b13e70f..d7b21f9 100644
--- a/arch/arm/kernel/ptrace.c
+++ b/arch/arm/kernel/ptrace.c
@@ -588,6 +588,10 @@ void ptrace_break(struct task_struct *tsk, struct pt_regs *regs)
 
 static int break_trap(struct pt_regs *regs, unsigned int instr)
 {
+
+	if (ipipe_trap_notify(IPIPE_TRAP_BREAK,regs))
+		return 0;
+
 	ptrace_break(current, regs);
 	return 0;
 }
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index 4539ebc..9ea207d 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -53,6 +53,25 @@ enum ipi_msg_type {
 	IPI_CALL_FUNC,
 	IPI_CALL_FUNC_SINGLE,
 	IPI_CPU_STOP,
+	IPI_CPU_DUMP,
+#ifdef CONFIG_IPIPE
+	IPI_IPIPE_CRITICAL,
+	IPI_IPIPE_0,
+	IPI_IPIPE_1,
+	IPI_IPIPE_2,
+	IPI_IPIPE_3,
+	IPI_IPIPE_VNMI,
+#define IPI_IPIPE_ALL				\
+	((1UL << IPI_IPIPE_CRITICAL)|		\
+	 (1UL << IPI_IPIPE_0)|			\
+	 (1UL << IPI_IPIPE_1)|			\
+	 (1UL << IPI_IPIPE_2)|			\
+	 (1UL << IPI_IPIPE_3)|			\
+	 (1UL << IPI_IPIPE_VNMI))
+#define IPI_ROOT_MASK  IPI_IPIPE_ALL
+#else /* !CONFIG_IPIPE */
+#define IPI_ROOT_MASK  0
+#endif /* !CONFIG_IPIPE */
 };
 
 int __cpuinit __cpu_up(unsigned int cpu)
@@ -287,7 +306,7 @@ asmlinkage void __cpuinit secondary_start_kernel(void)
 	atomic_inc(&mm->mm_count);
 	current->active_mm = mm;
 	cpumask_set_cpu(cpu, mm_cpumask(mm));
-	cpu_switch_mm(mm->pgd, mm);
+	cpu_switch_mm(mm->pgd, mm, 1);
 	enter_lazy_tlb(mm, current);
 	local_flush_tlb_all();
 
@@ -433,6 +452,13 @@ static DEFINE_PER_CPU(struct clock_event_device, percpu_clockevent);
 static void ipi_timer(void)
 {
 	struct clock_event_device *evt = &__get_cpu_var(percpu_clockevent);
+#ifdef CONFIG_IPIPE
+#ifndef CONFIG_IPIPE_ARM_KUSER_TSC
+	__ipipe_mach_update_tsc();
+#else /* CONFIG_IPIPE_ARM_KUSER_TSC */
+	__ipipe_tsc_update();
+#endif /* CONFIG_IPIPE_ARM_KUSER_TSC */
+#endif /* CONFIG_IPIPE */
 	irq_enter();
 	evt->event_handler(evt);
 	irq_exit();
@@ -465,6 +491,61 @@ void show_local_irqs(struct seq_file *p, int prec)
 }
 #endif
 
+#ifdef CONFIG_IPIPE
+
+void __ipipe_mach_release_timer(void)
+{
+	int cpu = ipipe_processor_id();
+	struct clock_event_device *evt = &per_cpu(percpu_clockevent, cpu);
+
+	evt->set_mode(evt->mode, evt);
+	if (evt->mode == CLOCK_EVT_MODE_ONESHOT)
+		evt->set_next_event(__ipipe_mach_ticks_per_jiffy, evt);
+}
+EXPORT_SYMBOL(__ipipe_mach_release_timer);
+
+int __ipipe_send_ipi(unsigned ipi, cpumask_t cpumask)
+{
+	enum ipi_msg_type msg = ipi - IPIPE_FIRST_IPI + IPI_IPIPE_CRITICAL;
+	smp_cross_call(&cpumask, msg);
+	return 0;
+}
+
+asmlinkage void __exception
+__ipipe_grab_ipi(unsigned svc, struct pt_regs *regs) /* hw IRQs off */
+{
+	unsigned int cpu = ipipe_processor_id();
+	int virq;
+
+	/*
+	 * Virtual NMIs ignore the root domain's stall
+	 * bit. When caught over high priority
+	 * domains, virtual VMIs are pipelined the
+	 * usual way as normal interrupts.
+	 */
+	if (svc == IPI_IPIPE_VNMI && ipipe_root_domain_p)
+		__ipipe_do_vnmi(IPIPE_SERVICE_VNMI, NULL);
+	else if ((1 << svc) & IPI_IPIPE_ALL) {
+		virq = svc - IPI_IPIPE_CRITICAL + IPIPE_FIRST_IPI;
+		__ipipe_handle_irq(virq, NULL);
+	} else
+		__ipipe_mach_relay_ipi(svc, cpu);
+
+	__ipipe_exit_irq(regs);
+}
+
+void  __ipipe_root_ipi(unsigned int ipinr, void *cookie)
+{
+	do_IPI(ipinr, &__raw_get_cpu_var(__ipipe_tick_regs));
+}
+
+void  __ipipe_root_localtimer(unsigned int irq, void *cookie)
+{
+	do_local_timer(&__raw_get_cpu_var(__ipipe_tick_regs));
+}
+
+#endif /* CONFIG_IPIPE */
+
 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
 static void smp_timer_broadcast(const struct cpumask *mask)
 {
diff --git a/arch/arm/kernel/smp_twd.c b/arch/arm/kernel/smp_twd.c
index 60636f4..461b25c 100644
--- a/arch/arm/kernel/smp_twd.c
+++ b/arch/arm/kernel/smp_twd.c
@@ -17,15 +17,143 @@
 #include <linux/clockchips.h>
 #include <linux/irq.h>
 #include <linux/io.h>
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/ipipe.h>
 
 #include <asm/smp_twd.h>
 #include <asm/hardware/gic.h>
 
 /* set up by the platform code */
 void __iomem *twd_base;
+static struct clk *twd_clk;
 
 static unsigned long twd_timer_rate;
 
+#if defined(CONFIG_IPIPE) && defined(CONFIG_SMP)
+void __iomem *gt_base;
+
+unsigned __ipipe_twd_hrtimer_freq;
+
+unsigned int __ipipe_mach_ticks_per_jiffy;
+EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy);
+
+int __ipipe_mach_timerstolen;
+EXPORT_SYMBOL(__ipipe_mach_timerstolen);
+
+static DEFINE_PER_CPU(int, timer_mode) = {
+	CLOCK_EVT_MODE_SHUTDOWN
+};
+
+void __ipipe_ack_localtimer(unsigned irq, struct irq_desc *desc)
+{
+	writel_relaxed(1, twd_base + TWD_TIMER_INTSTAT);
+}
+
+asmlinkage void __ipipe_grab_localtimer(struct pt_regs *regs)
+{
+	__ipipe_grab_irq(__ipipe_mach_hrtimer_irq, regs);
+}
+
+void __ipipe_mach_set_dec(unsigned long delay)
+{
+	if (delay <= 0xf) {
+		ipipe_trigger_irq(__ipipe_mach_hrtimer_irq);
+		return;
+	}
+
+	writel_relaxed(delay, twd_base + TWD_TIMER_COUNTER);
+}
+EXPORT_SYMBOL(__ipipe_mach_set_dec);
+
+unsigned long __ipipe_mach_get_dec(void)
+{
+	return __raw_readl(twd_base + TWD_TIMER_COUNTER);
+}
+
+int __ipipe_check_tickdev(const char *devname)
+{
+	return !strcmp(devname, "local_timer");
+}
+
+int twd_timer_ack(void)
+{
+	/*
+	 * Always true if we get there, already handled early by the
+	 * pipeline.
+	 */
+	return 1;
+}
+
+static struct __ipipe_tscinfo tsc_info;
+
+void __cpuinit gt_setup(unsigned long base_paddr, unsigned bits)
+{
+	if (!gt_base) {
+		gt_base = ioremap(base_paddr, SZ_256);
+		BUG_ON(!gt_base);
+
+		/* Start global timer */
+		__raw_writel(1, gt_base + 0x8);
+
+		tsc_info.type = IPIPE_TSC_TYPE_FREERUNNING;
+		tsc_info.freq = twd_timer_rate;
+		tsc_info.counter_vaddr = (unsigned long)gt_base;
+		tsc_info.u.counter_paddr = base_paddr;
+
+		switch(bits) {
+		case 64:
+			tsc_info.u.mask = 0xffffffffffffffffULL;
+			break;
+		case 32:
+			tsc_info.u.mask = 0xffffffff;
+			break;
+		default:
+			/* Only supported as a 32 bits or 64 bits */
+			BUG();
+		}
+
+		__ipipe_tsc_register(&tsc_info);
+
+	}
+}
+
+#ifdef CONFIG_IPIPE_DEBUG_INTERNAL
+
+static DEFINE_PER_CPU(int, irqs);
+
+void twd_hrtimer_debug(unsigned int irq) /* hw interrupt off */
+{
+	int cpu = ipipe_processor_id();
+
+	if ((++per_cpu(irqs, cpu) % HZ) == 0) {
+#if 0
+		__ipipe_serial_debug("%c", 'A' + cpu);
+#else
+		do { } while (0);
+#endif
+	}
+}
+#endif /* CONFIG_IPIPE_DEBUG_INTERNAL */
+#else /* !CONFIG_IPIPE || !CONFIG_SMP*/
+
+/*
+ * local_timer_ack: checks for a local timer interrupt.
+ *
+ * If a local timer interrupt has occurred, acknowledge and return 1.
+ * Otherwise, return 0.
+ */
+int twd_timer_ack(void)
+{
+	if (__raw_readl(twd_base + TWD_TIMER_INTSTAT)) {
+		__raw_writel(1, twd_base + TWD_TIMER_INTSTAT);
+		return 1;
+	}
+
+	return 0;
+}
+#endif /* !CONFIG_IPIPE || !CONFIG_SMP*/
+
 static void twd_set_mode(enum clock_event_mode mode,
 			struct clock_event_device *clk)
 {
@@ -33,10 +161,9 @@ static void twd_set_mode(enum clock_event_mode mode,
 
 	switch (mode) {
 	case CLOCK_EVT_MODE_PERIODIC:
-		/* timer load already set up */
 		ctrl = TWD_TIMER_CONTROL_ENABLE | TWD_TIMER_CONTROL_IT_ENABLE
 			| TWD_TIMER_CONTROL_PERIODIC;
-		__raw_writel(twd_timer_rate / HZ, twd_base + TWD_TIMER_LOAD);
+		__raw_writel((twd_timer_rate + HZ / 2) / HZ, twd_base + TWD_TIMER_LOAD);
 		break;
 	case CLOCK_EVT_MODE_ONESHOT:
 		/* period set, and timer enabled in 'next_event' hook */
@@ -49,6 +176,9 @@ static void twd_set_mode(enum clock_event_mode mode,
 	}
 
 	__raw_writel(ctrl, twd_base + TWD_TIMER_CONTROL);
+#if defined(CONFIG_IPIPE) && defined(CONFIG_SMP)
+	per_cpu(timer_mode, smp_processor_id()) = mode;
+#endif
 }
 
 static int twd_set_next_event(unsigned long evt,
@@ -64,22 +194,6 @@ static int twd_set_next_event(unsigned long evt,
 	return 0;
 }
 
-/*
- * local_timer_ack: checks for a local timer interrupt.
- *
- * If a local timer interrupt has occurred, acknowledge and return 1.
- * Otherwise, return 0.
- */
-int twd_timer_ack(void)
-{
-	if (__raw_readl(twd_base + TWD_TIMER_INTSTAT)) {
-		__raw_writel(1, twd_base + TWD_TIMER_INTSTAT);
-		return 1;
-	}
-
-	return 0;
-}
-
 static void __cpuinit twd_calibrate_rate(void)
 {
 	unsigned long count;
@@ -115,7 +229,7 @@ static void __cpuinit twd_calibrate_rate(void)
 		twd_timer_rate = (0xFFFFFFFFU - count) * (HZ / 5);
 
 		printk("%lu.%02luMHz.\n", twd_timer_rate / 1000000,
-			(twd_timer_rate / 1000000) % 100);
+			(twd_timer_rate / 10000) % 100);
 	}
 }
 
@@ -124,7 +238,29 @@ static void __cpuinit twd_calibrate_rate(void)
  */
 void __cpuinit twd_timer_setup(struct clock_event_device *clk)
 {
-	twd_calibrate_rate();
+	if (twd_clk == NULL) {
+		twd_clk = clk_get(NULL, "smp_twd");
+		if (IS_ERR(twd_clk))
+			pr_warn("%s: no clock found\n", __func__);
+		else
+			clk_enable(twd_clk);
+	}
+
+	if (!IS_ERR(twd_clk)) {
+		twd_timer_rate = clk_get_rate(twd_clk);
+		printk(KERN_INFO "local timer: %lu.%02luMHz.\n",
+		       twd_timer_rate / 1000000,
+		       (twd_timer_rate / 10000) % 100);
+	} else
+		twd_calibrate_rate();
+
+#if defined(CONFIG_IPIPE) && defined(CONFIG_SMP)
+	__ipipe_twd_hrtimer_freq = twd_timer_rate;
+	printk(KERN_INFO "I-pipe, %u.%03u MHz timer\n",
+	       __ipipe_twd_hrtimer_freq / 1000000,
+	       (__ipipe_twd_hrtimer_freq % 1000000) / 1000);
+	__ipipe_mach_ticks_per_jiffy = (twd_timer_rate + HZ / 2) / HZ;
+#endif
 
 	clk->name = "local_timer";
 	clk->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT |
diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c
index ee57640..c9477e2 100644
--- a/arch/arm/kernel/traps.c
+++ b/arch/arm/kernel/traps.c
@@ -765,8 +765,13 @@ void __init early_trap_init(void)
 #endif
 	extern char __stubs_start[], __stubs_end[];
 	extern char __vectors_start[], __vectors_end[];
+#ifndef CONFIG_IPIPE
 	extern char __kuser_helper_start[], __kuser_helper_end[];
 	int kuser_sz = __kuser_helper_end - __kuser_helper_start;
+#else /* !CONFIG_IPIPE */
+	extern char __ipipe_tsc_area_start[], __kuser_helper_end[];
+	int kuser_sz = __kuser_helper_end - __ipipe_tsc_area_start;
+#endif /* !CONFIG_IPIPE */
 
 	/*
 	 * Copy the vectors, stubs and kuser helpers (in entry-armv.S)
@@ -775,7 +780,12 @@ void __init early_trap_init(void)
 	 */
 	memcpy((void *)vectors, __vectors_start, __vectors_end - __vectors_start);
 	memcpy((void *)vectors + 0x200, __stubs_start, __stubs_end - __stubs_start);
+#ifndef CONFIG_IPIPE
 	memcpy((void *)vectors + 0x1000 - kuser_sz, __kuser_helper_start, kuser_sz);
+#else /* !CONFIG_IPIPE */
+	BUG_ON(0x1000 - kuser_sz < 0x200 + __stubs_end - __stubs_start);
+	memcpy((void *)vectors + 0x1000 - kuser_sz, __ipipe_tsc_area_start, kuser_sz);
+#endif /* !CONFIG_IPIPE */
 
 	/*
 	 * Do processor specific fixups for the kuser helpers
diff --git a/arch/arm/mach-at91/Kconfig b/arch/arm/mach-at91/Kconfig
index 1939023..f2db355 100644
--- a/arch/arm/mach-at91/Kconfig
+++ b/arch/arm/mach-at91/Kconfig
@@ -474,6 +474,19 @@ endif
 
 # ----------------------------------------------------------
 
+comment "Adeos I-pipe Options"
+
+config IPIPE_AT91_TC
+	depends on IPIPE
+	int "AT91 TC used as time base by Adeos I-pipe"
+	default 0
+	help
+	When Adeos interrupt pipeline is enabled, TC0 is used by default
+	as time base, but you can use TC1 or TC2 by setting this variable to 1
+	or 2. This should only be needed to avoid conflicts with other drivers.
+
+# ----------------------------------------------------------
+
 comment "AT91 Board Options"
 
 config MTD_AT91_DATAFLASH_CARD
diff --git a/arch/arm/mach-at91/Makefile b/arch/arm/mach-at91/Makefile
index a83835e..5e11f9f 100644
--- a/arch/arm/mach-at91/Makefile
+++ b/arch/arm/mach-at91/Makefile
@@ -95,3 +95,14 @@ obj-$(CONFIG_CPU_IDLE)	+= cpuidle.o
 ifeq ($(CONFIG_PM_DEBUG),y)
 CFLAGS_pm.o += -DDEBUG
 endif
+
+ifeq ($(CONFIG_IPIPE),y)
+obj-y := $(filter-out at91rm9200_time.o at91sam926x_time.o at91x40_time.o, $(obj-y))
+obj-$(CONFIG_ARCH_AT91RM9200)   += at91_ipipe_time.o
+obj-$(CONFIG_ARCH_AT91SAM9260)  += at91_ipipe_time.o
+obj-$(CONFIG_ARCH_AT91SAM9261)  += at91_ipipe_time.o
+obj-$(CONFIG_ARCH_AT91SAM9263)	+= at91_ipipe_time.o
+obj-$(CONFIG_ARCH_AT91SAM9RL)	+= at91_ipipe_time.o
+obj-$(CONFIG_ARCH_AT91SAM9G20)  += at91_ipipe_time.o
+obj-$(CONFIG_ARCH_AT91X40)	+= at91_ipipe_time.o
+endif
diff --git a/arch/arm/mach-at91/at91_ipipe_time.c b/arch/arm/mach-at91/at91_ipipe_time.c
new file mode 100644
index 0000000..b0aa6a4
--- /dev/null
+++ b/arch/arm/mach-at91/at91_ipipe_time.c
@@ -0,0 +1,316 @@
+/*
+ * linux/arch/arm/mach-at91/at91_ipipe_time.c
+ *
+ * Copyright (C) 2007 Gilles Chanteperdrix <gilles.chanteperdrix@xenomai.org>
+ *
+ * Adaptation to AT91SAM926x:
+ * Copyright (C) 2007 Gregory CLEMENT, Adeneo
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/kernel.h>
+#include <linux/clockchips.h>
+#include <linux/clk.h>
+#include <linux/stringify.h>
+#include <linux/err.h>
+#include <linux/console.h>
+#include <linux/module.h>
+#include <linux/ipipe.h>
+
+#include <asm/io.h>
+#include <asm/mach/time.h>
+
+#include <mach/hardware.h>
+#include <mach/at91_st.h>
+#include <mach/at91_tc.h>
+#include <mach/at91_pit.h>
+#include "clock.h"
+
+#if defined(CONFIG_ARCH_AT91RM9200)
+#define AT91_ID_TC0 AT91RM9200_ID_TC0
+#define AT91_ID_TC1 AT91RM9200_ID_TC1
+#define AT91_ID_TC2 AT91RM9200_ID_TC2
+#elif defined(CONFIG_ARCH_AT91SAM9260) || defined(CONFIG_ARCH_AT91SAM9G20)
+#define AT91_ID_TC0 AT91SAM9260_ID_TC0
+#define AT91_ID_TC1 AT91SAM9260_ID_TC1
+#define AT91_ID_TC2 AT91SAM9260_ID_TC2
+#elif defined(CONFIG_ARCH_AT91SAM9261)
+#define AT91_ID_TC0 AT91SAM9261_ID_TC0
+#define AT91_ID_TC1 AT91SAM9261_ID_TC1
+#define AT91_ID_TC2 AT91SAM9261_ID_TC2
+#elif defined(CONFIG_ARCH_AT91SAM9263)
+#define AT91_ID_TC0 AT91SAM9263_ID_TCB
+#define AT91_ID_TC1 AT91SAM9263_ID_TCB
+#define AT91_ID_TC2 AT91SAM9263_ID_TCB
+#elif defined(CONFIG_ARCH_AT91SAM9RL)
+#define AT91_ID_TC0 AT91SAM9RL_ID_TC0
+#define AT91_ID_TC1 AT91SAM9RL_ID_TC1
+#define AT91_ID_TC2 AT91SAM9RL_ID_TC2
+#elif defined(CONFIG_ARCH_AT91X40)
+#define AT91_ID_TC0 AT91X40_ID_TC0
+#define AT91_ID_TC1 AT91X40_ID_TC1
+#define AT91_ID_TC2 AT91X40_ID_TC2
+#else
+#error "AT91 processor unsupported by Adeos"
+#endif
+
+#if (CONFIG_IPIPE_AT91_TC==0)
+#   define KERNEL_TIMER_IRQ_NUM AT91_ID_TC0
+#elif (CONFIG_IPIPE_AT91_TC==1)
+#   define KERNEL_TIMER_IRQ_NUM AT91_ID_TC1
+#elif (CONFIG_IPIPE_AT91_TC==2)
+#   define KERNEL_TIMER_IRQ_NUM AT91_ID_TC2
+#else
+#error IPIPE_AT91_TC must be 0, 1 or 2.
+#endif
+
+#define TCNXCNS(timer,v) ((v) << ((timer)<<1))
+#define AT91_TC_REG_MASK (0xffff)
+
+static unsigned long next_match;
+
+static unsigned max_delta_ticks, min_delta_ticks;
+static struct clock_event_device clkevt;
+static int tc_timer_clock;
+
+static inline unsigned int at91_tc_read(unsigned int reg_offset)
+{
+	unsigned long addr =
+		(AT91_VA_BASE_TCB0 + 0x40 * CONFIG_IPIPE_AT91_TC);
+
+	return readl((void __iomem *)(addr + reg_offset));
+}
+
+static inline void at91_tc_write(unsigned int reg_offset, unsigned long value)
+{
+	unsigned long addr =
+		(AT91_VA_BASE_TCB0 + 0x40 * CONFIG_IPIPE_AT91_TC);
+
+	writel(value, (void __iomem *)(addr + reg_offset));
+}
+
+#define read_CV() at91_tc_read(AT91_TC_CV)
+#define read_RC() at91_tc_read(AT91_TC_RC)
+#define write_RC(value) at91_tc_write(AT91_TC_RC, value)
+
+int __ipipe_mach_timerint = KERNEL_TIMER_IRQ_NUM;
+EXPORT_SYMBOL(__ipipe_mach_timerint);
+
+int __ipipe_mach_timerstolen = 0;
+EXPORT_SYMBOL(__ipipe_mach_timerstolen);
+
+unsigned int __ipipe_mach_ticks_per_jiffy = LATCH;
+EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy);
+
+/*
+ * IRQ handler for the timer.
+ */
+static irqreturn_t at91_timer_interrupt(int irq, void *dev_id)
+{
+	clkevt.event_handler(&clkevt);
+	return IRQ_HANDLED;
+}
+
+static struct irqaction at91_timer_irq = {
+	.name		= "at91_tick",
+	.flags		= IRQF_DISABLED | IRQF_TIMER,
+	.handler	= &at91_timer_interrupt
+};
+
+void __ipipe_mach_acktimer(void)
+{
+	at91_tc_read(AT91_TC_SR);
+
+	if (unlikely(!__ipipe_mach_timerstolen)) {
+		__ipipe_tsc_update();
+		next_match = (next_match + __ipipe_mach_ticks_per_jiffy)
+			& AT91_TC_REG_MASK;
+		write_RC(next_match);
+	}
+}
+
+static void
+at91_tc_set_mode(enum clock_event_mode mode, struct clock_event_device *dev)
+{
+	/* Disable the channel */
+	at91_tc_write(AT91_TC_CCR, AT91_TC_CLKDIS);
+
+	/* Disable all interrupts. */
+	at91_tc_write(AT91_TC_IDR, ~0ul);
+
+	if (mode == CLOCK_EVT_MODE_PERIODIC) {
+		unsigned long v;
+
+#ifndef CONFIG_ARCH_AT91SAM9263
+		clk_enable(clk_get(NULL, "tc"
+				   __stringify(CONFIG_IPIPE_AT91_TC) "_clk"));
+#else /* AT91SAM9263 */
+		clk_enable(clk_get(NULL, "tcb_clk"));
+#endif /* AT91SAM9263 */
+
+		/* No Sync. */
+		at91_tc_write(AT91_TC_BCR, 0);
+
+		/* program NO signal on XCN */
+		v = readl((void __iomem *) (AT91_VA_BASE_TCB0 + AT91_TC_BMR));
+		v &= ~TCNXCNS(CONFIG_IPIPE_AT91_TC, 3);
+		v |= TCNXCNS(CONFIG_IPIPE_AT91_TC, 1); /* AT91_TC_TCNXCNS_NONE */
+		writel(v, (void __iomem *) (AT91_VA_BASE_TCB0 + AT91_TC_BMR));
+
+		/* Use the clock selected by at91_timer_init as input clock. */
+		at91_tc_write(AT91_TC_CMR, tc_timer_clock);
+
+		/* Load the TC register C. */
+		next_match = __ipipe_mach_ticks_per_jiffy;
+		write_RC(next_match);
+
+		/* Enable CPCS interrupt. */
+		at91_tc_write(AT91_TC_IER, AT91_TC_CPCS);
+
+		/* Enable the channel. */
+		at91_tc_write(AT91_TC_CCR, AT91_TC_CLKEN | AT91_TC_SWTRG);
+	}
+}
+
+/*
+ * Reprogram the timer
+ */
+void __ipipe_mach_set_dec(unsigned long delay)
+{
+	if (delay > max_delta_ticks)
+		delay = max_delta_ticks;
+
+	if (likely(delay > min_delta_ticks)) {
+		write_RC((read_CV() + delay) & AT91_TC_REG_MASK);
+		__ipipe_tsc_update();
+	} else
+		ipipe_trigger_irq(KERNEL_TIMER_IRQ_NUM);
+}
+EXPORT_SYMBOL(__ipipe_mach_set_dec);
+
+int __ipipe_check_tickdev(const char *devname)
+{
+	return !strcmp(devname, clkevt.name);
+}
+
+static struct clock_event_device clkevt = {
+	.name		= "at91_tc" __stringify(CONFIG_IPIPE_AT91_TC),
+	.features	= CLOCK_EVT_FEAT_PERIODIC,
+	.shift		= 20,
+	.rating		= 250,
+	.set_mode	= at91_tc_set_mode,
+};
+
+static struct __ipipe_tscinfo tsc_info = {
+	.type = IPIPE_TSC_TYPE_FREERUNNING,
+	.counter_vaddr = (AT91_VA_BASE_TCB0 +
+			  0x40 * CONFIG_IPIPE_AT91_TC + AT91_TC_CV),
+	.u = {
+		{
+			.counter_paddr = (AT91_BASE_TCB0 +
+					  0x40 * CONFIG_IPIPE_AT91_TC +
+					  AT91_TC_CV),
+			.mask = AT91_TC_REG_MASK,
+		},
+	},
+};
+
+void __ipipe_mach_release_timer(void)
+{
+	unsigned long flags;
+	local_irq_save_hw(flags);
+	__ipipe_mach_set_dec(__ipipe_mach_ticks_per_jiffy);
+	local_irq_restore_hw(flags);
+}
+EXPORT_SYMBOL(__ipipe_mach_release_timer);
+
+unsigned long __ipipe_mach_get_dec(void)
+{
+	return (read_RC() - read_CV()) & AT91_TC_REG_MASK;
+}
+
+void __init at91_timer_init(void)
+{
+	unsigned char tc_divisors[] = { 2, 8, 32, 128, 0, };
+	unsigned master_freq, divisor = 0, divided_freq = 0;
+	unsigned long long wrap_ns;
+
+	/* Disable (boot loader) timer interrupts. */
+#if defined(CONFIG_ARCH_AT91RM9200)
+	at91_sys_write(AT91_ST_IDR, AT91_ST_PITS | AT91_ST_WDOVF | AT91_ST_RTTINC | AT91_ST_ALMS);
+	(void) at91_sys_read(AT91_ST_SR);	/* Clear any pending interrupts */
+#elif defined(CONFIG_ARCH_AT91SAM9260) || defined(CONFIG_ARCH_AT91SAM9261) \
+	|| defined(CONFIG_ARCH_AT91SAM9263) || defined(CONFIG_ARCH_AT91SAM9RL) \
+	|| defined(CONFIG_ARCH_AT91SAM9G20)
+	at91_sys_write(AT91_PIT_MR, 0);
+
+	/* Clear any pending interrupts */
+	(void) at91_sys_read(AT91_PIT_PIVR);
+#endif /* CONFIG_ARCH_AT91SAM926x */
+
+	master_freq = clk_get_rate(clk_get(NULL, "mck"));
+	/* Find the first frequency above 1 MHz */
+	for (tc_timer_clock = ARRAY_SIZE(tc_divisors) - 1;
+	     tc_timer_clock >= 0; tc_timer_clock--) {
+		divisor = tc_divisors[tc_timer_clock];
+		divided_freq = (divisor
+				? master_freq / divisor : AT91_SLOW_CLOCK);
+		if (divided_freq > 1000000)
+			break;
+	}
+
+	wrap_ns = (unsigned long long) (AT91_TC_REG_MASK + 1) * NSEC_PER_SEC;
+	do_div(wrap_ns, divided_freq);
+
+	if (divided_freq < 1000000)
+		printk(KERN_INFO "AT91 I-pipe warning: could not find a"
+		       " frequency greater than 1MHz\n");
+
+	printk(KERN_INFO "AT91 I-pipe timer: div: %u, freq: %u.%06u MHz, wrap: "
+	       "%u.%06u ms\n", divisor,
+	       divided_freq / 1000000, divided_freq % 1000000,
+	       (unsigned) wrap_ns / 1000000, (unsigned) wrap_ns % 1000000);
+
+	/* Add a 1ms margin. It means that when an interrupt occurs, update_tsc
+	   must be called within 1ms. update_tsc is called by acktimer when no
+	   higher domain handles the timer, and called through set_dec when a
+	   higher domain handles the timer. */
+	wrap_ns -= 1000000;
+	/* Set up the interrupt. */
+	setup_irq(KERNEL_TIMER_IRQ_NUM, &at91_timer_irq);
+
+	clkevt.mult = div_sc(divided_freq, NSEC_PER_SEC, clkevt.shift);
+	clkevt.max_delta_ns = wrap_ns;
+	clkevt.min_delta_ns = 2000;
+	clkevt.cpumask = cpumask_of(0);
+	clockevents_register_device(&clkevt);
+
+	tsc_info.freq = divided_freq;
+	__ipipe_tsc_register(&tsc_info);
+
+	__ipipe_mach_ticks_per_jiffy = (divided_freq + HZ/2) / HZ;
+	max_delta_ticks = (wrap_ns * clkevt.mult) >> clkevt.shift;
+	min_delta_ticks = ((unsigned long long) clkevt.min_delta_ns
+			   * clkevt.mult) >> clkevt.shift;
+}
+
+#ifdef CONFIG_ARCH_AT91RM9200
+struct sys_timer at91rm9200_timer = {
+#elif defined(CONFIG_ARCH_AT91SAM9260) || defined(CONFIG_ARCH_AT91SAM9261) \
+	|| defined(CONFIG_ARCH_AT91SAM9263) || defined(CONFIG_ARCH_AT91SAM9RL) \
+	|| defined(CONFIG_ARCH_AT91SAM9G20)
+struct sys_timer at91sam926x_timer = {
+#elif defined(CONFIG_ARCH_AT91X40)
+struct sys_timer at91x40_timer = {
+#else
+#error "Unknown machine"
+#endif
+	.init		= at91_timer_init,
+	.suspend	= NULL,
+	.resume		= NULL,
+};
diff --git a/arch/arm/mach-at91/at91rm9200.c b/arch/arm/mach-at91/at91rm9200.c
index 2e9ecad..541da05 100644
--- a/arch/arm/mach-at91/at91rm9200.c
+++ b/arch/arm/mach-at91/at91rm9200.c
@@ -33,6 +33,13 @@ static struct map_desc at91rm9200_io_desc[] __initdata = {
 		.pfn		= __phys_to_pfn(AT91RM9200_BASE_EMAC),
 		.length		= SZ_16K,
 		.type		= MT_DEVICE,
+#ifdef CONFIG_IPIPE
+	}, {
+		.virtual	= AT91_VA_BASE_TCB0,
+		.pfn		= __phys_to_pfn(AT91_BASE_TCB0),
+		.length		= SZ_16K,
+		.type		= MT_DEVICE,
+#endif /* CONFIG_IPIPE */
 	}, {
 		.virtual	= AT91_IO_VIRT_BASE - AT91RM9200_SRAM_SIZE,
 		.pfn		= __phys_to_pfn(AT91RM9200_SRAM_BASE),
@@ -300,6 +307,7 @@ void __init at91rm9200_initialize(unsigned long main_clock, unsigned short banks
  * The default interrupt priority levels (0 = lowest, 7 = highest).
  */
 static unsigned int at91rm9200_default_irq_priority[NR_AIC_IRQS] __initdata = {
+#ifndef CONFIG_IPIPE
 	7,	/* Advanced Interrupt Controller (FIQ) */
 	7,	/* System Peripherals */
 	1,	/* Parallel IO Controller A */
@@ -332,6 +340,42 @@ static unsigned int at91rm9200_default_irq_priority[NR_AIC_IRQS] __initdata = {
 	0,	/* Advanced Interrupt Controller (IRQ4) */
 	0,	/* Advanced Interrupt Controller (IRQ5) */
 	0	/* Advanced Interrupt Controller (IRQ6) */
+#else /* CONFIG_IPIPE */
+/* Give the highest priority to TC, since they are used as timer interrupt by
+   I-pipe. */
+	7,	/* Advanced Interrupt Controller */
+	6,	/* System Peripheral */
+	0,	/* Parallel IO Controller A */
+	0,	/* Parallel IO Controller B */
+	0,	/* Parallel IO Controller C */
+	0,	/* Parallel IO Controller D */
+	5,	/* USART 0 */
+	5,	/* USART 1 */
+	5,	/* USART 2 */
+	5,	/* USART 3 */
+	0,	/* Multimedia Card Interface */
+	3,	/* USB Device Port */
+	0,	/* Two-Wire Interface */
+	5,	/* Serial Peripheral Interface */
+	4,	/* Serial Synchronous Controller */
+	4,	/* Serial Synchronous Controller */
+	4,	/* Serial Synchronous Controller */
+	7,	/* Timer Counter 0 */
+	7,	/* Timer Counter 1 */
+	7,	/* Timer Counter 2 */
+	0,	/* Timer Counter 3 */
+	0,	/* Timer Counter 4 */
+	0,	/* Timer Counter 5 */
+	2,	/* USB Host port */
+	2,	/* Ethernet MAC */
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+	0	/* Advanced Interrupt Controller */
+#endif /*CONFIG_IPIPE */
 };
 
 void __init at91rm9200_init_interrupts(unsigned int priority[NR_AIC_IRQS])
diff --git a/arch/arm/mach-at91/at91sam9260.c b/arch/arm/mach-at91/at91sam9260.c
index 195208b..daf7530 100644
--- a/arch/arm/mach-at91/at91sam9260.c
+++ b/arch/arm/mach-at91/at91sam9260.c
@@ -31,6 +31,13 @@ static struct map_desc at91sam9260_io_desc[] __initdata = {
 		.pfn		= __phys_to_pfn(AT91_BASE_SYS),
 		.length		= SZ_16K,
 		.type		= MT_DEVICE,
+#ifdef CONFIG_IPIPE
+	}, {
+		.virtual	= AT91_VA_BASE_TCB0,
+		.pfn		= __phys_to_pfn(AT91_BASE_TCB0),
+		.length		= SZ_16K,
+		.type		= MT_DEVICE,
+#endif /* CONFIG_IPIPE */
 	}
 };
 
@@ -345,6 +352,7 @@ void __init at91sam9260_initialize(unsigned long main_clock)
  * The default interrupt priority levels (0 = lowest, 7 = highest).
  */
 static unsigned int at91sam9260_default_irq_priority[NR_AIC_IRQS] __initdata = {
+#ifndef CONFIG_IPIPE
 	7,	/* Advanced Interrupt Controller */
 	7,	/* System Peripherals */
 	1,	/* Parallel IO Controller A */
@@ -377,6 +385,42 @@ static unsigned int at91sam9260_default_irq_priority[NR_AIC_IRQS] __initdata = {
 	0,	/* Advanced Interrupt Controller */
 	0,	/* Advanced Interrupt Controller */
 	0,	/* Advanced Interrupt Controller */
+#else /* CONFIG_IPIPE */
+/* Give the highest priority to TC, since they are used as timer interrupt by
+   I-pipe. */
+	7,	/* Advanced Interrupt Controller */
+	7,	/* System Peripherals */
+	0,	/* Parallel IO Controller A */
+	0,	/* Parallel IO Controller B */
+	0,	/* Parallel IO Controller C */
+	0,	/* Analog-to-Digital Converter */
+	6,	/* USART 0 */
+	6,	/* USART 1 */
+	6,	/* USART 2 */
+	0,	/* Multimedia Card Interface */
+	4,	/* USB Device Port */
+	0,	/* Two-Wire Interface */
+	6,	/* Serial Peripheral Interface 0 */
+	6,	/* Serial Peripheral Interface 1 */
+	5,	/* Serial Synchronous Controller */
+	0,
+	0,
+	7,	/* Timer Counter 0 */
+	7,	/* Timer Counter 1 */
+	7,	/* Timer Counter 2 */
+	3,	/* USB Host port */
+	3,	/* Ethernet */
+	0,	/* Image Sensor Interface */
+	6,	/* USART 3 */
+	6,	/* USART 4 */
+	6,	/* USART 5 */
+	7,	/* Timer Counter 3 */
+	7,	/* Timer Counter 4 */
+	7,	/* Timer Counter 5 */
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+#endif /*CONFIG_IPIPE */
 };
 
 void __init at91sam9260_init_interrupts(unsigned int priority[NR_AIC_IRQS])
diff --git a/arch/arm/mach-at91/at91sam9261.c b/arch/arm/mach-at91/at91sam9261.c
index fcad886..c645e21 100644
--- a/arch/arm/mach-at91/at91sam9261.c
+++ b/arch/arm/mach-at91/at91sam9261.c
@@ -31,6 +31,13 @@ static struct map_desc at91sam9261_io_desc[] __initdata = {
 		.pfn		= __phys_to_pfn(AT91_BASE_SYS),
 		.length		= SZ_16K,
 		.type		= MT_DEVICE,
+#ifdef CONFIG_IPIPE
+	}, {
+		.virtual	= AT91_VA_BASE_TCB0,
+		.pfn		= __phys_to_pfn(AT91_BASE_TCB0),
+		.length		= SZ_16K,
+		.type		= MT_DEVICE,
+#endif /* CONFIG_IPIPE */
 	},
 };
 
@@ -301,6 +308,7 @@ void __init at91sam9261_initialize(unsigned long main_clock)
  * The default interrupt priority levels (0 = lowest, 7 = highest).
  */
 static unsigned int at91sam9261_default_irq_priority[NR_AIC_IRQS] __initdata = {
+#ifndef CONFIG_IPIPE
 	7,	/* Advanced Interrupt Controller */
 	7,	/* System Peripherals */
 	1,	/* Parallel IO Controller A */
@@ -333,6 +341,42 @@ static unsigned int at91sam9261_default_irq_priority[NR_AIC_IRQS] __initdata = {
 	0,	/* Advanced Interrupt Controller */
 	0,	/* Advanced Interrupt Controller */
 	0,	/* Advanced Interrupt Controller */
+#else /* CONFIG_IPIPE */
+/* Give the highest priority to TC, since they are used as timer interrupt by
+   I-pipe. */
+	7,	/* Advanced Interrupt Controller */
+	7,	/* System Peripherals */
+	0,	/* Parallel IO Controller A */
+	0,	/* Parallel IO Controller B */
+	0,	/* Parallel IO Controller C */
+	0,
+	6,	/* USART 0 */
+	6,	/* USART 1 */
+	6,	/* USART 2 */
+	0,	/* Multimedia Card Interface */
+	4,	/* USB Device Port */
+	0,	/* Two-Wire Interface */
+	6,	/* Serial Peripheral Interface 0 */
+	6,	/* Serial Peripheral Interface 1 */
+	5,	/* Serial Synchronous Controller 0 */
+	5,	/* Serial Synchronous Controller 1 */
+	5,	/* Serial Synchronous Controller 2 */
+	7,	/* Timer Counter 0 */
+	7,	/* Timer Counter 1 */
+	7,	/* Timer Counter 2 */
+	3,	/* USB Host port */
+	3,	/* LCD Controller */
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+#endif /*CONFIG_IPIPE */
 };
 
 void __init at91sam9261_init_interrupts(unsigned int priority[NR_AIC_IRQS])
diff --git a/arch/arm/mach-at91/at91sam9263.c b/arch/arm/mach-at91/at91sam9263.c
index 249f900..b7d0ae2 100644
--- a/arch/arm/mach-at91/at91sam9263.c
+++ b/arch/arm/mach-at91/at91sam9263.c
@@ -30,6 +30,13 @@ static struct map_desc at91sam9263_io_desc[] __initdata = {
 		.pfn		= __phys_to_pfn(AT91_BASE_SYS),
 		.length		= SZ_16K,
 		.type		= MT_DEVICE,
+#ifdef CONFIG_IPIPE
+	}, {
+		.virtual	= AT91_VA_BASE_TCB0,
+		.pfn		= __phys_to_pfn(AT91_BASE_TCB0),
+		.length		= SZ_16K,
+		.type		= MT_DEVICE,
+#endif /* CONFIG_IPIPE */
 	}, {
 		.virtual	= AT91_IO_VIRT_BASE - AT91SAM9263_SRAM0_SIZE,
 		.pfn		= __phys_to_pfn(AT91SAM9263_SRAM0_BASE),
@@ -306,6 +313,7 @@ void __init at91sam9263_initialize(unsigned long main_clock)
  * The default interrupt priority levels (0 = lowest, 7 = highest).
  */
 static unsigned int at91sam9263_default_irq_priority[NR_AIC_IRQS] __initdata = {
+#ifndef CONFIG_IPIPE
 	7,	/* Advanced Interrupt Controller (FIQ) */
 	7,	/* System Peripherals */
 	1,	/* Parallel IO Controller A */
@@ -338,6 +346,42 @@ static unsigned int at91sam9263_default_irq_priority[NR_AIC_IRQS] __initdata = {
 	2,	/* USB Host port */
 	0,	/* Advanced Interrupt Controller (IRQ0) */
 	0,	/* Advanced Interrupt Controller (IRQ1) */
+#else /* CONFIG_IPIPE */
+/* Give the highest priority to TC, since they are used as timer interrupt by
+   I-pipe. */
+	7,	/* Advanced Interrupt Controller (FIQ) */
+	6,	/* System Peripherals */
+	0,	/* Parallel IO Controller A */
+	0,	/* Parallel IO Controller B */
+	0,	/* Parallel IO Controller C, D and E */
+	0,
+	0,
+	5,	/* USART 0 */
+	5,	/* USART 1 */
+	5,	/* USART 2 */
+	0,	/* Multimedia Card Interface 0 */
+	0,	/* Multimedia Card Interface 1 */
+	3,	/* CAN */
+	0,	/* Two-Wire Interface */
+	5,	/* Serial Peripheral Interface 0 */
+	5,	/* Serial Peripheral Interface 1 */
+	4,	/* Serial Synchronous Controller 0 */
+	4,	/* Serial Synchronous Controller 1 */
+	5,	/* AC97 Controller */
+	7,	/* Timer Counter 0, 1 and 2 */
+	0,	/* Pulse Width Modulation Controller */
+	2,	/* Ethernet */
+	0,
+	0,	/* 2D Graphic Engine */
+	2,	/* USB Device Port */
+	0,	/* Image Sensor Interface */
+	2,	/* LDC Controller */
+	0,	/* DMA Controller */
+	0,
+	2,	/* USB Host port */
+	0,	/* Advanced Interrupt Controller (IRQ0) */
+	0,	/* Advanced Interrupt Controller (IRQ1) */
+#endif /*CONFIG_IPIPE */
 };
 
 void __init at91sam9263_init_interrupts(unsigned int priority[NR_AIC_IRQS])
diff --git a/arch/arm/mach-at91/at91sam9rl.c b/arch/arm/mach-at91/at91sam9rl.c
index 6a9d24e..fc8cd68 100644
--- a/arch/arm/mach-at91/at91sam9rl.c
+++ b/arch/arm/mach-at91/at91sam9rl.c
@@ -30,6 +30,13 @@ static struct map_desc at91sam9rl_io_desc[] __initdata = {
 		.pfn		= __phys_to_pfn(AT91_BASE_SYS),
 		.length		= SZ_16K,
 		.type		= MT_DEVICE,
+#ifdef CONFIG_IPIPE
+	}, {
+		.virtual	= AT91_VA_BASE_TCB0,
+		.pfn		= __phys_to_pfn(AT91_BASE_TCB0),
+		.length		= SZ_16K,
+		.type		= MT_DEVICE,
+#endif /* CONFIG_IPIPE */
 	},
 };
 
@@ -298,6 +305,7 @@ void __init at91sam9rl_initialize(unsigned long main_clock)
  * The default interrupt priority levels (0 = lowest, 7 = highest).
  */
 static unsigned int at91sam9rl_default_irq_priority[NR_AIC_IRQS] __initdata = {
+#ifndef CONFIG_IPIPE
 	7,	/* Advanced Interrupt Controller */
 	7,	/* System Peripherals */
 	1,	/* Parallel IO Controller A */
@@ -330,6 +338,42 @@ static unsigned int at91sam9rl_default_irq_priority[NR_AIC_IRQS] __initdata = {
 	0,
 	0,
 	0,	/* Advanced Interrupt Controller */
+#else /* CONFIG_IPIPE */
+/* Give the highest priority to TC, since they are used as timer interrupt by
+   I-pipe. */
+	7,	/* Advanced Interrupt Controller */
+	6,	/* System Peripherals */
+	1,	/* Parallel IO Controller A */
+	1,	/* Parallel IO Controller B */
+	1,	/* Parallel IO Controller C */
+	1,	/* Parallel IO Controller D */
+	4,	/* USART 0 */
+	4,	/* USART 1 */
+	4,	/* USART 2 */
+	4,	/* USART 3 */
+	0,	/* Multimedia Card Interface */
+	5,	/* Two-Wire Interface 0 */
+	5,	/* Two-Wire Interface 1 */
+	4,	/* Serial Peripheral Interface */
+	3,	/* Serial Synchronous Controller 0 */
+	3,	/* Serial Synchronous Controller 1 */
+	7,	/* Timer Counter 0 */
+	7,	/* Timer Counter 1 */
+	7,	/* Timer Counter 2 */
+	0,
+	0,	/* Touch Screen Controller */
+	0,	/* DMA Controller */
+	2,	/* USB Device High speed port */
+	2,	/* LCD Controller */
+	5,	/* AC97 Controller */
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,	/* Advanced Interrupt Controller */
+#endif /*CONFIG_IPIPE */
 };
 
 void __init at91sam9rl_init_interrupts(unsigned int priority[NR_AIC_IRQS])
diff --git a/arch/arm/mach-at91/gpio.c b/arch/arm/mach-at91/gpio.c
index af818a2..caf2226 100644
--- a/arch/arm/mach-at91/gpio.c
+++ b/arch/arm/mach-at91/gpio.c
@@ -19,12 +19,20 @@
 #include <linux/list.h>
 #include <linux/module.h>
 #include <linux/io.h>
+#include <linux/ipipe.h>
 
 #include <mach/hardware.h>
 #include <mach/at91_pio.h>
 #include <mach/gpio.h>
 
 #include <asm/gpio.h>
+#ifdef CONFIG_IPIPE
+#include <asm/irq.h>
+
+#ifdef __IPIPE_FEATURE_PIC_MUTE
+DEFINE_PER_CPU(__ipipe_irqbits_t, __ipipe_muted_irqs);
+#endif /* __IPIPE_FEATURE_PIC_MUTE */
+#endif /* CONFIG_IPIPE */
 
 #include "generic.h"
 
@@ -33,6 +41,10 @@ struct at91_gpio_chip {
 	struct at91_gpio_chip	*next;		/* Bank sharing same clock */
 	struct at91_gpio_bank	*bank;		/* Bank definition */
 	void __iomem		*regbase;	/* Base of register bank */
+#ifdef CONFIG_IPIPE
+	unsigned *nonroot_gpios;
+	unsigned nonroot_gpios_storage;
+#endif
 };
 
 #define to_at91_gpio_chip(c) container_of(c, struct at91_gpio_chip, chip)
@@ -375,6 +387,10 @@ static int gpio_irq_type(struct irq_data *d, unsigned type)
 
 static struct irq_chip gpio_irqchip = {
 	.name		= "GPIO",
+#ifdef CONFIG_IPIPE
+	.irq_ack        = gpio_irq_mask,
+	.irq_mask_ack   = gpio_irq_mask,
+#endif /* CONFIG_IPIPE */
 	.irq_mask	= gpio_irq_mask,
 	.irq_unmask	= gpio_irq_unmask,
 	.irq_set_type	= gpio_irq_type,
@@ -394,6 +410,13 @@ static void gpio_irq_handler(unsigned irq, struct irq_desc *desc)
 
 	/* temporarily mask (level sensitive) parent IRQ */
 	desc->irq_data.chip->irq_ack(&desc->irq_data);
+#ifdef CONFIG_IPIPE
+	if (!(*at91_gpio->nonroot_gpios)) {
+		local_irq_enable_hw();
+		local_irq_disable_hw();
+	}
+#endif /* CONFIG_IPIPE */
+
 	for (;;) {
 		/* Reading ISR acks pending (edge triggered) GPIO interrupts.
 		 * When there none are pending, we're finished unless we need
@@ -420,9 +443,15 @@ static void gpio_irq_handler(unsigned irq, struct irq_desc *desc)
 					 * here to be disabled on the GPIO controller.
 					 */
 					gpio_irq_mask(irq_get_irq_data(pin));
+				} else {
+					ipipe_handle_chained_irq(pin);
+#ifdef CONFIG_IPIPE
+					if (!(*at91_gpio->nonroot_gpios)) {
+						local_irq_enable_hw();
+						local_irq_disable_hw();
+					}
+#endif /* CONFIG_IPIPE */
 				}
-				else
-					generic_handle_irq(pin);
 			}
 			pin++;
 			gpio++;
@@ -433,6 +462,79 @@ static void gpio_irq_handler(unsigned irq, struct irq_desc *desc)
 	/* now it may re-trigger */
 }
 
+#if defined(CONFIG_IPIPE) && defined(__IPIPE_FEATURE_PIC_MUTE)
+void __ipipe_mach_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
+{
+	struct irq_desc *desc = irq_to_desc(irq);
+
+	if (desc->chip == &gpio_irqchip) {
+		struct at91_gpio_chip *chip = &gpio_chip[(irq - PIN_BASE) / 32];
+		struct at91_gpio_bank *bank = chip->bank;
+
+		if (ipd != &ipipe_root && ++(*chip->nonroot_gpios) == 1)
+			__ipipe_irqbits[(bank->id / BITS_PER_LONG)]
+				&= ~(1 << (bank->id % BITS_PER_LONG));
+	}
+}
+
+void __ipipe_mach_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
+{
+	struct irq_desc *desc = irq_to_desc(irq);
+	if (desc->chip == &gpio_irqchip) {
+		struct at91_gpio_chip *chip = &gpio_chip[(irq - PIN_BASE) / 32];
+		struct at91_gpio_bank *bank = chip->bank;
+
+		if (ipd != &ipipe_root && --(*chip->nonroot_gpios) == 0)
+			__ipipe_irqbits[(bank->id / BITS_PER_LONG)]
+				|= (1 << (bank->id % BITS_PER_LONG));
+	}
+}
+
+void ipipe_mute_pic(void)
+{
+	struct at91_gpio_chip *prev, *chip = NULL;
+	unsigned long unmasked, muted;
+	unsigned i;
+
+	for (i = 0; i < gpio_banks; i++) {
+		prev = chip;
+		chip = &gpio_chip[i];
+		if (!(*chip->nonroot_gpios))
+			continue;
+
+		unmasked = __raw_readl(chip->regbase + PIO_IMR);
+		muted = unmasked & __ipipe_irqbits[i + 1];
+		__raw_get_cpu_var(__ipipe_muted_irqs)
+			[i + PIN_BASE / 32] = muted;
+		__raw_writel(muted, chip->regbase + PIO_IDR);
+	}
+
+	unmasked = at91_sys_read(AT91_AIC_IMR);
+	muted = unmasked & __ipipe_irqbits[0];
+	__raw_get_cpu_var(__ipipe_muted_irqs)[0] = muted;
+	at91_sys_write(AT91_AIC_IDCR, muted);
+}
+
+void ipipe_unmute_pic(void)
+{
+	struct at91_gpio_chip *prev, *chip = NULL;
+	unsigned long muted;
+	unsigned i;
+
+	at91_sys_write(AT91_AIC_IECR, __raw_get_cpu_var(__ipipe_muted_irqs)[0]);
+	for (i = 0; i < gpio_banks; i++) {
+		prev = chip;
+		chip = &gpio_chip[i];
+		if (!(*chip->nonroot_gpios))
+			continue;
+
+		muted = __raw_get_cpu_var(__ipipe_muted_irqs)
+			[i + PIN_BASE / 32];
+		__raw_writel(muted, chip->regbase + PIO_IER);
+	}
+}
+#endif /* CONFIG_IPIPE && __IPIPE_FEATURE_PIC_MUTE */
+
 /*--------------------------------------------------------------------------*/
 
 #ifdef CONFIG_DEBUG_FS
@@ -631,13 +733,19 @@ void __init at91_gpio_init(struct at91_gpio_bank *data, int nr_banks)
 		at91_gpio->chip.base = PIN_BASE + i * 32;
 		at91_gpio->regbase = at91_gpio->bank->offset +
 			(void __iomem *)AT91_VA_BASE_SYS;
-
+#ifdef CONFIG_IPIPE
+		at91_gpio->nonroot_gpios = &at91_gpio->nonroot_gpios_storage;
+#endif
 		/* enable PIO controller's clock */
 		clk_enable(at91_gpio->bank->clock);
 
 		/* AT91SAM9263_ID_PIOCDE groups PIOC, PIOD, PIOE */
-		if (last && last->bank->id == at91_gpio->bank->id)
+		if (last && last->bank->id == at91_gpio->bank->id) {
 			last->next = at91_gpio;
+#ifdef CONFIG_IPIPE
+			at91_gpio->nonroot_gpios = last->nonroot_gpios;
+#endif
+		}
 		last = at91_gpio;
 
 		gpiochip_add(&at91_gpio->chip);
diff --git a/arch/arm/mach-at91/include/mach/hardware.h b/arch/arm/mach-at91/include/mach/hardware.h
index 3d64a75..6aa41bd 100644
--- a/arch/arm/mach-at91/include/mach/hardware.h
+++ b/arch/arm/mach-at91/include/mach/hardware.h
@@ -65,6 +65,25 @@
 #define AT91_VA_BASE_SYS	AT91_IO_P2V(AT91_BASE_SYS)
 #define AT91_VA_BASE_EMAC	AT91_IO_P2V(AT91RM9200_BASE_EMAC)
 
+#ifdef CONFIG_IPIPE
+#if defined(CONFIG_ARCH_AT91RM9200)
+#define AT91_BASE_TCB0 AT91RM9200_BASE_TCB0
+#elif defined(CONFIG_ARCH_AT91SAM9260) || defined(CONFIG_ARCH_AT91SAM9G20)
+#define AT91_BASE_TCB0 AT91SAM9260_BASE_TCB0
+#elif defined(CONFIG_ARCH_AT91SAM9261)
+#define AT91_BASE_TCB0 AT91SAM9261_BASE_TCB0
+#elif defined(CONFIG_ARCH_AT91SAM9263)
+#define AT91_BASE_TCB0 AT91SAM9263_BASE_TCB0
+#elif defined(CONFIG_ARCH_AT91SAM9RL)
+#define AT91_BASE_TCB0 AT91SAM9RL_BASE_TCB0
+#elif defined(CONFIG_ARCH_AT91X40)
+#define AT91_BASE_TCB0 (AT91_BASE_SYS + AT91_TC)
+#else
+#error "AT91 processor unsupported by Adeos"
+#endif
+#define AT91_VA_BASE_TCB0 AT91_IO_P2V(AT91_BASE_TCB0)
+#endif
+
  /* Internal SRAM is mapped below the IO devices */
 #define AT91_SRAM_MAX		SZ_1M
 #define AT91_VIRT_BASE		(AT91_IO_VIRT_BASE - AT91_SRAM_MAX)
diff --git a/arch/arm/mach-at91/include/mach/irqs.h b/arch/arm/mach-at91/include/mach/irqs.h
index 36bd55f..cedaab2 100644
--- a/arch/arm/mach-at91/include/mach/irqs.h
+++ b/arch/arm/mach-at91/include/mach/irqs.h
@@ -45,4 +45,6 @@
 /* FIQ is AIC source 0. */
 #define FIQ_START AT91_ID_FIQ
 
+#define __IPIPE_FEATURE_PIC_MUTE
+
 #endif
diff --git a/arch/arm/mach-at91/include/mach/timex.h b/arch/arm/mach-at91/include/mach/timex.h
index 05a6e8a..f2e35f6 100644
--- a/arch/arm/mach-at91/include/mach/timex.h
+++ b/arch/arm/mach-at91/include/mach/timex.h
@@ -87,6 +87,6 @@
 #define AT572D940HF_MASTER_CLOCK	80000000
 #define CLOCK_TICK_RATE		(AT572D940HF_MASTER_CLOCK/16)
 
-#endif
+#endif /* arch specific */
 
 #endif
diff --git a/arch/arm/mach-at91/irq.c b/arch/arm/mach-at91/irq.c
index b56d6b3..efe86a5 100644
--- a/arch/arm/mach-at91/irq.c
+++ b/arch/arm/mach-at91/irq.c
@@ -121,6 +121,9 @@ static struct irq_chip at91_aic_chip = {
 	.name		= "AIC",
 	.irq_ack	= at91_aic_mask_irq,
 	.irq_mask	= at91_aic_mask_irq,
+#ifdef CONFIG_IPIPE
+	.irq_mask_ack   = at91_aic_mask_irq,
+#endif /* CONFIG_IPIPE */
 	.irq_unmask	= at91_aic_unmask_irq,
 	.irq_set_type	= at91_aic_set_type,
 	.irq_set_wake	= at91_aic_set_wake,
diff --git a/arch/arm/mach-imx/clock-imx1.c b/arch/arm/mach-imx/clock-imx1.c
index 3938a56..9bd585f 100644
--- a/arch/arm/mach-imx/clock-imx1.c
+++ b/arch/arm/mach-imx/clock-imx1.c
@@ -629,7 +629,7 @@ int __init mx1_clocks_init(unsigned long fref)
 	clk_enable(&fclk);
 
 	mxc_timer_init(&gpt_clk, MX1_IO_ADDRESS(MX1_TIM1_BASE_ADDR),
-			MX1_TIM1_INT);
+		       MX1_TIM1_BASE_ADDR, MX1_TIM1_INT);
 
 	return 0;
 }
diff --git a/arch/arm/mach-imx/clock-imx21.c b/arch/arm/mach-imx/clock-imx21.c
index bf30a8c..d1dc7b2 100644
--- a/arch/arm/mach-imx/clock-imx21.c
+++ b/arch/arm/mach-imx/clock-imx21.c
@@ -1234,6 +1234,6 @@ int __init mx21_clocks_init(unsigned long lref, unsigned long href)
 #endif
 
 	mxc_timer_init(&gpt_clk[0], MX21_IO_ADDRESS(MX21_GPT1_BASE_ADDR),
-			MX21_INT_GPT1);
+		       MX21_GPT1_BASE_ADDR, MX21_INT_GPT1);
 	return 0;
 }
diff --git a/arch/arm/mach-imx/clock-imx27.c b/arch/arm/mach-imx/clock-imx27.c
index 583f251..c4d0f97 100644
--- a/arch/arm/mach-imx/clock-imx27.c
+++ b/arch/arm/mach-imx/clock-imx27.c
@@ -756,8 +756,7 @@ int __init mx27_clocks_init(unsigned long fref)
 #endif
 
 	mxc_timer_init(&gpt1_clk, MX27_IO_ADDRESS(MX27_GPT1_BASE_ADDR),
-			MX27_INT_GPT1);
+		       MX27_GPT1_BASE_ADDR, MX27_INT_GPT1);
 
 	return 0;
 }
-
diff --git a/arch/arm/mach-integrator/core.c b/arch/arm/mach-integrator/core.c
index b8e884b..8c9c447 100644
--- a/arch/arm/mach-integrator/core.c
+++ b/arch/arm/mach-integrator/core.c
@@ -2,6 +2,7 @@
  *  linux/arch/arm/mach-integrator/core.c
  *
  *  Copyright (C) 2000-2003 Deep Blue Solutions Ltd
+ *  Copyright (C) 2005 Stelian Pop.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2, as
diff --git a/arch/arm/mach-integrator/include/mach/entry-macro.S b/arch/arm/mach-integrator/include/mach/entry-macro.S
index 3d029c9..876b1de 100644
--- a/arch/arm/mach-integrator/include/mach/entry-macro.S
+++ b/arch/arm/mach-integrator/include/mach/entry-macro.S
@@ -29,7 +29,11 @@
 		teq	\irqstat, #0
 		ldreq	\irqstat, [\base, #(INTEGRATOR_HDR_IC_OFFSET+IRQ_STATUS)]
 		moveq	\irqnr, #IRQ_CIC_START
-
+#ifdef CONFIG_IPIPE
+		tst	\irqstat, #0x00000040			@ check IRQ_TIMERINT1 first
+		movne	\irqnr, #6
+		bne	1003f
+#endif /* CONFIG_IPIPE */
 1001:		tst	\irqstat, #15
 		bne	1002f
 		add	\irqnr, \irqnr, #4
diff --git a/arch/arm/mach-integrator/include/mach/irqs.h b/arch/arm/mach-integrator/include/mach/irqs.h
index 1fbe6d1..dea6390 100644
--- a/arch/arm/mach-integrator/include/mach/irqs.h
+++ b/arch/arm/mach-integrator/include/mach/irqs.h
@@ -79,4 +79,3 @@
 #define IRQ_SIC_END			46
 
 #define NR_IRQS                         47
-
diff --git a/arch/arm/mach-integrator/include/mach/platform.h b/arch/arm/mach-integrator/include/mach/platform.h
index 5e6ea5c..c508323 100644
--- a/arch/arm/mach-integrator/include/mach/platform.h
+++ b/arch/arm/mach-integrator/include/mach/platform.h
@@ -390,7 +390,7 @@
  *  Timer definitions
  *
  *  Only use timer 1 & 2
- *  (both run at 24MHz and will need the clock divider set to 16).
+ *  (both run at 1MHZ on /CP and at 24MHz on /AP)
  *
  *  Timer 0 runs at bus frequency
  */
@@ -399,7 +399,11 @@
 #define INTEGRATOR_TIMER1_BASE          (INTEGRATOR_CT_BASE + 0x100)
 #define INTEGRATOR_TIMER2_BASE          (INTEGRATOR_CT_BASE + 0x200)
 
+#ifdef CONFIG_ARCH_INTEGRATOR_CP
+#define TICKS_PER_uSEC                  1
+#else
 #define TICKS_PER_uSEC                  24
+#endif
 
 /*
  *  These are useconds NOT ticks.
diff --git a/arch/arm/mach-integrator/include/mach/timex.h b/arch/arm/mach-integrator/include/mach/timex.h
index 1dcb420..a04653a 100644
--- a/arch/arm/mach-integrator/include/mach/timex.h
+++ b/arch/arm/mach-integrator/include/mach/timex.h
@@ -21,6 +21,6 @@
  */
 
 /*
- * ??
+ * Timer rate
  */
-#define CLOCK_TICK_RATE		(50000000 / 16)
+#define CLOCK_TICK_RATE		(1000000)
diff --git a/arch/arm/mach-integrator/integrator_cp.c b/arch/arm/mach-integrator/integrator_cp.c
index e9327da..c9d848a 100644
--- a/arch/arm/mach-integrator/integrator_cp.c
+++ b/arch/arm/mach-integrator/integrator_cp.c
@@ -2,6 +2,7 @@
  *  linux/arch/arm/mach-integrator/integrator_cp.c
  *
  *  Copyright (C) 2003 Deep Blue Solutions Ltd
+ *  Copyright (C) 2005 Stelian Pop.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -162,6 +163,9 @@ static struct irq_chip cic_chip = {
 	.name		= "CIC",
 	.irq_ack	= cic_mask_irq,
 	.irq_mask	= cic_mask_irq,
+#ifdef CONFIG_IPIPE
+	.irq_mask_ack	= cic_mask_irq,
+#endif /* CONFIG_IPIPE */
 	.irq_unmask	= cic_unmask_irq,
 };
 
@@ -181,6 +185,9 @@ static struct irq_chip pic_chip = {
 	.name		= "PIC",
 	.irq_ack	= pic_mask_irq,
 	.irq_mask	= pic_mask_irq,
+#ifdef CONFIG_IPIPE
+	.irq_mask_ack	= pic_mask_irq,
+#endif /* CONFIG_IPIPE */
 	.irq_unmask	= pic_unmask_irq,
 };
 
@@ -200,6 +207,9 @@ static struct irq_chip sic_chip = {
 	.name		= "SIC",
 	.irq_ack	= sic_mask_irq,
 	.irq_mask	= sic_mask_irq,
+#ifdef CONFIG_IPIPE
+	.irq_mask_ack	= sic_mask_irq,
+#endif /* CONFIG_IPIPE */
 	.irq_unmask	= sic_unmask_irq,
 };
 
@@ -219,7 +229,7 @@ sic_handle_irq(unsigned int irq, struct irq_desc *desc)
 
 		irq += IRQ_SIC_START;
 
-		generic_handle_irq(irq);
+		ipipe_handle_chained_irq(irq);
 	} while (status);
 }
 
@@ -588,7 +598,7 @@ static void __init intcp_timer_init(void)
 	writel(0, TIMER1_VA_BASE + TIMER_CTRL);
 	writel(0, TIMER2_VA_BASE + TIMER_CTRL);
 
-	sp804_clocksource_init(TIMER2_VA_BASE);
+	sp804_clocksource_init(TIMER2_VA_BASE, INTEGRATOR_TIMER2_BASE);
 	sp804_clockevents_init(TIMER1_VA_BASE, IRQ_TIMERINT1);
 }
 
diff --git a/arch/arm/mach-ixp4xx/common.c b/arch/arm/mach-ixp4xx/common.c
index 9fd8942..a6274a7 100644
--- a/arch/arm/mach-ixp4xx/common.c
+++ b/arch/arm/mach-ixp4xx/common.c
@@ -6,10 +6,10 @@
  * Maintainer: Deepak Saxena <dsaxena@plexity.net>
  *
  * Copyright 2002 (c) Intel Corporation
- * Copyright 2003-2004 (c) MontaVista, Software, Inc. 
- * 
- * This file is licensed under  the terms of the GNU General Public 
- * License version 2. This program is licensed "as is" without any 
+ * Copyright 2003-2004 (c) MontaVista, Software, Inc.
+ *
+ * This file is licensed under  the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
  * warranty of any kind, whether express or implied.
  */
 
@@ -45,6 +45,60 @@ static void __init ixp4xx_clocksource_init(void);
 static void __init ixp4xx_clockevent_init(void);
 static struct clock_event_device clockevent_ixp4xx;
 
+#ifdef CONFIG_IPIPE
+#include <linux/ipipe.h>
+int __ipipe_mach_timerint = IRQ_IXP4XX_TIMER1;
+int __ipipe_mach_timerstolen = 0;
+unsigned int __ipipe_mach_ticks_per_jiffy = LATCH;
+EXPORT_SYMBOL(__ipipe_mach_timerint);
+EXPORT_SYMBOL(__ipipe_mach_timerstolen);
+EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy);
+
+static int ixp4xx_timer_initialized;
+
+#define ONE_SHOT_ENABLE (IXP4XX_OST_ENABLE|IXP4XX_OST_ONE_SHOT)
+
+union tsc_reg {
+#ifdef __BIG_ENDIAN
+	struct {
+		unsigned long high;
+		unsigned long low;
+	};
+#else /* __LITTLE_ENDIAN */
+	struct {
+		unsigned long low;
+		unsigned long high;
+	};
+#endif /* __LITTLE_ENDIAN */
+	unsigned long long full;
+};
+
+#ifdef CONFIG_SMP
+static union tsc_reg tsc[NR_CPUS];
+
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info)
+{
+	info->type = IPIPE_TSC_TYPE_NONE;
+}
+
+#else /* !CONFIG_SMP */
+static union tsc_reg *tsc;
+
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info)
+{
+	info->type = IPIPE_TSC_TYPE_FREERUNNING;
+	info->u.fr.counter =
+		(unsigned *)
+		(IXP4XX_TIMER_BASE_PHYS + IXP4XX_OSTS_OFFSET);
+	info->u.fr.mask = 0xffffffff;
+	info->u.fr.tsc = &tsc->full;
+}
+#endif /* !CONFIG_SMP */
+
+static void ipipe_mach_update_tsc(void);
+
+#endif /* CONFIG_IPIPE */
+
 /*************************************************************************
  * IXP4xx chipset I/O mapping
  *************************************************************************/
@@ -240,7 +294,7 @@ void __init ixp4xx_init_irq(void)
 	*IXP4XX_ICLR = 0x0;
 
 	/* Disable all interrupt */
-	*IXP4XX_ICMR = 0x0; 
+	*IXP4XX_ICMR = 0x0;
 
 	if (cpu_is_ixp46x() || cpu_is_ixp43x()) {
 		/* Route upper 32 sources to IRQ instead of FIQ */
@@ -250,7 +304,7 @@ void __init ixp4xx_init_irq(void)
 		*IXP4XX_ICMR2 = 0x00;
 	}
 
-        /* Default to all level triggered */
+	/* Default to all level triggered */
 	for(i = 0; i < NR_IRQS; i++) {
 		set_irq_chip(i, &ixp4xx_irq_chip);
 		set_irq_handler(i, handle_level_irq);
@@ -261,7 +315,7 @@ void __init ixp4xx_init_irq(void)
 
 /*************************************************************************
  * IXP4xx timer tick
- * We use OS timer1 on the CPU for the timer tick and the timestamp 
+ * We use OS timer1 on the CPU for the timer tick and the timestamp
  * counter as a source of real clock ticks to account for missed jiffies.
  *************************************************************************/
 
@@ -269,8 +323,12 @@ static irqreturn_t ixp4xx_timer_interrupt(int irq, void *dev_id)
 {
 	struct clock_event_device *evt = dev_id;
 
+#ifndef CONFIG_IPIPE
 	/* Clear Pending Interrupt by writing '1' to it */
 	*IXP4XX_OSST = IXP4XX_OSST_TIMER_1_PEND;
+#else /* CONFIG_IPIPE */
+	ipipe_mach_update_tsc();
+#endif /* CONFIG_IPIPE */
 
 	evt->event_handler(evt);
 
@@ -295,6 +353,14 @@ void __init ixp4xx_timer_init(void)
 	/* Reset time-stamp counter */
 	*IXP4XX_OSTS = 0;
 
+#ifdef CONFIG_IPIPE
+#ifndef CONFIG_SMP
+	tsc = (union tsc_reg *) __ipipe_tsc_area;
+	barrier();
+#endif /* CONFIG_SMP */
+
+	ixp4xx_timer_initialized = 1;
+#endif
 	/* Connect the interrupt handler and enable the interrupt */
 	setup_irq(IRQ_IXP4XX_TIMER1, &ixp4xx_timer_irq);
 
@@ -485,6 +551,13 @@ static void ixp4xx_set_mode(enum clock_event_mode mode,
 	*IXP4XX_OSRT1 = osrt | opts;
 }
 
+#ifdef CONFIG_IPIPE
+int __ipipe_check_tickdev(const char *devname)
+{
+	return !strcmp(devname, clockevent_ixp4xx.name);
+}
+#endif /* CONFIG_IPIPE */
+
 static struct clock_event_device clockevent_ixp4xx = {
 	.name		= "ixp4xx timer1",
 	.features       = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT,
@@ -506,3 +579,94 @@ static void __init ixp4xx_clockevent_init(void)
 
 	clockevents_register_device(&clockevent_ixp4xx);
 }
+
+#ifdef CONFIG_IPIPE
+void __ipipe_mach_acktimer(void)
+{
+	/* Clear Pending Interrupt by writing '1' to it */
+	*IXP4XX_OSST = IXP4XX_OSST_TIMER_1_PEND;
+}
+
+static void ipipe_mach_update_tsc(void)
+{
+	union tsc_reg *local_tsc;
+	unsigned long stamp, flags;
+
+	local_irq_save_hw(flags);
+	local_tsc = &tsc[ipipe_processor_id()];
+	stamp = *IXP4XX_OSTS;
+	if (unlikely(stamp < local_tsc->low))
+		/* 32 bit counter wrapped, increment high word. */
+		local_tsc->high++;
+	local_tsc->low = stamp;
+	local_irq_restore_hw(flags);
+}
+
+notrace unsigned long long __ipipe_mach_get_tsc(void)
+{
+	if (likely(ixp4xx_timer_initialized)) {
+		union tsc_reg *local_tsc, result;
+		unsigned long stamp;
+
+		local_tsc = &tsc[ipipe_processor_id()];
+
+		__asm__ ("ldmia %1, %M0\n":
+			 "=r"(result.full): "r"(local_tsc), "m"(*local_tsc));
+		barrier();
+		stamp = *IXP4XX_OSTS;
+		if (unlikely(stamp < result.low))
+			/* 32 bit counter wrapped, increment high word. */
+			result.high++;
+		result.low = stamp;
+		return result.full;
+	}
+
+	return 0;
+}
+
+EXPORT_SYMBOL(__ipipe_mach_get_tsc);
+
+/*
+ * Reprogram the timer
+ *
+ * The timer is aperiodic (most of the time) when running Xenomai, so
+ * __ipipe_mach_set_dec is called for each timer tick and programs the
+ * timer hardware for the next tick.
+ *
+ */
+#define MIN_DELAY 333 /* 5 usec with the 66.66 MHz system clock */
+
+void __ipipe_mach_set_dec(unsigned long delay)
+{
+	if (delay > MIN_DELAY) {
+		*IXP4XX_OSRT1 = delay | ONE_SHOT_ENABLE;
+	} else {
+		ipipe_trigger_irq(IRQ_IXP4XX_TIMER1);
+	}
+}
+
+EXPORT_SYMBOL(__ipipe_mach_set_dec);
+
+/*
+ * This returns the number of clock ticks remaining.
+ */
+unsigned long __ipipe_mach_get_dec(void)
+{
+	return(*IXP4XX_OST1); /* remaining */
+}
+
+EXPORT_SYMBOL(__ipipe_mach_get_dec);
+
+void __ipipe_mach_release_timer(void)
+{
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
+	ixp4xx_set_mode(clockevent_ixp4xx.mode, &clockevent_ixp4xx);
+	if (clockevent_ixp4xx.mode == CLOCK_EVT_MODE_ONESHOT)
+		ixp4xx_set_next_event(LATCH, &clockevent_ixp4xx);
+	local_irq_restore_hw(flags);
+}
+EXPORT_SYMBOL(__ipipe_mach_release_timer);
+
+#endif /* CONFIG_IPIPE */
diff --git a/arch/arm/mach-ixp4xx/include/mach/platform.h b/arch/arm/mach-ixp4xx/include/mach/platform.h
index e824c02..1b99142 100644
--- a/arch/arm/mach-ixp4xx/include/mach/platform.h
+++ b/arch/arm/mach-ixp4xx/include/mach/platform.h
@@ -150,10 +150,14 @@ extern struct pci_bus *ixp4xx_scan_bus(int nr, struct pci_sys_data *sys);
 
 static inline void gpio_line_config(u8 line, u32 direction)
 {
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
 	if (direction == IXP4XX_GPIO_IN)
 		*IXP4XX_GPIO_GPOER |= (1 << line);
 	else
 		*IXP4XX_GPIO_GPOER &= ~(1 << line);
+	local_irq_restore_hw(flags);
 }
 
 static inline void gpio_line_get(u8 line, int *value)
@@ -163,11 +167,14 @@ static inline void gpio_line_get(u8 line, int *value)
 
 static inline void gpio_line_set(u8 line, int value)
 {
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
 	if (value == IXP4XX_GPIO_HIGH)
 	    *IXP4XX_GPIO_GPOUTR |= (1 << line);
 	else if (value == IXP4XX_GPIO_LOW)
 	    *IXP4XX_GPIO_GPOUTR &= ~(1 << line);
+	local_irq_restore_hw(flags);
 }
 
 #endif // __ASSEMBLY__
-
diff --git a/arch/arm/mach-mx3/clock-imx31.c b/arch/arm/mach-mx3/clock-imx31.c
index d423cac..6cea8d6 100644
--- a/arch/arm/mach-mx3/clock-imx31.c
+++ b/arch/arm/mach-mx3/clock-imx31.c
@@ -623,8 +623,7 @@ int __init mx31_clocks_init(unsigned long fref)
 	}
 
 	mxc_timer_init(&ipg_clk, MX31_IO_ADDRESS(MX31_GPT1_BASE_ADDR),
-			MX31_INT_GPT);
+		       MX31_GPT1_BASE_ADDR, MX31_INT_GPT);
 
 	return 0;
 }
-
diff --git a/arch/arm/mach-mx3/clock-imx35.c b/arch/arm/mach-mx3/clock-imx35.c
index 448a038..b9575f2 100644
--- a/arch/arm/mach-mx3/clock-imx35.c
+++ b/arch/arm/mach-mx3/clock-imx35.c
@@ -541,10 +541,9 @@ int __init mx35_clocks_init()
 	epit_timer_init(&epit1_clk,
 			MX35_IO_ADDRESS(MX35_EPIT1_BASE_ADDR), MX35_INT_EPIT1);
 #else
-	mxc_timer_init(&gpt_clk,
-			MX35_IO_ADDRESS(MX35_GPT1_BASE_ADDR), MX35_INT_GPT);
+	mxc_timer_init(&gpt_clk, MX35_IO_ADDRESS(MX35_GPT1_BASE_ADDR),
+		       MX35_GPT1_BASE_ADDR, MX35_INT_GPT);
 #endif
 
 	return 0;
 }
-
diff --git a/arch/arm/mach-mx3/devices.c b/arch/arm/mach-mx3/devices.c
index b6672db..5191ad8 100644
--- a/arch/arm/mach-mx3/devices.c
+++ b/arch/arm/mach-mx3/devices.c
@@ -22,6 +22,7 @@
 #include <linux/platform_device.h>
 #include <linux/serial.h>
 #include <linux/gpio.h>
+#include <mach/common.h>
 #include <mach/hardware.h>
 #include <mach/irqs.h>
 #include <mach/common.h>
diff --git a/arch/arm/mach-mx3/mach-mx31_3ds.c b/arch/arm/mach-mx3/mach-mx31_3ds.c
index 0d65db8..9c4ab15 100644
--- a/arch/arm/mach-mx3/mach-mx31_3ds.c
+++ b/arch/arm/mach-mx3/mach-mx31_3ds.c
@@ -17,6 +17,7 @@
 #include <linux/init.h>
 #include <linux/clk.h>
 #include <linux/irq.h>
+#include <linux/ipipe.h>
 #include <linux/gpio.h>
 #include <linux/platform_device.h>
 #include <linux/mfd/mc13783.h>
diff --git a/arch/arm/mach-mx3/mach-mx31ads.c b/arch/arm/mach-mx3/mach-mx31ads.c
index 88b97d6..2a46a02 100644
--- a/arch/arm/mach-mx3/mach-mx31ads.c
+++ b/arch/arm/mach-mx3/mach-mx31ads.c
@@ -21,6 +21,7 @@
 #include <linux/gpio.h>
 #include <linux/i2c.h>
 #include <linux/irq.h>
+#include <linux/ipipe.h>
 
 #include <asm/mach-types.h>
 #include <asm/mach/arch.h>
@@ -154,7 +155,7 @@ static void mx31ads_expio_irq_handler(u32 irq, struct irq_desc *desc)
 		if ((int_valid & 1) == 0)
 			continue;
 
-		generic_handle_irq(expio_irq);
+		ipipe_handle_chained_irq(expio_irq);
 	}
 }
 
diff --git a/arch/arm/mach-mx5/clock-mx51-mx53.c b/arch/arm/mach-mx5/clock-mx51-mx53.c
index 0a19e75..dae229f 100644
--- a/arch/arm/mach-mx5/clock-mx51-mx53.c
+++ b/arch/arm/mach-mx5/clock-mx51-mx53.c
@@ -1400,7 +1400,7 @@ int __init mx51_clocks_init(unsigned long ckil, unsigned long osc,
 
 	/* System timer */
 	mxc_timer_init(&gpt_clk, MX51_IO_ADDRESS(MX51_GPT1_BASE_ADDR),
-		MX51_MXC_INT_GPT);
+		       MX51_GPT1_BASE_ADDR, MX51_MXC_INT_GPT);
 	return 0;
 }
 
@@ -1429,6 +1429,6 @@ int __init mx53_clocks_init(unsigned long ckil, unsigned long osc,
 
 	/* System timer */
 	mxc_timer_init(&gpt_clk, MX53_IO_ADDRESS(MX53_GPT1_BASE_ADDR),
-		MX53_INT_GPT);
+		       MX53_GPT1_BASE_ADDR, MX53_INT_GPT);
 	return 0;
 }
diff --git a/arch/arm/mach-mxc91231/clock.c b/arch/arm/mach-mxc91231/clock.c
index 9fab505..ad18807 100644
--- a/arch/arm/mach-mxc91231/clock.c
+++ b/arch/arm/mach-mxc91231/clock.c
@@ -634,7 +634,8 @@ int __init mxc91231_clocks_init(unsigned long fref)
 	clkdev_add_table(lookups, ARRAY_SIZE(lookups));
 
 	gpt_base = MXC91231_IO_ADDRESS(MXC91231_GPT1_BASE_ADDR);
-	mxc_timer_init(&gpt_clk, gpt_base, MXC91231_INT_GPT);
+	mxc_timer_init(&gpt_clk, gpt_base,
+		       MXC91231_GPT1_BASE_ADDR, MXC91231_INT_GPT);
 
 	return 0;
 }
diff --git a/arch/arm/mach-omap2/board-cm-t35.c b/arch/arm/mach-omap2/board-cm-t35.c
index dac1416..651d13b 100644
--- a/arch/arm/mach-omap2/board-cm-t35.c
+++ b/arch/arm/mach-omap2/board-cm-t35.c
@@ -694,6 +694,7 @@ static void __init cm_t35_init_irq(void)
 	omap_init_irq();
 }
 
+#ifdef CONFIG_OMAP_MUX
 static struct omap_board_mux board_mux[] __initdata = {
 	/* nCS and IRQ for CM-T35 ethernet */
 	OMAP3_MUX(GPMC_NCS5, OMAP_MUX_MODE0),
@@ -791,6 +792,7 @@ static struct omap_board_mux board_mux[] __initdata = {
 
 	{ .reg_offset = OMAP_MUX_TERMINATOR },
 };
+#endif
 
 static struct omap_musb_board_data musb_board_data = {
 	.interface_type		= MUSB_INTERFACE_ULPI,
diff --git a/arch/arm/mach-omap2/board-cm-t3517.c b/arch/arm/mach-omap2/board-cm-t3517.c
index 8f9a64d..64a3039 100644
--- a/arch/arm/mach-omap2/board-cm-t3517.c
+++ b/arch/arm/mach-omap2/board-cm-t3517.c
@@ -264,6 +264,7 @@ static void __init cm_t3517_init_irq(void)
 	omap_init_irq();
 }
 
+#ifdef CONFIG_OMAP_MUX
 static struct omap_board_mux board_mux[] __initdata = {
 	/* GPIO186 - Green LED */
 	OMAP3_MUX(SYS_CLKOUT2, OMAP_MUX_MODE4 | OMAP_PIN_OUTPUT),
@@ -289,6 +290,7 @@ static struct omap_board_mux board_mux[] __initdata = {
 
 	{ .reg_offset = OMAP_MUX_TERMINATOR },
 };
+#endif
 
 static void __init cm_t3517_init(void)
 {
diff --git a/arch/arm/mach-omap2/board-devkit8000.c b/arch/arm/mach-omap2/board-devkit8000.c
index 9a2a31e..9680b8e 100644
--- a/arch/arm/mach-omap2/board-devkit8000.c
+++ b/arch/arm/mach-omap2/board-devkit8000.c
@@ -632,6 +632,7 @@ static const struct ehci_hcd_omap_platform_data ehci_pdata __initconst = {
 	.reset_gpio_port[2]  = -EINVAL
 };
 
+#ifdef CONFIG_OMAP_MUX
 static struct omap_board_mux board_mux[] __initdata = {
 	/* nCS and IRQ for Devkit8000 ethernet */
 	OMAP3_MUX(GPMC_NCS6, OMAP_MUX_MODE0),
@@ -785,6 +786,7 @@ static struct omap_board_mux board_mux[] __initdata = {
 
 	{ .reg_offset = OMAP_MUX_TERMINATOR },
 };
+#endif
 
 static void __init devkit8000_init(void)
 {
diff --git a/arch/arm/mach-omap2/clock44xx_data.c b/arch/arm/mach-omap2/clock44xx_data.c
index de9ec8d..964c1ad 100644
--- a/arch/arm/mach-omap2/clock44xx_data.c
+++ b/arch/arm/mach-omap2/clock44xx_data.c
@@ -3002,6 +3002,14 @@ static struct clk auxclkreq5_ck = {
 	.recalc		= &omap2_clksel_recalc,
 };
 
+static struct clk smp_twd = {
+       .name           = "smp_twd",
+       .parent         = &dpll_mpu_ck,
+       .ops            = &clkops_null,
+       .fixed_div      = 2,
+       .recalc         = &omap_fixed_divisor_recalc,
+};
+
 /*
  * clkdev
  */
@@ -3275,6 +3283,7 @@ static struct omap_clk omap44xx_clks[] = {
 	CLK(NULL,	"auxclkreq3_ck",		&auxclkreq3_ck,	CK_443X),
 	CLK(NULL,	"auxclkreq4_ck",		&auxclkreq4_ck,	CK_443X),
 	CLK(NULL,	"auxclkreq5_ck",		&auxclkreq5_ck,	CK_443X),
+	CLK(NULL,	"smp_twd",			&smp_twd,	CK_443X),
 };
 
 int __init omap4xxx_clk_init(void)
diff --git a/arch/arm/mach-omap2/irq.c b/arch/arm/mach-omap2/irq.c
index 23049c4..5413935 100644
--- a/arch/arm/mach-omap2/irq.c
+++ b/arch/arm/mach-omap2/irq.c
@@ -15,6 +15,7 @@
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <mach/hardware.h>
+#include <asm/ipipe.h>
 #include <asm/mach/irq.h>
 
 
@@ -32,9 +33,16 @@
 #define INTC_MIR_CLEAR0		0x0088
 #define INTC_MIR_SET0		0x008c
 #define INTC_PENDING_IRQ0	0x0098
+#define INTC_PRIO               0x0100
 /* Number of IRQ state bits in each MIR register */
 #define IRQ_BITS_PER_REG	32
 
+#if !defined(MULTI_OMAP1) && !defined(MULTI_OMAP2)
+#define inline_single inline
+#else
+#define inline_single
+#endif
+
 /*
  * OMAP2 has a number of different interrupt controllers, each interrupt
  * controller is identified as its own "bank". Register definitions are
@@ -65,12 +73,12 @@ static struct omap3_intc_regs intc_context[ARRAY_SIZE(irq_banks)];
 
 /* INTC bank register get/set */
 
-static void intc_bank_write_reg(u32 val, struct omap_irq_bank *bank, u16 reg)
+static inline_single void intc_bank_write_reg(u32 val, struct omap_irq_bank *bank, u16 reg)
 {
 	__raw_writel(val, bank->base_reg + reg);
 }
 
-static u32 intc_bank_read_reg(struct omap_irq_bank *bank, u16 reg)
+static inline_single u32 intc_bank_read_reg(struct omap_irq_bank *bank, u16 reg)
 {
 	return __raw_readl(bank->base_reg + reg);
 }
@@ -82,7 +90,7 @@ static int previous_irq;
  * an interrupt handler does not get posted before we unmask. Warn about
  * the interrupt handlers that need to flush posted writes.
  */
-static int omap_check_spurious(unsigned int irq)
+static inline_single int omap_check_spurious(unsigned int irq)
 {
 	u32 sir, spurious;
 
@@ -100,12 +108,13 @@ static int omap_check_spurious(unsigned int irq)
 }
 
 /* XXX: FIQ and additional INTC support (only MPU at the moment) */
-static void omap_ack_irq(struct irq_data *d)
+static inline_single void omap_ack_irq(struct irq_data *d)
 {
 	intc_bank_write_reg(0x1, &irq_banks[0], INTC_CONTROL);
+	dsb();
 }
 
-static void omap_mask_irq(struct irq_data *d)
+static inline_single void omap_mask_irq(struct irq_data *d)
 {
 	unsigned int irq = d->irq;
 	int offset = irq & (~(IRQ_BITS_PER_REG - 1));
@@ -149,6 +158,7 @@ static struct irq_chip omap_irq_chip = {
 	.name		= "INTC",
 	.irq_ack	= omap_mask_ack_irq,
 	.irq_mask	= omap_mask_irq,
+	.irq_mask_ack	= omap_mask_ack_irq,
 	.irq_unmask	= omap_unmask_irq,
 };
 
@@ -168,8 +178,15 @@ static void __init omap_irq_bank_init_one(struct omap_irq_bank *bank)
 	while (!(intc_bank_read_reg(bank, INTC_SYSSTATUS) & 0x1))
 		/* Wait for reset to complete */;
 
+#ifndef CONFIG_IPIPE
 	/* Enable autoidle */
 	intc_bank_write_reg(1 << 0, bank, INTC_SYSCONFIG);
+	intc_bank_write_reg(0x2, bank, INTC_IDLE);
+#else /* CONFIG_IPIPE */
+	/* Disable autoidle */
+	intc_bank_write_reg(0, bank, INTC_SYSCONFIG);
+	intc_bank_write_reg(0x1, bank, INTC_IDLE);
+#endif /* CONFIG_IPIPE */
 }
 
 int omap_irq_pending(void)
@@ -228,6 +245,68 @@ void __init omap_init_irq(void)
 	}
 }
 
+#if defined(CONFIG_IPIPE) && defined(__IPIPE_FEATURE_PIC_MUTE)
+DECLARE_PER_CPU(__ipipe_irqbits_t, __ipipe_muted_irqs);
+
+void omap3_intc_mute(void)
+{
+	struct omap_irq_bank *bank = &irq_banks[0];
+
+	intc_bank_write_reg(0x1, bank, INTC_THRESHOLD);
+	intc_bank_write_reg(0x1, bank, INTC_CONTROL);
+}
+
+void omap3_intc_unmute(void)
+{
+	struct omap_irq_bank *bank = &irq_banks[0];
+
+	intc_bank_write_reg(0xff, bank, INTC_THRESHOLD);
+}
+
+void omap3_intc_set_irq_prio(int irq, int hi)
+{
+	struct omap_irq_bank *bank = &irq_banks[0];
+
+	if (irq >= INTCPS_NR_MIR_REGS * 32)
+		return;
+
+	intc_bank_write_reg(hi ? 0 : 0xfc, bank, INTC_PRIO + 4 * irq);
+}
+
+void omap2_intc_mute(void)
+{
+	struct omap_irq_bank *bank = &irq_banks[0];
+	unsigned muted;
+	int i;
+
+	for (i = 0; i < INTCPS_NR_MIR_REGS; i++) {
+		muted = __ipipe_irqbits[i];
+		if (muted)
+			muted &= ~intc_bank_read_reg(bank,
+						     INTC_MIR0 + 0x20 * i);
+		__raw_get_cpu_var(__ipipe_muted_irqs)[i] = muted;
+		if (muted)
+			intc_bank_write_reg(muted, bank,
+					    INTC_MIR_SET0 + 0x20 * i);
+	}
+	intc_bank_write_reg(0x1, bank, INTC_CONTROL);
+}
+
+void omap2_intc_unmute(void)
+{
+	struct omap_irq_bank *bank = &irq_banks[0];
+	unsigned muted;
+	int i;
+
+	for (i = 0; i < INTCPS_NR_MIR_REGS; i++) {
+		muted = __raw_get_cpu_var(__ipipe_muted_irqs)[i];
+		if (muted)
+			intc_bank_write_reg(muted, bank,
+					    INTC_MIR_CLEAR0 + 0x20 * i);
+	}
+}
+#endif /* __IPIPE_FEATURE_PIC_MUTE */
+
 #ifdef CONFIG_ARCH_OMAP3
 void omap_intc_save_context(void)
 {
diff --git a/arch/arm/mach-omap2/timer-gp.c b/arch/arm/mach-omap2/timer-gp.c
index 0fc550e..9b0f924 100644
--- a/arch/arm/mach-omap2/timer-gp.c
+++ b/arch/arm/mach-omap2/timer-gp.c
@@ -35,6 +35,7 @@
 #include <linux/irq.h>
 #include <linux/clocksource.h>
 #include <linux/clockchips.h>
+#include <linux/ipipe.h>
 
 #include <asm/mach/time.h>
 #include <plat/dmtimer.h>
@@ -53,13 +54,31 @@ static struct clock_event_device clockevent_gpt;
 static u8 __initdata gptimer_id = 1;
 static u8 __initdata inited;
 struct omap_dm_timer *gptimer_wakeup;
+#ifndef CONFIG_OMAP_32K_TIMER
+static struct omap_dm_timer *gpt_clocksource;
+#endif /* !CONFIG_OMAP_32K_TIMER */
+
+#if defined(CONFIG_IPIPE) && !defined(CONFIG_SMP)
+int __ipipe_mach_timerint;
+EXPORT_SYMBOL(__ipipe_mach_timerint);
+
+unsigned int __ipipe_mach_ticks_per_jiffy;
+EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy);
+
+int __ipipe_mach_timerstolen;
+EXPORT_SYMBOL(__ipipe_mach_timerstolen);
+#endif /* CONFIG_IPIPE && !CONFIG_SMP */
 
 static irqreturn_t omap2_gp_timer_interrupt(int irq, void *dev_id)
 {
-	struct omap_dm_timer *gpt = (struct omap_dm_timer *)dev_id;
 	struct clock_event_device *evt = &clockevent_gpt;
+#if !defined(CONFIG_IPIPE) || defined(CONFIG_SMP)
+	struct omap_dm_timer *gpt = (struct omap_dm_timer *)dev_id;
 
 	omap_dm_timer_write_status(gpt, OMAP_TIMER_INT_OVERFLOW);
+#else /* CONFIG_IPIPE */
+	__ipipe_tsc_update();
+#endif /* CONFIG_IPIPE */
 
 	evt->event_handler(evt);
 	return IRQ_HANDLED;
@@ -129,6 +148,24 @@ int __init omap2_gp_clockevent_set_gptimer(u8 id)
 	return 0;
 }
 
+#ifdef CONFIG_IPIPE
+static struct __ipipe_tscinfo __maybe_unused tsc_info = {
+	.type = IPIPE_TSC_TYPE_FREERUNNING,
+	.u = {
+		{
+			.mask = 0xffffffff,
+		},
+	},
+};
+
+#ifndef CONFIG_SMP
+int __ipipe_check_tickdev(const char *devname)
+{
+	return !strcmp(devname, clockevent_gpt.name);
+}
+#endif /* !CONFIG_SMP */
+#endif /* CONFIG_IPIPE */
+
 static void __init omap2_gp_clockevent_init(void)
 {
 	u32 tick_rate;
@@ -136,7 +173,7 @@ static void __init omap2_gp_clockevent_init(void)
 
 	inited = 1;
 
-	gptimer = omap_dm_timer_request_specific(gptimer_id);
+	gptimer = omap_dm_timer_request_specific_nonposted(gptimer_id);
 	BUG_ON(gptimer == NULL);
 	gptimer_wakeup = gptimer;
 
@@ -158,6 +195,12 @@ static void __init omap2_gp_clockevent_init(void)
 		gptimer_id, tick_rate);
 
 	omap2_gp_timer_irq.dev_id = (void *)gptimer;
+
+#if defined(CONFIG_IPIPE) && !defined(CONFIG_SMP)
+	__ipipe_mach_timerint = omap_dm_timer_get_irq(gptimer);
+	__ipipe_mach_ticks_per_jiffy = (tick_rate + HZ / 2) / HZ;
+#endif /* CONFIG_IPIPE && !CONFIG_SMP*/
+
 	setup_irq(omap_dm_timer_get_irq(gptimer), &omap2_gp_timer_irq);
 	omap_dm_timer_set_int_enable(gptimer, OMAP_TIMER_INT_OVERFLOW);
 
@@ -168,7 +211,6 @@ static void __init omap2_gp_clockevent_init(void)
 	clockevent_gpt.min_delta_ns =
 		clockevent_delta2ns(3, &clockevent_gpt);
 		/* Timer internal resynch latency. */
-
 	clockevent_gpt.cpumask = cpumask_of(0);
 	clockevents_register_device(&clockevent_gpt);
 }
@@ -176,7 +218,7 @@ static void __init omap2_gp_clockevent_init(void)
 /* Clocksource code */
 
 #ifdef CONFIG_OMAP_32K_TIMER
-/* 
+/*
  * When 32k-timer is enabled, don't use GPTimer for clocksource
  * instead, just leave default clocksource which uses the 32k
  * sync counter.  See clocksource setup in plat-omap/counter_32k.c
@@ -193,7 +235,13 @@ static void __init omap2_gp_clocksource_init(void)
  */
 static DEFINE_CLOCK_DATA(cd);
 static struct omap_dm_timer *gpt_clocksource;
-static cycle_t clocksource_read_cycles(struct clocksource *cs)
+
+static cycle_t notrace clocksource_read_dummy(struct clocksource *cs)
+{
+	return 0;
+}
+
+static cycle_t notrace clocksource_read_cycles(struct clocksource *cs)
 {
 	return (cycle_t)omap_dm_timer_read_counter(gpt_clocksource);
 }
@@ -201,11 +249,17 @@ static cycle_t clocksource_read_cycles(struct clocksource *cs)
 static struct clocksource clocksource_gpt = {
 	.name		= "gp timer",
 	.rating		= 300,
-	.read		= clocksource_read_cycles,
+	.read		= clocksource_read_dummy,
 	.mask		= CLOCKSOURCE_MASK(32),
 	.flags		= CLOCK_SOURCE_IS_CONTINUOUS,
 };
 
+unsigned long long notrace sched_clock(void)
+{
+	u32 cyc = clocksource_gpt.read(&clocksource_gpt);
+	return cyc_to_sched_clock(&cd, cyc, (u32)~0);
+}
+
 static void notrace dmtimer_update_sched_clock(void)
 {
 	u32 cyc;
@@ -225,7 +279,12 @@ static void __init omap2_gp_clocksource_init(void)
 	static char err2[] __initdata = KERN_ERR
 		"%s: can't register clocksource!\n";
 
-	gpt = omap_dm_timer_request();
+#ifdef CONFIG_IPIPE
+	if (cpu_is_omap34xx())
+		gpt = omap_dm_timer_request_specific(3);
+	else
+#endif /* CONFIG_IPIPE */
+		gpt = omap_dm_timer_request();
 	if (!gpt)
 		printk(err1, clocksource_gpt.name);
 	gpt_clocksource = gpt;
@@ -235,8 +294,18 @@ static void __init omap2_gp_clocksource_init(void)
 
 	omap_dm_timer_set_load_start(gpt, 1, 0);
 
+	clocksource_gpt.read = &clocksource_read_cycles;
 	init_sched_clock(&cd, dmtimer_update_sched_clock, 32, tick_rate);
 
+#if defined(CONFIG_IPIPE) && !defined(CONFIG_SMP)
+	tsc_info.freq = tick_rate;
+	tsc_info.counter_vaddr =
+		omap_dm_timer_get_virt_counter_addr(gpt_clocksource);
+	tsc_info.u.counter_paddr =
+		omap_dm_timer_get_phys_counter_addr(gpt_clocksource);
+	__ipipe_tsc_register(&tsc_info);
+#endif
+
 	if (clocksource_register_hz(&clocksource_gpt, tick_rate))
 		printk(err2, clocksource_gpt.name);
 }
@@ -252,6 +321,9 @@ static void __init omap2_gp_timer_init(void)
 #endif
 	omap_dm_timer_init();
 
+#if defined(CONFIG_IPIPE) && !defined(CONFIG_SMP)
+	omap2_gp_clockevent_set_gptimer(2);
+#endif
 	omap2_gp_clockevent_init();
 	omap2_gp_clocksource_init();
 }
@@ -259,3 +331,38 @@ static void __init omap2_gp_timer_init(void)
 struct sys_timer omap_timer = {
 	.init	= omap2_gp_timer_init,
 };
+
+#if defined(CONFIG_IPIPE) && !defined(CONFIG_SMP)
+void __ipipe_mach_acktimer(void)
+{
+	omap_dm_timer_write_status(gptimer, OMAP_TIMER_INT_OVERFLOW);
+	omap_dm_timer_read_status(gptimer);
+}
+
+/*
+ * Reprogram the timer
+ */
+
+void __ipipe_mach_set_dec(unsigned long delay)
+{
+	if (delay > 3)
+		omap_dm_timer_set_load_start(gptimer, 0, 0xffffffff - delay);
+	else
+		ipipe_trigger_irq(__ipipe_mach_timerint);
+}
+EXPORT_SYMBOL(__ipipe_mach_set_dec);
+
+void __ipipe_mach_release_timer(void)
+{
+	struct clock_event_device *ckdev = &clockevent_gpt;
+	ckdev->set_mode(ckdev->mode, ckdev);
+	if (ckdev->mode == CLOCK_EVT_MODE_ONESHOT)
+		ckdev->set_next_event(__ipipe_mach_ticks_per_jiffy, ckdev);
+}
+EXPORT_SYMBOL(__ipipe_mach_release_timer);
+
+unsigned long __ipipe_mach_get_dec(void)
+{
+	return 0xffffffff - omap_dm_timer_read_counter(gptimer);
+}
+#endif /* CONFIG_IPIPE */
diff --git a/arch/arm/mach-omap2/timer-mpu.c b/arch/arm/mach-omap2/timer-mpu.c
index 954682e..d165d19 100644
--- a/arch/arm/mach-omap2/timer-mpu.c
+++ b/arch/arm/mach-omap2/timer-mpu.c
@@ -30,5 +30,7 @@ void __cpuinit local_timer_setup(struct clock_event_device *evt)
 {
 	evt->irq = OMAP44XX_IRQ_LOCALTIMER;
 	twd_timer_setup(evt);
+#ifdef CONFIG_IPIPE
+	gt_setup(OMAP44XX_LOCAL_TWD_BASE - 0x400, 32);
+#endif /* CONFIG_IPIPE */
 }
-
diff --git a/arch/arm/mach-pxa/irq.c b/arch/arm/mach-pxa/irq.c
index 2693e3c..d2f0899 100644
--- a/arch/arm/mach-pxa/irq.c
+++ b/arch/arm/mach-pxa/irq.c
@@ -86,6 +86,9 @@ static struct irq_chip pxa_internal_irq_chip = {
 	.name		= "SC",
 	.irq_ack	= pxa_mask_irq,
 	.irq_mask	= pxa_mask_irq,
+#ifdef CONFIG_IPIPE
+	.irq_mask_ack	= pxa_mask_irq,
+#endif /* CONFIG_IPIPE */
 	.irq_unmask	= pxa_unmask_irq,
 };
 
diff --git a/arch/arm/mach-pxa/leds-idp.c b/arch/arm/mach-pxa/leds-idp.c
index 8b9c171..144cdc1 100644
--- a/arch/arm/mach-pxa/leds-idp.c
+++ b/arch/arm/mach-pxa/leds-idp.c
@@ -13,6 +13,7 @@
 
 
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <mach/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-pxa/leds-lubbock.c b/arch/arm/mach-pxa/leds-lubbock.c
index e26d5ef..706cced 100644
--- a/arch/arm/mach-pxa/leds-lubbock.c
+++ b/arch/arm/mach-pxa/leds-lubbock.c
@@ -12,6 +12,7 @@
  */
 
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <mach/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-pxa/leds-mainstone.c b/arch/arm/mach-pxa/leds-mainstone.c
index db4af5e..c204cd4 100644
--- a/arch/arm/mach-pxa/leds-mainstone.c
+++ b/arch/arm/mach-pxa/leds-mainstone.c
@@ -11,6 +11,7 @@
  */
 
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <mach/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-pxa/lpd270.c b/arch/arm/mach-pxa/lpd270.c
index c9a3e77..8465a16 100644
--- a/arch/arm/mach-pxa/lpd270.c
+++ b/arch/arm/mach-pxa/lpd270.c
@@ -24,6 +24,7 @@
 #include <linux/mtd/mtd.h>
 #include <linux/mtd/partitions.h>
 #include <linux/pwm_backlight.h>
+#include <linux/ipipe.h>
 
 #include <asm/types.h>
 #include <asm/setup.h>
@@ -130,7 +131,7 @@ static void lpd270_irq_handler(unsigned int irq, struct irq_desc *desc)
 		desc->irq_data.chip->irq_ack(&desc->irq_data);
 		if (likely(pending)) {
 			irq = LPD270_IRQ(0) + __ffs(pending);
-			generic_handle_irq(irq);
+			ipipe_handle_chained_irq(irq);
 
 			pending = __raw_readw(LPD270_INT_STATUS) &
 						lpd270_irq_enabled;
diff --git a/arch/arm/mach-pxa/lubbock.c b/arch/arm/mach-pxa/lubbock.c
index dca20de..a6691ac 100644
--- a/arch/arm/mach-pxa/lubbock.c
+++ b/arch/arm/mach-pxa/lubbock.c
@@ -22,6 +22,7 @@
 #include <linux/mtd/mtd.h>
 #include <linux/mtd/partitions.h>
 #include <linux/smc91x.h>
+#include <linux/ipipe.h>
 
 #include <linux/spi/spi.h>
 #include <linux/spi/ads7846.h>
@@ -151,7 +152,7 @@ static void lubbock_irq_handler(unsigned int irq, struct irq_desc *desc)
 		desc->irq_data.chip->irq_ack(&desc->irq_data);
 		if (likely(pending)) {
 			irq = LUBBOCK_IRQ(0) + __ffs(pending);
-			generic_handle_irq(irq);
+			ipipe_handle_chained_irq(irq);
 		}
 		pending = LUB_IRQ_SET_CLR & lubbock_irq_enabled;
 	} while (pending);
diff --git a/arch/arm/mach-pxa/mainstone.c b/arch/arm/mach-pxa/mainstone.c
index d4b6f23..25c474d 100644
--- a/arch/arm/mach-pxa/mainstone.c
+++ b/arch/arm/mach-pxa/mainstone.c
@@ -27,6 +27,7 @@
 #include <linux/gpio_keys.h>
 #include <linux/pwm_backlight.h>
 #include <linux/smc91x.h>
+#include <linux/ipipe.h>
 
 #include <asm/types.h>
 #include <asm/setup.h>
@@ -152,7 +153,7 @@ static void mainstone_irq_handler(unsigned int irq, struct irq_desc *desc)
 		desc->irq_data.chip->irq_ack(&desc->irq_data);
 		if (likely(pending)) {
 			irq = MAINSTONE_IRQ(0) + __ffs(pending);
-			generic_handle_irq(irq);
+			ipipe_handle_chained_irq(irq);
 		}
 		pending = MST_INTSETCLR & mainstone_irq_enabled;
 	} while (pending);
diff --git a/arch/arm/mach-pxa/pcm990-baseboard.c b/arch/arm/mach-pxa/pcm990-baseboard.c
index 90820fa..2e130fe 100644
--- a/arch/arm/mach-pxa/pcm990-baseboard.c
+++ b/arch/arm/mach-pxa/pcm990-baseboard.c
@@ -24,6 +24,7 @@
 #include <linux/platform_device.h>
 #include <linux/i2c.h>
 #include <linux/pwm_backlight.h>
+#include <linux/ipipe.h>
 
 #include <media/soc_camera.h>
 
@@ -269,7 +270,7 @@ static void pcm990_irq_handler(unsigned int irq, struct irq_desc *desc)
 		desc->irq_data.chip->irq_ack(&desc->irq_data);
 		if (likely(pending)) {
 			irq = PCM027_IRQ(0) + __ffs(pending);
-			generic_handle_irq(irq);
+			ipipe_handle_chained_irq(irq);
 		}
 		pending = (~PCM990_INTSETCLR) & pcm990_irq_enabled;
 	} while (pending);
diff --git a/arch/arm/mach-pxa/time.c b/arch/arm/mach-pxa/time.c
index e7f64d9..4bc2262 100644
--- a/arch/arm/mach-pxa/time.c
+++ b/arch/arm/mach-pxa/time.c
@@ -24,6 +24,28 @@
 #include <asm/sched_clock.h>
 #include <mach/regs-ost.h>
 
+#ifdef CONFIG_IPIPE
+int __ipipe_mach_timerint = IRQ_OST0;
+EXPORT_SYMBOL(__ipipe_mach_timerint);
+
+int __ipipe_mach_timerstolen = 0;
+EXPORT_SYMBOL(__ipipe_mach_timerstolen);
+
+unsigned int __ipipe_mach_ticks_per_jiffy = LATCH;
+EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy);
+
+static struct __ipipe_tscinfo tsc_info = {
+	.type = IPIPE_TSC_TYPE_FREERUNNING,
+	.counter_vaddr = io_p2v(0x40A00010UL),
+	.u = {
+		{
+			.counter_paddr = 0x40A00010UL,
+			.mask = 0xffffffff,
+		},
+	},
+};
+#endif /* CONFIG_IPIPE */
+
 /*
  * This is PXA's sched_clock implementation. This has a resolution
  * of at least 308 ns and a maximum value of 208 days.
@@ -55,8 +77,12 @@ pxa_ost0_interrupt(int irq, void *dev_id)
 	struct clock_event_device *c = dev_id;
 
 	/* Disarm the compare/match, signal the event. */
+#ifndef CONFIG_IPIPE
 	OIER &= ~OIER_E0;
 	OSSR = OSSR_M0;
+#else /* CONFIG_IPIPE */
+	__ipipe_tsc_update();
+#endif /* CONFIG_IPIPE */
 	c->event_handler(c);
 
 	return IRQ_HANDLED;
@@ -67,8 +93,8 @@ pxa_osmr0_set_next_event(unsigned long delta, struct clock_event_device *dev)
 {
 	unsigned long next, oscr;
 
-	OIER |= OIER_E0;
 	next = OSCR + delta;
+	OIER |= OIER_E0;
 	OSMR0 = next;
 	oscr = OSCR;
 
@@ -106,6 +132,46 @@ static struct clock_event_device ckevt_pxa_osmr0 = {
 	.set_mode	= pxa_osmr0_set_mode,
 };
 
+#ifdef CONFIG_IPIPE
+int __ipipe_check_tickdev(const char *devname)
+{
+	return !strcmp(devname, ckevt_pxa_osmr0.name);
+}
+
+void __ipipe_mach_acktimer(void)
+{
+	OSSR = OSSR_M0;  /* Clear match on timer 0 */
+	OIER &= ~OIER_E0;
+}
+
+/*
+ * Reprogram the timer
+ */
+
+void __ipipe_mach_set_dec(unsigned long delay)
+{
+	if (delay > MIN_OSCR_DELTA) {
+		OSMR0 = delay + OSCR;
+		OIER |= OIER_E0;
+	} else
+		ipipe_trigger_irq(IRQ_OST0);
+}
+EXPORT_SYMBOL(__ipipe_mach_set_dec);
+
+void __ipipe_mach_release_timer(void)
+{
+	pxa_osmr0_set_mode(ckevt_pxa_osmr0.mode, &ckevt_pxa_osmr0);
+	if (ckevt_pxa_osmr0.mode == CLOCK_EVT_MODE_ONESHOT)
+		pxa_osmr0_set_next_event(LATCH, &ckevt_pxa_osmr0);
+}
+EXPORT_SYMBOL(__ipipe_mach_release_timer);
+
+unsigned long __ipipe_mach_get_dec(void)
+{
+	return OSMR0 - OSCR;
+}
+#endif /* CONFIG_IPIPE */
+
 static cycle_t pxa_read_oscr(struct clocksource *cs)
 {
 	return OSCR;
@@ -145,6 +211,11 @@ static void __init pxa_timer_init(void)
 
 	setup_irq(IRQ_OST0, &pxa_ost0_irq);
 
+#ifdef CONFIG_IPIPE
+	tsc_info.freq = clock_tick_rate;
+	__ipipe_tsc_register(&tsc_info);
+#endif /* CONFIG_IPIPE */
+
 	clocksource_register_hz(&cksrc_pxa_oscr0, clock_tick_rate);
 	clockevents_register_device(&ckevt_pxa_osmr0);
 }
diff --git a/arch/arm/mach-pxa/viper.c b/arch/arm/mach-pxa/viper.c
index 49eeeab..7bdec58 100644
--- a/arch/arm/mach-pxa/viper.c
+++ b/arch/arm/mach-pxa/viper.c
@@ -43,6 +43,7 @@
 #include <linux/mtd/mtd.h>
 #include <linux/mtd/partitions.h>
 #include <linux/mtd/physmap.h>
+#include <linux/ipipe.h>
 
 #include <mach/pxa25x.h>
 #include <mach/audio.h>
@@ -287,7 +288,7 @@ static void viper_irq_handler(unsigned int irq, struct irq_desc *desc)
 
 		if (likely(pending)) {
 			irq = viper_bit_to_irq(__ffs(pending));
-			generic_handle_irq(irq);
+			ipipe_handle_chained_irq(irq);
 		}
 		pending = viper_irq_pending();
 	} while (pending);
diff --git a/arch/arm/mach-realview/core.c b/arch/arm/mach-realview/core.c
index 1c6602c..5df2619 100644
--- a/arch/arm/mach-realview/core.c
+++ b/arch/arm/mach-realview/core.c
@@ -668,6 +668,7 @@ void __iomem *timer0_va_base;
 void __iomem *timer1_va_base;
 void __iomem *timer2_va_base;
 void __iomem *timer3_va_base;
+void __iomem *timer3_pa_base;
 
 /*
  * Set up the clock source and clock events devices
@@ -678,14 +679,14 @@ void __init realview_timer_init(unsigned int timer_irq)
 
 	versatile_sched_clock_init(REFCOUNTER, 24000000);
 
-	/* 
-	 * set clock frequency: 
+	/*
+	 * set clock frequency:
 	 *	REALVIEW_REFCLK is 32KHz
 	 *	REALVIEW_TIMCLK is 1MHz
 	 */
 	val = readl(__io_address(REALVIEW_SCTL_BASE));
 	writel((REALVIEW_TIMCLK << REALVIEW_TIMER1_EnSel) |
-	       (REALVIEW_TIMCLK << REALVIEW_TIMER2_EnSel) | 
+	       (REALVIEW_TIMCLK << REALVIEW_TIMER2_EnSel) |
 	       (REALVIEW_TIMCLK << REALVIEW_TIMER3_EnSel) |
 	       (REALVIEW_TIMCLK << REALVIEW_TIMER4_EnSel) | val,
 	       __io_address(REALVIEW_SCTL_BASE));
@@ -698,7 +699,7 @@ void __init realview_timer_init(unsigned int timer_irq)
 	writel(0, timer2_va_base + TIMER_CTRL);
 	writel(0, timer3_va_base + TIMER_CTRL);
 
-	sp804_clocksource_init(timer3_va_base);
+	sp804_clocksource_init(timer3_va_base, timer3_pa_base);
 	sp804_clockevents_init(timer0_va_base, timer_irq);
 }
 
diff --git a/arch/arm/mach-realview/core.h b/arch/arm/mach-realview/core.h
index 693239d..a4398a5 100644
--- a/arch/arm/mach-realview/core.h
+++ b/arch/arm/mach-realview/core.h
@@ -57,6 +57,7 @@ extern void __iomem *timer0_va_base;
 extern void __iomem *timer1_va_base;
 extern void __iomem *timer2_va_base;
 extern void __iomem *timer3_va_base;
+extern void __iomem *timer3_pa_base;
 
 extern void realview_leds_event(led_event_t ledevt);
 extern void realview_timer_init(unsigned int timer_irq);
diff --git a/arch/arm/mach-realview/realview_eb.c b/arch/arm/mach-realview/realview_eb.c
index 6ef5c5e..8166741 100644
--- a/arch/arm/mach-realview/realview_eb.c
+++ b/arch/arm/mach-realview/realview_eb.c
@@ -420,6 +420,7 @@ static void __init realview_eb_timer_init(void)
 	timer1_va_base = __io_address(REALVIEW_EB_TIMER0_1_BASE) + 0x20;
 	timer2_va_base = __io_address(REALVIEW_EB_TIMER2_3_BASE);
 	timer3_va_base = __io_address(REALVIEW_EB_TIMER2_3_BASE) + 0x20;
+	timer3_pa_base = REALVIEW_EB_TIMER2_3_BASE + 0x20;
 
 	if (core_tile_eb11mp() || core_tile_a9mp()) {
 #ifdef CONFIG_LOCAL_TIMERS
diff --git a/arch/arm/mach-realview/realview_pb1176.c b/arch/arm/mach-realview/realview_pb1176.c
index cbdc97a..005571e 100644
--- a/arch/arm/mach-realview/realview_pb1176.c
+++ b/arch/arm/mach-realview/realview_pb1176.c
@@ -321,6 +321,7 @@ static void __init realview_pb1176_timer_init(void)
 	timer1_va_base = __io_address(REALVIEW_PB1176_TIMER0_1_BASE) + 0x20;
 	timer2_va_base = __io_address(REALVIEW_PB1176_TIMER2_3_BASE);
 	timer3_va_base = __io_address(REALVIEW_PB1176_TIMER2_3_BASE) + 0x20;
+	timer3_pa_base = REALVIEW_PB1176_TIMER2_3_BASE + 0x20;
 
 	realview_timer_init(IRQ_DC1176_TIMER0);
 }
diff --git a/arch/arm/mach-realview/realview_pb11mp.c b/arch/arm/mach-realview/realview_pb11mp.c
index 8e8ab7d..8fa2f71 100644
--- a/arch/arm/mach-realview/realview_pb11mp.c
+++ b/arch/arm/mach-realview/realview_pb11mp.c
@@ -325,6 +325,7 @@ static void __init realview_pb11mp_timer_init(void)
 	timer1_va_base = __io_address(REALVIEW_PB11MP_TIMER0_1_BASE) + 0x20;
 	timer2_va_base = __io_address(REALVIEW_PB11MP_TIMER2_3_BASE);
 	timer3_va_base = __io_address(REALVIEW_PB11MP_TIMER2_3_BASE) + 0x20;
+	timer3_pa_base = REALVIEW_PB11MP_TIMER2_3_BASE + 0x20;
 
 #ifdef CONFIG_LOCAL_TIMERS
 	twd_base = __io_address(REALVIEW_TC11MP_TWD_BASE);
diff --git a/arch/arm/mach-realview/realview_pba8.c b/arch/arm/mach-realview/realview_pba8.c
index 841118e..b2233e8 100644
--- a/arch/arm/mach-realview/realview_pba8.c
+++ b/arch/arm/mach-realview/realview_pba8.c
@@ -284,6 +284,7 @@ static void __init realview_pba8_timer_init(void)
 	timer1_va_base = __io_address(REALVIEW_PBA8_TIMER0_1_BASE) + 0x20;
 	timer2_va_base = __io_address(REALVIEW_PBA8_TIMER2_3_BASE);
 	timer3_va_base = __io_address(REALVIEW_PBA8_TIMER2_3_BASE) + 0x20;
+	timer3_pa_base = REALVIEW_PBA8_TIMER2_3_BASE + 0x20;
 
 	realview_timer_init(IRQ_PBA8_TIMER0_1);
 }
diff --git a/arch/arm/mach-realview/realview_pbx.c b/arch/arm/mach-realview/realview_pbx.c
index 02b755b..4298aae 100644
--- a/arch/arm/mach-realview/realview_pbx.c
+++ b/arch/arm/mach-realview/realview_pbx.c
@@ -328,6 +328,7 @@ static void __init realview_pbx_timer_init(void)
 	timer1_va_base = __io_address(REALVIEW_PBX_TIMER0_1_BASE) + 0x20;
 	timer2_va_base = __io_address(REALVIEW_PBX_TIMER2_3_BASE);
 	timer3_va_base = __io_address(REALVIEW_PBX_TIMER2_3_BASE) + 0x20;
+	timer3_pa_base = REALVIEW_PBX_TIMER2_3_BASE + 0x20;
 
 #ifdef CONFIG_LOCAL_TIMERS
 	if (core_tile_pbx11mp() || core_tile_pbxa9mp())
diff --git a/arch/arm/mach-s3c2410/include/mach/irqs.h b/arch/arm/mach-s3c2410/include/mach/irqs.h
index e5a68ea..579f9c6 100644
--- a/arch/arm/mach-s3c2410/include/mach/irqs.h
+++ b/arch/arm/mach-s3c2410/include/mach/irqs.h
@@ -3,6 +3,8 @@
  * Copyright (c) 2003-2005 Simtec Electronics
  *   Ben Dooks <ben@simtec.co.uk>
  *
+ * Copyright (C) 2006, 2007 Sebastian Smolorz <ssmolorz@emlix.com>, emlix GmbH
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
diff --git a/arch/arm/mach-s3c2440/irq.c b/arch/arm/mach-s3c2440/irq.c
index acad442..7327b5f 100644
--- a/arch/arm/mach-s3c2440/irq.c
+++ b/arch/arm/mach-s3c2440/irq.c
@@ -3,6 +3,8 @@
  * Copyright (c) 2003-2004 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
+ * Copyright (C) 2006, 2007 Sebastian Smolorz <ssmolorz@emlix.com>, emlix GmbH
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
@@ -25,6 +27,7 @@
 #include <linux/ioport.h>
 #include <linux/sysdev.h>
 #include <linux/io.h>
+#include <linux/ipipe.h>
 
 #include <mach/hardware.h>
 #include <asm/irq.h>
@@ -57,10 +60,10 @@ static void s3c_irq_demux_wdtac97(unsigned int irq,
 
 	if (subsrc != 0) {
 		if (subsrc & 1) {
-			generic_handle_irq(IRQ_S3C2440_WDT);
+			ipipe_handle_chained_irq(IRQ_S3C2440_WDT);
 		}
 		if (subsrc & 2) {
-			generic_handle_irq(IRQ_S3C2440_AC97);
+			ipipe_handle_chained_irq(IRQ_S3C2440_AC97);
 		}
 	}
 }
diff --git a/arch/arm/mach-s3c2440/s3c244x-irq.c b/arch/arm/mach-s3c2440/s3c244x-irq.c
index 83daf4e..cdf1cdd 100644
--- a/arch/arm/mach-s3c2440/s3c244x-irq.c
+++ b/arch/arm/mach-s3c2440/s3c244x-irq.c
@@ -3,6 +3,8 @@
  * Copyright (c) 2003-2004 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
+ * Copyright (C) 2007 Sebastian Smolorz <ssmolorz@emlix.com>, emlix GmbH
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
@@ -25,6 +27,7 @@
 #include <linux/ioport.h>
 #include <linux/sysdev.h>
 #include <linux/io.h>
+#include <linux/ipipe.h>
 
 #include <mach/hardware.h>
 #include <asm/irq.h>
@@ -57,10 +60,10 @@ static void s3c_irq_demux_cam(unsigned int irq,
 
 	if (subsrc != 0) {
 		if (subsrc & 1) {
-			generic_handle_irq(IRQ_S3C2440_CAM_C);
+			ipipe_handle_chained_irq(IRQ_S3C2440_CAM_C);
 		}
 		if (subsrc & 2) {
-			generic_handle_irq(IRQ_S3C2440_CAM_P);
+			ipipe_handle_chained_irq(IRQ_S3C2440_CAM_P);
 		}
 	}
 }
@@ -87,6 +90,9 @@ s3c_irq_cam_ack(struct irq_data *data)
 
 static struct irq_chip s3c_irq_cam = {
 	.irq_mask	= s3c_irq_cam_mask,
+#ifdef CONFIG_IPIPE
+	.irq_mask_ack   = s3c_irq_cam_ack,
+#endif /* CONFIG_IPIPE */
 	.irq_unmask	= s3c_irq_cam_unmask,
 	.irq_ack	= s3c_irq_cam_ack,
 };
diff --git a/arch/arm/mach-sa1100/gpio.c b/arch/arm/mach-sa1100/gpio.c
index 0d3829a..873f883 100644
--- a/arch/arm/mach-sa1100/gpio.c
+++ b/arch/arm/mach-sa1100/gpio.c
@@ -32,9 +32,9 @@ static int sa1100_direction_input(struct gpio_chip *chip, unsigned offset)
 {
 	unsigned long flags;
 
-	local_irq_save(flags);
+	local_irq_save_hw(flags);
 	GPDR &= ~GPIO_GPIO(offset);
-	local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 	return 0;
 }
 
@@ -42,10 +42,10 @@ static int sa1100_direction_output(struct gpio_chip *chip, unsigned offset, int
 {
 	unsigned long flags;
 
-	local_irq_save(flags);
+	local_irq_save_hw(flags);
 	sa1100_gpio_set(chip, offset, value);
 	GPDR |= GPIO_GPIO(offset);
-	local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 	return 0;
 }
 
diff --git a/arch/arm/mach-sa1100/irq.c b/arch/arm/mach-sa1100/irq.c
index 3d85dfa..cb6e8e5 100644
--- a/arch/arm/mach-sa1100/irq.c
+++ b/arch/arm/mach-sa1100/irq.c
@@ -15,6 +15,7 @@
 #include <linux/irq.h>
 #include <linux/ioport.h>
 #include <linux/sysdev.h>
+#include <linux/ipipe.h>
 
 #include <mach/hardware.h>
 #include <asm/mach/irq.h>
@@ -125,7 +126,7 @@ sa1100_high_gpio_handler(unsigned int irq, struct irq_desc *desc)
 		mask >>= 11;
 		do {
 			if (mask & 1)
-				generic_handle_irq(irq);
+				ipipe_handle_chained_irq(irq);
 			mask >>= 1;
 			irq++;
 		} while (mask);
@@ -217,6 +218,9 @@ static struct irq_chip sa1100_normal_chip = {
 	.name		= "SC",
 	.irq_ack	= sa1100_mask_irq,
 	.irq_mask	= sa1100_mask_irq,
+#ifdef CONFIG_IPIPE
+	.irq_mask_ack	= sa1100_mask_irq,
+#endif /* CONFIG_IPIPE */
 	.irq_unmask	= sa1100_unmask_irq,
 	.irq_set_wake	= sa1100_set_wake,
 };
@@ -255,7 +259,7 @@ static int sa1100irq_suspend(struct sys_device *dev, pm_message_t state)
 	 */
 	GRER = PWER & GPIO_IRQ_rising_edge;
 	GFER = PWER & GPIO_IRQ_falling_edge;
-	
+
 	/*
 	 * Clear any pending GPIO interrupts.
 	 */
diff --git a/arch/arm/mach-sa1100/leds-assabet.c b/arch/arm/mach-sa1100/leds-assabet.c
index 64e9b4b..0d4e5a4 100644
--- a/arch/arm/mach-sa1100/leds-assabet.c
+++ b/arch/arm/mach-sa1100/leds-assabet.c
@@ -10,6 +10,7 @@
  *   - Red   - on if system is not idle
  */
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <mach/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-sa1100/leds-badge4.c b/arch/arm/mach-sa1100/leds-badge4.c
index cf1e384..e41140f 100644
--- a/arch/arm/mach-sa1100/leds-badge4.c
+++ b/arch/arm/mach-sa1100/leds-badge4.c
@@ -11,6 +11,7 @@
  */
 
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <mach/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-sa1100/leds-cerf.c b/arch/arm/mach-sa1100/leds-cerf.c
index 259b48e..3ab9eff 100644
--- a/arch/arm/mach-sa1100/leds-cerf.c
+++ b/arch/arm/mach-sa1100/leds-cerf.c
@@ -4,6 +4,7 @@
  * Author: ???
  */
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <mach/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-sa1100/leds-hackkit.c b/arch/arm/mach-sa1100/leds-hackkit.c
index 2bce137..0a4fe91 100644
--- a/arch/arm/mach-sa1100/leds-hackkit.c
+++ b/arch/arm/mach-sa1100/leds-hackkit.c
@@ -10,6 +10,7 @@
  * as cpu led, the green one is used as timer led.
  */
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <mach/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-sa1100/leds-lart.c b/arch/arm/mach-sa1100/leds-lart.c
index 0505a1f..f03128d 100644
--- a/arch/arm/mach-sa1100/leds-lart.c
+++ b/arch/arm/mach-sa1100/leds-lart.c
@@ -10,6 +10,7 @@
  *  pace of the LED.
  */
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <mach/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-sa1100/leds-simpad.c b/arch/arm/mach-sa1100/leds-simpad.c
index d50f4ee..2c154e3 100644
--- a/arch/arm/mach-sa1100/leds-simpad.c
+++ b/arch/arm/mach-sa1100/leds-simpad.c
@@ -4,6 +4,7 @@
  * Author: Juergen Messerer <juergen.messerer@siemens.ch>
  */
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <mach/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-sa1100/time.c b/arch/arm/mach-sa1100/time.c
index ae4f3d8..fa700e2 100644
--- a/arch/arm/mach-sa1100/time.c
+++ b/arch/arm/mach-sa1100/time.c
@@ -15,6 +15,7 @@
 #include <linux/sched.h>	/* just for sched_clock() - funny that */
 #include <linux/timex.h>
 #include <linux/clockchips.h>
+#include <linux/ipipe.h>
 
 #include <asm/mach/time.h>
 #include <asm/sched_clock.h>
@@ -47,13 +48,40 @@ static void notrace sa1100_update_sched_clock(void)
 
 #define MIN_OSCR_DELTA 2
 
+#ifdef CONFIG_IPIPE
+int __ipipe_mach_timerint = IRQ_OST0;
+EXPORT_SYMBOL(__ipipe_mach_timerint);
+
+int __ipipe_mach_timerstolen = 0;
+EXPORT_SYMBOL(__ipipe_mach_timerstolen);
+
+unsigned int __ipipe_mach_ticks_per_jiffy = LATCH;
+EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy);
+
+static struct __ipipe_tscinfo tsc_info = {
+	.type = IPIPE_TSC_TYPE_FREERUNNING,
+	.freq = CLOCK_TICK_RATE,
+	.counter_vaddr = io_p2v(0x90000010UL),
+	.u = {
+		{
+			.counter_paddr = 0x90000010UL,
+			.mask = 0xffffffff,
+		},
+	},
+};
+#endif /* CONFIG_IPIPE */
+
 static irqreturn_t sa1100_ost0_interrupt(int irq, void *dev_id)
 {
 	struct clock_event_device *c = dev_id;
 
 	/* Disarm the compare/match, signal the event. */
+#ifndef CONFIG_IPIPE
 	OIER &= ~OIER_E0;
 	OSSR = OSSR_M0;
+#else /* CONFIG_IPIPE */
+	__ipipe_tsc_update();
+#endif /* CONFIG_IPIPE */
 	c->event_handler(c);
 
 	return IRQ_HANDLED;
@@ -98,7 +126,14 @@ static struct clock_event_device ckevt_sa1100_osmr0 = {
 	.set_mode	= sa1100_osmr0_set_mode,
 };
 
-static cycle_t sa1100_read_oscr(struct clocksource *s)
+#ifdef CONFIG_IPIPE
+int __ipipe_check_tickdev(const char *devname)
+{
+	return !strcmp(devname, ckevt_sa1100_osmr0.name);
+}
+#endif /* CONFIG_IPIPE */
+
+static cycle_t sa1100_read_oscr(struct clocksource *cs)
 {
 	return OSCR;
 }
@@ -136,6 +171,10 @@ static void __init sa1100_timer_init(void)
 
 	setup_irq(IRQ_OST0, &sa1100_timer_irq);
 
+#ifdef CONFIG_IPIPE
+	__ipipe_tsc_register(&tsc_info);
+#endif /* CONFIG_IPIPE */
+
 	clocksource_register_hz(&cksrc_sa1100_oscr, CLOCK_TICK_RATE);
 	clockevents_register_device(&ckevt_sa1100_osmr0);
 }
@@ -176,3 +215,38 @@ struct sys_timer sa1100_timer = {
 	.suspend	= sa1100_timer_suspend,
 	.resume		= sa1100_timer_resume,
 };
+
+#ifdef CONFIG_IPIPE
+void __ipipe_mach_acktimer(void)
+{
+	OSSR = OSSR_M0;  /* Clear match on timer 0 */
+	OIER &= ~OIER_E0;
+}
+
+/*
+ * Reprogram the timer
+ */
+
+void __ipipe_mach_set_dec(unsigned long delay)
+{
+	if (delay > MIN_OSCR_DELTA) {
+		OSMR0 = delay + OSCR;
+		OIER |= OIER_E0;
+	} else
+		ipipe_trigger_irq(IRQ_OST0);
+}
+EXPORT_SYMBOL(__ipipe_mach_set_dec);
+
+void __ipipe_mach_release_timer(void)
+{
+	sa1100_osmr0_set_mode(ckevt_sa1100_osmr0.mode, &ckevt_sa1100_osmr0);
+	if (ckevt_sa1100_osmr0.mode == CLOCK_EVT_MODE_ONESHOT)
+		sa1100_osmr0_set_next_event(LATCH, &ckevt_sa1100_osmr0);
+}
+EXPORT_SYMBOL(__ipipe_mach_release_timer);
+
+unsigned long __ipipe_mach_get_dec(void)
+{
+	return OSMR0 - OSCR;
+}
+#endif /* CONFIG_IPIPE */
diff --git a/arch/arm/mach-versatile/core.c b/arch/arm/mach-versatile/core.c
index 136c32e..d0f9f62 100644
--- a/arch/arm/mach-versatile/core.c
+++ b/arch/arm/mach-versatile/core.c
@@ -911,14 +911,14 @@ static void __init versatile_timer_init(void)
 
 	versatile_sched_clock_init(REFCOUNTER, 24000000);
 
-	/* 
-	 * set clock frequency: 
+	/*
+	 * set clock frequency:
 	 *	VERSATILE_REFCLK is 32KHz
 	 *	VERSATILE_TIMCLK is 1MHz
 	 */
 	val = readl(__io_address(VERSATILE_SCTL_BASE));
 	writel((VERSATILE_TIMCLK << VERSATILE_TIMER1_EnSel) |
-	       (VERSATILE_TIMCLK << VERSATILE_TIMER2_EnSel) | 
+	       (VERSATILE_TIMCLK << VERSATILE_TIMER2_EnSel) |
 	       (VERSATILE_TIMCLK << VERSATILE_TIMER3_EnSel) |
 	       (VERSATILE_TIMCLK << VERSATILE_TIMER4_EnSel) | val,
 	       __io_address(VERSATILE_SCTL_BASE));
@@ -931,11 +931,10 @@ static void __init versatile_timer_init(void)
 	writel(0, TIMER2_VA_BASE + TIMER_CTRL);
 	writel(0, TIMER3_VA_BASE + TIMER_CTRL);
 
-	sp804_clocksource_init(TIMER3_VA_BASE);
+	sp804_clocksource_init(TIMER3_VA_BASE, VERSATILE_TIMER2_3_BASE + 0x20);
 	sp804_clockevents_init(TIMER0_VA_BASE, IRQ_TIMERINT0_1);
 }
 
 struct sys_timer versatile_timer = {
 	.init		= versatile_timer_init,
 };
-
diff --git a/arch/arm/mach-vexpress/ct-ca9x4.c b/arch/arm/mach-vexpress/ct-ca9x4.c
index e628402..612a7a4 100644
--- a/arch/arm/mach-vexpress/ct-ca9x4.c
+++ b/arch/arm/mach-vexpress/ct-ca9x4.c
@@ -71,7 +71,7 @@ static void __init ct_ca9x4_timer_init(void)
 	writel(0, MMIO_P2V(CT_CA9X4_TIMER0) + TIMER_CTRL);
 	writel(0, MMIO_P2V(CT_CA9X4_TIMER1) + TIMER_CTRL);
 
-	sp804_clocksource_init(MMIO_P2V(CT_CA9X4_TIMER1));
+	sp804_clocksource_init(MMIO_P2V(CT_CA9X4_TIMER1), CT_CA9X4_TIMER1);
 	sp804_clockevents_init(MMIO_P2V(CT_CA9X4_TIMER0), IRQ_CT_CA9X4_TIMER0);
 }
 
diff --git a/arch/arm/mach-vexpress/v2m.c b/arch/arm/mach-vexpress/v2m.c
index 1edae65..2d8cdf9 100644
--- a/arch/arm/mach-vexpress/v2m.c
+++ b/arch/arm/mach-vexpress/v2m.c
@@ -64,7 +64,7 @@ static void __init v2m_timer_init(void)
 	writel(0, MMIO_P2V(V2M_TIMER0) + TIMER_CTRL);
 	writel(0, MMIO_P2V(V2M_TIMER1) + TIMER_CTRL);
 
-	sp804_clocksource_init(MMIO_P2V(V2M_TIMER1));
+	sp804_clocksource_init(MMIO_P2V(V2M_TIMER1), V2M_TIMER1);
 	sp804_clockevents_init(MMIO_P2V(V2M_TIMER0), IRQ_V2M_TIMER0);
 }
 
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index e4509ba..9d079d2 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -608,7 +608,7 @@ config CPU_CP15_MPU
 config CPU_USE_DOMAINS
 	bool
 	depends on MMU
-	default y if !CPU_32v6K
+	default y if !CPU_32v6K || IPIPE
 	help
 	  This option enables or disables the use of domain switching
 	  via the set_fs() function.
@@ -876,3 +876,85 @@ config ARCH_HAS_BARRIERS
 	help
 	  This option allows the use of custom mandatory barriers
 	  included via the mach/barriers.h file.
+
+config ARM_FCSE
+	bool "Fast Context Switch Extension (EXPERIMENTAL)"
+	depends on EXPERIMENTAL && !SMP && (CPU_32v4 || CPU_32v4T || CPU_32v5)
+	help
+	  The Fast Context Switch Extension (FCSE for short) is an extension of
+	  some ARM processors which allows to switch contexts between processes
+	  without flushing cache and saves a few tens of microseconds in the
+	  worst case.
+
+	  Enabling this option makes linux use the FCSE.
+
+	  We propose two modes:
+	  - the guaranteed mode: we guarantee that there will never be any cache
+	    flush when switching context, but this means that there can not be
+	    more than 95 running processes in the system, each with a virtual
+	    memory space smaller than 32MB, and that the shared memory
+	    mappings do not use cache;
+	  - the best effort mode: we allow some cache flushes to happen from
+	    time to time, but do not limit the number of processes or the
+	    virtual memory space available for each process, and the shared
+	    memory mappings use cache.
+
+if ARM_FCSE
+
+choice
+	prompt "FCSE mode"
+	default ARM_FCSE_BEST_EFFORT
+	help
+	  This option allow setting which FCSE mode will be used.
+
+config ARM_FCSE_GUARANTEED
+	bool "guaranteed"
+	help
+	  Select guaranteed mode.
+
+config ARM_FCSE_BEST_EFFORT
+	bool "best effort"
+	help
+	  Select best-effort mode.
+
+endchoice
+
+config ARM_FCSE_DYNPID
+	bool "Dynamic FCSE pid allocation"
+	depends on ARM_FCSE_BEST_EFFORT && !IPIPE
+	help
+	  When this option is enabled, the kernel may decide to change the FCSE
+	  pid of a process when switching context. This helps reducing the
+	  number of cache flushes, but adds some overhead to the context
+	  switches.
+
+config ARM_FCSE_PREEMPT_FLUSH
+	bool "Preemptible cache flushes"
+	default ARM_FCSE_GUARANTEED
+	help
+	  When FCSE is enabled, some cache flushes happen with preemption
+	  disabled by default, this allows avoiding more cache
+	  flushes, but increases the latency. This option allows making
+	  them preemptible. It probably only make sense in guaranteed mode.
+
+config ARM_FCSE_MESSAGES
+	bool "help messages"
+	default ARM_FCSE_BEST_EFFORT
+	help
+	  When FCSE is enabled in best-effort mode, due to the VM space
+	  reduction, a too large stack size limit may result in processes
+	  exceeding the 32MB limit too easily. A too small stack size may result
+	  in stack overflows. Enabling this option will print messages in these
+	  situations to assist you in tuning the stack size limit.
+
+	  In guaranteed mode, this option will cause message to be printed if
+	  one of the hard limits (95 proceses, 32 MB VM space) is exceeded.
+
+config ARM_FCSE_DEBUG
+       bool "FCSE debug"
+       select ARM_FCSE_MESSAGES
+       help
+	  This option enables some internal debug checks. It has a high
+	  overhead, and is only useful for debugging the FCSE code.
+
+endif
diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile
index 00d74a0..c7aff03 100644
--- a/arch/arm/mm/Makefile
+++ b/arch/arm/mm/Makefile
@@ -99,3 +99,4 @@ obj-$(CONFIG_CACHE_FEROCEON_L2)	+= cache-feroceon-l2.o
 obj-$(CONFIG_CACHE_L2X0)	+= cache-l2x0.o
 obj-$(CONFIG_CACHE_XSC3L2)	+= cache-xsc3l2.o
 obj-$(CONFIG_CACHE_TAUROS2)	+= cache-tauros2.o
+obj-$(CONFIG_ARM_FCSE)		+= fcse.o
diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c
index 724ba3b..46693d9 100644
--- a/arch/arm/mm/alignment.c
+++ b/arch/arm/mm/alignment.c
@@ -727,6 +727,9 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 	int isize = 4;
 	int thumb2_32b = 0;
 
+	if (ipipe_trap_notify(IPIPE_TRAP_ALIGNMENT,regs))
+		return 0;
+
 	instrptr = instruction_pointer(regs);
 
 	fs = get_fs();
diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c
index f2ce38e..9053b27 100644
--- a/arch/arm/mm/cache-l2x0.c
+++ b/arch/arm/mm/cache-l2x0.c
@@ -19,6 +19,7 @@
 #include <linux/init.h>
 #include <linux/spinlock.h>
 #include <linux/io.h>
+#include <linux/ipipe.h>
 
 #include <asm/cacheflush.h>
 #include <asm/hardware/cache-l2x0.h>
@@ -26,7 +27,7 @@
 #define CACHE_LINE_SIZE		32
 
 static void __iomem *l2x0_base;
-static DEFINE_SPINLOCK(l2x0_lock);
+static IPIPE_DEFINE_SPINLOCK(l2x0_lock);
 static uint32_t l2x0_way_mask;	/* Bitmask of active ways */
 static uint32_t l2x0_size;
 
diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c
index b0ee9ba..a95f4db 100644
--- a/arch/arm/mm/context.c
+++ b/arch/arm/mm/context.c
@@ -16,12 +16,47 @@
 #include <asm/mmu_context.h>
 #include <asm/tlbflush.h>
 
-static DEFINE_SPINLOCK(cpu_asid_lock);
+static IPIPE_DEFINE_SPINLOCK(cpu_asid_lock);
 unsigned int cpu_last_asid = ASID_FIRST_VERSION;
 #ifdef CONFIG_SMP
 DEFINE_PER_CPU(struct mm_struct *, current_mm);
 #endif
 
+#if defined(CONFIG_IPIPE) && defined(CONFIG_SMP)
+/*
+ * We shall be able to serve interrupts while attempting to grab the
+ * ASID lock on entry to __new_context(). This is a prerequisite for
+ * broadcasting VNMIs to other CPUs later on, to have them reset their
+ * current ASID, without risking deadlocks. I.e. each CPU shall be
+ * able to reset the current ASID upon a remote request, while trying
+ * to get a new ASID.
+ */
+#define asid_lock(flags)						\
+	do {								\
+		WARN_ON_ONCE(irqs_disabled_hw());			\
+		while (!spin_trylock_irqsave(&cpu_asid_lock, (flags)))	\
+			cpu_relax();					\
+	} while (0)							\
+
+#define asid_unlock(flags)	\
+	spin_unlock_irqrestore(&cpu_asid_lock, flags)
+
+#define asid_broadcast_reset()	\
+	__ipipe_send_vnmi(reset_context, cpu_online_map, NULL);
+
+#else /* !(CONFIG_IPIPE && CONFIG_SMP) */
+
+#define asid_lock(flags)	\
+	spin_lock_irqsave_cond(&cpu_asid_lock, flags)
+
+#define asid_unlock(flags)	\
+	spin_unlock_irqrestore_cond(&cpu_asid_lock, flags)
+
+#define asid_broadcast_reset()	\
+	smp_call_function(reset_context, NULL, 1);
+
+#endif /* !(CONFIG_IPIPE && CONFIG_SMP) */
+
 /*
  * We fork()ed a process, and we need a new context for the child
  * to run in.  We reserve version 0 for initial tasks so we will
@@ -72,7 +107,7 @@ static void set_mm_context(struct mm_struct *mm, unsigned int asid)
 	/*
 	 * Set the mm_cpumask(mm) bit for the current CPU.
 	 */
-	cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
+	cpumask_set_cpu(ipipe_processor_id(), mm_cpumask(mm));
 }
 
 /*
@@ -82,7 +117,7 @@ static void set_mm_context(struct mm_struct *mm, unsigned int asid)
 static void reset_context(void *info)
 {
 	unsigned int asid;
-	unsigned int cpu = smp_processor_id();
+	unsigned int cpu = ipipe_processor_id();
 	struct mm_struct *mm = per_cpu(current_mm, cpu);
 
 	/*
@@ -108,24 +143,26 @@ static void reset_context(void *info)
 static inline void set_mm_context(struct mm_struct *mm, unsigned int asid)
 {
 	mm->context.id = asid;
-	cpumask_copy(mm_cpumask(mm), cpumask_of(smp_processor_id()));
+	cpumask_copy(mm_cpumask(mm), cpumask_of(ipipe_processor_id()));
 }
 
 #endif
 
 void __new_context(struct mm_struct *mm)
 {
+	int cpu = ipipe_processor_id();
+	unsigned long flags;
 	unsigned int asid;
 
-	spin_lock(&cpu_asid_lock);
+	asid_lock(flags);
 #ifdef CONFIG_SMP
 	/*
 	 * Check the ASID again, in case the change was broadcast from
 	 * another CPU before we acquired the lock.
 	 */
 	if (unlikely(((mm->context.id ^ cpu_last_asid) >> ASID_BITS) == 0)) {
-		cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
-		spin_unlock(&cpu_asid_lock);
+		cpumask_set_cpu(cpu, mm_cpumask(mm));
+		asid_unlock(flags);
 		return;
 	}
 #endif
@@ -143,15 +180,15 @@ void __new_context(struct mm_struct *mm)
 	 * to start a new version and flush the TLB.
 	 */
 	if (unlikely((asid & ~ASID_MASK) == 0)) {
-		asid = cpu_last_asid + smp_processor_id() + 1;
+		asid = cpu_last_asid + cpu + 1;
 		flush_context();
 #ifdef CONFIG_SMP
 		smp_wmb();
-		smp_call_function(reset_context, NULL, 1);
+		asid_broadcast_reset();
 #endif
 		cpu_last_asid += NR_CPUS;
 	}
 
 	set_mm_context(mm, asid);
-	spin_unlock(&cpu_asid_lock);
+	asid_unlock(flags);
 }
diff --git a/arch/arm/mm/copypage-v4mc.c b/arch/arm/mm/copypage-v4mc.c
index b806151..f7f06c2 100644
--- a/arch/arm/mm/copypage-v4mc.c
+++ b/arch/arm/mm/copypage-v4mc.c
@@ -44,7 +44,7 @@ static DEFINE_SPINLOCK(minicache_lock);
  * instruction.  If your processor does not supply this, you have to write your
  * own copy_user_highpage that does the right thing.
  */
-static void __naked
+static void notrace __naked
 mc_copy_user_page(void *from, void *to)
 {
 	asm volatile(
diff --git a/arch/arm/mm/copypage-xscale.c b/arch/arm/mm/copypage-xscale.c
index 649bbcd..9400f0f 100644
--- a/arch/arm/mm/copypage-xscale.c
+++ b/arch/arm/mm/copypage-xscale.c
@@ -42,7 +42,7 @@ static DEFINE_SPINLOCK(minicache_lock);
  * Dcache aliasing issue.  The writes will be forwarded to the write buffer,
  * and merged as appropriate.
  */
-static void __naked
+static void notrace __naked
 mc_copy_user_page(void *from, void *to)
 {
 	/*
diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c
index 01210db..03dc056 100644
--- a/arch/arm/mm/fault-armv.c
+++ b/arch/arm/mm/fault-armv.c
@@ -29,6 +29,30 @@
 static pteval_t shared_pte_mask = L_PTE_MT_BUFFERABLE;
 
 #if __LINUX_ARM_ARCH__ < 6
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+static void fcse_set_pte_shared(struct vm_area_struct *vma,
+				unsigned long address, pte_t *ptep)
+{
+	pte_t entry;
+
+	if (!(vma->vm_flags & VM_MAYSHARE) || address >= TASK_SIZE)
+		return;
+
+	entry = *ptep;
+	if ((pte_val(entry)
+	     & (L_PTE_PRESENT | PTE_CACHEABLE | L_PTE_RDONLY | L_PTE_DIRTY | L_PTE_SHARED))
+	    == (L_PTE_PRESENT | PTE_CACHEABLE | L_PTE_DIRTY)) {
+		pte_val(entry) |= L_PTE_SHARED;
+		/* Bypass set_pte_at here, we are not changing
+		   hardware bits, flush is not needed */
+		++vma->vm_mm->context.fcse.shared_dirty_pages;
+		*ptep = entry;
+	}
+}
+#else /* !CONFIG_ARM_FCSE_BEST_EFFORT */
+#define fcse_set_pte_shared(vma, addr, ptep) do { } while (0)
+#endif /* !CONFIG_ARM_FCSE_BEST_EFFORT */
+
 /*
  * We take the easy way out of this problem - we make the
  * PTE uncacheable.  However, we leave the write buffer on.
@@ -66,6 +90,7 @@ static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address,
 	return ret;
 }
 
+#ifndef CONFIG_ARM_FCSE_GUARANTEED
 #if USE_SPLIT_PTLOCKS
 /*
  * If we are using split PTE locks, then we need to take the page
@@ -123,11 +148,13 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address,
 
 	return ret;
 }
+#endif /* CONFIG_ARM_FCSE_GUARANTEED */
 
 static void
 make_coherent(struct address_space *mapping, struct vm_area_struct *vma,
 	unsigned long addr, pte_t *ptep, unsigned long pfn)
 {
+#ifndef CONFIG_ARM_FCSE_GUARANTEED
 	struct mm_struct *mm = vma->vm_mm;
 	struct vm_area_struct *mpnt;
 	struct prio_tree_iter iter;
@@ -159,6 +186,12 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma,
 	flush_dcache_mmap_unlock(mapping);
 	if (aliases)
 		do_adjust_pte(vma, addr, pfn, ptep);
+	else
+		fcse_set_pte_shared(vma, addr, ptep);
+#else /* CONFIG_ARM_FCSE_GUARANTEED */
+	if (vma->vm_flags & VM_MAYSHARE)
+		do_adjust_pte(vma, addr, pfn, ptep);
+#endif /* CONFIG_ARM_FCSE_GUARANTEED */
 }
 
 /*
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index f10f9ba..1fa450d 100644
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -74,6 +74,10 @@ void show_pte(struct mm_struct *mm, unsigned long addr)
 	if (!mm)
 		mm = &init_mm;
 
+#ifdef CONFIG_ARM_FCSE
+	printk(KERN_ALERT "fcse pid: %ld, 0x%08lx\n",
+	       mm->context.fcse.pid >> FCSE_PID_SHIFT, mm->context.fcse.pid);
+#endif /* CONFIG_ARM_FCSE */
 	printk(KERN_ALERT "pgd = %p\n", mm->pgd);
 	pgd = pgd_offset(mm, addr);
 	printk(KERN_ALERT "[%08lx] *pgd=%08lx", addr, pgd_val(*pgd));
@@ -162,11 +166,21 @@ __do_user_fault(struct task_struct *tsk, unsigned long addr,
 	if (user_debug & UDBG_SEGV) {
 		printk(KERN_DEBUG "%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n",
 		       tsk->comm, sig, addr, fsr);
+#ifdef CONFIG_ARM_FCSE_DYNPID
+		/* Disable preemption to avoid page tables changing under our
+		   feet */
+		preempt_disable();
+#endif /* CONFIG_ARM_FCSE_DYNPID */
 		show_pte(tsk->mm, addr);
+#ifdef CONFIG_ARM_FCSE_DYNPID
+		preempt_enable();
+#endif /* CONFIG_ARM_FCSE_DYNPID */
 		show_regs(regs);
 	}
 #endif
 
+	fcse_notify_segv(tsk->mm, addr, regs);
+
 	tsk->thread.address = addr;
 	tsk->thread.error_code = fsr;
 	tsk->thread.trap_no = 14;
@@ -267,6 +281,9 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 	if (notify_page_fault(regs, fsr))
 		return 0;
 
+	if (ipipe_trap_notify(IPIPE_TRAP_ACCESS,regs))
+		return 0;
+
 	tsk = current;
 	mm  = tsk->mm;
 
@@ -393,6 +410,9 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
 	if (addr < TASK_SIZE)
 		return do_page_fault(addr, fsr, regs);
 
+	if (ipipe_trap_notify(IPIPE_TRAP_ACCESS,regs))
+		return 0;
+
 	if (user_mode(regs))
 		goto bad_area;
 
@@ -448,6 +468,10 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
 static int
 do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 {
+
+	if (ipipe_trap_notify(IPIPE_TRAP_SECTION,regs))
+		return 0;
+
 	do_bad_area(addr, fsr, regs);
 	return 0;
 }
@@ -458,6 +482,9 @@ do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 static int
 do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 {
+	if (ipipe_trap_notify(IPIPE_TRAP_DABT,regs))
+		return 0;
+
 	return 1;
 }
 
@@ -535,6 +562,9 @@ do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 	if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs))
 		return;
 
+	if (ipipe_trap_notify(IPIPE_TRAP_UNKNOWN,regs))
+		return;
+
 	printk(KERN_ALERT "Unhandled fault: %s (0x%03x) at 0x%08lx\n",
 		inf->name, fsr, addr);
 
@@ -635,3 +665,43 @@ static int __init exceptions_init(void)
 }
 
 arch_initcall(exceptions_init);
+
+#ifdef CONFIG_IPIPE
+extern spinlock_t pgd_lock;
+extern struct page *pgd_list;
+
+static void vmalloc_sync_one(pgd_t *pgd, unsigned long addr)
+{
+	unsigned int index = pgd_index(addr);
+	pgd_t *pgd_k;
+	pmd_t *pmd, *pmd_k;
+
+	pgd += index;
+	pgd_k = init_mm.pgd + index;
+
+	if (!pgd_present(*pgd))
+		set_pgd(pgd, *pgd_k);
+
+	pmd_k = pmd_offset(pgd_k, addr);
+	pmd   = pmd_offset(pgd, addr);
+
+	copy_pmd(pmd, pmd_k);
+}
+
+void __ipipe_pin_range_globally(unsigned long start, unsigned long end)
+{
+	unsigned long next, addr = start;
+
+	do {
+		unsigned long flags;
+		struct page *page;
+
+		next = pgd_addr_end(addr, end);
+		spin_lock_irqsave(&pgd_lock, flags);
+		for (page = pgd_list; page; page = (struct page *)page->index)
+			vmalloc_sync_one(page_address(page), addr);
+		spin_unlock_irqrestore(&pgd_lock, flags);
+
+	} while (addr = next, addr != end);
+}
+#endif /* CONFIG_IPIPE */
diff --git a/arch/arm/mm/fcse.c b/arch/arm/mm/fcse.c
new file mode 100644
index 0000000..24e7a67
--- /dev/null
+++ b/arch/arm/mm/fcse.c
@@ -0,0 +1,466 @@
+/*
+ * arch/arm/kernel/fcse.c
+ *
+ * Helper functions for using the ARM Fast Context Switch Extension with
+ * processors supporting it.
+ *
+ * Copyright (C) 2008 Richard Cochran
+ * Copyright (C) 2009-2011 Gilles Chanteperdrix <gch@xenomai.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/bitops.h>
+#include <linux/memory.h>
+#include <linux/spinlock.h>
+#include <linux/mm.h>
+#include <linux/kernel_stat.h>
+#include <linux/mman.h>
+#include <linux/dcache.h>
+#include <linux/fs.h>
+#include <linux/hardirq.h>
+
+#include <asm/fcse.h>
+#include <asm/cacheflush.h>
+#include <asm/tlbflush.h>
+
+#define NR_PIDS (TASK_SIZE / FCSE_PID_TASK_SIZE)
+#define PIDS_LONGS ((NR_PIDS + BITS_PER_LONG - 1) / BITS_PER_LONG)
+
+static IPIPE_DEFINE_SPINLOCK(fcse_lock);
+static unsigned long fcse_pids_bits[PIDS_LONGS];
+unsigned long fcse_pids_cache_dirty[PIDS_LONGS];
+EXPORT_SYMBOL(fcse_pids_cache_dirty);
+
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+static unsigned random_pid;
+struct fcse_user fcse_pids_user[NR_PIDS];
+#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */
+
+static inline void fcse_pid_reference_inner(unsigned pid)
+{
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+	if (++fcse_pids_user[pid].count == 1)
+#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */
+		__set_bit(pid, fcse_pids_bits);
+}
+
+static inline void fcse_pid_dereference(struct mm_struct *mm)
+{
+	unsigned fcse_pid = mm->context.fcse.pid >> FCSE_PID_SHIFT;
+
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+	if (--fcse_pids_user[fcse_pid].count == 0)
+		__clear_bit(fcse_pid, fcse_pids_bits);
+
+	/*
+	 * The following means we suppose that by the time this
+	 * function is called, this mm is out of cache:
+	 * - when the caller is destroy_context, exit_mmap is called
+	 * by mmput before, which flushes the cache;
+	 * - when the caller is fcse_relocate_mm_to_pid from
+	 * fcse_switch_mm_inner, we only relocate when the mm is out
+	 * of cache;
+	 * - when the caller is fcse_relocate_mm_to_pid from
+	 * fcse_relocate_mm_to_null_pid, we flush the cache in this
+	 * function.
+	 */
+	if (fcse_pids_user[fcse_pid].mm == mm) {
+		fcse_pids_user[fcse_pid].mm = NULL;
+		__clear_bit(fcse_pid, fcse_pids_cache_dirty);
+	}
+#else /* CONFIG_ARM_FCSE_BEST_EFFORT */
+	__clear_bit(fcse_pid, fcse_pids_bits);
+	__clear_bit(fcse_pid, fcse_pids_cache_dirty);
+#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */
+}
+
+static inline unsigned find_free_pid(unsigned long bits[])
+{
+	unsigned fcse_pid;
+
+	fcse_pid = find_next_zero_bit(bits, NR_PIDS, 1);
+	if (fcse_pid == NR_PIDS)
+		/* Allocate zero pid last, since zero pid is also used by
+		   processes with address space larger than 32MB in
+		   best-effort mode. */
+		if (!test_bit(0, bits))
+			fcse_pid = 0;
+
+	return fcse_pid;
+}
+
+void fcse_pid_free(struct mm_struct *mm)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&fcse_lock, flags);
+	fcse_pid_dereference(mm);
+	spin_unlock_irqrestore(&fcse_lock, flags);
+}
+
+int fcse_pid_alloc(struct mm_struct *mm)
+{
+	unsigned long flags;
+	unsigned fcse_pid;
+
+	spin_lock_irqsave(&fcse_lock, flags);
+	fcse_pid = find_free_pid(fcse_pids_bits);
+	if (fcse_pid == NR_PIDS) {
+		/* Allocate zero pid last, since zero pid is also used by
+		   processes with address space larger than 32MB in
+		   best-effort mode. */
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+		if(++random_pid == NR_PIDS)
+			random_pid = 0;
+		fcse_pid = random_pid;
+#else /* CONFIG_ARM_FCSE_GUARANTEED */
+		spin_unlock_irqrestore(&fcse_lock, flags);
+#ifdef CONFIG_ARM_FCSE_MESSAGES
+		printk(KERN_WARNING "FCSE: %s[%d] would exceed the %lu processes limit.\n",
+		       current->comm, current->pid, NR_PIDS);
+#endif /* CONFIG_ARM_FCSE_MESSAGES */
+		return -EAGAIN;
+#endif /* CONFIG_ARM_FCSE_GUARANTEED */
+	}
+	fcse_pid_reference_inner(fcse_pid);
+	spin_unlock_irqrestore(&fcse_lock, flags);
+
+	return fcse_pid;
+}
+
+static inline void fcse_clear_dirty_all(void)
+{
+	switch(ARRAY_SIZE(fcse_pids_cache_dirty)) {
+	case 4:
+		fcse_pids_cache_dirty[3] = 0UL;
+	case 3:
+		fcse_pids_cache_dirty[2] = 0UL;
+	case 2:
+		fcse_pids_cache_dirty[1] = 0UL;
+	case 1:
+		fcse_pids_cache_dirty[0] = 0UL;
+	}
+}
+
+unsigned fcse_flush_all_start(void)
+{
+	if (!cache_is_vivt())
+		return 0;
+
+#ifndef CONFIG_ARM_FCSE_PREEMPT_FLUSH
+	preempt_disable();
+#endif /* CONFIG_ARM_FCSE_PREEMPT_FLUSH */
+
+#if defined(CONFIG_IPIPE)
+	clear_ti_thread_flag(current_thread_info(), TIF_SWITCHED);
+#elif defined(CONFIG_ARM_FCSE_PREEMPT_FLUSH)
+	return nr_context_switches();
+#endif /* CONFIG_ARM_FCSE_PREEMPT_FLUSH */
+
+	return 0;
+}
+
+noinline void
+fcse_flush_all_done(unsigned seq, unsigned dirty)
+{
+	unsigned long flags;
+
+	if (!cache_is_vivt())
+		return;
+
+	spin_lock_irqsave(&fcse_lock, flags);
+#if defined(CONFIG_IPIPE)
+	if (!test_ti_thread_flag(current_thread_info(), TIF_SWITCHED))
+#elif defined(CONFIG_ARM_FCSE_PREEMPT_FLUSH)
+	if (seq == nr_context_switches())
+#endif /* CONFIG_ARM_FCSE_PREEMPT_FLUSH */
+		fcse_clear_dirty_all();
+
+	if (dirty && current->mm != &init_mm && current->mm) {
+		unsigned fcse_pid =
+			current->mm->context.fcse.pid >> FCSE_PID_SHIFT;
+		__set_bit(fcse_pid, fcse_pids_cache_dirty);
+	}
+	spin_unlock_irqrestore(&fcse_lock, flags);
+#ifndef CONFIG_ARM_FCSE_PREEMPT_FLUSH
+	preempt_enable();
+#endif /* CONFIG_ARM_FCSE_PREEMPT_FLUSH */
+}
+
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+/* Called with preemption disabled, mm->mmap_sem being held for writing. */
+static noinline int fcse_relocate_mm_to_pid(struct mm_struct *mm, int fcse_pid)
+{
+	const unsigned len = pgd_index(FCSE_TASK_SIZE) * sizeof(pgd_t);
+	unsigned long flags;
+	pgd_t *from, *to;
+
+	spin_lock_irqsave(&fcse_lock, flags);
+#if defined(CONFIG_ARM_FCSE_DYNPID)
+	/* pid == -1 means find a free pid. */
+	if (fcse_pid == -1) {
+		fcse_pid = find_free_pid(fcse_pids_bits);
+		if (fcse_pid == NR_PIDS) {
+			fcse_pid = find_free_pid(fcse_pids_cache_dirty);
+			if (unlikely(fcse_pid == NR_PIDS)) {
+				spin_unlock_irqrestore(&fcse_lock, flags);
+				return -ENOENT;
+			}
+		}
+	}
+#endif /* CONFIG_ARM_FCSE_DYNPID */
+	fcse_pid_dereference(mm);
+	fcse_pid_reference_inner(fcse_pid);
+	fcse_pids_user[fcse_pid].mm = mm;
+	__set_bit(fcse_pid, fcse_pids_cache_dirty);
+	spin_unlock_irqrestore(&fcse_lock, flags);
+
+	from = pgd_offset(mm, 0);
+	mm->context.fcse.pid = fcse_pid << FCSE_PID_SHIFT;
+	to = pgd_offset(mm, 0);
+
+	memcpy(to, from, len);
+	memset(from, '\0', len);
+	barrier();
+	clean_dcache_area(from, len);
+	clean_dcache_area(to, len);
+
+	return fcse_pid;
+}
+
+int fcse_switch_mm_inner(struct mm_struct *prev, struct mm_struct *next)
+{
+	unsigned fcse_pid = next->context.fcse.pid >> FCSE_PID_SHIFT;
+	unsigned flush_needed, reused_pid = 0;
+	unsigned long flags;
+
+	if (unlikely(next == &init_mm)) {
+		spin_lock_irqsave(&fcse_lock, flags);
+		goto is_flush_needed;
+	}
+
+#ifdef CONFIG_ARM_FCSE_DYNPID
+	/*
+	 * If the next mm's pid is currently in use, and not by that
+	 * mm, try and find a new, free, pid.
+	 */
+	if (unlikely(fcse_pids_user[fcse_pid].mm != next)
+	    && test_bit(fcse_pid, fcse_pids_cache_dirty)
+	    && fcse_pids_user[fcse_pid].mm
+	    && !rwsem_is_locked(&next->mmap_sem)
+	    && !next->context.fcse.large
+	    && !next->core_state) {
+		int new_fcse_pid = fcse_relocate_mm_to_pid(next, -1);
+		if (new_fcse_pid >= 0)
+			fcse_pid = new_fcse_pid;
+	}
+#endif /* CONFIG_ARM_FCSE_DYNPID */
+
+	spin_lock_irqsave(&fcse_lock, flags);
+	if (fcse_pids_user[fcse_pid].mm != next) {
+		if (fcse_pids_user[fcse_pid].mm)
+			reused_pid = test_bit(fcse_pid, fcse_pids_cache_dirty);
+		fcse_pids_user[fcse_pid].mm = next;
+	}
+
+  is_flush_needed:
+	flush_needed = reused_pid
+		|| next->context.fcse.high_pages
+		|| !prev
+		|| prev->context.fcse.shared_dirty_pages
+		|| prev->context.fcse.high_pages;
+
+	fcse_pid_set(fcse_pid << FCSE_PID_SHIFT);
+	if (flush_needed)
+		fcse_clear_dirty_all();
+	if (next != &init_mm)
+		__set_bit(fcse_pid, fcse_pids_cache_dirty);
+	spin_unlock_irqrestore(&fcse_lock, flags);
+
+	return flush_needed;
+}
+EXPORT_SYMBOL_GPL(fcse_switch_mm_inner);
+
+void fcse_pid_reference(unsigned fcse_pid)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&fcse_lock, flags);
+	fcse_pid_reference_inner(fcse_pid);
+	spin_unlock_irqrestore(&fcse_lock, flags);
+}
+
+/* Called with mm->mmap_sem write-locked. */
+static noinline void fcse_relocate_mm_to_null_pid(struct mm_struct *mm)
+{
+	if (!cache_is_vivt())
+		return;
+
+	preempt_disable();
+	while (fcse_mm_in_cache(mm)) {
+		unsigned seq;
+
+		preempt_enable();
+
+		seq = fcse_flush_all_start();
+		flush_cache_all();
+
+		preempt_disable();
+		fcse_flush_all_done(seq, 0);
+	}
+
+	fcse_relocate_mm_to_pid(mm, 0);
+	barrier();
+	flush_tlb_mm(mm);
+	fcse_pid_set(0);
+
+	preempt_enable();
+}
+#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */
+
+unsigned long
+fcse_check_mmap_inner(struct mm_struct *mm, unsigned long start_addr,
+		      unsigned long addr, unsigned long len, unsigned long fl)
+{
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+	unsigned long stack_reserved =
+		current->signal->rlim[RLIMIT_STACK].rlim_cur;
+	unsigned long stack_base = PAGE_ALIGN(mm->start_stack) - stack_reserved;
+
+	/* We enfore the RLIMIT_STACK stack size, and here, the return
+	   address would fall in that reserved stack area */
+	if ((unsigned long)(addr + len - stack_base) < stack_reserved) {
+		/* Restart to try and find a hole, once. */
+		if (start_addr != TASK_UNMAPPED_BASE && !(fl & MAP_FIXED)
+		    && !mm->context.fcse.large)
+			return TASK_UNMAPPED_BASE;
+
+		/* Forcibly restart from above the stack */
+		if (!(fl & MAP_FIXED))
+			return PAGE_ALIGN(mm->start_stack);
+
+		/* If MAP_FIXED is set, we encroach upon the reserved
+		   stack area. No choice. */
+	}
+
+	/* Address above 32MB */
+	if (addr + len > FCSE_TASK_SIZE && !mm->context.fcse.high_pages) {
+		/* Restart to try and find a hole, once. */
+		if (start_addr != TASK_UNMAPPED_BASE && !(fl & MAP_FIXED))
+			return TASK_UNMAPPED_BASE;
+
+		if (!mm->context.fcse.large) {
+			/* Ok, the process is going to be larger than 32MB */
+#ifdef CONFIG_ARM_FCSE_MSSAGES
+			printk(KERN_INFO "FCSE: process %u(%s) VM exceeds 32MB.\n",
+			       current->pid, current->comm);
+#endif /* CONFIG_ARM_FCSE_MESSAGES */
+			mm->context.fcse.large = 1;
+		}
+		if (mm->context.fcse.pid)
+			fcse_relocate_mm_to_null_pid(mm);
+	}
+
+	return addr;
+
+#else /* CONFIG_ARM_FCSE_GUARANTEED */
+	/* Address above 32MB */
+	/* Restart to try and find a hole, once. */
+	if (start_addr != TASK_UNMAPPED_BASE && !(fl & MAP_FIXED))
+		return TASK_UNMAPPED_BASE;
+
+	/* Fail, no 32MB processes in guaranteed mode. */
+#ifdef CONFIG_ARM_FCSE_MESSAGES
+	printk(KERN_WARNING "FCSE: process %u(%s) VM would exceed the 32MB limit.\n",
+	       current->pid, current->comm);
+#endif /* CONFIG_ARM_FCSE_MESSAGES */
+	return -ENOMEM;
+#endif /* CONFIG_ARM_FCSE_GUARANTEED */
+}
+
+#ifdef CONFIG_ARM_FCSE_MESSAGES
+#define addr_in_vma(vma, addr)						\
+	({								\
+		struct vm_area_struct *_vma = (vma);			\
+		((unsigned long)((addr) - _vma->vm_start)		\
+		 < (unsigned long)((_vma->vm_end - _vma->vm_start)));	\
+	})
+
+#ifdef CONFIG_DEBUG_USER
+static noinline void
+dump_vmas(struct mm_struct *mm, unsigned long addr, struct pt_regs *regs)
+{
+	struct vm_area_struct *vma;
+	char path[128];
+	int locked = 0;
+
+	printk("mappings:\n");
+	if (!in_atomic())
+		locked = down_read_trylock(&mm->mmap_sem);
+	for(vma = mm->mmap; vma; vma = vma->vm_next) {
+		struct file *file = vma->vm_file;
+		int flags = vma->vm_flags;
+		const char *name;
+
+		printk("0x%08lx-0x%08lx %c%c%c%c 0x%08llx ",
+		       vma->vm_start,
+		       vma->vm_end,
+		       flags & VM_READ ? 'r' : '-',
+		       flags & VM_WRITE ? 'w' : '-',
+		       flags & VM_EXEC ? 'x' : '-',
+		       flags & VM_MAYSHARE ? 's' : 'p',
+		       ((loff_t)vma->vm_pgoff) << PAGE_SHIFT);
+
+		if (file)
+			name = d_path(&file->f_path, path, sizeof(path));
+		else if ((name = arch_vma_name(vma)))
+			;
+		else if (!vma->vm_mm)
+			name = "[vdso]";
+		else if (vma->vm_start <= mm->start_brk
+			 && vma->vm_end >= mm->brk)
+			name = "[heap]";
+		else if (vma->vm_start <= mm->start_stack &&
+			 vma->vm_end >= mm->start_stack)
+			name = "[stack]";
+		else
+			name = "";
+		printk("%s", name);
+		if (addr_in_vma(vma, regs->ARM_pc))
+			printk(" <- PC");
+		if (addr_in_vma(vma, regs->ARM_sp))
+			printk(" <- SP");
+		if (addr_in_vma(vma, addr))
+			printk("%s fault",
+			       (addr_in_vma(vma, regs->ARM_pc)
+				|| addr_in_vma(vma, regs->ARM_sp)
+				? "," : " <-"));
+		printk("\n");
+	}
+	if (locked)
+		up_read(&mm->mmap_sem);
+}
+#endif /* CONFIG_DEBUG_USER */
+
+void fcse_notify_segv(struct mm_struct *mm,
+		       unsigned long addr, struct pt_regs *regs)
+{
+	int locked = 0;
+
+#if defined(CONFIG_DEBUG_USER)
+	if (user_debug & UDBG_SEGV)
+		dump_vmas(mm, addr, regs);
+#endif /* CONFIG_DEBUG_USER */
+
+	if (!in_atomic())
+		locked = down_read_trylock(&mm->mmap_sem);
+	if (find_vma(mm, addr) == find_vma(mm, regs->ARM_sp))
+		printk(KERN_INFO "FCSE: process %u(%s) probably overflowed stack at 0x%08lx.\n",
+		       current->pid, current->comm, regs->ARM_pc);
+	if (locked)
+		up_read(&mm->mmap_sem);
+}
+#endif /* CONFIG_ARM_FCSE_MESSAGES */
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index 2b269c9..3917d8c 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -118,7 +118,7 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
 			 unsigned long uaddr, void *kaddr, unsigned long len)
 {
 	if (cache_is_vivt()) {
-		if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) {
+		if (fcse_mm_in_cache(vma->vm_mm)) {
 			unsigned long addr = (unsigned long)kaddr;
 			__cpuc_coherent_kern_range(addr, addr + len);
 		}
@@ -158,6 +158,7 @@ void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
 #ifdef CONFIG_SMP
 	preempt_disable();
 #endif
+	fcse_flush_cache_user_range(vma, uaddr, uaddr + len);
 	memcpy(dst, src, len);
 	flush_ptrace_access(vma, page, uaddr, dst, len);
 #ifdef CONFIG_SMP
diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c
index ab50627..ad5a366 100644
--- a/arch/arm/mm/ioremap.c
+++ b/arch/arm/mm/ioremap.c
@@ -242,6 +242,7 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn,
  	}
 
 	flush_cache_vmap(addr, addr + size);
+	__ipipe_pin_range_globally(addr, addr + size);
 	return (void __iomem *) (offset + addr);
 }
 
diff --git a/arch/arm/mm/mmap.c b/arch/arm/mm/mmap.c
index b0a9830..1afcb3e 100644
--- a/arch/arm/mm/mmap.c
+++ b/arch/arm/mm/mmap.c
@@ -30,7 +30,8 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr,
 {
 	struct mm_struct *mm = current->mm;
 	struct vm_area_struct *vma;
-	unsigned long start_addr;
+	unsigned long start_addr = addr;
+
 #ifdef CONFIG_CPU_V6
 	unsigned int cache_type;
 	int do_align = 0, aliasing = 0;
@@ -51,6 +52,9 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr,
 #define aliasing 0
 #endif
 
+#ifdef CONFIG_ARM_FCSE
+	start_addr = addr;
+#endif /* CONFIG_ARM_FCSE */
 	/*
 	 * We enforce the MAP_FIXED case.
 	 */
@@ -58,7 +62,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr,
 		if (aliasing && flags & MAP_SHARED &&
 		    (addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1))
 			return -EINVAL;
-		return addr;
+		goto found_addr;
 	}
 
 	if (len > TASK_SIZE)
@@ -73,13 +77,13 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr,
 		vma = find_vma(mm, addr);
 		if (TASK_SIZE - len >= addr &&
 		    (!vma || addr + len <= vma->vm_start))
-			return addr;
+			goto found_addr;
 	}
 	if (len > mm->cached_hole_size) {
-	        start_addr = addr = mm->free_area_cache;
+		start_addr = addr = mm->free_area_cache;
 	} else {
-	        start_addr = addr = TASK_UNMAPPED_BASE;
-	        mm->cached_hole_size = 0;
+		start_addr = addr = TASK_UNMAPPED_BASE;
+		mm->cached_hole_size = 0;
 	}
 	/* 8 bits of randomness in 20 address space bits */
 	if (current->flags & PF_RANDOMIZE)
@@ -110,14 +114,31 @@ full_search:
 			 * Remember the place where we stopped the search:
 			 */
 			mm->free_area_cache = addr + len;
-			return addr;
+			goto found_addr;
 		}
 		if (addr + mm->cached_hole_size < vma->vm_start)
-		        mm->cached_hole_size = vma->vm_start - addr;
+			mm->cached_hole_size = vma->vm_start - addr;
 		addr = vma->vm_end;
 		if (do_align)
 			addr = COLOUR_ALIGN(addr, pgoff);
 	}
+
+  found_addr:
+#ifdef CONFIG_ARM_FCSE
+	{
+		unsigned long new_addr = fcse_check_mmap_addr(mm, start_addr,
+							      addr, len, flags);
+		if (new_addr != addr) {
+			addr = new_addr;
+			if (!(addr & ~PAGE_MASK)) {
+				start_addr = TASK_UNMAPPED_BASE;
+				mm->cached_hole_size = 0;
+				goto full_search;
+			}
+		}
+	}
+#endif /* CONFIG_ARM_FCSE */
+	return addr;
 }
 
 
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index 3c67e92..7df195d 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -213,7 +213,7 @@ static struct mem_type mem_types[] = {
 		.prot_l1	= PMD_TYPE_TABLE,
 		.prot_sect	= PROT_SECT_DEVICE | PMD_SECT_WB,
 		.domain		= DOMAIN_IO,
-	},	
+	},
 	[MT_DEVICE_WC] = {	/* ioremap_wc */
 		.prot_pte	= PROT_PTE_DEVICE | L_PTE_MT_DEV_WC,
 		.prot_l1	= PMD_TYPE_TABLE,
diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
index 709244c..88350d0 100644
--- a/arch/arm/mm/pgd.c
+++ b/arch/arm/mm/pgd.c
@@ -17,6 +17,43 @@
 
 #include "mm.h"
 
+#define FIRST_KERNEL_PGD_NR	(FIRST_USER_PGD_NR + USER_PTRS_PER_PGD)
+
+#ifdef CONFIG_IPIPE
+/* Copied from arch/i386/mm/pgdtable.c, maintains the list of pgds for the
+   implementation of ipipe_pin_range_globally in arch/arm/mm/fault.c. */
+DEFINE_SPINLOCK(pgd_lock);
+struct page *pgd_list;
+
+#define pgd_list_lock(flags) spin_lock_irqsave(&pgd_lock, flags)
+#define pgd_list_unlock(flags) spin_unlock_irqrestore(&pgd_lock, flags)
+
+static inline void pgd_list_add(pgd_t *pgd)
+{
+	struct page *page = virt_to_page(pgd);
+	page->index = (unsigned long)pgd_list;
+	if (pgd_list)
+		set_page_private(pgd_list, (unsigned long)&page->index);
+	pgd_list = page;
+	set_page_private(page, (unsigned long)&pgd_list);
+}
+
+static inline void pgd_list_del(pgd_t *pgd)
+{
+	struct page *next, **pprev, *page = virt_to_page(pgd);
+	next = (struct page *)page->index;
+	pprev = (struct page **)page_private(page);
+	*pprev = next;
+	if (next)
+		set_page_private(next, (unsigned long)pprev);
+}
+#else /* !CONFIG_IPIPE */
+#define pgd_list_lock(flags) ((void) (flags))
+#define pgd_list_unlock(flags) ((void) (flags))
+#define pgd_list_add(pgd) do { } while (0)
+#define pgd_list_del(pgd) do { } while (0)
+#endif /* !CONFIG_IPIPE */
+
 /*
  * need to get a 16k page for level 1
  */
@@ -25,6 +62,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
 	pgd_t *new_pgd, *init_pgd;
 	pmd_t *new_pmd, *init_pmd;
 	pte_t *new_pte, *init_pte;
+	unsigned long flags;
 
 	new_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, 2);
 	if (!new_pgd)
@@ -36,12 +74,20 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
 	 * Copy over the kernel and IO PGD entries
 	 */
 	init_pgd = pgd_offset_k(0);
+	pgd_list_lock(flags);
 	memcpy(new_pgd + USER_PTRS_PER_PGD, init_pgd + USER_PTRS_PER_PGD,
 		       (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
+	pgd_list_add(new_pgd);
+	pgd_list_unlock(flags);
 
 	clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t));
 
 	if (!vectors_high()) {
+#ifdef CONFIG_ARM_FCSE
+		/* FCSE does not work without high vectors. */
+		BUG();
+#endif /* CONFIG_ARM_FCSE */
+
 		/*
 		 * On ARM, first page must always be allocated since it
 		 * contains the machine vectors.
@@ -74,6 +120,7 @@ no_pgd:
 void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
 {
 	pgd_t *pgd;
+	unsigned long flags;
 	pmd_t *pmd;
 	pgtable_t pte;
 
@@ -84,7 +131,7 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
 	if (pgd_none_or_clear_bad(pgd))
 		goto no_pgd;
 
-	pmd = pmd_offset(pgd, 0);
+	pmd = pmd_offset(pgd + pgd_index(fcse_va_to_mva(mm, 0)), 0);
 	if (pmd_none_or_clear_bad(pmd))
 		goto no_pmd;
 
@@ -95,5 +142,8 @@ no_pmd:
 	pgd_clear(pgd);
 	pmd_free(mm, pmd);
 no_pgd:
+	pgd_list_lock(flags);
+	pgd_list_del(pgd);
+	pgd_list_unlock(flags);
 	free_pages((unsigned long) pgd_base, 2);
 }
diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S
index 6109f27..9b95a2c 100644
--- a/arch/arm/mm/proc-arm920.S
+++ b/arch/arm/mm/proc-arm920.S
@@ -350,6 +350,11 @@ ENTRY(cpu_arm920_dcache_clean_area)
 ENTRY(cpu_arm920_switch_mm)
 #ifdef CONFIG_MMU
 	mov	ip, #0
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+	cmp	r2, #0
+	beq	3f
+#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */
+#ifndef CONFIG_ARM_FCSE_GUARANTEED
 #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH
 	mcr	p15, 0, ip, c7, c6, 0		@ invalidate D cache
 #else
@@ -367,6 +372,10 @@ ENTRY(cpu_arm920_switch_mm)
 #endif
 	mcr	p15, 0, ip, c7, c5, 0		@ invalidate I cache
 	mcr	p15, 0, ip, c7, c10, 4		@ drain WB
+#endif /* !CONFIG_ARM_FCSE_GUARANTEED */
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+3:
+#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */
 	mcr	p15, 0, r0, c2, c0, 0		@ load page table pointer
 	mcr	p15, 0, ip, c8, c7, 0		@ invalidate I & D TLBs
 #endif
diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S
index 42eb431..3f801db 100644
--- a/arch/arm/mm/proc-arm926.S
+++ b/arch/arm/mm/proc-arm926.S
@@ -370,6 +370,11 @@ ENTRY(cpu_arm926_dcache_clean_area)
 ENTRY(cpu_arm926_switch_mm)
 #ifdef CONFIG_MMU
 	mov	ip, #0
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+	cmp	r2, #0
+	beq	2f
+#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */
+#ifndef CONFIG_ARM_FCSE_GUARANTEED
 #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH
 	mcr	p15, 0, ip, c7, c6, 0		@ invalidate D cache
 #else
@@ -379,6 +384,10 @@ ENTRY(cpu_arm926_switch_mm)
 #endif
 	mcr	p15, 0, ip, c7, c5, 0		@ invalidate I cache
 	mcr	p15, 0, ip, c7, c10, 4		@ drain WB
+#endif /* !CONFIG_ARM_FCSE_GUARANTEED */
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+2:
+#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */
 	mcr	p15, 0, r0, c2, c0, 0		@ load page table pointer
 	mcr	p15, 0, ip, c8, c7, 0		@ invalidate I & D TLBs
 #endif
diff --git a/arch/arm/mm/proc-feroceon.S b/arch/arm/mm/proc-feroceon.S
index b4597ed..3e67357 100644
--- a/arch/arm/mm/proc-feroceon.S
+++ b/arch/arm/mm/proc-feroceon.S
@@ -468,6 +468,12 @@ ENTRY(cpu_feroceon_dcache_clean_area)
 	.align	5
 ENTRY(cpu_feroceon_switch_mm)
 #ifdef CONFIG_MMU
+#ifndef CONFIG_ARM_FCSE_GUARANTEED
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+	cmp	r2, #0
+	mov	r2, lr
+	beq	2f
+#else /* !CONFIG_ARM_FCSE */
 	/*
 	 * Note: we wish to call __flush_whole_cache but we need to preserve
 	 * lr to do so.  The only way without touching main memory is to
@@ -475,12 +481,19 @@ ENTRY(cpu_feroceon_switch_mm)
 	 * compensate locally for the skipped ops if it is not set.
 	 */
 	mov	r2, lr				@ abuse r2 to preserve lr
+#endif /* !CONFIG_ARM_FCSE */
 	bl	__flush_whole_cache
 	@ if r2 contains the VM_EXEC bit then the next 2 ops are done already
 	tst	r2, #VM_EXEC
 	mcreq	p15, 0, ip, c7, c5, 0		@ invalidate I cache
 	mcreq	p15, 0, ip, c7, c10, 4		@ drain WB
 
+#ifdef CONFIG_ARM_FCSE
+2:
+#endif
+#else /* CONFIG_ARM_FCSE_GUARANTEED */
+	mov	r2, lr
+#endif /* CONFIG_ARM_FCSE_GUARANTEED */
 	mcr	p15, 0, r0, c2, c0, 0		@ load page table pointer
 	mcr	p15, 0, ip, c8, c7, 0		@ invalidate I & D TLBs
 	mov	pc, r2
diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S
index 5a37c5e..333b5a2 100644
--- a/arch/arm/mm/proc-xscale.S
+++ b/arch/arm/mm/proc-xscale.S
@@ -461,9 +461,18 @@ ENTRY(cpu_xscale_dcache_clean_area)
  */
 	.align	5
 ENTRY(cpu_xscale_switch_mm)
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+	cmp	r2, #0
+	beq	2f
+#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */
+#ifndef CONFIG_ARM_FCSE_GUARANTEED
 	clean_d_cache r1, r2
 	mcr	p15, 0, ip, c7, c5, 0		@ Invalidate I cache & BTB
 	mcr	p15, 0, ip, c7, c10, 4		@ Drain Write (& Fill) Buffer
+#endif /* CONFIG_ARM_FCSE_GUARANTEED */
+#ifdef CONFIG_ARM_FCSE_BEST_EFFORT
+2:
+#endif /* !CONFIG_ARM_FCSE_GUARANTEED */
 	mcr	p15, 0, r0, c2, c0, 0		@ load page table pointer
 	mcr	p15, 0, ip, c8, c7, 0		@ invalidate I & D TLBs
 	cpwait_ret lr, ip
diff --git a/arch/arm/plat-mxc/avic.c b/arch/arm/plat-mxc/avic.c
index deb284b..d90bd0e 100644
--- a/arch/arm/plat-mxc/avic.c
+++ b/arch/arm/plat-mxc/avic.c
@@ -104,6 +104,9 @@ static struct mxc_irq_chip mxc_avic_chip = {
 	.base = {
 		.irq_ack = mxc_mask_irq,
 		.irq_mask = mxc_mask_irq,
+#ifdef CONFIG_IPIPE
+		.irq_mask_ack = mxc_mask_irq,
+#endif
 		.irq_unmask = mxc_unmask_irq,
 	},
 #ifdef CONFIG_MXC_IRQ_PRIOR
@@ -155,4 +158,3 @@ void __init mxc_init_irq(void __iomem *irqbase)
 
 	printk(KERN_INFO "MXC IRQ initialized\n");
 }
-
diff --git a/arch/arm/plat-mxc/cpu.c b/arch/arm/plat-mxc/cpu.c
index 386e0d5..5cafd19 100644
--- a/arch/arm/plat-mxc/cpu.c
+++ b/arch/arm/plat-mxc/cpu.c
@@ -1,5 +1,6 @@
-
 #include <linux/module.h>
+#include <linux/io.h>
+#include <mach/hardware.h>
 
 unsigned int __mxc_cpu_type;
 EXPORT_SYMBOL(__mxc_cpu_type);
@@ -9,3 +10,35 @@ void mxc_set_cpu_type(unsigned int type)
 	__mxc_cpu_type = type;
 }
 
+#ifdef CONFIG_IPIPE
+void ipipe_mach_allow_hwtimer_uaccess(unsigned long aips1, unsigned long aips2)
+{
+	volatile unsigned long aips_reg;
+
+	if (!cpu_is_mx27()) {
+		/*
+		 * S/W workaround: Clear the off platform peripheral modules
+		 * Supervisor Protect bit for SDMA to access them.
+		 */
+		__raw_writel(0x0, aips1 + 0x40);
+		__raw_writel(0x0, aips1 + 0x44);
+		__raw_writel(0x0, aips1 + 0x48);
+		__raw_writel(0x0, aips1 + 0x4C);
+		aips_reg = __raw_readl(aips1 + 0x50);
+		aips_reg &= 0x00FFFFFF;
+		__raw_writel(aips_reg, aips1 + 0x50);
+
+		__raw_writel(0x0, aips2 + 0x40);
+		__raw_writel(0x0, aips2 + 0x44);
+		__raw_writel(0x0, aips2 + 0x48);
+		__raw_writel(0x0, aips2 + 0x4C);
+		aips_reg = __raw_readl(aips2 + 0x50);
+		aips_reg &= 0x00FFFFFF;
+		__raw_writel(aips_reg, aips2 + 0x50);
+	} else {
+		aips_reg = __raw_readl(aips1 + 8);
+		aips_reg &= ~(1 << aips2);
+		__raw_writel(aips_reg, aips1 + 8);
+	}
+}
+#endif /* CONFIG_IPIPE */
diff --git a/arch/arm/plat-mxc/devices.c b/arch/arm/plat-mxc/devices.c
index e9bcefe..98f03c4 100644
--- a/arch/arm/plat-mxc/devices.c
+++ b/arch/arm/plat-mxc/devices.c
@@ -22,6 +22,7 @@
 #include <linux/err.h>
 #include <linux/platform_device.h>
 #include <mach/common.h>
+#include <mach/hardware.h>
 
 int __init mxc_register_device(struct platform_device *pdev, void *data)
 {
@@ -87,3 +88,24 @@ err:
 
 	return pdev;
 }
+
+static int post_cpu_init(void)
+{
+#ifdef CONFIG_MACH_MX27
+	if (cpu_is_mx27())
+		ipipe_mach_allow_hwtimer_uaccess(MX27_IO_P2V(MX27_AIPI_BASE_ADDR), 3);
+#endif /* CONFIG_MACH_MX27 */
+#ifdef CONFIG_MACH_MX25
+	if (cpu_is_mx25())
+		ipipe_mach_allow_hwtimer_uaccess(MX25_AIPS1_BASE_ADDR_VIRT,
+						 MX25_AIPS2_BASE_ADDR_VIRT);
+#endif /* CONFIG_MACH_MX25 */
+#ifdef CONFIG_MACH_MX31
+	if (cpu_is_mx31())
+		ipipe_mach_allow_hwtimer_uaccess(AIPS1_BASE_ADDR_VIRT,
+						 AIPS2_BASE_ADDR_VIRT);
+#endif /* CONFIG_MACH_MX31 */
+	return 0;
+}
+
+postcore_initcall(post_cpu_init);
diff --git a/arch/arm/plat-mxc/gpio.c b/arch/arm/plat-mxc/gpio.c
index d17b3c9..3ef2e4f 100644
--- a/arch/arm/plat-mxc/gpio.c
+++ b/arch/arm/plat-mxc/gpio.c
@@ -24,6 +24,7 @@
 #include <linux/io.h>
 #include <linux/irq.h>
 #include <linux/gpio.h>
+#include <linux/ipipe.h>
 #include <mach/hardware.h>
 #include <asm-generic/bug.h>
 
@@ -87,6 +88,7 @@ static int gpio_set_irq_type(struct irq_data *d, u32 type)
 {
 	u32 gpio = irq_to_gpio(d->irq);
 	struct mxc_gpio_port *port = &mxc_gpio_ports[gpio / 32];
+	unsigned long flags;
 	u32 bit, val;
 	int edge;
 	void __iomem *reg = port->base;
@@ -120,11 +122,13 @@ static int gpio_set_irq_type(struct irq_data *d, u32 type)
 		return -EINVAL;
 	}
 
+	spin_lock_irqsave(&port->lock, flags);
 	reg += GPIO_ICR1 + ((gpio & 0x10) >> 2); /* lower or upper register */
 	bit = gpio & 0xf;
 	val = __raw_readl(reg) & ~(0x3 << (bit << 1));
 	__raw_writel(val | (edge << (bit << 1)), reg);
 	_clear_gpio_irqstatus(port, gpio & 0x1f);
+	spin_unlock_irqrestore(&port->lock, flags);
 
 	return 0;
 }
@@ -159,13 +163,35 @@ static void mxc_gpio_irq_handler(struct mxc_gpio_port *port, u32 irq_stat)
 {
 	u32 gpio_irq_no_base = port->virtual_irq_start;
 
+#ifdef CONFIG_IPIPE
+	/* Handle high priority domains interrupts first */
+	u32 nonroot_gpios = irq_stat & port->nonroot_gpios;
+
+	irq_stat &= ~nonroot_gpios;
+	while (nonroot_gpios != 0) {
+		int irqoffset = fls(nonroot_gpios) - 1;
+
+		if (port->both_edges & (1 << irqoffset))
+			mxc_flip_edge(port, irqoffset);
+
+		ipipe_handle_chained_irq(gpio_irq_no_base + irqoffset);
+
+		nonroot_gpios &= ~(1 << irqoffset);
+	}
+#endif
+
 	while (irq_stat != 0) {
 		int irqoffset = fls(irq_stat) - 1;
 
+#ifdef CONFIG_IPIPE
+		local_irq_enable_hw();
+		local_irq_disable_hw();
+#endif /* CONFIG_IPIPE */
+
 		if (port->both_edges & (1 << irqoffset))
 			mxc_flip_edge(port, irqoffset);
 
-		generic_handle_irq(gpio_irq_no_base + irqoffset);
+		ipipe_handle_chained_irq(gpio_irq_no_base + irqoffset);
 
 		irq_stat &= ~(1 << irqoffset);
 	}
@@ -177,10 +203,18 @@ static void mx3_gpio_irq_handler(u32 irq, struct irq_desc *desc)
 	u32 irq_stat;
 	struct mxc_gpio_port *port = get_irq_data(irq);
 
+#ifdef CONFIG_IPIPE
+	desc->irq_data.chip->irq_mask_ack(&desc->irq_data);
+#endif /* CONFIG_IPIPE */
+
 	irq_stat = __raw_readl(port->base + GPIO_ISR) &
 			__raw_readl(port->base + GPIO_IMR);
 
 	mxc_gpio_irq_handler(port, irq_stat);
+
+#ifdef CONFIG_IPIPE
+	desc->irq_data.chip->irq_unmask(&desc->irq_data);
+#endif /* CONFIG_IPIPE */
 }
 
 /* MX2 has one interrupt *for all* gpio ports */
@@ -190,6 +224,10 @@ static void mx2_gpio_irq_handler(u32 irq, struct irq_desc *desc)
 	u32 irq_msk, irq_stat;
 	struct mxc_gpio_port *port = get_irq_data(irq);
 
+#ifdef CONFIG_IPIPE
+	desc->irq_data.chip->irq_mask_ack(&desc->irq_data);
+#endif /* CONFIG_IPIPE */
+
 	/* walk through all interrupt status registers */
 	for (i = 0; i < gpio_table_size; i++) {
 		irq_msk = __raw_readl(port[i].base + GPIO_IMR);
@@ -200,6 +238,10 @@ static void mx2_gpio_irq_handler(u32 irq, struct irq_desc *desc)
 		if (irq_stat)
 			mxc_gpio_irq_handler(&port[i], irq_stat);
 	}
+
+#ifdef CONFIG_IPIPE
+	desc->irq_data.chip->irq_unmask(&desc->irq_data);
+#endif /* CONFIG_IPIPE */
 }
 
 /*
@@ -459,3 +501,48 @@ static struct mxc_gpio_port imx50_gpio_ports[] = {
 DEFINE_REGISTER_FUNCTION(imx50)
 
 #endif /* if defined(CONFIG_SOC_IMX50) */
+
+#if defined(CONFIG_IPIPE) && defined(__IPIPE_FEATURE_PIC_MUTE)
+extern void tzic_set_irq_prio(int irq, int hi);
+
+void __ipipe_mach_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
+{
+	struct irq_desc *desc = irq_to_desc(irq);
+
+	if (desc->chip == &gpio_irq_chip) {
+		/* It is a gpio. */
+		u32 gpio = irq_to_gpio(irq);
+		struct mxc_gpio_port *port = &mxc_gpio_ports[gpio / 32];
+
+		if (ipd != &ipipe_root) {
+			port->nonroot_gpios |= (1 << (gpio % 32));
+			if (port->nonroot_gpios == (1 << (gpio % 32))) {
+				__ipipe_irqbits[(port->irq / 32)]
+					&= ~(1 << (port->irq % 32));
+				tzic_set_irq_prio(port->irq, 1);
+			}
+		}
+	} else
+		tzic_set_irq_prio(irq, ipd != &ipipe_root);
+}
+
+void __ipipe_mach_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
+{
+	struct irq_desc *desc = irq_to_desc(irq);
+	if (desc->chip == &gpio_irq_chip) {
+		/* It is a gpio. */
+		u32 gpio = irq_to_gpio(irq);
+		struct mxc_gpio_port *port = &mxc_gpio_ports[gpio / 32];
+
+		if (ipd != &ipipe_root) {
+			port->nonroot_gpios &= ~(1 << (gpio % 32));
+			if (!port->nonroot_gpios) {
+				tzic_set_irq_prio(port->irq, 0);
+				__ipipe_irqbits[(port->irq / 32)]
+					|= (1 << (port->irq % 32));
+			}
+		}
+	} else if (ipd != &ipipe_root)
+		tzic_set_irq_prio(irq, 0);
+}
+#endif /* CONFIG_IPIPE && __IPIPE_FEATURE_PIC_MUTE */
diff --git a/arch/arm/plat-mxc/include/mach/common.h b/arch/arm/plat-mxc/include/mach/common.h
index aea2cd3..7d8cf14 100644
--- a/arch/arm/plat-mxc/include/mach/common.h
+++ b/arch/arm/plat-mxc/include/mach/common.h
@@ -37,7 +37,7 @@ extern void mx51_init_irq(void);
 extern void mx53_init_irq(void);
 extern void mxc91231_init_irq(void);
 extern void epit_timer_init(struct clk *timer_clk, void __iomem *base, int irq);
-extern void mxc_timer_init(struct clk *timer_clk, void __iomem *, int);
+extern void mxc_timer_init(struct clk *timer_clk, void __iomem *, unsigned long, int);
 extern int mx1_clocks_init(unsigned long fref);
 extern int mx21_clocks_init(unsigned long lref, unsigned long fref);
 extern int mx25_clocks_init(void);
@@ -58,4 +58,9 @@ extern void mxc91231_arch_reset(int, const char *);
 extern void mxc91231_prepare_idle(void);
 extern void mx51_efikamx_reset(void);
 extern int mx53_revision(void);
+
+#ifdef CONFIG_IPIPE
+void ipipe_mach_allow_hwtimer_uaccess(unsigned long aips1, unsigned long aips2);
+#endif
+
 #endif
diff --git a/arch/arm/plat-mxc/include/mach/gpio.h b/arch/arm/plat-mxc/include/mach/gpio.h
index 0044e2f..c0b391d 100644
--- a/arch/arm/plat-mxc/include/mach/gpio.h
+++ b/arch/arm/plat-mxc/include/mach/gpio.h
@@ -43,7 +43,10 @@ struct mxc_gpio_port {
 	int virtual_irq_start;
 	struct gpio_chip chip;
 	u32 both_edges;
-	spinlock_t lock;
+	ipipe_spinlock_t lock;
+#ifdef CONFIG_IPIPE
+	unsigned nonroot_gpios;
+#endif /* CONFIG_IPIPE */
 };
 
 int mxc_gpio_init(struct mxc_gpio_port*, int);
diff --git a/arch/arm/plat-mxc/include/mach/irqs.h b/arch/arm/plat-mxc/include/mach/irqs.h
index ba65c92..ad3a1c1 100644
--- a/arch/arm/plat-mxc/include/mach/irqs.h
+++ b/arch/arm/plat-mxc/include/mach/irqs.h
@@ -73,4 +73,8 @@ extern int imx_irq_set_priority(unsigned char irq, unsigned char prio);
 /* switch between IRQ and FIQ */
 extern int mxc_set_irq_fiq(unsigned int irq, unsigned int type);
 
+#ifdef CONFIG_MXC_TZIC
+#define __IPIPE_FEATURE_PIC_MUTE
+#endif /* CONFIG_MXC_TZIC */
+
 #endif /* __ASM_ARCH_MXC_IRQS_H__ */
diff --git a/arch/arm/plat-mxc/time.c b/arch/arm/plat-mxc/time.c
index 9f0c261..b6dc64d 100644
--- a/arch/arm/plat-mxc/time.c
+++ b/arch/arm/plat-mxc/time.c
@@ -25,6 +25,7 @@
 #include <linux/irq.h>
 #include <linux/clockchips.h>
 #include <linux/clk.h>
+#include <linux/ipipe.h>
 
 #include <mach/hardware.h>
 #include <asm/mach/time.h>
@@ -66,6 +67,19 @@
 #define timer_is_v1()	(cpu_is_mx1() || cpu_is_mx21() || cpu_is_mx27())
 #define timer_is_v2()	(!timer_is_v1())
 
+#ifdef CONFIG_IPIPE
+int __ipipe_mach_timerint;
+EXPORT_SYMBOL(__ipipe_mach_timerint);
+
+int __ipipe_mach_timerstolen = 0;
+EXPORT_SYMBOL(__ipipe_mach_timerstolen);
+
+unsigned int __ipipe_mach_ticks_per_jiffy = LATCH;
+EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy);
+
+static unsigned mxc_min_delay;
+#endif /* CONFIG_IPIPE */
+
 static struct clock_event_device clockevent_mxc;
 static enum clock_event_mode clockevent_mode = CLOCK_EVT_MODE_UNUSED;
 
@@ -246,7 +260,11 @@ static irqreturn_t mxc_timer_interrupt(int irq, void *dev_id)
 	else
 		tstat = __raw_readl(timer_base + MX1_2_TSTAT);
 
+#ifndef CONFIG_IPIPE
 	gpt_irq_acknowledge();
+#else /* !CONFIG_IPIPE */
+	__ipipe_tsc_update();
+#endif /* !CONFIG_IPIPE */
 
 	evt->event_handler(evt);
 
@@ -289,7 +307,68 @@ static int __init mxc_clockevent_init(struct clk *timer_clk)
 	return 0;
 }
 
-void __init mxc_timer_init(struct clk *timer_clk, void __iomem *base, int irq)
+#ifdef CONFIG_IPIPE
+static struct __ipipe_tscinfo tsc_info = {
+	.type = IPIPE_TSC_TYPE_FREERUNNING,
+	.u = {
+		{
+			.mask = 0xffffffff,
+		},
+	},
+};
+
+int __ipipe_check_tickdev(const char *devname)
+{
+	return !strcmp(devname, clockevent_mxc.name);
+}
+
+void __ipipe_mach_acktimer(void)
+{
+	gpt_irq_acknowledge();
+}
+/*
+ * Reprogram the timer
+ */
+
+void __ipipe_mach_set_dec(unsigned long delay)
+{
+	if (delay > mxc_min_delay) {
+		unsigned long tcmp;
+
+		if (!timer_is_v2()) {
+			tcmp = __raw_readl(timer_base + MX1_2_TCN) + delay;
+			__raw_writel(tcmp, timer_base + MX1_2_TCMP);
+		} else {
+			tcmp = __raw_readl(timer_base + V2_TCN) + delay;
+			__raw_writel(tcmp, timer_base + V2_TCMP);
+		}
+	} else
+		ipipe_trigger_irq(__ipipe_mach_timerint);
+}
+EXPORT_SYMBOL(__ipipe_mach_set_dec);
+
+void __ipipe_mach_release_timer(void)
+{
+	mxc_set_mode(clockevent_mxc.mode, &clockevent_mxc);
+	if (clockevent_mxc.mode == CLOCK_EVT_MODE_ONESHOT)
+		clockevent_mxc.set_next_event(LATCH, &clockevent_mxc);
+}
+EXPORT_SYMBOL(__ipipe_mach_release_timer);
+
+unsigned long __ipipe_mach_get_dec(void)
+{
+	if (!timer_is_v2())
+		return __raw_readl(timer_base + MX1_2_TCMP)
+			- __raw_readl(timer_base + MX1_2_TCN);
+	else
+		return __raw_readl(timer_base + V2_TCMP)
+			- __raw_readl(timer_base + V2_TCN);
+}
+#endif /* CONFIG_IPIPE */
+
+void __init
+mxc_timer_init(struct clk *timer_clk,
+	       void __iomem *base, unsigned long phys, int irq)
 {
 	uint32_t tctl_val;
 
@@ -317,4 +396,20 @@ void __init mxc_timer_init(struct clk *timer_clk, void __iomem *base, int irq)
 
 	/* Make irqs happen */
 	setup_irq(irq, &mxc_timer_irq);
+
+#ifdef CONFIG_IPIPE
+	__ipipe_mach_timerint = irq;
+	__ipipe_mach_ticks_per_jiffy = (clk_get_rate(timer_clk) + HZ/2) / HZ;
+	tsc_info.freq = clk_get_rate(timer_clk);
+	mxc_min_delay = ((__ipipe_cpu_freq + 500000) / 1000000) ?: 1;
+
+	if (timer_is_v1()) {
+		tsc_info.u.counter_paddr = phys + MX1_2_TCN;
+		tsc_info.counter_vaddr =(unsigned long)(phys + MX1_2_TCN);
+	} else {
+		tsc_info.u.counter_paddr = phys + V2_TCN;
+		tsc_info.counter_vaddr = (unsigned long)(timer_base + V2_TCN);
+	}
+	__ipipe_tsc_register(&tsc_info);
+#endif /* CONFIG_IPIPE */
 }
diff --git a/arch/arm/plat-mxc/tzic.c b/arch/arm/plat-mxc/tzic.c
index bc3a6be..b1a9d7a 100644
--- a/arch/arm/plat-mxc/tzic.c
+++ b/arch/arm/plat-mxc/tzic.c
@@ -15,6 +15,7 @@
 #include <linux/device.h>
 #include <linux/errno.h>
 #include <linux/io.h>
+#include <linux/ipipe.h>
 
 #include <asm/mach/irq.h>
 
@@ -130,6 +131,9 @@ static struct mxc_irq_chip mxc_tzic_chip = {
 		.name = "MXC_TZIC",
 		.irq_ack = tzic_mask_irq,
 		.irq_mask = tzic_mask_irq,
+#ifdef CONFIG_IPIPE
+		.irq_mask_ack = tzic_mask_irq,
+#endif /* CONFIG_IPIPE */
 		.irq_unmask = tzic_unmask_irq,
 		.irq_set_wake = tzic_set_wake_irq,
 	},
@@ -137,6 +141,27 @@ static struct mxc_irq_chip mxc_tzic_chip = {
 	.set_irq_fiq = tzic_set_irq_fiq,
 #endif
 };
+#if defined(CONFIG_IPIPE) && defined(__IPIPE_FEATURE_PIC_MUTE)
+DEFINE_PER_CPU(__ipipe_irqbits_t, __ipipe_muted_irqs);
+
+void tzic_set_irq_prio(unsigned irq, unsigned hi)
+{
+	if (irq >= MXC_INTERNAL_IRQS)
+		return;
+
+	__raw_writeb(hi ? 0 : 0x80, tzic_base + TZIC_PRIORITY0 + irq);
+}
+
+void ipipe_mute_pic(void)
+{
+	__raw_writel(0x10, tzic_base + TZIC_PRIOMASK);
+}
+
+void ipipe_unmute_pic(void)
+{
+	__raw_writel(0xf0, tzic_base + TZIC_PRIOMASK);
+}
+#endif /* CONFIG_IPIPE && __IPIPE_FEATURE_PIC_MUTE */
 
 /*
  * This function initializes the TZIC hardware and disables all the
@@ -154,8 +179,13 @@ void __init tzic_init_irq(void __iomem *irqbase)
 	i = __raw_readl(tzic_base + TZIC_INTCNTL);
 
 	__raw_writel(0x80010001, tzic_base + TZIC_INTCNTL);
+#ifndef CONFIG_IPIPE
 	__raw_writel(0x1f, tzic_base + TZIC_PRIOMASK);
 	__raw_writel(0x02, tzic_base + TZIC_SYNCCTRL);
+#else
+	__raw_writel(0xf0, tzic_base + TZIC_PRIOMASK);
+	__raw_writel(0, tzic_base + TZIC_SYNCCTRL);
+#endif
 
 	for (i = 0; i < 4; i++)
 		__raw_writel(0xFFFFFFFF, tzic_base + TZIC_INTSEC0(i));
diff --git a/arch/arm/plat-omap/Kconfig b/arch/arm/plat-omap/Kconfig
index b6333ae..de9cad3 100644
--- a/arch/arm/plat-omap/Kconfig
+++ b/arch/arm/plat-omap/Kconfig
@@ -136,10 +136,10 @@ config OMAP_IOMMU_DEBUG
        tristate "Export OMAP IOMMU internals in DebugFS"
        depends on OMAP_IOMMU && DEBUG_FS
        help
-         Select this to see extensive information about
-         the internal state of OMAP IOMMU in debugfs.
+	 Select this to see extensive information about
+	 the internal state of OMAP IOMMU in debugfs.
 
-         Say N unless you know you need this.
+	 Say N unless you know you need this.
 
 config OMAP_IOMMU_IVA2
 	bool
@@ -152,6 +152,7 @@ config OMAP_MPU_TIMER
 	  timer provides more intra-tick resolution than the 32KHz timer,
 	  but consumes more power.
 
+if !IPIPE || SMP
 config OMAP_32K_TIMER
 	bool "Use 32KHz timer"
 	depends on ARCH_OMAP16XX || ARCH_OMAP2PLUS
@@ -162,6 +163,7 @@ config OMAP_32K_TIMER
 	  support for no tick during idle. The 32KHz timer provides less
 	  intra-tick resolution than OMAP_MPU_TIMER. The 32KHz timer is
 	  currently only available for OMAP16XX, 24XX, 34XX and OMAP4.
+endif
 
 config OMAP3_L2_AUX_SECURE_SAVE_RESTORE
 	bool "OMAP3 HS/EMU save and restore for L2 AUX control register"
diff --git a/arch/arm/plat-omap/counter_32k.c b/arch/arm/plat-omap/counter_32k.c
index 862dda9..9f43c9f 100644
--- a/arch/arm/plat-omap/counter_32k.c
+++ b/arch/arm/plat-omap/counter_32k.c
@@ -126,7 +126,7 @@ static inline unsigned long long notrace _omap_32k_sched_clock(void)
 	return cyc_to_fixed_sched_clock(&cd, cyc, (u32)~0, SC_MULT, SC_SHIFT);
 }
 
-#ifndef CONFIG_OMAP_MPU_TIMER
+#if !defined(CONFIG_OMAP_MPU_TIMER) && defined(CONFIG_OMAP_32K_TIMER)
 unsigned long long notrace sched_clock(void)
 {
 	return _omap_32k_sched_clock();
diff --git a/arch/arm/plat-omap/dmtimer.c b/arch/arm/plat-omap/dmtimer.c
index 1d706cf..ac1c1fd 100644
--- a/arch/arm/plat-omap/dmtimer.c
+++ b/arch/arm/plat-omap/dmtimer.c
@@ -37,6 +37,7 @@
 #include <linux/delay.h>
 #include <linux/io.h>
 #include <linux/module.h>
+#include <linux/sched.h>
 #include <mach/hardware.h>
 #include <plat/dmtimer.h>
 #include <mach/irqs.h>
@@ -328,7 +329,7 @@ static void omap_dm_timer_wait_for_reset(struct omap_dm_timer *timer)
 	}
 }
 
-static void omap_dm_timer_reset(struct omap_dm_timer *timer)
+static void omap_dm_timer_reset(struct omap_dm_timer *timer, int posted)
 {
 	u32 l;
 
@@ -339,8 +340,13 @@ static void omap_dm_timer_reset(struct omap_dm_timer *timer)
 	omap_dm_timer_set_source(timer, OMAP_TIMER_SRC_32_KHZ);
 
 	l = omap_dm_timer_read_reg(timer, OMAP_TIMER_OCP_CFG_REG);
+#if !defined(CONFIG_IPIPE) || defined(CONFIG_SMP)
 	l |= 0x02 << 3;  /* Set to smart-idle mode */
-	l |= 0x2 << 8;   /* Set clock activity to perserve f-clock on idle */
+	l |= 0x2 << 8;   /* Set clock activity to perserve f-clock on
+			  * idle */
+#else /* IPIPE */
+	l = (0x3 << 8) | (l & (1 << 5)) | (0x1 << 3) | (l & (1 << 2));
+#endif /* IPIPE */
 
 	/*
 	 * Enable wake-up on OMAP2 CPUs.
@@ -351,14 +357,14 @@ static void omap_dm_timer_reset(struct omap_dm_timer *timer)
 
 	/* Match hardware reset default of posted mode */
 	omap_dm_timer_write_reg(timer, OMAP_TIMER_IF_CTRL_REG,
-			OMAP_TIMER_CTRL_POSTED);
-	timer->posted = 1;
+				posted ? OMAP_TIMER_CTRL_POSTED : 0);
+	timer->posted = posted;
 }
 
-static void omap_dm_timer_prepare(struct omap_dm_timer *timer)
+static void omap_dm_timer_prepare(struct omap_dm_timer *timer, int posted)
 {
 	omap_dm_timer_enable(timer);
-	omap_dm_timer_reset(timer);
+	omap_dm_timer_reset(timer, posted);
 }
 
 struct omap_dm_timer *omap_dm_timer_request(void)
@@ -379,13 +385,14 @@ struct omap_dm_timer *omap_dm_timer_request(void)
 	spin_unlock_irqrestore(&dm_timer_lock, flags);
 
 	if (timer != NULL)
-		omap_dm_timer_prepare(timer);
+		omap_dm_timer_prepare(timer, 1);
 
 	return timer;
 }
 EXPORT_SYMBOL_GPL(omap_dm_timer_request);
 
-struct omap_dm_timer *omap_dm_timer_request_specific(int id)
+static struct omap_dm_timer *
+omap_dm_timer_request_specific_inner(int id, int posted)
 {
 	struct omap_dm_timer *timer;
 	unsigned long flags;
@@ -403,16 +410,26 @@ struct omap_dm_timer *omap_dm_timer_request_specific(int id)
 	timer->reserved = 1;
 	spin_unlock_irqrestore(&dm_timer_lock, flags);
 
-	omap_dm_timer_prepare(timer);
+	omap_dm_timer_prepare(timer, posted);
 
 	return timer;
 }
+struct omap_dm_timer *omap_dm_timer_request_specific(int id)
+{
+	return omap_dm_timer_request_specific_inner(id, 1);
+}
 EXPORT_SYMBOL_GPL(omap_dm_timer_request_specific);
 
+struct omap_dm_timer *omap_dm_timer_request_specific_nonposted(int id)
+{
+	return omap_dm_timer_request_specific_inner(id, 0);
+}
+EXPORT_SYMBOL_GPL(omap_dm_timer_request_specific_nonposted);
+
 void omap_dm_timer_free(struct omap_dm_timer *timer)
 {
 	omap_dm_timer_enable(timer);
-	omap_dm_timer_reset(timer);
+	omap_dm_timer_reset(timer, 1);
 	omap_dm_timer_disable(timer);
 
 	WARN_ON(!timer->reserved);
@@ -458,6 +475,18 @@ int omap_dm_timer_get_irq(struct omap_dm_timer *timer)
 }
 EXPORT_SYMBOL_GPL(omap_dm_timer_get_irq);
 
+#ifdef CONFIG_IPIPE
+unsigned long omap_dm_timer_get_phys_counter_addr(struct omap_dm_timer *timer)
+{
+	return timer->phys_base + (OMAP_TIMER_COUNTER_REG & 0xff);
+}
+
+unsigned long omap_dm_timer_get_virt_counter_addr(struct omap_dm_timer *timer)
+{
+	return (unsigned long)timer->io_base + (OMAP_TIMER_COUNTER_REG & 0xff);
+}
+#endif /* CONFIG_IPIPE */
+
 #if defined(CONFIG_ARCH_OMAP1)
 
 /**
@@ -608,7 +637,7 @@ EXPORT_SYMBOL_GPL(omap_dm_timer_set_load);
 
 /* Optimized set_load which removes costly spin wait in timer_start */
 void omap_dm_timer_set_load_start(struct omap_dm_timer *timer, int autoreload,
-                            unsigned int load)
+			    unsigned int load)
 {
 	u32 l;
 
diff --git a/arch/arm/plat-omap/gpio.c b/arch/arm/plat-omap/gpio.c
index 971d186..30cb7d4 100644
--- a/arch/arm/plat-omap/gpio.c
+++ b/arch/arm/plat-omap/gpio.c
@@ -20,9 +20,11 @@
 #include <linux/sysdev.h>
 #include <linux/err.h>
 #include <linux/clk.h>
+#include <linux/irq.h>		/* For irq_desc */
 #include <linux/io.h>
 #include <linux/slab.h>
 #include <linux/pm_runtime.h>
+#include <linux/ipipe.h>
 
 #include <mach/hardware.h>
 #include <asm/irq.h>
@@ -151,7 +153,7 @@ struct gpio_bank {
 	u32 saved_risingdetect;
 	u32 level_mask;
 	u32 toggle_mask;
-	spinlock_t lock;
+	ipipe_spinlock_t lock;
 	struct gpio_chip chip;
 	struct clk *dbck;
 	u32 mod_usage;
@@ -159,8 +161,95 @@ struct gpio_bank {
 	struct device *dev;
 	bool dbck_flag;
 	int stride;
+#ifdef CONFIG_IPIPE
+	unsigned nonroot_gpios;
+#endif
+};
+
+#define METHOD_MPUIO		0
+#define METHOD_GPIO_1510	1
+#define METHOD_GPIO_1610	2
+#define METHOD_GPIO_7XX		3
+#define METHOD_GPIO_24XX	5
+#define METHOD_GPIO_44XX	6
+
+#if !defined(MULTI_OMAP1) && !defined(MULTI_OMAP2)
+#define inline_single inline
+#else
+#define inline_single
+#endif
+
+#ifdef CONFIG_ARCH_OMAP16XX
+static struct gpio_bank gpio_bank_1610[5] = {
+	{ OMAP1_MPUIO_VBASE, NULL, INT_MPUIO, IH_MPUIO_BASE,
+		METHOD_MPUIO },
+	{ OMAP1610_GPIO1_BASE, NULL, INT_GPIO_BANK1, IH_GPIO_BASE,
+		METHOD_GPIO_1610 },
+	{ OMAP1610_GPIO2_BASE, NULL, INT_1610_GPIO_BANK2, IH_GPIO_BASE + 16,
+		METHOD_GPIO_1610 },
+	{ OMAP1610_GPIO3_BASE, NULL, INT_1610_GPIO_BANK3, IH_GPIO_BASE + 32,
+		METHOD_GPIO_1610 },
+	{ OMAP1610_GPIO4_BASE, NULL, INT_1610_GPIO_BANK4, IH_GPIO_BASE + 48,
+		METHOD_GPIO_1610 },
+};
+#endif
+
+#ifdef CONFIG_ARCH_OMAP15XX
+static struct gpio_bank gpio_bank_1510[2] = {
+	{ OMAP1_MPUIO_VBASE, NULL, INT_MPUIO, IH_MPUIO_BASE,
+		METHOD_MPUIO },
+	{ OMAP1510_GPIO_BASE, NULL, INT_GPIO_BANK1, IH_GPIO_BASE,
+		METHOD_GPIO_1510 }
+};
+#endif
+
+#if defined(CONFIG_ARCH_OMAP730) || defined(CONFIG_ARCH_OMAP850)
+static struct gpio_bank gpio_bank_7xx[7] = {
+	{ OMAP1_MPUIO_VBASE, NULL, INT_7XX_MPUIO, IH_MPUIO_BASE,
+		METHOD_MPUIO },
+	{ OMAP7XX_GPIO1_BASE, NULL, INT_7XX_GPIO_BANK1, IH_GPIO_BASE,
+		METHOD_GPIO_7XX },
+	{ OMAP7XX_GPIO2_BASE, NULL, INT_7XX_GPIO_BANK2, IH_GPIO_BASE + 32,
+		METHOD_GPIO_7XX },
+	{ OMAP7XX_GPIO3_BASE, NULL, INT_7XX_GPIO_BANK3, IH_GPIO_BASE + 64,
+		METHOD_GPIO_7XX },
+	{ OMAP7XX_GPIO4_BASE, NULL, INT_7XX_GPIO_BANK4,  IH_GPIO_BASE + 96,
+		METHOD_GPIO_7XX },
+	{ OMAP7XX_GPIO5_BASE, NULL, INT_7XX_GPIO_BANK5,  IH_GPIO_BASE + 128,
+		METHOD_GPIO_7XX },
+	{ OMAP7XX_GPIO6_BASE, NULL, INT_7XX_GPIO_BANK6,  IH_GPIO_BASE + 160,
+		METHOD_GPIO_7XX },
+};
+#endif
+
+#ifdef CONFIG_ARCH_OMAP2
+
+static struct gpio_bank gpio_bank_242x[4] = {
+	{ OMAP242X_GPIO1_BASE, NULL, INT_24XX_GPIO_BANK1, IH_GPIO_BASE,
+		METHOD_GPIO_24XX },
+	{ OMAP242X_GPIO2_BASE, NULL, INT_24XX_GPIO_BANK2, IH_GPIO_BASE + 32,
+		METHOD_GPIO_24XX },
+	{ OMAP242X_GPIO3_BASE, NULL, INT_24XX_GPIO_BANK3, IH_GPIO_BASE + 64,
+		METHOD_GPIO_24XX },
+	{ OMAP242X_GPIO4_BASE, NULL, INT_24XX_GPIO_BANK4, IH_GPIO_BASE + 96,
+		METHOD_GPIO_24XX },
+};
+
+static struct gpio_bank gpio_bank_243x[5] = {
+	{ OMAP243X_GPIO1_BASE, NULL, INT_24XX_GPIO_BANK1, IH_GPIO_BASE,
+		METHOD_GPIO_24XX },
+	{ OMAP243X_GPIO2_BASE, NULL, INT_24XX_GPIO_BANK2, IH_GPIO_BASE + 32,
+		METHOD_GPIO_24XX },
+	{ OMAP243X_GPIO3_BASE, NULL, INT_24XX_GPIO_BANK3, IH_GPIO_BASE + 64,
+		METHOD_GPIO_24XX },
+	{ OMAP243X_GPIO4_BASE, NULL, INT_24XX_GPIO_BANK4, IH_GPIO_BASE + 96,
+		METHOD_GPIO_24XX },
+	{ OMAP243X_GPIO5_BASE, NULL, INT_24XX_GPIO_BANK5, IH_GPIO_BASE + 128,
+		METHOD_GPIO_24XX },
 };
 
+#endif
+
 #ifdef CONFIG_ARCH_OMAP3
 struct omap3_gpio_regs {
 	u32 irqenable1;
@@ -189,6 +278,12 @@ static int bank_width;
 /* TODO: Analyze removing gpio_bank_count usage from driver code */
 int gpio_bank_count;
 
+#ifdef CONFIG_IPIPE
+#ifdef __IPIPE_FEATURE_PIC_MUTE
+DEFINE_PER_CPU(__ipipe_irqbits_t, __ipipe_muted_irqs);
+#endif /* __IPIPE_FEATURE_PIC_MUTE */
+#endif /* CONFIG_IPIPE */
+
 static inline struct gpio_bank *get_gpio_bank(int gpio)
 {
 	if (cpu_is_omap15xx()) {
@@ -249,7 +344,7 @@ static inline int gpio_valid(int gpio)
 	return -1;
 }
 
-static int check_gpio(int gpio)
+static inline_single int check_gpio(int gpio)
 {
 	if (unlikely(gpio_valid(gpio) < 0)) {
 		printk(KERN_ERR "omap-gpio: invalid GPIO %d\n", gpio);
@@ -613,7 +708,7 @@ static inline void set_24xx_gpio_triggering(struct gpio_bank *bank, int gpio,
  * This only applies to chips that can't do both rising and falling edge
  * detection at once.  For all other chips, this function is a noop.
  */
-static void _toggle_gpio_edge_triggering(struct gpio_bank *bank, int gpio)
+static inline_single void _toggle_gpio_edge_triggering(struct gpio_bank *bank, int gpio)
 {
 	void __iomem *reg = bank->base;
 	u32 l = 0;
@@ -646,7 +741,7 @@ static void _toggle_gpio_edge_triggering(struct gpio_bank *bank, int gpio)
 }
 #endif
 
-static int _set_gpio_triggering(struct gpio_bank *bank, int gpio, int trigger)
+static inline_single int _set_gpio_triggering(struct gpio_bank *bank, int gpio, int trigger)
 {
 	void __iomem *reg = bank->base;
 	u32 l = 0;
@@ -771,7 +866,7 @@ static int gpio_irq_type(struct irq_data *d, unsigned type)
 	return retval;
 }
 
-static void _clear_gpio_irqbank(struct gpio_bank *bank, int gpio_mask)
+static inline_single void _clear_gpio_irqbank(struct gpio_bank *bank, int gpio_mask)
 {
 	void __iomem *reg = bank->base;
 
@@ -832,7 +927,7 @@ static inline void _clear_gpio_irqstatus(struct gpio_bank *bank, int gpio)
 	_clear_gpio_irqbank(bank, 1 << get_gpio_index(gpio));
 }
 
-static u32 _get_gpio_irqbank_mask(struct gpio_bank *bank)
+static inline_single u32 _get_gpio_irqbank_mask(struct gpio_bank *bank)
 {
 	void __iomem *reg = bank->base;
 	int inv = 0;
@@ -891,7 +986,7 @@ static u32 _get_gpio_irqbank_mask(struct gpio_bank *bank)
 	return l;
 }
 
-static void _enable_gpio_irqbank(struct gpio_bank *bank, int gpio_mask, int enable)
+static inline_single void _enable_gpio_irqbank(struct gpio_bank *bank, int gpio_mask, int enable)
 {
 	void __iomem *reg = bank->base;
 	u32 l;
@@ -1126,6 +1221,40 @@ static void omap_gpio_free(struct gpio_chip *chip, unsigned offset)
 	spin_unlock_irqrestore(&bank->lock, flags);
 }
 
+static void gpio_demux_inner(struct gpio_bank *bank, u32 isr, int nonroot)
+{
+	unsigned int gpio_irq, gpio_index;
+
+	gpio_irq = bank->virtual_irq_start;
+	for (; isr != 0; isr >>= 1, gpio_irq++) {
+		if (!(isr & 1))
+			continue;
+
+#ifdef CONFIG_IPIPE
+		if (nonroot) {
+			local_irq_enable_hw();
+			local_irq_disable_hw();
+		}
+#endif /* CONFIG_IPIPE */
+
+#ifdef CONFIG_ARCH_OMAP1
+		gpio_index = get_gpio_index(irq_to_gpio(gpio_irq));
+
+		/*
+		 * Some chips can't respond to both rising and falling
+		 * at the same time.  If this irq was requested with
+		 * both flags, we need to flip the ICR data for the IRQ
+		 * to respond to the IRQ for the opposite direction.
+		 * This will be indicated in the bank toggle_mask.
+		 */
+		if (bank->toggle_mask & (1 << gpio_index))
+			_toggle_gpio_edge_triggering(bank, gpio_index);
+#endif
+		ipipe_handle_chained_irq(gpio_irq);
+	}
+}
+
+
 /*
  * We need to unmask the GPIO bank interrupt as soon as possible to
  * avoid missing GPIO interrupts for other lines in the bank.
@@ -1139,7 +1268,6 @@ static void gpio_irq_handler(unsigned int irq, struct irq_desc *desc)
 {
 	void __iomem *isr_reg = NULL;
 	u32 isr;
-	unsigned int gpio_irq, gpio_index;
 	struct gpio_bank *bank;
 	u32 retrigger = 0;
 	int unmasked = 0;
@@ -1180,6 +1308,13 @@ static void gpio_irq_handler(unsigned int irq, struct irq_desc *desc)
 		u32 isr_saved, level_mask = 0;
 		u32 enabled;
 
+#ifdef CONFIG_IPIPE
+		if (!bank->nonroot_gpios) {
+			local_irq_enable_hw();
+			local_irq_disable_hw();
+		}
+#endif /* CONFIG_IPIPE */
+
 		enabled = _get_gpio_irqbank_mask(bank);
 		isr_saved = isr = __raw_readl(isr_reg) & enabled;
 
@@ -1209,27 +1344,14 @@ static void gpio_irq_handler(unsigned int irq, struct irq_desc *desc)
 		if (!isr)
 			break;
 
-		gpio_irq = bank->virtual_irq_start;
-		for (; isr != 0; isr >>= 1, gpio_irq++) {
-			gpio_index = get_gpio_index(irq_to_gpio(gpio_irq));
-
-			if (!(isr & 1))
-				continue;
+#ifdef CONFIG_IPIPE
+		if (bank->nonroot_gpios)
+			gpio_demux_inner(bank, isr & bank->nonroot_gpios, 1);
+		gpio_demux_inner(bank, isr & ~bank->nonroot_gpios, 0);
+#else /* !CONFIG_IPIPE */
+		gpio_demux_inner(bank, isr, 0);
+#endif /* !CONFIG_IPIPE */
 
-#ifdef CONFIG_ARCH_OMAP1
-			/*
-			 * Some chips can't respond to both rising and falling
-			 * at the same time.  If this irq was requested with
-			 * both flags, we need to flip the ICR data for the IRQ
-			 * to respond to the IRQ for the opposite direction.
-			 * This will be indicated in the bank toggle_mask.
-			 */
-			if (bank->toggle_mask & (1 << gpio_index))
-				_toggle_gpio_edge_triggering(bank, gpio_index);
-#endif
-
-			generic_handle_irq(gpio_irq);
-		}
 	}
 	/* if bank has any level sensitive GPIO pin interrupt
 	configured, we must unmask the bank interrupt only after
@@ -1265,6 +1387,16 @@ static void gpio_mask_irq(struct irq_data *d)
 	_set_gpio_triggering(bank, get_gpio_index(gpio), IRQ_TYPE_NONE);
 }
 
+static void gpio_mask_ack_irq(struct irq_data *d)
+{
+	unsigned int gpio = d->irq - IH_GPIO_BASE;
+	struct gpio_bank *bank = get_irq_chip_data(d->irq);
+
+	_set_gpio_irqenable(bank, gpio, 0);
+	_set_gpio_triggering(bank, get_gpio_index(gpio), IRQ_TYPE_NONE);
+	_clear_gpio_irqstatus(bank, gpio);
+}
+
 static void gpio_unmask_irq(struct irq_data *d)
 {
 	unsigned int gpio = d->irq - IH_GPIO_BASE;
@@ -1291,6 +1423,7 @@ static struct irq_chip gpio_irq_chip = {
 	.irq_shutdown	= gpio_irq_shutdown,
 	.irq_ack	= gpio_ack_irq,
 	.irq_mask	= gpio_mask_irq,
+	.irq_mask_ack	= gpio_mask_ack_irq,
 	.irq_unmask	= gpio_unmask_irq,
 	.irq_set_type	= gpio_irq_type,
 	.irq_set_wake	= gpio_wake_enable,
@@ -1682,7 +1815,14 @@ static void __init omap_gpio_chip_init(struct gpio_bank *bank)
 			set_irq_chip(j, &gpio_irq_chip);
 		set_irq_handler(j, handle_simple_irq);
 		set_irq_flags(j, IRQF_VALID);
+
 	}
+
+#if defined(CONFIG_IPIPE) && defined(__IPIPE_FEATURE_PIC_MUTE)
+	__ipipe_irqbits[1 << (bank->irq / 32)]
+		|= (1 << (bank->irq % 32));
+#endif
+
 	set_irq_chained_handler(bank->irq, gpio_irq_handler);
 	set_irq_data(bank->irq, bank);
 }
@@ -1753,6 +1893,166 @@ static int __devinit omap_gpio_probe(struct platform_device *pdev)
 	return 0;
 }
 
+#if defined(CONFIG_IPIPE) && defined(__IPIPE_FEATURE_PIC_MUTE)
+
+extern void omap2_intc_mute(void);
+extern void omap2_intc_unmute(void);
+extern void omap3_intc_mute(void);
+extern void omap3_intc_unmute(void);
+extern void omap3_intc_set_irq_prio(int irq, int hi);
+extern void gic_mute(void);
+extern void gic_unmute(void);
+extern void gic_set_irq_prio(int irq, int hi);
+
+void __ipipe_mach_pic_set_irq_prio(int irq, int hi)
+{
+#if defined(CONFIG_ARCH_OMAP3)
+	if (cpu_is_omap34xx())
+		omap3_intc_set_irq_prio(irq, hi);
+#endif /* OMAP3 */
+#if defined(CONFIG_ARCH_OMAP4)
+	if (cpu_is_omap44xx())
+		gic_set_irq_prio(irq, hi);
+#endif /* OMAP4 */
+}
+
+/* Multi OMAP2: run-time selection */
+#ifdef MULTI_OMAP2
+void __ipipe_mach_pic_mute(void)
+{
+#if defined(CONFIG_ARCH_OMAP2420) || defined (CONFIG_ARM_OMAP2430)
+	if (cpu_is_omap24xx())
+		omap2_intc_mute();
+#endif /* OMAP2 */
+#if defined(CONFIG_ARCH_OMAP3)
+	if (cpu_is_omap34xx())
+		omap3_intc_mute();
+#endif /* OMAP3 */
+#if defined(CONFIG_ARCH_OMAP4)
+	if (cpu_is_omap44xx())
+		gic_mute();
+#endif /* OMAP4 */
+}
+
+void __ipipe_mach_pic_unmute(void)
+{
+#if defined(CONFIG_ARCH_OMAP2420) || defined (CONFIG_ARM_OMAP2430)
+	if (cpu_is_omap24xx())
+		omap2_intc_unmute();
+#endif /* OMAP2 */
+#if defined(CONFIG_ARCH_OMAP3)
+	if (cpu_is_omap34xx())
+		omap3_intc_unmute();
+#endif /* OMAP3 */
+#if defined(CONFIG_ARCH_OMAP4)
+	if (cpu_is_omap44xx())
+		gic_unmute();
+#endif /* OMAP4 */
+}
+
+/* OMAP2 */
+#elif defined(CONFIG_ARCH_OMAP2420) || defined (CONFIG_ARM_OMAP2430)
+#define __ipipe_mach_pic_mute omap2_intc_mute
+#define __ipipe_mach_pic_unmute omap2_intc_unmute
+
+/* OMAP3 */
+#elif defined(CONFIG_ARCH_OMAP3)
+#define __ipipe_mach_pic_mute omap3_intc_mute
+#define __ipipe_mach_pic_unmute omap3_intc_unmute
+
+/* OMAP4 */
+#elif defined(CONFIG_ARCH_OMAP4)
+#define __ipipe_mach_pic_mute gic_mute
+#define __ipipe_mach_pic_unmute gic_unmute
+#endif /* OMAP4 */
+
+void __ipipe_mach_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
+{
+	struct irq_desc *desc = irq_to_desc(irq);
+
+	if (desc->chip == &gpio_irq_chip
+#ifdef CONFIG_ARCH_OMAP1
+	    || desc->chip == &mpuio_irq_chip
+#endif
+	    ) {
+		/* It is a gpio. */
+		struct gpio_bank *bank = get_irq_chip_data(irq);
+
+		if (ipd != &ipipe_root) {
+			bank->nonroot_gpios |= (1 << (irq % 32));
+			if (bank->nonroot_gpios == (1 << (irq % 32))) {
+				__ipipe_irqbits[(bank->irq / 32)]
+					&= ~(1 << (bank->irq % 32));
+				__ipipe_mach_pic_set_irq_prio(bank->irq, 1);
+			}
+		}
+	} else
+		__ipipe_mach_pic_set_irq_prio(irq, ipd != &ipipe_root);
+}
+
+void __ipipe_mach_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
+{
+	struct irq_desc *desc = irq_to_desc(irq);
+	if (desc->chip == &gpio_irq_chip
+#ifdef CONFIG_ARCH_OMAP1
+	    || desc->chip == &mpuio_irq_chip
+#endif
+	    ) {
+		/* It is a gpio. */
+		struct gpio_bank *bank = get_irq_chip_data(irq);
+
+		if (ipd != &ipipe_root) {
+			bank->nonroot_gpios &= ~(1 << (irq % 32));
+			if (!bank->nonroot_gpios) {
+				__ipipe_mach_pic_set_irq_prio(bank->irq, 0);
+				__ipipe_irqbits[(bank->irq / 32)]
+					|= (1 << (bank->irq % 32));
+			}
+		}
+	} else if (ipd != &ipipe_root)
+		__ipipe_mach_pic_set_irq_prio(irq, 0);
+}
+
+void ipipe_mute_pic(void)
+{
+	unsigned muted;
+	unsigned i;
+
+	__ipipe_mach_pic_mute();
+
+	for (i = 0; i < gpio_bank_count; i++) {
+		struct gpio_bank *bank = &gpio_bank[i];
+		if (!bank->nonroot_gpios)
+			continue;
+
+		muted = __ipipe_irqbits[IH_GPIO_BASE / 32 + i];
+		if (muted)
+			muted &= _get_gpio_irqbank_mask(bank);
+		__raw_get_cpu_var(__ipipe_muted_irqs)[IH_GPIO_BASE / 32 + i] = muted;
+		if (muted)
+			_enable_gpio_irqbank(bank, muted, 0);
+	}
+}
+
+void ipipe_unmute_pic(void)
+{
+	unsigned muted;
+	unsigned i;
+
+	for (i = 0; i < gpio_bank_count; i++) {
+		struct gpio_bank *bank = &gpio_bank[i];
+		if (!bank->nonroot_gpios)
+			continue;
+
+		muted = __raw_get_cpu_var(__ipipe_muted_irqs)[IH_GPIO_BASE / 32 + i];
+		if (muted)
+			_enable_gpio_irqbank(bank, muted, 1);
+	}
+
+	__ipipe_mach_pic_unmute();
+}
+#endif /* CONFIG_IPIPE && __IPIPE_FEATURE_PIC_MUTE */
+
 #if defined(CONFIG_ARCH_OMAP16XX) || defined(CONFIG_ARCH_OMAP2PLUS)
 static int omap_gpio_suspend(struct sys_device *dev, pm_message_t mesg)
 {
diff --git a/arch/arm/plat-omap/include/plat/dmtimer.h b/arch/arm/plat-omap/include/plat/dmtimer.h
index dfa3aff..1ac8dae 100644
--- a/arch/arm/plat-omap/include/plat/dmtimer.h
+++ b/arch/arm/plat-omap/include/plat/dmtimer.h
@@ -58,6 +58,11 @@ void omap_dm_timer_enable(struct omap_dm_timer *timer);
 void omap_dm_timer_disable(struct omap_dm_timer *timer);
 
 int omap_dm_timer_get_irq(struct omap_dm_timer *timer);
+struct omap_dm_timer *omap_dm_timer_request_specific_nonposted(int timer_id);
+#ifdef CONFIG_IPIPE
+unsigned long omap_dm_timer_get_phys_counter_addr(struct omap_dm_timer *timer);
+unsigned long omap_dm_timer_get_virt_counter_addr(struct omap_dm_timer *timer);
+#endif /* CONFIG_IPIPE */
 
 u32 omap_dm_timer_modify_idlect_mask(u32 inputmask);
 struct clk *omap_dm_timer_get_fclk(struct omap_dm_timer *timer);
diff --git a/arch/arm/plat-omap/include/plat/irqs.h b/arch/arm/plat-omap/include/plat/irqs.h
index 2910de9..c704d0d 100644
--- a/arch/arm/plat-omap/include/plat/irqs.h
+++ b/arch/arm/plat-omap/include/plat/irqs.h
@@ -418,7 +418,16 @@
 #define INTCPS_NR_MIR_REGS	3
 #define INTCPS_NR_IRQS		96
 
+#ifndef CONFIG_ARCH_OMAP1
+#define __IPIPE_FEATURE_PIC_MUTE
+#endif /* OMAP2 || OMAP3 || OMAP4 */
+
 #ifndef __ASSEMBLY__
+
+#if defined(CONFIG_ARCH_OMAP4) && defined(CONFIG_SMP)
+#include <asm/smp_twd.h>
+#endif /* CONFIG_ARCH_OMAP4 */
+
 extern void omap_init_irq(void);
 extern int omap_irq_pending(void);
 void omap_intc_save_context(void);
diff --git a/arch/arm/plat-pxa/gpio.c b/arch/arm/plat-pxa/gpio.c
index e7de6ae..6f704fe 100644
--- a/arch/arm/plat-pxa/gpio.c
+++ b/arch/arm/plat-pxa/gpio.c
@@ -17,6 +17,7 @@
 #include <linux/io.h>
 #include <linux/sysdev.h>
 #include <linux/slab.h>
+#include <linux/ipipe.h>
 
 #include <mach/gpio.h>
 
@@ -39,7 +40,7 @@ struct pxa_gpio_chip {
 #endif
 };
 
-static DEFINE_SPINLOCK(gpio_lock);
+static IPIPE_DEFINE_SPINLOCK(gpio_lock);
 static struct pxa_gpio_chip *pxa_gpio_chips;
 
 #define for_each_gpio_chip(i, c)			\
@@ -220,7 +221,7 @@ static void pxa_gpio_demux_handler(unsigned int irq, struct irq_desc *desc)
 			while (n < BITS_PER_LONG) {
 				loop = 1;
 
-				generic_handle_irq(gpio_to_irq(gpio_base + n));
+				ipipe_handle_chained_irq(gpio_to_irq(gpio_base + n));
 				n = find_next_bit(&gedr, BITS_PER_LONG, n + 1);
 			}
 		}
diff --git a/arch/arm/plat-s3c24xx/irq.c b/arch/arm/plat-s3c24xx/irq.c
index 4434cb5..db41f85 100644
--- a/arch/arm/plat-s3c24xx/irq.c
+++ b/arch/arm/plat-s3c24xx/irq.c
@@ -3,6 +3,8 @@
  * Copyright (c) 2003-2004 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
+ * Copyright (C) 2006, 2007 Sebastian Smolorz <ssmolorz@emlix.com>, emlix GmbH
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
@@ -23,6 +25,7 @@
 #include <linux/interrupt.h>
 #include <linux/ioport.h>
 #include <linux/sysdev.h>
+#include <linux/ipipe.h>
 
 #include <asm/irq.h>
 #include <asm/mach/irq.h>
@@ -87,6 +90,9 @@ struct irq_chip s3c_irq_level_chip = {
 	.name		= "s3c-level",
 	.irq_ack	= s3c_irq_maskack,
 	.irq_mask	= s3c_irq_mask,
+#ifdef CONFIG_IPIPE
+	.irq_mask_ack   = s3c_irq_maskack,
+#endif /* CONFIG_IPIPE */
 	.irq_unmask	= s3c_irq_unmask,
 	.irq_set_wake	= s3c_irq_wake
 };
@@ -273,6 +279,9 @@ s3c_irq_uart0_ack(struct irq_data *data)
 static struct irq_chip s3c_irq_uart0 = {
 	.name		= "s3c-uart0",
 	.irq_mask	= s3c_irq_uart0_mask,
+#ifdef CONFIG_IPIPE
+	.irq_mask_ack   = s3c_irq_uart0_ack,
+#endif /* CONFIG_IPIPE */
 	.irq_unmask	= s3c_irq_uart0_unmask,
 	.irq_ack	= s3c_irq_uart0_ack,
 };
@@ -300,6 +309,9 @@ s3c_irq_uart1_ack(struct irq_data *data)
 static struct irq_chip s3c_irq_uart1 = {
 	.name		= "s3c-uart1",
 	.irq_mask	= s3c_irq_uart1_mask,
+#ifdef CONFIG_IPIPE
+	.irq_mask_ack	= s3c_irq_uart1_ack,
+#endif /* CONFIG_IPIPE */
 	.irq_unmask	= s3c_irq_uart1_unmask,
 	.irq_ack	= s3c_irq_uart1_ack,
 };
@@ -327,6 +339,9 @@ s3c_irq_uart2_ack(struct irq_data *data)
 static struct irq_chip s3c_irq_uart2 = {
 	.name		= "s3c-uart2",
 	.irq_mask	= s3c_irq_uart2_mask,
+#ifdef CONFIG_IPIPE
+	.irq_mask_ack	= s3c_irq_uart2_ack,
+#endif /* CONFIG_IPIPE */
 	.irq_unmask	= s3c_irq_uart2_unmask,
 	.irq_ack	= s3c_irq_uart2_ack,
 };
@@ -377,10 +392,10 @@ static void s3c_irq_demux_adc(unsigned int irq,
 
 	if (subsrc != 0) {
 		if (subsrc & 1) {
-			generic_handle_irq(IRQ_TC);
+			ipipe_handle_chained_irq(IRQ_TC);
 		}
 		if (subsrc & 2) {
-			generic_handle_irq(IRQ_ADC);
+			ipipe_handle_chained_irq(IRQ_ADC);
 		}
 	}
 }
@@ -405,13 +420,13 @@ static void s3c_irq_demux_uart(unsigned int start)
 
 	if (subsrc != 0) {
 		if (subsrc & 1)
-			generic_handle_irq(start);
+			ipipe_handle_chained_irq(start);
 
 		if (subsrc & 2)
-			generic_handle_irq(start+1);
+			ipipe_handle_chained_irq(start+1);
 
 		if (subsrc & 4)
-			generic_handle_irq(start+2);
+			ipipe_handle_chained_irq(start+2);
 	}
 }
 
@@ -458,7 +473,7 @@ s3c_irq_demux_extint8(unsigned int irq,
 		eintpnd &= ~(1<<irq);
 
 		irq += (IRQ_EINT4 - 4);
-		generic_handle_irq(irq);
+		ipipe_handle_chained_irq(irq);
 	}
 
 }
@@ -481,7 +496,7 @@ s3c_irq_demux_extint4t7(unsigned int irq,
 
 		irq += (IRQ_EINT4 - 4);
 
-		generic_handle_irq(irq);
+		ipipe_handle_chained_irq(irq);
 	}
 }
 
@@ -604,7 +619,7 @@ void __init s3c24xx_init_irq(void)
 		default:
 			//irqdbf("registering irq %d (s3c irq)\n", irqno);
 			set_irq_chip(irqno, &s3c_irq_chip);
-			set_irq_handler(irqno, handle_edge_irq);
+			set_irq_handler(irqno, handle_level_irq);
 			set_irq_flags(irqno, IRQF_VALID);
 		}
 	}
@@ -624,14 +639,14 @@ void __init s3c24xx_init_irq(void)
 	for (irqno = IRQ_EINT0; irqno <= IRQ_EINT3; irqno++) {
 		irqdbf("registering irq %d (ext int)\n", irqno);
 		set_irq_chip(irqno, &s3c_irq_eint0t4);
-		set_irq_handler(irqno, handle_edge_irq);
+		set_irq_handler(irqno, handle_level_irq);
 		set_irq_flags(irqno, IRQF_VALID);
 	}
 
 	for (irqno = IRQ_EINT4; irqno <= IRQ_EINT23; irqno++) {
 		irqdbf("registering irq %d (extended s3c irq)\n", irqno);
 		set_irq_chip(irqno, &s3c_irqext_chip);
-		set_irq_handler(irqno, handle_edge_irq);
+		set_irq_handler(irqno, handle_level_irq);
 		set_irq_flags(irqno, IRQF_VALID);
 	}
 
@@ -663,7 +678,7 @@ void __init s3c24xx_init_irq(void)
 	for (irqno = IRQ_TC; irqno <= IRQ_ADC; irqno++) {
 		irqdbf("registering irq %d (s3c adc irq)\n", irqno);
 		set_irq_chip(irqno, &s3c_irq_adc);
-		set_irq_handler(irqno, handle_edge_irq);
+		set_irq_handler(irqno, handle_level_irq);
 		set_irq_flags(irqno, IRQF_VALID);
 	}
 
diff --git a/arch/arm/plat-samsung/include/plat/gpio-core.h b/arch/arm/plat-samsung/include/plat/gpio-core.h
index dac35d0..e1303fe 100644
--- a/arch/arm/plat-samsung/include/plat/gpio-core.h
+++ b/arch/arm/plat-samsung/include/plat/gpio-core.h
@@ -67,7 +67,7 @@ struct s3c_gpio_chip {
 	void __iomem		*base;
 	int			irq_base;
 	int			group;
-	spinlock_t		 lock;
+	ipipe_spinlock_t	lock;
 #ifdef CONFIG_PM
 	u32			pm_save[4];
 #endif
diff --git a/arch/arm/plat-samsung/irq-uart.c b/arch/arm/plat-samsung/irq-uart.c
index 4e77035..2cc6e40 100644
--- a/arch/arm/plat-samsung/irq-uart.c
+++ b/arch/arm/plat-samsung/irq-uart.c
@@ -88,13 +88,13 @@ static void s3c_irq_demux_uart(unsigned int irq, struct irq_desc *desc)
 	int base = uirq->base_irq;
 
 	if (pend & (1 << 0))
-		generic_handle_irq(base);
+		ipipe_handle_chained_irq(base);
 	if (pend & (1 << 1))
-		generic_handle_irq(base + 1);
+		ipipe_handle_chained_irq(base + 1);
 	if (pend & (1 << 2))
-		generic_handle_irq(base + 2);
+		ipipe_handle_chained_irq(base + 2);
 	if (pend & (1 << 3))
-		generic_handle_irq(base + 3);
+		ipipe_handle_chained_irq(base + 3);
 }
 
 static struct irq_chip s3c_irq_uart = {
diff --git a/arch/arm/plat-samsung/time.c b/arch/arm/plat-samsung/time.c
index 2231d80..baf61f6 100644
--- a/arch/arm/plat-samsung/time.c
+++ b/arch/arm/plat-samsung/time.c
@@ -3,6 +3,8 @@
  * Copyright (C) 2003-2005 Simtec Electronics
  *	Ben Dooks, <ben@simtec.co.uk>
  *
+ * Copyright (C) 2006, 2007 Sebastian Smolorz <ssmolorz@emlix.com>, emlix GmbH
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
@@ -27,6 +29,7 @@
 #include <linux/clk.h>
 #include <linux/io.h>
 #include <linux/platform_device.h>
+#include <linux/module.h>
 
 #include <asm/system.h>
 #include <asm/leds.h>
@@ -42,7 +45,6 @@
 #include <plat/clock.h>
 #include <plat/cpu.h>
 
-static unsigned long timer_startval;
 static unsigned long timer_usec_ticks;
 
 #ifndef TICK_MAX
@@ -61,6 +63,35 @@ static unsigned long timer_usec_ticks;
  * Original patch by Dimitry Andric, updated by Ben Dooks
 */
 
+static unsigned long last_free_running_tcnt = 0;
+static unsigned long free_running_tcon = 0;
+static unsigned long timer_lxlost = 0;
+
+#ifdef CONFIG_IPIPE
+unsigned int __ipipe_mach_ticks_per_jiffy;
+EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy);
+
+int __ipipe_mach_timerint = IRQ_TIMER4;
+EXPORT_SYMBOL(__ipipe_mach_timerint);
+
+static unsigned long timer_ackval = 1UL << (IRQ_TIMER4 - IRQ_EINT0);
+
+static struct __ipipe_tscinfo tsc_info = {
+	.type = IPIPE_TSC_TYPE_DECREMENTER,
+	.counter_vaddr = (unsigned long)S3C2410_TCNTO(3),
+	.u = {
+		{
+			.counter_paddr = 0x51000038UL,
+			.mask = 0xffff,
+		},
+	},
+};
+
+static IPIPE_DEFINE_SPINLOCK(timer_lock);
+
+int __ipipe_mach_timerstolen = 0;
+EXPORT_SYMBOL(__ipipe_mach_timerstolen);
+#endif /* CONFIG_IPIPE */
 
 /* timer_mask_usec_ticks
  *
@@ -91,40 +122,45 @@ static inline unsigned long timer_ticks_to_usec(unsigned long ticks)
 	return res >> TIMER_USEC_SHIFT;
 }
 
-/***
- * Returns microsecond  since last clock interrupt.  Note that interrupts
- * will have been disabled by do_gettimeoffset()
- * IRQs are disabled before entering here from do_gettimeofday()
- */
-
-static unsigned long s3c2410_gettimeoffset (void)
+static inline unsigned long timer_freerunning_getvalue(void)
 {
-	unsigned long tdone;
-	unsigned long tval;
-
-	/* work out how many ticks have gone since last timer interrupt */
+	return __raw_readl(S3C2410_TCNTO(3));
+}
 
-	tval =  __raw_readl(S3C2410_TCNTO(4));
-	tdone = timer_startval - tval;
+static inline unsigned long timer_freerunning_getticksoffset(unsigned long tval)
+{
+	long tdone;
 
-	/* check to see if there is an interrupt pending */
+	tdone =  last_free_running_tcnt - tval;
+	if (tdone < 0)
+		tdone += 0x10000;
 
-	if (s3c24xx_ostimer_pending()) {
-		/* re-read the timer, and try and fix up for the missed
-		 * interrupt. Note, the interrupt may go off before the
-		 * timer has re-loaded from wrapping.
-		 */
+	return tdone;
+}
 
-		tval =  __raw_readl(S3C2410_TCNTO(4));
-		tdone = timer_startval - tval;
+static inline unsigned long getticksoffset(void)
+{
+	return timer_freerunning_getticksoffset(timer_freerunning_getvalue());
+}
 
-		if (tval != 0)
-			tdone += timer_startval;
-	}
+#ifdef CONFIG_IPIPE
+static inline unsigned long getticksoffset_tscupdate(void)
+{
+	unsigned long tval;
+	unsigned long ticks;
 
-	return timer_ticks_to_usec(tdone);
+	tval = timer_freerunning_getvalue();
+	ticks = timer_freerunning_getticksoffset(tval);
+	last_free_running_tcnt = tval;
+	__ipipe_tsc_update();
+	return ticks;
 }
-
+#else
+static unsigned long s3c2410_gettimeoffset (void)
+{
+	return timer_ticks_to_usec(timer_lxlost + getticksoffset());
+}
+#endif /* CONFIG_IPIPE */
 
 /*
  * IRQ handler for the timer
@@ -132,6 +168,16 @@ static unsigned long s3c2410_gettimeoffset (void)
 static irqreturn_t
 s3c2410_timer_interrupt(int irq, void *dev_id)
 {
+#ifdef CONFIG_IPIPE
+	timer_lxlost = 0;
+
+	if (!__ipipe_mach_timerstolen) {
+		spin_lock(&timer_lock);
+		getticksoffset_tscupdate();
+		spin_unlock(&timer_lock);
+	}
+#endif /* CONFIG_IPIPE */
+
 	timer_tick();
 	return IRQ_HANDLED;
 }
@@ -153,10 +199,10 @@ static struct clk *tdiv;
 static struct clk *timerclk;
 
 /*
- * Set up timer interrupt, and return the current time in seconds.
+ * Set up timer interrupt.
  *
- * Currently we only use timer4, as it is the only timer which has no
- * other function that can be exploited externally
+ * Currently we use timer4 as event timer and timer3 as tick counter which
+ * permanently counts ticks without interrupt generation.
  */
 static void s3c2410_timer_setup (void)
 {
@@ -164,6 +210,7 @@ static void s3c2410_timer_setup (void)
 	unsigned long tcnt;
 	unsigned long tcfg1;
 	unsigned long tcfg0;
+	unsigned long intmask;
 
 	tcnt = TICK_MAX;  /* default value for tcnt */
 
@@ -175,8 +222,8 @@ static void s3c2410_timer_setup (void)
 		tcnt = 12000000 / HZ;
 
 		tcfg1 = __raw_readl(S3C2410_TCFG1);
-		tcfg1 &= ~S3C2410_TCFG1_MUX4_MASK;
-		tcfg1 |= S3C2410_TCFG1_MUX4_TCLK1;
+		tcfg1 &= ~(S3C2410_TCFG1_MUX4_MASK | S3C2410_TCFG1_MUX3_MASK);
+		tcfg1 |= (S3C2410_TCFG1_MUX4_TCLK1 | S3C2410_TCFG1_MUX3_TCLK1);
 		__raw_writel(tcfg1, S3C2410_TCFG1);
 	} else {
 		unsigned long pclk;
@@ -210,6 +257,12 @@ static void s3c2410_timer_setup (void)
 	tcfg0 = __raw_readl(S3C2410_TCFG0);
 	tcfg1 = __raw_readl(S3C2410_TCFG1);
 
+#ifdef CONFIG_IPIPE
+	__ipipe_mach_ticks_per_jiffy = tcnt;
+	tsc_info.freq = tcnt * HZ;
+	__ipipe_tsc_register(&tsc_info);
+#endif /* CONFIG_IPIPE */
+
 	/* timers reload after counting zero, so reduce the count by 1 */
 
 	tcnt--;
@@ -226,23 +279,37 @@ static void s3c2410_timer_setup (void)
 	__raw_writel(tcfg1, S3C2410_TCFG1);
 	__raw_writel(tcfg0, S3C2410_TCFG0);
 
-	timer_startval = tcnt;
-	__raw_writel(tcnt, S3C2410_TCNTB(4));
-
-	/* ensure timer is stopped... */
+	/* ensure timers are stopped... */
+	tcon &= ~(0x3f<<17);
+	__raw_writel(tcon, S3C2410_TCON);
 
-	tcon &= ~(7<<20);
-	tcon |= S3C2410_TCON_T4RELOAD;
-	tcon |= S3C2410_TCON_T4MANUALUPD;
+	/* Mask timer3 interrupt. */
+	intmask = __raw_readl(S3C2410_INTMSK);
+	intmask |= 1UL << (IRQ_TIMER3 - IRQ_EINT0);
+	__raw_writel(intmask, S3C2410_INTMSK);
 
-	__raw_writel(tcon, S3C2410_TCON);
+	/* Set timer values */
 	__raw_writel(tcnt, S3C2410_TCNTB(4));
 	__raw_writel(tcnt, S3C2410_TCMPB(4));
+	__raw_writel(0xffff, S3C2410_TCNTB(3));
+	__raw_writel(0xffff, S3C2410_TCMPB(3));
+
+	/* Set base tcon value for later programming of timer 4 by Xenomai. */
+	free_running_tcon = tcon |  S3C2410_TCON_T3RELOAD | S3C2410_TCON_T3START;
 
-	/* start the timer running */
-	tcon |= S3C2410_TCON_T4START;
-	tcon &= ~S3C2410_TCON_T4MANUALUPD;
+	/* Set auto reloads for both timers. */
+	tcon |= S3C2410_TCON_T3RELOAD | S3C2410_TCON_T4RELOAD;
+
+	/* Manual update */
+	__raw_writel(tcon | S3C2410_TCON_T3MANUALUPD
+			  | S3C2410_TCON_T4MANUALUPD, S3C2410_TCON);
+
+	tcon |= S3C2410_TCON_T3START | S3C2410_TCON_T4START;
+	/* Start timers.*/
 	__raw_writel(tcon, S3C2410_TCON);
+
+	/* Save start value of timer 3 as begining of first period. */
+	last_free_running_tcnt = 0xffff;
 }
 
 static void __init s3c2410_timer_resources(void)
@@ -280,6 +347,49 @@ static void __init s3c2410_timer_init(void)
 
 struct sys_timer s3c24xx_timer = {
 	.init		= s3c2410_timer_init,
+#ifndef CONFIG_IPIPE
 	.offset		= s3c2410_gettimeoffset,
+#endif
 	.resume		= s3c2410_timer_setup
 };
+
+#ifdef CONFIG_IPIPE
+void __ipipe_mach_acktimer(void)
+{
+	__raw_writel(timer_ackval, S3C2410_SRCPND);
+	__raw_writel(timer_ackval, S3C2410_INTPND);
+}
+
+static inline void set_dec(unsigned long reload)
+{
+	__raw_writel(reload, S3C2410_TCNTB(4));
+	/* Manual update */
+	__raw_writel(free_running_tcon | S3C2410_TCON_T4MANUALUPD, S3C2410_TCON);
+	/* Start timer */
+	__raw_writel(free_running_tcon | S3C2410_TCON_T4START, S3C2410_TCON);
+}
+
+void __ipipe_mach_set_dec(unsigned long reload)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&timer_lock, flags);
+	timer_lxlost += getticksoffset_tscupdate();
+	set_dec(reload);
+	spin_unlock_irqrestore(&timer_lock, flags);
+}
+EXPORT_SYMBOL(__ipipe_mach_set_dec);
+
+void __ipipe_mach_release_timer(void)
+{
+	free_running_tcon |= S3C2410_TCON_T4RELOAD;
+	__ipipe_mach_set_dec(__ipipe_mach_ticks_per_jiffy - 1);
+	free_running_tcon &= ~S3C2410_TCON_T4RELOAD;
+}
+EXPORT_SYMBOL(__ipipe_mach_release_timer);
+
+unsigned long __ipipe_mach_get_dec(void)
+{
+	return __raw_readl(S3C2410_TCNTO(4));
+}
+#endif /* CONFIG_IPIPE */
diff --git a/arch/arm/vfp/entry.S b/arch/arm/vfp/entry.S
index 4fa9903..e46bae3 100644
--- a/arch/arm/vfp/entry.S
+++ b/arch/arm/vfp/entry.S
@@ -25,7 +25,6 @@ ENTRY(do_vfp)
 	add	r11, r4, #1		@ increment it
 	str	r11, [r10, #TI_PREEMPT]
 #endif
-	enable_irq
  	ldr	r4, .LCvfp
 	ldr	r11, [r10, #TI_CPU]	@ CPU number
 	add	r10, r10, #TI_VFPSTATE	@ r10 = workspace
@@ -33,6 +32,7 @@ ENTRY(do_vfp)
 ENDPROC(do_vfp)
 
 ENTRY(vfp_null_entry)
+	enable_irq
 #ifdef CONFIG_PREEMPT
 	get_thread_info	r10
 	ldr	r4, [r10, #TI_PREEMPT]	@ get preempt count
@@ -51,6 +51,7 @@ ENDPROC(vfp_null_entry)
 
 	__INIT
 ENTRY(vfp_testing_entry)
+	enable_irq
 #ifdef CONFIG_PREEMPT
 	get_thread_info	r10
 	ldr	r4, [r10, #TI_PREEMPT]	@ get preempt count
diff --git a/arch/arm/vfp/vfphw.S b/arch/arm/vfp/vfphw.S
index 9897dcf..de4ae09 100644
--- a/arch/arm/vfp/vfphw.S
+++ b/arch/arm/vfp/vfphw.S
@@ -19,7 +19,7 @@
 #include "../kernel/entry-header.S"
 
 	.macro	DBGSTR, str
-#ifdef DEBUG
+#if defined(DEBUG) && !defined(CONFIG_IPIPE)
 	stmfd	sp!, {r0-r3, ip, lr}
 	add	r0, pc, #4
 	bl	printk
@@ -31,7 +31,7 @@
 	.endm
 
 	.macro  DBGSTR1, str, arg
-#ifdef DEBUG
+#if defined(DEBUG) && !defined(CONFIG_IPIPE)
 	stmfd	sp!, {r0-r3, ip, lr}
 	mov	r1, \arg
 	add	r0, pc, #4
@@ -44,7 +44,7 @@
 	.endm
 
 	.macro  DBGSTR3, str, arg1, arg2, arg3
-#ifdef DEBUG
+#if defined(DEBUG) && !defined(CONFIG_IPIPE)
 	stmfd	sp!, {r0-r3, ip, lr}
 	mov	r3, \arg3
 	mov	r2, \arg2
@@ -87,6 +87,14 @@ ENTRY(vfp_support_entry)
 					@ still there.  In this case, we do
 					@ not want to drop a pending exception.
 
+	enable_irq
+#ifdef CONFIG_IPIPE
+	disable_irq
+#ifdef CONFIG_SMP
+	mrc	p15, 0, r11, c0, c0, 5  @ reload current CPU number
+#endif
+	ldr	r4, [r3, r11, lsl #2]	@ reload last_VFP_context pointer
+#endif
 	VFPFMXR	FPEXC, r5		@ enable VFP, disable any pending
 					@ exceptions, so we can get at the
 					@ rest of it
@@ -139,6 +147,7 @@ check_for_exception:
 					@ out before setting an FPEXC that
 					@ stops us reading stuff
 	VFPFMXR	FPEXC, r1		@ restore FPEXC last
+	enable_irq_cond
 	sub	r2, r2, #4
 	str	r2, [sp, #S_PC]		@ retry the instruction
 #ifdef CONFIG_PREEMPT
@@ -164,6 +173,7 @@ look_for_VFP_exceptions:
 	@ Fall into hand on to next handler - appropriate coproc instr
 	@ not recognised by VFP
 
+	enable_irq_cond
 	DBGSTR	"not VFP"
 #ifdef CONFIG_PREEMPT
 	get_thread_info	r10
diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c
index 0797cb5..94ce836 100644
--- a/arch/arm/vfp/vfpmodule.c
+++ b/arch/arm/vfp/vfpmodule.c
@@ -48,6 +48,7 @@ unsigned int VFP_arch;
 static void vfp_thread_flush(struct thread_info *thread)
 {
 	union vfp_state *vfp = &thread->vfpstate;
+	unsigned long flags;
 	unsigned int cpu;
 
 	memset(vfp, 0, sizeof(union vfp_state));
@@ -60,22 +61,23 @@ static void vfp_thread_flush(struct thread_info *thread)
 	 * that the modification of last_VFP_context[] and hardware disable
 	 * are done for the same CPU and without preemption.
 	 */
-	cpu = get_cpu();
+	cpu = ipipe_get_cpu(flags);
 	if (last_VFP_context[cpu] == vfp)
 		last_VFP_context[cpu] = NULL;
 	fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
-	put_cpu();
+	ipipe_put_cpu(flags);
 }
 
 static void vfp_thread_exit(struct thread_info *thread)
 {
 	/* release case: Per-thread VFP cleanup. */
 	union vfp_state *vfp = &thread->vfpstate;
-	unsigned int cpu = get_cpu();
+	unsigned long flags;
+	unsigned int cpu = ipipe_get_cpu(flags);
 
 	if (last_VFP_context[cpu] == vfp)
 		last_VFP_context[cpu] = NULL;
-	put_cpu();
+	ipipe_put_cpu(flags);
 }
 
 /*
@@ -104,12 +106,16 @@ static void vfp_thread_exit(struct thread_info *thread)
 static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
 {
 	struct thread_info *thread = v;
+	unsigned long flags;
+	unsigned int cpu;
 
 	if (likely(cmd == THREAD_NOTIFY_SWITCH)) {
-		u32 fpexc = fmrx(FPEXC);
+		u32 fpexc;
 
+		local_irq_save_hw_cond(flags);
+		fpexc = fmrx(FPEXC);
 #ifdef CONFIG_SMP
-		unsigned int cpu = thread->cpu;
+		cpu = thread->cpu;
 
 		/*
 		 * On SMP, if VFP is enabled, save the old state in
@@ -127,6 +133,8 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
 		 */
 		if (thread->vfpstate.hard.cpu != cpu)
 			last_VFP_context[cpu] = NULL;
+#else
+		(void)cpu;
 #endif
 
 		/*
@@ -134,6 +142,7 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
 		 * old state.
 		 */
 		fmxr(FPEXC, fpexc & ~FPEXC_EN);
+		local_irq_restore_hw_cond(flags);
 		return NOTIFY_DONE;
 	}
 
@@ -270,7 +279,7 @@ static u32 vfp_emulate_instruction(u32 inst, u32 fpscr, struct pt_regs *regs)
  */
 void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
 {
-	u32 fpscr, orig_fpscr, fpsid, exceptions;
+	u32 fpscr, orig_fpscr, fpsid, exceptions, next_trigger = 0;
 
 	pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc);
 
@@ -300,6 +309,7 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
 		/*
 		 * Synchronous exception, emulate the trigger instruction
 		 */
+		local_irq_enable_hw_cond();
 		goto emulate;
 	}
 
@@ -312,7 +322,18 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
 		trigger = fmrx(FPINST);
 		regs->ARM_pc -= 4;
 #endif
-	} else if (!(fpexc & FPEXC_DEX)) {
+		if (fpexc & FPEXC_FP2V) {
+			/*
+			 * The barrier() here prevents fpinst2 being read
+			 * before the condition above.
+			 */
+			barrier();
+			next_trigger = fmrx(FPINST2);
+		}
+	}
+	local_irq_enable_hw_cond();
+
+	if (!(fpexc & (FPEXC_EX | FPEXC_DEX))) {
 		/*
 		 * Illegal combination of bits. It can be caused by an
 		 * unallocated VFP instruction but with FPSCR.IXE set and not
@@ -352,18 +373,14 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
 	if (fpexc ^ (FPEXC_EX | FPEXC_FP2V))
 		goto exit;
 
-	/*
-	 * The barrier() here prevents fpinst2 being read
-	 * before the condition above.
-	 */
-	barrier();
-	trigger = fmrx(FPINST2);
+	trigger = next_trigger;
 
  emulate:
 	exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs);
 	if (exceptions)
 		vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);
  exit:
+	local_irq_enable_hw_cond();
 	preempt_enable();
 }
 
@@ -434,7 +451,8 @@ static inline void vfp_pm_init(void) { }
 
 void vfp_sync_hwstate(struct thread_info *thread)
 {
-	unsigned int cpu = get_cpu();
+	unsigned long flags;
+	unsigned int cpu = ipipe_get_cpu(flags);
 
 	/*
 	 * If the thread we're interested in is the current owner of the
@@ -451,12 +469,13 @@ void vfp_sync_hwstate(struct thread_info *thread)
 		fmxr(FPEXC, fpexc);
 	}
 
-	put_cpu();
+	ipipe_put_cpu(flags);
 }
 
 void vfp_flush_hwstate(struct thread_info *thread)
 {
-	unsigned int cpu = get_cpu();
+	unsigned long flags;
+	unsigned int cpu = ipipe_get_cpu(flags);
 
 	/*
 	 * If the thread we're interested in is the current owner of the
@@ -484,7 +503,7 @@ void vfp_flush_hwstate(struct thread_info *thread)
 	 */
 	thread->vfpstate.hard.cpu = NR_CPUS;
 #endif
-	put_cpu();
+	ipipe_put_cpu(flags);
 }
 
 /*
diff --git a/drivers/mfd/twl4030-irq.c b/drivers/mfd/twl4030-irq.c
index 63a30e8..729943d 100644
--- a/drivers/mfd/twl4030-irq.c
+++ b/drivers/mfd/twl4030-irq.c
@@ -336,7 +336,11 @@ static int twl4030_irq_thread(void *data)
 					note_interrupt(module_irq, d,
 							IRQ_NONE);
 				else
+#ifndef CONFIG_IPIPE
 					d->handle_irq(module_irq, d);
+#else
+					d->ipipe_ack(module_irq, d);
+#endif
 			}
 		}
 		local_irq_enable();
diff --git a/drivers/mfd/twl6030-irq.c b/drivers/mfd/twl6030-irq.c
index 4082ed7..2c14628 100644
--- a/drivers/mfd/twl6030-irq.c
+++ b/drivers/mfd/twl6030-irq.c
@@ -155,7 +155,11 @@ static int twl6030_irq_thread(void *data)
 					note_interrupt(module_irq, d,
 							IRQ_NONE);
 				else
+#ifndef CONFIG_IPIPE
 					d->handle_irq(module_irq, d);
+#else
+					d->ipipe_ack(module_irq, d);
+#endif
 
 			}
 		local_irq_enable();
@@ -359,6 +363,7 @@ int twl6030_init_irq(int irq_num, unsigned irq_base, unsigned irq_end)
 		pr_err("twl6030: could not claim irq%d: %d\n", irq_num, status);
 		goto fail_irq;
 	}
+
 	return status;
 fail_irq:
 	free_irq(irq_num, &irq_event);
@@ -378,4 +383,3 @@ int twl6030_exit_irq(void)
 	}
 	return 0;
 }
-
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
index cc8e49d..2a5f4fa 100644
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -81,7 +81,7 @@ config ATMEL_TCLIB
 
 config ATMEL_TCB_CLKSRC
 	bool "TC Block Clocksource"
-	depends on ATMEL_TCLIB
+	depends on ATMEL_TCLIB && !IPIPE
 	default y
 	help
 	  Select this to get a high precision clocksource based on a
@@ -344,14 +344,14 @@ config SENSORS_BH1780
 	  will be called bh1780gli.
 
 config SENSORS_BH1770
-         tristate "BH1770GLC / SFH7770 combined ALS - Proximity sensor"
-         depends on I2C
-         ---help---
-           Say Y here if you want to build a driver for BH1770GLC (ROHM) or
+	 tristate "BH1770GLC / SFH7770 combined ALS - Proximity sensor"
+	 depends on I2C
+	 ---help---
+	   Say Y here if you want to build a driver for BH1770GLC (ROHM) or
 	   SFH7770 (Osram) combined ambient light and proximity sensor chip.
 
-           To compile this driver as a module, choose M here: the
-           module will be called bh1770glc. If unsure, say N here.
+	   To compile this driver as a module, choose M here: the
+	   module will be called bh1770glc. If unsure, say N here.
 
 config SENSORS_APDS990X
 	 tristate "APDS990X combined als and proximity sensors"
diff --git a/drivers/pci/htirq.c b/drivers/pci/htirq.c
index 834842a..86f2406 100644
--- a/drivers/pci/htirq.c
+++ b/drivers/pci/htirq.c
@@ -20,7 +20,7 @@
  * With multiple simultaneous hypertransport irq devices it might pay
  * to make this more fine grained.  But start with simple, stupid, and correct.
  */
-static DEFINE_SPINLOCK(ht_irq_lock);
+static IPIPE_DEFINE_SPINLOCK(ht_irq_lock);
 
 struct ht_irq_cfg {
 	struct pci_dev *dev;
diff --git a/fs/exec.c b/fs/exec.c
index 52a447d..4bd63f3 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -771,6 +771,7 @@ static int exec_mmap(struct mm_struct *mm)
 {
 	struct task_struct *tsk;
 	struct mm_struct * old_mm, *active_mm;
+	unsigned long flags;
 
 	/* Notify parent that we're no longer interested in the old VM */
 	tsk = current;
@@ -794,12 +795,14 @@ static int exec_mmap(struct mm_struct *mm)
 	task_lock(tsk);
 	active_mm = tsk->active_mm;
 	tsk->mm = mm;
+	ipipe_mm_switch_protect(flags);
 	tsk->active_mm = mm;
 	activate_mm(active_mm, mm);
 	if (old_mm && tsk->signal->oom_score_adj == OOM_SCORE_ADJ_MIN) {
 		atomic_dec(&old_mm->oom_disable_count);
 		atomic_inc(&tsk->mm->oom_disable_count);
 	}
+	ipipe_mm_switch_unprotect(flags);
 	task_unlock(tsk);
 	arch_pick_mmap_layout(mm);
 	if (old_mm) {
diff --git a/fs/proc/array.c b/fs/proc/array.c
index 5e4f776..a18c14f 100644
--- a/fs/proc/array.c
+++ b/fs/proc/array.c
@@ -142,6 +142,10 @@ static const char *task_state_array[] = {
 	"x (dead)",		/*  64 */
 	"K (wakekill)",		/* 128 */
 	"W (waking)",		/* 256 */
+#ifdef CONFIG_IPIPE
+	"A (atomic switch)",	/* 512 */
+	"N (wakeup disabled)",	/* 1024 */
+#endif
 };
 
 static inline const char *get_task_state(struct task_struct *tsk)
diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h
index e994197..4914a26 100644
--- a/include/asm-generic/atomic.h
+++ b/include/asm-generic/atomic.h
@@ -58,11 +58,11 @@ static inline int atomic_add_return(int i, atomic_t *v)
 	unsigned long flags;
 	int temp;
 
-	raw_local_irq_save(flags); /* Don't trace it in an irqsoff handler */
+	local_irq_save_hw(flags); /* Don't trace it in an irqsoff handler */
 	temp = v->counter;
 	temp += i;
 	v->counter = temp;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return temp;
 }
@@ -79,11 +79,11 @@ static inline int atomic_sub_return(int i, atomic_t *v)
 	unsigned long flags;
 	int temp;
 
-	raw_local_irq_save(flags); /* Don't trace it in an irqsoff handler */
+	local_irq_save_hw(flags);
 	temp = v->counter;
 	temp -= i;
 	v->counter = temp;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return temp;
 }
@@ -145,9 +145,9 @@ static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
 	unsigned long flags;
 
 	mask = ~mask;
-	raw_local_irq_save(flags); /* Don't trace it in a irqsoff handler */
+	local_irq_save_hw(flags);
 	*addr &= mask;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 }
 
 /* Assume that atomic operations are already serializing */
diff --git a/include/asm-generic/bitops/atomic.h b/include/asm-generic/bitops/atomic.h
index ecc44a8..5caf6e9 100644
--- a/include/asm-generic/bitops/atomic.h
+++ b/include/asm-generic/bitops/atomic.h
@@ -21,20 +21,20 @@ extern arch_spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] __lock_aligned;
  * this is the substitute */
 #define _atomic_spin_lock_irqsave(l,f) do {	\
 	arch_spinlock_t *s = ATOMIC_HASH(l);	\
-	local_irq_save(f);			\
+	local_irq_save_hw(f);			\
 	arch_spin_lock(s);			\
 } while(0)
 
 #define _atomic_spin_unlock_irqrestore(l,f) do {	\
 	arch_spinlock_t *s = ATOMIC_HASH(l);		\
 	arch_spin_unlock(s);				\
-	local_irq_restore(f);				\
+	local_irq_restore_hw(f);			\
 } while(0)
 
 
 #else
-#  define _atomic_spin_lock_irqsave(l,f) do { local_irq_save(f); } while (0)
-#  define _atomic_spin_unlock_irqrestore(l,f) do { local_irq_restore(f); } while (0)
+#  define _atomic_spin_lock_irqsave(l,f) do { local_irq_save_hw(f); } while (0)
+#  define _atomic_spin_unlock_irqrestore(l,f) do { local_irq_restore_hw(f); } while (0)
 #endif
 
 /*
diff --git a/include/asm-generic/cmpxchg-local.h b/include/asm-generic/cmpxchg-local.h
index 2533fdd..41865db 100644
--- a/include/asm-generic/cmpxchg-local.h
+++ b/include/asm-generic/cmpxchg-local.h
@@ -21,7 +21,7 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr,
 	if (size == 8 && sizeof(unsigned long) != 8)
 		wrong_size_cmpxchg(ptr);
 
-	local_irq_save(flags);
+	local_irq_save_hw(flags);
 	switch (size) {
 	case 1: prev = *(u8 *)ptr;
 		if (prev == old)
@@ -42,7 +42,7 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr,
 	default:
 		wrong_size_cmpxchg(ptr);
 	}
-	local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 	return prev;
 }
 
@@ -55,11 +55,11 @@ static inline u64 __cmpxchg64_local_generic(volatile void *ptr,
 	u64 prev;
 	unsigned long flags;
 
-	local_irq_save(flags);
+	local_irq_save_hw(flags);
 	prev = *(u64 *)ptr;
 	if (prev == old)
 		*(u64 *)ptr = new;
-	local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 	return prev;
 }
 
diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h
index d17784e..d2a6f27 100644
--- a/include/asm-generic/percpu.h
+++ b/include/asm-generic/percpu.h
@@ -67,6 +67,20 @@ extern unsigned long __per_cpu_offset[NR_CPUS];
 
 #define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
 #define __raw_get_cpu_var(var) (*__this_cpu_ptr(&(var)))
+#ifdef CONFIG_IPIPE
+#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP)
+extern int __ipipe_check_percpu_access(void);
+#define __ipipe_local_cpu_offset				\
+	({							\
+		WARN_ON_ONCE(__ipipe_check_percpu_access());	\
+		__my_cpu_offset;				\
+	})
+#else
+#define __ipipe_local_cpu_offset  __my_cpu_offset
+#endif
+#define __ipipe_get_cpu_var(var) \
+	(*SHIFT_PERCPU_PTR(&(var), __ipipe_local_cpu_offset))
+#endif /* CONFIG_IPIPE */
 
 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
 extern void setup_per_cpu_areas(void);
@@ -82,6 +96,7 @@ extern void setup_per_cpu_areas(void);
 #define per_cpu(var, cpu)	(*((void)(cpu), VERIFY_PERCPU_PTR(&(var))))
 #define __get_cpu_var(var)	(*VERIFY_PERCPU_PTR(&(var)))
 #define __raw_get_cpu_var(var)	(*VERIFY_PERCPU_PTR(&(var)))
+#define __ipipe_get_cpu_var(var) __raw_get_cpu_var(var)
 #define this_cpu_ptr(ptr)	per_cpu_ptr(ptr, 0)
 #define __this_cpu_ptr(ptr)	this_cpu_ptr(ptr)
 
diff --git a/include/asm-generic/resource.h b/include/asm-generic/resource.h
index 587566f..66d60cd 100644
--- a/include/asm-generic/resource.h
+++ b/include/asm-generic/resource.h
@@ -58,6 +58,14 @@
 #endif
 
 /*
+ * Limit the stack by to some sane default: root can always
+ * increase this limit if needed..  8MB seems reasonable.
+ */
+#ifndef _STK_LIM
+# define _STK_LIM		(8*1024*1024)
+#endif
+
+/*
  * RLIMIT_STACK default maximum - some architectures override it:
  */
 #ifndef _STK_LIM_MAX
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index 32f9fd6..0fcf21b 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -217,6 +217,7 @@ extern void irq_exit(void);
 
 #define nmi_enter()						\
 	do {							\
+		ipipe_nmi_enter();				\
 		ftrace_nmi_enter();				\
 		BUG_ON(in_nmi());				\
 		add_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET);	\
@@ -233,6 +234,7 @@ extern void irq_exit(void);
 		BUG_ON(!in_nmi());				\
 		sub_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET);	\
 		ftrace_nmi_exit();				\
+		ipipe_nmi_exit();				\
 	} while (0)
 
 #endif /* LINUX_HARDIRQ_H */
diff --git a/include/linux/ipipe.h b/include/linux/ipipe.h
new file mode 100644
index 0000000..aff6c3c
--- /dev/null
+++ b/include/linux/ipipe.h
@@ -0,0 +1,744 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe.h
+ *
+ * Copyright (C) 2002-2007 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_H
+#define __LINUX_IPIPE_H
+
+#include <linux/spinlock.h>
+#include <linux/cache.h>
+#include <linux/percpu.h>
+#include <linux/mutex.h>
+#include <linux/linkage.h>
+#include <linux/ipipe_base.h>
+#include <asm/ipipe.h>
+#include <asm/bug.h>
+
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+
+#include <linux/cpumask.h>
+#include <asm/system.h>
+
+static inline int ipipe_disable_context_check(int cpu)
+{
+	return xchg(&per_cpu(ipipe_percpu_context_check, cpu), 0);
+}
+
+static inline void ipipe_restore_context_check(int cpu, int old_state)
+{
+	per_cpu(ipipe_percpu_context_check, cpu) = old_state;
+}
+
+static inline void ipipe_context_check_off(void)
+{
+	int cpu;
+	for_each_online_cpu(cpu)
+		per_cpu(ipipe_percpu_context_check, cpu) = 0;
+}
+
+#else	/* !CONFIG_IPIPE_DEBUG_CONTEXT */
+
+static inline int ipipe_disable_context_check(int cpu)
+{
+	return 0;
+}
+
+static inline void ipipe_restore_context_check(int cpu, int old_state) { }
+
+static inline void ipipe_context_check_off(void) { }
+
+#endif	/* !CONFIG_IPIPE_DEBUG_CONTEXT */
+
+#ifdef CONFIG_IPIPE_DEBUG_INTERNAL
+#define IPIPE_WARN(c)		WARN_ON(c)
+#define IPIPE_WARN_ONCE(c)	WARN_ON_ONCE(c)
+#else
+#define IPIPE_WARN(c)		do { (void)(c); } while (0)
+#define IPIPE_WARN_ONCE(c)	do { (void)(c); } while (0)
+#endif
+
+#ifdef CONFIG_IPIPE
+
+#define IPIPE_VERSION_STRING	IPIPE_ARCH_STRING
+#define IPIPE_RELEASE_NUMBER	((IPIPE_MAJOR_NUMBER << 16) | \
+				 (IPIPE_MINOR_NUMBER <<  8) | \
+				 (IPIPE_PATCH_NUMBER))
+
+#ifndef BROKEN_BUILTIN_RETURN_ADDRESS
+#define __BUILTIN_RETURN_ADDRESS0 ((unsigned long)__builtin_return_address(0))
+#define __BUILTIN_RETURN_ADDRESS1 ((unsigned long)__builtin_return_address(1))
+#endif /* !BUILTIN_RETURN_ADDRESS */
+
+#define IPIPE_ROOT_PRIO		100
+#define IPIPE_ROOT_ID		0
+#define IPIPE_ROOT_NPTDKEYS	4	/* Must be <= BITS_PER_LONG */
+
+#define IPIPE_RESET_TIMER	0x1
+#define IPIPE_GRAB_TIMER	0x2
+
+/* Global domain flags */
+#define IPIPE_SPRINTK_FLAG	0	/* Synchronous printk() allowed */
+#define IPIPE_AHEAD_FLAG	1	/* Domain always heads the pipeline */
+
+/* Interrupt control bits */
+#define IPIPE_HANDLE_FLAG	0
+#define IPIPE_PASS_FLAG		1
+#define IPIPE_ENABLE_FLAG	2
+#define IPIPE_DYNAMIC_FLAG	IPIPE_HANDLE_FLAG
+#define IPIPE_STICKY_FLAG	3
+#define IPIPE_SYSTEM_FLAG	4
+#define IPIPE_LOCK_FLAG		5
+#define IPIPE_WIRED_FLAG	6
+#define IPIPE_EXCLUSIVE_FLAG	7
+
+#define IPIPE_HANDLE_MASK	(1 << IPIPE_HANDLE_FLAG)
+#define IPIPE_PASS_MASK		(1 << IPIPE_PASS_FLAG)
+#define IPIPE_ENABLE_MASK	(1 << IPIPE_ENABLE_FLAG)
+#define IPIPE_DYNAMIC_MASK	IPIPE_HANDLE_MASK
+#define IPIPE_STICKY_MASK	(1 << IPIPE_STICKY_FLAG)
+#define IPIPE_SYSTEM_MASK	(1 << IPIPE_SYSTEM_FLAG)
+#define IPIPE_LOCK_MASK		(1 << IPIPE_LOCK_FLAG)
+#define IPIPE_WIRED_MASK	(1 << IPIPE_WIRED_FLAG)
+#define IPIPE_EXCLUSIVE_MASK	(1 << IPIPE_EXCLUSIVE_FLAG)
+
+#define IPIPE_DEFAULT_MASK	(IPIPE_HANDLE_MASK|IPIPE_PASS_MASK)
+#define IPIPE_STDROOT_MASK	(IPIPE_HANDLE_MASK|IPIPE_PASS_MASK|IPIPE_SYSTEM_MASK)
+
+#define IPIPE_EVENT_SELF        0x80000000
+
+#define IPIPE_NR_CPUS		NR_CPUS
+
+/* This accessor assumes hw IRQs are off on SMP; allows assignment. */
+#define __ipipe_current_domain	__ipipe_get_cpu_var(ipipe_percpu_domain)
+/* This read-only accessor makes sure that hw IRQs are off on SMP. */
+#define ipipe_current_domain				\
+	({						\
+		struct ipipe_domain *__ipd__;		\
+		unsigned long __flags__;		\
+		local_irq_save_hw_smp(__flags__);	\
+		__ipd__ = __ipipe_current_domain;	\
+		local_irq_restore_hw_smp(__flags__);	\
+		__ipd__;				\
+	})
+
+#define ipipe_virtual_irq_p(irq)	((irq) >= IPIPE_VIRQ_BASE && \
+					 (irq) < IPIPE_NR_IRQS)
+
+#define IPIPE_SAME_HANDLER	((ipipe_irq_handler_t)(-1))
+
+struct irq_desc;
+
+typedef void (*ipipe_irq_ackfn_t)(unsigned irq, struct irq_desc *desc);
+
+typedef int (*ipipe_event_handler_t)(unsigned event,
+				     struct ipipe_domain *from,
+				     void *data);
+struct ipipe_domain {
+
+	int slot;			/* Slot number in percpu domain data array. */
+	struct list_head p_link;	/* Link in pipeline */
+	ipipe_event_handler_t evhand[IPIPE_NR_EVENTS]; /* Event handlers. */
+	unsigned long long evself;	/* Self-monitored event bits. */
+
+	struct ipipe_irqdesc {
+		unsigned long control;
+		ipipe_irq_ackfn_t acknowledge;
+		ipipe_irq_handler_t handler;
+		void *cookie;
+	} ____cacheline_aligned irqs[IPIPE_NR_IRQS];
+
+	int priority;
+	void *pdd;
+	unsigned long flags;
+	unsigned domid;
+	const char *name;
+	struct mutex mutex;
+};
+
+#define IPIPE_HEAD_PRIORITY	(-1) /* For domains always heading the pipeline */
+
+struct ipipe_domain_attr {
+
+	unsigned domid;		/* Domain identifier -- Magic value set by caller */
+	const char *name;	/* Domain name -- Warning: won't be dup'ed! */
+	int priority;		/* Priority in interrupt pipeline */
+	void (*entry) (void);	/* Domain entry point */
+	void *pdd;		/* Per-domain (opaque) data pointer */
+};
+
+#define __ipipe_irq_cookie(ipd, irq)		(ipd)->irqs[irq].cookie
+#define __ipipe_irq_handler(ipd, irq)		(ipd)->irqs[irq].handler
+#define __ipipe_cpudata_irq_hits(ipd, cpu, irq)	ipipe_percpudom(ipd, irqall, cpu)[irq]
+
+extern unsigned __ipipe_printk_virq;
+
+extern unsigned long __ipipe_virtual_irq_map;
+
+extern struct list_head __ipipe_pipeline;
+
+extern int __ipipe_event_monitors[];
+
+/* Private interface */
+
+void ipipe_init_early(void);
+
+void ipipe_init(void);
+
+#ifdef CONFIG_PROC_FS
+void ipipe_init_proc(void);
+
+#ifdef CONFIG_IPIPE_TRACE
+void __ipipe_init_tracer(void);
+#else /* !CONFIG_IPIPE_TRACE */
+#define __ipipe_init_tracer()       do { } while(0)
+#endif /* CONFIG_IPIPE_TRACE */
+
+#else	/* !CONFIG_PROC_FS */
+#define ipipe_init_proc()	do { } while(0)
+#endif	/* CONFIG_PROC_FS */
+
+void __ipipe_init_stage(struct ipipe_domain *ipd);
+
+void __ipipe_cleanup_domain(struct ipipe_domain *ipd);
+
+void __ipipe_add_domain_proc(struct ipipe_domain *ipd);
+
+void __ipipe_remove_domain_proc(struct ipipe_domain *ipd);
+
+void __ipipe_flush_printk(unsigned irq, void *cookie);
+
+void __ipipe_walk_pipeline(struct list_head *pos);
+
+void __ipipe_pend_irq(unsigned irq, struct list_head *head);
+
+int __ipipe_dispatch_event(unsigned event, void *data);
+
+void __ipipe_dispatch_wired_nocheck(struct ipipe_domain *head, unsigned irq);
+
+void __ipipe_dispatch_wired(struct ipipe_domain *head, unsigned irq);
+
+void __ipipe_sync_stage(void);
+
+void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned irq);
+
+void __ipipe_lock_irq(struct ipipe_domain *ipd, int cpu, unsigned irq);
+
+void __ipipe_unlock_irq(struct ipipe_domain *ipd, unsigned irq);
+
+void __ipipe_pin_range_globally(unsigned long start, unsigned long end);
+
+/* Must be called hw IRQs off. */
+static inline void ipipe_irq_lock(unsigned irq)
+{
+	__ipipe_lock_irq(__ipipe_current_domain, ipipe_processor_id(), irq);
+}
+
+/* Must be called hw IRQs off. */
+static inline void ipipe_irq_unlock(unsigned irq)
+{
+	__ipipe_unlock_irq(__ipipe_current_domain, irq);
+}
+
+#ifndef __ipipe_sync_pipeline
+#define __ipipe_sync_pipeline() __ipipe_sync_stage()
+#endif
+
+#ifndef __ipipe_do_root_xirq
+#define __ipipe_do_root_xirq(ipd, irq)			\
+	(ipd)->irqs[irq].handler(irq, (ipd)->irqs[irq].cookie)
+#endif
+
+#ifndef __ipipe_check_root_resched
+#ifdef CONFIG_PREEMPT
+#define __ipipe_check_root_resched()	\
+	(preempt_count() == 0 && need_resched())
+#else
+#define __ipipe_check_root_resched()	0
+#endif
+#endif
+
+#ifndef __ipipe_run_irqtail
+#define __ipipe_run_irqtail(irq) do { } while(0)
+#endif
+
+#define __ipipe_pipeline_head_p(ipd) (&(ipd)->p_link == __ipipe_pipeline.next)
+
+#define __ipipe_ipending_p(p)	((p)->irqpend_himap != 0)
+
+/*
+ * Keep the following as a macro, so that client code could check for
+ * the support of the invariant pipeline head optimization.
+ */
+#define __ipipe_pipeline_head() \
+	list_entry(__ipipe_pipeline.next, struct ipipe_domain, p_link)
+
+#define local_irq_enable_hw_cond()		local_irq_enable_hw()
+#define local_irq_disable_hw_cond()		local_irq_disable_hw()
+#define local_irq_save_hw_cond(flags)		local_irq_save_hw(flags)
+#define local_irq_restore_hw_cond(flags)	local_irq_restore_hw(flags)
+
+#ifdef CONFIG_SMP
+cpumask_t __ipipe_set_irq_affinity(unsigned irq, cpumask_t cpumask);
+int __ipipe_send_ipi(unsigned ipi, cpumask_t cpumask);
+#define local_irq_save_hw_smp(flags)		local_irq_save_hw(flags)
+#define local_irq_restore_hw_smp(flags)		local_irq_restore_hw(flags)
+#else /* !CONFIG_SMP */
+#define local_irq_save_hw_smp(flags)		do { (void)(flags); } while(0)
+#define local_irq_restore_hw_smp(flags)		do { } while(0)
+#endif /* CONFIG_SMP */
+
+#define local_irq_save_full(vflags, rflags)		\
+	do {						\
+		local_irq_save(vflags);			\
+		local_irq_save_hw(rflags);		\
+	} while(0)
+
+#define local_irq_restore_full(vflags, rflags)		\
+	do {						\
+		local_irq_restore_hw(rflags);		\
+		local_irq_restore(vflags);		\
+	} while(0)
+
+static inline void __local_irq_restore_nosync(unsigned long x)
+{
+	struct ipipe_percpu_domain_data *p = ipipe_root_cpudom_ptr();
+
+	if (raw_irqs_disabled_flags(x)) {
+		__set_bit(IPIPE_STALL_FLAG, &p->status);
+		trace_hardirqs_off();
+	} else {
+		trace_hardirqs_on();
+		__clear_bit(IPIPE_STALL_FLAG, &p->status);
+	}
+}
+
+static inline void local_irq_restore_nosync(unsigned long x)
+{
+	unsigned long flags;
+	local_irq_save_hw_smp(flags);
+	__local_irq_restore_nosync(x);
+	local_irq_restore_hw_smp(flags);
+}
+
+#define __ipipe_root_domain_p	(__ipipe_current_domain == ipipe_root_domain)
+#define ipipe_root_domain_p	(ipipe_current_domain == ipipe_root_domain)
+
+/* This has to be called with hw IRQs off. */
+#define __ipipe_head_domain_p   __ipipe_pipeline_head_p(__ipipe_current_domain)
+
+static inline int __ipipe_event_monitored_p(int ev)
+{
+	if (__ipipe_event_monitors[ev] > 0)
+		return 1;
+
+	return (ipipe_current_domain->evself & (1LL << ev)) != 0;
+}
+
+#define ipipe_notifier_enabled_p(p)	\
+	(((p)->ipipe_flags) & PF_EVNOTIFY)
+
+#define ipipe_sigwake_notify(p)						\
+	do {								\
+		if (ipipe_notifier_enabled_p(p) &&			\
+		    __ipipe_event_monitored_p(IPIPE_EVENT_SIGWAKE))	\
+			__ipipe_dispatch_event(IPIPE_EVENT_SIGWAKE, p);	\
+	} while (0)
+
+#define ipipe_exit_notify(p)						\
+	do {								\
+		if (ipipe_notifier_enabled_p(p) &&			\
+		    __ipipe_event_monitored_p(IPIPE_EVENT_EXIT))	\
+			__ipipe_dispatch_event(IPIPE_EVENT_EXIT, p);	\
+	} while (0)
+
+#define ipipe_setsched_notify(p)					\
+	do {								\
+		if (ipipe_notifier_enabled_p(p) &&			\
+		    __ipipe_event_monitored_p(IPIPE_EVENT_SETSCHED))	\
+			__ipipe_dispatch_event(IPIPE_EVENT_SETSCHED, p); \
+	} while (0)
+
+#define ipipe_schedule_notify(prev, next)				\
+do {									\
+	if ((ipipe_notifier_enabled_p(next) ||				\
+	     ipipe_notifier_enabled_p(prev)) &&				\
+	    __ipipe_event_monitored_p(IPIPE_EVENT_SCHEDULE))		\
+		__ipipe_dispatch_event(IPIPE_EVENT_SCHEDULE, next);	\
+} while (0)
+
+#define ipipe_trap_notify(ex, regs)					\
+	({								\
+		unsigned long __flags__;				\
+		int __ret__ = 0;					\
+		local_irq_save_hw_smp(__flags__);			\
+		if ((test_bit(IPIPE_NOSTACK_FLAG, &ipipe_this_cpudom_var(status)) || \
+		     ipipe_notifier_enabled_p(current)) &&		\
+		    __ipipe_event_monitored_p(ex)) {			\
+			local_irq_restore_hw_smp(__flags__);		\
+			__ret__ = __ipipe_dispatch_event(ex, regs);	\
+		} else							\
+			local_irq_restore_hw_smp(__flags__);		\
+		__ret__;						\
+	})
+
+#define ipipe_init_notify(p)						\
+	do {								\
+		if (__ipipe_event_monitored_p(IPIPE_EVENT_INIT))	\
+			__ipipe_dispatch_event(IPIPE_EVENT_INIT, p);	\
+	} while (0)
+
+#define ipipe_cleanup_notify(mm)					\
+	do {								\
+		if (__ipipe_event_monitored_p(IPIPE_EVENT_CLEANUP))	\
+			__ipipe_dispatch_event(IPIPE_EVENT_CLEANUP, mm); \
+	} while (0)
+
+/* Public interface */
+
+int ipipe_register_domain(struct ipipe_domain *ipd,
+			  struct ipipe_domain_attr *attr);
+
+int ipipe_unregister_domain(struct ipipe_domain *ipd);
+
+void ipipe_suspend_domain(void);
+
+int ipipe_virtualize_irq(struct ipipe_domain *ipd,
+			 unsigned irq,
+			 ipipe_irq_handler_t handler,
+			 void *cookie,
+			 ipipe_irq_ackfn_t acknowledge,
+			 unsigned modemask);
+
+int ipipe_control_irq(struct ipipe_domain *ipd,
+		      unsigned int irq,
+		      unsigned clrmask,
+		      unsigned setmask);
+
+unsigned ipipe_alloc_virq(void);
+
+int ipipe_free_virq(unsigned virq);
+
+int ipipe_trigger_irq(unsigned irq);
+
+static inline void __ipipe_propagate_irq(unsigned irq)
+{
+	struct list_head *next = __ipipe_current_domain->p_link.next;
+	if (next == &ipipe_root.p_link) {
+		/* Fast path: root must handle all interrupts. */
+		__ipipe_set_irq_pending(&ipipe_root, irq);
+		return;
+	}
+	__ipipe_pend_irq(irq, next);
+}
+
+static inline void __ipipe_schedule_irq(unsigned irq)
+{
+	__ipipe_pend_irq(irq, &__ipipe_current_domain->p_link);
+}
+
+static inline void __ipipe_schedule_irq_head(unsigned irq)
+{
+	__ipipe_set_irq_pending(__ipipe_pipeline_head(), irq);
+}
+
+static inline void __ipipe_schedule_irq_root(unsigned irq)
+{
+	__ipipe_set_irq_pending(&ipipe_root, irq);
+}
+
+static inline void ipipe_propagate_irq(unsigned irq)
+{
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
+	__ipipe_propagate_irq(irq);
+	local_irq_restore_hw(flags);
+}
+
+static inline void ipipe_schedule_irq(unsigned irq)
+{
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
+	__ipipe_schedule_irq(irq);
+	local_irq_restore_hw(flags);
+}
+
+static inline void ipipe_schedule_irq_head(unsigned irq)
+{
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
+	__ipipe_schedule_irq_head(irq);
+	local_irq_restore_hw(flags);
+}
+
+static inline void ipipe_schedule_irq_root(unsigned irq)
+{
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
+	__ipipe_schedule_irq_root(irq);
+	local_irq_restore_hw(flags);
+}
+
+void ipipe_stall_pipeline_from(struct ipipe_domain *ipd);
+
+unsigned long ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd);
+
+unsigned long ipipe_test_and_unstall_pipeline_from(struct ipipe_domain *ipd);
+
+static inline void ipipe_unstall_pipeline_from(struct ipipe_domain *ipd)
+{
+	ipipe_test_and_unstall_pipeline_from(ipd);
+}
+
+void ipipe_restore_pipeline_from(struct ipipe_domain *ipd,
+					  unsigned long x);
+
+static inline unsigned long ipipe_test_pipeline_from(struct ipipe_domain *ipd)
+{
+	return test_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status));
+}
+
+static inline void ipipe_stall_pipeline_head(void)
+{
+	local_irq_disable_hw();
+	__set_bit(IPIPE_STALL_FLAG, &ipipe_head_cpudom_var(status));
+}
+
+static inline unsigned long ipipe_test_and_stall_pipeline_head(void)
+{
+	local_irq_disable_hw();
+	return __test_and_set_bit(IPIPE_STALL_FLAG, &ipipe_head_cpudom_var(status));
+}
+
+void ipipe_unstall_pipeline_head(void);
+
+void __ipipe_restore_pipeline_head(unsigned long x);
+
+static inline void ipipe_restore_pipeline_head(unsigned long x)
+{
+#ifdef CONFIG_IPIPE_DEBUG
+	if (WARN_ON_ONCE(!irqs_disabled_hw()))
+		local_irq_disable_hw();
+#endif
+	if ((x ^ test_bit(IPIPE_STALL_FLAG, &ipipe_head_cpudom_var(status))) & 1)
+		__ipipe_restore_pipeline_head(x);
+}
+
+#define ipipe_unstall_pipeline() \
+	ipipe_unstall_pipeline_from(ipipe_current_domain)
+
+#define ipipe_test_and_unstall_pipeline() \
+	ipipe_test_and_unstall_pipeline_from(ipipe_current_domain)
+
+#define ipipe_test_pipeline() \
+	ipipe_test_pipeline_from(ipipe_current_domain)
+
+#define ipipe_test_and_stall_pipeline() \
+	ipipe_test_and_stall_pipeline_from(ipipe_current_domain)
+
+#define ipipe_stall_pipeline() \
+	ipipe_stall_pipeline_from(ipipe_current_domain)
+
+#define ipipe_restore_pipeline(x) \
+	ipipe_restore_pipeline_from(ipipe_current_domain, (x))
+
+void ipipe_init_attr(struct ipipe_domain_attr *attr);
+
+int ipipe_get_sysinfo(struct ipipe_sysinfo *sysinfo);
+
+void __ipipe_do_critical_sync(unsigned irq, void *cookie);
+
+unsigned long ipipe_critical_enter(void (*syncfn) (void));
+
+void ipipe_critical_exit(unsigned long flags);
+
+void ipipe_prepare_panic(void);
+
+static inline void ipipe_set_printk_sync(struct ipipe_domain *ipd)
+{
+	set_bit(IPIPE_SPRINTK_FLAG, &ipd->flags);
+}
+
+static inline void ipipe_set_printk_async(struct ipipe_domain *ipd)
+{
+	clear_bit(IPIPE_SPRINTK_FLAG, &ipd->flags);
+}
+
+static inline void ipipe_set_foreign_stack(struct ipipe_domain *ipd)
+{
+	/* Must be called hw interrupts off. */
+	__set_bit(IPIPE_NOSTACK_FLAG, &ipipe_cpudom_var(ipd, status));
+}
+
+static inline void ipipe_clear_foreign_stack(struct ipipe_domain *ipd)
+{
+	/* Must be called hw interrupts off. */
+	__clear_bit(IPIPE_NOSTACK_FLAG, &ipipe_cpudom_var(ipd, status));
+}
+
+static inline int ipipe_test_foreign_stack(void)
+{
+	/* Must be called hw interrupts off. */
+	return test_bit(IPIPE_NOSTACK_FLAG, &ipipe_this_cpudom_var(status));
+}
+
+#ifndef ipipe_safe_current
+#define ipipe_safe_current()					\
+({								\
+	struct task_struct *p;					\
+	unsigned long flags;					\
+	local_irq_save_hw_smp(flags);				\
+	p = ipipe_test_foreign_stack() ? &init_task : current;	\
+	local_irq_restore_hw_smp(flags);			\
+	p; \
+})
+#endif
+
+ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd,
+					unsigned event,
+					ipipe_event_handler_t handler);
+
+cpumask_t ipipe_set_irq_affinity(unsigned irq,
+				 cpumask_t cpumask);
+
+int ipipe_send_ipi(unsigned ipi,
+		   cpumask_t cpumask);
+
+int ipipe_setscheduler_root(struct task_struct *p,
+			    int policy,
+			    int prio);
+
+int ipipe_reenter_root(struct task_struct *prev,
+		       int policy,
+		       int prio);
+
+int ipipe_alloc_ptdkey(void);
+
+int ipipe_free_ptdkey(int key);
+
+int ipipe_set_ptd(int key,
+		  void *value);
+
+void *ipipe_get_ptd(int key);
+
+int ipipe_disable_ondemand_mappings(struct task_struct *tsk);
+
+static inline void ipipe_nmi_enter(void)
+{
+	int cpu = ipipe_processor_id();
+
+	per_cpu(ipipe_nmi_saved_root, cpu) = ipipe_root_cpudom_var(status);
+	__set_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
+
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+	per_cpu(ipipe_saved_context_check_state, cpu) =
+		ipipe_disable_context_check(cpu);
+#endif /* CONFIG_IPIPE_DEBUG_CONTEXT */
+}
+
+static inline void ipipe_nmi_exit(void)
+{
+	int cpu = ipipe_processor_id();
+
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+	ipipe_restore_context_check
+		(cpu, per_cpu(ipipe_saved_context_check_state, cpu));
+#endif /* CONFIG_IPIPE_DEBUG_CONTEXT */
+
+	if (!test_bit(IPIPE_STALL_FLAG, &per_cpu(ipipe_nmi_saved_root, cpu)))
+		__clear_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
+}
+
+#define ipipe_enable_notifier(p)			\
+	do {						\
+		(p)->ipipe_flags |= PF_EVNOTIFY;	\
+	} while (0)
+
+#define ipipe_disable_notifier(p)				\
+	do {							\
+		(p)->ipipe_flags &= ~(PF_EVNOTIFY|PF_EVTRET);	\
+	} while (0)
+
+/* hw IRQs off. */
+#define ipipe_return_notify(p)						\
+	do {								\
+		if (ipipe_notifier_enabled_p(p) &&			\
+		    __ipipe_event_monitored_p(IPIPE_EVENT_RETURN))	\
+			(p)->ipipe_flags |= PF_EVTRET;			\
+	} while (0)
+
+#define ipipe_clear_flags(p)		do { (p)->ipipe_flags = 0; } while (0)
+
+#else	/* !CONFIG_IPIPE */
+
+#define ipipe_init_early()		do { } while(0)
+#define ipipe_init()			do { } while(0)
+#define ipipe_suspend_domain()		do { } while(0)
+#define ipipe_sigwake_notify(p)		do { } while(0)
+#define ipipe_setsched_notify(p)	do { } while(0)
+#define ipipe_init_notify(p)		do { } while(0)
+#define ipipe_exit_notify(p)		do { } while(0)
+#define ipipe_cleanup_notify(mm)	do { } while(0)
+#define ipipe_trap_notify(t,r)		0
+#define ipipe_init_proc()		do { } while(0)
+
+static inline void __ipipe_pin_range_globally(unsigned long start,
+					      unsigned long end)
+{
+}
+
+static inline int ipipe_test_foreign_stack(void)
+{
+	return 0;
+}
+
+#define local_irq_enable_hw_cond()		do { } while(0)
+#define local_irq_disable_hw_cond()		do { } while(0)
+#define local_irq_save_hw_cond(flags)		do { (void)(flags); } while(0)
+#define local_irq_restore_hw_cond(flags)	do { } while(0)
+#define local_irq_save_hw_smp(flags)		do { (void)(flags); } while(0)
+#define local_irq_restore_hw_smp(flags)		do { } while(0)
+
+#define ipipe_irq_lock(irq)		do { } while(0)
+#define ipipe_irq_unlock(irq)		do { } while(0)
+
+#define __ipipe_root_domain_p		1
+#define ipipe_root_domain_p		1
+#define ipipe_safe_current()		current
+#define ipipe_processor_id()		smp_processor_id()
+#define ipipe_clear_flags(p)		do { } while (0)
+
+#define ipipe_nmi_enter()		do { } while (0)
+#define ipipe_nmi_exit()		do { } while (0)
+
+#define local_irq_disable_head()	local_irq_disable()
+
+#define local_irq_save_full(vflags, rflags)	do { (void)(vflags); local_irq_save(rflags); } while(0)
+#define local_irq_restore_full(vflags, rflags)	do { (void)(vflags); local_irq_restore(rflags); } while(0)
+#define local_irq_restore_nosync(vflags)	local_irq_restore(vflags)
+
+#define __ipipe_pipeline_head_p(ipd)	1
+
+#endif	/* CONFIG_IPIPE */
+
+#endif	/* !__LINUX_IPIPE_H */
diff --git a/include/linux/ipipe_base.h b/include/linux/ipipe_base.h
new file mode 100644
index 0000000..3f43ba5
--- /dev/null
+++ b/include/linux/ipipe_base.h
@@ -0,0 +1,134 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_base.h
+ *
+ * Copyright (C) 2002-2007 Philippe Gerum.
+ *               2007 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_BASE_H
+#define __LINUX_IPIPE_BASE_H
+
+#ifdef CONFIG_IPIPE
+
+#include <asm/ipipe_base.h>
+
+#define __bpl_up(x)		(((x)+(BITS_PER_LONG-1)) & ~(BITS_PER_LONG-1))
+/* Number of virtual IRQs (must be a multiple of BITS_PER_LONG) */
+#define IPIPE_NR_VIRQS		BITS_PER_LONG
+/* First virtual IRQ # (must be aligned on BITS_PER_LONG) */
+#define IPIPE_VIRQ_BASE		__bpl_up(IPIPE_NR_XIRQS)
+/* Total number of IRQ slots */
+#define IPIPE_NR_IRQS		(IPIPE_VIRQ_BASE+IPIPE_NR_VIRQS)
+
+#define IPIPE_IRQ_LOMAPSZ	(IPIPE_NR_IRQS / BITS_PER_LONG)
+#if IPIPE_IRQ_LOMAPSZ > BITS_PER_LONG
+/*
+ * We need a 3-level mapping. This allows us to handle up to 32k IRQ
+ * vectors on 32bit machines, 256k on 64bit ones.
+ */
+#define __IPIPE_3LEVEL_IRQMAP	1
+#define IPIPE_IRQ_MDMAPSZ	(__bpl_up(IPIPE_IRQ_LOMAPSZ) / BITS_PER_LONG)
+#else
+/*
+ * 2-level mapping is enough. This allows us to handle up to 1024 IRQ
+ * vectors on 32bit machines, 4096 on 64bit ones.
+ */
+#define __IPIPE_2LEVEL_IRQMAP	1
+#endif
+
+/* Per-cpu pipeline status */
+#define IPIPE_STALL_FLAG	0	/* Stalls a pipeline stage -- guaranteed at bit #0 */
+#define IPIPE_NOSTACK_FLAG	1	/* Domain currently runs on a foreign stack */
+
+#define IPIPE_STALL_MASK	(1L << IPIPE_STALL_FLAG)
+#define IPIPE_NOSTACK_MASK	(1L << IPIPE_NOSTACK_FLAG)
+
+typedef void (*ipipe_irq_handler_t)(unsigned int irq,
+				    void *cookie);
+
+extern struct ipipe_domain ipipe_root;
+
+#define ipipe_root_domain (&ipipe_root)
+
+void __ipipe_unstall_root(void);
+
+void __ipipe_restore_root(unsigned long x);
+
+#define ipipe_preempt_disable(flags)		\
+	do {					\
+		local_irq_save_hw(flags);	\
+		if (__ipipe_root_domain_p)	\
+			preempt_disable();	\
+	} while (0)
+
+#define ipipe_preempt_enable(flags)			\
+	do {						\
+		if (__ipipe_root_domain_p) {		\
+			preempt_enable_no_resched();	\
+			local_irq_restore_hw(flags);	\
+			preempt_check_resched();	\
+		} else					\
+			local_irq_restore_hw(flags);	\
+	} while (0)
+
+#define ipipe_get_cpu(flags)	({ ipipe_preempt_disable(flags); ipipe_processor_id(); })
+#define ipipe_put_cpu(flags)	ipipe_preempt_enable(flags)
+ 
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+void ipipe_check_context(struct ipipe_domain *border_ipd);
+#else /* !CONFIG_IPIPE_DEBUG_CONTEXT */
+static inline void ipipe_check_context(struct ipipe_domain *border_ipd) { }
+#endif /* !CONFIG_IPIPE_DEBUG_CONTEXT */
+
+/* Generic features */
+
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+#define __IPIPE_FEATURE_REQUEST_TICKDEV		1
+#endif
+#define __IPIPE_FEATURE_DELAYED_ATOMICSW	1
+#define __IPIPE_FEATURE_FASTPEND_IRQ		1
+#define __IPIPE_FEATURE_TRACE_EVENT		1
+#define __IPIPE_FEATURE_ENABLE_NOTIFIER		1
+#ifdef CONFIG_HAVE_IPIPE_HOSTRT
+#define __IPIPE_FEATURE_HOSTRT			1
+#endif
+#define __IPIPE_FEATURE_PREPARE_PANIC		1
+#define __IPIPE_FEATURE_ROOT_PREEMPT_NOTIFIER	1
+#define __IPIPE_FEATURE_CONTROL_IRQ		1
+
+#else /* !CONFIG_IPIPE */
+
+#define ipipe_preempt_disable(flags)	   \
+	do {				   \
+		preempt_disable();	   \
+		(void)(flags);		   \
+	} while (0)
+#define ipipe_preempt_enable(flags)	preempt_enable()
+
+#define ipipe_get_cpu(flags)		({ (void)(flags); get_cpu(); })
+#define ipipe_put_cpu(flags)		\
+	do {				\
+		(void)(flags);		\
+		put_cpu();		\
+	} while (0)
+	
+#define ipipe_check_context(ipd)	do { } while(0)
+
+#endif	/* CONFIG_IPIPE */
+
+#endif	/* !__LINUX_IPIPE_BASE_H */
diff --git a/include/linux/ipipe_lock.h b/include/linux/ipipe_lock.h
new file mode 100644
index 0000000..5382208
--- /dev/null
+++ b/include/linux/ipipe_lock.h
@@ -0,0 +1,241 @@
+/*   -*- linux-c -*-
+ *   include/linux/ipipe_lock.h
+ *
+ *   Copyright (C) 2009 Philippe Gerum.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ *   USA; either version 2 of the License, or (at your option) any later
+ *   version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the Free Software
+ *   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_LOCK_H
+#define __LINUX_IPIPE_LOCK_H
+
+typedef struct {
+	arch_spinlock_t arch_lock;
+} __ipipe_spinlock_t;
+
+#define ipipe_spinlock_p(lock)						\
+	__builtin_types_compatible_p(typeof(lock), __ipipe_spinlock_t *)
+
+#define std_spinlock_raw_p(lock)					\
+	__builtin_types_compatible_p(typeof(lock), raw_spinlock_t *)
+
+#define std_spinlock_p(lock)						\
+	__builtin_types_compatible_p(typeof(lock), spinlock_t *)
+
+#define ipipe_spinlock(lock)	((__ipipe_spinlock_t *)(lock))
+#define std_spinlock_raw(lock)	((raw_spinlock_t *)(lock))
+#define std_spinlock(lock)	((spinlock_t *)(lock))
+
+#define PICK_SPINLOCK_IRQSAVE(lock, flags)				\
+	do {								\
+		if (ipipe_spinlock_p(lock))				\
+			(flags) = __ipipe_spin_lock_irqsave(ipipe_spinlock(lock)); \
+		else if (std_spinlock_raw_p(lock))				\
+			__real_raw_spin_lock_irqsave(std_spinlock_raw(lock), flags); \
+		else if (std_spinlock_p(lock))				\
+			__real_raw_spin_lock_irqsave(&std_spinlock(lock)->rlock, flags); \
+		else __bad_lock_type();					\
+	} while (0)
+
+#define PICK_SPINTRYLOCK_IRQSAVE(lock, flags)				\
+	({								\
+		int __ret__;						\
+		if (ipipe_spinlock_p(lock))				\
+			__ret__ = __ipipe_spin_trylock_irqsave(ipipe_spinlock(lock), &(flags)); \
+		else if (std_spinlock_raw_p(lock))				\
+			__ret__ = __real_raw_spin_trylock_irqsave(std_spinlock_raw(lock), flags); \
+		else if (std_spinlock_p(lock))				\
+			__ret__ = __real_raw_spin_trylock_irqsave(&std_spinlock(lock)->rlock, flags); \
+		else __bad_lock_type();					\
+		__ret__;						\
+	 })
+
+#define PICK_SPINTRYLOCK_IRQ(lock)					\
+	({								\
+		int __ret__;						\
+		if (ipipe_spinlock_p(lock))				\
+			__ret__ = __ipipe_spin_trylock_irq(ipipe_spinlock(lock)); \
+		else if (std_spinlock_raw_p(lock))				\
+			__ret__ = __real_raw_spin_trylock_irq(std_spinlock_raw(lock)); \
+		else if (std_spinlock_p(lock))				\
+			__ret__ = __real_raw_spin_trylock_irq(&std_spinlock(lock)->rlock); \
+		else __bad_lock_type();					\
+		__ret__;						\
+	 })
+
+#define PICK_SPINUNLOCK_IRQRESTORE(lock, flags)				\
+	do {								\
+		if (ipipe_spinlock_p(lock))				\
+			__ipipe_spin_unlock_irqrestore(ipipe_spinlock(lock), flags); \
+		else {							\
+			__ipipe_spin_unlock_debug(flags);		\
+			if (std_spinlock_raw_p(lock))			\
+				__real_raw_spin_unlock_irqrestore(std_spinlock_raw(lock), flags); \
+			else if (std_spinlock_p(lock))			\
+				__real_raw_spin_unlock_irqrestore(&std_spinlock(lock)->rlock, flags); \
+		}							\
+	} while (0)
+
+#define PICK_SPINOP(op, lock)						\
+	do {								\
+		if (ipipe_spinlock_p(lock))				\
+			arch_spin##op(&ipipe_spinlock(lock)->arch_lock); \
+		else if (std_spinlock_raw_p(lock))			\
+			__real_raw_spin##op(std_spinlock_raw(lock));	\
+		else if (std_spinlock_p(lock))				\
+			__real_raw_spin##op(&std_spinlock(lock)->rlock); \
+		else __bad_lock_type();					\
+	} while (0)
+
+#define PICK_SPINOP_RET(op, lock, type)					\
+	({								\
+		type __ret__;						\
+		if (ipipe_spinlock_p(lock))				\
+			__ret__ = arch_spin##op(&ipipe_spinlock(lock)->arch_lock); \
+		else if (std_spinlock_raw_p(lock))			\
+			__ret__ = __real_raw_spin##op(std_spinlock_raw(lock)); \
+		else if (std_spinlock_p(lock))				\
+			__ret__ = __real_raw_spin##op(&std_spinlock(lock)->rlock); \
+		else { __ret__ = -1; __bad_lock_type(); }		\
+		__ret__;						\
+	})
+
+#define arch_spin_lock_init(lock)					\
+	do {								\
+		IPIPE_DEFINE_SPINLOCK(__lock__);			\
+		*((ipipe_spinlock_t *)lock) = __lock__;			\
+	} while (0)
+
+#define arch_spin_lock_irq(lock)					\
+	do {								\
+		local_irq_disable_hw();					\
+		arch_spin_lock(lock);					\
+	} while (0)
+
+#define arch_spin_unlock_irq(lock)					\
+	do {								\
+		arch_spin_unlock(lock);					\
+		local_irq_enable_hw();					\
+	} while (0)
+
+typedef struct {
+	arch_rwlock_t arch_lock;
+} __ipipe_rwlock_t;
+
+#define ipipe_rwlock_p(lock)						\
+	__builtin_types_compatible_p(typeof(lock), __ipipe_rwlock_t *)
+
+#define std_rwlock_p(lock)						\
+	__builtin_types_compatible_p(typeof(lock), rwlock_t *)
+
+#define ipipe_rwlock(lock)	((__ipipe_rwlock_t *)(lock))
+#define std_rwlock(lock)	((rwlock_t *)(lock))
+
+#define PICK_RWOP(op, lock)						\
+	do {								\
+		if (ipipe_rwlock_p(lock))				\
+			arch##op(&ipipe_rwlock(lock)->arch_lock);	\
+		else if (std_rwlock_p(lock))				\
+			_raw##op(std_rwlock(lock));			\
+		else __bad_lock_type();					\
+	} while (0)
+
+extern int __bad_lock_type(void);
+
+#ifdef CONFIG_IPIPE
+
+#define ipipe_spinlock_t		__ipipe_spinlock_t
+#define IPIPE_DEFINE_RAW_SPINLOCK(x)	ipipe_spinlock_t x = IPIPE_SPIN_LOCK_UNLOCKED
+#define IPIPE_DECLARE_RAW_SPINLOCK(x)	extern ipipe_spinlock_t x
+#define IPIPE_DEFINE_SPINLOCK(x)	IPIPE_DEFINE_RAW_SPINLOCK(x)
+#define IPIPE_DECLARE_SPINLOCK(x)	IPIPE_DECLARE_RAW_SPINLOCK(x)
+
+#define IPIPE_SPIN_LOCK_UNLOCKED	\
+	(__ipipe_spinlock_t) {	.arch_lock = __ARCH_SPIN_LOCK_UNLOCKED }
+
+#define spin_lock_irqsave_cond(lock, flags) \
+	spin_lock_irqsave(lock, flags)
+
+#define spin_unlock_irqrestore_cond(lock, flags) \
+	spin_unlock_irqrestore(lock, flags)
+
+void __ipipe_spin_lock_irq(ipipe_spinlock_t *lock);
+
+int __ipipe_spin_trylock_irq(ipipe_spinlock_t *lock);
+
+void __ipipe_spin_unlock_irq(ipipe_spinlock_t *lock);
+
+unsigned long __ipipe_spin_lock_irqsave(ipipe_spinlock_t *lock);
+
+int __ipipe_spin_trylock_irqsave(ipipe_spinlock_t *lock,
+				 unsigned long *x);
+
+void __ipipe_spin_unlock_irqrestore(ipipe_spinlock_t *lock,
+				    unsigned long x);
+
+void __ipipe_spin_unlock_irqbegin(ipipe_spinlock_t *lock);
+
+void __ipipe_spin_unlock_irqcomplete(unsigned long x);
+
+#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP)
+void __ipipe_spin_unlock_debug(unsigned long flags);
+#else
+#define __ipipe_spin_unlock_debug(flags)  do { } while (0)
+#endif
+
+#define ipipe_rwlock_t			__ipipe_rwlock_t
+#define IPIPE_DEFINE_RWLOCK(x)		ipipe_rwlock_t x = IPIPE_RW_LOCK_UNLOCKED
+#define IPIPE_DECLARE_RWLOCK(x)		extern ipipe_rwlock_t x
+
+#define IPIPE_RW_LOCK_UNLOCKED	\
+	(__ipipe_rwlock_t) { .arch_lock = __ARCH_RW_LOCK_UNLOCKED }
+
+#else /* !CONFIG_IPIPE */
+
+#define ipipe_spinlock_t		spinlock_t
+#define IPIPE_DEFINE_SPINLOCK(x)	DEFINE_SPINLOCK(x)
+#define IPIPE_DECLARE_SPINLOCK(x)	extern spinlock_t x
+#define IPIPE_SPIN_LOCK_UNLOCKED	SPIN_LOCK_UNLOCKED
+#define IPIPE_DEFINE_RAW_SPINLOCK(x)	DEFINE_RAW_SPINLOCK(x)
+#define IPIPE_DECLARE_RAW_SPINLOCK(x)	extern raw_spinlock_t x
+
+#define spin_lock_irqsave_cond(lock, flags)		\
+	do {						\
+		(void)(flags);				\
+		spin_lock(lock);			\
+	} while(0)
+
+#define spin_unlock_irqrestore_cond(lock, flags)	\
+	spin_unlock(lock)
+
+#define __ipipe_spin_lock_irq(lock)		do { } while (0)
+#define __ipipe_spin_unlock_irq(lock)		do { } while (0)
+#define __ipipe_spin_lock_irqsave(lock)		0
+#define __ipipe_spin_trylock_irq(lock)		1
+#define __ipipe_spin_trylock_irqsave(lock, x)	({ (void)(x); 1; })
+#define __ipipe_spin_unlock_irqrestore(lock, x)	do { (void)(x); } while (0)
+#define __ipipe_spin_unlock_irqbegin(lock)	do { } while (0)
+#define __ipipe_spin_unlock_irqcomplete(x)	do { (void)(x); } while (0)
+#define __ipipe_spin_unlock_debug(flags)	do { } while (0)
+
+#define ipipe_rwlock_t			rwlock_t
+#define IPIPE_DEFINE_RWLOCK(x)		DEFINE_RWLOCK(x)
+#define IPIPE_DECLARE_RWLOCK(x)		extern rwlock_t x
+#define IPIPE_RW_LOCK_UNLOCKED		RW_LOCK_UNLOCKED
+
+#endif /* !CONFIG_IPIPE */
+
+#endif /* !__LINUX_IPIPE_LOCK_H */
diff --git a/include/linux/ipipe_percpu.h b/include/linux/ipipe_percpu.h
new file mode 100644
index 0000000..0b42e8c
--- /dev/null
+++ b/include/linux/ipipe_percpu.h
@@ -0,0 +1,89 @@
+/*   -*- linux-c -*-
+ *   include/linux/ipipe_percpu.h
+ *
+ *   Copyright (C) 2007 Philippe Gerum.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ *   USA; either version 2 of the License, or (at your option) any later
+ *   version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the Free Software
+ *   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_PERCPU_H
+#define __LINUX_IPIPE_PERCPU_H
+
+#include <asm/percpu.h>
+#include <asm/ptrace.h>
+
+struct ipipe_domain;
+
+struct ipipe_percpu_domain_data {
+	unsigned long status;	/* <= Must be first in struct. */
+	unsigned long irqpend_himap;
+#ifdef __IPIPE_3LEVEL_IRQMAP
+	unsigned long irqpend_mdmap[IPIPE_IRQ_MDMAPSZ];
+#endif
+	unsigned long irqpend_lomap[IPIPE_IRQ_LOMAPSZ];
+	unsigned long irqheld_map[IPIPE_IRQ_LOMAPSZ];
+	unsigned long irqall[IPIPE_NR_IRQS];
+	u64 evsync;
+};
+
+/*
+ * CAREFUL: all accessors based on __raw_get_cpu_var() you may find in
+ * this file should be used only while hw interrupts are off, to
+ * prevent from CPU migration regardless of the running domain.
+ */
+#ifdef CONFIG_SMP
+#define ipipe_percpudom_ptr(ipd, cpu)	\
+	(&per_cpu(ipipe_percpu_darray, cpu)[(ipd)->slot])
+#define ipipe_cpudom_ptr(ipd)	\
+	(&__ipipe_get_cpu_var(ipipe_percpu_darray)[(ipd)->slot])
+#else
+DECLARE_PER_CPU(struct ipipe_percpu_domain_data *, ipipe_percpu_daddr[CONFIG_IPIPE_DOMAINS]);
+#define ipipe_percpudom_ptr(ipd, cpu)	\
+	(per_cpu(ipipe_percpu_daddr, cpu)[(ipd)->slot])
+#define ipipe_cpudom_ptr(ipd)	\
+	(__ipipe_get_cpu_var(ipipe_percpu_daddr)[(ipd)->slot])
+#endif
+#define ipipe_percpudom(ipd, var, cpu)	(ipipe_percpudom_ptr(ipd, cpu)->var)
+#define ipipe_cpudom_var(ipd, var)	(ipipe_cpudom_ptr(ipd)->var)
+
+#define IPIPE_ROOT_SLOT			0
+#define IPIPE_HEAD_SLOT			(CONFIG_IPIPE_DOMAINS - 1)
+
+DECLARE_PER_CPU(struct ipipe_percpu_domain_data, ipipe_percpu_darray[CONFIG_IPIPE_DOMAINS]);
+
+DECLARE_PER_CPU(struct ipipe_domain *, ipipe_percpu_domain);
+
+DECLARE_PER_CPU(unsigned long, ipipe_nmi_saved_root);
+
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+DECLARE_PER_CPU(int, ipipe_percpu_context_check);
+DECLARE_PER_CPU(int, ipipe_saved_context_check_state);
+#endif
+
+#define ipipe_root_cpudom_ptr()		\
+	(&__ipipe_get_cpu_var(ipipe_percpu_darray)[IPIPE_ROOT_SLOT])
+
+#define ipipe_root_cpudom_var(var)	ipipe_root_cpudom_ptr()->var
+
+#define ipipe_this_cpudom_var(var)	\
+	ipipe_cpudom_var(__ipipe_current_domain, var)
+
+#define ipipe_head_cpudom_ptr()		\
+	(&__ipipe_get_cpu_var(ipipe_percpu_darray)[IPIPE_HEAD_SLOT])
+
+#define ipipe_head_cpudom_var(var)	ipipe_head_cpudom_ptr()->var
+
+#endif	/* !__LINUX_IPIPE_PERCPU_H */
diff --git a/include/linux/ipipe_tickdev.h b/include/linux/ipipe_tickdev.h
new file mode 100644
index 0000000..572e391
--- /dev/null
+++ b/include/linux/ipipe_tickdev.h
@@ -0,0 +1,87 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_tickdev.h
+ *
+ * Copyright (C) 2007 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_TICKDEV_H
+#define __LINUX_IPIPE_TICKDEV_H
+
+#include <linux/clocksource.h>
+
+#ifdef CONFIG_IPIPE
+/*
+ * NOTE: When modifying this structure, make sure to keep the Xenomai
+ * definition include/nucleus/vdso.h in sync.
+ */
+struct ipipe_hostrt_data {
+	short live;
+	seqcount_t seqcount;
+	time_t wall_time_sec;
+	u32 wall_time_nsec;
+	struct timespec wall_to_monotonic;
+	cycle_t cycle_last;
+	cycle_t mask;
+	u32 mult;
+	u32 shift;
+};
+
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+
+#include <linux/clockchips.h>
+
+struct tick_device;
+
+struct ipipe_tick_device {
+
+	void (*emul_set_mode)(enum clock_event_mode,
+			      struct clock_event_device *cdev);
+	int (*emul_set_tick)(unsigned long delta,
+			     struct clock_event_device *cdev);
+	void (*real_set_mode)(enum clock_event_mode mode,
+			      struct clock_event_device *cdev);
+	int (*real_set_tick)(unsigned long delta,
+			     struct clock_event_device *cdev);
+	struct tick_device *slave;
+	unsigned long real_max_delta_ns;
+	unsigned long real_mult;
+	int real_shift;
+};
+
+int ipipe_request_tickdev(const char *devname,
+			  void (*emumode)(enum clock_event_mode mode,
+					  struct clock_event_device *cdev),
+			  int (*emutick)(unsigned long evt,
+					 struct clock_event_device *cdev),
+			  int cpu, unsigned long *tmfreq);
+
+void ipipe_release_tickdev(int cpu);
+
+#endif /* CONFIG_GENERIC_CLOCKEVENTS */
+
+#ifdef CONFIG_HAVE_IPIPE_HOSTRT
+void ipipe_update_hostrt(struct timespec *wall_time,
+			 struct clocksource *clock);
+#else /* !CONFIG_IPIPE_HOSTRT */
+static inline void
+ipipe_update_hostrt(struct timespec *wall_time, struct clocksource *clock) {}
+#endif
+
+#endif /* CONFIG_IPIPE */
+
+#endif /* !__LINUX_IPIPE_TICKDEV_H */
diff --git a/include/linux/ipipe_trace.h b/include/linux/ipipe_trace.h
new file mode 100644
index 0000000..627b354
--- /dev/null
+++ b/include/linux/ipipe_trace.h
@@ -0,0 +1,72 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_trace.h
+ *
+ * Copyright (C) 2005 Luotao Fu.
+ *               2005-2007 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef _LINUX_IPIPE_TRACE_H
+#define _LINUX_IPIPE_TRACE_H
+
+#ifdef CONFIG_IPIPE_TRACE
+
+#include <linux/types.h>
+
+void ipipe_trace_begin(unsigned long v);
+void ipipe_trace_end(unsigned long v);
+void ipipe_trace_freeze(unsigned long v);
+void ipipe_trace_special(unsigned char special_id, unsigned long v);
+void ipipe_trace_pid(pid_t pid, short prio);
+void ipipe_trace_event(unsigned char id, unsigned long delay_tsc);
+int ipipe_trace_max_reset(void);
+int ipipe_trace_frozen_reset(void);
+
+#else /* !CONFIG_IPIPE_TRACE */
+
+#define ipipe_trace_begin(v)			do { (void)(v); } while(0)
+#define ipipe_trace_end(v)			do { (void)(v); } while(0)
+#define ipipe_trace_freeze(v)			do { (void)(v); } while(0)
+#define ipipe_trace_special(id, v)		do { (void)(id); (void)(v); } while(0)
+#define ipipe_trace_pid(pid, prio)		do { (void)(pid); (void)(prio); } while(0)
+#define ipipe_trace_event(id, delay_tsc)	do { (void)(id); (void)(delay_tsc); } while(0)
+#define ipipe_trace_max_reset()			do { } while(0)
+#define ipipe_trace_froze_reset()		do { } while(0)
+
+#endif /* !CONFIG_IPIPE_TRACE */
+
+#ifdef CONFIG_IPIPE_TRACE_PANIC
+void ipipe_trace_panic_freeze(void);
+void ipipe_trace_panic_dump(void);
+#else
+static inline void ipipe_trace_panic_freeze(void) { }
+static inline void ipipe_trace_panic_dump(void) { }
+#endif
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+#define ipipe_trace_irq_entry(irq)	ipipe_trace_begin(irq)
+#define ipipe_trace_irq_exit(irq)	ipipe_trace_end(irq)
+#define ipipe_trace_irqsoff()		ipipe_trace_begin(0x80000000UL)
+#define ipipe_trace_irqson()		ipipe_trace_end(0x80000000UL)
+#else
+#define ipipe_trace_irq_entry(irq)	do { (void)(irq);} while(0)
+#define ipipe_trace_irq_exit(irq)	do { (void)(irq);} while(0)
+#define ipipe_trace_irqsoff()		do { } while(0)
+#define ipipe_trace_irqson()		do { } while(0)
+#endif
+
+#endif	/* !__LINUX_IPIPE_TRACE_H */
diff --git a/include/linux/irq.h b/include/linux/irq.h
index 80fcb53..3d93391 100644
--- a/include/linux/irq.h
+++ b/include/linux/irq.h
@@ -195,6 +195,9 @@ struct irq_chip {
 
 	void		(*irq_bus_lock)(struct irq_data *data);
 	void		(*irq_bus_sync_unlock)(struct irq_data *data);
+#ifdef CONFIG_IPIPE
+	void		(*irq_move)(struct irq_data *data);
+#endif /* CONFIG_IPIPE */
 
 	/* Currently used only by UML, might disappear one day.*/
 #ifdef CONFIG_IRQ_RELEASE_METHOD
@@ -343,7 +346,7 @@ extern int set_irq_data(unsigned int irq, void *data);
 extern int set_irq_chip_data(unsigned int irq, void *data);
 extern int set_irq_type(unsigned int irq, unsigned int type);
 extern int set_irq_msi(unsigned int irq, struct msi_desc *entry);
-extern struct irq_data *irq_get_irq_data(unsigned int irq);
+extern struct irq_data *irq_get_irq_data(unsigned int irq) __attribute__((const));
 
 static inline struct irq_chip *get_irq_chip(unsigned int irq)
 {
diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h
index c1a95b7..323a4b7 100644
--- a/include/linux/irqdesc.h
+++ b/include/linux/irqdesc.h
@@ -55,6 +55,12 @@ struct irq_desc {
 		};
 	};
 #endif
+#ifdef CONFIG_IPIPE
+	void			(*ipipe_ack)(unsigned int irq,
+					     struct irq_desc *desc);
+	void			(*ipipe_end)(unsigned int irq,
+					     struct irq_desc *desc);
+#endif /* CONFIG_IPIPE */
 
 	struct timer_rand_state *timer_rand_state;
 	unsigned int __percpu	*kstat_irqs;
@@ -131,6 +137,10 @@ static inline int irq_balancing_disabled(unsigned int irq)
 	return desc->status & IRQ_NO_BALANCING_MASK;
 }
 
+irq_flow_handler_t
+__fixup_irq_handler(struct irq_desc *desc, irq_flow_handler_t handle,
+		    int is_chained);
+
 /* caller has locked the irq_desc and both params are valid */
 static inline void __set_irq_handler_unlocked(int irq,
 					      irq_flow_handler_t handler)
@@ -138,6 +148,7 @@ static inline void __set_irq_handler_unlocked(int irq,
 	struct irq_desc *desc;
 
 	desc = irq_to_desc(irq);
+	handler = __fixup_irq_handler(desc, handler, 0);
 	desc->handle_irq = handler;
 }
 #endif
diff --git a/include/linux/irqnr.h b/include/linux/irqnr.h
index 3bc4dca..fb1f848 100644
--- a/include/linux/irqnr.h
+++ b/include/linux/irqnr.h
@@ -24,7 +24,11 @@
 #else /* CONFIG_GENERIC_HARDIRQS */
 
 extern int nr_irqs;
+#if !defined(CONFIG_IPIPE) || defined(CONFIG_SPARSE_IRQ)
 extern struct irq_desc *irq_to_desc(unsigned int irq);
+#else
+#define irq_to_desc(irq)	(&irq_desc[irq])
+#endif
 unsigned int irq_get_next_irq(unsigned int offset);
 
 # define for_each_irq_desc(irq, desc)					\
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index 00cec4d..e4f44b9 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -16,6 +16,7 @@
 #include <linux/compiler.h>
 #include <linux/bitops.h>
 #include <linux/log2.h>
+#include <linux/ipipe_base.h>
 #include <linux/typecheck.h>
 #include <linux/printk.h>
 #include <linux/dynamic_debug.h>
@@ -116,9 +117,12 @@ struct user;
 
 #ifdef CONFIG_PREEMPT_VOLUNTARY
 extern int _cond_resched(void);
-# define might_resched() _cond_resched()
+# define might_resched() do { \
+		ipipe_check_context(ipipe_root_domain); \
+		_cond_resched(); \
+	} while (0)
 #else
-# define might_resched() do { } while (0)
+# define might_resched() ipipe_check_context(ipipe_root_domain)
 #endif
 
 #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index 2e681d9..130b7d5 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -9,13 +9,20 @@
 #include <linux/thread_info.h>
 #include <linux/linkage.h>
 #include <linux/list.h>
+#include <linux/ipipe_base.h>
 
 #if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER)
   extern void add_preempt_count(int val);
   extern void sub_preempt_count(int val);
 #else
-# define add_preempt_count(val)	do { preempt_count() += (val); } while (0)
-# define sub_preempt_count(val)	do { preempt_count() -= (val); } while (0)
+# define add_preempt_count(val)	do {		\
+    ipipe_check_context(ipipe_root_domain);	\
+    preempt_count() += (val);			\
+  } while (0)
+# define sub_preempt_count(val)	do {		\
+    ipipe_check_context(ipipe_root_domain);	\
+    preempt_count() -= (val);			\
+  } while (0)
 #endif
 
 #define inc_preempt_count() add_preempt_count(1)
diff --git a/include/linux/resource.h b/include/linux/resource.h
index d01c96c..9e4eb7c 100644
--- a/include/linux/resource.h
+++ b/include/linux/resource.h
@@ -59,12 +59,6 @@ struct rlimit64 {
 #define	PRIO_USER	2
 
 /*
- * Limit the stack by to some sane default: root can always
- * increase this limit if needed..  8MB seems reasonable.
- */
-#define _STK_LIM	(8*1024*1024)
-
-/*
  * GPG2 wants 64kB of mlocked memory, to make sure pass phrases
  * and other sensitive information are never written to disk.
  */
diff --git a/include/linux/rwlock.h b/include/linux/rwlock.h
index bc2994e..5e2da8d 100644
--- a/include/linux/rwlock.h
+++ b/include/linux/rwlock.h
@@ -61,8 +61,8 @@ do {								\
 #define read_trylock(lock)	__cond_lock(lock, _raw_read_trylock(lock))
 #define write_trylock(lock)	__cond_lock(lock, _raw_write_trylock(lock))
 
-#define write_lock(lock)	_raw_write_lock(lock)
-#define read_lock(lock)		_raw_read_lock(lock)
+#define write_lock(lock)	PICK_RWOP(_write_lock, lock)
+#define read_lock(lock)		PICK_RWOP(_read_lock, lock)
 
 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 
@@ -96,8 +96,8 @@ do {								\
 #define read_lock_bh(lock)		_raw_read_lock_bh(lock)
 #define write_lock_irq(lock)		_raw_write_lock_irq(lock)
 #define write_lock_bh(lock)		_raw_write_lock_bh(lock)
-#define read_unlock(lock)		_raw_read_unlock(lock)
-#define write_unlock(lock)		_raw_write_unlock(lock)
+#define read_unlock(lock)		PICK_RWOP(_read_unlock, lock)
+#define write_unlock(lock)		PICK_RWOP(_write_unlock, lock)
 #define read_unlock_irq(lock)		_raw_read_unlock_irq(lock)
 #define write_unlock_irq(lock)		_raw_write_unlock_irq(lock)
 
diff --git a/include/linux/rwlock_api_smp.h b/include/linux/rwlock_api_smp.h
index 9c9f049..62c8941 100644
--- a/include/linux/rwlock_api_smp.h
+++ b/include/linux/rwlock_api_smp.h
@@ -141,7 +141,9 @@ static inline int __raw_write_trylock(rwlock_t *lock)
  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
  */
-#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
+#if !defined(CONFIG_GENERIC_LOCKBREAK) ||	\
+	defined(CONFIG_DEBUG_LOCK_ALLOC) ||	\
+	defined(CONFIG_IPIPE)
 
 static inline void __raw_read_lock(rwlock_t *lock)
 {
diff --git a/include/linux/sched.h b/include/linux/sched.h
index d2a5da9..061b662 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -62,6 +62,7 @@ struct sched_param {
 #include <linux/errno.h>
 #include <linux/nodemask.h>
 #include <linux/mm_types.h>
+#include <linux/ipipe.h>
 
 #include <asm/system.h>
 #include <asm/page.h>
@@ -190,9 +191,17 @@ print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
 #define TASK_DEAD		64
 #define TASK_WAKEKILL		128
 #define TASK_WAKING		256
+#ifdef CONFIG_IPIPE
+#define TASK_ATOMICSWITCH	512
+#define TASK_NOWAKEUP		1024
+#define TASK_STATE_MAX		2048
+#define TASK_STATE_TO_CHAR_STR "RSDTtZXxKWAN"
+#else  /* !CONFIG_IPIPE */
+#define TASK_ATOMICSWITCH	0
+#define TASK_NOWAKEUP		0
 #define TASK_STATE_MAX		512
-
 #define TASK_STATE_TO_CHAR_STR "RSDTtZXxKW"
+#endif /* CONFIG_IPIPE */
 
 extern char ___assert_task_state[1 - 2*!!(
 		sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1)];
@@ -304,6 +313,15 @@ extern void trap_init(void);
 extern void update_process_times(int user);
 extern void scheduler_tick(void);
 
+#ifdef CONFIG_IPIPE
+void update_root_process_times(struct pt_regs *regs);
+#else  /* !CONFIG_IPIPE */
+static inline void update_root_process_times(struct pt_regs *regs)
+{
+	update_process_times(user_mode(regs));
+}
+#endif /* CONFIG_IPIPE */
+
 extern void sched_show_task(struct task_struct *p);
 
 #ifdef CONFIG_LOCKUP_DETECTOR
@@ -358,7 +376,7 @@ extern signed long schedule_timeout(signed long timeout);
 extern signed long schedule_timeout_interruptible(signed long timeout);
 extern signed long schedule_timeout_killable(signed long timeout);
 extern signed long schedule_timeout_uninterruptible(signed long timeout);
-asmlinkage void schedule(void);
+asmlinkage int schedule(void);
 extern int mutex_spin_on_owner(struct mutex *lock, struct thread_info *owner);
 
 struct nsproxy;
@@ -435,6 +453,9 @@ extern int get_dumpable(struct mm_struct *mm);
 					/* leave room for more dump flags */
 #define MMF_VM_MERGEABLE	16	/* KSM may merge identical pages */
 #define MMF_VM_HUGEPAGE		17	/* set when VM_HUGEPAGE is set on vma */
+#ifdef CONFIG_IPIPE
+#define MMF_VM_PINNED		31	/* ondemand load up and COW disabled */
+#endif
 
 #define MMF_INIT_MASK		(MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK)
 
@@ -1478,6 +1499,10 @@ struct task_struct {
 #endif
 	atomic_t fs_excl;	/* holding fs exclusive resources */
 	struct rcu_head rcu;
+#ifdef CONFIG_IPIPE
+	unsigned int ipipe_flags;
+	void *ptd[IPIPE_ROOT_NPTDKEYS];
+#endif
 
 	/*
 	 * cache last used pipe for splice
@@ -1753,6 +1778,10 @@ extern void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *
 #define PF_FREEZER_SKIP	0x40000000	/* Freezer should not count it as freezable */
 #define PF_FREEZER_NOSIG 0x80000000	/* Freezer won't send signals to it */
 
+/* p->ipipe_flags */
+#define PF_EVTRET	0x1	/* EVENT_RETURN is pending */
+#define PF_EVNOTIFY	0x2	/* Notify other domains about internal events */
+
 /*
  * Only the _current_ task can read/write to tsk->flags, but other
  * tasks can access tsk->flags in readonly mode for example
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index 80e5358..2e29b16 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -89,10 +89,12 @@
 # include <linux/spinlock_up.h>
 #endif
 
+#include <linux/ipipe_lock.h>
+
 #ifdef CONFIG_DEBUG_SPINLOCK
   extern void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
 				   struct lock_class_key *key);
-# define raw_spin_lock_init(lock)				\
+# define __real_raw_spin_lock_init(lock)			\
 do {								\
 	static struct lock_class_key __key;			\
 								\
@@ -100,9 +102,10 @@ do {								\
 } while (0)
 
 #else
-# define raw_spin_lock_init(lock)				\
+# define __real_raw_spin_lock_init(lock)			\
 	do { *(lock) = __RAW_SPIN_LOCK_UNLOCKED(lock); } while (0)
 #endif
+#define raw_spin_lock_init(lock)	PICK_SPINOP(_lock_init, lock)
 
 #define raw_spin_is_locked(lock)	arch_spin_is_locked(&(lock)->raw_lock)
 
@@ -165,9 +168,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
  * various methods are defined as nops in the case they are not
  * required.
  */
-#define raw_spin_trylock(lock)	__cond_lock(lock, _raw_spin_trylock(lock))
+#define __real_raw_spin_trylock(lock)	__cond_lock(lock, _raw_spin_trylock(lock))
+#define raw_spin_trylock(lock)		PICK_SPINOP_RET(_trylock, lock, int)
 
-#define raw_spin_lock(lock)	_raw_spin_lock(lock)
+#define __real_raw_spin_lock(lock)	_raw_spin_lock(lock)
+#define raw_spin_lock(lock)		PICK_SPINOP(_lock, lock)
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 # define raw_spin_lock_nested(lock, subclass) \
@@ -185,7 +190,7 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
 
 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 
-#define raw_spin_lock_irqsave(lock, flags)			\
+#define __real_raw_spin_lock_irqsave(lock, flags)	\
 	do {						\
 		typecheck(unsigned long, flags);	\
 		flags = _raw_spin_lock_irqsave(lock);	\
@@ -207,7 +212,7 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
 
 #else
 
-#define raw_spin_lock_irqsave(lock, flags)		\
+#define __real_raw_spin_lock_irqsave(lock, flags)	\
 	do {						\
 		typecheck(unsigned long, flags);	\
 		_raw_spin_lock_irqsave(lock, flags);	\
@@ -218,34 +223,46 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
 
 #endif
 
-#define raw_spin_lock_irq(lock)		_raw_spin_lock_irq(lock)
+#define raw_spin_lock_irqsave(lock, flags)  \
+	PICK_SPINLOCK_IRQSAVE(lock, flags)
+
+#define __real_raw_spin_lock_irq(lock)	_raw_spin_lock_irq(lock)
+#define raw_spin_lock_irq(lock)		PICK_SPINOP(_lock_irq, lock)
 #define raw_spin_lock_bh(lock)		_raw_spin_lock_bh(lock)
-#define raw_spin_unlock(lock)		_raw_spin_unlock(lock)
-#define raw_spin_unlock_irq(lock)	_raw_spin_unlock_irq(lock)
+#define __real_raw_spin_unlock(lock)	_raw_spin_unlock(lock)
+#define raw_spin_unlock(lock)		PICK_SPINOP(_unlock, lock)
+#define __real_raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock)
+#define raw_spin_unlock_irq(lock)	PICK_SPINOP(_unlock_irq, lock)
 
-#define raw_spin_unlock_irqrestore(lock, flags)		\
+#define __real_raw_spin_unlock_irqrestore(lock, flags)		\
 	do {							\
 		typecheck(unsigned long, flags);		\
 		_raw_spin_unlock_irqrestore(lock, flags);	\
 	} while (0)
+#define raw_spin_unlock_irqrestore(lock, flags)	\
+	PICK_SPINUNLOCK_IRQRESTORE(lock, flags)
+
 #define raw_spin_unlock_bh(lock)	_raw_spin_unlock_bh(lock)
 
 #define raw_spin_trylock_bh(lock) \
 	__cond_lock(lock, _raw_spin_trylock_bh(lock))
 
-#define raw_spin_trylock_irq(lock) \
+#define __real_raw_spin_trylock_irq(lock) \
 ({ \
 	local_irq_disable(); \
-	raw_spin_trylock(lock) ? \
+	__real_raw_spin_trylock(lock) ? \
 	1 : ({ local_irq_enable(); 0;  }); \
 })
+#define raw_spin_trylock_irq(lock)	PICK_SPINTRYLOCK_IRQ(lock)
 
-#define raw_spin_trylock_irqsave(lock, flags) \
+#define __real_raw_spin_trylock_irqsave(lock, flags) \
 ({ \
 	local_irq_save(flags); \
 	raw_spin_trylock(lock) ? \
 	1 : ({ local_irq_restore(flags); 0; }); \
 })
+#define raw_spin_trylock_irqsave(lock, flags)	\
+	PICK_SPINTRYLOCK_IRQSAVE(lock, flags)
 
 /**
  * raw_spin_can_lock - would raw_spin_trylock() succeed?
@@ -276,24 +293,17 @@ static inline raw_spinlock_t *spinlock_check(spinlock_t *lock)
 
 #define spin_lock_init(_lock)				\
 do {							\
-	spinlock_check(_lock);				\
-	raw_spin_lock_init(&(_lock)->rlock);		\
+	raw_spin_lock_init(_lock);			\
 } while (0)
 
-static inline void spin_lock(spinlock_t *lock)
-{
-	raw_spin_lock(&lock->rlock);
-}
+#define spin_lock(lock)		raw_spin_lock(lock)
 
 static inline void spin_lock_bh(spinlock_t *lock)
 {
 	raw_spin_lock_bh(&lock->rlock);
 }
 
-static inline int spin_trylock(spinlock_t *lock)
-{
-	return raw_spin_trylock(&lock->rlock);
-}
+#define spin_trylock(lock)	raw_spin_trylock(lock)
 
 #define spin_lock_nested(lock, subclass)			\
 do {								\
@@ -305,14 +315,11 @@ do {									\
 	raw_spin_lock_nest_lock(spinlock_check(lock), nest_lock);	\
 } while (0)
 
-static inline void spin_lock_irq(spinlock_t *lock)
-{
-	raw_spin_lock_irq(&lock->rlock);
-}
+#define spin_lock_irq(lock)	raw_spin_lock_irq(lock)
 
 #define spin_lock_irqsave(lock, flags)				\
 do {								\
-	raw_spin_lock_irqsave(spinlock_check(lock), flags);	\
+	raw_spin_lock_irqsave(lock, flags);			\
 } while (0)
 
 #define spin_lock_irqsave_nested(lock, flags, subclass)			\
@@ -320,39 +327,28 @@ do {									\
 	raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \
 } while (0)
 
-static inline void spin_unlock(spinlock_t *lock)
-{
-	raw_spin_unlock(&lock->rlock);
-}
+#define spin_unlock(lock)	raw_spin_unlock(lock)
 
 static inline void spin_unlock_bh(spinlock_t *lock)
 {
 	raw_spin_unlock_bh(&lock->rlock);
 }
 
-static inline void spin_unlock_irq(spinlock_t *lock)
-{
-	raw_spin_unlock_irq(&lock->rlock);
-}
+#define spin_unlock_irq(lock)	raw_spin_unlock_irq(lock)
 
-static inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
-{
-	raw_spin_unlock_irqrestore(&lock->rlock, flags);
-}
+#define spin_unlock_irqrestore(lock, flags)	\
+	raw_spin_unlock_irqrestore(lock, flags)
 
 static inline int spin_trylock_bh(spinlock_t *lock)
 {
 	return raw_spin_trylock_bh(&lock->rlock);
 }
 
-static inline int spin_trylock_irq(spinlock_t *lock)
-{
-	return raw_spin_trylock_irq(&lock->rlock);
-}
+#define spin_trylock_irq(lock)	raw_spin_trylock_irq(lock)
 
 #define spin_trylock_irqsave(lock, flags)			\
 ({								\
-	raw_spin_trylock_irqsave(spinlock_check(lock), flags); \
+	raw_spin_trylock_irqsave(lock, flags);			\
 })
 
 static inline void spin_unlock_wait(spinlock_t *lock)
diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h
index e253ccd..378e01e 100644
--- a/include/linux/spinlock_api_smp.h
+++ b/include/linux/spinlock_api_smp.h
@@ -99,7 +99,9 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock)
  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
  */
-#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
+#if !defined(CONFIG_GENERIC_LOCKBREAK) ||	\
+	defined(CONFIG_DEBUG_LOCK_ALLOC) ||	\
+	defined(CONFIG_IPIPE)
 
 static inline unsigned long __raw_spin_lock_irqsave(raw_spinlock_t *lock)
 {
@@ -113,7 +115,7 @@ static inline unsigned long __raw_spin_lock_irqsave(raw_spinlock_t *lock)
 	 * do_raw_spin_lock_flags() code, because lockdep assumes
 	 * that interrupts are not re-enabled during lock-acquire:
 	 */
-#ifdef CONFIG_LOCKDEP
+#if defined(CONFIG_LOCKDEP) || defined(CONFIG_IPIPE)
 	LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
 #else
 	do_raw_spin_lock_flags(lock, &flags);
diff --git a/include/linux/spinlock_up.h b/include/linux/spinlock_up.h
index b14f6a9..e400972 100644
--- a/include/linux/spinlock_up.h
+++ b/include/linux/spinlock_up.h
@@ -49,13 +49,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock)
 /*
  * Read-write spinlocks. No debug version.
  */
-#define arch_read_lock(lock)		do { (void)(lock); } while (0)
-#define arch_write_lock(lock)		do { (void)(lock); } while (0)
-#define arch_read_trylock(lock)	({ (void)(lock); 1; })
-#define arch_write_trylock(lock)	({ (void)(lock); 1; })
-#define arch_read_unlock(lock)		do { (void)(lock); } while (0)
-#define arch_write_unlock(lock)	do { (void)(lock); } while (0)
-
 #else /* DEBUG_SPINLOCK */
 #define arch_spin_is_locked(lock)	((void)(lock), 0)
 /* for sched.c and kernel_lock.c: */
@@ -65,6 +58,13 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock)
 # define arch_spin_trylock(lock)	({ (void)(lock); 1; })
 #endif /* DEBUG_SPINLOCK */
 
+#define arch_read_lock(lock)		do { (void)(lock); } while (0)
+#define arch_write_lock(lock)		do { (void)(lock); } while (0)
+#define arch_read_trylock(lock)		({ (void)(lock); 1; })
+#define arch_write_trylock(lock)	({ (void)(lock); 1; })
+#define arch_read_unlock(lock)		do { (void)(lock); } while (0)
+#define arch_write_unlock(lock)		do { (void)(lock); } while (0)
+
 #define arch_spin_is_contended(lock)	(((void)(lock), 0))
 
 #define arch_read_can_lock(lock)	(((void)(lock), 1))
diff --git a/init/Kconfig b/init/Kconfig
index 47dd02f..e432dc8 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -93,6 +93,7 @@ config CROSS_COMPILE
 
 config LOCALVERSION
 	string "Local version - append to kernel release"
+	default "-ipipe"
 	help
 	  Append an extra string to the end of your kernel version.
 	  This will show up when you type uname, for example.
diff --git a/init/main.c b/init/main.c
index a7fcde2..5054188 100644
--- a/init/main.c
+++ b/init/main.c
@@ -562,7 +562,7 @@ asmlinkage void __init start_kernel(void)
 
 	cgroup_init_early();
 
-	local_irq_disable();
+	local_irq_disable_hw();
 	early_boot_irqs_disabled = true;
 
 /*
@@ -595,6 +595,7 @@ asmlinkage void __init start_kernel(void)
 	pidhash_init();
 	vfs_caches_init_early();
 	sort_main_extable();
+	ipipe_init_early();
 	trap_init();
 	mm_init();
 	/*
@@ -626,6 +627,11 @@ asmlinkage void __init start_kernel(void)
 	softirq_init();
 	timekeeping_init();
 	time_init();
+	/*
+	 * We need to wait for the interrupt and time subsystems to be
+	 * initialized before enabling the pipeline.
+	 */
+	ipipe_init();
 	profile_init();
 	if (!irqs_disabled())
 		printk(KERN_CRIT "start_kernel(): bug: interrupts were "
@@ -802,6 +808,7 @@ static void __init do_basic_setup(void)
 	init_tmpfs();
 	driver_init();
 	init_irq_proc();
+  	ipipe_init_proc();
 	do_ctors();
 	do_initcalls();
 }
diff --git a/kernel/Makefile b/kernel/Makefile
index 353d3fe..3ec3ffc 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -88,6 +88,7 @@ obj-$(CONFIG_TREE_RCU_TRACE) += rcutree_trace.o
 obj-$(CONFIG_TINY_RCU) += rcutiny.o
 obj-$(CONFIG_TINY_PREEMPT_RCU) += rcutiny.o
 obj-$(CONFIG_RELAY) += relay.o
+obj-$(CONFIG_IPIPE) += ipipe/
 obj-$(CONFIG_SYSCTL) += utsname_sysctl.o
 obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
 obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
diff --git a/kernel/exit.c b/kernel/exit.c
index 557a348..da81dae 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -989,6 +989,7 @@ NORET_TYPE void do_exit(long code)
 		acct_process();
 	trace_sched_process_exit(tsk);
 
+  	ipipe_exit_notify(tsk);
 	exit_sem(tsk);
 	exit_files(tsk);
 	exit_fs(tsk);
diff --git a/kernel/fork.c b/kernel/fork.c
index 25e4291..b0d6fe8 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -552,6 +552,7 @@ void mmput(struct mm_struct *mm)
 		ksm_exit(mm);
 		khugepaged_exit(mm); /* must run before exit_mmap */
 		exit_mmap(mm);
+ 		ipipe_cleanup_notify(mm);
 		set_mm_exe_file(mm, NULL);
 		if (!list_empty(&mm->mmlist)) {
 			spin_lock(&mmlist_lock);
@@ -937,6 +938,7 @@ static void copy_flags(unsigned long clone_flags, struct task_struct *p)
 	new_flags |= PF_FORKNOEXEC;
 	new_flags |= PF_STARTING;
 	p->flags = new_flags;
+	ipipe_clear_flags(p);
 	clear_freeze_flag(p);
 }
 
@@ -1309,6 +1311,10 @@ static struct task_struct *copy_process(unsigned long clone_flags,
 	write_unlock_irq(&tasklist_lock);
 	proc_fork_connector(p);
 	cgroup_post_fork(p);
+#ifdef CONFIG_IPIPE
+	p->ipipe_flags = 0;
+	memset(p->ptd, 0, sizeof(p->ptd));
+#endif /* CONFIG_IPIPE */
 	perf_event_fork(p);
 	return p;
 
@@ -1690,15 +1696,18 @@ SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
 		}
 
 		if (new_mm) {
+			unsigned long flags;
 			mm = current->mm;
 			active_mm = current->active_mm;
 			current->mm = new_mm;
+			ipipe_mm_switch_protect(flags);
 			current->active_mm = new_mm;
 			if (current->signal->oom_score_adj == OOM_SCORE_ADJ_MIN) {
 				atomic_dec(&mm->oom_disable_count);
 				atomic_inc(&new_mm->oom_disable_count);
 			}
 			activate_mm(active_mm, new_mm);
+			ipipe_mm_switch_unprotect(flags);
 			new_mm = mm;
 		}
 
diff --git a/kernel/ipipe/Kconfig b/kernel/ipipe/Kconfig
new file mode 100644
index 0000000..693a7d2
--- /dev/null
+++ b/kernel/ipipe/Kconfig
@@ -0,0 +1,26 @@
+config IPIPE
+	bool "Interrupt pipeline"
+	default y
+	---help---
+	  Activate this option if you want the interrupt pipeline to be
+	  compiled in.
+
+config IPIPE_DOMAINS
+	int "Max domains"
+	depends on IPIPE
+	default 4
+	---help---
+	The maximum number of I-pipe domains to run concurrently.
+
+config IPIPE_DELAYED_ATOMICSW
+       bool
+       depends on IPIPE
+       default n
+
+config IPIPE_UNMASKED_CONTEXT_SWITCH
+       bool
+       depends on IPIPE
+       default n
+
+config HAVE_IPIPE_HOSTRT
+       bool
diff --git a/kernel/ipipe/Kconfig.debug b/kernel/ipipe/Kconfig.debug
new file mode 100644
index 0000000..ba87335
--- /dev/null
+++ b/kernel/ipipe/Kconfig.debug
@@ -0,0 +1,95 @@
+config IPIPE_DEBUG
+	bool "I-pipe debugging"
+	depends on IPIPE
+
+config IPIPE_DEBUG_CONTEXT
+	bool "Check for illicit cross-domain calls"
+	depends on IPIPE_DEBUG
+	default y
+	---help---
+	  Enable this feature to arm checkpoints in the kernel that
+	  verify the correct invocation context. On entry of critical
+	  Linux services a warning is issued if the caller is not
+	  running over the root domain.
+
+config IPIPE_DEBUG_INTERNAL
+	bool "Enable internal debug checks"
+	depends on IPIPE_DEBUG
+	default y
+	---help---
+	  When this feature is enabled, I-pipe will perform internal
+	  consistency checks of its subsystems, e.g. on per-cpu variable
+	  access.
+
+config IPIPE_TRACE
+	bool "Latency tracing"
+	depends on IPIPE_DEBUG
+	select ARCH_WANT_FRAME_POINTERS
+	select FRAME_POINTER
+	select KALLSYMS
+	select PROC_FS
+	---help---
+	  Activate this option if you want to use per-function tracing of
+	  the kernel. The tracer will collect data via instrumentation
+	  features like the one below or with the help of explicite calls
+	  of ipipe_trace_xxx(). See include/linux/ipipe_trace.h for the
+	  in-kernel tracing API. The collected data and runtime control
+	  is available via /proc/ipipe/trace/*.
+
+if IPIPE_TRACE
+
+config IPIPE_TRACE_ENABLE
+	bool "Enable tracing on boot"
+	default y
+	---help---
+	  Disable this option if you want to arm the tracer after booting
+	  manually ("echo 1 > /proc/ipipe/tracer/enable"). This can reduce
+	  boot time on slow embedded devices due to the tracer overhead.
+
+config IPIPE_TRACE_MCOUNT
+	bool "Instrument function entries"
+	default y
+	select FTRACE
+	select FUNCTION_TRACER
+	---help---
+	  When enabled, records every kernel function entry in the tracer
+	  log. While this slows down the system noticeably, it provides
+	  the highest level of information about the flow of events.
+	  However, it can be switch off in order to record only explicit
+	  I-pipe trace points.
+
+config IPIPE_TRACE_IRQSOFF
+	bool "Trace IRQs-off times"
+	default y
+	---help---
+	  Activate this option if I-pipe shall trace the longest path
+	  with hard-IRQs switched off.
+
+config IPIPE_TRACE_SHIFT
+	int "Depth of trace log (14 => 16Kpoints, 15 => 32Kpoints)"
+	range 10 18
+	default 14
+	---help---
+	  The number of trace points to hold tracing data for each
+	  trace path, as a power of 2.
+
+config IPIPE_TRACE_VMALLOC
+	bool "Use vmalloc'ed trace buffer"
+	default y if EMBEDDED
+	---help---
+	  Instead of reserving static kernel data, the required buffer
+	  is allocated via vmalloc during boot-up when this option is
+	  enabled. This can help to start systems that are low on memory,
+	  but it slightly degrades overall performance. Try this option
+	  when a traced kernel hangs unexpectedly at boot time.
+
+config IPIPE_TRACE_PANIC
+	bool "Enable panic back traces"
+	default y
+	---help---
+	  Provides services to freeze and dump a back trace on panic
+	  situations. This is used on IPIPE_DEBUG_CONTEXT exceptions
+	  as well as ordinary kernel oopses. You can control the number
+	  of printed back trace points via /proc/ipipe/trace.
+
+endif
diff --git a/kernel/ipipe/Makefile b/kernel/ipipe/Makefile
new file mode 100644
index 0000000..6257dfa
--- /dev/null
+++ b/kernel/ipipe/Makefile
@@ -0,0 +1,3 @@
+
+obj-$(CONFIG_IPIPE)	+= core.o
+obj-$(CONFIG_IPIPE_TRACE) += tracer.o
diff --git a/kernel/ipipe/core.c b/kernel/ipipe/core.c
new file mode 100644
index 0000000..81a5cc4
--- /dev/null
+++ b/kernel/ipipe/core.c
@@ -0,0 +1,2194 @@
+/* -*- linux-c -*-
+ * linux/kernel/ipipe/core.c
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-independent I-PIPE core support.
+ */
+
+#include <linux/version.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/sched.h>
+#include <linux/kallsyms.h>
+#include <linux/interrupt.h>
+#include <linux/bitops.h>
+#include <linux/tick.h>
+#ifdef CONFIG_PROC_FS
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#endif	/* CONFIG_PROC_FS */
+#include <linux/ipipe_trace.h>
+#include <linux/ipipe_tickdev.h>
+#include <linux/irq.h>
+
+static int __ipipe_ptd_key_count;
+
+static unsigned long __ipipe_ptd_key_map;
+
+static unsigned long __ipipe_domain_slot_map;
+
+struct ipipe_domain ipipe_root;
+
+#ifdef CONFIG_SMP
+
+#define IPIPE_CRITICAL_TIMEOUT	1000000
+
+static cpumask_t __ipipe_cpu_sync_map;
+
+static cpumask_t __ipipe_cpu_lock_map;
+
+static cpumask_t __ipipe_cpu_pass_map;
+
+static unsigned long __ipipe_critical_lock;
+
+static IPIPE_DEFINE_SPINLOCK(__ipipe_cpu_barrier);
+
+static atomic_t __ipipe_critical_count = ATOMIC_INIT(0);
+
+static void (*__ipipe_cpu_sync) (void);
+
+#else /* !CONFIG_SMP */
+
+/*
+ * Create an alias to the unique root status, so that arch-dep code
+ * may get simple and easy access to this percpu variable.  We also
+ * create an array of pointers to the percpu domain data; this tends
+ * to produce a better code when reaching non-root domains. We make
+ * sure that the early boot code would be able to dereference the
+ * pointer to the root domain data safely by statically initializing
+ * its value (local_irq*() routines depend on this).
+ */
+#if __GNUC__ >= 4
+extern unsigned long __ipipe_root_status
+__attribute__((alias(__stringify(ipipe_percpu_darray))));
+EXPORT_SYMBOL(__ipipe_root_status);
+#else /* __GNUC__ < 4 */
+/*
+ * Work around a GCC 3.x issue making alias symbols unusable as
+ * constant initializers.
+ */
+unsigned long *const __ipipe_root_status_addr =
+	&__raw_get_cpu_var(ipipe_percpu_darray)[IPIPE_ROOT_SLOT].status;
+EXPORT_SYMBOL(__ipipe_root_status_addr);
+#endif /* __GNUC__ < 4 */
+
+DEFINE_PER_CPU(struct ipipe_percpu_domain_data *, ipipe_percpu_daddr[CONFIG_IPIPE_DOMAINS]) =
+{ [IPIPE_ROOT_SLOT] = (struct ipipe_percpu_domain_data *)ipipe_percpu_darray };
+EXPORT_PER_CPU_SYMBOL(ipipe_percpu_daddr);
+#endif /* !CONFIG_SMP */
+
+DEFINE_PER_CPU(struct ipipe_percpu_domain_data, ipipe_percpu_darray[CONFIG_IPIPE_DOMAINS]) =
+{ [IPIPE_ROOT_SLOT] = { .status = IPIPE_STALL_MASK } }; /* Root domain stalled on each CPU at startup. */
+
+DEFINE_PER_CPU(struct ipipe_domain *, ipipe_percpu_domain) = { &ipipe_root };
+
+DEFINE_PER_CPU(unsigned long, ipipe_nmi_saved_root); /* Copy of root status during NMI */
+
+static IPIPE_DEFINE_SPINLOCK(__ipipe_pipelock);
+
+LIST_HEAD(__ipipe_pipeline);
+
+unsigned long __ipipe_virtual_irq_map;
+
+#ifdef CONFIG_PRINTK
+unsigned __ipipe_printk_virq;
+#endif /* CONFIG_PRINTK */
+
+int __ipipe_event_monitors[IPIPE_NR_EVENTS];
+
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+
+DECLARE_PER_CPU(struct tick_device, tick_cpu_device);
+
+static DEFINE_PER_CPU(struct ipipe_tick_device, ipipe_tick_cpu_device);
+
+int ipipe_request_tickdev(const char *devname,
+			  void (*emumode)(enum clock_event_mode mode,
+					  struct clock_event_device *cdev),
+			  int (*emutick)(unsigned long delta,
+					 struct clock_event_device *cdev),
+			  int cpu, unsigned long *tmfreq)
+{
+	struct ipipe_tick_device *itd;
+	struct tick_device *slave;
+	struct clock_event_device *evtdev;
+	unsigned long long freq;
+	unsigned long flags;
+	int status;
+
+	flags = ipipe_critical_enter(NULL);
+
+	itd = &per_cpu(ipipe_tick_cpu_device, cpu);
+
+	if (itd->slave != NULL) {
+		status = -EBUSY;
+		goto out;
+	}
+
+	slave = &per_cpu(tick_cpu_device, cpu);
+
+	if (strcmp(slave->evtdev->name, devname)) {
+		/*
+		 * No conflict so far with the current tick device,
+		 * check whether the requested device is sane and has
+		 * been blessed by the kernel.
+		 */
+		status = __ipipe_check_tickdev(devname) ?
+			CLOCK_EVT_MODE_UNUSED : CLOCK_EVT_MODE_SHUTDOWN;
+		goto out;
+	}
+
+	/*
+	 * Our caller asks for using the same clock event device for
+	 * ticking than we do, let's create a tick emulation device to
+	 * interpose on the set_next_event() method, so that we may
+	 * both manage the device in oneshot mode. Only the tick
+	 * emulation code will actually program the clockchip hardware
+	 * for the next shot, though.
+	 *
+	 * CAUTION: we still have to grab the tick device even when it
+	 * current runs in periodic mode, since the kernel may switch
+	 * to oneshot dynamically (highres/no_hz tick mode).
+	 */
+
+	evtdev = slave->evtdev;
+	status = evtdev->mode;
+
+	if (status == CLOCK_EVT_MODE_SHUTDOWN)
+		goto out;
+
+	itd->slave = slave;
+	itd->emul_set_mode = emumode;
+	itd->emul_set_tick = emutick;
+	itd->real_set_mode = evtdev->set_mode;
+	itd->real_set_tick = evtdev->set_next_event;
+	itd->real_max_delta_ns = evtdev->max_delta_ns;
+	itd->real_mult = evtdev->mult;
+	itd->real_shift = evtdev->shift;
+	freq = (1000000000ULL * evtdev->mult) >> evtdev->shift;
+	*tmfreq = (unsigned long)freq;
+	evtdev->set_mode = emumode;
+	evtdev->set_next_event = emutick;
+	evtdev->max_delta_ns = ULONG_MAX;
+	evtdev->mult = 1;
+	evtdev->shift = 0;
+out:
+	ipipe_critical_exit(flags);
+
+	return status;
+}
+
+void ipipe_release_tickdev(int cpu)
+{
+	struct ipipe_tick_device *itd;
+	struct tick_device *slave;
+	struct clock_event_device *evtdev;
+	unsigned long flags;
+
+	flags = ipipe_critical_enter(NULL);
+
+	itd = &per_cpu(ipipe_tick_cpu_device, cpu);
+
+	if (itd->slave != NULL) {
+		slave = &per_cpu(tick_cpu_device, cpu);
+		evtdev = slave->evtdev;
+		evtdev->set_mode = itd->real_set_mode;
+		evtdev->set_next_event = itd->real_set_tick;
+		evtdev->max_delta_ns = itd->real_max_delta_ns;
+		evtdev->mult = itd->real_mult;
+		evtdev->shift = itd->real_shift;
+		itd->slave = NULL;
+	}
+
+	ipipe_critical_exit(flags);
+}
+
+#endif /* CONFIG_GENERIC_CLOCKEVENTS */
+
+void __init ipipe_init_early(void)
+{
+	struct ipipe_domain *ipd = &ipipe_root;
+
+	/*
+	 * Do the early init stuff. At this point, the kernel does not
+	 * provide much services yet: be careful.
+	 */
+	__ipipe_check_platform(); /* Do platform dependent checks first. */
+
+	/*
+	 * A lightweight registration code for the root domain. We are
+	 * running on the boot CPU, hw interrupts are off, and
+	 * secondary CPUs are still lost in space.
+	 */
+
+	/* Reserve percpu data slot #0 for the root domain. */
+	ipd->slot = 0;
+	set_bit(0, &__ipipe_domain_slot_map);
+
+	ipd->name = "Linux";
+	ipd->domid = IPIPE_ROOT_ID;
+	ipd->priority = IPIPE_ROOT_PRIO;
+
+	__ipipe_init_stage(ipd);
+
+	list_add_tail(&ipd->p_link, &__ipipe_pipeline);
+
+	__ipipe_init_platform();
+
+#ifdef CONFIG_PRINTK
+	__ipipe_printk_virq = ipipe_alloc_virq();	/* Cannot fail here. */
+	ipd->irqs[__ipipe_printk_virq].handler = &__ipipe_flush_printk;
+	ipd->irqs[__ipipe_printk_virq].cookie = NULL;
+	ipd->irqs[__ipipe_printk_virq].acknowledge = NULL;
+	ipd->irqs[__ipipe_printk_virq].control = IPIPE_HANDLE_MASK;
+#endif /* CONFIG_PRINTK */
+}
+
+void __init ipipe_init(void)
+{
+	/* Now we may engage the pipeline. */
+	__ipipe_enable_pipeline();
+
+	printk(KERN_INFO "I-pipe %s: pipeline enabled.\n",
+	       IPIPE_VERSION_STRING);
+}
+
+void __ipipe_init_stage(struct ipipe_domain *ipd)
+{
+	struct ipipe_percpu_domain_data *p;
+	unsigned long status;
+	int cpu, n;
+
+	for_each_online_cpu(cpu) {
+		p = ipipe_percpudom_ptr(ipd, cpu);
+		status = p->status;
+		memset(p, 0, sizeof(*p));
+		p->status = status;
+	}
+
+	for (n = 0; n < IPIPE_NR_IRQS; n++) {
+		ipd->irqs[n].acknowledge = NULL;
+		ipd->irqs[n].handler = NULL;
+		ipd->irqs[n].control = IPIPE_PASS_MASK;	/* Pass but don't handle */
+	}
+
+	for (n = 0; n < IPIPE_NR_EVENTS; n++)
+		ipd->evhand[n] = NULL;
+
+	ipd->evself = 0LL;
+	mutex_init(&ipd->mutex);
+
+	__ipipe_hook_critical_ipi(ipd);
+}
+
+void __ipipe_cleanup_domain(struct ipipe_domain *ipd)
+{
+	ipipe_unstall_pipeline_from(ipd);
+
+#ifdef CONFIG_SMP
+	{
+		struct ipipe_percpu_domain_data *p;
+		int cpu;
+
+		for_each_online_cpu(cpu) {
+			p = ipipe_percpudom_ptr(ipd, cpu);
+			while (__ipipe_ipending_p(p))
+				cpu_relax();
+		}
+	}
+#else
+	__raw_get_cpu_var(ipipe_percpu_daddr)[ipd->slot] = NULL;
+#endif
+
+	clear_bit(ipd->slot, &__ipipe_domain_slot_map);
+}
+
+void __ipipe_unstall_root(void)
+{
+	struct ipipe_percpu_domain_data *p;
+
+	local_irq_disable_hw();
+
+	/* This helps catching bad usage from assembly call sites. */
+	ipipe_check_context(ipipe_root_domain);
+
+	p = ipipe_root_cpudom_ptr();
+
+	__clear_bit(IPIPE_STALL_FLAG, &p->status);
+
+	if (unlikely(__ipipe_ipending_p(p)))
+		__ipipe_sync_pipeline();
+
+	local_irq_enable_hw();
+}
+
+void __ipipe_restore_root(unsigned long x)
+{
+	ipipe_check_context(ipipe_root_domain);
+
+	if (x)
+		__ipipe_stall_root();
+	else
+		__ipipe_unstall_root();
+}
+
+void ipipe_stall_pipeline_from(struct ipipe_domain *ipd)
+{
+	unsigned long flags;
+	/*
+	 * We have to prevent against race on updating the status
+	 * variable _and_ CPU migration at the same time, so disable
+	 * hw IRQs here.
+	 */
+	local_irq_save_hw(flags);
+
+	__set_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status));
+
+	if (!__ipipe_pipeline_head_p(ipd))
+		local_irq_restore_hw(flags);
+}
+
+unsigned long ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd)
+{
+	unsigned long flags, x;
+
+	/* See ipipe_stall_pipeline_from() */
+	local_irq_save_hw(flags);
+
+	x = __test_and_set_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status));
+
+	if (!__ipipe_pipeline_head_p(ipd))
+		local_irq_restore_hw(flags);
+
+	return x;
+}
+
+unsigned long ipipe_test_and_unstall_pipeline_from(struct ipipe_domain *ipd)
+{
+	unsigned long flags, x;
+	struct list_head *pos;
+
+	local_irq_save_hw(flags);
+
+	x = __test_and_clear_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status));
+
+	if (ipd == __ipipe_current_domain)
+		pos = &ipd->p_link;
+	else
+		pos = __ipipe_pipeline.next;
+
+	__ipipe_walk_pipeline(pos);
+
+	if (likely(__ipipe_pipeline_head_p(ipd)))
+		local_irq_enable_hw();
+	else
+		local_irq_restore_hw(flags);
+
+	return x;
+}
+
+void ipipe_restore_pipeline_from(struct ipipe_domain *ipd,
+					  unsigned long x)
+{
+	if (x)
+		ipipe_stall_pipeline_from(ipd);
+	else
+		ipipe_unstall_pipeline_from(ipd);
+}
+
+void ipipe_unstall_pipeline_head(void)
+{
+	struct ipipe_percpu_domain_data *p = ipipe_head_cpudom_ptr();
+	struct ipipe_domain *head_domain;
+
+	local_irq_disable_hw();
+
+	__clear_bit(IPIPE_STALL_FLAG, &p->status);
+
+	if (unlikely(__ipipe_ipending_p(p))) {
+		head_domain = __ipipe_pipeline_head();
+		if (likely(head_domain == __ipipe_current_domain))
+			__ipipe_sync_pipeline();
+		else
+			__ipipe_walk_pipeline(&head_domain->p_link);
+	}
+
+	local_irq_enable_hw();
+}
+
+void __ipipe_restore_pipeline_head(unsigned long x) /* hw interrupt off */
+{
+	struct ipipe_percpu_domain_data *p = ipipe_head_cpudom_ptr();
+	struct ipipe_domain *head_domain;
+
+	if (x) {
+#ifdef CONFIG_DEBUG_KERNEL
+		static int warned;
+		if (!warned &&
+		    __test_and_set_bit(IPIPE_STALL_FLAG, &p->status)) {
+			/*
+			 * Already stalled albeit ipipe_restore_pipeline_head()
+			 * should have detected it? Send a warning once.
+			 */
+			local_irq_enable_hw();
+			warned = 1;
+			printk(KERN_WARNING
+				   "I-pipe: ipipe_restore_pipeline_head() optimization failed.\n");
+			dump_stack();
+			local_irq_disable_hw();
+		}
+#else /* !CONFIG_DEBUG_KERNEL */
+		__set_bit(IPIPE_STALL_FLAG, &p->status);
+#endif /* CONFIG_DEBUG_KERNEL */
+	}
+	else {
+		__clear_bit(IPIPE_STALL_FLAG, &p->status);
+		if (unlikely(__ipipe_ipending_p(p))) {
+			head_domain = __ipipe_pipeline_head();
+			if (likely(head_domain == __ipipe_current_domain))
+				__ipipe_sync_pipeline();
+			else
+				__ipipe_walk_pipeline(&head_domain->p_link);
+		}
+		local_irq_enable_hw();
+	}
+}
+
+void __ipipe_spin_lock_irq(ipipe_spinlock_t *lock)
+{
+	local_irq_disable_hw();
+	arch_spin_lock(&lock->arch_lock);
+	__set_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
+}
+
+void __ipipe_spin_unlock_irq(ipipe_spinlock_t *lock)
+{
+	arch_spin_unlock(&lock->arch_lock);
+	__clear_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
+	local_irq_enable_hw();
+}
+
+unsigned long __ipipe_spin_lock_irqsave(ipipe_spinlock_t *lock)
+{
+	unsigned long flags;
+	int s;
+
+	local_irq_save_hw(flags);
+	arch_spin_lock(&lock->arch_lock);
+	s = __test_and_set_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
+
+	return arch_mangle_irq_bits(s, flags);
+}
+
+int __ipipe_spin_trylock_irqsave(ipipe_spinlock_t *lock,
+				 unsigned long *x)
+{
+	unsigned long flags;
+	int s;
+
+	local_irq_save_hw(flags);
+	if (!arch_spin_trylock(&lock->arch_lock)) {
+		local_irq_restore_hw(flags);
+		return 0;
+	}
+	s = __test_and_set_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
+	*x = arch_mangle_irq_bits(s, flags);
+
+	return 1;
+}
+
+void __ipipe_spin_unlock_irqrestore(ipipe_spinlock_t *lock,
+				    unsigned long x)
+{
+	arch_spin_unlock(&lock->arch_lock);
+	if (!arch_demangle_irq_bits(&x))
+		__clear_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
+	local_irq_restore_hw(x);
+}
+
+int __ipipe_spin_trylock_irq(ipipe_spinlock_t *lock)
+{
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
+	if (!arch_spin_trylock(&lock->arch_lock)) {
+		local_irq_restore_hw(flags);
+		return 0;
+	}
+	__set_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
+
+	return 1;
+}
+
+void __ipipe_spin_unlock_irqbegin(ipipe_spinlock_t *lock)
+{
+	arch_spin_unlock(&lock->arch_lock);
+}
+
+void __ipipe_spin_unlock_irqcomplete(unsigned long x)
+{
+	if (!arch_demangle_irq_bits(&x))
+		__clear_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
+	local_irq_restore_hw(x);
+}
+
+#ifdef __IPIPE_3LEVEL_IRQMAP
+
+/* Must be called hw IRQs off. */
+static inline void __ipipe_set_irq_held(struct ipipe_percpu_domain_data *p,
+					unsigned int irq)
+{
+	set_bit(irq, p->irqheld_map);
+	p->irqall[irq]++;
+}
+
+/* Must be called hw IRQs off. */
+void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned int irq)
+{
+	struct ipipe_percpu_domain_data *p = ipipe_cpudom_ptr(ipd);
+	int l0b, l1b;
+
+	IPIPE_WARN_ONCE(!irqs_disabled_hw());
+
+	l0b = irq / (BITS_PER_LONG * BITS_PER_LONG);
+	l1b = irq / BITS_PER_LONG;
+
+	if (likely(!test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) {
+		set_bit(irq, p->irqpend_lomap);
+		set_bit(l1b, p->irqpend_mdmap);
+		set_bit(l0b, &p->irqpend_himap);
+	} else
+		set_bit(irq, p->irqheld_map);
+
+	p->irqall[irq]++;
+}
+
+/* Must be called hw IRQs off. */
+void __ipipe_lock_irq(struct ipipe_domain *ipd, int cpu, unsigned int irq)
+{
+	struct ipipe_percpu_domain_data *p;
+	int l0b, l1b;
+
+	IPIPE_WARN_ONCE(!irqs_disabled_hw());
+
+	/* Wired interrupts cannot be locked (it is useless). */
+	if (test_bit(IPIPE_WIRED_FLAG, &ipd->irqs[irq].control) ||
+	    test_and_set_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
+		return;
+
+	l0b = irq / (BITS_PER_LONG * BITS_PER_LONG);
+	l1b = irq / BITS_PER_LONG;
+
+	p = ipipe_percpudom_ptr(ipd, cpu);
+	if (test_and_clear_bit(irq, p->irqpend_lomap)) {
+		set_bit(irq, p->irqheld_map);
+		if (p->irqpend_lomap[l1b] == 0) {
+			clear_bit(l1b, p->irqpend_mdmap);
+			if (p->irqpend_mdmap[l0b] == 0)
+				clear_bit(l0b, &p->irqpend_himap);
+		}
+	}
+}
+
+/* Must be called hw IRQs off. */
+void __ipipe_unlock_irq(struct ipipe_domain *ipd, unsigned int irq)
+{
+	struct ipipe_percpu_domain_data *p;
+	int l0b, l1b, cpu;
+
+	IPIPE_WARN_ONCE(!irqs_disabled_hw());
+
+	if (unlikely(!test_and_clear_bit(IPIPE_LOCK_FLAG,
+					 &ipd->irqs[irq].control)))
+		return;
+
+	l0b = irq / (BITS_PER_LONG * BITS_PER_LONG);
+	l1b = irq / BITS_PER_LONG;
+
+	for_each_online_cpu(cpu) {
+		p = ipipe_percpudom_ptr(ipd, cpu);
+		if (test_and_clear_bit(irq, p->irqheld_map)) {
+			set_bit(irq, p->irqpend_lomap);
+			set_bit(l1b, p->irqpend_mdmap);
+			set_bit(l0b, &p->irqpend_himap);
+		}
+	}
+}
+
+static inline int __ipipe_next_irq(struct ipipe_percpu_domain_data *p)
+{
+	int l0b, l1b, l2b;
+	unsigned long l0m, l1m, l2m;
+	unsigned int irq;
+
+	l0m = p->irqpend_himap;
+	if (unlikely(l0m == 0))
+		return -1;
+
+	l0b = __ipipe_ffnz(l0m);
+	l1m = p->irqpend_mdmap[l0b];
+	if (unlikely(l1m == 0))
+		return -1;
+
+	l1b = __ipipe_ffnz(l1m) + l0b * BITS_PER_LONG;
+	l2m = p->irqpend_lomap[l1b];
+	if (unlikely(l2m == 0))
+		return -1;
+
+	l2b = __ipipe_ffnz(l2m);
+	irq = l1b * BITS_PER_LONG + l2b;
+
+	clear_bit(irq, p->irqpend_lomap);
+	if (p->irqpend_lomap[l1b] == 0) {
+		clear_bit(l1b, p->irqpend_mdmap);
+		if (p->irqpend_mdmap[l0b] == 0)
+			clear_bit(l0b, &p->irqpend_himap);
+	}
+
+	return irq;
+}
+
+#else /* __IPIPE_2LEVEL_IRQMAP */
+
+/* Must be called hw IRQs off. */
+static inline void __ipipe_set_irq_held(struct ipipe_percpu_domain_data *p,
+					unsigned int irq)
+{
+	set_bit(irq, p->irqheld_map);
+	p->irqall[irq]++;
+}
+
+/* Must be called hw IRQs off. */
+void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned irq)
+{
+	struct ipipe_percpu_domain_data *p = ipipe_cpudom_ptr(ipd);
+	int l0b = irq / BITS_PER_LONG;
+
+	IPIPE_WARN_ONCE(!irqs_disabled_hw());
+
+	if (likely(!test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) {
+		set_bit(irq, p->irqpend_lomap);
+		set_bit(l0b, &p->irqpend_himap);
+	} else
+		set_bit(irq, p->irqheld_map);
+
+	p->irqall[irq]++;
+}
+
+/* Must be called hw IRQs off. */
+void __ipipe_lock_irq(struct ipipe_domain *ipd, int cpu, unsigned irq)
+{
+	struct ipipe_percpu_domain_data *p;
+	int l0b = irq / BITS_PER_LONG;
+
+	IPIPE_WARN_ONCE(!irqs_disabled_hw());
+
+	/* Wired interrupts cannot be locked (it is useless). */
+	if (test_bit(IPIPE_WIRED_FLAG, &ipd->irqs[irq].control) ||
+	    test_and_set_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
+		return;
+
+	p = ipipe_percpudom_ptr(ipd, cpu);
+	if (test_and_clear_bit(irq, p->irqpend_lomap)) {
+		set_bit(irq, p->irqheld_map);
+		if (p->irqpend_lomap[l0b] == 0)
+			clear_bit(l0b, &p->irqpend_himap);
+	}
+}
+
+/* Must be called hw IRQs off. */
+void __ipipe_unlock_irq(struct ipipe_domain *ipd, unsigned irq)
+{
+	struct ipipe_percpu_domain_data *p;
+	int l0b = irq / BITS_PER_LONG, cpu;
+
+	IPIPE_WARN_ONCE(!irqs_disabled_hw());
+
+	if (unlikely(!test_and_clear_bit(IPIPE_LOCK_FLAG,
+					 &ipd->irqs[irq].control)))
+		return;
+
+	for_each_online_cpu(cpu) {
+		p = ipipe_percpudom_ptr(ipd, cpu);
+		if (test_and_clear_bit(irq, p->irqheld_map)) {
+			set_bit(irq, p->irqpend_lomap);
+			set_bit(l0b, &p->irqpend_himap);
+		}
+	}
+}
+
+static inline int __ipipe_next_irq(struct ipipe_percpu_domain_data *p)
+{
+	unsigned long l0m, l1m;
+	int l0b, l1b;
+
+	l0m = p->irqpend_himap;
+	if (unlikely(l0m == 0))
+		return -1;
+
+	l0b = __ipipe_ffnz(l0m);
+	l1m = p->irqpend_lomap[l0b];
+	if (unlikely(l1m == 0))
+		return -1;
+
+	l1b = __ipipe_ffnz(l1m);
+	clear_bit(l1b, &p->irqpend_lomap[l0b]);
+	if (p->irqpend_lomap[l0b] == 0)
+		clear_bit(l0b, &p->irqpend_himap);
+
+	return l0b * BITS_PER_LONG + l1b;
+}
+
+#endif /* __IPIPE_2LEVEL_IRQMAP */
+
+/*
+ * __ipipe_walk_pipeline(): Plays interrupts pending in the log. Must
+ * be called with local hw interrupts disabled.
+ */
+void __ipipe_walk_pipeline(struct list_head *pos)
+{
+	struct ipipe_domain *this_domain = __ipipe_current_domain, *next_domain;
+	struct ipipe_percpu_domain_data *p, *np;
+
+	p = ipipe_cpudom_ptr(this_domain);
+
+	while (pos != &__ipipe_pipeline) {
+
+		next_domain = list_entry(pos, struct ipipe_domain, p_link);
+		np = ipipe_cpudom_ptr(next_domain);
+
+		if (test_bit(IPIPE_STALL_FLAG, &np->status))
+			break;	/* Stalled stage -- do not go further. */
+
+		if (__ipipe_ipending_p(np)) {
+			if (next_domain == this_domain)
+				__ipipe_sync_pipeline();
+			else {
+
+				p->evsync = 0;
+				__ipipe_current_domain = next_domain;
+				ipipe_suspend_domain();	/* Sync stage and propagate interrupts. */
+
+				if (__ipipe_current_domain == next_domain)
+					__ipipe_current_domain = this_domain;
+				/*
+				 * Otherwise, something changed the current domain under our
+				 * feet recycling the register set; do not override the new
+				 * domain.
+				 */
+
+				if (__ipipe_ipending_p(p) &&
+				    !test_bit(IPIPE_STALL_FLAG, &p->status))
+					__ipipe_sync_pipeline();
+			}
+			break;
+		} else if (next_domain == this_domain)
+			break;
+
+		pos = next_domain->p_link.next;
+	}
+}
+
+/*
+ * ipipe_suspend_domain() -- Suspend the current domain, switching to
+ * the next one which has pending work down the pipeline.
+ */
+void ipipe_suspend_domain(void)
+{
+	struct ipipe_domain *this_domain, *next_domain;
+	struct ipipe_percpu_domain_data *p;
+	struct list_head *ln;
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
+
+	this_domain = next_domain = __ipipe_current_domain;
+	p = ipipe_cpudom_ptr(this_domain);
+	p->status &= ~IPIPE_STALL_MASK;
+
+	if (__ipipe_ipending_p(p))
+		goto sync_stage;
+
+	for (;;) {
+		ln = next_domain->p_link.next;
+
+		if (ln == &__ipipe_pipeline)
+			break;
+
+		next_domain = list_entry(ln, struct ipipe_domain, p_link);
+		p = ipipe_cpudom_ptr(next_domain);
+
+		if (p->status & IPIPE_STALL_MASK)
+			break;
+
+		if (!__ipipe_ipending_p(p))
+			continue;
+
+		__ipipe_current_domain = next_domain;
+sync_stage:
+		__ipipe_sync_pipeline();
+
+		if (__ipipe_current_domain != next_domain)
+			/*
+			 * Something has changed the current domain under our
+			 * feet, recycling the register set; take note.
+			 */
+			this_domain = __ipipe_current_domain;
+	}
+
+	__ipipe_current_domain = this_domain;
+
+	local_irq_restore_hw(flags);
+}
+
+
+/* ipipe_alloc_virq() -- Allocate a pipelined virtual/soft interrupt.
+ * Virtual interrupts are handled in exactly the same way than their
+ * hw-generated counterparts wrt pipelining.
+ */
+unsigned ipipe_alloc_virq(void)
+{
+	unsigned long flags, irq = 0;
+	int ipos;
+
+	spin_lock_irqsave(&__ipipe_pipelock, flags);
+
+	if (__ipipe_virtual_irq_map != ~0) {
+		ipos = ffz(__ipipe_virtual_irq_map);
+		set_bit(ipos, &__ipipe_virtual_irq_map);
+		irq = ipos + IPIPE_VIRQ_BASE;
+	}
+
+	spin_unlock_irqrestore(&__ipipe_pipelock, flags);
+
+	return irq;
+}
+
+/*
+ * ipipe_virtualize_irq() -- Set a per-domain pipelined interrupt
+ * handler.
+ */
+int ipipe_virtualize_irq(struct ipipe_domain *ipd,
+			 unsigned int irq,
+			 ipipe_irq_handler_t handler,
+			 void *cookie,
+			 ipipe_irq_ackfn_t acknowledge,
+			 unsigned modemask)
+{
+	ipipe_irq_handler_t old_handler;
+	struct irq_desc *desc;
+	unsigned long flags;
+	int ret = 0;
+
+	if (irq >= IPIPE_NR_IRQS)
+		return -EINVAL;
+
+	if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK)
+		return -EPERM;
+
+	if (!test_bit(IPIPE_AHEAD_FLAG, &ipd->flags))
+		/* Silently unwire interrupts for non-heading domains. */
+		modemask &= ~IPIPE_WIRED_MASK;
+
+	spin_lock_irqsave(&__ipipe_pipelock, flags);
+
+	old_handler = ipd->irqs[irq].handler;
+
+	if (handler == NULL) {
+		modemask &=
+		    ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK |
+		      IPIPE_EXCLUSIVE_MASK | IPIPE_WIRED_MASK);
+
+		ipd->irqs[irq].handler = NULL;
+		ipd->irqs[irq].cookie = NULL;
+		ipd->irqs[irq].acknowledge = NULL;
+		ipd->irqs[irq].control = modemask;
+
+		if (irq < NR_IRQS && !ipipe_virtual_irq_p(irq)) {
+			desc = irq_to_desc(irq);
+			if (old_handler && desc)
+				__ipipe_disable_irqdesc(ipd, irq);
+		}
+
+		goto unlock_and_exit;
+	}
+
+	if (handler == IPIPE_SAME_HANDLER) {
+		cookie = ipd->irqs[irq].cookie;
+		handler = old_handler;
+		if (handler == NULL) {
+			ret = -EINVAL;
+			goto unlock_and_exit;
+		}
+	} else if ((modemask & IPIPE_EXCLUSIVE_MASK) != 0 && old_handler) {
+		ret = -EBUSY;
+		goto unlock_and_exit;
+	}
+
+	/*
+	 * Wired interrupts can only be delivered to domains always
+	 * heading the pipeline, and using dynamic propagation.
+	 */
+	if ((modemask & IPIPE_WIRED_MASK) != 0) {
+		if ((modemask & (IPIPE_PASS_MASK | IPIPE_STICKY_MASK)) != 0) {
+			ret = -EINVAL;
+			goto unlock_and_exit;
+		}
+		modemask |= IPIPE_HANDLE_MASK;
+	}
+
+	if ((modemask & IPIPE_STICKY_MASK) != 0)
+		modemask |= IPIPE_HANDLE_MASK;
+
+	if (acknowledge == NULL)
+		/*
+		 * Acknowledge handler unspecified for a hw interrupt:
+		 * use the Linux-defined handler instead.
+		 */
+		acknowledge = ipipe_root_domain->irqs[irq].acknowledge;
+
+	ipd->irqs[irq].handler = handler;
+	ipd->irqs[irq].cookie = cookie;
+	ipd->irqs[irq].acknowledge = acknowledge;
+	ipd->irqs[irq].control = modemask;
+
+	desc = irq_to_desc(irq);
+	if (desc == NULL)
+		goto unlock_and_exit;
+
+	if (irq < NR_IRQS && !ipipe_virtual_irq_p(irq)) {
+		__ipipe_enable_irqdesc(ipd, irq);
+		/*
+		 * IRQ enable/disable state is domain-sensitive, so we
+		 * may not change it for another domain. What is
+		 * allowed however is forcing some domain to handle an
+		 * interrupt source, by passing the proper 'ipd'
+		 * descriptor which thus may be different from
+		 * __ipipe_current_domain.
+		 */
+		if ((modemask & IPIPE_ENABLE_MASK) != 0) {
+			if (ipd != __ipipe_current_domain)
+				ret = -EPERM;
+			else
+				__ipipe_enable_irq(irq);
+		}
+	}
+
+unlock_and_exit:
+
+	spin_unlock_irqrestore(&__ipipe_pipelock, flags);
+
+	return ret;
+}
+
+/* ipipe_control_irq() -- Change control mode of a pipelined interrupt. */
+
+int ipipe_control_irq(struct ipipe_domain *ipd, unsigned int irq,
+		      unsigned clrmask, unsigned setmask)
+{
+	unsigned long flags;
+	int ret = 0;
+
+	if (irq >= IPIPE_NR_IRQS)
+		return -EINVAL;
+
+	flags = ipipe_critical_enter(NULL);
+
+	if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK) {
+		ret = -EPERM;
+		goto out;
+	}
+
+	if (ipd->irqs[irq].handler == NULL)
+		setmask &= ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK);
+
+	if ((setmask & IPIPE_STICKY_MASK) != 0)
+		setmask |= IPIPE_HANDLE_MASK;
+
+	if ((clrmask & (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK)) != 0)	/* If one goes, both go. */
+		clrmask |= (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK);
+
+	ipd->irqs[irq].control &= ~clrmask;
+	ipd->irqs[irq].control |= setmask;
+
+	if ((setmask & IPIPE_ENABLE_MASK) != 0)
+		__ipipe_enable_irq(irq);
+	else if ((clrmask & IPIPE_ENABLE_MASK) != 0)
+		__ipipe_disable_irq(irq);
+
+out:
+	ipipe_critical_exit(flags);
+
+	return ret;
+}
+
+/* __ipipe_dispatch_event() -- Low-level event dispatcher. */
+
+int __ipipe_dispatch_event (unsigned event, void *data)
+{
+	struct ipipe_domain *start_domain, *this_domain, *next_domain;
+	struct ipipe_percpu_domain_data *np;
+	ipipe_event_handler_t evhand;
+	struct list_head *pos, *npos;
+	unsigned long flags;
+	int propagate = 1;
+
+	local_irq_save_hw(flags);
+
+	start_domain = this_domain = __ipipe_current_domain;
+
+	list_for_each_safe(pos, npos, &__ipipe_pipeline) {
+		/*
+		 * Note: Domain migration may occur while running
+		 * event or interrupt handlers, in which case the
+		 * current register set is going to be recycled for a
+		 * different domain than the initiating one. We do
+		 * care for that, always tracking the current domain
+		 * descriptor upon return from those handlers.
+		 */
+		next_domain = list_entry(pos, struct ipipe_domain, p_link);
+		np = ipipe_cpudom_ptr(next_domain);
+
+		/*
+		 * Keep a cached copy of the handler's address since
+		 * ipipe_catch_event() may clear it under our feet.
+		 */
+		evhand = next_domain->evhand[event];
+
+		if (evhand != NULL) {
+			__ipipe_current_domain = next_domain;
+			np->evsync |= (1LL << event);
+			local_irq_restore_hw(flags);
+			propagate = !evhand(event, start_domain, data);
+			local_irq_save_hw(flags);
+			/*
+			 * We may have a migration issue here, if the
+			 * current task is migrated to another CPU on
+			 * behalf of the invoked handler, usually when
+			 * a syscall event is processed. However,
+			 * ipipe_catch_event() will make sure that a
+			 * CPU that clears a handler for any given
+			 * event will not attempt to wait for itself
+			 * to clear the evsync bit for that event,
+			 * which practically plugs the hole, without
+			 * resorting to a much more complex strategy.
+			 */
+			np->evsync &= ~(1LL << event);
+			if (__ipipe_current_domain != next_domain)
+				this_domain = __ipipe_current_domain;
+		}
+
+		/* NEVER sync the root stage here. */
+		if (next_domain != ipipe_root_domain &&
+		    __ipipe_ipending_p(np) &&
+		    !test_bit(IPIPE_STALL_FLAG, &np->status)) {
+			__ipipe_current_domain = next_domain;
+			__ipipe_sync_pipeline();
+			if (__ipipe_current_domain != next_domain)
+				this_domain = __ipipe_current_domain;
+		}
+
+		__ipipe_current_domain = this_domain;
+
+		if (next_domain == this_domain || !propagate)
+			break;
+	}
+
+	local_irq_restore_hw(flags);
+
+	return !propagate;
+}
+
+/*
+ * __ipipe_dispatch_wired -- Wired interrupt dispatcher. Wired
+ * interrupts are immediately and unconditionally delivered to the
+ * domain heading the pipeline upon receipt, and such domain must have
+ * been registered as an invariant head for the system (priority ==
+ * IPIPE_HEAD_PRIORITY). The motivation for using wired interrupts is
+ * to get an extra-fast dispatching path for those IRQs, by relying on
+ * a straightforward logic based on assumptions that must always be
+ * true for invariant head domains.  The following assumptions are
+ * made when dealing with such interrupts:
+ *
+ * 1- Wired interrupts are purely dynamic, i.e. the decision to
+ * propagate them down the pipeline must be done from the head domain
+ * ISR.
+ * 2- Wired interrupts cannot be shared or sticky.
+ * 3- The root domain cannot be an invariant pipeline head, in
+ * consequence of what the root domain cannot handle wired
+ * interrupts.
+ * 4- Wired interrupts must have a valid acknowledge handler for the
+ * head domain (if needed, see __ipipe_handle_irq).
+ *
+ * Called with hw interrupts off.
+ */
+
+void __ipipe_dispatch_wired(struct ipipe_domain *head, unsigned irq)
+{
+	struct ipipe_percpu_domain_data *p = ipipe_cpudom_ptr(head);
+
+	if (test_bit(IPIPE_STALL_FLAG, &p->status)) {
+		__ipipe_set_irq_pending(head, irq);
+		return;
+	}
+
+	__ipipe_dispatch_wired_nocheck(head, irq);
+}
+
+void __ipipe_dispatch_wired_nocheck(struct ipipe_domain *head, unsigned irq) /* hw interrupts off */
+{
+	struct ipipe_percpu_domain_data *p = ipipe_cpudom_ptr(head);
+	struct ipipe_domain *old;
+
+	old = __ipipe_current_domain;
+	__ipipe_current_domain = head; /* Switch to the head domain. */
+
+	p->irqall[irq]++;
+	__set_bit(IPIPE_STALL_FLAG, &p->status);
+	barrier();
+	head->irqs[irq].handler(irq, head->irqs[irq].cookie); /* Call the ISR. */
+	__ipipe_run_irqtail(irq);
+	barrier();
+	p = ipipe_cpudom_ptr(head);
+	__clear_bit(IPIPE_STALL_FLAG, &p->status);
+
+	if (__ipipe_current_domain == head) {
+		__ipipe_current_domain = old;
+		if (old == head) {
+			if (__ipipe_ipending_p(p))
+				__ipipe_sync_pipeline();
+			return;
+		}
+	}
+
+	__ipipe_walk_pipeline(&head->p_link);
+}
+
+#ifdef CONFIG_TRACE_IRQFLAGS
+#define root_stall_after_handler()	local_irq_disable()
+#else
+#define root_stall_after_handler()	do { } while (0)
+#endif
+
+#ifdef CONFIG_PREEMPT
+
+asmlinkage void preempt_schedule_irq(void);
+
+asmlinkage void __sched __ipipe_preempt_schedule_irq(void)
+{
+	struct ipipe_percpu_domain_data *p;
+	unsigned long flags;
+
+	BUG_ON(!irqs_disabled_hw());
+	local_irq_save(flags);
+	local_irq_enable_hw();
+	preempt_schedule_irq(); /* Ok, may reschedule now. */
+	local_irq_disable_hw();
+
+	/*
+	 * Flush any pending interrupt that may have been logged after
+	 * preempt_schedule_irq() stalled the root stage before
+	 * returning to us, and now.
+	 */
+	p = ipipe_root_cpudom_ptr();
+	if (unlikely(__ipipe_ipending_p(p))) {
+		add_preempt_count(PREEMPT_ACTIVE);
+		trace_hardirqs_on();
+		__clear_bit(IPIPE_STALL_FLAG, &p->status);
+		__ipipe_sync_pipeline();
+		sub_preempt_count(PREEMPT_ACTIVE);
+	}
+
+	__local_irq_restore_nosync(flags);
+}
+
+#else /* !CONFIG_PREEMPT */
+
+#define __ipipe_preempt_schedule_irq()	do { } while (0)
+
+#endif	/* !CONFIG_PREEMPT */
+
+/*
+ * __ipipe_sync_stage() -- Flush the pending IRQs for the current
+ * domain (and processor). This routine flushes the interrupt log
+ * (see "Optimistic interrupt protection" from D. Stodolsky et al. for
+ * more on the deferred interrupt scheme). Every interrupt that
+ * occurred while the pipeline was stalled gets played. WARNING:
+ * callers on SMP boxen should always check for CPU migration on
+ * return of this routine.
+ *
+ * This routine must be called with hw interrupts off.
+ */
+void __ipipe_sync_stage(void)
+{
+	struct ipipe_percpu_domain_data *p;
+	struct ipipe_domain *ipd;
+	int irq;
+
+	ipd = __ipipe_current_domain;
+	p = ipipe_cpudom_ptr(ipd);
+
+	__set_bit(IPIPE_STALL_FLAG, &p->status);
+	smp_wmb();
+
+	if (ipd == ipipe_root_domain)
+		trace_hardirqs_off();
+
+	for (;;) {
+		irq = __ipipe_next_irq(p);
+		if (irq < 0)
+			break;
+		/*
+		 * Make sure the compiler does not reorder wrongly, so
+		 * that all updates to maps are done before the
+		 * handler gets called.
+		 */
+		barrier();
+
+		if (test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
+			continue;
+
+		if (!__ipipe_pipeline_head_p(ipd))
+			local_irq_enable_hw();
+
+		if (likely(ipd != ipipe_root_domain)) {
+			ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie);
+			__ipipe_run_irqtail(irq);
+			local_irq_disable_hw();
+		} else if (ipipe_virtual_irq_p(irq)) {
+			irq_enter();
+			ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie);
+			irq_exit();
+			root_stall_after_handler();
+			local_irq_disable_hw();
+			while (__ipipe_check_root_resched())
+				__ipipe_preempt_schedule_irq();
+		} else {
+			__ipipe_do_root_xirq(ipd, irq);
+			root_stall_after_handler();
+			local_irq_disable_hw();
+		}
+
+		p = ipipe_cpudom_ptr(__ipipe_current_domain);
+	}
+
+	if (ipd == ipipe_root_domain)
+		trace_hardirqs_on();
+
+	__clear_bit(IPIPE_STALL_FLAG, &p->status);
+}
+
+/* ipipe_register_domain() -- Link a new domain to the pipeline. */
+
+int ipipe_register_domain(struct ipipe_domain *ipd,
+			  struct ipipe_domain_attr *attr)
+{
+	struct ipipe_percpu_domain_data *p;
+	struct list_head *pos = NULL;
+	struct ipipe_domain *_ipd;
+	unsigned long flags;
+
+	if (!ipipe_root_domain_p) {
+		printk(KERN_WARNING
+		       "I-pipe: Only the root domain may register a new domain.\n");
+		return -EPERM;
+	}
+
+	flags = ipipe_critical_enter(NULL);
+
+	if (attr->priority == IPIPE_HEAD_PRIORITY) {
+		if (test_bit(IPIPE_HEAD_SLOT, &__ipipe_domain_slot_map)) {
+			ipipe_critical_exit(flags);
+			return -EAGAIN;	/* Cannot override current head. */
+		}
+		ipd->slot = IPIPE_HEAD_SLOT;
+	} else
+		ipd->slot = ffz(__ipipe_domain_slot_map);
+
+	if (ipd->slot < CONFIG_IPIPE_DOMAINS) {
+		set_bit(ipd->slot, &__ipipe_domain_slot_map);
+		list_for_each(pos, &__ipipe_pipeline) {
+			_ipd = list_entry(pos, struct ipipe_domain, p_link);
+			if (_ipd->domid == attr->domid)
+				break;
+		}
+	}
+
+	ipipe_critical_exit(flags);
+
+	if (pos != &__ipipe_pipeline) {
+		if (ipd->slot < CONFIG_IPIPE_DOMAINS)
+			clear_bit(ipd->slot, &__ipipe_domain_slot_map);
+		return -EBUSY;
+	}
+
+#ifndef CONFIG_SMP
+	/*
+	 * Set up the perdomain pointers for direct access to the
+	 * percpu domain data. This saves a costly multiply each time
+	 * we need to refer to the contents of the percpu domain data
+	 * array.
+	 */
+	__raw_get_cpu_var(ipipe_percpu_daddr)[ipd->slot] = &__raw_get_cpu_var(ipipe_percpu_darray)[ipd->slot];
+#endif
+
+	ipd->name = attr->name;
+	ipd->domid = attr->domid;
+	ipd->pdd = attr->pdd;
+	ipd->flags = 0;
+
+	if (attr->priority == IPIPE_HEAD_PRIORITY) {
+		ipd->priority = INT_MAX;
+		__set_bit(IPIPE_AHEAD_FLAG,&ipd->flags);
+	}
+	else
+		ipd->priority = attr->priority;
+
+	__ipipe_init_stage(ipd);
+
+	INIT_LIST_HEAD(&ipd->p_link);
+
+#ifdef CONFIG_PROC_FS
+	__ipipe_add_domain_proc(ipd);
+#endif /* CONFIG_PROC_FS */
+
+	flags = ipipe_critical_enter(NULL);
+
+	list_for_each(pos, &__ipipe_pipeline) {
+		_ipd = list_entry(pos, struct ipipe_domain, p_link);
+		if (ipd->priority > _ipd->priority)
+			break;
+	}
+
+	list_add_tail(&ipd->p_link, pos);
+
+	ipipe_critical_exit(flags);
+
+	printk(KERN_INFO "I-pipe: Domain %s registered.\n", ipd->name);
+
+	if (attr->entry == NULL)
+		return 0;
+
+	/*
+	 * Finally, allow the new domain to perform its initialization
+	 * duties.
+	 */
+	local_irq_save_hw_smp(flags);
+	__ipipe_current_domain = ipd;
+	local_irq_restore_hw_smp(flags);
+	attr->entry();
+	local_irq_save_hw(flags);
+	__ipipe_current_domain = ipipe_root_domain;
+	p = ipipe_root_cpudom_ptr();
+
+	if (__ipipe_ipending_p(p) &&
+	    !test_bit(IPIPE_STALL_FLAG, &p->status))
+		__ipipe_sync_pipeline();
+
+	local_irq_restore_hw(flags);
+
+	return 0;
+}
+
+/* ipipe_unregister_domain() -- Remove a domain from the pipeline. */
+
+int ipipe_unregister_domain(struct ipipe_domain *ipd)
+{
+	unsigned long flags;
+
+	if (!ipipe_root_domain_p) {
+		printk(KERN_WARNING
+		       "I-pipe: Only the root domain may unregister a domain.\n");
+		return -EPERM;
+	}
+
+	if (ipd == ipipe_root_domain) {
+		printk(KERN_WARNING
+		       "I-pipe: Cannot unregister the root domain.\n");
+		return -EPERM;
+	}
+#ifdef CONFIG_SMP
+	{
+		struct ipipe_percpu_domain_data *p;
+		unsigned int irq;
+		int cpu;
+
+		/*
+		 * In the SMP case, wait for the logged events to drain on
+		 * other processors before eventually removing the domain
+		 * from the pipeline.
+		 */
+
+		ipipe_unstall_pipeline_from(ipd);
+
+		flags = ipipe_critical_enter(NULL);
+
+		for (irq = 0; irq < IPIPE_NR_IRQS; irq++) {
+			clear_bit(IPIPE_HANDLE_FLAG, &ipd->irqs[irq].control);
+			clear_bit(IPIPE_WIRED_FLAG, &ipd->irqs[irq].control);
+			clear_bit(IPIPE_STICKY_FLAG, &ipd->irqs[irq].control);
+			set_bit(IPIPE_PASS_FLAG, &ipd->irqs[irq].control);
+		}
+
+		ipipe_critical_exit(flags);
+
+		for_each_online_cpu(cpu) {
+			p = ipipe_percpudom_ptr(ipd, cpu);
+			while (__ipipe_ipending_p(p))
+				cpu_relax();
+		}
+	}
+#endif	/* CONFIG_SMP */
+
+	mutex_lock(&ipd->mutex);
+
+#ifdef CONFIG_PROC_FS
+	__ipipe_remove_domain_proc(ipd);
+#endif /* CONFIG_PROC_FS */
+
+	/*
+	 * Simply remove the domain from the pipeline and we are almost done.
+	 */
+
+	flags = ipipe_critical_enter(NULL);
+	list_del_init(&ipd->p_link);
+	ipipe_critical_exit(flags);
+
+	__ipipe_cleanup_domain(ipd);
+
+	mutex_unlock(&ipd->mutex);
+
+	printk(KERN_INFO "I-pipe: Domain %s unregistered.\n", ipd->name);
+
+	return 0;
+}
+
+/*
+ * ipipe_propagate_irq() -- Force a given IRQ propagation on behalf of
+ * a running interrupt handler to the next domain down the pipeline.
+ * ipipe_schedule_irq() -- Does almost the same as above, but attempts
+ * to pend the interrupt for the current domain first.
+ * Must be called hw IRQs off.
+ */
+void __ipipe_pend_irq(unsigned irq, struct list_head *head)
+{
+	struct ipipe_domain *ipd;
+	struct list_head *ln;
+
+#ifdef CONFIG_IPIPE_DEBUG
+	BUG_ON(irq >= IPIPE_NR_IRQS ||
+	       (ipipe_virtual_irq_p(irq)
+		&& !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)));
+#endif
+	for (ln = head; ln != &__ipipe_pipeline; ln = ipd->p_link.next) {
+		ipd = list_entry(ln, struct ipipe_domain, p_link);
+		if (test_bit(IPIPE_HANDLE_FLAG, &ipd->irqs[irq].control)) {
+			__ipipe_set_irq_pending(ipd, irq);
+			return;
+		}
+	}
+}
+
+/* ipipe_free_virq() -- Release a virtual/soft interrupt. */
+
+int ipipe_free_virq(unsigned virq)
+{
+	if (!ipipe_virtual_irq_p(virq))
+		return -EINVAL;
+
+	clear_bit(virq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map);
+
+	return 0;
+}
+
+void ipipe_init_attr(struct ipipe_domain_attr *attr)
+{
+	attr->name = "anon";
+	attr->domid = 1;
+	attr->entry = NULL;
+	attr->priority = IPIPE_ROOT_PRIO;
+	attr->pdd = NULL;
+}
+
+/*
+ * ipipe_catch_event() -- Interpose or remove an event handler for a
+ * given domain.
+ */
+ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd,
+					unsigned event,
+					ipipe_event_handler_t handler)
+{
+	ipipe_event_handler_t old_handler;
+	unsigned long flags;
+	int self = 0, cpu;
+
+	if (event & IPIPE_EVENT_SELF) {
+		event &= ~IPIPE_EVENT_SELF;
+		self = 1;
+	}
+
+	if (event >= IPIPE_NR_EVENTS)
+		return NULL;
+
+	flags = ipipe_critical_enter(NULL);
+
+	if (!(old_handler = xchg(&ipd->evhand[event],handler)))	{
+		if (handler) {
+			if (self)
+				ipd->evself |= (1LL << event);
+			else
+				__ipipe_event_monitors[event]++;
+		}
+	}
+	else if (!handler) {
+		if (ipd->evself & (1LL << event))
+			ipd->evself &= ~(1LL << event);
+		else
+			__ipipe_event_monitors[event]--;
+	} else if ((ipd->evself & (1LL << event)) && !self) {
+			__ipipe_event_monitors[event]++;
+			ipd->evself &= ~(1LL << event);
+	} else if (!(ipd->evself & (1LL << event)) && self) {
+			__ipipe_event_monitors[event]--;
+			ipd->evself |= (1LL << event);
+	}
+
+	ipipe_critical_exit(flags);
+
+	if (!handler && ipipe_root_domain_p) {
+		/*
+		 * If we cleared a handler on behalf of the root
+		 * domain, we have to wait for any current invocation
+		 * to drain, since our caller might subsequently unmap
+		 * the target domain. To this aim, this code
+		 * synchronizes with __ipipe_dispatch_event(),
+		 * guaranteeing that either the dispatcher sees a null
+		 * handler in which case it discards the invocation
+		 * (which also prevents from entering a livelock), or
+		 * finds a valid handler and calls it. Symmetrically,
+		 * ipipe_catch_event() ensures that the called code
+		 * won't be unmapped under our feet until the event
+		 * synchronization flag is cleared for the given event
+		 * on all CPUs.
+		 */
+		preempt_disable();
+		cpu = smp_processor_id();
+		/*
+		 * Hack: this solves the potential migration issue
+		 * raised in __ipipe_dispatch_event(). This is a
+		 * work-around which makes the assumption that other
+		 * CPUs will subsequently, either process at least one
+		 * interrupt for the target domain, or call
+		 * __ipipe_dispatch_event() without going through a
+		 * migration while running the handler at least once;
+		 * practically, this is safe on any normally running
+		 * system.
+		 */
+		ipipe_percpudom(ipd, evsync, cpu) &= ~(1LL << event);
+		preempt_enable();
+
+		for_each_online_cpu(cpu) {
+			while (ipipe_percpudom(ipd, evsync, cpu) & (1LL << event))
+				schedule_timeout_interruptible(HZ / 50);
+		}
+	}
+
+	return old_handler;
+}
+
+cpumask_t ipipe_set_irq_affinity (unsigned irq, cpumask_t cpumask)
+{
+#ifdef CONFIG_SMP
+	if (irq >= IPIPE_NR_XIRQS)
+		/* Allow changing affinity of external IRQs only. */
+		return CPU_MASK_NONE;
+
+	if (num_online_cpus() > 1)
+		return __ipipe_set_irq_affinity(irq,cpumask);
+#endif /* CONFIG_SMP */
+
+	return CPU_MASK_NONE;
+}
+
+int ipipe_send_ipi (unsigned ipi, cpumask_t cpumask)
+
+{
+#ifdef CONFIG_SMP
+	if (!ipipe_ipi_p(ipi))
+		return -EINVAL;
+	return __ipipe_send_ipi(ipi,cpumask);
+#else /* !CONFIG_SMP */
+	return -EINVAL;
+#endif /* CONFIG_SMP */
+}
+
+#ifdef CONFIG_SMP
+
+/* Always called with hw interrupts off. */
+void __ipipe_do_critical_sync(unsigned irq, void *cookie)
+{
+	int cpu = ipipe_processor_id();
+
+	cpu_set(cpu, __ipipe_cpu_sync_map);
+
+	/* Now we are in sync with the lock requestor running on another
+	   CPU. Enter a spinning wait until he releases the global
+	   lock. */
+	spin_lock(&__ipipe_cpu_barrier);
+
+	/* Got it. Now get out. */
+
+	if (__ipipe_cpu_sync)
+		/* Call the sync routine if any. */
+		__ipipe_cpu_sync();
+
+	cpu_set(cpu, __ipipe_cpu_pass_map);
+
+	spin_unlock(&__ipipe_cpu_barrier);
+
+	cpu_clear(cpu, __ipipe_cpu_sync_map);
+}
+#endif	/* CONFIG_SMP */
+
+/*
+ * ipipe_critical_enter() -- Grab the superlock excluding all CPUs but
+ * the current one from a critical section. This lock is used when we
+ * must enforce a global critical section for a single CPU in a
+ * possibly SMP system whichever context the CPUs are running.
+ */
+unsigned long ipipe_critical_enter(void (*syncfn)(void))
+{
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
+
+#ifdef CONFIG_SMP
+	if (num_online_cpus() > 1) {
+		int cpu = ipipe_processor_id();
+		cpumask_t allbutself;
+		unsigned long loops;
+
+		if (!cpu_test_and_set(cpu, __ipipe_cpu_lock_map)) {
+			while (test_and_set_bit(0, &__ipipe_critical_lock)) {
+				int n = 0;
+
+				local_irq_enable_hw();
+
+				do {
+					cpu_relax();
+				} while (++n < cpu);
+
+				local_irq_disable_hw();
+			}
+
+restart:
+			spin_lock(&__ipipe_cpu_barrier);
+
+			__ipipe_cpu_sync = syncfn;
+
+			cpus_clear(__ipipe_cpu_pass_map);
+			cpu_set(cpu, __ipipe_cpu_pass_map);
+
+			/*
+			 * Send the sync IPI to all processors but the current
+			 * one.
+			 */
+			cpus_andnot(allbutself, cpu_online_map,
+				    __ipipe_cpu_pass_map);
+			__ipipe_send_ipi(IPIPE_CRITICAL_IPI, allbutself);
+
+			loops = IPIPE_CRITICAL_TIMEOUT;
+
+			while (!cpus_equal(__ipipe_cpu_sync_map, allbutself)) {
+				cpu_relax();
+
+				if (--loops == 0) {
+					/*
+					 * We ran into a deadlock due to a
+					 * contended rwlock. Cancel this round
+					 * and retry.
+					 */
+					__ipipe_cpu_sync = NULL;
+
+					spin_unlock(&__ipipe_cpu_barrier);
+
+					/*
+					 * Ensure all CPUs consumed the IPI to
+					 * avoid running __ipipe_cpu_sync
+					 * prematurely. This usually resolves
+					 * the deadlock reason too.
+					 */
+					while (!cpus_equal(cpu_online_map,
+							   __ipipe_cpu_pass_map))
+						cpu_relax();
+
+					goto restart;
+				}
+			}
+		}
+
+		atomic_inc(&__ipipe_critical_count);
+	}
+#endif	/* CONFIG_SMP */
+
+	return flags;
+}
+
+/* ipipe_critical_exit() -- Release the superlock. */
+
+void ipipe_critical_exit(unsigned long flags)
+{
+#ifdef CONFIG_SMP
+	if (num_online_cpus() > 1 &&
+	    atomic_dec_and_test(&__ipipe_critical_count)) {
+		spin_unlock(&__ipipe_cpu_barrier);
+
+		while (!cpus_empty(__ipipe_cpu_sync_map))
+			cpu_relax();
+
+		cpu_clear(ipipe_processor_id(), __ipipe_cpu_lock_map);
+		clear_bit(0, &__ipipe_critical_lock);
+		smp_mb__after_clear_bit();
+	}
+#endif	/* CONFIG_SMP */
+
+	local_irq_restore_hw(flags);
+}
+
+#ifdef CONFIG_HAVE_IPIPE_HOSTRT
+/*
+ * NOTE: The architecture specific code must only call this function
+ * when a clocksource suitable for CLOCK_HOST_REALTIME is enabled.
+ */
+void ipipe_update_hostrt(struct timespec *wall_time, struct clocksource *clock)
+{
+	struct ipipe_hostrt_data hostrt_data;
+
+	hostrt_data.live = 1;
+	hostrt_data.cycle_last = clock->cycle_last;
+	hostrt_data.mask = clock->mask;
+	hostrt_data.mult = clock->mult;
+	hostrt_data.shift = clock->shift;
+	hostrt_data.wall_time_sec = wall_time->tv_sec;
+	hostrt_data.wall_time_nsec = wall_time->tv_nsec;
+	hostrt_data.wall_to_monotonic = __get_wall_to_monotonic();
+
+	/* Note: The event receiver is responsible for providing
+	   proper locking */
+	if (__ipipe_event_monitored_p(IPIPE_EVENT_HOSTRT))
+		__ipipe_dispatch_event(IPIPE_EVENT_HOSTRT, &hostrt_data);
+}
+#endif /* CONFIG_HAVE_IPIPE_HOSTRT */
+
+int ipipe_alloc_ptdkey (void)
+{
+	unsigned long flags;
+	int key = -1;
+
+	spin_lock_irqsave(&__ipipe_pipelock,flags);
+
+	if (__ipipe_ptd_key_count < IPIPE_ROOT_NPTDKEYS) {
+		key = ffz(__ipipe_ptd_key_map);
+		set_bit(key,&__ipipe_ptd_key_map);
+		__ipipe_ptd_key_count++;
+	}
+
+	spin_unlock_irqrestore(&__ipipe_pipelock,flags);
+
+	return key;
+}
+
+int ipipe_free_ptdkey (int key)
+{
+	unsigned long flags;
+
+	if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+		return -EINVAL;
+
+	spin_lock_irqsave(&__ipipe_pipelock,flags);
+
+	if (test_and_clear_bit(key,&__ipipe_ptd_key_map))
+		__ipipe_ptd_key_count--;
+
+	spin_unlock_irqrestore(&__ipipe_pipelock,flags);
+
+	return 0;
+}
+
+int ipipe_set_ptd (int key, void *value)
+
+{
+	if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+		return -EINVAL;
+
+	current->ptd[key] = value;
+
+	return 0;
+}
+
+void *ipipe_get_ptd (int key)
+
+{
+	if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+		return NULL;
+
+	return current->ptd[key];
+}
+
+#ifdef CONFIG_PROC_FS
+
+struct proc_dir_entry *ipipe_proc_root;
+
+static int __ipipe_version_info_proc(char *page,
+				     char **start,
+				     off_t off, int count, int *eof, void *data)
+{
+	int len = sprintf(page, "%s\n", IPIPE_VERSION_STRING);
+
+	len -= off;
+
+	if (len <= off + count)
+		*eof = 1;
+
+	*start = page + off;
+
+	if(len > count)
+		len = count;
+
+	if(len < 0)
+		len = 0;
+
+	return len;
+}
+
+static int __ipipe_common_info_show(struct seq_file *p, void *data)
+{
+	struct ipipe_domain *ipd = (struct ipipe_domain *)p->private;
+	char handling, stickiness, lockbit, exclusive, virtuality;
+
+	unsigned long ctlbits;
+	unsigned irq;
+
+	seq_printf(p, "       +----- Handling ([A]ccepted, [G]rabbed, [W]ired, [D]iscarded)\n");
+	seq_printf(p, "       |+---- Sticky\n");
+	seq_printf(p, "       ||+--- Locked\n");
+	seq_printf(p, "       |||+-- Exclusive\n");
+	seq_printf(p, "       ||||+- Virtual\n");
+	seq_printf(p, "[IRQ]  |||||\n");
+
+	mutex_lock(&ipd->mutex);
+
+	for (irq = 0; irq < IPIPE_NR_IRQS; irq++) {
+		/* Remember to protect against
+		 * ipipe_virtual_irq/ipipe_control_irq if more fields
+		 * get involved. */
+		ctlbits = ipd->irqs[irq].control;
+
+		if (irq >= IPIPE_NR_XIRQS && !ipipe_virtual_irq_p(irq))
+			/*
+			 * There might be a hole between the last external
+			 * IRQ and the first virtual one; skip it.
+			 */
+			continue;
+
+		if (ipipe_virtual_irq_p(irq)
+		    && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map))
+			/* Non-allocated virtual IRQ; skip it. */
+			continue;
+
+		/*
+		 * Statuses are as follows:
+		 * o "accepted" means handled _and_ passed down the pipeline.
+		 * o "grabbed" means handled, but the interrupt might be
+		 * terminated _or_ passed down the pipeline depending on
+		 * what the domain handler asks for to the I-pipe.
+		 * o "wired" is basically the same as "grabbed", except that
+		 * the interrupt is unconditionally delivered to an invariant
+		 * pipeline head domain.
+		 * o "passed" means unhandled by the domain but passed
+		 * down the pipeline.
+		 * o "discarded" means unhandled and _not_ passed down the
+		 * pipeline. The interrupt merely disappears from the
+		 * current domain down to the end of the pipeline.
+		 */
+		if (ctlbits & IPIPE_HANDLE_MASK) {
+			if (ctlbits & IPIPE_PASS_MASK)
+				handling = 'A';
+			else if (ctlbits & IPIPE_WIRED_MASK)
+				handling = 'W';
+			else
+				handling = 'G';
+		} else if (ctlbits & IPIPE_PASS_MASK)
+			/* Do not output if no major action is taken. */
+			continue;
+		else
+			handling = 'D';
+
+		if (ctlbits & IPIPE_STICKY_MASK)
+			stickiness = 'S';
+		else
+			stickiness = '.';
+
+		if (ctlbits & IPIPE_LOCK_MASK)
+			lockbit = 'L';
+		else
+			lockbit = '.';
+
+		if (ctlbits & IPIPE_EXCLUSIVE_MASK)
+			exclusive = 'X';
+		else
+			exclusive = '.';
+
+		if (ipipe_virtual_irq_p(irq))
+			virtuality = 'V';
+		else
+			virtuality = '.';
+
+		seq_printf(p, " %3u:  %c%c%c%c%c\n",
+			     irq, handling, stickiness, lockbit, exclusive, virtuality);
+	}
+
+	seq_printf(p, "[Domain info]\n");
+
+	seq_printf(p, "id=0x%.8x\n", ipd->domid);
+
+	if (test_bit(IPIPE_AHEAD_FLAG,&ipd->flags))
+		seq_printf(p, "priority=topmost\n");
+	else
+		seq_printf(p, "priority=%d\n", ipd->priority);
+
+	mutex_unlock(&ipd->mutex);
+
+	return 0;
+}
+
+static int __ipipe_common_info_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, __ipipe_common_info_show, PROC_I(inode)->pde->data);
+}
+
+static struct file_operations __ipipe_info_proc_ops = {
+	.owner		= THIS_MODULE,
+	.open		= __ipipe_common_info_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+void __ipipe_add_domain_proc(struct ipipe_domain *ipd)
+{
+	struct proc_dir_entry *e = create_proc_entry(ipd->name, 0444, ipipe_proc_root);
+	if (e) {
+		e->proc_fops = &__ipipe_info_proc_ops;
+		e->data = (void*) ipd;
+	}
+}
+
+void __ipipe_remove_domain_proc(struct ipipe_domain *ipd)
+{
+	remove_proc_entry(ipd->name,ipipe_proc_root);
+}
+
+void __init ipipe_init_proc(void)
+{
+	ipipe_proc_root = create_proc_entry("ipipe",S_IFDIR, 0);
+	create_proc_read_entry("version",0444,ipipe_proc_root,&__ipipe_version_info_proc,NULL);
+	__ipipe_add_domain_proc(ipipe_root_domain);
+
+	__ipipe_init_tracer();
+}
+
+#endif	/* CONFIG_PROC_FS */
+
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+
+DEFINE_PER_CPU(int, ipipe_percpu_context_check) = { 1 };
+DEFINE_PER_CPU(int, ipipe_saved_context_check_state);
+
+void ipipe_check_context(struct ipipe_domain *border_domain)
+{
+	struct ipipe_percpu_domain_data *p;
+	struct ipipe_domain *this_domain;
+	unsigned long flags;
+	int cpu;
+
+	local_irq_save_hw_smp(flags);
+
+	this_domain = __ipipe_current_domain;
+	p = ipipe_head_cpudom_ptr();
+	if (likely(this_domain->priority <= border_domain->priority &&
+		   !test_bit(IPIPE_STALL_FLAG, &p->status))) {
+		local_irq_restore_hw_smp(flags);
+		return;
+	}
+
+	cpu = ipipe_processor_id();
+	if (!per_cpu(ipipe_percpu_context_check, cpu)) {
+		local_irq_restore_hw_smp(flags);
+		return;
+	}
+
+	local_irq_restore_hw_smp(flags);
+
+	ipipe_context_check_off();
+	ipipe_trace_panic_freeze();
+	ipipe_set_printk_sync(__ipipe_current_domain);
+
+	if (this_domain->priority > border_domain->priority)
+		printk(KERN_ERR "I-pipe: Detected illicit call from domain "
+				"'%s'\n"
+		       KERN_ERR "        into a service reserved for domain "
+				"'%s' and below.\n",
+		       this_domain->name, border_domain->name);
+	else
+		printk(KERN_ERR "I-pipe: Detected stalled topmost domain, "
+				"probably caused by a bug.\n"
+				"        A critical section may have been "
+				"left unterminated.\n");
+	dump_stack();
+	ipipe_trace_panic_dump();
+}
+
+EXPORT_SYMBOL(ipipe_check_context);
+
+#endif /* CONFIG_IPIPE_DEBUG_CONTEXT */
+
+#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP)
+
+int notrace __ipipe_check_percpu_access(void)
+{
+	struct ipipe_percpu_domain_data *p;
+	struct ipipe_domain *this_domain;
+	unsigned long flags;
+	int ret = 0;
+
+	local_irq_save_hw_notrace(flags);
+
+	this_domain = __raw_get_cpu_var(ipipe_percpu_domain);
+
+	/*
+	 * Only the root domain may implement preemptive CPU migration
+	 * of tasks, so anything above in the pipeline should be fine.
+	 */
+	if (this_domain->priority > IPIPE_ROOT_PRIO)
+		goto out;
+
+	if (raw_irqs_disabled_flags(flags))
+		goto out;
+
+	/*
+	 * Last chance: hw interrupts were enabled on entry while
+	 * running over the root domain, but the root stage might be
+	 * currently stalled, in which case preemption would be
+	 * disabled, and no migration could occur.
+	 */
+	if (this_domain == ipipe_root_domain) {
+		p = ipipe_root_cpudom_ptr();
+		if (test_bit(IPIPE_STALL_FLAG, &p->status))
+			goto out;
+	}
+	/*
+	 * Our caller may end up accessing the wrong per-cpu variable
+	 * instance due to CPU migration; tell it to complain about
+	 * this.
+	 */
+	ret = 1;
+out:
+	local_irq_restore_hw_notrace(flags);
+
+	return ret;
+}
+
+void __ipipe_spin_unlock_debug(unsigned long flags)
+{
+	/*
+	 * We catch a nasty issue where spin_unlock_irqrestore() on a
+	 * regular kernel spinlock is about to re-enable hw interrupts
+	 * in a section entered with hw irqs off. This is clearly the
+	 * sign of a massive breakage coming. Usual suspect is a
+	 * regular spinlock which was overlooked, used within a
+	 * section which must run with hw irqs disabled.
+	 */
+	WARN_ON_ONCE(!raw_irqs_disabled_flags(flags) && irqs_disabled_hw());
+}
+EXPORT_SYMBOL(__ipipe_spin_unlock_debug);
+
+#endif /* CONFIG_IPIPE_DEBUG_INTERNAL && CONFIG_SMP */
+
+
+void ipipe_prepare_panic(void)
+{
+	ipipe_set_printk_sync(ipipe_current_domain);
+	ipipe_context_check_off();
+}
+
+EXPORT_SYMBOL_GPL(ipipe_prepare_panic);
+
+EXPORT_SYMBOL(ipipe_virtualize_irq);
+EXPORT_SYMBOL(ipipe_control_irq);
+EXPORT_SYMBOL(ipipe_suspend_domain);
+EXPORT_SYMBOL(ipipe_alloc_virq);
+EXPORT_PER_CPU_SYMBOL(ipipe_percpu_domain);
+EXPORT_PER_CPU_SYMBOL(ipipe_percpu_darray);
+EXPORT_SYMBOL(ipipe_root);
+EXPORT_SYMBOL(ipipe_stall_pipeline_from);
+EXPORT_SYMBOL(ipipe_test_and_stall_pipeline_from);
+EXPORT_SYMBOL(ipipe_test_and_unstall_pipeline_from);
+EXPORT_SYMBOL(ipipe_restore_pipeline_from);
+EXPORT_SYMBOL(ipipe_unstall_pipeline_head);
+EXPORT_SYMBOL(__ipipe_restore_pipeline_head);
+EXPORT_SYMBOL(__ipipe_unstall_root);
+EXPORT_SYMBOL(__ipipe_restore_root);
+EXPORT_SYMBOL(__ipipe_spin_lock_irq);
+EXPORT_SYMBOL(__ipipe_spin_unlock_irq);
+EXPORT_SYMBOL(__ipipe_spin_lock_irqsave);
+EXPORT_SYMBOL(__ipipe_spin_trylock_irq);
+EXPORT_SYMBOL(__ipipe_spin_trylock_irqsave);
+EXPORT_SYMBOL(__ipipe_spin_unlock_irqrestore);
+EXPORT_SYMBOL(__ipipe_pipeline);
+EXPORT_SYMBOL(__ipipe_lock_irq);
+EXPORT_SYMBOL(__ipipe_unlock_irq);
+EXPORT_SYMBOL(ipipe_register_domain);
+EXPORT_SYMBOL(ipipe_unregister_domain);
+EXPORT_SYMBOL(ipipe_free_virq);
+EXPORT_SYMBOL(ipipe_init_attr);
+EXPORT_SYMBOL(ipipe_catch_event);
+EXPORT_SYMBOL(ipipe_alloc_ptdkey);
+EXPORT_SYMBOL(ipipe_free_ptdkey);
+EXPORT_SYMBOL(ipipe_set_ptd);
+EXPORT_SYMBOL(ipipe_get_ptd);
+EXPORT_SYMBOL(ipipe_set_irq_affinity);
+EXPORT_SYMBOL(ipipe_send_ipi);
+EXPORT_SYMBOL(__ipipe_pend_irq);
+EXPORT_SYMBOL(__ipipe_set_irq_pending);
+EXPORT_SYMBOL(__ipipe_event_monitors);
+#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP)
+EXPORT_SYMBOL(__ipipe_check_percpu_access);
+#endif
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+EXPORT_SYMBOL(ipipe_request_tickdev);
+EXPORT_SYMBOL(ipipe_release_tickdev);
+#endif
+
+EXPORT_SYMBOL(ipipe_critical_enter);
+EXPORT_SYMBOL(ipipe_critical_exit);
+EXPORT_SYMBOL(ipipe_trigger_irq);
+EXPORT_SYMBOL(ipipe_get_sysinfo);
diff --git a/kernel/ipipe/tracer.c b/kernel/ipipe/tracer.c
new file mode 100644
index 0000000..f013ef4
--- /dev/null
+++ b/kernel/ipipe/tracer.c
@@ -0,0 +1,1442 @@
+/* -*- linux-c -*-
+ * kernel/ipipe/tracer.c
+ *
+ * Copyright (C) 2005 Luotao Fu.
+ *               2005-2008 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/kallsyms.h>
+#include <linux/seq_file.h>
+#include <linux/proc_fs.h>
+#include <linux/ctype.h>
+#include <linux/vmalloc.h>
+#include <linux/pid.h>
+#include <linux/vermagic.h>
+#include <linux/sched.h>
+#include <linux/ipipe.h>
+#include <linux/ftrace.h>
+#include <asm/uaccess.h>
+
+#define IPIPE_TRACE_PATHS           4 /* <!> Do not lower below 3 */
+#define IPIPE_DEFAULT_ACTIVE        0
+#define IPIPE_DEFAULT_MAX           1
+#define IPIPE_DEFAULT_FROZEN        2
+
+#define IPIPE_TRACE_POINTS          (1 << CONFIG_IPIPE_TRACE_SHIFT)
+#define WRAP_POINT_NO(point)        ((point) & (IPIPE_TRACE_POINTS-1))
+
+#define IPIPE_DEFAULT_PRE_TRACE     10
+#define IPIPE_DEFAULT_POST_TRACE    10
+#define IPIPE_DEFAULT_BACK_TRACE    100
+
+#define IPIPE_DELAY_NOTE            1000  /* in nanoseconds */
+#define IPIPE_DELAY_WARN            10000 /* in nanoseconds */
+
+#define IPIPE_TFLG_NMI_LOCK         0x0001
+#define IPIPE_TFLG_NMI_HIT          0x0002
+#define IPIPE_TFLG_NMI_FREEZE_REQ   0x0004
+
+#define IPIPE_TFLG_HWIRQ_OFF        0x0100
+#define IPIPE_TFLG_FREEZING         0x0200
+#define IPIPE_TFLG_CURRDOM_SHIFT    10   /* bits 10..11: current domain */
+#define IPIPE_TFLG_CURRDOM_MASK     0x0C00
+#define IPIPE_TFLG_DOMSTATE_SHIFT   12   /* bits 12..15: domain stalled? */
+#define IPIPE_TFLG_DOMSTATE_BITS    3
+
+#define IPIPE_TFLG_DOMAIN_STALLED(point, n) \
+	(point->flags & (1 << (n + IPIPE_TFLG_DOMSTATE_SHIFT)))
+#define IPIPE_TFLG_CURRENT_DOMAIN(point) \
+	((point->flags & IPIPE_TFLG_CURRDOM_MASK) >> IPIPE_TFLG_CURRDOM_SHIFT)
+
+struct ipipe_trace_point {
+	short type;
+	short flags;
+	unsigned long eip;
+	unsigned long parent_eip;
+	unsigned long v;
+	unsigned long long timestamp;
+};
+
+struct ipipe_trace_path {
+	volatile int flags;
+	int dump_lock; /* separated from flags due to cross-cpu access */
+	int trace_pos; /* next point to fill */
+	int begin, end; /* finalised path begin and end */
+	int post_trace; /* non-zero when in post-trace phase */
+	unsigned long long length; /* max path length in cycles */
+	unsigned long nmi_saved_eip; /* for deferred requests from NMIs */
+	unsigned long nmi_saved_parent_eip;
+	unsigned long nmi_saved_v;
+	struct ipipe_trace_point point[IPIPE_TRACE_POINTS];
+} ____cacheline_aligned_in_smp;
+
+enum ipipe_trace_type
+{
+	IPIPE_TRACE_FUNC = 0,
+	IPIPE_TRACE_BEGIN,
+	IPIPE_TRACE_END,
+	IPIPE_TRACE_FREEZE,
+	IPIPE_TRACE_SPECIAL,
+	IPIPE_TRACE_PID,
+	IPIPE_TRACE_EVENT,
+};
+
+#define IPIPE_TYPE_MASK             0x0007
+#define IPIPE_TYPE_BITS             3
+
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
+static DEFINE_PER_CPU(struct ipipe_trace_path *, trace_path);
+#else /* !CONFIG_IPIPE_TRACE_VMALLOC */
+static DEFINE_PER_CPU(struct ipipe_trace_path, trace_path[IPIPE_TRACE_PATHS]) =
+	{ [0 ... IPIPE_TRACE_PATHS-1] = { .begin = -1, .end = -1 } };
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
+
+int ipipe_trace_enable = 0;
+
+static DEFINE_PER_CPU(int, active_path) = { IPIPE_DEFAULT_ACTIVE };
+static DEFINE_PER_CPU(int, max_path) = { IPIPE_DEFAULT_MAX };
+static DEFINE_PER_CPU(int, frozen_path) = { IPIPE_DEFAULT_FROZEN };
+static IPIPE_DEFINE_SPINLOCK(global_path_lock);
+static int pre_trace = IPIPE_DEFAULT_PRE_TRACE;
+static int post_trace = IPIPE_DEFAULT_POST_TRACE;
+static int back_trace = IPIPE_DEFAULT_BACK_TRACE;
+static int verbose_trace = 1;
+static unsigned long trace_overhead;
+
+static unsigned long trigger_begin;
+static unsigned long trigger_end;
+
+static DEFINE_MUTEX(out_mutex);
+static struct ipipe_trace_path *print_path;
+#ifdef CONFIG_IPIPE_TRACE_PANIC
+static struct ipipe_trace_path *panic_path;
+#endif /* CONFIG_IPIPE_TRACE_PANIC */
+static int print_pre_trace;
+static int print_post_trace;
+
+
+static long __ipipe_signed_tsc2us(long long tsc);
+static void
+__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point);
+static void __ipipe_print_symname(struct seq_file *m, unsigned long eip);
+
+
+static notrace void
+__ipipe_store_domain_states(struct ipipe_trace_point *point)
+{
+	struct ipipe_domain *ipd;
+	struct list_head *pos;
+	int i = 0;
+
+	list_for_each_prev(pos, &__ipipe_pipeline) {
+		ipd = list_entry(pos, struct ipipe_domain, p_link);
+
+		if (test_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status)))
+			point->flags |= 1 << (i + IPIPE_TFLG_DOMSTATE_SHIFT);
+
+		if (ipd == __ipipe_current_domain)
+			point->flags |= i << IPIPE_TFLG_CURRDOM_SHIFT;
+
+		if (++i > IPIPE_TFLG_DOMSTATE_BITS)
+			break;
+	}
+}
+
+static notrace int __ipipe_get_free_trace_path(int old, int cpu)
+{
+	int new_active = old;
+	struct ipipe_trace_path *tp;
+
+	do {
+		if (++new_active == IPIPE_TRACE_PATHS)
+			new_active = 0;
+		tp = &per_cpu(trace_path, cpu)[new_active];
+	} while (new_active == per_cpu(max_path, cpu) ||
+	         new_active == per_cpu(frozen_path, cpu) ||
+	         tp->dump_lock);
+
+	return new_active;
+}
+
+static notrace void
+__ipipe_migrate_pre_trace(struct ipipe_trace_path *new_tp,
+                          struct ipipe_trace_path *old_tp, int old_pos)
+{
+	int i;
+
+	new_tp->trace_pos = pre_trace+1;
+
+	for (i = new_tp->trace_pos; i > 0; i--)
+		memcpy(&new_tp->point[WRAP_POINT_NO(new_tp->trace_pos-i)],
+		       &old_tp->point[WRAP_POINT_NO(old_pos-i)],
+		       sizeof(struct ipipe_trace_point));
+
+	/* mark the end (i.e. the point before point[0]) invalid */
+	new_tp->point[IPIPE_TRACE_POINTS-1].eip = 0;
+}
+
+static notrace struct ipipe_trace_path *
+__ipipe_trace_end(int cpu, struct ipipe_trace_path *tp, int pos)
+{
+	struct ipipe_trace_path *old_tp = tp;
+	long active = per_cpu(active_path, cpu);
+	unsigned long long length;
+
+	/* do we have a new worst case? */
+	length = tp->point[tp->end].timestamp -
+	         tp->point[tp->begin].timestamp;
+	if (length > per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)].length) {
+		/* we need protection here against other cpus trying
+		   to start a proc dump */
+		spin_lock(&global_path_lock);
+
+		/* active path holds new worst case */
+		tp->length = length;
+		per_cpu(max_path, cpu) = active;
+
+		/* find next unused trace path */
+		active = __ipipe_get_free_trace_path(active, cpu);
+
+		spin_unlock(&global_path_lock);
+
+		tp = &per_cpu(trace_path, cpu)[active];
+
+		/* migrate last entries for pre-tracing */
+		__ipipe_migrate_pre_trace(tp, old_tp, pos);
+	}
+
+	return tp;
+}
+
+static notrace struct ipipe_trace_path *
+__ipipe_trace_freeze(int cpu, struct ipipe_trace_path *tp, int pos)
+{
+	struct ipipe_trace_path *old_tp = tp;
+	long active = per_cpu(active_path, cpu);
+	int n;
+
+	/* frozen paths have no core (begin=end) */
+	tp->begin = tp->end;
+
+	/* we need protection here against other cpus trying
+	 * to set their frozen path or to start a proc dump */
+	spin_lock(&global_path_lock);
+
+	per_cpu(frozen_path, cpu) = active;
+
+	/* find next unused trace path */
+	active = __ipipe_get_free_trace_path(active, cpu);
+
+	/* check if this is the first frozen path */
+	for_each_possible_cpu(n) {
+		if (n != cpu &&
+		    per_cpu(trace_path, n)[per_cpu(frozen_path, n)].end >= 0)
+			tp->end = -1;
+	}
+
+	spin_unlock(&global_path_lock);
+
+	tp = &per_cpu(trace_path, cpu)[active];
+
+	/* migrate last entries for pre-tracing */
+	__ipipe_migrate_pre_trace(tp, old_tp, pos);
+
+	return tp;
+}
+
+void notrace
+__ipipe_trace(enum ipipe_trace_type type, unsigned long eip,
+              unsigned long parent_eip, unsigned long v)
+{
+	struct ipipe_trace_path *tp, *old_tp;
+	int pos, next_pos, begin;
+	struct ipipe_trace_point *point;
+	unsigned long flags;
+	int cpu;
+
+	local_irq_save_hw_notrace(flags);
+
+	cpu = ipipe_processor_id();
+ restart:
+	tp = old_tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
+
+	/* here starts a race window with NMIs - catched below */
+
+	/* check for NMI recursion */
+	if (unlikely(tp->flags & IPIPE_TFLG_NMI_LOCK)) {
+		tp->flags |= IPIPE_TFLG_NMI_HIT;
+
+		/* first freeze request from NMI context? */
+		if ((type == IPIPE_TRACE_FREEZE) &&
+		    !(tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)) {
+			/* save arguments and mark deferred freezing */
+			tp->flags |= IPIPE_TFLG_NMI_FREEZE_REQ;
+			tp->nmi_saved_eip = eip;
+			tp->nmi_saved_parent_eip = parent_eip;
+			tp->nmi_saved_v = v;
+		}
+		return; /* no need for restoring flags inside IRQ */
+	}
+
+	/* clear NMI events and set lock (atomically per cpu) */
+	tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT |
+	                           IPIPE_TFLG_NMI_FREEZE_REQ))
+	                       | IPIPE_TFLG_NMI_LOCK;
+
+	/* check active_path again - some nasty NMI may have switched
+	 * it meanwhile */
+	if (unlikely(tp !=
+		     &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)])) {
+		/* release lock on wrong path and restart */
+		tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+		/* there is no chance that the NMI got deferred
+		 * => no need to check for pending freeze requests */
+		goto restart;
+	}
+
+	/* get the point buffer */
+	pos = tp->trace_pos;
+	point = &tp->point[pos];
+
+	/* store all trace point data */
+	point->type = type;
+	point->flags = raw_irqs_disabled_flags(flags) ? IPIPE_TFLG_HWIRQ_OFF : 0;
+	point->eip = eip;
+	point->parent_eip = parent_eip;
+	point->v = v;
+	ipipe_read_tsc(point->timestamp);
+
+	__ipipe_store_domain_states(point);
+
+	/* forward to next point buffer */
+	next_pos = WRAP_POINT_NO(pos+1);
+	tp->trace_pos = next_pos;
+
+	/* only mark beginning if we haven't started yet */
+	begin = tp->begin;
+	if (unlikely(type == IPIPE_TRACE_BEGIN) && (begin < 0))
+		tp->begin = pos;
+
+	/* end of critical path, start post-trace if not already started */
+	if (unlikely(type == IPIPE_TRACE_END) &&
+	    (begin >= 0) && !tp->post_trace)
+		tp->post_trace = post_trace + 1;
+
+	/* freeze only if the slot is free and we are not already freezing */
+	if ((unlikely(type == IPIPE_TRACE_FREEZE) ||
+	     (unlikely(eip >= trigger_begin && eip <= trigger_end) &&
+	     type == IPIPE_TRACE_FUNC)) &&
+	    per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)].begin < 0 &&
+	    !(tp->flags & IPIPE_TFLG_FREEZING)) {
+		tp->post_trace = post_trace + 1;
+		tp->flags |= IPIPE_TFLG_FREEZING;
+	}
+
+	/* enforce end of trace in case of overflow */
+	if (unlikely(WRAP_POINT_NO(next_pos + 1) == begin)) {
+		tp->end = pos;
+		goto enforce_end;
+	}
+
+	/* stop tracing this path if we are in post-trace and
+	 *  a) that phase is over now or
+	 *  b) a new TRACE_BEGIN came in but we are not freezing this path */
+	if (unlikely((tp->post_trace > 0) && ((--tp->post_trace == 0) ||
+	             ((type == IPIPE_TRACE_BEGIN) &&
+	              !(tp->flags & IPIPE_TFLG_FREEZING))))) {
+		/* store the path's end (i.e. excluding post-trace) */
+		tp->end = WRAP_POINT_NO(pos - post_trace + tp->post_trace);
+
+ enforce_end:
+		if (tp->flags & IPIPE_TFLG_FREEZING)
+			tp = __ipipe_trace_freeze(cpu, tp, pos);
+		else
+			tp = __ipipe_trace_end(cpu, tp, pos);
+
+		/* reset the active path, maybe already start a new one */
+		tp->begin = (type == IPIPE_TRACE_BEGIN) ?
+			WRAP_POINT_NO(tp->trace_pos - 1) : -1;
+		tp->end = -1;
+		tp->post_trace = 0;
+		tp->flags = 0;
+
+		/* update active_path not earlier to avoid races with NMIs */
+		per_cpu(active_path, cpu) = tp - per_cpu(trace_path, cpu);
+	}
+
+	/* we still have old_tp and point,
+	 * let's reset NMI lock and check for catches */
+	old_tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+	if (unlikely(old_tp->flags & IPIPE_TFLG_NMI_HIT)) {
+		/* well, this late tagging may not immediately be visible for
+		 * other cpus already dumping this path - a minor issue */
+		point->flags |= IPIPE_TFLG_NMI_HIT;
+
+		/* handle deferred freezing from NMI context */
+		if (old_tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)
+			__ipipe_trace(IPIPE_TRACE_FREEZE, old_tp->nmi_saved_eip,
+			              old_tp->nmi_saved_parent_eip,
+			              old_tp->nmi_saved_v);
+	}
+
+	local_irq_restore_hw_notrace(flags);
+}
+
+static unsigned long __ipipe_global_path_lock(void)
+{
+	unsigned long flags;
+	int cpu;
+	struct ipipe_trace_path *tp;
+
+	spin_lock_irqsave(&global_path_lock, flags);
+
+	cpu = ipipe_processor_id();
+ restart:
+	tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
+
+	/* here is small race window with NMIs - catched below */
+
+	/* clear NMI events and set lock (atomically per cpu) */
+	tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT |
+	                           IPIPE_TFLG_NMI_FREEZE_REQ))
+	                       | IPIPE_TFLG_NMI_LOCK;
+
+	/* check active_path again - some nasty NMI may have switched
+	 * it meanwhile */
+	if (tp != &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)]) {
+		/* release lock on wrong path and restart */
+		tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+		/* there is no chance that the NMI got deferred
+		 * => no need to check for pending freeze requests */
+		goto restart;
+	}
+
+	return flags;
+}
+
+static void __ipipe_global_path_unlock(unsigned long flags)
+{
+	int cpu;
+	struct ipipe_trace_path *tp;
+
+	/* release spinlock first - it's not involved in the NMI issue */
+	__ipipe_spin_unlock_irqbegin(&global_path_lock);
+
+	cpu = ipipe_processor_id();
+	tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
+
+	tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+	/* handle deferred freezing from NMI context */
+	if (tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)
+		__ipipe_trace(IPIPE_TRACE_FREEZE, tp->nmi_saved_eip,
+		              tp->nmi_saved_parent_eip, tp->nmi_saved_v);
+
+	/* See __ipipe_spin_lock_irqsave() and friends. */
+	__ipipe_spin_unlock_irqcomplete(flags);
+}
+
+void notrace ipipe_trace_begin(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_BEGIN, __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_begin);
+
+void notrace ipipe_trace_end(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_END, __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_end);
+
+void notrace ipipe_trace_freeze(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_FREEZE, __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_freeze);
+
+void notrace ipipe_trace_special(unsigned char id, unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_SPECIAL | (id << IPIPE_TYPE_BITS),
+	              __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_special);
+
+void notrace ipipe_trace_pid(pid_t pid, short prio)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_PID | (prio << IPIPE_TYPE_BITS),
+	              __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, pid);
+}
+EXPORT_SYMBOL(ipipe_trace_pid);
+
+void notrace ipipe_trace_event(unsigned char id, unsigned long delay_tsc)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_EVENT | (id << IPIPE_TYPE_BITS),
+	              __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, delay_tsc);
+}
+EXPORT_SYMBOL(ipipe_trace_event);
+
+int ipipe_trace_max_reset(void)
+{
+	int cpu;
+	unsigned long flags;
+	struct ipipe_trace_path *path;
+	int ret = 0;
+
+	flags = __ipipe_global_path_lock();
+
+	for_each_possible_cpu(cpu) {
+		path = &per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)];
+
+		if (path->dump_lock) {
+			ret = -EBUSY;
+			break;
+		}
+
+		path->begin     = -1;
+		path->end       = -1;
+		path->trace_pos = 0;
+		path->length    = 0;
+	}
+
+	__ipipe_global_path_unlock(flags);
+
+	return ret;
+}
+EXPORT_SYMBOL(ipipe_trace_max_reset);
+
+int ipipe_trace_frozen_reset(void)
+{
+	int cpu;
+	unsigned long flags;
+	struct ipipe_trace_path *path;
+	int ret = 0;
+
+	flags = __ipipe_global_path_lock();
+
+	for_each_online_cpu(cpu) {
+		path = &per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)];
+
+		if (path->dump_lock) {
+			ret = -EBUSY;
+			break;
+		}
+
+		path->begin = -1;
+		path->end = -1;
+		path->trace_pos = 0;
+		path->length    = 0;
+	}
+
+	__ipipe_global_path_unlock(flags);
+
+	return ret;
+}
+EXPORT_SYMBOL(ipipe_trace_frozen_reset);
+
+static void
+__ipipe_get_task_info(char *task_info, struct ipipe_trace_point *point,
+                      int trylock)
+{
+	struct task_struct *task = NULL;
+	char buf[8];
+	int i;
+	int locked = 1;
+
+	if (trylock) {
+		if (!read_trylock(&tasklist_lock))
+			locked = 0;
+	} else
+		read_lock(&tasklist_lock);
+
+	if (locked)
+		task = find_task_by_pid_ns((pid_t)point->v, &init_pid_ns);
+
+	if (task)
+		strncpy(task_info, task->comm, 11);
+	else
+		strcpy(task_info, "-<?>-");
+
+	if (locked)
+		read_unlock(&tasklist_lock);
+
+	for (i = strlen(task_info); i < 11; i++)
+		task_info[i] = ' ';
+
+	sprintf(buf, " %d ", point->type >> IPIPE_TYPE_BITS);
+	strcpy(task_info + (11 - strlen(buf)), buf);
+}
+
+static void
+__ipipe_get_event_date(char *buf,struct ipipe_trace_path *path,
+		       struct ipipe_trace_point *point)
+{
+	long time;
+	int type;
+
+	time = __ipipe_signed_tsc2us(point->timestamp -
+				     path->point[path->begin].timestamp + point->v);
+	type = point->type >> IPIPE_TYPE_BITS;
+
+	if (type == 0)
+		/*
+		 * Event type #0 is predefined, stands for the next
+		 * timer tick.
+		 */
+		sprintf(buf, "tick@%-6ld", time);
+	else
+		sprintf(buf, "%3d@%-7ld", type, time);
+}
+
+#ifdef CONFIG_IPIPE_TRACE_PANIC
+void ipipe_trace_panic_freeze(void)
+{
+	unsigned long flags;
+	int cpu;
+
+	if (!ipipe_trace_enable)
+		return;
+
+	ipipe_trace_enable = 0;
+	local_irq_save_hw_notrace(flags);
+
+	cpu = ipipe_processor_id();
+
+	panic_path = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
+
+	local_irq_restore_hw(flags);
+}
+EXPORT_SYMBOL(ipipe_trace_panic_freeze);
+
+void ipipe_trace_panic_dump(void)
+{
+	int cnt = back_trace;
+	int start, pos;
+	char buf[16];
+
+	if (!panic_path)
+		return;
+
+	ipipe_context_check_off();
+
+	printk("I-pipe tracer log (%d points):\n", cnt);
+
+	start = pos = WRAP_POINT_NO(panic_path->trace_pos-1);
+
+	while (cnt-- > 0) {
+		struct ipipe_trace_point *point = &panic_path->point[pos];
+		long time;
+		char info[16];
+		int i;
+
+		printk(" %c",
+		       (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' ');
+
+		for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--)
+			printk("%c",
+			       (IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ?
+				(IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
+					'#' : '+') :
+				(IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
+					'*' : ' '));
+
+		if (!point->eip)
+			printk("-<invalid>-\n");
+		else {
+			__ipipe_trace_point_type(buf, point);
+			printk("%s", buf);
+
+			switch (point->type & IPIPE_TYPE_MASK) {
+				case IPIPE_TRACE_FUNC:
+					printk("           ");
+					break;
+
+				case IPIPE_TRACE_PID:
+					__ipipe_get_task_info(info,
+							      point, 1);
+					printk("%s", info);
+					break;
+
+				case IPIPE_TRACE_EVENT:
+					__ipipe_get_event_date(info,
+							       panic_path, point);
+					printk("%s", info);
+					break;
+
+				default:
+					printk("0x%08lx ", point->v);
+			}
+
+			time = __ipipe_signed_tsc2us(point->timestamp -
+				panic_path->point[start].timestamp);
+			printk(" %5ld ", time);
+
+			__ipipe_print_symname(NULL, point->eip);
+			printk(" (");
+			__ipipe_print_symname(NULL, point->parent_eip);
+			printk(")\n");
+		}
+		pos = WRAP_POINT_NO(pos - 1);
+	}
+
+	panic_path = NULL;
+}
+EXPORT_SYMBOL(ipipe_trace_panic_dump);
+#endif /* CONFIG_IPIPE_TRACE_PANIC */
+
+
+/* --- /proc output --- */
+
+static notrace int __ipipe_in_critical_trpath(long point_no)
+{
+	return ((WRAP_POINT_NO(point_no-print_path->begin) <
+	         WRAP_POINT_NO(print_path->end-print_path->begin)) ||
+	        ((print_path->end == print_path->begin) &&
+	         (WRAP_POINT_NO(point_no-print_path->end) >
+	          print_post_trace)));
+}
+
+static long __ipipe_signed_tsc2us(long long tsc)
+{
+        unsigned long long abs_tsc;
+        long us;
+
+	/* ipipe_tsc2us works on unsigned => handle sign separately */
+        abs_tsc = (tsc >= 0) ? tsc : -tsc;
+        us = ipipe_tsc2us(abs_tsc);
+        if (tsc < 0)
+                return -us;
+        else
+                return us;
+}
+
+static void
+__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point)
+{
+	switch (point->type & IPIPE_TYPE_MASK) {
+		case IPIPE_TRACE_FUNC:
+			strcpy(buf, "func    ");
+			break;
+
+		case IPIPE_TRACE_BEGIN:
+			strcpy(buf, "begin   ");
+			break;
+
+		case IPIPE_TRACE_END:
+			strcpy(buf, "end     ");
+			break;
+
+		case IPIPE_TRACE_FREEZE:
+			strcpy(buf, "freeze  ");
+			break;
+
+		case IPIPE_TRACE_SPECIAL:
+			sprintf(buf, "(0x%02x)  ",
+				point->type >> IPIPE_TYPE_BITS);
+			break;
+
+		case IPIPE_TRACE_PID:
+			sprintf(buf, "[%5d] ", (pid_t)point->v);
+			break;
+
+		case IPIPE_TRACE_EVENT:
+			sprintf(buf, "event   ");
+			break;
+	}
+}
+
+static void
+__ipipe_print_pathmark(struct seq_file *m, struct ipipe_trace_point *point)
+{
+	char mark = ' ';
+	int point_no = point - print_path->point;
+	int i;
+
+	if (print_path->end == point_no)
+		mark = '<';
+	else if (print_path->begin == point_no)
+		mark = '>';
+	else if (__ipipe_in_critical_trpath(point_no))
+		mark = ':';
+	seq_printf(m, "%c%c", mark,
+	           (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' ');
+
+	if (!verbose_trace)
+		return;
+
+	for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--)
+		seq_printf(m, "%c",
+			(IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ?
+			    (IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
+				'#' : '+') :
+			(IPIPE_TFLG_DOMAIN_STALLED(point, i) ? '*' : ' '));
+}
+
+static void
+__ipipe_print_delay(struct seq_file *m, struct ipipe_trace_point *point)
+{
+	unsigned long delay = 0;
+	int next;
+	char *mark = "  ";
+
+	next = WRAP_POINT_NO(point+1 - print_path->point);
+
+	if (next != print_path->trace_pos)
+		delay = ipipe_tsc2ns(print_path->point[next].timestamp -
+		                     point->timestamp);
+
+	if (__ipipe_in_critical_trpath(point - print_path->point)) {
+		if (delay > IPIPE_DELAY_WARN)
+			mark = "! ";
+		else if (delay > IPIPE_DELAY_NOTE)
+			mark = "+ ";
+	}
+	seq_puts(m, mark);
+
+	if (verbose_trace)
+		seq_printf(m, "%3lu.%03lu%c ", delay/1000, delay%1000,
+		           (point->flags & IPIPE_TFLG_NMI_HIT) ? 'N' : ' ');
+	else
+		seq_puts(m, " ");
+}
+
+static void __ipipe_print_symname(struct seq_file *m, unsigned long eip)
+{
+	char namebuf[KSYM_NAME_LEN+1];
+	unsigned long size, offset;
+	const char *sym_name;
+	char *modname;
+
+	sym_name = kallsyms_lookup(eip, &size, &offset, &modname, namebuf);
+
+#ifdef CONFIG_IPIPE_TRACE_PANIC
+	if (!m) {
+		/* panic dump */
+		if (sym_name) {
+			printk("%s+0x%lx", sym_name, offset);
+			if (modname)
+				printk(" [%s]", modname);
+		}
+	} else
+#endif /* CONFIG_IPIPE_TRACE_PANIC */
+	{
+		if (sym_name) {
+			if (verbose_trace) {
+				seq_printf(m, "%s+0x%lx", sym_name, offset);
+				if (modname)
+					seq_printf(m, " [%s]", modname);
+			} else
+				seq_puts(m, sym_name);
+		} else
+			seq_printf(m, "<%08lx>", eip);
+	}
+}
+
+static void __ipipe_print_headline(struct seq_file *m)
+{
+	seq_printf(m, "Calibrated minimum trace-point overhead: %lu.%03lu "
+		   "us\n\n", trace_overhead/1000, trace_overhead%1000);
+
+	if (verbose_trace) {
+		const char *name[4] = { [0 ... 3] = "<unused>" };
+		struct list_head *pos;
+		int i = 0;
+
+		list_for_each_prev(pos, &__ipipe_pipeline) {
+			struct ipipe_domain *ipd =
+				list_entry(pos, struct ipipe_domain, p_link);
+
+			name[i] = ipd->name;
+			if (++i > 3)
+				break;
+		}
+
+		seq_printf(m,
+		           " +----- Hard IRQs ('|': locked)\n"
+		           " |+---- %s\n"
+		           " ||+--- %s\n"
+		           " |||+-- %s\n"
+		           " ||||+- %s%s\n"
+		           " |||||                        +---------- "
+		               "Delay flag ('+': > %d us, '!': > %d us)\n"
+		           " |||||                        |        +- "
+		               "NMI noise ('N')\n"
+		           " |||||                        |        |\n"
+		           "      Type    User Val.   Time    Delay  Function "
+		               "(Parent)\n",
+		           name[3], name[2], name[1], name[0],
+		           name[0] ? " ('*': domain stalled, '+': current, "
+		               "'#': current+stalled)" : "",
+		           IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000);
+	} else
+		seq_printf(m,
+		           " +--------------- Hard IRQs ('|': locked)\n"
+		           " |             +- Delay flag "
+		               "('+': > %d us, '!': > %d us)\n"
+		           " |             |\n"
+		           "  Type     Time   Function (Parent)\n",
+		           IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000);
+}
+
+static void *__ipipe_max_prtrace_start(struct seq_file *m, loff_t *pos)
+{
+	loff_t n = *pos;
+
+	mutex_lock(&out_mutex);
+
+	if (!n) {
+		struct ipipe_trace_path *tp;
+		unsigned long length_usecs;
+		int points, cpu;
+		unsigned long flags;
+
+		/* protect against max_path/frozen_path updates while we
+		 * haven't locked our target path, also avoid recursively
+		 * taking global_path_lock from NMI context */
+		flags = __ipipe_global_path_lock();
+
+		/* find the longest of all per-cpu paths */
+		print_path = NULL;
+		for_each_online_cpu(cpu) {
+			tp = &per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)];
+			if ((print_path == NULL) ||
+			    (tp->length > print_path->length)) {
+				print_path = tp;
+				break;
+			}
+		}
+		print_path->dump_lock = 1;
+
+		__ipipe_global_path_unlock(flags);
+
+		/* does this path actually contain data? */
+		if (print_path->end == print_path->begin)
+			return NULL;
+
+		/* number of points inside the critical path */
+		points = WRAP_POINT_NO(print_path->end-print_path->begin+1);
+
+		/* pre- and post-tracing length, post-trace length was frozen
+		   in __ipipe_trace, pre-trace may have to be reduced due to
+		   buffer overrun */
+		print_pre_trace  = pre_trace;
+		print_post_trace = WRAP_POINT_NO(print_path->trace_pos -
+		                                 print_path->end - 1);
+		if (points+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1)
+			print_pre_trace = IPIPE_TRACE_POINTS - 1 - points -
+				print_post_trace;
+
+		length_usecs = ipipe_tsc2us(print_path->length);
+		seq_printf(m, "I-pipe worst-case tracing service on %s/ipipe-%s\n"
+			"------------------------------------------------------------\n",
+			UTS_RELEASE, IPIPE_ARCH_STRING);
+		seq_printf(m, "CPU: %d, Begin: %lld cycles, Trace Points: "
+			"%d (-%d/+%d), Length: %lu us\n",
+			cpu, print_path->point[print_path->begin].timestamp,
+			points, print_pre_trace, print_post_trace, length_usecs);
+		__ipipe_print_headline(m);
+	}
+
+	/* check if we are inside the trace range */
+	if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 +
+	                       print_pre_trace + print_post_trace))
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin -
+	                                        print_pre_trace + n)];
+}
+
+static void *__ipipe_prtrace_next(struct seq_file *m, void *p, loff_t *pos)
+{
+	loff_t n = ++*pos;
+
+	/* check if we are inside the trace range with the next entry */
+	if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 +
+	                       print_pre_trace + print_post_trace))
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin -
+	                                        print_pre_trace + *pos)];
+}
+
+static void __ipipe_prtrace_stop(struct seq_file *m, void *p)
+{
+	if (print_path)
+		print_path->dump_lock = 0;
+	mutex_unlock(&out_mutex);
+}
+
+static int __ipipe_prtrace_show(struct seq_file *m, void *p)
+{
+	long time;
+	struct ipipe_trace_point *point = p;
+	char buf[16];
+
+	if (!point->eip) {
+		seq_puts(m, "-<invalid>-\n");
+		return 0;
+	}
+
+	__ipipe_print_pathmark(m, point);
+	__ipipe_trace_point_type(buf, point);
+	seq_puts(m, buf);
+	if (verbose_trace)
+		switch (point->type & IPIPE_TYPE_MASK) {
+			case IPIPE_TRACE_FUNC:
+				seq_puts(m, "           ");
+				break;
+
+			case IPIPE_TRACE_PID:
+				__ipipe_get_task_info(buf, point, 0);
+				seq_puts(m, buf);
+				break;
+
+			case IPIPE_TRACE_EVENT:
+				__ipipe_get_event_date(buf, print_path, point);
+				seq_puts(m, buf);
+				break;
+
+			default:
+				seq_printf(m, "0x%08lx ", point->v);
+		}
+
+	time = __ipipe_signed_tsc2us(point->timestamp -
+		print_path->point[print_path->begin].timestamp);
+	seq_printf(m, "%5ld", time);
+
+	__ipipe_print_delay(m, point);
+	__ipipe_print_symname(m, point->eip);
+	seq_puts(m, " (");
+	__ipipe_print_symname(m, point->parent_eip);
+	seq_puts(m, ")\n");
+
+	return 0;
+}
+
+static struct seq_operations __ipipe_max_ptrace_ops = {
+	.start = __ipipe_max_prtrace_start,
+	.next  = __ipipe_prtrace_next,
+	.stop  = __ipipe_prtrace_stop,
+	.show  = __ipipe_prtrace_show
+};
+
+static int __ipipe_max_prtrace_open(struct inode *inode, struct file *file)
+{
+	return seq_open(file, &__ipipe_max_ptrace_ops);
+}
+
+static ssize_t
+__ipipe_max_reset(struct file *file, const char __user *pbuffer,
+                  size_t count, loff_t *data)
+{
+	mutex_lock(&out_mutex);
+	ipipe_trace_max_reset();
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+struct file_operations __ipipe_max_prtrace_fops = {
+	.open       = __ipipe_max_prtrace_open,
+	.read       = seq_read,
+	.write      = __ipipe_max_reset,
+	.llseek     = seq_lseek,
+	.release    = seq_release,
+};
+
+static void *__ipipe_frozen_prtrace_start(struct seq_file *m, loff_t *pos)
+{
+	loff_t n = *pos;
+
+	mutex_lock(&out_mutex);
+
+	if (!n) {
+		struct ipipe_trace_path *tp;
+		int cpu;
+		unsigned long flags;
+
+		/* protect against max_path/frozen_path updates while we
+		 * haven't locked our target path, also avoid recursively
+		 * taking global_path_lock from NMI context */
+		flags = __ipipe_global_path_lock();
+
+		/* find the first of all per-cpu frozen paths */
+		print_path = NULL;
+		for_each_online_cpu(cpu) {
+			tp = &per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)];
+			if (tp->end >= 0) {
+				print_path = tp;
+				break;
+			}
+		}
+		if (print_path)
+			print_path->dump_lock = 1;
+
+		__ipipe_global_path_unlock(flags);
+
+		if (!print_path)
+			return NULL;
+
+		/* back- and post-tracing length, post-trace length was frozen
+		   in __ipipe_trace, back-trace may have to be reduced due to
+		   buffer overrun */
+		print_pre_trace  = back_trace-1; /* substract freeze point */
+		print_post_trace = WRAP_POINT_NO(print_path->trace_pos -
+		                                 print_path->end - 1);
+		if (1+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1)
+			print_pre_trace = IPIPE_TRACE_POINTS - 2 -
+				print_post_trace;
+
+		seq_printf(m, "I-pipe frozen back-tracing service on %s/ipipe-%s\n"
+			"------------------------------------------------------"
+			"------\n",
+			UTS_RELEASE, IPIPE_ARCH_STRING);
+		seq_printf(m, "CPU: %d, Freeze: %lld cycles, Trace Points: %d (+%d)\n",
+			cpu, print_path->point[print_path->begin].timestamp,
+			print_pre_trace+1, print_post_trace);
+		__ipipe_print_headline(m);
+	}
+
+	/* check if we are inside the trace range */
+	if (n >= print_pre_trace + 1 + print_post_trace)
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin-
+	                                        print_pre_trace+n)];
+}
+
+static struct seq_operations __ipipe_frozen_ptrace_ops = {
+	.start = __ipipe_frozen_prtrace_start,
+	.next  = __ipipe_prtrace_next,
+	.stop  = __ipipe_prtrace_stop,
+	.show  = __ipipe_prtrace_show
+};
+
+static int __ipipe_frozen_prtrace_open(struct inode *inode, struct file *file)
+{
+	return seq_open(file, &__ipipe_frozen_ptrace_ops);
+}
+
+static ssize_t
+__ipipe_frozen_ctrl(struct file *file, const char __user *pbuffer,
+                    size_t count, loff_t *data)
+{
+	char *end, buf[16];
+	int val;
+	int n;
+
+	n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+	if (copy_from_user(buf, pbuffer, n))
+		return -EFAULT;
+
+	buf[n] = '\0';
+	val = simple_strtol(buf, &end, 0);
+
+	if (((*end != '\0') && !isspace(*end)) || (val < 0))
+		return -EINVAL;
+
+	mutex_lock(&out_mutex);
+	ipipe_trace_frozen_reset();
+	if (val > 0)
+		ipipe_trace_freeze(-1);
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+struct file_operations __ipipe_frozen_prtrace_fops = {
+	.open       = __ipipe_frozen_prtrace_open,
+	.read       = seq_read,
+	.write      = __ipipe_frozen_ctrl,
+	.llseek     = seq_lseek,
+	.release    = seq_release,
+};
+
+static int __ipipe_rd_proc_val(char *page, char **start, off_t off,
+                               int count, int *eof, void *data)
+{
+	int len;
+
+	len = sprintf(page, "%u\n", *(int *)data);
+	len -= off;
+	if (len <= off + count)
+		*eof = 1;
+	*start = page + off;
+	if (len > count)
+		len = count;
+	if (len < 0)
+		len = 0;
+
+	return len;
+}
+
+static int __ipipe_wr_proc_val(struct file *file, const char __user *buffer,
+                               unsigned long count, void *data)
+{
+	char *end, buf[16];
+	int val;
+	int n;
+
+	n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+	if (copy_from_user(buf, buffer, n))
+		return -EFAULT;
+
+	buf[n] = '\0';
+	val = simple_strtol(buf, &end, 0);
+
+	if (((*end != '\0') && !isspace(*end)) || (val < 0))
+		return -EINVAL;
+
+	mutex_lock(&out_mutex);
+	*(int *)data = val;
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+static int __ipipe_rd_trigger(char *page, char **start, off_t off, int count,
+			      int *eof, void *data)
+{
+	int len;
+
+	if (!trigger_begin)
+		return 0;
+
+	len = sprint_symbol(page, trigger_begin);
+	page[len++] = '\n';
+
+	len -= off;
+	if (len <= off + count)
+		*eof = 1;
+	*start = page + off;
+	if (len > count)
+		len = count;
+	if (len < 0)
+		len = 0;
+
+	return len;
+}
+
+static int __ipipe_wr_trigger(struct file *file, const char __user *buffer,
+			      unsigned long count, void *data)
+{
+	char buf[KSYM_SYMBOL_LEN];
+	unsigned long begin, end;
+
+	if (count > sizeof(buf) - 1)
+		count = sizeof(buf) - 1;
+	if (copy_from_user(buf, buffer, count))
+		return -EFAULT;
+	buf[count] = 0;
+	if (buf[count-1] == '\n')
+		buf[count-1] = 0;
+
+	begin = kallsyms_lookup_name(buf);
+	if (!begin || !kallsyms_lookup_size_offset(begin, &end, NULL))
+		return -ENOENT;
+	end += begin - 1;
+
+	mutex_lock(&out_mutex);
+	/* invalidate the current range before setting a new one */
+	trigger_end = 0;
+	wmb();
+	ipipe_trace_frozen_reset();
+
+	/* set new range */
+	trigger_begin = begin;
+	wmb();
+	trigger_end = end;
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+#ifdef CONFIG_IPIPE_TRACE_MCOUNT
+static void notrace
+ipipe_trace_function(unsigned long ip, unsigned long parent_ip)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_FUNC, ip, parent_ip, 0);
+}
+
+static struct ftrace_ops ipipe_trace_ops = {
+	.func = ipipe_trace_function
+};
+
+static int __ipipe_wr_enable(struct file *file, const char __user *buffer,
+			     unsigned long count, void *data)
+{
+	char *end, buf[16];
+	int val;
+	int n;
+
+	n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+	if (copy_from_user(buf, buffer, n))
+		return -EFAULT;
+
+	buf[n] = '\0';
+	val = simple_strtol(buf, &end, 0);
+
+	if (((*end != '\0') && !isspace(*end)) || (val < 0))
+		return -EINVAL;
+
+	mutex_lock(&out_mutex);
+
+	if (ipipe_trace_enable) {
+		if (!val)
+			unregister_ftrace_function(&ipipe_trace_ops);
+	} else if (val)
+		register_ftrace_function(&ipipe_trace_ops);
+
+	ipipe_trace_enable = val;
+
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+#endif /* CONFIG_IPIPE_TRACE_MCOUNT */
+
+extern struct proc_dir_entry *ipipe_proc_root;
+
+static struct proc_dir_entry * __init
+__ipipe_create_trace_proc_val(struct proc_dir_entry *trace_dir,
+                              const char *name, int *value_ptr)
+{
+	struct proc_dir_entry *entry;
+
+	entry = create_proc_entry(name, 0644, trace_dir);
+	if (entry) {
+		entry->data = value_ptr;
+		entry->read_proc = __ipipe_rd_proc_val;
+		entry->write_proc = __ipipe_wr_proc_val;
+	}
+	return entry;
+}
+
+void __init __ipipe_init_tracer(void)
+{
+	struct proc_dir_entry *trace_dir;
+	struct proc_dir_entry *entry;
+	unsigned long long start, end, min = ULLONG_MAX;
+	int i;
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
+	int cpu, path;
+
+	for_each_possible_cpu(cpu) {
+		struct ipipe_trace_path *tp_buf;
+
+		tp_buf = vmalloc_node(sizeof(struct ipipe_trace_path) *
+				      IPIPE_TRACE_PATHS, cpu_to_node(cpu));
+		if (!tp_buf) {
+			printk(KERN_ERR "I-pipe: "
+			       "insufficient memory for trace buffer.\n");
+			return;
+		}
+		memset(tp_buf, 0,
+		       sizeof(struct ipipe_trace_path) * IPIPE_TRACE_PATHS);
+		for (path = 0; path < IPIPE_TRACE_PATHS; path++) {
+			tp_buf[path].begin = -1;
+			tp_buf[path].end   = -1;
+		}
+		per_cpu(trace_path, cpu) = tp_buf;
+	}
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
+
+	/* Calculate minimum overhead of __ipipe_trace() */
+	local_irq_disable_hw();
+	for (i = 0; i < 100; i++) {
+		ipipe_read_tsc(start);
+		__ipipe_trace(IPIPE_TRACE_FUNC, __BUILTIN_RETURN_ADDRESS0,
+			      __BUILTIN_RETURN_ADDRESS1, 0);
+		ipipe_read_tsc(end);
+
+		end -= start;
+		if (end < min)
+			min = end;
+	}
+	local_irq_enable_hw();
+	trace_overhead = ipipe_tsc2ns(min);
+
+#ifdef CONFIG_IPIPE_TRACE_ENABLE
+	ipipe_trace_enable = 1;
+#ifdef CONFIG_IPIPE_TRACE_MCOUNT
+	ftrace_enabled = 1;
+	register_ftrace_function(&ipipe_trace_ops);
+#endif /* CONFIG_IPIPE_TRACE_MCOUNT */
+#endif /* CONFIG_IPIPE_TRACE_ENABLE */
+
+	trace_dir = create_proc_entry("trace", S_IFDIR, ipipe_proc_root);
+
+	entry = create_proc_entry("max", 0644, trace_dir);
+	if (entry)
+		entry->proc_fops = &__ipipe_max_prtrace_fops;
+
+	entry = create_proc_entry("frozen", 0644, trace_dir);
+	if (entry)
+		entry->proc_fops = &__ipipe_frozen_prtrace_fops;
+
+	entry = create_proc_entry("trigger", 0644, trace_dir);
+	if (entry) {
+		entry->read_proc = __ipipe_rd_trigger;
+		entry->write_proc = __ipipe_wr_trigger;
+	}
+
+	__ipipe_create_trace_proc_val(trace_dir, "pre_trace_points",
+	                              &pre_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "post_trace_points",
+	                              &post_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "back_trace_points",
+	                              &back_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "verbose",
+	                              &verbose_trace);
+	entry = __ipipe_create_trace_proc_val(trace_dir, "enable",
+					      &ipipe_trace_enable);
+#ifdef CONFIG_IPIPE_TRACE_MCOUNT
+	if (entry)
+		entry->write_proc = __ipipe_wr_enable;
+#endif /* CONFIG_IPIPE_TRACE_MCOUNT */
+}
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index baa5c4a..9671529 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -15,6 +15,7 @@
 #include <linux/module.h>
 #include <linux/interrupt.h>
 #include <linux/kernel_stat.h>
+#include <linux/ipipe.h>
 
 #include "internals.h"
 
@@ -505,7 +506,9 @@ handle_level_irq(unsigned int irq, struct irq_desc *desc)
 	irqreturn_t action_ret;
 
 	raw_spin_lock(&desc->lock);
+#ifndef CONFIG_IPIPE
 	mask_ack_irq(desc);
+#endif
 
 	if (unlikely(desc->status & IRQ_INPROGRESS))
 		goto out_unlock;
@@ -582,8 +585,13 @@ handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc)
 
 	raw_spin_lock(&desc->lock);
 	desc->status &= ~IRQ_INPROGRESS;
+#ifdef CONFIG_IPIPE
+	desc->irq_data.chip->irq_unmask(&desc->irq_data);
+out:
+#else
 out:
 	desc->irq_data.chip->irq_eoi(&desc->irq_data);
+#endif
 
 	raw_spin_unlock(&desc->lock);
 }
@@ -625,7 +633,9 @@ handle_edge_irq(unsigned int irq, struct irq_desc *desc)
 	kstat_incr_irqs_this_cpu(irq, desc);
 
 	/* Start handling the irq */
+#ifndef CONFIG_IPIPE
 	desc->irq_data.chip->irq_ack(&desc->irq_data);
+#endif
 
 	/* Mark the IRQ currently in progress.*/
 	desc->status |= IRQ_INPROGRESS;
@@ -678,8 +688,10 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc)
 
 	kstat_incr_irqs_this_cpu(irq, desc);
 
+#ifndef CONFIG_IPIPE
 	if (desc->irq_data.chip->irq_ack)
 		desc->irq_data.chip->irq_ack(&desc->irq_data);
+#endif /* CONFIG_IPIPE */
 
 	action_ret = handle_IRQ_event(irq, desc->action);
 	if (!noirqdebug)
@@ -689,6 +701,135 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc)
 		desc->irq_data.chip->irq_eoi(&desc->irq_data);
 }
 
+#ifdef CONFIG_IPIPE
+
+void __ipipe_ack_simple_irq(unsigned irq, struct irq_desc *desc)
+{
+}
+
+void __ipipe_end_simple_irq(unsigned irq, struct irq_desc *desc)
+{
+}
+
+void __ipipe_ack_level_irq(unsigned irq, struct irq_desc *desc)
+{
+	mask_ack_irq(desc);
+}
+
+void __ipipe_end_level_irq(unsigned irq, struct irq_desc *desc)
+{
+	if (desc->irq_data.chip->irq_unmask)
+		desc->irq_data.chip->irq_unmask(&desc->irq_data);
+}
+
+void __ipipe_ack_fasteoi_irq(unsigned irq, struct irq_desc *desc)
+{
+	if (desc->irq_data.chip->irq_eoi)
+		desc->irq_data.chip->irq_eoi(&desc->irq_data);
+}
+
+void __ipipe_end_fasteoi_irq(unsigned irq, struct irq_desc *desc)
+{
+	/*
+	 * Non-requestable IRQs should not be masked in EOI handler.
+	 */
+	if (!(desc->status & IRQ_NOREQUEST))
+		desc->irq_data.chip->irq_unmask(&desc->irq_data);
+}
+
+void __ipipe_ack_edge_irq(unsigned irq, struct irq_desc *desc)
+{
+	desc->irq_data.chip->irq_ack(&desc->irq_data);
+}
+
+void __ipipe_ack_percpu_irq(unsigned irq, struct irq_desc *desc)
+{
+	if (desc->irq_data.chip->irq_ack)
+		desc->irq_data.chip->irq_ack(&desc->irq_data);
+}
+
+void __ipipe_end_percpu_irq(unsigned irq, struct irq_desc *desc)
+{
+	if (desc->irq_data.chip->irq_eoi)
+		desc->irq_data.chip->irq_eoi(&desc->irq_data);
+}
+
+void __ipipe_end_edge_irq(unsigned irq, struct irq_desc *desc)
+{
+}
+
+void __ipipe_ack_bad_irq(unsigned irq, struct irq_desc *desc)
+{
+	static int done;
+
+	handle_bad_irq(irq, desc);
+
+	if (!done) {
+		printk(KERN_WARNING "%s: unknown flow handler for IRQ %d\n",
+		       __FUNCTION__, irq);
+		done = 1;
+	}
+}
+
+void __ipipe_noack_irq(unsigned irq, struct irq_desc *desc)
+{
+}
+
+void __ipipe_noend_irq(unsigned irq, struct irq_desc *desc)
+{
+}
+
+irq_flow_handler_t
+__fixup_irq_handler(struct irq_desc *desc, irq_flow_handler_t handle, int is_chained)
+{
+	if (unlikely(handle == NULL)) {
+		desc->ipipe_ack = &__ipipe_ack_bad_irq;
+		desc->ipipe_end = &__ipipe_noend_irq;
+	} else {
+		if (is_chained) {
+			desc->ipipe_ack = handle;
+			desc->ipipe_end = &__ipipe_noend_irq;
+			handle = __ipipe_noack_irq;
+		} else if (handle == &handle_simple_irq) {
+			desc->ipipe_ack = &__ipipe_ack_simple_irq;
+			desc->ipipe_end = &__ipipe_end_simple_irq;
+		} else if (handle == &handle_level_irq) {
+			desc->ipipe_ack = &__ipipe_ack_level_irq;
+			desc->ipipe_end = &__ipipe_end_level_irq;
+		} else if (handle == &handle_edge_irq) {
+			desc->ipipe_ack = &__ipipe_ack_edge_irq;
+			desc->ipipe_end = &__ipipe_end_edge_irq;
+		} else if (handle == &handle_fasteoi_irq) {
+			desc->ipipe_ack = &__ipipe_ack_fasteoi_irq;
+			desc->ipipe_end = &__ipipe_end_fasteoi_irq;
+		} else if (handle == &handle_percpu_irq) {
+			desc->ipipe_ack = &__ipipe_ack_percpu_irq;
+			desc->ipipe_end = &__ipipe_end_percpu_irq;
+		} else if (get_irq_desc_chip(desc) == &no_irq_chip) {
+			desc->ipipe_ack = &__ipipe_noack_irq;
+			desc->ipipe_end = &__ipipe_noend_irq;
+		} else {
+			desc->ipipe_ack = &__ipipe_ack_bad_irq;
+			desc->ipipe_end = &__ipipe_noend_irq;
+		}
+	}
+
+	/* Suppress intermediate trampoline routine. */
+	ipipe_root_domain->irqs[desc->irq_data.irq].acknowledge = desc->ipipe_ack;
+
+	return handle;
+}
+
+#else /* !CONFIG_IPIPE */
+
+irq_flow_handler_t
+__fixup_irq_handler(struct irq_desc *desc, irq_flow_handler_t handle, int is_chained)
+{
+	return handle;
+}
+
+#endif /* !CONFIG_IPIPE */
+
 void
 __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
 		  const char *name)
@@ -720,6 +861,8 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
 	chip_bus_lock(desc);
 	raw_spin_lock_irqsave(&desc->lock, flags);
 
+	handle = __fixup_irq_handler(desc, handle, is_chained);
+
 	/* Uninstall? */
 	if (handle == handle_bad_irq) {
 		if (desc->irq_data.chip != &no_irq_chip)
diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c
index 2039bea..17bad02 100644
--- a/kernel/irq/irqdesc.c
+++ b/kernel/irq/irqdesc.c
@@ -269,10 +269,12 @@ int __init early_irq_init(void)
 	return arch_early_irq_init();
 }
 
+#ifndef CONFIG_IPIPE
 struct irq_desc *irq_to_desc(unsigned int irq)
 {
 	return (irq < NR_IRQS) ? irq_desc + irq : NULL;
 }
+#endif /* CONFIG_IPIPE */
 
 struct irq_desc *irq_to_desc_alloc_node(unsigned int irq, int node)
 {
diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index 0d2058d..1ac7e20 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -2318,7 +2318,7 @@ void trace_hardirqs_on_caller(unsigned long ip)
 	/* we'll do an OFF -> ON transition: */
 	curr->hardirqs_enabled = 1;
 
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !irqs_disabled_hw()))
 		return;
 	if (DEBUG_LOCKS_WARN_ON(current->hardirq_context))
 		return;
@@ -2361,7 +2361,7 @@ void trace_hardirqs_off_caller(unsigned long ip)
 	if (unlikely(!debug_locks || current->lockdep_recursion))
 		return;
 
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !irqs_disabled_hw()))
 		return;
 
 	if (curr->hardirqs_enabled) {
@@ -2393,7 +2393,7 @@ void trace_softirqs_on(unsigned long ip)
 	if (unlikely(!debug_locks))
 		return;
 
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !irqs_disabled_hw()))
 		return;
 
 	if (curr->softirqs_enabled) {
@@ -2427,7 +2427,7 @@ void trace_softirqs_off(unsigned long ip)
 	if (unlikely(!debug_locks))
 		return;
 
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !irqs_disabled_hw()))
 		return;
 
 	if (curr->softirqs_enabled) {
diff --git a/kernel/panic.c b/kernel/panic.c
index 991bb87..7145489 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -23,6 +23,7 @@
 #include <linux/init.h>
 #include <linux/nmi.h>
 #include <linux/dmi.h>
+#include <linux/ipipe_trace.h>
 
 #define PANIC_TIMER_STEP 100
 #define PANIC_BLINK_SPD 18
@@ -319,6 +320,8 @@ void oops_enter(void)
 {
 	tracing_off();
 	/* can't trust the integrity of the kernel anymore: */
+	ipipe_trace_panic_freeze();
+	ipipe_disable_context_check(ipipe_processor_id());
 	debug_locks_off();
 	do_oops_enter_exit();
 }
diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c
index 1832bd2..9f2a552 100644
--- a/kernel/power/hibernate.c
+++ b/kernel/power/hibernate.c
@@ -270,6 +270,7 @@ static int create_image(int platform_mode)
 		goto Enable_cpus;
 
 	local_irq_disable();
+	local_irq_disable_hw_cond();
 
 	error = sysdev_suspend(PMSG_FREEZE);
 	if (error) {
@@ -301,6 +302,7 @@ static int create_image(int platform_mode)
 	 */
 
  Enable_irqs:
+	local_irq_enable_hw_cond();
 	local_irq_enable();
 
  Enable_cpus:
@@ -401,6 +403,7 @@ static int resume_target_kernel(bool platform_mode)
 		goto Enable_cpus;
 
 	local_irq_disable();
+	local_irq_disable_hw_cond();
 
 	error = sysdev_suspend(PMSG_QUIESCE);
 	if (error)
@@ -432,6 +435,7 @@ static int resume_target_kernel(bool platform_mode)
 	sysdev_resume();
 
  Enable_irqs:
+	local_irq_enable_hw_cond();
 	local_irq_enable();
 
  Enable_cpus:
@@ -515,6 +519,7 @@ int hibernation_platform_enter(void)
 		goto Platform_finish;
 
 	local_irq_disable();
+	local_irq_disable_hw_cond();
 	sysdev_suspend(PMSG_HIBERNATE);
 	if (pm_wakeup_pending()) {
 		error = -EAGAIN;
diff --git a/kernel/printk.c b/kernel/printk.c
index 3623152..beb8497 100644
--- a/kernel/printk.c
+++ b/kernel/printk.c
@@ -614,6 +614,41 @@ static int have_callable_console(void)
 	return 0;
 }
 
+#ifdef CONFIG_IPIPE
+
+static IPIPE_DEFINE_SPINLOCK(__ipipe_printk_lock);
+
+static int __ipipe_printk_fill;
+
+static char __ipipe_printk_buf[__LOG_BUF_LEN];
+
+void __ipipe_flush_printk (unsigned virq, void *cookie)
+{
+	char *p = __ipipe_printk_buf;
+	int len, lmax, out = 0;
+	unsigned long flags;
+
+	goto start;
+
+	do {
+		spin_unlock_irqrestore(&__ipipe_printk_lock, flags);
+ start:
+		lmax = __ipipe_printk_fill;
+		while (out < lmax) {
+			len = strlen(p) + 1;
+			printk("%s",p);
+			p += len;
+			out += len;
+		}
+		spin_lock_irqsave(&__ipipe_printk_lock, flags);
+	}
+	while (__ipipe_printk_fill != lmax);
+
+	__ipipe_printk_fill = 0;
+
+	spin_unlock_irqrestore(&__ipipe_printk_lock, flags);
+}
+
 /**
  * printk - print a kernel message
  * @fmt: format string
@@ -638,6 +673,68 @@ static int have_callable_console(void)
 
 asmlinkage int printk(const char *fmt, ...)
 {
+	int r, fbytes, oldcount;
+	unsigned long flags;
+	int sprintk = 1;
+	int cs = -1;
+	va_list args;
+
+	va_start(args, fmt);
+
+	local_irq_save_hw(flags);
+
+	if (test_bit(IPIPE_SPRINTK_FLAG, &__ipipe_current_domain->flags) ||
+	    oops_in_progress)
+		cs = ipipe_disable_context_check(ipipe_processor_id());
+	else if (__ipipe_root_domain_p) {
+		struct ipipe_domain *dom;
+
+		list_for_each_entry(dom, &__ipipe_pipeline, p_link) {
+			if (dom == ipipe_root_domain)
+				break;
+			if (test_bit(IPIPE_STALL_FLAG,
+				     &ipipe_cpudom_var(dom, status))
+			    || raw_irqs_disabled_flags(flags)) {
+				sprintk = 0;
+				break;
+			}
+		}
+	} else
+		sprintk = 0;
+
+	local_irq_restore_hw(flags);
+
+	if (sprintk) {
+		r = vprintk(fmt, args);
+		if (cs != -1)
+			ipipe_restore_context_check(ipipe_processor_id(), cs);
+		goto out;
+	}
+
+	spin_lock_irqsave(&__ipipe_printk_lock, flags);
+
+	oldcount = __ipipe_printk_fill;
+	fbytes = __LOG_BUF_LEN - oldcount;
+
+	if (fbytes > 1)	{
+		r = vscnprintf(__ipipe_printk_buf + __ipipe_printk_fill,
+			       fbytes, fmt, args) + 1; /* account for the null byte */
+		__ipipe_printk_fill += r;
+	} else
+		r = 0;
+
+	spin_unlock_irqrestore(&__ipipe_printk_lock, flags);
+
+	if (oldcount == 0)
+		ipipe_trigger_irq(__ipipe_printk_virq);
+out:
+	va_end(args);
+
+	return r;
+}
+#else /* !CONFIG_IPIPE */
+asmlinkage int printk(const char *fmt, ...)
+{
 	va_list args;
 	int r;
 
@@ -655,6 +752,7 @@ asmlinkage int printk(const char *fmt, ...)
 
 	return r;
 }
+#endif /* CONFIG_IPIPE */
 
 /* cpu currently holding logbuf_lock */
 static volatile unsigned int printk_cpu = UINT_MAX;
@@ -841,7 +939,7 @@ asmlinkage int vprintk(const char *fmt, va_list args)
 	 * Try to acquire and then immediately release the
 	 * console semaphore. The release will do all the
 	 * actual magic (print out buffers, wake up klogd,
-	 * etc). 
+	 * etc).
 	 *
 	 * The console_trylock_for_printk() function
 	 * will release 'logbuf_lock' regardless of whether it
@@ -1412,7 +1510,7 @@ EXPORT_SYMBOL(register_console);
 
 int unregister_console(struct console *console)
 {
-        struct console *a, *b;
+	struct console *a, *b;
 	int res = 1;
 
 #ifdef CONFIG_A11Y_BRAILLE_CONSOLE
diff --git a/kernel/sched.c b/kernel/sched.c
index c164920..3651908 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -2423,7 +2423,8 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state,
 
 	smp_wmb();
 	rq = task_rq_lock(p, &flags);
-	if (!(p->state & state))
+	if (!(p->state & state) ||
+	    (p->state & (TASK_NOWAKEUP|TASK_ATOMICSWITCH)))
 		goto out;
 
 	if (p->se.on_rq)
@@ -2898,13 +2899,15 @@ asmlinkage void schedule_tail(struct task_struct *prev)
 #endif
 	if (current->set_child_tid)
 		put_user(task_pid_vnr(current), current->set_child_tid);
+
+ 	ipipe_init_notify(current);
 }
 
 /*
  * context_switch - switch to the new MM and the new
  * thread's register state.
  */
-static inline void
+static inline int
 context_switch(struct rq *rq, struct task_struct *prev,
 	       struct task_struct *next)
 {
@@ -2946,12 +2949,23 @@ context_switch(struct rq *rq, struct task_struct *prev,
 	switch_to(prev, next, prev);
 
 	barrier();
+
+#ifdef CONFIG_IPIPE_DELAYED_ATOMICSW
+	current->state &= ~TASK_ATOMICSWITCH;
+#else
+	prev->state &= ~TASK_ATOMICSWITCH;
+#endif
+	if (task_hijacked(prev))
+		return 1;
+
 	/*
 	 * this_rq must be evaluated again because prev may have moved
 	 * CPUs since it called schedule(), thus the 'rq' on its stack
 	 * frame will be invalid.
 	 */
 	finish_task_switch(this_rq(), prev);
+
+	return 0;
 }
 
 /*
@@ -3804,6 +3818,7 @@ notrace unsigned long get_parent_ip(unsigned long addr)
 
 void __kprobes add_preempt_count(int val)
 {
+ 	ipipe_check_context(ipipe_root_domain);
 #ifdef CONFIG_DEBUG_PREEMPT
 	/*
 	 * Underflow?
@@ -3826,6 +3841,7 @@ EXPORT_SYMBOL(add_preempt_count);
 
 void __kprobes sub_preempt_count(int val)
 {
+ 	ipipe_check_context(ipipe_root_domain);
 #ifdef CONFIG_DEBUG_PREEMPT
 	/*
 	 * Underflow?
@@ -3874,6 +3890,7 @@ static noinline void __schedule_bug(struct task_struct *prev)
  */
 static inline void schedule_debug(struct task_struct *prev)
 {
+	ipipe_check_context(ipipe_root_domain);
 	/*
 	 * Test if we are atomic. Since do_exit() needs to call into
 	 * schedule() atomically, we ignore that path for now.
@@ -3931,7 +3948,7 @@ pick_next_task(struct rq *rq)
 /*
  * schedule() is the main scheduler function.
  */
-asmlinkage void __sched schedule(void)
+asmlinkage int __sched schedule(void)
 {
 	struct task_struct *prev, *next;
 	unsigned long *switch_count;
@@ -3945,6 +3962,10 @@ need_resched:
 	rcu_note_context_switch(cpu);
 	prev = rq->curr;
 
+ 	if (unlikely(prev->state & TASK_ATOMICSWITCH))
+		/* Pop one disable level -- one still remains. */
+		preempt_enable();
+
 	release_kernel_lock(prev);
 need_resched_nonpreemptible:
 
@@ -3996,7 +4017,8 @@ need_resched_nonpreemptible:
 		rq->curr = next;
 		++*switch_count;
 
-		context_switch(rq, prev, next); /* unlocks the rq */
+ 		if (context_switch(rq, prev, next)) /* unlocks the rq */
+  			return 1; /* task hijacked by higher domain */
 		/*
 		 * The context switch have flipped the stack from under us
 		 * and restored the local variables which were saved when
@@ -4005,8 +4027,10 @@ need_resched_nonpreemptible:
 		 */
 		cpu = smp_processor_id();
 		rq = cpu_rq(cpu);
-	} else
+	} else {
+  		prev->state &= ~TASK_ATOMICSWITCH;
 		raw_spin_unlock_irq(&rq->lock);
+	}
 
 	post_schedule(rq);
 
@@ -4016,6 +4040,8 @@ need_resched_nonpreemptible:
 	preempt_enable_no_resched();
 	if (need_resched())
 		goto need_resched;
+
+	return 0;
 }
 EXPORT_SYMBOL(schedule);
 
@@ -4107,7 +4133,8 @@ asmlinkage void __sched notrace preempt_schedule(void)
 
 	do {
 		add_preempt_count_notrace(PREEMPT_ACTIVE);
-		schedule();
+		if (schedule())
+			return;
 		sub_preempt_count_notrace(PREEMPT_ACTIVE);
 
 		/*
@@ -4900,6 +4927,7 @@ recheck:
 	oldprio = p->prio;
 	prev_class = p->sched_class;
 	__setscheduler(rq, p, policy, param->sched_priority);
+  	ipipe_setsched_notify(p);
 
 	if (running)
 		p->sched_class->set_curr_task(rq);
@@ -5568,6 +5596,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
 #else
 	task_thread_info(idle)->preempt_count = 0;
 #endif
+	ipipe_check_context(ipipe_root_domain);
 	/*
 	 * The idle tasks have their own, simple scheduling class:
 	 */
@@ -9255,3 +9284,63 @@ struct cgroup_subsys cpuacct_subsys = {
 };
 #endif	/* CONFIG_CGROUP_CPUACCT */
 
+
+#ifdef CONFIG_IPIPE
+
+int ipipe_setscheduler_root(struct task_struct *p, int policy, int prio)
+{
+	const struct sched_class *prev_class;
+	int oldprio, on_rq, running;
+	unsigned long flags;
+	struct rq *rq;
+
+	raw_spin_lock_irqsave(&p->pi_lock, flags);
+	rq = __task_rq_lock(p);
+	on_rq = p->se.on_rq;
+	running = task_current(rq, p);
+	if (on_rq)
+		deactivate_task(rq, p, 0);
+	if (running)
+		p->sched_class->put_prev_task(rq, p);
+
+	p->sched_reset_on_fork = 0;
+
+	oldprio = p->prio;
+	prev_class = p->sched_class;
+	__setscheduler(rq, p, policy, prio);
+	ipipe_setsched_notify(p);
+
+	if (running)
+		p->sched_class->set_curr_task(rq);
+	if (on_rq) {
+		activate_task(rq, p, 0);
+		check_class_changed(rq, p, prev_class, oldprio, running);
+	}
+	__task_rq_unlock(rq);
+	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
+
+	rt_mutex_adjust_pi(p);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipipe_setscheduler_root);
+
+int ipipe_reenter_root(struct task_struct *prev, int policy, int prio)
+{
+	struct rq *rq = this_rq();
+
+	finish_task_switch(rq, prev);
+
+	post_schedule(rq);
+
+	reacquire_kernel_lock(current);
+	preempt_enable_no_resched();
+
+	if (current->policy != policy || current->rt_priority != prio)
+		return ipipe_setscheduler_root(current, policy, prio);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipipe_reenter_root);
+
+#endif /* CONFIG_IPIPE */
diff --git a/kernel/signal.c b/kernel/signal.c
index bf11d26..9e907e1 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -558,6 +558,7 @@ void signal_wake_up(struct task_struct *t, int resume)
 	unsigned int mask;
 
 	set_tsk_thread_flag(t, TIF_SIGPENDING);
+	ipipe_sigwake_notify(t); /* TIF_SIGPENDING must be set first. */
 
 	/*
 	 * For SIGKILL, we want to wake it up in the stopped/traced/killable
diff --git a/kernel/spinlock.c b/kernel/spinlock.c
index be6517f..862aed4 100644
--- a/kernel/spinlock.c
+++ b/kernel/spinlock.c
@@ -26,7 +26,9 @@
  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
  */
-#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
+#if !defined(CONFIG_GENERIC_LOCKBREAK) ||			\
+	defined(CONFIG_DEBUG_LOCK_ALLOC) ||			\
+	defined(CONFIG_IPIPE)
 /*
  * The __lock_function inlines are taken from
  * include/linux/spinlock_api_smp.h
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
index ed228ef..33cf47d 100644
--- a/kernel/time/tick-common.c
+++ b/kernel/time/tick-common.c
@@ -73,7 +73,7 @@ static void tick_periodic(int cpu)
 		write_sequnlock(&xtime_lock);
 	}
 
-	update_process_times(user_mode(get_irq_regs()));
+	update_root_process_times(get_irq_regs());
 	profile_tick(CPU_PROFILING);
 }
 
@@ -181,6 +181,10 @@ static void tick_setup_device(struct tick_device *td,
 
 	td->evtdev = newdev;
 
+	/* I-pipe: derive global tick IRQ from CPU 0 */
+	if (cpu == 0)
+		ipipe_update_tick_evtdev(newdev);
+
 	/*
 	 * When the device is not per cpu, pin the interrupt to the
 	 * current cpu:
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index c55ea24..bd9c203 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -598,7 +598,7 @@ static void tick_nohz_handler(struct clock_event_device *dev)
 		ts->idle_jiffies++;
 	}
 
-	update_process_times(user_mode(regs));
+	update_root_process_times(regs);
 	profile_tick(CPU_PROFILING);
 
 	while (tick_nohz_reprogram(ts, now)) {
@@ -757,7 +757,7 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
 			touch_softlockup_watchdog();
 			ts->idle_jiffies++;
 		}
-		update_process_times(user_mode(regs));
+		update_root_process_times(regs);
 		profile_tick(CPU_PROFILING);
 	}
 
diff --git a/kernel/timer.c b/kernel/timer.c
index d645992..240bb42 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -1273,6 +1273,25 @@ void update_process_times(int user_tick)
 	run_posix_cpu_timers(p);
 }
 
+#ifdef CONFIG_IPIPE
+
+void update_root_process_times(struct pt_regs *regs)
+{
+	int cpu, user_tick = user_mode(regs);
+
+	if (__ipipe_root_tick_p(regs)) {
+		update_process_times(user_tick);
+		return;
+	}
+
+	run_local_timers();
+	cpu = smp_processor_id();
+	rcu_check_callbacks(cpu, user_tick);
+	run_posix_cpu_timers(current);
+}
+
+#endif
+
 /*
  * This function runs timers and the timer-tq in bottom half context.
  */
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 14674dc..7d5e153 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -389,6 +389,7 @@ config DYNAMIC_FTRACE
 	bool "enable/disable ftrace tracepoints dynamically"
 	depends on FUNCTION_TRACER
 	depends on HAVE_DYNAMIC_FTRACE
+	depends on !IPIPE_TRACE_MCOUNT
 	default y
 	help
           This option will modify all the calls to ftrace dynamically
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 9803b68..bb3ab19 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -29,6 +29,7 @@
 #include <linux/list.h>
 #include <linux/hash.h>
 #include <linux/rcupdate.h>
+#include <linux/ipipe.h>
 
 #include <trace/events/sched.h>
 
@@ -1156,6 +1157,9 @@ static int __ftrace_modify_code(void *data)
 
 static void ftrace_run_update_code(int command)
 {
+#ifdef CONFIG_IPIPE
+	unsigned long flags;
+#endif /* CONFIG_IPIPE */
 	int ret;
 
 	ret = ftrace_arch_code_modify_prepare();
@@ -1163,7 +1167,13 @@ static void ftrace_run_update_code(int command)
 	if (ret)
 		return;
 
+#ifdef CONFIG_IPIPE
+	flags = ipipe_critical_enter(NULL);
+	__ftrace_modify_code(&command);
+	ipipe_critical_exit(flags);
+#else  /* !CONFIG_IPIPE */
 	stop_machine(__ftrace_modify_code, &command, NULL);
+#endif /* !CONFIG_IPIPE */
 
 	ret = ftrace_arch_code_modify_post_process();
 	FTRACE_WARN_ON(ret);
@@ -2724,9 +2734,9 @@ static int ftrace_process_locs(struct module *mod,
 	}
 
 	/* disable interrupts to prevent kstop machine */
-	local_irq_save(flags);
+	local_irq_save_hw_notrace(flags);
 	ftrace_update_code(mod);
-	local_irq_restore(flags);
+	local_irq_restore_hw_notrace(flags);
 	mutex_unlock(&ftrace_lock);
 
 	return 0;
@@ -2805,9 +2815,9 @@ void __init ftrace_init(void)
 	/* Keep the ftrace pointer to the stub */
 	addr = (unsigned long)ftrace_stub;
 
-	local_irq_save(flags);
+	local_irq_save_hw_notrace(flags);
 	ftrace_dyn_arch_init(&addr);
-	local_irq_restore(flags);
+	local_irq_restore_hw_notrace(flags);
 
 	/* ftrace_dyn_arch_init places the return code in addr */
 	if (addr)
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index d1b5951..0cf26e4 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -136,6 +136,8 @@ config DEBUG_SECTION_MISMATCH
 	  - Enable verbose reporting from modpost to help solving
 	    the section mismatches reported.
 
+source "kernel/ipipe/Kconfig.debug"
+
 config DEBUG_KERNEL
 	bool "Kernel debugging"
 	help
diff --git a/lib/bust_spinlocks.c b/lib/bust_spinlocks.c
index 9681d54..2dba50c 100644
--- a/lib/bust_spinlocks.c
+++ b/lib/bust_spinlocks.c
@@ -13,6 +13,7 @@
 #include <linux/wait.h>
 #include <linux/vt_kern.h>
 #include <linux/console.h>
+#include <linux/ipipe_trace.h>
 
 
 void __attribute__((weak)) bust_spinlocks(int yes)
@@ -24,6 +25,7 @@ void __attribute__((weak)) bust_spinlocks(int yes)
 		unblank_screen();
 #endif
 		console_unblank();
+  		ipipe_trace_panic_dump();
 		if (--oops_in_progress == 0)
 			wake_up_klogd();
 	}
diff --git a/lib/ioremap.c b/lib/ioremap.c
index da4e2ad..729c995 100644
--- a/lib/ioremap.c
+++ b/lib/ioremap.c
@@ -86,8 +86,8 @@ int ioremap_page_range(unsigned long addr,
 		if (err)
 			break;
 	} while (pgd++, addr = next, addr != end);
-
-	flush_cache_vmap(start, end);
+	__ipipe_pin_range_globally(start, end);
+ 	flush_cache_vmap(start, end);
 
 	return err;
 }
diff --git a/lib/smp_processor_id.c b/lib/smp_processor_id.c
index 4689cb0..3d12764 100644
--- a/lib/smp_processor_id.c
+++ b/lib/smp_processor_id.c
@@ -12,10 +12,13 @@ notrace unsigned int debug_smp_processor_id(void)
 	unsigned long preempt_count = preempt_count();
 	int this_cpu = raw_smp_processor_id();
 
+	if (!ipipe_root_domain_p)
+		goto out;
+
 	if (likely(preempt_count))
 		goto out;
 
-	if (irqs_disabled())
+	if (irqs_disabled() || irqs_disabled_hw())
 		goto out;
 
 	/*
diff --git a/mm/memory.c b/mm/memory.c
index c8fff70..949d965 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -647,6 +647,32 @@ out:
 	return pfn_to_page(pfn);
 }
 
+static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
+{
+	/*
+	 * If the source page was a PFN mapping, we don't have
+	 * a "struct page" for it. We do a best-effort copy by
+	 * just copying from the original user address. If that
+	 * fails, we just zero-fill it. Live with it.
+	 */
+	if (unlikely(!src)) {
+		void *kaddr = kmap_atomic(dst, KM_USER0);
+		void __user *uaddr = (void __user *)(va & PAGE_MASK);
+
+		/*
+		 * This really shouldn't fail, because the page is there
+		 * in the page tables. But it might just be unreadable,
+		 * in which case we just give up and fill the result with
+		 * zeroes.
+		 */
+		if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
+			clear_page(kaddr);
+		kunmap_atomic(kaddr, KM_USER0);
+		flush_dcache_page(dst);
+	} else
+		copy_user_highpage(dst, src, va, vma);
+}
+
 /*
  * copy one vm_area from one task to the other. Assumes the page tables
  * already present in the new task to be cleared in the whole range
@@ -655,8 +681,8 @@ out:
 
 static inline unsigned long
 copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
-		pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
-		unsigned long addr, int *rss)
+	     pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
+	     unsigned long addr, int *rss, struct page *uncow_page)
 {
 	unsigned long vm_flags = vma->vm_flags;
 	pte_t pte = *src_pte;
@@ -699,6 +725,21 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 	 * in the parent and the child
 	 */
 	if (is_cow_mapping(vm_flags)) {
+#ifdef CONFIG_IPIPE
+		if (uncow_page) {
+			struct page *old_page = vm_normal_page(vma, addr, pte);
+			cow_user_page(uncow_page, old_page, addr, vma);
+			pte = mk_pte(uncow_page, vma->vm_page_prot);
+
+			if (vm_flags & VM_SHARED)
+				pte = pte_mkclean(pte);
+			pte = pte_mkold(pte);
+
+			page_add_new_anon_rmap(uncow_page, vma, addr);
+			rss[!!PageAnon(uncow_page)]++;
+			goto out_set_pte;
+		}
+#endif /* CONFIG_IPIPE */
 		ptep_set_wrprotect(src_mm, addr, src_pte);
 		pte = pte_wrprotect(pte);
 	}
@@ -736,13 +777,27 @@ int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 	int progress = 0;
 	int rss[NR_MM_COUNTERS];
 	swp_entry_t entry = (swp_entry_t){0};
-
+	struct page *uncow_page = NULL;
+#ifdef CONFIG_IPIPE
+	int do_cow_break = 0;
+again:
+ 	if (do_cow_break) {
+ 		uncow_page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
+		if (uncow_page == NULL)
+ 			return -ENOMEM;
+		do_cow_break = 0;
+	}
+#else
 again:
+#endif
 	init_rss_vec(rss);
 
 	dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
-	if (!dst_pte)
+	if (!dst_pte) {
+		if (uncow_page)
+			page_cache_release(uncow_page);
 		return -ENOMEM;
+	}
 	src_pte = pte_offset_map(src_pmd, addr);
 	src_ptl = pte_lockptr(src_mm, src_pmd);
 	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
@@ -765,8 +820,25 @@ again:
 			progress++;
 			continue;
 		}
+#ifdef CONFIG_IPIPE
+		if (likely(uncow_page == NULL) && likely(pte_present(*src_pte))) {
+			if (is_cow_mapping(vma->vm_flags) &&
+			    test_bit(MMF_VM_PINNED, &src_mm->flags) &&
+			    ((vma->vm_flags|src_mm->def_flags) & VM_LOCKED)) {
+				arch_leave_lazy_mmu_mode();
+				spin_unlock(src_ptl);
+				pte_unmap(src_pte);
+				add_mm_rss_vec(dst_mm, rss);
+				pte_unmap_unlock(dst_pte, dst_ptl);
+				cond_resched();
+				do_cow_break = 1;
+				goto again;
+			}
+		}
+#endif
 		entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte,
-							vma, addr, rss);
+					 vma, addr, rss, uncow_page);
+		uncow_page = NULL;
 		if (entry.val)
 			break;
 		progress += 8;
@@ -2143,32 +2215,6 @@ static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd,
 	return same;
 }
 
-static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
-{
-	/*
-	 * If the source page was a PFN mapping, we don't have
-	 * a "struct page" for it. We do a best-effort copy by
-	 * just copying from the original user address. If that
-	 * fails, we just zero-fill it. Live with it.
-	 */
-	if (unlikely(!src)) {
-		void *kaddr = kmap_atomic(dst, KM_USER0);
-		void __user *uaddr = (void __user *)(va & PAGE_MASK);
-
-		/*
-		 * This really shouldn't fail, because the page is there
-		 * in the page tables. But it might just be unreadable,
-		 * in which case we just give up and fill the result with
-		 * zeroes.
-		 */
-		if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
-			clear_page(kaddr);
-		kunmap_atomic(kaddr, KM_USER0);
-		flush_dcache_page(dst);
-	} else
-		copy_user_highpage(dst, src, va, vma);
-}
-
 /*
  * This routine handles present pages, when users try to write
  * to a shared page. It is done by copying the page to a new address
@@ -3795,3 +3841,111 @@ void copy_user_huge_page(struct page *dst, struct page *src,
 	}
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
+
+#ifdef CONFIG_IPIPE
+
+static inline int ipipe_pin_pte_range(struct mm_struct *mm, pmd_t *pmd,
+				      struct vm_area_struct *vma,
+				      unsigned long addr, unsigned long end)
+{
+	spinlock_t *ptl;
+	pte_t *pte;
+
+	do {
+		pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
+		if (!pte)
+			continue;
+
+		if (!pte_present(*pte) || pte_write(*pte)) {
+			pte_unmap_unlock(pte, ptl);
+			continue;
+		}
+
+		if (do_wp_page(mm, vma, addr, pte, pmd, ptl, *pte) == VM_FAULT_OOM)
+			return -ENOMEM;
+	} while (addr += PAGE_SIZE, addr != end);
+	return 0;
+}
+
+static inline int ipipe_pin_pmd_range(struct mm_struct *mm, pud_t *pud,
+				      struct vm_area_struct *vma,
+				      unsigned long addr, unsigned long end)
+{
+	unsigned long next;
+	pmd_t *pmd;
+
+	pmd = pmd_offset(pud, addr);
+	do {
+		next = pmd_addr_end(addr, end);
+		if (pmd_none_or_clear_bad(pmd))
+			continue;
+		if (ipipe_pin_pte_range(mm, pmd, vma, addr, next))
+			return -ENOMEM;
+	} while (pmd++, addr = next, addr != end);
+	return 0;
+}
+
+static inline int ipipe_pin_pud_range(struct mm_struct *mm, pgd_t *pgd,
+				      struct vm_area_struct *vma,
+				      unsigned long addr, unsigned long end)
+{
+	unsigned long next;
+	pud_t *pud;
+
+	pud = pud_offset(pgd, addr);
+	do {
+		next = pud_addr_end(addr, end);
+		if (pud_none_or_clear_bad(pud))
+			continue;
+		if (ipipe_pin_pmd_range(mm, pud, vma, addr, next))
+			return -ENOMEM;
+	} while (pud++, addr = next, addr != end);
+	return 0;
+}
+
+int ipipe_disable_ondemand_mappings(struct task_struct *tsk)
+{
+	unsigned long addr, next, end;
+	struct vm_area_struct *vma;
+	struct mm_struct *mm;
+	int result = 0;
+	pgd_t *pgd;
+
+	mm = get_task_mm(tsk);
+	if (!mm)
+		return -EPERM;
+
+	down_write(&mm->mmap_sem);
+	if (test_bit(MMF_VM_PINNED, &mm->flags))
+		goto done_mm;
+
+	for (vma = mm->mmap; vma; vma = vma->vm_next) {
+		if (!is_cow_mapping(vma->vm_flags)
+		    || !(vma->vm_flags & VM_WRITE))
+			continue;
+
+		addr = vma->vm_start;
+		end = vma->vm_end;
+
+		pgd = pgd_offset(mm, addr);
+		do {
+			next = pgd_addr_end(addr, end);
+			if (pgd_none_or_clear_bad(pgd))
+				continue;
+			if (ipipe_pin_pud_range(mm, pgd, vma, addr, next)) {
+				result = -ENOMEM;
+				goto done_mm;
+			}
+		} while (pgd++, addr = next, addr != end);
+	}
+	set_bit(MMF_VM_PINNED, &mm->flags);
+
+  done_mm:
+	up_write(&mm->mmap_sem);
+	mmput(mm);
+	return result;
+}
+
+EXPORT_SYMBOL(ipipe_disable_ondemand_mappings);
+
+#endif
diff --git a/mm/mmu_context.c b/mm/mmu_context.c
index 9e82e93..a4bd34d 100644
--- a/mm/mmu_context.c
+++ b/mm/mmu_context.c
@@ -24,15 +24,18 @@ void use_mm(struct mm_struct *mm)
 {
 	struct mm_struct *active_mm;
 	struct task_struct *tsk = current;
+	unsigned long flags;
 
 	task_lock(tsk);
 	active_mm = tsk->active_mm;
+ 	ipipe_mm_switch_protect(flags);
 	if (active_mm != mm) {
 		atomic_inc(&mm->mm_count);
 		tsk->active_mm = mm;
 	}
 	tsk->mm = mm;
-	switch_mm(active_mm, mm, tsk);
+	__switch_mm(active_mm, mm, tsk);
+ 	ipipe_mm_switch_unprotect(flags);
 	task_unlock(tsk);
 
 	if (active_mm != mm)
diff --git a/mm/mprotect.c b/mm/mprotect.c
index 5a688a2..3f234b3 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -147,6 +147,7 @@ mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
 {
 	struct mm_struct *mm = vma->vm_mm;
 	unsigned long oldflags = vma->vm_flags;
+	unsigned long protflags;
 	long nrpages = (end - start) >> PAGE_SHIFT;
 	unsigned long charged = 0;
 	pgoff_t pgoff;
@@ -205,8 +206,17 @@ success:
 	 * held in write mode.
 	 */
 	vma->vm_flags = newflags;
+	protflags = newflags;
+#ifdef CONFIG_IPIPE
+	/*
+	 * Enforce non-COW vm_page_prot by faking VM_SHARED on locked regions.
+	 */
+	if (test_bit(MMF_VM_PINNED, &mm->flags) &&
+	    ((vma->vm_flags | mm->def_flags) & VM_LOCKED))
+		protflags |= VM_SHARED;
+#endif
 	vma->vm_page_prot = pgprot_modify(vma->vm_page_prot,
-					  vm_get_page_prot(newflags));
+					  vm_get_page_prot(protflags));
 
 	if (vma_wants_writenotify(vma)) {
 		vma->vm_page_prot = vm_get_page_prot(newflags & ~VM_SHARED);
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index f9b1667..c2a5ed2 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -171,6 +171,8 @@ static int vmap_page_range_noflush(unsigned long start, unsigned long end,
 			return err;
 	} while (pgd++, addr = next, addr != end);
 
+ 	__ipipe_pin_range_globally(start, end);
+ 
 	return nr;
 }
 
