diff --git a/Makefile b/Makefile
index e3c5eb6..6f37e4f 100644
--- a/Makefile
+++ b/Makefile
@@ -517,6 +517,10 @@ KBUILD_CFLAGS += $(call cc-option, -fno-stack-protector)
 
 include $(srctree)/arch/$(SRCARCH)/Makefile
 
+ifdef CONFIG_IPIPE_TRACE_MCOUNT
+KBUILD_CFLAGS	+= -pg
+endif
+
 ifdef CONFIG_FRAME_POINTER
 KBUILD_CFLAGS	+= -fno-omit-frame-pointer -fno-optimize-sibling-calls
 else
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index b786e68..4091576 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -215,6 +215,8 @@ config ARCH_VERSATILE
 config ARCH_AT91
 	bool "Atmel AT91"
 	select GENERIC_GPIO
+	select GENERIC_TIME if IPIPE
+	select GENERIC_CLOCKEVENTS if IPIPE
 	help
 	  This enables support for systems based on the Atmel AT91RM9200,
 	  AT91SAM9 and AT91CAP9 processors.
@@ -690,6 +692,8 @@ config LOCAL_TIMERS
 	  accounting to be spread across the timer interval, preventing a
 	  "thundering herd" at every timer tick.
 
+source "kernel/ipipe/Kconfig"
+
 config PREEMPT
 	bool "Preemptible Kernel (EXPERIMENTAL)"
 	depends on EXPERIMENTAL
diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S
index 3c2c8f2..51f68a0 100644
--- a/arch/arm/boot/compressed/head.S
+++ b/arch/arm/boot/compressed/head.S
@@ -940,6 +940,15 @@ memdump:	mov	r12, r0
 		mov	pc, r10
 #endif
 
+#ifdef CONFIG_IPIPE_TRACE_MCOUNT
+                .text
+                .align 0
+                .type mcount %function
+                .global mcount
+mcount:
+		mov pc, lr	@ just return
+#endif
+
 		.ltorg
 reloc_end:
 
diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile
index ad455ff..72cea9c 100644
--- a/arch/arm/kernel/Makefile
+++ b/arch/arm/kernel/Makefile
@@ -10,6 +10,7 @@ obj-y		:= compat.o entry-armv.o entry-common.o irq.o \
 		   process.o ptrace.o setup.o signal.o \
 		   sys_arm.o stacktrace.o time.o traps.o
 
+obj-$(CONFIG_ARM_FCSE)		+= fcse.o
 obj-$(CONFIG_ISA_DMA_API)	+= dma.o
 obj-$(CONFIG_ARCH_ACORN)	+= ecard.o 
 obj-$(CONFIG_FIQ)		+= fiq.o
@@ -23,6 +24,8 @@ obj-$(CONFIG_KPROBES)		+= kprobes.o kprobes-decode.o
 obj-$(CONFIG_ATAGS_PROC)	+= atags.o
 obj-$(CONFIG_OABI_COMPAT)	+= sys_oabi-compat.o
 obj-$(CONFIG_ARM_THUMBEE)	+= thumbee.o
+obj-$(CONFIG_IPIPE)		+= ipipe.o
+obj-$(CONFIG_IPIPE_TRACE_MCOUNT)	+= ipipe-mcount.o
 
 obj-$(CONFIG_CRUNCH)		+= crunch.o crunch-bits.o
 AFLAGS_crunch-bits.o		:= -Wa,-mcpu=ep9312
diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S
index 7dca225..77c5ea1 100644
--- a/arch/arm/kernel/entry-armv.S
+++ b/arch/arm/kernel/entry-armv.S
@@ -4,6 +4,7 @@
  *  Copyright (C) 1996,1997,1998 Russell King.
  *  ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk)
  *  nommu support by Hyok S. Choi (hyok.choi@samsung.com)
+ *  Copyright (C) 2005 Stelian Pop.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -27,14 +28,22 @@
  * Interrupt handling.  Preserves r7, r8, r9
  */
 	.macro	irq_handler
+#ifdef CONFIG_IPIPE
+	mov r0, #2
+#endif
 	get_irqnr_preamble r5, lr
-1:	get_irqnr_and_base r0, r6, r5, lr
+1:	get_irqnr_and_base r1, r6, r5, lr
+	movne	r0, r1
 	movne	r1, sp
 	@
 	@ routine called with r0 = irq number, r1 = struct pt_regs *
 	@
 	adrne	lr, 1b
+#ifdef CONFIG_IPIPE
+	bne	__ipipe_grab_irq
+#else
 	bne	asm_do_IRQ
+#endif
 
 #ifdef CONFIG_SMP
 	/*
@@ -43,18 +52,22 @@
 	 * this macro assumes that irqstat (r6) and base (r5) are
 	 * preserved from get_irqnr_and_base above
 	 */
-	test_for_ipi r0, r6, r5, lr
+	test_for_ipi r1, r6, r5, lr
 	movne	r0, sp
 	adrne	lr, 1b
 	bne	do_IPI
 
 #ifdef CONFIG_LOCAL_TIMERS
-	test_for_ltirq r0, r6, r5, lr
+	test_for_ltirq r1, r6, r5, lr
 	movne	r0, sp
 	adrne	lr, 1b
 	bne	do_local_timer
 #endif
 #endif
+#ifdef CONFIG_IPIPE
+	cmp	r0, #2
+	bleq	__ipipe_check_root_interruptible
+#endif	
 
 	.endm
 
@@ -202,20 +215,34 @@ __irq_svc:
 #endif
 #ifdef CONFIG_PREEMPT
 	get_thread_info tsk
+#ifndef CONFIG_IPIPE
 	ldr	r8, [tsk, #TI_PREEMPT]		@ get preempt count
 	add	r7, r8, #1			@ increment it
 	str	r7, [tsk, #TI_PREEMPT]
 #endif
+#endif
 
 	irq_handler
+#ifdef CONFIG_IPIPE
+	cmp	r0, #0
+	beq	__ipipe_fast_svc_irq_exit
+#endif
+
 #ifdef CONFIG_PREEMPT
+#ifndef	CONFIG_IPIPE
 	str	r8, [tsk, #TI_PREEMPT]		@ restore preempt count
+#else
+	ldr	r8, [tsk, #TI_PREEMPT]
+#endif
 	ldr	r0, [tsk, #TI_FLAGS]		@ get flags
 	teq	r8, #0				@ if preempt count != 0
 	movne	r0, #0				@ force flags to 0
 	tst	r0, #_TIF_NEED_RESCHED
 	blne	svc_preempt
 #endif
+#ifdef CONFIG_IPIPE
+__ipipe_fast_svc_irq_exit:
+#endif
 	ldr	r0, [sp, #S_PSR]		@ irqs are already disabled
 	msr	spsr_cxsf, r0
 #ifdef CONFIG_TRACE_IRQFLAGS
@@ -229,12 +256,20 @@ __irq_svc:
 #ifdef CONFIG_PREEMPT
 svc_preempt:
 	mov	r8, lr
+	ipipe_fast_stall_root r4, r0		@ r4 must be preserved
+	enable_irq_cond				@ reused by unstall
 1:	bl	preempt_schedule_irq		@ irq en/disable is done inside
 	ldr	r0, [tsk, #TI_FLAGS]		@ get new tasks TI_FLAGS
 	tst	r0, #_TIF_NEED_RESCHED
-	moveq	pc, r8				@ go again
-	b	1b
-#endif
+	bne	1b
+	disable_irq_cond
+	ipipe_fast_unstall_root r4, r0
+	mov	pc, r8				@ go again
+#ifdef CONFIG_IPIPE
+.LCipipe_root_status:
+	.word	__ipipe_root_status
+#endif /* CONFIG_IPIPE */
+#endif /* CONFIG_PREEMPT */
 
 	.align	5
 __und_svc:
@@ -247,6 +282,17 @@ __und_svc:
 	svc_entry
 #endif
 
+#ifdef CONFIG_IPIPE
+	/* Beware, do_undefinstr in arch/arm/traps.c will not signal the trap
+	when in svc mode because of this hunk. */
+	mov	r4, r2
+	mov	r0, #7				@ r0 = IPIPE_TRAP_UNDEFINSTR
+	mov	r1, sp				@ r1 = &regs
+	bl	__ipipe_dispatch_event		@ branch to trap handler
+	cmp	r0, #0
+	bne	1f
+	mov	r2, r4
+#endif /* CONFIG_IPIPE */
 	@
 	@ call emulation code, which returns using r9 if it has emulated
 	@ the instruction, or the more conventional lr if we are to treat
@@ -423,18 +469,28 @@ __irq_usr:
 #endif
 	get_thread_info tsk
 #ifdef CONFIG_PREEMPT
+#ifndef CONFIG_IPIPE
 	ldr	r8, [tsk, #TI_PREEMPT]		@ get preempt count
 	add	r7, r8, #1			@ increment it
 	str	r7, [tsk, #TI_PREEMPT]
 #endif
+#endif
 
 	irq_handler
+#ifdef CONFIG_IPIPE
+	cmp	r0, #0
+	bne	__ipipe_usr_irq_continue
+	slow_restore_user_regs			@ Fast exit path over non-root domains
+__ipipe_usr_irq_continue:
+#endif
 #ifdef CONFIG_PREEMPT
+#ifndef CONFIG_IPIPE
 	ldr	r0, [tsk, #TI_PREEMPT]
 	str	r8, [tsk, #TI_PREEMPT]
 	teq	r0, r7
 	strne	r0, [r0, -r0]
 #endif
+#endif
 #ifdef CONFIG_TRACE_IRQFLAGS
 	bl	trace_hardirqs_on
 #endif
@@ -539,7 +595,7 @@ call_fpe:
 	mov	r7, #1
 	strb	r7, [r10, #TI_USED_CP + 10]	@ mark CP#10 as used
 	strb	r7, [r10, #TI_USED_CP + 11]	@ mark CP#11 as used
-	b	do_vfp				@ let VFP handler handle this
+	b	_do_vfp				@ let VFP handler handle this
 1:
 #endif
 	tst	r0, #0x08000000			@ only CDP/CPRT/LDC/STC have bit 27
@@ -581,8 +637,8 @@ call_fpe:
 	mov	pc, lr				@ CP#8
 	mov	pc, lr				@ CP#9
 #ifdef CONFIG_VFP
-	b	do_vfp				@ CP#10 (VFP)
-	b	do_vfp				@ CP#11 (VFP)
+	b	_do_vfp				@ CP#10 (VFP)
+	b	_do_vfp				@ CP#11 (VFP)
 #else
 	mov	pc, lr				@ CP#10 (VFP)
 	mov	pc, lr				@ CP#11 (VFP)
@@ -617,11 +673,43 @@ call_fpe:
 #endif
 
 do_fpe:
+#ifdef CONFIG_IPIPE
+	mov	r4, r0
+	mov	r5, r2
+	mov	r6, lr
+	mov	r0, #5				@ r0 = IPIPE_TRAP_FPU
+	mov	r1, sp				@ r1 = &regs
+	bl	__ipipe_dispatch_event		@ branch to trap handler
+	cmp	r0, #0
+	movne	pc, r9
+	mov	lr, r6
+	mov	r2, r5
+	mov	r0, r4
+#endif
 	enable_irq
 	ldr	r4, .LCfp
 	add	r10, r10, #TI_FPSTATE		@ r10 = workspace
 	ldr	pc, [r4]			@ Call FP module USR entry point
 
+#ifdef CONFIG_VFP
+_do_vfp:
+#ifdef CONFIG_IPIPE
+	mov	r4, r0
+	mov	r5, r2
+	mov	r6, lr
+	mov	r0, #6				@ r0 = IPIPE_TRAP_VFP
+	mov	r1, sp				@ r1 = &regs
+	bl	__ipipe_dispatch_event		@ branch to trap handler
+	cmp	r0, #0
+	movne	pc, r9
+	mov	lr, r6
+	mov	r2, r5
+	mov	r0, r4
+#endif
+	b	do_vfp
+#endif
+
+
 /*
  * The FP module is called with these registers set:
  *  r0  = instruction
@@ -663,6 +751,13 @@ __pabt_usr:
  * This is the return code to user mode for abort handlers
  */
 ENTRY(ret_from_exception)
+#ifdef CONFIG_IPIPE
+	bl     __ipipe_check_root
+	cmp     r0, #0
+	bne     1f
+	slow_restore_user_regs          @ Fast exit path over non-root domains
+1:
+#endif /* CONFIG_IPIPE */
 	get_thread_info tsk
 	mov	why, #0
 	b	ret_to_user
@@ -735,6 +830,18 @@ ENTRY(__switch_to)
  * purpose.
  */
 
+#ifdef CONFIG_IPIPE
+/* We use the same mechanism as Linux user helpers to store variables related to
+   TSC emulation, so that they can be used in user-space. */
+	.align 5
+	.globl  __ipipe_tsc_area_start
+
+__ipipe_tsc_area_start:
+	.rep	12
+	.word	0
+	.endr
+#endif /* CONFIG_IPIPE */
+
 	.macro	usr_ret, reg
 #ifdef CONFIG_ARM_THUMB
 	bx	\reg
diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S
index 597ed00..1e1f086 100644
--- a/arch/arm/kernel/entry-common.S
+++ b/arch/arm/kernel/entry-common.S
@@ -2,6 +2,7 @@
  *  linux/arch/arm/kernel/entry-common.S
  *
  *  Copyright (C) 2000 Russell King
+ *  Copyright (C) 2005 Stelian Pop.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -20,16 +21,21 @@
  * possible here, and this includes saving r0 back into the SVC
  * stack.
  */
+#ifdef CONFIG_IPIPE
+__ipipe_ret_fast_syscall:
+	ldr	r0, [sp, #S_R0+S_OFF]		@ returned r0
+	/* fall through */
+#endif
 ret_fast_syscall:
 	disable_irq				@ disable interrupts
 	ldr	r1, [tsk, #TI_FLAGS]
 	tst	r1, #_TIF_WORK_MASK
 	bne	fast_work_pending
 
+fast_restore_user_regs:
 	/* perform architecture specific actions before user return */
 	arch_ret_to_user r1, lr
 
-	@ fast_restore_user_regs
 	ldr	r1, [sp, #S_OFF + S_PSR]	@ get calling cpsr
 	ldr	lr, [sp, #S_OFF + S_PC]!	@ get pc
 	msr	spsr_cxsf, r1			@ save in spsr_svc
@@ -38,6 +44,13 @@ ret_fast_syscall:
 	add	sp, sp, #S_FRAME_SIZE - S_PC
 	movs	pc, lr				@ return & move spsr_svc into cpsr
 
+#ifdef CONFIG_IPIPE
+__ipipe_fast_exit_syscall:
+	ldr	r0, [sp, #S_R0+S_OFF]		@ returned r0
+	disable_irq				@ disable interrupts
+	b	fast_restore_user_regs
+#endif /* CONFIG_IPIPE */
+
 /*
  * Ok, we need to do extra processing, enter the slow path.
  */
@@ -48,13 +61,16 @@ work_pending:
 	bne	work_resched
 	tst	r1, #_TIF_SIGPENDING
 	beq	no_work_pending
+	enable_irq_cond
 	mov	r0, sp				@ 'regs'
 	mov	r2, why				@ 'syscall'
 	bl	do_notify_resume
 	b	ret_slow_syscall		@ Check work again
 
 work_resched:
+	enable_irq_cond
 	bl	schedule
+
 /*
  * "slow" syscall return path.  "why" tells us if this was a real syscall.
  */
@@ -65,22 +81,13 @@ ret_slow_syscall:
 	tst	r1, #_TIF_WORK_MASK
 	bne	work_pending
 no_work_pending:
-	/* perform architecture specific actions before user return */
-	arch_ret_to_user r1, lr
-
-	@ slow_restore_user_regs
-	ldr	r1, [sp, #S_PSR]		@ get calling cpsr
-	ldr	lr, [sp, #S_PC]!		@ get pc
-	msr	spsr_cxsf, r1			@ save in spsr_svc
-	ldmdb	sp, {r0 - lr}^			@ get calling r0 - lr
-	mov	r0, r0
-	add	sp, sp, #S_FRAME_SIZE - S_PC
-	movs	pc, lr				@ return & move spsr_svc into cpsr
+	slow_restore_user_regs
 
 /*
  * This is how we return from a fork.
  */
 ENTRY(ret_from_fork)
+	enable_irq_cond
 	bl	schedule_tail
 	get_thread_info tsk
 	ldr	r1, [tsk, #TI_FLAGS]		@ check for syscall tracing
@@ -205,6 +212,18 @@ ENTRY(vector_swi)
 #endif
 
 	stmdb	sp!, {r4, r5}			@ push fifth and sixth args
+#ifdef CONFIG_IPIPE
+	stmfd	sp!, {r0-r3, ip}
+	sub	sp, sp, #4
+	add	r1, sp, #S_OFF+24
+	mov	r0, scno
+	bl	__ipipe_syscall_root
+	cmp	r0, #0
+	add	sp, sp, #4
+	ldmfd	sp!, {r0-r3, ip}
+	blt	__ipipe_ret_fast_syscall
+	bgt	__ipipe_fast_exit_syscall
+#endif /* CONFIG_IPIPE */
 	tst	ip, #_TIF_SYSCALL_TRACE		@ are we tracing syscalls?
 	bne	__sys_trace
 
@@ -251,6 +270,9 @@ __sys_trace_return:
 __cr_alignment:
 	.word	cr_alignment
 #endif
+#ifdef CONFIG_IPIPE
+	.word	__ipipe_syscall_root
+#endif
 	.ltorg
 
 /*
@@ -402,3 +424,28 @@ ENTRY(sys_oabi_call_table)
 
 #endif
 
+
+#ifdef CONFIG_FRAME_POINTER
+
+	.text
+	.align 0
+	.type arm_return_addr %function
+	.global arm_return_addr
+
+arm_return_addr:
+	mov	ip, r0
+	mov	r0, fp
+3:
+	cmp	r0, #0
+	beq	1f		@ frame list hit end, bail
+	cmp	ip, #0
+	beq	2f		@ reached desired frame
+	ldr	r0, [r0, #-12]  @ else continue, get next fp
+	sub	ip, ip, #1
+	b	3b
+2:
+	ldr	r0, [r0, #-4]   @ get target return address
+1:
+	mov	pc, lr
+
+#endif
diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S
index 87ab4e1..c4892b5 100644
--- a/arch/arm/kernel/entry-header.S
+++ b/arch/arm/kernel/entry-header.S
@@ -49,6 +49,34 @@
 #endif
 	.endm
 
+	.macro slow_restore_user_regs
+	/* perform architecture specific actions before user return */
+	arch_ret_to_user r1, lr
+	ldr	r1, [sp, #S_PSR]		@ get calling cpsr
+	ldr	lr, [sp, #S_PC]!		@ get pc
+	msr	spsr_cxsf, r1			@ save in spsr_svc
+	ldmdb	sp, {r0 - lr}^			@ get calling r0 - lr
+	mov	r0, r0
+	add	sp, sp, #S_FRAME_SIZE - S_PC
+	movs	pc, lr				@ return & move spsr_svc into cpsr
+	.endm
+
+	.macro ipipe_fast_stall_root, raddr, rtemp1
+#ifdef CONFIG_IPIPE
+	ldr	\raddr, .LCipipe_root_status
+	ldr	\rtemp1, [\raddr]
+	orr	\rtemp1, \rtemp1, #1
+	str	\rtemp1, [\raddr]
+#endif /* CONFIG_IPIPE */
+	.endm
+
+	.macro ipipe_fast_unstall_root, raddr, rtemp1
+#ifdef CONFIG_IPIPE
+	ldr	\rtemp1, [\raddr]
+	bic	\rtemp1, \rtemp1, #1
+	str	\rtemp1, [\raddr]
+#endif /* CONFIG_IPIPE */
+	.endm
 
 /*
  * These are the registers used in the syscall handler, and allow us to
diff --git a/arch/arm/kernel/fcse.c b/arch/arm/kernel/fcse.c
new file mode 100644
index 0000000..b314aae
--- /dev/null
+++ b/arch/arm/kernel/fcse.c
@@ -0,0 +1,37 @@
+#include <linux/bitops.h>
+#include <linux/memory.h>
+#include <linux/spinlock.h>
+#include <linux/bitops.h>
+#include <asm/fcse.h>
+
+#define MAX_PID (MODULE_START / FCSE_PID_TASK_SIZE)
+#define PIDS_LONGS ((MAX_PID + 8 * sizeof(long) - 1) / (8 * sizeof(long)))
+
+static DEFINE_SPINLOCK(fcse_lock);
+static unsigned long fcse_pids_bits[PIDS_LONGS];
+
+int fcse_pid_alloc(void)
+{
+	unsigned long flags;
+	unsigned bit;
+
+	spin_lock_irqsave(&fcse_lock, flags);
+	bit = find_first_zero_bit(fcse_pids_bits, MAX_PID);
+	if (bit == MAX_PID) {
+		spin_unlock(&fcse_lock);
+		return -1;
+	}
+	set_bit(bit, fcse_pids_bits);
+	spin_unlock_irqrestore(&fcse_lock, flags);
+
+	return bit;
+}
+
+void fcse_pid_free(unsigned pid)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&fcse_lock, flags);
+	pid = test_and_clear_bit(pid, fcse_pids_bits);
+	spin_unlock_irqrestore(&fcse_lock, flags);
+}
diff --git a/arch/arm/kernel/ipipe-mcount.S b/arch/arm/kernel/ipipe-mcount.S
new file mode 100644
index 0000000..ce61e0c
--- /dev/null
+++ b/arch/arm/kernel/ipipe-mcount.S
@@ -0,0 +1,38 @@
+/*
+ *  linux/arch/arm/kernel/ipipe-mcount.S
+ *
+ *  Copyright (C) 2006 Sebastian Smolorz <ssmolorz@emlix.com>, emlix GmbH
+ */
+
+#ifdef CONFIG_FRAME_POINTER
+
+	.text
+	.align 0
+	.type mcount %function
+	.global mcount
+
+mcount:
+
+	ldr	ip, =ipipe_trace_enable	@ leave early, if disabled
+	ldr	ip, [ip]
+	cmp	ip, #0
+	moveq	pc,lr
+
+	mov	ip,  sp
+	stmdb   sp!, {r0 - r3, fp, ip, lr, pc}	@ create stack frame
+
+	mov	r3, #0			@ no additional value (v)
+	ldr	r2, [fp, #-4]		@ get lr (the return address
+					@ of the caller of the
+					@ instrumented function)
+	mov	r1, lr			@ get lr - (the return address
+					@ of the instrumented function)
+	mov	r0, #0			@ IPIPE_TRACE_FN
+
+	sub	fp, ip, #4		@ point fp at this frame
+
+	bl	__ipipe_trace
+
+	ldmdb   fp, {r0 - r3, fp, sp, pc}	@ pop entry frame and return
+
+#endif
diff --git a/arch/arm/kernel/ipipe.c b/arch/arm/kernel/ipipe.c
new file mode 100644
index 0000000..29adc96
--- /dev/null
+++ b/arch/arm/kernel/ipipe.c
@@ -0,0 +1,442 @@
+/* -*- linux-c -*-
+ * linux/arch/arm/kernel/ipipe.c
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ * Copyright (C) 2004 Wolfgang Grandegger (Adeos/arm port over 2.4).
+ * Copyright (C) 2005 Heikki Lindholm (PowerPC 970 fixes).
+ * Copyright (C) 2005 Stelian Pop.
+ * Copyright (C) 2006-2008 Gilles Chanteperdrix.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-dependent I-PIPE support for ARM.
+ */
+
+#include <linux/kernel.h>
+#include <linux/smp.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/bitops.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <asm/system.h>
+#include <asm/atomic.h>
+#include <asm/hardirq.h>
+#include <asm/io.h>
+#include <asm/irq.h>
+#include <asm/unistd.h>
+#include <asm/mach/irq.h>
+#include <asm/mmu_context.h>
+
+/* Next tick date (timebase value). */
+DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs);
+DEFINE_PER_CPU(struct mm_struct *,ipipe_active_mm);
+EXPORT_PER_CPU_SYMBOL(ipipe_active_mm);
+
+extern struct irq_desc irq_desc[];
+asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs);
+
+#ifdef CONFIG_SMP
+
+static cpumask_t __ipipe_cpu_sync_map;
+
+static cpumask_t __ipipe_cpu_lock_map;
+
+static IPIPE_DEFINE_SPINLOCK(__ipipe_cpu_barrier);
+
+static atomic_t __ipipe_critical_count = ATOMIC_INIT(0);
+
+static void (*__ipipe_cpu_sync) (void);
+
+/* Always called with hw interrupts off. */
+
+void __ipipe_do_critical_sync(unsigned irq)
+{
+	int cpu = ipipe_processor_id();
+
+	cpu_set(cpu, __ipipe_cpu_sync_map);
+
+	/*
+	 * Now we are in sync with the lock requestor running on another
+	 * CPU. Enter a spinning wait until he releases the global
+	 * lock.
+	 */
+	spin_lock(&__ipipe_cpu_barrier);
+
+	/* Got it. Now get out. */
+
+	if (__ipipe_cpu_sync)
+		/* Call the sync routine if any. */
+		__ipipe_cpu_sync();
+
+	spin_unlock(&__ipipe_cpu_barrier);
+
+	cpu_clear(cpu, __ipipe_cpu_sync_map);
+}
+
+#endif	/* CONFIG_SMP */
+
+/*
+ * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline
+ * just like if it has been actually received from a hw source. Also
+ * works for virtual interrupts.
+ */
+int ipipe_trigger_irq(unsigned irq)
+{
+	unsigned long flags;
+
+	if (irq >= IPIPE_NR_IRQS ||
+	    (ipipe_virtual_irq_p(irq)
+	     && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)))
+		return -EINVAL;
+
+	local_irq_save_hw(flags);
+
+	__ipipe_handle_irq(irq, NULL);
+
+	local_irq_restore_hw(flags);
+
+	return 1;
+}
+
+int ipipe_get_sysinfo(struct ipipe_sysinfo *info)
+{
+	info->ncpus = num_online_cpus();
+	info->cpufreq = ipipe_cpu_freq();
+	info->archdep.tmirq = __ipipe_mach_timerint;
+	info->archdep.tmfreq = info->cpufreq;
+        __ipipe_mach_get_tscinfo(&info->archdep.tsc);
+
+	return 0;
+}
+
+static int __ipipe_ack_irq(unsigned irq)
+{
+	irq_desc_t *desc = irq_desc + irq;
+	desc->ipipe_ack(irq, desc);
+#ifdef irq_finish
+ 	/* AT91 specific workaround */
+        irq_finish(irq);
+#endif /* irq_finish */
+	return 1;
+}
+
+static int __ipipe_ack_timerirq(unsigned irq)
+{
+	irq_desc_t *desc = irq_desc + irq;
+	desc->ipipe_ack(irq, desc);
+#ifdef irq_finish
+ 	/* AT91 specific workaround */
+	irq_finish(irq);
+#endif /* irq_finish */
+	__ipipe_mach_acktimer();
+	desc->ipipe_end(irq, desc);
+	return 1;
+}
+
+void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
+{
+	irq_desc[irq].status &= ~IRQ_DISABLED;
+}
+
+/*
+ * __ipipe_enable_pipeline() -- We are running on the boot CPU, hw
+ * interrupts are off, and secondary CPUs are still lost in space.
+ */
+void __ipipe_enable_pipeline(void)
+{
+	unsigned long flags;
+	unsigned irq;
+
+	flags = ipipe_critical_enter(NULL);
+
+	/* First, virtualize all interrupts from the root domain. */
+
+	for (irq = 0; irq < NR_IRQS; irq++)
+		ipipe_virtualize_irq(ipipe_root_domain,
+				     irq,
+				     (ipipe_irq_handler_t)&asm_do_IRQ, NULL,
+				     ((irq == __ipipe_mach_timerint)
+				      ? &__ipipe_ack_timerirq
+				      : &__ipipe_ack_irq),
+				     IPIPE_HANDLE_MASK | IPIPE_PASS_MASK);
+
+	ipipe_critical_exit(flags);
+}
+
+/*
+ * ipipe_critical_enter() -- Grab the superlock excluding all CPUs
+ * but the current one from a critical section. This lock is used when
+ * we must enforce a global critical section for a single CPU in a
+ * possibly SMP system whichever context the CPUs are running.
+ */
+unsigned long ipipe_critical_enter(void (*syncfn) (void))
+{
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
+
+#ifdef CONFIG_SMP
+	if (num_online_cpus() > 1) {	/* We might be running a SMP-kernel on a UP box... */
+		int cpu = ipipe_processor_id();
+		cpumask_t lock_map;
+
+		if (!cpu_test_and_set(cpu, __ipipe_cpu_lock_map)) {
+			while (cpu_test_and_set(BITS_PER_LONG - 1,
+						__ipipe_cpu_lock_map)) {
+				int n = 0;
+				do {
+					cpu_relax();
+				} while (++n < cpu);
+			}
+
+			spin_lock(&__ipipe_cpu_barrier);
+
+			__ipipe_cpu_sync = syncfn;
+
+			/* Send the sync IPI to all processors but the current one. */
+			send_IPI_allbutself(IPIPE_CRITICAL_VECTOR);
+
+			cpus_andnot(lock_map, cpu_online_map,
+				    __ipipe_cpu_lock_map);
+
+			while (!cpus_equal(__ipipe_cpu_sync_map, lock_map))
+				cpu_relax();
+		}
+
+		atomic_inc(&__ipipe_critical_count);
+	}
+#endif	/* CONFIG_SMP */
+
+	return flags;
+}
+
+/* ipipe_critical_exit() -- Release the superlock. */
+
+void ipipe_critical_exit(unsigned long flags)
+{
+#ifdef CONFIG_SMP
+	if (num_online_cpus() > 1) {	/* We might be running a SMP-kernel on a UP box... */
+		if (atomic_dec_and_test(&__ipipe_critical_count)) {
+			spin_unlock(&__ipipe_cpu_barrier);
+
+			while (!cpus_empty(__ipipe_cpu_sync_map))
+				cpu_relax();
+
+			cpu_clear(ipipe_processor_id(), __ipipe_cpu_lock_map);
+			cpu_clear(BITS_PER_LONG - 1, __ipipe_cpu_lock_map);
+		}
+	}
+#endif	/* CONFIG_SMP */
+
+	local_irq_restore_hw(flags);
+}
+
+asmlinkage int __ipipe_check_root(void)
+{
+	return ipipe_root_domain_p;
+}
+
+asmlinkage int __ipipe_check_root_interruptible(void)
+{
+        return ipipe_root_domain_p && !__ipipe_test_root();
+}
+
+asmlinkage int __ipipe_syscall_root(unsigned long scno, struct pt_regs *regs)
+{
+	unsigned long flags, origr7;
+
+	/* We use r7 to pass the syscall number to the other domains */
+	origr7 = regs->ARM_r7;
+	regs->ARM_r7 = __NR_SYSCALL_BASE + scno;
+	/*
+	 * This routine either returns:
+	 * 0 -- if the syscall is to be passed to Linux;
+	 * >0 -- if the syscall should not be passed to Linux, and no
+	 * tail work should be performed;
+	 * <0 -- if the syscall should not be passed to Linux but the
+	 * tail work has to be performed (for handling signals etc).
+	 */
+
+	if (__ipipe_syscall_watched_p(current, regs->ARM_r7) &&
+	    __ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL) &&
+	    __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL,regs) > 0) {
+		if (ipipe_root_domain_p && !in_atomic()) {
+			/*
+			 * Sync pending VIRQs before _TIF_NEED_RESCHED
+			 * is tested.
+			 */
+			local_irq_save_hw(flags);
+			if ((ipipe_root_cpudom_var(irqpend_himask) & IPIPE_IRQMASK_VIRT) != 0)
+				__ipipe_sync_pipeline(IPIPE_IRQMASK_VIRT);
+			local_irq_restore_hw(flags);
+			regs->ARM_r7 = origr7;
+			return -1;
+		}
+		regs->ARM_r7 = origr7;
+		return 1;
+	}
+
+	regs->ARM_r7 = origr7;
+	return 0;
+}
+
+/*
+ * __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic
+ * interrupt protection log is maintained here for each domain. Hw
+ * interrupts are off on entry.
+ */
+int __ipipe_handle_irq(int irq, struct pt_regs *regs)
+{
+	struct ipipe_domain *this_domain, *next_domain;
+	struct list_head *head, *pos;
+	int m_ack;
+
+	m_ack = (regs == NULL);
+
+	if (irq >= IPIPE_NR_IRQS) {
+		printk(KERN_ERR "I-pipe: spurious interrupt %d\n", irq);
+		goto finalize_nosync;
+	}
+
+	head = __ipipe_pipeline.next;
+	next_domain = list_entry(head, struct ipipe_domain, p_link);
+	if (likely(test_bit(IPIPE_WIRED_FLAG, &next_domain->irqs[irq].control))) {
+		if (!m_ack && next_domain->irqs[irq].acknowledge != NULL)
+			next_domain->irqs[irq].acknowledge(irq);
+		if (likely(__ipipe_dispatch_wired(next_domain, irq))) {
+			goto finalize;
+		} else
+			goto finalize_nosync;
+	}
+
+	this_domain = ipipe_current_domain;
+
+	if (test_bit(IPIPE_STICKY_FLAG, &this_domain->irqs[irq].control))
+		head = &this_domain->p_link;
+
+	/* Ack the interrupt. */
+
+	pos = head;
+
+	while (pos != &__ipipe_pipeline) {
+		next_domain = list_entry(pos, struct ipipe_domain, p_link);
+
+		/*
+		 * For each domain handling the incoming IRQ, mark it
+		 * as pending in its log.
+		 */
+		if (test_bit(IPIPE_HANDLE_FLAG, &next_domain->irqs[irq].control)) {
+			/*
+			 * Domains that handle this IRQ are polled for
+			 * acknowledging it by decreasing priority
+			 * order. The interrupt must be made pending
+			 * _first_ in the domain's status flags before
+			 * the PIC is unlocked.
+			 */
+			__ipipe_set_irq_pending(next_domain, irq);
+
+			if (!m_ack && next_domain->irqs[irq].acknowledge != NULL)
+				m_ack = next_domain->irqs[irq].acknowledge(irq);
+		}
+
+		/*
+		 * If the domain does not want the IRQ to be passed
+		 * down the interrupt pipe, exit the loop now.
+		 */
+
+		if (!test_bit(IPIPE_PASS_FLAG, &next_domain->irqs[irq].control))
+			break;
+
+		pos = next_domain->p_link.next;
+	}
+
+finalize:
+
+	/*
+	 * Now walk the pipeline, yielding control to the highest
+	 * priority domain that has pending interrupt(s) or
+	 * immediately to the current domain if the interrupt has been
+	 * marked as 'sticky'. This search does not go beyond the
+	 * current domain in the pipeline.
+	 */
+
+	__ipipe_walk_pipeline(head);
+
+finalize_nosync:
+        if (!ipipe_root_domain_p || __ipipe_test_root())
+                return 0;
+
+#ifdef CONFIG_SMP
+	/*
+	 * Prevent a spurious rescheduling from being triggered on
+	 * preemptible kernels along the way out through
+	 * ret_from_intr.
+	 */
+        if (!regs)
+                __set_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
+#endif	/* CONFIG_SMP */
+
+        return 1;
+}
+
+asmlinkage int __ipipe_grab_irq(int irq, struct pt_regs *regs)
+{
+        int status;
+
+	if (irq == __ipipe_mach_timerint) {
+                /*
+		 * Given our deferred dispatching model for regular IRQs, we
+                 * only record CPU regs for the last timer interrupt, so that
+                 * the timer handler charges CPU times properly. It is assumed
+                 * that other interrupt handlers don't actually care for such
+                 * information.
+		 */
+		__raw_get_cpu_var(__ipipe_tick_regs).ARM_cpsr = regs->ARM_cpsr;
+		__raw_get_cpu_var(__ipipe_tick_regs).ARM_pc = regs->ARM_pc;
+	}
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+	ipipe_trace_begin(regs->ARM_ORIG_r0);
+#endif
+
+	if (__ipipe_mach_irq_mux_p(irq)) {
+                __ipipe_mach_demux_irq(irq, regs);
+                status = ipipe_root_domain_p && !__ipipe_test_root();
+        } else
+		status = __ipipe_handle_irq(irq, regs);
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+	ipipe_trace_end(regs->ARM_ORIG_r0);
+#endif
+
+        return status;
+}
+
+EXPORT_SYMBOL_GPL(show_stack);
+#ifndef MULTI_CPU
+EXPORT_SYMBOL_GPL(cpu_do_switch_mm);
+#endif
+EXPORT_SYMBOL_GPL(__check_kvm_seq);
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
+EXPORT_SYMBOL(tasklist_lock);
+#endif /* CONFIG_SMP || CONFIG_DEBUG_SPINLOCK */
+
+#ifdef CONFIG_IPIPE_TRACE_MCOUNT
+void notrace mcount(void);
+EXPORT_SYMBOL(mcount);
+#endif /* CONFIG_IPIPE_TRACE_MCOUNT */
diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c
index 11dcd52..1af02cd 100644
--- a/arch/arm/kernel/irq.c
+++ b/arch/arm/kernel/irq.c
@@ -124,8 +124,10 @@ asmlinkage void __exception asm_do_IRQ(unsigned int irq, struct pt_regs *regs)
 
 	desc_handle_irq(irq, desc);
 
+#ifndef CONFIG_IPIPE
 	/* AT91 specific workaround */
 	irq_finish(irq);
+#endif /* !CONFIG_IPIPE */
 
 	irq_exit();
 	set_irq_regs(old_regs);
diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c
index 46bf2ed..8fa54e2 100644
--- a/arch/arm/kernel/process.c
+++ b/arch/arm/kernel/process.c
@@ -135,6 +135,12 @@ static void default_idle(void)
 		local_irq_disable();
 		if (!need_resched()) {
 			timer_dyn_reprogram();
+#ifdef CONFIG_IPIPE
+			__ipipe_unstall_root();
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+			ipipe_trace_end(0x8000000E);
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+#endif /* CONFIG_IPIPE */
 			arch_idle();
 		}
 		local_irq_enable();
@@ -163,6 +169,7 @@ void cpu_idle(void)
 
 		if (!idle)
 			idle = default_idle;
+ 		ipipe_suspend_domain();
 		leds_event(led_idle_start);
 		tick_nohz_stop_sched_tick();
 		while (!need_resched())
diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c
index 4b05dc5..78905e2 100644
--- a/arch/arm/kernel/ptrace.c
+++ b/arch/arm/kernel/ptrace.c
@@ -481,6 +481,10 @@ void ptrace_break(struct task_struct *tsk, struct pt_regs *regs)
 
 static int break_trap(struct pt_regs *regs, unsigned int instr)
 {
+
+	if (ipipe_trap_notify(IPIPE_TRAP_BREAK,regs))
+		return 0;
+
 	ptrace_break(current, regs);
 	return 0;
 }
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index eefae1d..b13d8a5 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -32,6 +32,7 @@
 #include <asm/processor.h>
 #include <asm/tlbflush.h>
 #include <asm/ptrace.h>
+#include <asm/fcse.h>
 
 /*
  * bitmask of present and online CPUs.
@@ -736,14 +737,14 @@ void flush_tlb_all(void)
 
 void flush_tlb_mm(struct mm_struct *mm)
 {
-	cpumask_t mask = mm->cpu_vm_mask;
+	cpumask_t mask = fcse_tlb_mask(mm);
 
 	on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, 1, mask);
 }
 
 void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
 {
-	cpumask_t mask = vma->vm_mm->cpu_vm_mask;
+	cpumask_t mask = fcse_tlb_mask(vma->vm_mm);
 	struct tlb_args ta;
 
 	ta.ta_vma = vma;
@@ -764,7 +765,7 @@ void flush_tlb_kernel_page(unsigned long kaddr)
 void flush_tlb_range(struct vm_area_struct *vma,
                      unsigned long start, unsigned long end)
 {
-	cpumask_t mask = vma->vm_mm->cpu_vm_mask;
+	cpumask_t mask = fcse_tlb_mask(vma->vm_mm);
 	struct tlb_args ta;
 
 	ta.ta_vma = vma;
diff --git a/arch/arm/kernel/sys_arm.c b/arch/arm/kernel/sys_arm.c
index 0128687..66ff375 100644
--- a/arch/arm/kernel/sys_arm.c
+++ b/arch/arm/kernel/sys_arm.c
@@ -61,6 +61,16 @@ inline long do_mmap2(
 
 	if (file)
 		fput(file);
+#ifdef CONFIG_ARM_FCSE
+	/* FIXME, this really sucks, and we should really recheck in mremap,
+	   mprotect, and munmap */
+	if (likely((unsigned) error < (unsigned)(-4096))
+	    && (flags & MAP_SHARED) && (prot & PROT_WRITE)) {
+		struct vm_area_struct *vma = find_vma(current->mm, error);
+		if (vma->vm_page_prot & (L_PTE_CACHEABLE | L_PTE_BUFFERABLE))
+			++current->mm->context.mappings_needing_flush;
+	}
+#endif
 out:
 	return error;
 }
diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c
index 5595fdd..171f707 100644
--- a/arch/arm/kernel/traps.c
+++ b/arch/arm/kernel/traps.c
@@ -297,6 +297,11 @@ asmlinkage void __exception do_undefinstr(struct pt_regs *regs)
 	void __user *pc;
 	unsigned long flags;
 
+	/* In kernel mode, the trap was already signaled at the beginning of
+	 * __und_svc in arch/arm/kernel/entry-armv.S */
+	if (user_mode(regs) && ipipe_trap_notify(IPIPE_TRAP_UNDEFINSTR, regs))
+		return;
+
 	/*
 	 * According to the ARM ARM, PC is 2 or 4 bytes ahead,
 	 * depending whether we're in Thumb mode or not.
@@ -706,13 +711,22 @@ void abort(void)
 }
 EXPORT_SYMBOL(abort);
 
+#ifdef CONFIG_IPIPE
+void *__ipipe_tsc_area;
+#endif /* CONFIG_IPIPE */
+
 void __init trap_init(void)
 {
 	unsigned long vectors = CONFIG_VECTORS_BASE;
 	extern char __stubs_start[], __stubs_end[];
 	extern char __vectors_start[], __vectors_end[];
+#ifndef CONFIG_IPIPE
 	extern char __kuser_helper_start[], __kuser_helper_end[];
 	int kuser_sz = __kuser_helper_end - __kuser_helper_start;
+#else /* !CONFIG_IPIPE */
+	extern char __ipipe_tsc_area_start[], __kuser_helper_end[];
+	int kuser_sz = __kuser_helper_end - __ipipe_tsc_area_start;
+#endif /* !CONFIG_IPIPE */
 
 	/*
 	 * Copy the vectors, stubs and kuser helpers (in entry-armv.S)
@@ -721,7 +735,13 @@ void __init trap_init(void)
 	 */
 	memcpy((void *)vectors, __vectors_start, __vectors_end - __vectors_start);
 	memcpy((void *)vectors + 0x200, __stubs_start, __stubs_end - __stubs_start);
+#ifndef CONFIG_IPIPE
 	memcpy((void *)vectors + 0x1000 - kuser_sz, __kuser_helper_start, kuser_sz);
+#else /* !CONFIG_IPIPE */
+	BUG_ON(0x1000 - kuser_sz < 0x200 + __stubs_end - __stubs_start);
+	memcpy((void *)vectors + 0x1000 - kuser_sz, __ipipe_tsc_area_start, kuser_sz);
+	__ipipe_tsc_area = (void *)vectors + 0x1000 - kuser_sz;
+#endif /* !CONFIG_IPIPE */
 
 	/*
 	 * Copy signal return handlers into the vector page, and
diff --git a/arch/arm/mach-at91/Kconfig b/arch/arm/mach-at91/Kconfig
index 0fc07b6..ab4bf2b 100644
--- a/arch/arm/mach-at91/Kconfig
+++ b/arch/arm/mach-at91/Kconfig
@@ -243,6 +243,19 @@ endif
 
 # ----------------------------------------------------------
 
+comment "Adeos I-pipe Options"
+
+config IPIPE_AT91_TC
+	depends on IPIPE
+	int "AT91 TC used as time base by Adeos I-pipe"
+	default 0
+	help
+	When Adeos interrupt pipeline is enabled, TC0 is used by default
+	as time base, but you can use TC1 or TC2 by setting this variable to 1
+	or 2. This should only be needed to avoid conflicts with other drivers.
+
+# ----------------------------------------------------------
+
 comment "AT91 Board Options"
 
 config MTD_AT91_DATAFLASH_CARD
diff --git a/arch/arm/mach-at91/Makefile b/arch/arm/mach-at91/Makefile
index 8d9bc01..43a407d 100644
--- a/arch/arm/mach-at91/Makefile
+++ b/arch/arm/mach-at91/Makefile
@@ -60,3 +60,13 @@ obj-$(CONFIG_PM)		+= pm.o
 ifeq ($(CONFIG_PM_DEBUG),y)
 CFLAGS_pm.o += -DDEBUG
 endif
+
+ifeq ($(CONFIG_IPIPE),y)
+obj-y := $(filter-out at91rm9200_time.o at91sam926x_time.o at91x40_time.o, $(obj-y))
+obj-$(CONFIG_ARCH_AT91RM9200)   += at91_ipipe_time.o
+obj-$(CONFIG_ARCH_AT91SAM9260)  += at91_ipipe_time.o
+obj-$(CONFIG_ARCH_AT91SAM9261)  += at91_ipipe_time.o
+obj-$(CONFIG_ARCH_AT91SAM9263)	+= at91_ipipe_time.o
+obj-$(CONFIG_ARCH_AT91SAM9RL)	+= at91_ipipe_time.o
+obj-$(CONFIG_ARCH_AT91X40)	+= at91_ipipe_time.o
+endif
diff --git a/arch/arm/mach-at91/at91_ipipe_time.c b/arch/arm/mach-at91/at91_ipipe_time.c
new file mode 100644
index 0000000..a645e85
--- /dev/null
+++ b/arch/arm/mach-at91/at91_ipipe_time.c
@@ -0,0 +1,413 @@
+/*
+ * linux/arch/arm/mach-at91/at91_ipipe_time.c
+ *
+ * Copyright (C) 2007 Gilles Chanteperdrix <gilles.chanteperdrix@laposte.net>
+ *
+ * Adaptation to AT91SAM926x:
+ * Copyright (C) 2007 Gregory CLEMENT, Adeneo
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/kernel.h>
+#include <linux/clockchips.h>
+
+#include <asm/hardware.h>
+#include <asm/io.h>
+#include <asm/mach/time.h>
+
+#include <asm/arch/at91_st.h>
+
+#include <linux/clk.h>
+#include <linux/stringify.h>
+#include <linux/err.h>
+#include <linux/console.h>
+#include <linux/module.h>
+#include <asm/arch/at91_tc.h>
+#include <asm/arch/at91_pit.h>
+#include "clock.h"
+
+#if defined(CONFIG_ARCH_AT91RM9200)
+#define AT91_ID_TC0 AT91RM9200_ID_TC0
+#define AT91_ID_TC1 AT91RM9200_ID_TC1
+#define AT91_ID_TC2 AT91RM9200_ID_TC2
+#elif defined(CONFIG_ARCH_AT91SAM9260)
+#define AT91_ID_TC0 AT91SAM9260_ID_TC0
+#define AT91_ID_TC1 AT91SAM9260_ID_TC1
+#define AT91_ID_TC2 AT91SAM9260_ID_TC2
+#elif defined(CONFIG_ARCH_AT91SAM9261)
+#define AT91_ID_TC0 AT91SAM9261_ID_TC0
+#define AT91_ID_TC1 AT91SAM9261_ID_TC1
+#define AT91_ID_TC2 AT91SAM9261_ID_TC2
+#elif defined(CONFIG_ARCH_AT91SAM9263)
+#define AT91_ID_TC0 AT91SAM9263_ID_TCB
+#define AT91_ID_TC1 AT91SAM9263_ID_TCB
+#define AT91_ID_TC2 AT91SAM9263_ID_TCB
+#elif defined(CONFIG_ARCH_AT91SAM9RL)
+#define AT91_ID_TC0 AT91SAM9RL_ID_TC0
+#define AT91_ID_TC1 AT91SAM9RL_ID_TC1
+#define AT91_ID_TC2 AT91SAM9RL_ID_TC2
+#elif defined(CONFIG_ARCH_AT91X40)
+#define AT91_ID_TC0 AT91X40_ID_TC0
+#define AT91_ID_TC1 AT91X40_ID_TC1
+#define AT91_ID_TC2 AT91X40_ID_TC2
+#else
+#error "AT91 processor unsupported by Adeos"
+#endif
+
+#if (CONFIG_IPIPE_AT91_TC==0)
+#   define KERNEL_TIMER_IRQ_NUM AT91_ID_TC0
+#elif (CONFIG_IPIPE_AT91_TC==1)
+#   define KERNEL_TIMER_IRQ_NUM AT91_ID_TC1
+#elif (CONFIG_IPIPE_AT91_TC==2)
+#   define KERNEL_TIMER_IRQ_NUM AT91_ID_TC2
+#else
+#error IPIPE_AT91_TC must be 0, 1 or 2.
+#endif
+
+#ifdef CONFIG_NO_IDLE_HZ
+#error "dynamic tick timer not yet supported with IPIPE"
+#endif /* CONFIG_NO_IDLE_HZ */
+
+#define TCNXCNS(timer,v) ((v) << ((timer)<<1))
+#define AT91_TC_REG_MASK (0xffff)
+
+static unsigned long next_match;
+
+union tsc_reg {
+#ifdef __BIG_ENDIAN
+	struct {
+		unsigned long high;
+		union {
+			struct {
+				unsigned short mid;
+				unsigned short low;
+			};
+			unsigned long mid_low;
+		};
+	};
+#else /* __LITTLE_ENDIAN */
+	struct {
+		union {
+			struct {
+				unsigned short low;
+				unsigned short mid;
+			};
+			unsigned long mid_low;
+		};
+		unsigned long high;
+	};
+#endif /* __LITTLE_ENDIAN */
+	unsigned long long full;
+};
+static unsigned max_delta_ticks, min_delta_ticks;
+static struct clock_event_device clkevt;
+static int tc_timer_clock;
+
+#ifdef CONFIG_SMP
+static union tsc_reg tsc[NR_CPUS];
+
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info)
+{
+	info->type = IPIPE_TSC_TYPE_NONE;
+}
+
+#else /* !CONFIG_SMP */
+static union tsc_reg *tsc;
+
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info)
+{
+	info->type = IPIPE_TSC_TYPE_FREERUNNING;
+	info->u.fr.counter =
+		(unsigned *)
+		(AT91_BASE_TCB0 + 0x40 * CONFIG_IPIPE_AT91_TC + AT91_TC_CV);
+	info->u.fr.mask = AT91_TC_REG_MASK;
+	info->u.fr.tsc = &tsc->full;
+}
+#endif /* !CONFIG_SMP */
+
+static inline unsigned int at91_tc_read(unsigned int reg_offset)
+{
+	unsigned long addr =
+		(AT91_VA_BASE_TCB0 + 0x40 * CONFIG_IPIPE_AT91_TC);
+
+	return readl((void __iomem *)(addr + reg_offset));
+}
+
+static inline void at91_tc_write(unsigned int reg_offset, unsigned long value)
+{
+	unsigned long addr =
+		(AT91_VA_BASE_TCB0 + 0x40 * CONFIG_IPIPE_AT91_TC);
+
+	writel(value, (void __iomem *)(addr + reg_offset));
+}
+
+#define read_CV() at91_tc_read(AT91_TC_CV)
+#define read_RC() at91_tc_read(AT91_TC_RC)
+#define write_RC(value) at91_tc_write(AT91_TC_RC, value)
+
+int __ipipe_mach_timerint = KERNEL_TIMER_IRQ_NUM;
+EXPORT_SYMBOL(__ipipe_mach_timerint);
+
+int __ipipe_mach_timerstolen = 0;
+EXPORT_SYMBOL(__ipipe_mach_timerstolen);
+
+unsigned int __ipipe_mach_ticks_per_jiffy = LATCH;
+EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy);
+
+static void ipipe_mach_update_tsc(unsigned short stamp);
+
+static int at91_timer_initialized;
+
+/*
+ * IRQ handler for the timer.
+ */
+static irqreturn_t at91_timer_interrupt(int irq, void *dev_id)
+{
+	clkevt.event_handler(&clkevt);
+	return IRQ_HANDLED;
+}
+
+static struct irqaction at91_timer_irq = {
+	.name		= "at91_tick",
+	.flags		= IRQF_DISABLED | IRQF_TIMER,
+	.handler	= &at91_timer_interrupt
+};
+
+static void ipipe_mach_update_tsc(unsigned short stamp)
+{
+	union tsc_reg *local_tsc;
+
+	local_tsc = &tsc[ipipe_processor_id()];
+	if (unlikely(stamp < local_tsc->low))
+		local_tsc->full += AT91_TC_REG_MASK + 1;
+	local_tsc->low = stamp;
+}
+
+void __ipipe_mach_acktimer(void)
+{
+	at91_tc_read(AT91_TC_SR);
+
+	if (unlikely(!__ipipe_mach_timerstolen)) {
+		ipipe_mach_update_tsc(read_CV());
+		next_match = (next_match + __ipipe_mach_ticks_per_jiffy)
+			& AT91_TC_REG_MASK;
+		write_RC(next_match);
+	}
+}
+
+notrace unsigned long long __ipipe_mach_get_tsc(void)
+{
+	if (likely(at91_timer_initialized)) {
+		union tsc_reg *local_tsc, before, after;
+		unsigned short stamp;
+
+		local_tsc = &tsc[ipipe_processor_id()];
+
+		__asm__ ("ldmia %1, %M0\n":
+			 "=r"(after.full): "r"(local_tsc), "m"(*local_tsc));
+		do {
+			before = after;
+			stamp = read_CV();
+			barrier();
+			__asm__ ("ldmia %1, %M0\n":
+				 "=r"(after.full):
+				 "r"(local_tsc), "m"(*local_tsc));
+		} while (after.mid_low != before.mid_low);
+		if (unlikely(stamp < before.low))
+			before.full += AT91_TC_REG_MASK + 1;
+		before.low = stamp;
+		return before.full;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(__ipipe_mach_get_tsc);
+
+static struct clocksource clksrc = {
+	.name   = "at91_tc" __stringify(CONFIG_IPIPE_AT91_TC),
+	.rating = 250,
+	.read   = __ipipe_mach_get_tsc,
+	.mask   = CLOCKSOURCE_MASK(64),
+	.shift  = 20,
+	.flags  = CLOCK_SOURCE_IS_CONTINUOUS,
+};
+
+static void
+at91_tc_set_mode(enum clock_event_mode mode, struct clock_event_device *dev)
+{
+	/* Disable the channel */
+	at91_tc_write(AT91_TC_CCR, AT91_TC_CLKDIS);
+
+	/* Disable all interrupts. */
+	at91_tc_write(AT91_TC_IDR, ~0ul);	
+
+	if (mode == CLOCK_EVT_MODE_PERIODIC) {
+		unsigned long v;
+
+#ifndef CONFIG_ARCH_AT91SAM9263
+		clk_enable(clk_get(NULL, "tc"
+				   __stringify(CONFIG_IPIPE_AT91_TC) "_clk"));
+#else /* AT91SAM9263 */
+		clk_enable(clk_get(NULL, "tcb_clk"));
+#endif /* AT91SAM9263 */
+
+		/* No Sync. */
+		at91_tc_write(AT91_TC_BCR, 0);
+
+		/* program NO signal on XCN */
+		v = readl((void __iomem *) (AT91_VA_BASE_TCB0 + AT91_TC_BMR));
+		v &= ~TCNXCNS(CONFIG_IPIPE_AT91_TC, 3);
+		v |= TCNXCNS(CONFIG_IPIPE_AT91_TC, 1); /* AT91_TC_TCNXCNS_NONE */
+		writel(v, (void __iomem *) (AT91_VA_BASE_TCB0 + AT91_TC_BMR));
+
+		/* Use the clock selected by at91_timer_init as input clock. */
+		at91_tc_write(AT91_TC_CMR, tc_timer_clock);
+
+		/* Load the TC register C. */
+		next_match = __ipipe_mach_ticks_per_jiffy;
+		write_RC(next_match);
+
+		/* Enable CPCS interrupt. */
+		at91_tc_write(AT91_TC_IER, AT91_TC_CPCS);
+
+		/* Enable the channel. */
+		at91_tc_write(AT91_TC_CCR, AT91_TC_CLKEN | AT91_TC_SWTRG);
+
+		at91_timer_initialized = 1;
+	}
+}
+
+/*
+ * Reprogram the timer
+ */
+void __ipipe_mach_set_dec(unsigned long delay)
+{
+	unsigned short stamp;
+	unsigned long flags;
+
+	if (delay > max_delta_ticks)
+		delay = max_delta_ticks;
+
+	if (likely(delay > min_delta_ticks)) {
+		local_irq_save_hw(flags);
+		stamp = read_CV();
+		ipipe_mach_update_tsc(stamp);
+		write_RC((stamp + delay) & AT91_TC_REG_MASK);
+		local_irq_restore_hw(flags);
+	} else
+		ipipe_trigger_irq(KERNEL_TIMER_IRQ_NUM);
+}
+EXPORT_SYMBOL(__ipipe_mach_set_dec);
+
+int __ipipe_check_tickdev(const char *devname)
+{
+	return !strcmp(devname, clkevt.name);
+}
+
+static struct clock_event_device clkevt = {
+	.name		= "at91_tc" __stringify(CONFIG_IPIPE_AT91_TC),
+	.features	= CLOCK_EVT_FEAT_PERIODIC,
+	.shift		= 32,
+	.rating		= 250,
+	.cpumask	= CPU_MASK_CPU0,
+	.set_mode	= at91_tc_set_mode,
+};
+
+void __ipipe_mach_release_timer(void)
+{
+	__ipipe_mach_set_dec(__ipipe_mach_ticks_per_jiffy);
+}
+EXPORT_SYMBOL(__ipipe_mach_release_timer);
+
+unsigned long __ipipe_mach_get_dec(void)
+{
+	return (read_RC() - read_CV()) & AT91_TC_REG_MASK;
+}
+
+void __init at91_timer_init(void)
+{
+	unsigned char tc_divisors[] = { 2, 8, 32, 128, 0, };
+	unsigned master_freq, divisor, divided_freq;
+	unsigned long long wrap_ns;
+
+	/* Disable (boot loader) timer interrupts. */
+#if defined(CONFIG_ARCH_AT91RM9200)
+	at91_sys_write(AT91_ST_IDR, AT91_ST_PITS | AT91_ST_WDOVF | AT91_ST_RTTINC | AT91_ST_ALMS);
+	(void) at91_sys_read(AT91_ST_SR);	/* Clear any pending interrupts */	
+#elif defined(CONFIG_ARCH_AT91SAM9260) || defined(CONFIG_ARCH_AT91SAM9261) \
+	|| defined(CONFIG_ARCH_AT91SAM9263) || defined(CONFIG_ARCH_AT91SAM9RL)
+	at91_sys_write(AT91_PIT_MR, 0);
+
+	/* Clear any pending interrupts */
+	(void) at91_sys_read(AT91_PIT_PIVR);
+#endif /* CONFIG_ARCH_AT91SAM926x */
+
+	master_freq = clk_get_rate(clk_get(NULL, "mck"));
+	/* Find the first frequency above 1 MHz */
+	for (tc_timer_clock = ARRAY_SIZE(tc_divisors) - 1;
+	     tc_timer_clock >= 0; tc_timer_clock--) {
+		divisor = tc_divisors[tc_timer_clock];
+		divided_freq = (divisor
+				? master_freq / divisor : AT91_SLOW_CLOCK);
+		if (divided_freq > 1000000)
+			break;
+	}
+
+	wrap_ns = (unsigned long long) (AT91_TC_REG_MASK + 1) * NSEC_PER_SEC;
+	do_div(wrap_ns, divided_freq);
+
+	if (divided_freq < 1000000)
+		printk(KERN_INFO "AT91 I-pipe warning: could not find a"
+		       " frequency greater than 1MHz\n");
+
+	printk(KERN_INFO "AT91 I-pipe timer: div: %u, freq: %u.%06u MHz, wrap: "
+	       "%u.%06u ms\n", divisor,
+	       divided_freq / 1000000, divided_freq % 1000000,
+	       (unsigned) wrap_ns / 1000000, (unsigned) wrap_ns % 1000000);
+
+	/* Add a 1ms margin. It means that when an interrupt occurs, update_tsc
+	   must be called within 1ms. update_tsc is called by acktimer when no
+	   higher domain handles the timer, and called through set_dec when a
+	   higher domain handles the timer. */
+	wrap_ns -= 1000000;
+	/* Set up the interrupt. */
+	setup_irq(KERNEL_TIMER_IRQ_NUM, &at91_timer_irq);
+
+#ifndef CONFIG_SMP
+	tsc = (union tsc_reg *) __ipipe_tsc_area;
+#endif /* CONFIG_SMP */
+
+	clkevt.mult = div_sc(divided_freq, NSEC_PER_SEC, clkevt.shift);
+	clkevt.max_delta_ns = wrap_ns;
+	clkevt.min_delta_ns = 2000;
+	clkevt.cpumask = cpumask_of_cpu(0);
+	clockevents_register_device(&clkevt);
+
+	clksrc.mult = clocksource_hz2mult(divided_freq, clksrc.shift);
+	clocksource_register(&clksrc);
+
+	__ipipe_mach_ticks_per_jiffy = (divided_freq + HZ/2) / HZ;
+	max_delta_ticks = (wrap_ns * clkevt.mult) >> clkevt.shift;
+	min_delta_ticks = ((unsigned long long) clkevt.min_delta_ns
+			   * clkevt.mult) >> clkevt.shift;
+}
+
+#ifdef CONFIG_ARCH_AT91RM9200
+struct sys_timer at91rm9200_timer = {
+#elif defined(CONFIG_ARCH_AT91SAM9260) || defined(CONFIG_ARCH_AT91SAM9261) \
+	|| defined(CONFIG_ARCH_AT91SAM9263) || defined(CONFIG_ARCH_AT91SAM9RL)
+struct sys_timer at91sam926x_timer = {
+#elif defined(CONFIG_ARCH_AT91X40)
+struct sys_timer at91x40_timer = {
+#else
+#error "Unknown machine"
+#endif
+	.init		= at91_timer_init,
+	.suspend	= NULL,
+	.resume		= NULL,
+};
diff --git a/arch/arm/mach-at91/at91rm9200.c b/arch/arm/mach-at91/at91rm9200.c
index d688c1d..00961cc 100644
--- a/arch/arm/mach-at91/at91rm9200.c
+++ b/arch/arm/mach-at91/at91rm9200.c
@@ -32,7 +32,14 @@ static struct map_desc at91rm9200_io_desc[] __initdata = {
 		.pfn		= __phys_to_pfn(AT91RM9200_BASE_EMAC),
 		.length		= SZ_16K,
 		.type		= MT_DEVICE,
+#ifdef CONFIG_IPIPE
 	}, {
+		.virtual	= AT91_VA_BASE_TCB0,
+		.pfn		= __phys_to_pfn(AT91_BASE_TCB0),
+		.length		= SZ_16K,
+		.type		= MT_DEVICE,
+#endif /* CONFIG_IPIPE */
+        }, {
 		.virtual	= AT91_IO_VIRT_BASE - AT91RM9200_SRAM_SIZE,
 		.pfn		= __phys_to_pfn(AT91RM9200_SRAM_BASE),
 		.length		= AT91RM9200_SRAM_SIZE,
@@ -299,6 +306,7 @@ void __init at91rm9200_initialize(unsigned long main_clock, unsigned short banks
  * The default interrupt priority levels (0 = lowest, 7 = highest).
  */
 static unsigned int at91rm9200_default_irq_priority[NR_AIC_IRQS] __initdata = {
+#ifndef CONFIG_IPIPE
 	7,	/* Advanced Interrupt Controller (FIQ) */
 	7,	/* System Peripherals */
 	1,	/* Parallel IO Controller A */
@@ -331,6 +339,42 @@ static unsigned int at91rm9200_default_irq_priority[NR_AIC_IRQS] __initdata = {
 	0,	/* Advanced Interrupt Controller (IRQ4) */
 	0,	/* Advanced Interrupt Controller (IRQ5) */
 	0	/* Advanced Interrupt Controller (IRQ6) */
+#else /* CONFIG_IPIPE */
+/* Give the highest priority to TC, since they are used as timer interrupt by
+   I-pipe. */
+	7,	/* Advanced Interrupt Controller */
+	6,	/* System Peripheral */
+	0,	/* Parallel IO Controller A */
+	0,	/* Parallel IO Controller B */
+	0,	/* Parallel IO Controller C */
+	0,	/* Parallel IO Controller D */
+	5,	/* USART 0 */
+	5,	/* USART 1 */
+	5,	/* USART 2 */
+	5,	/* USART 3 */
+	0,	/* Multimedia Card Interface */
+	3,	/* USB Device Port */
+	0,	/* Two-Wire Interface */
+	5,	/* Serial Peripheral Interface */
+	4,	/* Serial Synchronous Controller */
+	4,	/* Serial Synchronous Controller */
+	4,	/* Serial Synchronous Controller */
+	7,	/* Timer Counter 0 */
+	7,	/* Timer Counter 1 */
+	7,	/* Timer Counter 2 */
+	0,	/* Timer Counter 3 */
+	0,	/* Timer Counter 4 */
+	0,	/* Timer Counter 5 */
+	2,	/* USB Host port */
+	2,	/* Ethernet MAC */
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+	0	/* Advanced Interrupt Controller */
+#endif /*CONFIG_IPIPE */
 };
 
 void __init at91rm9200_init_interrupts(unsigned int priority[NR_AIC_IRQS])
diff --git a/arch/arm/mach-at91/at91sam9260.c b/arch/arm/mach-at91/at91sam9260.c
index ee26550..eb5fd77 100644
--- a/arch/arm/mach-at91/at91sam9260.c
+++ b/arch/arm/mach-at91/at91sam9260.c
@@ -30,6 +30,13 @@ static struct map_desc at91sam9260_io_desc[] __initdata = {
 		.pfn		= __phys_to_pfn(AT91_BASE_SYS),
 		.length		= SZ_16K,
 		.type		= MT_DEVICE,
+#ifdef CONFIG_IPIPE
+	}, {
+		.virtual	= AT91_VA_BASE_TCB0,
+		.pfn		= __phys_to_pfn(AT91_BASE_TCB0),
+		.length		= SZ_16K,
+		.type		= MT_DEVICE,
+#endif /* CONFIG_IPIPE */
 	}
 };
 
@@ -333,6 +340,7 @@ void __init at91sam9260_initialize(unsigned long main_clock)
  * The default interrupt priority levels (0 = lowest, 7 = highest).
  */
 static unsigned int at91sam9260_default_irq_priority[NR_AIC_IRQS] __initdata = {
+#ifndef CONFIG_IPIPE
 	7,	/* Advanced Interrupt Controller */
 	7,	/* System Peripherals */
 	1,	/* Parallel IO Controller A */
@@ -365,6 +373,42 @@ static unsigned int at91sam9260_default_irq_priority[NR_AIC_IRQS] __initdata = {
 	0,	/* Advanced Interrupt Controller */
 	0,	/* Advanced Interrupt Controller */
 	0,	/* Advanced Interrupt Controller */
+#else /* CONFIG_IPIPE */
+/* Give the highest priority to TC, since they are used as timer interrupt by
+   I-pipe. */
+	7,	/* Advanced Interrupt Controller */
+	7,	/* System Peripherals */
+	0,	/* Parallel IO Controller A */
+	0,	/* Parallel IO Controller B */
+	0,	/* Parallel IO Controller C */
+	0,	/* Analog-to-Digital Converter */
+	6,	/* USART 0 */
+	6,	/* USART 1 */
+	6,	/* USART 2 */
+	0,	/* Multimedia Card Interface */
+	4,	/* USB Device Port */
+	0,	/* Two-Wire Interface */
+	6,	/* Serial Peripheral Interface 0 */
+	6,	/* Serial Peripheral Interface 1 */
+	5,	/* Serial Synchronous Controller */
+	0,
+	0,
+	7,	/* Timer Counter 0 */
+	7,	/* Timer Counter 1 */
+	7,	/* Timer Counter 2 */
+	3,	/* USB Host port */
+	3,	/* Ethernet */
+	0,	/* Image Sensor Interface */
+	6,	/* USART 3 */
+	6,	/* USART 4 */
+	6,	/* USART 5 */
+	7,	/* Timer Counter 3 */
+	7,	/* Timer Counter 4 */
+	7,	/* Timer Counter 5 */
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+#endif /*CONFIG_IPIPE */
 };
 
 void __init at91sam9260_init_interrupts(unsigned int priority[NR_AIC_IRQS])
diff --git a/arch/arm/mach-at91/at91sam9261.c b/arch/arm/mach-at91/at91sam9261.c
index 35bf6fd..b05e98b 100644
--- a/arch/arm/mach-at91/at91sam9261.c
+++ b/arch/arm/mach-at91/at91sam9261.c
@@ -29,6 +29,13 @@ static struct map_desc at91sam9261_io_desc[] __initdata = {
 		.pfn		= __phys_to_pfn(AT91_BASE_SYS),
 		.length		= SZ_16K,
 		.type		= MT_DEVICE,
+#ifdef CONFIG_IPIPE
+	}, {
+		.virtual	= AT91_VA_BASE_TCB0,
+		.pfn		= __phys_to_pfn(AT91_BASE_TCB0),
+		.length		= SZ_16K,
+		.type		= MT_DEVICE,
+#endif /* CONFIG_IPIPE */
 	}, {
 		.virtual	= AT91_IO_VIRT_BASE - AT91SAM9261_SRAM_SIZE,
 		.pfn		= __phys_to_pfn(AT91SAM9261_SRAM_BASE),
@@ -285,6 +292,7 @@ void __init at91sam9261_initialize(unsigned long main_clock)
  * The default interrupt priority levels (0 = lowest, 7 = highest).
  */
 static unsigned int at91sam9261_default_irq_priority[NR_AIC_IRQS] __initdata = {
+#ifndef CONFIG_IPIPE
 	7,	/* Advanced Interrupt Controller */
 	7,	/* System Peripherals */
 	1,	/* Parallel IO Controller A */
@@ -317,6 +325,42 @@ static unsigned int at91sam9261_default_irq_priority[NR_AIC_IRQS] __initdata = {
 	0,	/* Advanced Interrupt Controller */
 	0,	/* Advanced Interrupt Controller */
 	0,	/* Advanced Interrupt Controller */
+#else /* CONFIG_IPIPE */
+/* Give the highest priority to TC, since they are used as timer interrupt by
+   I-pipe. */
+	7,	/* Advanced Interrupt Controller */
+	7,	/* System Peripherals */
+	0,	/* Parallel IO Controller A */
+	0,	/* Parallel IO Controller B */
+	0,	/* Parallel IO Controller C */
+	0,
+	6,	/* USART 0 */
+	6,	/* USART 1 */
+	6,	/* USART 2 */
+	0,	/* Multimedia Card Interface */
+	4,	/* USB Device Port */
+	0,	/* Two-Wire Interface */
+	6,	/* Serial Peripheral Interface 0 */
+	6,	/* Serial Peripheral Interface 1 */
+	5,	/* Serial Synchronous Controller 0 */
+	5,	/* Serial Synchronous Controller 1 */
+	5,	/* Serial Synchronous Controller 2 */
+	7,	/* Timer Counter 0 */
+	7,	/* Timer Counter 1 */
+	7,	/* Timer Counter 2 */
+	3,	/* USB Host port */
+	3,	/* LCD Controller */
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+	0,	/* Advanced Interrupt Controller */
+#endif /*CONFIG_IPIPE */
 };
 
 void __init at91sam9261_init_interrupts(unsigned int priority[NR_AIC_IRQS])
diff --git a/arch/arm/mach-at91/at91sam9263.c b/arch/arm/mach-at91/at91sam9263.c
index 052074a..fb1a27a 100644
--- a/arch/arm/mach-at91/at91sam9263.c
+++ b/arch/arm/mach-at91/at91sam9263.c
@@ -29,6 +29,13 @@ static struct map_desc at91sam9263_io_desc[] __initdata = {
 		.pfn		= __phys_to_pfn(AT91_BASE_SYS),
 		.length		= SZ_16K,
 		.type		= MT_DEVICE,
+#ifdef CONFIG_IPIPE
+	}, {
+		.virtual	= AT91_VA_BASE_TCB0,
+		.pfn		= __phys_to_pfn(AT91_BASE_TCB0),
+		.length		= SZ_16K,
+		.type		= MT_DEVICE,
+#endif /* CONFIG_IPIPE */
 	}, {
 		.virtual	= AT91_IO_VIRT_BASE - AT91SAM9263_SRAM0_SIZE,
 		.pfn		= __phys_to_pfn(AT91SAM9263_SRAM0_BASE),
@@ -310,6 +317,7 @@ void __init at91sam9263_initialize(unsigned long main_clock)
  * The default interrupt priority levels (0 = lowest, 7 = highest).
  */
 static unsigned int at91sam9263_default_irq_priority[NR_AIC_IRQS] __initdata = {
+#ifndef CONFIG_IPIPE
 	7,	/* Advanced Interrupt Controller (FIQ) */
 	7,	/* System Peripherals */
 	1,	/* Parallel IO Controller A */
@@ -342,6 +350,42 @@ static unsigned int at91sam9263_default_irq_priority[NR_AIC_IRQS] __initdata = {
 	2,	/* USB Host port */
 	0,	/* Advanced Interrupt Controller (IRQ0) */
 	0,	/* Advanced Interrupt Controller (IRQ1) */
+#else /* CONFIG_IPIPE */
+/* Give the highest priority to TC, since they are used as timer interrupt by
+   I-pipe. */
+	7,	/* Advanced Interrupt Controller (FIQ) */
+	6,	/* System Peripherals */
+	0,	/* Parallel IO Controller A */
+	0,	/* Parallel IO Controller B */
+	0,	/* Parallel IO Controller C, D and E */
+	0,
+	0,
+	5,	/* USART 0 */
+	5,	/* USART 1 */
+	5,	/* USART 2 */
+	0,	/* Multimedia Card Interface 0 */
+	0,	/* Multimedia Card Interface 1 */
+	3,	/* CAN */
+	0,	/* Two-Wire Interface */
+	5,	/* Serial Peripheral Interface 0 */
+	5,	/* Serial Peripheral Interface 1 */
+	4,	/* Serial Synchronous Controller 0 */
+	4,	/* Serial Synchronous Controller 1 */
+	5,	/* AC97 Controller */
+	7,	/* Timer Counter 0, 1 and 2 */
+	0,	/* Pulse Width Modulation Controller */
+	2,	/* Ethernet */
+	0,
+	0,	/* 2D Graphic Engine */
+	2,	/* USB Device Port */
+	0,	/* Image Sensor Interface */
+	2,	/* LDC Controller */
+	0,	/* DMA Controller */
+	0,
+	2,	/* USB Host port */
+	0,	/* Advanced Interrupt Controller (IRQ0) */
+	0,	/* Advanced Interrupt Controller (IRQ1) */
+#endif /*CONFIG_IPIPE */
 };
 
 void __init at91sam9263_init_interrupts(unsigned int priority[NR_AIC_IRQS])
diff --git a/arch/arm/mach-at91/at91sam9rl.c b/arch/arm/mach-at91/at91sam9rl.c
index 902c798..e2498f1 100644
--- a/arch/arm/mach-at91/at91sam9rl.c
+++ b/arch/arm/mach-at91/at91sam9rl.c
@@ -29,6 +29,13 @@ static struct map_desc at91sam9rl_io_desc[] __initdata = {
 		.pfn		= __phys_to_pfn(AT91_BASE_SYS),
 		.length		= SZ_16K,
 		.type		= MT_DEVICE,
+#ifdef CONFIG_IPIPE
+	}, {
+		.virtual	= AT91_VA_BASE_TCB0,
+		.pfn		= __phys_to_pfn(AT91_BASE_TCB0),
+		.length		= SZ_16K,
+		.type		= MT_DEVICE,
+#endif /* CONFIG_IPIPE */
 	},
 };
 
@@ -302,6 +309,7 @@ void __init at91sam9rl_initialize(unsigned long main_clock)
  * The default interrupt priority levels (0 = lowest, 7 = highest).
  */
 static unsigned int at91sam9rl_default_irq_priority[NR_AIC_IRQS] __initdata = {
+#ifndef CONFIG_IPIPE
 	7,	/* Advanced Interrupt Controller */
 	7,	/* System Peripherals */
 	1,	/* Parallel IO Controller A */
@@ -334,6 +342,42 @@ static unsigned int at91sam9rl_default_irq_priority[NR_AIC_IRQS] __initdata = {
 	0,
 	0,
 	0,	/* Advanced Interrupt Controller */
+#else /* CONFIG_IPIPE */
+/* Give the highest priority to TC, since they are used as timer interrupt by
+   I-pipe. */
+	7,	/* Advanced Interrupt Controller */
+	6,	/* System Peripherals */
+	1,	/* Parallel IO Controller A */
+	1,	/* Parallel IO Controller B */
+	1,	/* Parallel IO Controller C */
+	1,	/* Parallel IO Controller D */
+	4,	/* USART 0 */
+	4,	/* USART 1 */
+	4,	/* USART 2 */
+	4,	/* USART 3 */
+	0,	/* Multimedia Card Interface */
+	5,	/* Two-Wire Interface 0 */
+	5,	/* Two-Wire Interface 1 */
+	4,	/* Serial Peripheral Interface */
+	3,	/* Serial Synchronous Controller 0 */
+	3,	/* Serial Synchronous Controller 1 */
+	7,	/* Timer Counter 0 */
+	7,	/* Timer Counter 1 */
+	7,	/* Timer Counter 2 */
+	0,
+	0,	/* Touch Screen Controller */
+	0,	/* DMA Controller */
+	2,	/* USB Device High speed port */
+	2,	/* LCD Controller */
+	5,	/* AC97 Controller */
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,	/* Advanced Interrupt Controller */
+#endif /*CONFIG_IPIPE */
 };
 
 void __init at91sam9rl_init_interrupts(unsigned int priority[NR_AIC_IRQS])
diff --git a/arch/arm/mach-at91/gpio.c b/arch/arm/mach-at91/gpio.c
index ee4964a..427d032 100644
--- a/arch/arm/mach-at91/gpio.c
+++ b/arch/arm/mach-at91/gpio.c
@@ -23,6 +23,11 @@
 #include <asm/hardware.h>
 #include <asm/arch/at91_pio.h>
 #include <asm/arch/gpio.h>
+#ifdef CONFIG_IPIPE
+#include <asm/irq.h>
+
+unsigned __ipipe_at91_gpio_banks = 0;
+#endif /* CONFIG_IPIPE */
 
 #include "generic.h"
 
@@ -370,6 +375,10 @@ static int gpio_irq_type(unsigned pin, unsigned type)
 
 static struct irq_chip gpio_irqchip = {
 	.name		= "GPIO",
+#ifdef CONFIG_IPIPE
+	.ack            = gpio_irq_mask,
+	.mask_ack       = gpio_irq_mask,
+#endif /* CONFIG_IPIPE */
 	.mask		= gpio_irq_mask,
 	.unmask		= gpio_irq_unmask,
 	.set_type	= gpio_irq_type,
@@ -428,6 +437,61 @@ static void gpio_irq_handler(unsigned irq, struct irq_desc *desc)
 	/* now it may re-trigger */
 }
 
+#ifdef CONFIG_IPIPE
+void __ipipe_mach_demux_irq(unsigned irq, struct pt_regs *regs)
+{
+	struct irq_desc *desc = &irq_desc[irq];
+	unsigned	pin;
+	struct irq_desc	*gpio;
+	struct at91_gpio_bank *bank;
+	void __iomem	*pio;
+	u32		isr;
+
+	bank = get_irq_chip_data(irq);
+	pio = bank->regbase;
+
+	/* temporarily mask (level sensitive) parent IRQ */
+	desc->chip->ack(irq);
+	for (;;) {
+		/* Reading ISR acks pending (edge triggered) GPIO interrupts.
+		 * When there none are pending, we're finished unless we need
+		 * to process multiple banks (like ID_PIOCDE on sam9263).
+		 */
+		isr = __raw_readl(pio + PIO_ISR) & __raw_readl(pio + PIO_IMR);
+		if (!isr) {
+			if (!bank->next)
+				break;
+			bank = bank->next;
+			pio = bank->regbase;
+			continue;
+		}
+
+		pin = bank->chipbase;
+		gpio = &irq_desc[pin];
+
+		while (isr) {
+			if (isr & 1) {
+				if (unlikely(gpio->depth)) {
+					/*
+					 * The core ARM interrupt handler lazily disables IRQs so
+					 * another IRQ must be generated before it actually gets
+					 * here to be disabled on the GPIO controller.
+					 */
+					gpio_irq_mask(pin);
+				}
+				else
+					__ipipe_handle_irq(pin, regs);
+			}
+			pin++;
+			gpio++;
+			isr >>= 1;
+		}
+	}
+	desc->chip->unmask(irq);
+	/* now it may re-trigger */
+}
+#endif /* CONFIG_IPIPE */
+
 /*--------------------------------------------------------------------------*/
 
 #ifdef CONFIG_DEBUG_FS
@@ -537,6 +601,9 @@ void __init at91_gpio_irq_setup(void)
 		set_irq_chained_handler(id, gpio_irq_handler);
 	}
 	pr_info("AT91: %d gpio irqs in %d banks\n", pin - PIN_BASE, gpio_banks);
+#ifdef CONFIG_IPIPE
+	__ipipe_at91_gpio_banks = gpio_banks;
+#endif /* CONFIG_IPIPE */
 }
 
 /*
diff --git a/arch/arm/mach-at91/irq.c b/arch/arm/mach-at91/irq.c
index 78a5cdb..32b59fb 100644
--- a/arch/arm/mach-at91/irq.c
+++ b/arch/arm/mach-at91/irq.c
@@ -122,6 +122,9 @@ static struct irq_chip at91_aic_chip = {
 	.name		= "AIC",
 	.ack		= at91_aic_mask_irq,
 	.mask		= at91_aic_mask_irq,
+#ifdef CONFIG_IPIPE
+	.mask_ack       = at91_aic_mask_irq,
+#endif /* CONFIG_IPIPE */
 	.unmask		= at91_aic_unmask_irq,
 	.set_type	= at91_aic_set_type,
 	.set_wake	= at91_aic_set_wake,
diff --git a/arch/arm/mach-imx/irq.c b/arch/arm/mach-imx/irq.c
index e6695c4..5fbc882 100644
--- a/arch/arm/mach-imx/irq.c
+++ b/arch/arm/mach-imx/irq.c
@@ -26,12 +26,87 @@
 #include <linux/init.h>
 #include <linux/list.h>
 #include <linux/timer.h>
+#ifdef CONFIG_IPIPE
+#include <linux/ipipe.h>
+#endif /* CONFIG_IPIPE */
 
 #include <asm/hardware.h>
 #include <asm/irq.h>
 #include <asm/io.h>
 
 #include <asm/mach/irq.h>
+#include <asm/arch/imx-regs.h>
+
+/* Used for IMX INTERRUPT priority: Still Experimental */ 
+#ifdef CONFIG_IPIPE 
+
+static unsigned int imx_default_irq_priority[IMX_IRQS] __initdata = {
+        0, /* unused */
+        0, /* unused */
+        0, /* unused */
+        0, /* unused */
+        0, /* unused */
+        0, /* unused */
+        5, /* Common Sensor Interface */
+        5, /* Multimedia Accelerator MAC */
+        5, /* Multimedia Accelerator */
+        0, /* unused */
+        5, /* MS */    
+        4, /* GPIO Port A */
+        4, /* GPIO Port B */
+        4, /* GPIO Port C */                 
+        5,  /* LCDC */   
+        0, /* unused */
+        0, /* unused */
+        7, /* RTC */
+        7, /* RTC */
+        5, /* UART 2 PFerr*/
+        5, /* UART 2 RTS */
+        5, /* UART 2 DTR */
+        5, /* UART 2 UARTC*/
+        5, /* UART 2 Tx*/
+        5, /* UART 2 Rx*/
+        5, /* UART 1 PFerr*/
+        5, /* UART 1 RTS */
+        5, /* UART 1 DTR */
+        5, /* UART 1 UARTC*/
+        5, /* UART 1 Tx*/
+        5, /* UART 1 Rx*/
+        0, /* unused */
+        0, /* unused */
+        0, /* unused */
+        5,/* PWM */
+        5, /* MMC */
+        0, /* unused */
+        0, /* unused */
+        0, /* unused */
+        5, /* I2C */
+        5, /*SPI 2 */
+        5, /*SPI 1 */
+        5, /* SSI Tx*/
+        5, /* SSI Tx Err*/
+        5, /* SSI Rx */
+        5, /* SSI Rx Err*/
+        0, /* unused */
+        5, /* USB0 */
+        5, /* USB1 */
+        5, /* USB2 */
+        5, /* USB3 */
+        5, /* USB4 */
+        5, /* USB5 */
+        5, /* USB6 */
+        0, /* unused */
+        0, /* unused */
+        0, /* unused */
+        0, /* unused */
+        5, /* Timer2 */
+        15, /* Timer1 This is use by IPIPE */
+        12, /* DMA Err */
+        12, /* DMA */
+        4, /* GPIO D */
+        12 /* Watch dog */        /* Advanced Interrupt Controller (IRQ6) */
+        };
+#endif 
 
 /*
  *
@@ -238,6 +313,9 @@ static struct irq_chip imx_internal_chip = {
 	.name = "MPU",
 	.ack = imx_mask_irq,
 	.mask = imx_mask_irq,
+#ifdef CONFIG_IPIPE
+	.mask_ack = imx_mask_irq,
+#endif /* CONFIG_IPIPE */
 	.unmask = imx_unmask_irq,
 };
 
@@ -249,6 +327,25 @@ static struct irq_chip imx_gpio_chip = {
 	.set_type = imx_gpio_irq_type,
 };
 
+#ifdef CONFIG_IPIPE 
+void __init imx_init_priority(void)
+{
+        unsigned int irq; 
+        IMX_PRIO0=0;   
+        IMX_PRIO1=0;     
+        IMX_PRIO2=0;
+        IMX_PRIO3=0;   
+        IMX_PRIO4=0;     
+        IMX_PRIO5=0;  
+        IMX_PRIO6=0;    
+        IMX_PRIO7=0; 
+        printk(KERN_INFO "Initializing imx interrupt priorities\n");
+        for (irq = 0; irq < IMX_IRQS; irq++) {
+	        IMX_PRIO(irq)|=((imx_default_irq_priority[irq]&15)<<((irq%8)*4));
+        }             
+}
+#endif
+
 void __init
 imx_init_irq(void)
 {
@@ -278,7 +375,9 @@ imx_init_irq(void)
 		set_irq_handler(irq, handle_edge_irq);
 		set_irq_flags(irq, IRQF_VALID);
 	}
-
+#ifdef CONFIG_IPIPE 
+	imx_init_priority(); 
+#endif 
 	set_irq_chained_handler(GPIO_INT_PORTA, imx_gpioa_demux_handler);
 	set_irq_chained_handler(GPIO_INT_PORTB, imx_gpiob_demux_handler);
 	set_irq_chained_handler(GPIO_INT_PORTC, imx_gpioc_demux_handler);
@@ -287,3 +386,18 @@ imx_init_irq(void)
 	/* Release masking of interrupts according to priority */
 	__raw_writel(-1, IMX_AITC_NIMASK);
 }
+
+#ifdef CONFIG_IPIPE
+void __ipipe_mach_demux_irq(unsigned irq, struct pt_regs *regs)
+{
+        unsigned int i, base, mask;
+
+        /* GPPIOA, GPIOB, GPIOC, GPIOD are INT 11,12,13 and 62 */
+	i = ((irq & 7) - 3);
+	base = (i << 5) + IMX_IRQS;
+
+	/* Get all multiplexed ITs from given GPIO */
+	while ((mask = ISR(i)))
+		__ipipe_handle_irq(base + __ffs(mask), regs);
+}
+#endif /* CONFIG_IPIPE */
diff --git a/arch/arm/mach-imx/time.c b/arch/arm/mach-imx/time.c
index d86d124..bd7027f 100644
--- a/arch/arm/mach-imx/time.c
+++ b/arch/arm/mach-imx/time.c
@@ -24,6 +24,57 @@
 #include <asm/irq.h>
 #include <asm/mach/time.h>
 
+#ifdef CONFIG_IPIPE
+#ifdef CONFIG_NO_IDLE_HZ
+#error "dynamic tick timer not yet supported with IPIPE"
+#endif				/* CONFIG_NO_IDLE_HZ */
+int __ipipe_mach_timerint = TIM1_INT;
+EXPORT_SYMBOL(__ipipe_mach_timerint);
+
+int __ipipe_mach_timerstolen = 0;
+EXPORT_SYMBOL(__ipipe_mach_timerstolen);
+
+unsigned int __ipipe_mach_ticks_per_jiffy = LATCH;
+EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy);
+
+static int imx_timer_initialized;
+union tsc_reg {
+#ifdef __BIG_ENDIAN
+	struct {
+		unsigned long high;
+		unsigned long low;
+	};
+#else				/* __LITTLE_ENDIAN */
+	struct {
+		unsigned long low;
+		unsigned long high;
+	};
+#endif				/* __LITTLE_ENDIAN */
+	unsigned long long full;
+};
+#ifdef CONFIG_SMP
+static union tsc_reg tsc[NR_CPUS];
+
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info)
+{
+	info->type = IPIPE_TSC_TYPE_NONE;
+}
+#else				/* !CONFIG_SMP */
+static union tsc_reg *tsc;
+
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info)
+{
+	info->type = IPIPE_TSC_TYPE_FREERUNNING;
+	info->u.fr.counter = (unsigned *)(0x10 + IMX_TIM1_BASE);
+	info->u.fr.mask = 0xffffffff;
+	info->u.fr.tsc = &tsc->full;
+}
+#endif				/* !CONFIG_SMP */
+
+static void ipipe_mach_update_tsc(void);
+
+#endif				/* CONFIG_IPIPE */
+
 /* Use timer 1 as system timer */
 #define TIMER_BASE IMX_TIM1_BASE
 
@@ -37,6 +88,7 @@ static irqreturn_t
 imx_timer_interrupt(int irq, void *dev_id)
 {
 	struct clock_event_device *evt = &clockevent_imx;
+#ifndef CONFIG_IPIPE
 	uint32_t tstat;
 	irqreturn_t ret = IRQ_NONE;
 
@@ -50,6 +102,11 @@ imx_timer_interrupt(int irq, void *dev_id)
 	}
 
 	return ret;
+#else /* CONFIG_IPIPE */
+	ipipe_mach_update_tsc();
+	evt->event_handler(evt);
+	return  IRQ_HANDLED;
+#endif /* CONFIG_IPIPE */
 }
 
 static struct irqaction imx_timer_irq = {
@@ -123,7 +180,7 @@ static void imx_set_mode(enum clock_event_mode mode, struct clock_event_device *
 	 * The timer interrupt generation is disabled at least
 	 * for enough time to call imx_set_next_event()
 	 */
-	local_irq_save(flags);
+	local_irq_save_hw(flags);
 	/* Disable interrupt in GPT module */
 	IMX_TCTL(TIMER_BASE) &= ~TCTL_IRQEN;
 	if (mode != clockevent_mode) {
@@ -140,7 +197,7 @@ static void imx_set_mode(enum clock_event_mode mode, struct clock_event_device *
 
 	/* Remember timer mode */
 	clockevent_mode = mode;
-	local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	switch (mode) {
 	case CLOCK_EVT_MODE_PERIODIC:
@@ -153,9 +210,9 @@ static void imx_set_mode(enum clock_event_mode mode, struct clock_event_device *
 		 * to call imx_set_next_event() or shutdown clock after
 		 * mode switching
 		 */
-		local_irq_save(flags);
+		local_irq_save_hw(flags);
 		IMX_TCTL(TIMER_BASE) |= TCTL_IRQEN;
-		local_irq_restore(flags);
+		local_irq_restore_hw(flags);
 		break;
 	case CLOCK_EVT_MODE_SHUTDOWN:
 	case CLOCK_EVT_MODE_UNUSED:
@@ -165,6 +222,13 @@ static void imx_set_mode(enum clock_event_mode mode, struct clock_event_device *
 	}
 }
 
+#ifdef CONFIG_IPIPE
+int __ipipe_check_tickdev(const char *devname)
+{
+	return !strcmp(devname, clockevent_imx.name);
+}
+#endif /* CONFIG_IPIPE */
+
 static struct clock_event_device clockevent_imx = {
 	.name		= "imx_timer1",
 	.features	= CLOCK_EVT_FEAT_ONESHOT,
@@ -198,6 +262,14 @@ static void __init imx_timer_init(void)
 
 	imx_clockevent_init();
 
+#ifdef CONFIG_IPIPE
+#ifndef CONFIG_SMP
+	tsc = (union tsc_reg *)__ipipe_tsc_area;
+	barrier();
+#endif				/* CONFIG_SMP */
+	imx_timer_initialized = 1;
+
+#endif				/* CONFIG_IPIPE */
 	/*
 	 * Make irqs happen for the system timer
 	 */
@@ -207,3 +279,75 @@ static void __init imx_timer_init(void)
 struct sys_timer imx_timer = {
 	.init		= imx_timer_init,
 };
+
+#ifdef CONFIG_IPIPE
+void __ipipe_mach_acktimer(void)
+{
+	IMX_TSTAT(TIMER_BASE) = 0;
+}
+
+static void ipipe_mach_update_tsc(void)
+{
+	union tsc_reg *local_tsc;
+	unsigned long stamp, flags;
+
+	local_irq_save_hw(flags);
+	local_tsc = &tsc[ipipe_processor_id()];
+	stamp = IMX_TCN(TIMER_BASE);
+	if (unlikely(stamp < local_tsc->low))
+		/* 32 bit counter wrapped, increment high word. */
+		local_tsc->high++;
+	local_tsc->low = stamp;
+	local_irq_restore_hw(flags);
+}
+
+notrace unsigned long long __ipipe_mach_get_tsc(void)
+{
+	if (likely(imx_timer_initialized)) {
+		union tsc_reg *local_tsc, result;
+		unsigned long stamp;
+
+		local_tsc = &tsc[ipipe_processor_id()];
+
+		__asm__("ldmia %1, %M0\n":
+			"=r"(result.full): "r"(local_tsc), "m"(*local_tsc));
+		barrier();
+		stamp = IMX_TCN(TIMER_BASE);
+		if (unlikely(stamp < result.low))
+			result.high++;
+		result.low = stamp;
+		return result.full;
+	}
+	return 0;
+}
+
+EXPORT_SYMBOL(__ipipe_mach_get_tsc);
+
+/*
+ * Reprogram the timer
+ */
+void __ipipe_mach_set_dec(unsigned long delay)
+{
+	unsigned long flags;
+	if (delay > 8) {
+		local_irq_save_hw(flags);
+		IMX_TCMP(TIMER_BASE) = IMX_TCN(TIMER_BASE) + delay;
+		local_irq_restore_hw(flags);
+	} else
+		ipipe_trigger_irq(TIM1_INT);
+}
+
+EXPORT_SYMBOL(__ipipe_mach_set_dec);
+
+void __ipipe_mach_release_timer(void)
+{
+	__ipipe_mach_set_dec(__ipipe_mach_ticks_per_jiffy);
+}
+
+EXPORT_SYMBOL(__ipipe_mach_release_timer);
+
+unsigned long __ipipe_mach_get_dec(void)
+{
+	return IMX_TCMP(TIMER_BASE) - IMX_TCN(TIMER_BASE);
+}
+#endif				/* CONFIG_IPIPE */
diff --git a/arch/arm/mach-integrator/core.c b/arch/arm/mach-integrator/core.c
index 7fbbc17..f9366c6 100644
--- a/arch/arm/mach-integrator/core.c
+++ b/arch/arm/mach-integrator/core.c
@@ -2,6 +2,7 @@
  *  linux/arch/arm/mach-integrator/core.c
  *
  *  Copyright (C) 2000-2003 Deep Blue Solutions Ltd
+ *  Copyright (C) 2005 Stelian Pop.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2, as
@@ -195,53 +196,62 @@ EXPORT_SYMBOL(cm_control);
 /*
  * How long is the timer interval?
  */
-#define TIMER_INTERVAL	(TICKS_PER_uSEC * mSEC_10)
-#if TIMER_INTERVAL >= 0x100000
-#define TICKS2USECS(x)	(256 * (x) / TICKS_PER_uSEC)
-#elif TIMER_INTERVAL >= 0x10000
-#define TICKS2USECS(x)	(16 * (x) / TICKS_PER_uSEC)
-#else
 #define TICKS2USECS(x)	((x) / TICKS_PER_uSEC)
-#endif
 
 static unsigned long timer_reload;
+static unsigned long timer_interval;
+static unsigned long timer_lxlost;
+static int tscok;
+
+#ifdef CONFIG_IPIPE
+int __ipipe_mach_timerint = IRQ_TIMERINT1;
+static unsigned long long __ipipe_mach_tsc;
+static IPIPE_DEFINE_SPINLOCK(timer_lock);
+
+int __ipipe_mach_timerstolen = 0;
+EXPORT_SYMBOL(__ipipe_mach_timerstolen);
+
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info)
+{
+	info->type = IPIPE_TSC_TYPE_NONE;
+}
+#endif
 
 /*
- * Returns number of ms since last clock interrupt.  Note that interrupts
- * will have been disabled by do_gettimeoffset()
+ * Called with IRQ disabled from do_gettimeofday().
  */
-unsigned long integrator_gettimeoffset(void)
+static inline unsigned long integrator_getticksoffset(void)
 {
-	unsigned long ticks1, ticks2, status;
+	unsigned long ticks;
 
-	/*
-	 * Get the current number of ticks.  Note that there is a race
-	 * condition between us reading the timer and checking for
-	 * an interrupt.  We get around this by ensuring that the
-	 * counter has not reloaded between our two reads.
-	 */
-	ticks2 = readl(TIMER1_VA_BASE + TIMER_VALUE) & 0xffff;
-	do {
-		ticks1 = ticks2;
-		status = __raw_readl(VA_IC_BASE + IRQ_RAW_STATUS);
-		ticks2 = readl(TIMER1_VA_BASE + TIMER_VALUE) & 0xffff;
-	} while (ticks2 > ticks1);
+	if (!tscok)
+		return 0;
 
-	/*
-	 * Number of ticks since last interrupt.
-	 */
-	ticks1 = timer_reload - ticks2;
+	ticks = readl(TIMER1_VA_BASE + TIMER_VALUE) & 0xffff;
 
-	/*
-	 * Interrupt pending?  If so, we've reloaded once already.
-	 */
-	if (status & (1 << IRQ_TIMERINT1))
-		ticks1 += timer_reload;
+	if (ticks > timer_reload)
+		ticks = 0xffff + timer_reload - ticks;
+	else
+		ticks = timer_reload - ticks;
 
+	if (timer_interval < 0x10000)
+		return ticks;
+	else if (timer_interval < 0x100000)
+		return ticks * 16;
+	else
+		return ticks * 256;
+}
+
+/*
+ * Returns number of ms since last clock interrupt.  Note that interrupts
+ * will have been disabled by do_gettimeoffset()
+ */
+unsigned long integrator_gettimeoffset(void)
+{
 	/*
 	 * Convert the ticks to usecs
 	 */
-	return TICKS2USECS(ticks1);
+	return TICKS2USECS(timer_lxlost + integrator_getticksoffset());
 }
 
 /*
@@ -250,11 +260,22 @@ unsigned long integrator_gettimeoffset(void)
 static irqreturn_t
 integrator_timer_interrupt(int irq, void *dev_id)
 {
+	timer_lxlost = 0;
+
+#ifdef CONFIG_IPIPE
 	/*
-	 * clear the interrupt
+	 * If Linux is the only domain, ack the timer and reprogram it
 	 */
-	writel(1, TIMER1_VA_BASE + TIMER_INTCLR);
+	if (!__ipipe_mach_timerstolen) {
+		__ipipe_mach_tsc += integrator_getticksoffset();
+#else
+		writel(1, TIMER1_VA_BASE + TIMER_INTCLR);
+#endif
 
+		writel(timer_reload, TIMER1_VA_BASE + TIMER_LOAD);
+#ifdef CONFIG_IPIPE
+	}
+#endif
 	timer_tick();
 
 	return IRQ_HANDLED;
@@ -266,24 +287,30 @@ static struct irqaction integrator_timer_irq = {
 	.handler	= integrator_timer_interrupt,
 };
 
-/*
- * Set up timer interrupt, and return the current time in seconds.
- */
-void __init integrator_time_init(unsigned long reload, unsigned int ctrl)
+static inline void set_dec(unsigned long reload)
 {
-	unsigned int timer_ctrl = TIMER_CTRL_ENABLE | TIMER_CTRL_PERIODIC;
+	unsigned int ctrl = TIMER_CTRL_ENABLE | TIMER_CTRL_IE;
 
 	timer_reload = reload;
-	timer_ctrl |= ctrl;
+	timer_interval = reload;
 
-	if (timer_reload > 0x100000) {
+	if (timer_reload >= 0x100000) {
 		timer_reload >>= 8;
-		timer_ctrl |= TIMER_CTRL_DIV256;
-	} else if (timer_reload > 0x010000) {
+		ctrl |= TIMER_CTRL_DIV256;
+	} else if (timer_reload >= 0x010000) {
 		timer_reload >>= 4;
-		timer_ctrl |= TIMER_CTRL_DIV16;
+		ctrl |= TIMER_CTRL_DIV16;
 	}
 
+	writel(ctrl, TIMER1_VA_BASE + TIMER_CTRL);
+	writel(timer_reload, TIMER1_VA_BASE + TIMER_LOAD);
+}
+
+/*
+ * Set up timer interrupt, and return the current time in seconds.
+ */
+void __init integrator_time_init(unsigned long reload, unsigned int ctrl)
+{
 	/*
 	 * Initialise to a known state (all timers off)
 	 */
@@ -291,12 +318,60 @@ void __init integrator_time_init(unsigned long reload, unsigned int ctrl)
 	writel(0, TIMER1_VA_BASE + TIMER_CTRL);
 	writel(0, TIMER2_VA_BASE + TIMER_CTRL);
 
-	writel(timer_reload, TIMER1_VA_BASE + TIMER_LOAD);
-	writel(timer_reload, TIMER1_VA_BASE + TIMER_VALUE);
-	writel(timer_ctrl, TIMER1_VA_BASE + TIMER_CTRL);
+	set_dec(reload);
 
 	/*
 	 * Make irqs happen for the system timer
 	 */
 	setup_irq(IRQ_TIMERINT1, &integrator_timer_irq);
+
+	tscok = 1;
+}
+
+#ifdef CONFIG_IPIPE
+
+void __ipipe_mach_acktimer(void)
+{
+	writel(1, TIMER1_VA_BASE + TIMER_INTCLR);
+}
+
+notrace unsigned long long __ipipe_mach_get_tsc(void)
+{
+	unsigned long long result;
+	unsigned long flags;
+
+	local_irq_save_hw_notrace(flags);
+	spin_lock(&timer_lock);
+	result = __ipipe_mach_tsc + integrator_getticksoffset();
+	spin_unlock(&timer_lock);
+	local_irq_restore_hw_notrace(flags);
+	return result;
+}
+EXPORT_SYMBOL(__ipipe_mach_get_tsc);
+
+void __ipipe_mach_set_dec(unsigned long reload)
+{
+	unsigned long ticks;
+	unsigned long flags;
+
+	spin_lock_irqsave(&timer_lock, flags);
+	ticks = integrator_getticksoffset();
+	__ipipe_mach_tsc += ticks;
+	timer_lxlost += ticks;
+
+	set_dec(reload);
+	spin_unlock_irqrestore(&timer_lock, flags);
+}
+EXPORT_SYMBOL(__ipipe_mach_set_dec);
+
+void __ipipe_mach_release_timer(void)
+{
+       __ipipe_mach_set_dec(__ipipe_mach_ticks_per_jiffy);
+}
+EXPORT_SYMBOL(__ipipe_mach_release_timer);
+
+unsigned long __ipipe_mach_get_dec(void)
+{
+	return readl(TIMER1_VA_BASE + TIMER_VALUE);
 }
+#endif /* CONFIG_IPIPE */
diff --git a/arch/arm/mach-integrator/integrator_cp.c b/arch/arm/mach-integrator/integrator_cp.c
index 913f64b..8a01e22 100644
--- a/arch/arm/mach-integrator/integrator_cp.c
+++ b/arch/arm/mach-integrator/integrator_cp.c
@@ -2,6 +2,7 @@
  *  linux/arch/arm/mach-integrator/integrator_cp.c
  *
  *  Copyright (C) 2003 Deep Blue Solutions Ltd
+ *  Copyright (C) 2005 Stelian Pop.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -19,6 +20,7 @@
 #include <linux/amba/bus.h>
 #include <linux/amba/kmi.h>
 #include <linux/amba/clcd.h>
+#include <linux/ipipe.h>
 
 #include <asm/hardware.h>
 #include <asm/io.h>
@@ -160,6 +162,9 @@ static struct irq_chip cic_chip = {
 	.name	= "CIC",
 	.ack	= cic_mask_irq,
 	.mask	= cic_mask_irq,
+#ifdef CONFIG_IPIPE
+	.mask_ack = cic_mask_irq,
+#endif /* CONFIG_IPIPE */
 	.unmask	= cic_unmask_irq,
 };
 
@@ -179,6 +184,9 @@ static struct irq_chip pic_chip = {
 	.name	= "PIC",
 	.ack	= pic_mask_irq,
 	.mask	= pic_mask_irq,
+#ifdef CONFIG_IPIPE
+	.mask_ack = pic_mask_irq,
+#endif /* CONFIG_IPIPE */
 	.unmask = pic_unmask_irq,
 };
 
@@ -198,6 +206,9 @@ static struct irq_chip sic_chip = {
 	.name	= "SIC",
 	.ack	= sic_mask_irq,
 	.mask	= sic_mask_irq,
+#ifdef CONFIG_IPIPE
+	.mask_ack = sic_mask_irq,
+#endif /* CONFIG_IPIPE */
 	.unmask	= sic_unmask_irq,
 };
 
@@ -222,6 +233,30 @@ sic_handle_irq(unsigned int irq, struct irq_desc *desc)
 	} while (status);
 }
 
+#ifdef CONFIG_IPIPE
+void __ipipe_mach_demux_irq(unsigned irq, struct pt_regs *regs)
+{
+	unsigned long status = sic_readl(INTCP_VA_SIC_BASE + IRQ_STATUS);
+	struct irq_desc *desc_unused = irq_desc + irq;
+
+	if (status == 0) {
+		do_bad_IRQ(irq, desc_unused);
+		return;
+	}
+
+	do {
+		irq = ffs(status) - 1;
+		status &= ~(1 << irq);
+
+		irq += IRQ_SIC_START;
+
+		__ipipe_handle_irq(irq, regs);
+
+		status = sic_readl(INTCP_VA_SIC_BASE + IRQ_STATUS);
+	} while (status);
+}
+#endif /* CONFIG_IPIPE */
+
 static void __init intcp_init_irq(void)
 {
 	unsigned int i;
@@ -568,9 +603,14 @@ static void __init intcp_init(void)
 
 #define TIMER_CTRL_IE	(1 << 5)			/* Interrupt Enable */
 
+#ifdef CONFIG_IPIPE
+unsigned int __ipipe_mach_ticks_per_jiffy = 1000000 * TICKS_PER_uSEC / HZ;
+EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy);
+#endif
+
 static void __init intcp_timer_init(void)
 {
-	integrator_time_init(1000000 / HZ, TIMER_CTRL_IE);
+	integrator_time_init(1000000 * TICKS_PER_uSEC / HZ, TIMER_CTRL_IE);
 }
 
 static struct sys_timer cp_timer = {
diff --git a/arch/arm/mach-ixp4xx/common.c b/arch/arm/mach-ixp4xx/common.c
index f6d66dc..988b86c 100644
--- a/arch/arm/mach-ixp4xx/common.c
+++ b/arch/arm/mach-ixp4xx/common.c
@@ -45,6 +45,68 @@ static int __init ixp4xx_clocksource_init(void);
 static int __init ixp4xx_clockevent_init(void);
 static struct clock_event_device clockevent_ixp4xx;
 
+#ifdef CONFIG_IPIPE
+#include <linux/ipipe.h>
+
+/* We have no cascaded interrupts. */
+void __ipipe_mach_demux_irq(unsigned irq, struct pt_regs *regs) {}
+
+#ifdef CONFIG_NO_IDLE_HZ
+#error "dynamic tick timer not yet supported with IPIPE"
+#endif
+
+int __ipipe_mach_timerint = IRQ_IXP4XX_TIMER1;
+int __ipipe_mach_timerstolen = 0;
+unsigned int __ipipe_mach_ticks_per_jiffy = LATCH;
+EXPORT_SYMBOL(__ipipe_mach_timerint);
+EXPORT_SYMBOL(__ipipe_mach_timerstolen);
+EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy);
+
+static int ixp4xx_timer_initialized;
+
+#define ONE_SHOT_ENABLE (IXP4XX_OST_ENABLE|IXP4XX_OST_ONE_SHOT)
+
+union tsc_reg {
+#ifdef __BIG_ENDIAN
+	struct {
+		unsigned long high;
+		unsigned long low;
+	};
+#else /* __LITTLE_ENDIAN */
+	struct {
+		unsigned long low;
+		unsigned long high;
+	};
+#endif /* __LITTLE_ENDIAN */
+	unsigned long long full;
+};
+
+#ifdef CONFIG_SMP
+static union tsc_reg tsc[NR_CPUS];
+
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info)
+{
+	info->type = IPIPE_TSC_TYPE_NONE;
+}
+
+#else /* !CONFIG_SMP */
+static union tsc_reg *tsc;
+
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info)
+{
+	info->type = IPIPE_TSC_TYPE_FREERUNNING;
+	info->u.fr.counter =
+		(unsigned *)
+		(IXP4XX_TIMER_BASE_PHYS + IXP4XX_OSTS_OFFSET);
+	info->u.fr.mask = 0xffffffff;
+	info->u.fr.tsc = &tsc->full;
+}
+#endif /* !CONFIG_SMP */
+
+static void ipipe_mach_update_tsc(void);
+
+#endif /* CONFIG_IPIPE */
+
 /*************************************************************************
  * IXP4xx chipset I/O mapping
  *************************************************************************/
@@ -269,8 +331,12 @@ static irqreturn_t ixp4xx_timer_interrupt(int irq, void *dev_id)
 {
 	struct clock_event_device *evt = &clockevent_ixp4xx;
 
+#ifndef CONFIG_IPIPE
 	/* Clear Pending Interrupt by writing '1' to it */
 	*IXP4XX_OSST = IXP4XX_OSST_TIMER_1_PEND;
+#else /* CONFIG_IPIPE */
+	ipipe_mach_update_tsc();
+#endif /* CONFIG_IPIPE */
 
 	evt->event_handler(evt);
 
@@ -294,6 +360,14 @@ void __init ixp4xx_timer_init(void)
 	/* Reset time-stamp counter */
 	*IXP4XX_OSTS = 0;
 
+#ifdef CONFIG_IPIPE
+#ifndef CONFIG_SMP
+	tsc = (union tsc_reg *) __ipipe_tsc_area;
+	barrier();
+#endif /* CONFIG_SMP */
+
+	ixp4xx_timer_initialized = 1;
+#endif
 	/* Connect the interrupt handler and enable the interrupt */
 	setup_irq(IRQ_IXP4XX_TIMER1, &ixp4xx_timer_irq);
 
@@ -470,6 +544,13 @@ static void ixp4xx_set_mode(enum clock_event_mode mode,
 	*IXP4XX_OSRT1 = osrt | opts;
 }
 
+#ifdef CONFIG_IPIPE
+int __ipipe_check_tickdev(const char *devname)
+{
+	return !strcmp(devname, clockevent_ixp4xx.name);
+}
+#endif /* CONFIG_IPIPE */
+
 static struct clock_event_device clockevent_ixp4xx = {
 	.name		= "ixp4xx timer1",
 	.features       = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT,
@@ -492,3 +573,98 @@ static int __init ixp4xx_clockevent_init(void)
 	clockevents_register_device(&clockevent_ixp4xx);
 	return 0;
 }
+
+#ifdef CONFIG_IPIPE
+void __ipipe_mach_acktimer(void)
+{
+	/* Clear Pending Interrupt by writing '1' to it */
+	*IXP4XX_OSST = IXP4XX_OSST_TIMER_1_PEND;
+}
+
+static void ipipe_mach_update_tsc(void)
+{
+	union tsc_reg *local_tsc;
+	unsigned long stamp, flags;
+
+	local_irq_save_hw(flags);
+	local_tsc = &tsc[ipipe_processor_id()];
+	stamp = *IXP4XX_OSTS;
+	if (unlikely(stamp < local_tsc->low))
+		/* 32 bit counter wrapped, increment high word. */
+		local_tsc->high++;
+	local_tsc->low = stamp;
+	local_irq_restore_hw(flags);
+}
+
+notrace unsigned long long __ipipe_mach_get_tsc(void)
+{
+	if (likely(ixp4xx_timer_initialized)) {
+		union tsc_reg *local_tsc, result;
+		unsigned long stamp;
+
+		local_tsc = &tsc[ipipe_processor_id()];
+
+		__asm__ ("ldmia %1, %M0\n":
+			 "=r"(result.full): "r"(local_tsc), "m"(*local_tsc));
+		barrier();
+		stamp = *IXP4XX_OSTS;
+		if (unlikely(stamp < result.low))
+			/* 32 bit counter wrapped, increment high word. */
+			result.high++;
+		result.low = stamp;
+		return result.full;
+	}
+
+        return 0;
+}
+
+EXPORT_SYMBOL(__ipipe_mach_get_tsc);
+
+/*
+ * Reprogram the timer
+ *
+ * The timer is aperiodic (most of the time) when running Xenomai, so
+ * __ipipe_mach_set_dec is called for each timer tick and programs the
+ * timer hardware for the next tick.
+ *
+ */
+#define MIN_DELAY 333 /* 5 usec with the 66.66 MHz system clock */
+
+void __ipipe_mach_set_dec(unsigned long delay)
+{
+	unsigned long flags;
+	if (delay > MIN_DELAY) {
+		local_irq_save_hw(flags);
+		*IXP4XX_OSRT1 = delay | ONE_SHOT_ENABLE;
+		local_irq_restore_hw(flags);
+	} else {
+		ipipe_trigger_irq(IRQ_IXP4XX_TIMER1);
+	}
+}
+
+EXPORT_SYMBOL(__ipipe_mach_set_dec);
+
+/*
+ * This returns the number of clock ticks remaining.
+ */
+unsigned long __ipipe_mach_get_dec(void)
+{
+	return(*IXP4XX_OST1); /* remaining */
+}
+
+EXPORT_SYMBOL(__ipipe_mach_get_dec);
+
+void __ipipe_mach_release_timer(void)
+{
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
+	ixp4xx_set_mode(clockevent_ixp4xx.mode, &clockevent_ixp4xx);
+	if (clockevent_ixp4xx.mode == CLOCK_EVT_MODE_ONESHOT)
+		ixp4xx_set_next_event(LATCH, &clockevent_ixp4xx);
+	local_irq_restore_hw(flags);
+}
+
+EXPORT_SYMBOL(__ipipe_mach_release_timer);
+
+#endif /* CONFIG_IPIPE */
diff --git a/arch/arm/mach-pxa/gpio.c b/arch/arm/mach-pxa/gpio.c
index 7d3e169..d91374f 100644
--- a/arch/arm/mach-pxa/gpio.c
+++ b/arch/arm/mach-pxa/gpio.c
@@ -16,6 +16,7 @@
 #include <linux/module.h>
 #include <linux/irq.h>
 #include <linux/sysdev.h>
+#include <linux/ipipe.h>
 
 #include <asm/gpio.h>
 #include <asm/hardware.h>
@@ -282,6 +283,35 @@ static void pxa_gpio_demux_handler(unsigned int irq, struct irq_desc *desc)
 	} while (loop);
 }
 
+#ifdef CONFIG_IPIPE
+void __ipipe_mach_demux_irq(unsigned irq, struct pt_regs *regs)
+{
+	int loop, bit, n;
+	unsigned long gedr[4];
+
+	do {
+		gedr[0] = GEDR0 & GPIO_IRQ_mask[0] & ~3;
+		gedr[1] = GEDR1 & GPIO_IRQ_mask[1];
+		gedr[2] = GEDR2 & GPIO_IRQ_mask[2];
+		gedr[3] = GEDR3 & GPIO_IRQ_mask[3];
+
+		GEDR0 = gedr[0]; GEDR1 = gedr[1];
+		GEDR2 = gedr[2]; GEDR3 = gedr[3];
+
+		loop = 0;
+		bit = find_first_bit(gedr, GEDR_BITS);
+		while (bit < GEDR_BITS) {
+			loop = 1;
+
+			n = PXA_GPIO_IRQ_BASE + bit;
+			__ipipe_handle_irq(n, regs);
+
+			bit = find_next_bit(gedr, GEDR_BITS, bit + 1);
+		}
+	} while (loop);
+}
+#endif /* CONFIG_IPIPE */
+
 static void pxa_ack_muxed_gpio(unsigned int irq)
 {
 	int gpio = irq - IRQ_GPIO(2) + 2;
diff --git a/arch/arm/mach-pxa/irq.c b/arch/arm/mach-pxa/irq.c
index a9a0c3f..b2b42ed 100644
--- a/arch/arm/mach-pxa/irq.c
+++ b/arch/arm/mach-pxa/irq.c
@@ -49,6 +49,9 @@ static struct irq_chip pxa_internal_irq_chip = {
 	.name		= "SC",
 	.ack		= pxa_mask_irq,
 	.mask		= pxa_mask_irq,
+#ifdef CONFIG_IPIPE
+	.mask_ack	= pxa_mask_irq,
+#endif /* CONFIG_IPIPE */
 	.unmask		= pxa_unmask_irq,
 };
 
diff --git a/arch/arm/mach-pxa/leds-idp.c b/arch/arm/mach-pxa/leds-idp.c
index 38aa927..1c4a9fe 100644
--- a/arch/arm/mach-pxa/leds-idp.c
+++ b/arch/arm/mach-pxa/leds-idp.c
@@ -13,6 +13,7 @@
 
 
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <asm/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-pxa/leds-lubbock.c b/arch/arm/mach-pxa/leds-lubbock.c
index afbc669..d9e87de 100644
--- a/arch/arm/mach-pxa/leds-lubbock.c
+++ b/arch/arm/mach-pxa/leds-lubbock.c
@@ -12,6 +12,7 @@
  */
 
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <asm/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-pxa/leds-mainstone.c b/arch/arm/mach-pxa/leds-mainstone.c
index 065293e..28b2cab 100644
--- a/arch/arm/mach-pxa/leds-mainstone.c
+++ b/arch/arm/mach-pxa/leds-mainstone.c
@@ -11,6 +11,7 @@
  */
 
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <asm/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-pxa/leds-trizeps4.c b/arch/arm/mach-pxa/leds-trizeps4.c
index 21880da..a2c6aad 100644
--- a/arch/arm/mach-pxa/leds-trizeps4.c
+++ b/arch/arm/mach-pxa/leds-trizeps4.c
@@ -11,6 +11,7 @@
  */
 
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <asm/hardware.h>
 #include <asm/system.h>
diff --git a/arch/arm/mach-pxa/time.c b/arch/arm/mach-pxa/time.c
index 7b7c017..91fd151 100644
--- a/arch/arm/mach-pxa/time.c
+++ b/arch/arm/mach-pxa/time.c
@@ -25,6 +25,60 @@
 #include <asm/arch/pxa-regs.h>
 #include <asm/mach-types.h>
 
+#ifdef CONFIG_IPIPE
+#ifdef CONFIG_NO_IDLE_HZ
+#error "dynamic tick timer not yet supported with IPIPE"
+#endif /* CONFIG_NO_IDLE_HZ */
+int __ipipe_mach_timerint = IRQ_OST0;
+EXPORT_SYMBOL(__ipipe_mach_timerint);
+
+int __ipipe_mach_timerstolen = 0;
+EXPORT_SYMBOL(__ipipe_mach_timerstolen);
+
+unsigned int __ipipe_mach_ticks_per_jiffy = LATCH;
+EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy);
+
+static int pxa_timer_initialized;
+
+union tsc_reg {
+#ifdef __BIG_ENDIAN
+	struct {
+		unsigned long high;
+		unsigned long low;
+	};
+#else /* __LITTLE_ENDIAN */
+	struct {
+		unsigned long low;
+		unsigned long high;
+	};
+#endif /* __LITTLE_ENDIAN */
+	unsigned long long full;
+};
+
+#ifdef CONFIG_SMP
+static union tsc_reg tsc[NR_CPUS];
+
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info)
+{
+	info->type = IPIPE_TSC_TYPE_NONE;
+}
+
+#else /* !CONFIG_SMP */
+static union tsc_reg *tsc;
+
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info)
+{
+	info->type = IPIPE_TSC_TYPE_FREERUNNING;
+	info->u.fr.counter = (unsigned *) 0x40A00010;
+	info->u.fr.mask = 0xffffffff;
+	info->u.fr.tsc = &tsc->full;
+}
+#endif /* !CONFIG_SMP */
+
+static void ipipe_mach_update_tsc(void);
+
+#endif /* CONFIG_IPIPE */
+
 /*
  * This is PXA's sched_clock implementation. This has a resolution
  * of at least 308 ns and a maximum value of 208 days.
@@ -67,8 +121,12 @@ pxa_ost0_interrupt(int irq, void *dev_id)
 	struct clock_event_device *c = dev_id;
 
 	/* Disarm the compare/match, signal the event. */
+#ifndef CONFIG_IPIPE
 	OIER &= ~OIER_E0;
 	OSSR = OSSR_M0;
+#else /* CONFIG_IPIPE */
+	ipipe_mach_update_tsc();
+#endif /* CONFIG_IPIPE */
 	c->event_handler(c);
 
 	return IRQ_HANDLED;
@@ -80,8 +138,8 @@ pxa_osmr0_set_next_event(unsigned long delta, struct clock_event_device *dev)
 	unsigned long flags, next, oscr;
 
 	raw_local_irq_save(flags);
-	OIER |= OIER_E0;
 	next = OSCR + delta;
+	OIER |= OIER_E0;
 	OSMR0 = next;
 	oscr = OSCR;
 	raw_local_irq_restore(flags);
@@ -127,6 +185,13 @@ static struct clock_event_device ckevt_pxa_osmr0 = {
 	.set_mode	= pxa_osmr0_set_mode,
 };
 
+#ifdef CONFIG_IPIPE
+int __ipipe_check_tickdev(const char *devname)
+{
+	return !strcmp(devname, ckevt_pxa_osmr0.name);
+}
+#endif /* CONFIG_IPIPE */
+
 static cycle_t pxa_read_oscr(void)
 {
 	return OSCR;
@@ -176,6 +241,15 @@ static void __init pxa_timer_init(void)
 
 	setup_irq(IRQ_OST0, &pxa_ost0_irq);
 
+#ifdef CONFIG_IPIPE
+#ifndef CONFIG_SMP
+	tsc = (union tsc_reg *) __ipipe_tsc_area;
+	barrier();
+#endif /* CONFIG_SMP */
+
+	pxa_timer_initialized = 1;
+#endif /* CONFIG_IPIPE */
+
 	clocksource_register(&cksrc_pxa_oscr0);
 	clockevents_register_device(&ckevt_pxa_osmr0);
 }
@@ -221,3 +295,81 @@ struct sys_timer pxa_timer = {
 	.suspend	= pxa_timer_suspend,
 	.resume		= pxa_timer_resume,
 };
+
+#ifdef CONFIG_IPIPE
+void __ipipe_mach_acktimer(void)
+{
+	OSSR = OSSR_M0;  /* Clear match on timer 0 */
+	OIER &= ~OIER_E0;
+}
+
+static void ipipe_mach_update_tsc(void)
+{
+	union tsc_reg *local_tsc;
+	unsigned long stamp, flags;
+
+	local_irq_save_hw(flags);
+	local_tsc = &tsc[ipipe_processor_id()];
+	stamp = OSCR;
+	if (unlikely(stamp < local_tsc->low))
+		/* 32 bit counter wrapped, increment high word. */
+		local_tsc->high++;
+	local_tsc->low = stamp;
+	local_irq_restore_hw(flags);
+}
+
+notrace unsigned long long __ipipe_mach_get_tsc(void)
+{
+	if (likely(pxa_timer_initialized)) {
+		union tsc_reg *local_tsc, result;
+		unsigned long stamp;
+
+		local_tsc = &tsc[ipipe_processor_id()];
+
+		__asm__ ("ldmia %1, %M0\n":
+			 "=r"(result.full): "r"(local_tsc), "m"(*local_tsc));
+		barrier();
+		stamp = OSCR;
+		if (unlikely(stamp < result.low))
+			/* 32 bit counter wrapped, increment high word. */
+			result.high++;
+		result.low = stamp;
+
+		return result.full;
+	}
+
+        return 0;
+}
+EXPORT_SYMBOL(__ipipe_mach_get_tsc);
+
+/*
+ * Reprogram the timer
+ */
+
+void __ipipe_mach_set_dec(unsigned long delay)
+{
+	if (delay > MIN_OSCR_DELTA) {
+		unsigned long flags;
+
+		local_irq_save_hw(flags);
+		OSMR0 = delay + OSCR;
+		OIER |= OIER_E0;
+		local_irq_restore_hw(flags);
+	} else
+		ipipe_trigger_irq(IRQ_OST0);
+}
+EXPORT_SYMBOL(__ipipe_mach_set_dec);
+
+void __ipipe_mach_release_timer(void)
+{
+	pxa_osmr0_set_mode(ckevt_pxa_osmr0.mode, &ckevt_pxa_osmr0);
+	if (ckevt_pxa_osmr0.mode == CLOCK_EVT_MODE_ONESHOT)
+		pxa_osmr0_set_next_event(LATCH, &ckevt_pxa_osmr0);
+}
+EXPORT_SYMBOL(__ipipe_mach_release_timer);
+
+unsigned long __ipipe_mach_get_dec(void)
+{
+	return OSMR0 - OSCR;
+}
+#endif /* CONFIG_IPIPE */
diff --git a/arch/arm/mach-s3c2440/irq.c b/arch/arm/mach-s3c2440/irq.c
index a87608b..a9b347f 100644
--- a/arch/arm/mach-s3c2440/irq.c
+++ b/arch/arm/mach-s3c2440/irq.c
@@ -3,6 +3,8 @@
  * Copyright (c) 2003,2004 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
+ * Copyright (C) 2006, 2007 Sebastian Smolorz <ssmolorz@emlix.com>, emlix GmbH
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
@@ -24,6 +26,7 @@
 #include <linux/interrupt.h>
 #include <linux/ioport.h>
 #include <linux/sysdev.h>
+#include <linux/ipipe.h>
 
 #include <asm/hardware.h>
 #include <asm/irq.h>
@@ -95,6 +98,25 @@ static struct irq_chip s3c_irq_wdtac97 = {
 	.ack	    = s3c_irq_wdtac97_ack,
 };
 
+#ifdef CONFIG_IPIPE
+void __ipipe_s3c_irq_demux_wdtac97(unsigned int subsrc, struct pt_regs *regs)
+{
+	subsrc >>= 13;
+	subsrc &= 3;
+
+	if (subsrc != 0) {
+		if (subsrc & 1) {
+			__ipipe_handle_irq(IRQ_S3C2440_WDT, regs);
+			return;
+		}
+		if (subsrc & 2) {
+			__ipipe_handle_irq(IRQ_S3C2440_AC97, regs);
+			return;
+		}
+	}
+}
+#endif /* CONFIG_IPIPE */
+
 static int s3c2440_irq_add(struct sys_device *sysdev)
 {
 	unsigned int irqno;
diff --git a/arch/arm/mach-sa1100/irq.c b/arch/arm/mach-sa1100/irq.c
index fa0403a..0dd140f 100644
--- a/arch/arm/mach-sa1100/irq.c
+++ b/arch/arm/mach-sa1100/irq.c
@@ -15,6 +15,7 @@
 #include <linux/irq.h>
 #include <linux/ioport.h>
 #include <linux/sysdev.h>
+#include <linux/ipipe.h>
 
 #include <asm/hardware.h>
 #include <asm/mach/irq.h>
@@ -136,6 +137,33 @@ sa1100_high_gpio_handler(unsigned int irq, struct irq_desc *desc)
 	} while (mask);
 }
 
+#ifdef CONFIG_IPIPE
+void __ipipe_mach_demux_irq(unsigned irq, struct pt_regs *regs)
+{
+	unsigned int mask;
+
+	mask = GEDR & 0xfffff800;
+	do {
+		/*
+		 * clear down all currently active IRQ sources.
+		 * We will be processing them all.
+		 */
+		GEDR = mask;
+
+		irq = IRQ_GPIO11;
+		mask >>= 11;
+		do {
+			if (mask & 1)
+				__ipipe_handle_irq(irq, regs);
+			mask >>= 1;
+			irq++;
+		} while (mask);
+
+		mask = GEDR & 0xfffff800;
+	} while (mask);
+}
+#endif /* CONFIG_IPIPE */
+
 /*
  * Like GPIO0 to 10, GPIO11-27 IRQs need to be handled specially.
  * In addition, the IRQs are all collected up into one bit in the
@@ -219,6 +247,9 @@ static struct irq_chip sa1100_normal_chip = {
 	.name		= "SC",
 	.ack		= sa1100_mask_irq,
 	.mask		= sa1100_mask_irq,
+#ifdef CONFIG_IPIPE
+	.mask_ack	= sa1100_mask_irq,
+#endif /* CONFIG_IPIPE */
 	.unmask		= sa1100_unmask_irq,
 	.set_wake	= sa1100_set_wake,
 };
diff --git a/arch/arm/mach-sa1100/leds-assabet.c b/arch/arm/mach-sa1100/leds-assabet.c
index ee97889..6f28e5b 100644
--- a/arch/arm/mach-sa1100/leds-assabet.c
+++ b/arch/arm/mach-sa1100/leds-assabet.c
@@ -10,6 +10,7 @@
  *   - Red   - on if system is not idle
  */
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <asm/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-sa1100/leds-badge4.c b/arch/arm/mach-sa1100/leds-badge4.c
index 280929b..b82b6d7 100644
--- a/arch/arm/mach-sa1100/leds-badge4.c
+++ b/arch/arm/mach-sa1100/leds-badge4.c
@@ -11,6 +11,7 @@
  */
 
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <asm/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-sa1100/leds-cerf.c b/arch/arm/mach-sa1100/leds-cerf.c
index f38eedd..08bc4e4 100644
--- a/arch/arm/mach-sa1100/leds-cerf.c
+++ b/arch/arm/mach-sa1100/leds-cerf.c
@@ -4,6 +4,7 @@
  * Author: ???
  */
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <asm/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-sa1100/leds-hackkit.c b/arch/arm/mach-sa1100/leds-hackkit.c
index 7e91cc9..b38db4e 100644
--- a/arch/arm/mach-sa1100/leds-hackkit.c
+++ b/arch/arm/mach-sa1100/leds-hackkit.c
@@ -10,6 +10,7 @@
  * as cpu led, the green one is used as timer led.
  */
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <asm/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-sa1100/leds-lart.c b/arch/arm/mach-sa1100/leds-lart.c
index 2d27d76..2c2c80b 100644
--- a/arch/arm/mach-sa1100/leds-lart.c
+++ b/arch/arm/mach-sa1100/leds-lart.c
@@ -10,6 +10,7 @@
  *  pace of the LED.
  */
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <asm/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-sa1100/leds-simpad.c b/arch/arm/mach-sa1100/leds-simpad.c
index def090a..6ed3032 100644
--- a/arch/arm/mach-sa1100/leds-simpad.c
+++ b/arch/arm/mach-sa1100/leds-simpad.c
@@ -4,6 +4,7 @@
  * Author: Juergen Messerer <juergen.messerer@siemens.ch>
  */
 #include <linux/init.h>
+#include <linux/ipipe_base.h>
 
 #include <asm/hardware.h>
 #include <asm/leds.h>
diff --git a/arch/arm/mach-sa1100/time.c b/arch/arm/mach-sa1100/time.c
index a9799cb..8de19f9 100644
--- a/arch/arm/mach-sa1100/time.c
+++ b/arch/arm/mach-sa1100/time.c
@@ -20,13 +20,71 @@
 
 #define MIN_OSCR_DELTA 2
 
+#ifdef CONFIG_IPIPE
+#ifdef CONFIG_NO_IDLE_HZ
+#error "dynamic tick timer not yet supported with IPIPE"
+#endif /* CONFIG_NO_IDLE_HZ */
+int __ipipe_mach_timerint = IRQ_OST0;
+EXPORT_SYMBOL(__ipipe_mach_timerint);
+
+int __ipipe_mach_timerstolen = 0;
+EXPORT_SYMBOL(__ipipe_mach_timerstolen);
+
+unsigned int __ipipe_mach_ticks_per_jiffy = LATCH;
+EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy);
+
+static int sa1100_timer_initialized;
+
+union tsc_reg {
+#ifdef __BIG_ENDIAN
+	struct {
+		unsigned long high;
+		unsigned long low;
+	};
+#else /* __LITTLE_ENDIAN */
+	struct {
+		unsigned long low;
+		unsigned long high;
+	};
+#endif /* __LITTLE_ENDIAN */
+	unsigned long long full;
+};
+
+#ifdef CONFIG_SMP
+static union tsc_reg tsc[NR_CPUS];
+
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info)
+{
+	info->type = IPIPE_TSC_TYPE_NONE;
+}
+
+#else /* !CONFIG_SMP */
+static union tsc_reg *tsc;
+
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info)
+{
+	info->type = IPIPE_TSC_TYPE_FREERUNNING;
+	info->u.fr.counter = (unsigned *)0x90000010;
+	info->u.fr.mask = 0xffffffff;
+	info->u.fr.tsc = &tsc->full;
+}
+#endif /* !CONFIG_SMP */
+
+static void ipipe_mach_update_tsc(void);
+
+#endif /* CONFIG_IPIPE */
+
 static irqreturn_t sa1100_ost0_interrupt(int irq, void *dev_id)
 {
 	struct clock_event_device *c = dev_id;
 
 	/* Disarm the compare/match, signal the event. */
+#ifndef CONFIG_IPIPE
 	OIER &= ~OIER_E0;
 	OSSR = OSSR_M0;
+#else /* CONFIG_IPIPE */
+	ipipe_mach_update_tsc();
+#endif /* CONFIG_IPIPE */
 	c->event_handler(c);
 
 	return IRQ_HANDLED;
@@ -78,6 +136,13 @@ static struct clock_event_device ckevt_sa1100_osmr0 = {
 	.set_mode	= sa1100_osmr0_set_mode,
 };
 
+#ifdef CONFIG_IPIPE
+int __ipipe_check_tickdev(const char *devname)
+{
+	return !strcmp(devname, ckevt_sa1100_osmr0.name);
+}
+#endif /* CONFIG_IPIPE */
+
 static cycle_t sa1100_read_oscr(void)
 {
 	return OSCR;
@@ -116,6 +181,15 @@ static void __init sa1100_timer_init(void)
 
 	setup_irq(IRQ_OST0, &sa1100_timer_irq);
 
+#ifdef CONFIG_IPIPE
+#ifndef CONFIG_SMP
+	tsc = (union tsc_reg *) __ipipe_tsc_area;
+	barrier();
+#endif /* CONFIG_SMP */
+
+	sa1100_timer_initialized = 1;
+#endif /* CONFIG_IPIPE */
+
 	clocksource_register(&cksrc_sa1100_oscr);
 	clockevents_register_device(&ckevt_sa1100_osmr0);
 }
@@ -156,3 +230,80 @@ struct sys_timer sa1100_timer = {
 	.suspend	= sa1100_timer_suspend,
 	.resume		= sa1100_timer_resume,
 };
+
+#ifdef CONFIG_IPIPE
+void __ipipe_mach_acktimer(void)
+{
+	OSSR = OSSR_M0;  /* Clear match on timer 0 */
+	OIER &= ~OIER_E0;
+}
+
+static void ipipe_mach_update_tsc(void)
+{
+	union tsc_reg *local_tsc;
+	unsigned long stamp, flags;
+
+	local_irq_save_hw(flags);
+	local_tsc = &tsc[ipipe_processor_id()];
+	stamp = OSCR;
+	if (unlikely(stamp < local_tsc->low))
+		/* 32 bit counter wrapped, increment high word. */
+		local_tsc->high++;
+	local_tsc->low = stamp;
+	local_irq_restore_hw(flags);
+}
+
+notrace unsigned long long __ipipe_mach_get_tsc(void)
+{
+	if (likely(sa1100_timer_initialized)) {
+		union tsc_reg *local_tsc, result;
+		unsigned long stamp;
+
+		local_tsc = &tsc[ipipe_processor_id()];
+
+		__asm__ ("ldmia %1, %M0\n":
+			 "=r"(result.full): "r"(local_tsc), "m"(*local_tsc));
+		barrier();
+		stamp = OSCR;
+		if (unlikely(stamp < result.low))
+			/* 32 bit counter wrapped, increment high word. */
+			result.high++;
+		result.low = stamp;
+		return result.full;
+	}
+	
+        return 0;
+}
+EXPORT_SYMBOL(__ipipe_mach_get_tsc);
+
+/*
+ * Reprogram the timer
+ */
+
+void __ipipe_mach_set_dec(unsigned long delay)
+{
+	if (delay > MIN_OSCR_DELTA) {
+		unsigned long flags;
+
+		local_irq_save_hw(flags);
+		OSMR0 = delay + OSCR;
+		OIER |= OIER_E0;
+		local_irq_restore_hw(flags);
+	} else
+		ipipe_trigger_irq(IRQ_OST0);
+}
+EXPORT_SYMBOL(__ipipe_mach_set_dec);
+
+void __ipipe_mach_release_timer(void)
+{
+	sa1100_osmr0_set_mode(ckevt_sa1100_osmr0.mode, &ckevt_sa1100_osmr0);
+	if (ckevt_sa1100_osmr0.mode == CLOCK_EVT_MODE_ONESHOT)
+		sa1100_osmr0_set_next_event(LATCH, &ckevt_sa1100_osmr0);
+}
+EXPORT_SYMBOL(__ipipe_mach_release_timer);
+
+unsigned long __ipipe_mach_get_dec(void)
+{
+	return OSMR0 - OSCR;
+}
+#endif /* CONFIG_IPIPE */
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index 33ed048..8e8e1e4 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -716,3 +716,19 @@ config CACHE_L2X0
 	select OUTER_CACHE
 	help
 	  This option enables the L2x0 PrimeCell.
+
+config ARM_FCSE
+	bool "Fast Context Switch Extension (EXPERIMENTAL)"
+	depends on EXPERIMENTAL
+	default n
+	help
+
+	  The Fast Context Switch Extension (FCSE) avoids the need for
+	  costly MMU-related operations when switching task contexts
+	  (like flushing caches), hence providing a significant
+	  improvement in terms of latency. However, this result is
+	  obtained at the expense of fewer concurrent processes
+	  available (95) and a smaller address space for each process
+	  (32M).
+
+	  Say Y here to enable the ARM FCSE. If unsure, say N.
diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c
index e162cca..8860e0f 100644
--- a/arch/arm/mm/alignment.c
+++ b/arch/arm/mm/alignment.c
@@ -626,6 +626,9 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 	unsigned int fault;
 	u16 tinstr = 0;
 
+	if (ipipe_trap_notify(IPIPE_TRAP_ALIGNMENT,regs))
+		return 0;
+
 	instrptr = instruction_pointer(regs);
 
 	fs = get_fs();
diff --git a/arch/arm/mm/copypage-v4mc.c b/arch/arm/mm/copypage-v4mc.c
index ded0e96..bb79f37 100644
--- a/arch/arm/mm/copypage-v4mc.c
+++ b/arch/arm/mm/copypage-v4mc.c
@@ -44,7 +44,7 @@ static DEFINE_SPINLOCK(minicache_lock);
  * instruction.  If your processor does not supply this, you have to write your
  * own copy_user_page that does the right thing.
  */
-static void __attribute__((naked))
+static void notrace __attribute__((naked))
 mc_copy_user_page(void *from, void *to)
 {
 	asm volatile(
@@ -88,7 +88,7 @@ void v4_mc_copy_user_page(void *kto, const void *kfrom, unsigned long vaddr)
 /*
  * ARMv4 optimised clear_user_page
  */
-void __attribute__((naked))
+void notrace __attribute__((naked))
 v4_mc_clear_user_page(void *kaddr, unsigned long vaddr)
 {
 	asm volatile(
diff --git a/arch/arm/mm/copypage-xscale.c b/arch/arm/mm/copypage-xscale.c
index 2e455f8..accf6ed 100644
--- a/arch/arm/mm/copypage-xscale.c
+++ b/arch/arm/mm/copypage-xscale.c
@@ -42,7 +42,7 @@ static DEFINE_SPINLOCK(minicache_lock);
  * Dcache aliasing issue.  The writes will be forwarded to the write buffer,
  * and merged as appropriate.
  */
-static void __attribute__((naked))
+static void notrace __attribute__((naked))
 mc_copy_user_page(void *from, void *to)
 {
 	/*
@@ -110,7 +110,7 @@ void xscale_mc_copy_user_page(void *kto, const void *kfrom, unsigned long vaddr)
 /*
  * XScale optimised clear_user_page
  */
-void __attribute__((naked))
+void notrace __attribute__((naked))
 xscale_mc_clear_user_page(void *kaddr, unsigned long vaddr)
 {
 	asm volatile(
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index 28ad7ab..78efa71 100644
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -18,6 +18,7 @@
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>
 #include <asm/uaccess.h>
+#include <asm/fcse.h>
 
 #include "fault.h"
 
@@ -55,6 +56,10 @@ void show_pte(struct mm_struct *mm, unsigned long addr)
 	if (!mm)
 		mm = &init_mm;
 
+#ifdef CONFIG_ARM_FCSE
+	printk(KERN_ALERT "fcse pid: %ld, 0x%08lx\n",
+	       mm->context.pid >> FCSE_PID_SHIFT, mm->context.pid);
+#endif /* CONFIG_ARM_FCSE */
 	printk(KERN_ALERT "pgd = %p\n", mm->pgd);
 	pgd = pgd_offset(mm, addr);
 	printk(KERN_ALERT "[%08lx] *pgd=%08lx", addr, pgd_val(*pgd));
@@ -249,6 +254,9 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 	if (notify_page_fault(regs, fsr))
 		return 0;
 
+	if (ipipe_trap_notify(IPIPE_TRAP_ACCESS,regs))
+		return 0;
+
 	tsk = current;
 	mm  = tsk->mm;
 
@@ -349,6 +357,9 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
 	if (addr < TASK_SIZE)
 		return do_page_fault(addr, fsr, regs);
 
+	if (ipipe_trap_notify(IPIPE_TRAP_ACCESS,regs))
+		return 0;
+
 	index = pgd_index(addr);
 
 	/*
@@ -384,6 +395,10 @@ bad_area:
 static int
 do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 {
+
+	if (ipipe_trap_notify(IPIPE_TRAP_SECTION,regs))
+		return 0;
+
 	do_bad_area(addr, fsr, regs);
 	return 0;
 }
@@ -394,6 +409,9 @@ do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 static int
 do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 {
+	if (ipipe_trap_notify(IPIPE_TRAP_DABT,regs))
+		return 0;
+
 	return 1;
 }
 
@@ -466,9 +484,14 @@ do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 	const struct fsr_info *inf = fsr_info + (fsr & 15) + ((fsr & (1 << 10)) >> 6);
 	struct siginfo info;
 
+	addr = fcse_mva_to_va(addr);
+
 	if (!inf->fn(addr, fsr, regs))
 		return;
 
+	if (ipipe_trap_notify(IPIPE_TRAP_UNKNOWN,regs))
+		return;
+
 	printk(KERN_ALERT "Unhandled fault: %s (0x%03x) at 0x%08lx\n",
 		inf->name, fsr, addr);
 
@@ -485,3 +508,42 @@ do_PrefetchAbort(unsigned long addr, struct pt_regs *regs)
 	do_translation_fault(addr, 0, regs);
 }
 
+#ifdef CONFIG_IPIPE
+extern spinlock_t pgd_lock;
+extern struct page *pgd_list;
+
+static void vmalloc_sync_one(pgd_t *pgd, unsigned long addr)
+{
+	unsigned int index = pgd_index(addr);
+	pgd_t *pgd_k;
+	pmd_t *pmd, *pmd_k;
+
+	pgd += index;
+	pgd_k = init_mm.pgd + index;
+
+	if (!pgd_present(*pgd))
+		set_pgd(pgd, *pgd_k);
+
+	pmd_k = pmd_offset(pgd_k, addr);
+	pmd   = pmd_offset(pgd, addr);
+
+	copy_pmd(pmd, pmd_k);
+}
+
+void __ipipe_pin_range_globally(unsigned long start, unsigned long end)
+{
+	unsigned long next, addr = start;
+
+	do {
+		unsigned long flags;
+		struct page *page;
+
+		next = pgd_addr_end(addr, end);
+		spin_lock_irqsave(&pgd_lock, flags);
+		for (page = pgd_list; page; page = (struct page *)page->index)
+			vmalloc_sync_one(page_address(page), addr);
+		spin_unlock_irqrestore(&pgd_lock, flags);
+
+	} while (addr = next, addr != end);
+}
+#endif /* CONFIG_IPIPE */
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index 9df507d..426941e 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -14,6 +14,7 @@
 #include <asm/cacheflush.h>
 #include <asm/system.h>
 #include <asm/tlbflush.h>
+#include <asm/fcse.h>
 
 #include "mm.h"
 
@@ -58,9 +59,11 @@ void flush_cache_mm(struct mm_struct *mm)
 void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
 {
 	if (cache_is_vivt()) {
-		if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask))
-			__cpuc_flush_user_range(start & PAGE_MASK, PAGE_ALIGN(end),
-						vma->vm_flags);
+		if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) {
+			start = fcse_va_to_mva(vma->vm_mm, start) & PAGE_MASK;
+			end = PAGE_ALIGN(fcse_va_to_mva(vma->vm_mm, end));
+			__cpuc_flush_user_range(start, end, vma->vm_flags);
+		}
 		return;
 	}
 
@@ -78,8 +81,11 @@ void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsig
 {
 	if (cache_is_vivt()) {
 		if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) {
-			unsigned long addr = user_addr & PAGE_MASK;
-			__cpuc_flush_user_range(addr, addr + PAGE_SIZE, vma->vm_flags);
+			unsigned long addr;
+			addr = fcse_va_to_mva(vma->vm_mm, user_addr);
+			addr &= PAGE_MASK;
+			__cpuc_flush_user_range(addr, addr + PAGE_SIZE,
+						vma->vm_flags);
 		}
 		return;
 	}
diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
index e0f19ab..ec39e6e 100644
--- a/arch/arm/mm/pgd.c
+++ b/arch/arm/mm/pgd.c
@@ -16,7 +16,46 @@
 
 #include "mm.h"
 
+#ifndef CONFIG_ARM_FCSE
 #define FIRST_KERNEL_PGD_NR	(FIRST_USER_PGD_NR + USER_PTRS_PER_PGD)
+#else /* CONFIG_ARM_FCSE */
+#define FIRST_KERNEL_PGD_NR	(MODULE_START / PGDIR_SIZE)
+#endif /* CONFIG_ARM_FCSE */
+
+#ifdef CONFIG_IPIPE
+/* Copied from arch/i386/mm/pgdtable.c, maintains the list of pgds for the
+   implementation of ipipe_pin_range_globally in arch/arm/mm/fault.c. */
+DEFINE_SPINLOCK(pgd_lock);
+struct page *pgd_list;
+
+#define pgd_list_lock(flags) spin_lock_irqsave(&pgd_lock, flags)
+#define pgd_list_unlock(flags) spin_unlock_irqrestore(&pgd_lock, flags)
+
+static inline void pgd_list_add(pgd_t *pgd)
+{
+	struct page *page = virt_to_page(pgd);
+	page->index = (unsigned long)pgd_list;
+	if (pgd_list)
+		set_page_private(pgd_list, (unsigned long)&page->index);
+	pgd_list = page;
+	set_page_private(page, (unsigned long)&pgd_list);
+}
+
+static inline void pgd_list_del(pgd_t *pgd)
+{
+	struct page *next, **pprev, *page = virt_to_page(pgd);
+	next = (struct page *)page->index;
+	pprev = (struct page **)page_private(page);
+	*pprev = next;
+	if (next)
+		set_page_private(next, (unsigned long)pprev);
+}
+#else /* !CONFIG_IPIPE */
+#define pgd_list_lock(flags) (flags) = (flags)
+#define pgd_list_unlock(flags) (flags) = (flags)
+#define pgd_list_add(pgd) do { } while (0)
+#define pgd_list_del(pgd) do { } while (0)
+#endif /* !CONFIG_IPIPE */
 
 /*
  * need to get a 16k page for level 1
@@ -26,6 +65,16 @@ pgd_t *get_pgd_slow(struct mm_struct *mm)
 	pgd_t *new_pgd, *init_pgd;
 	pmd_t *new_pmd, *init_pmd;
 	pte_t *new_pte, *init_pte;
+	unsigned long flags;
+#ifdef CONFIG_ARM_FCSE
+	int pid;
+
+	pid = fcse_pid_alloc();
+	if (pid == -1)
+		goto no_pgd;
+
+	mm->context.pid = pid << FCSE_PID_SHIFT;
+#endif /* CONFIG_ARM_FCSE */
 
 	new_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, 2);
 	if (!new_pgd)
@@ -37,17 +86,24 @@ pgd_t *get_pgd_slow(struct mm_struct *mm)
 	 * Copy over the kernel and IO PGD entries
 	 */
 	init_pgd = pgd_offset_k(0);
+	pgd_list_lock(flags);
 	memcpy(new_pgd + FIRST_KERNEL_PGD_NR, init_pgd + FIRST_KERNEL_PGD_NR,
 		       (PTRS_PER_PGD - FIRST_KERNEL_PGD_NR) * sizeof(pgd_t));
+	pgd_list_add(new_pgd);
+	pgd_list_unlock(flags);
 
 	clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t));
 
 	if (!vectors_high()) {
+		/* We can not use pgd_offset here since mm->pgd is not yet
+		   initialized. */
+		pgd_t *pgd = new_pgd + pgd_index(fcse_va_to_mva(mm, 0));
+
 		/*
 		 * On ARM, first page must always be allocated since it
 		 * contains the machine vectors.
 		 */
-		new_pmd = pmd_alloc(mm, new_pgd, 0);
+		new_pmd = pmd_alloc(mm, pgd, 0);
 		if (!new_pmd)
 			goto no_pmd;
 
@@ -74,6 +130,7 @@ no_pgd:
 
 void free_pgd_slow(struct mm_struct *mm, pgd_t *pgd)
 {
+	unsigned long flags;
 	pmd_t *pmd;
 	pgtable_t pte;
 
@@ -95,5 +152,11 @@ void free_pgd_slow(struct mm_struct *mm, pgd_t *pgd)
 	pte_free(mm, pte);
 	pmd_free(mm, pmd);
 free:
+	pgd_list_lock(flags);
+	pgd_list_del(pgd);
+	pgd_list_unlock(flags);
 	free_pages((unsigned long) pgd, 2);
+#ifdef CONFIG_ARM_FCSE
+	fcse_pid_free(mm->context.pid >> FCSE_PID_SHIFT);
+#endif /* CONFIG_ARM_FCSE */
 }
diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S
index 28cdb06..2454d0b 100644
--- a/arch/arm/mm/proc-arm920.S
+++ b/arch/arm/mm/proc-arm920.S
@@ -321,6 +321,10 @@ ENTRY(cpu_arm920_dcache_clean_area)
 ENTRY(cpu_arm920_switch_mm)
 #ifdef CONFIG_MMU
 	mov	ip, #0
+#ifdef CONFIG_ARM_FCSE
+	cmp	r2, #0
+	beq	.LCnoflush
+#endif
 #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH
 	mcr	p15, 0, ip, c7, c6, 0		@ invalidate D cache
 #else
@@ -338,6 +342,9 @@ ENTRY(cpu_arm920_switch_mm)
 #endif
 	mcr	p15, 0, ip, c7, c5, 0		@ invalidate I cache
 	mcr	p15, 0, ip, c7, c10, 4		@ drain WB
+#ifdef CONFIG_ARM_FCSE
+.LCnoflush:
+#endif
 	mcr	p15, 0, r0, c2, c0, 0		@ load page table pointer
 	mcr	p15, 0, ip, c8, c7, 0		@ invalidate I & D TLBs
 #endif
diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S
index 4cd3316..ca7f4ee 100644
--- a/arch/arm/mm/proc-arm926.S
+++ b/arch/arm/mm/proc-arm926.S
@@ -337,6 +337,10 @@ ENTRY(cpu_arm926_dcache_clean_area)
 ENTRY(cpu_arm926_switch_mm)
 #ifdef CONFIG_MMU
 	mov	ip, #0
+#ifdef CONFIG_ARM_FCSE
+	cmp	r2, #0
+	beq	.LCnoflush
+#endif
 #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH
 	mcr	p15, 0, ip, c7, c6, 0		@ invalidate D cache
 #else
@@ -346,6 +350,9 @@ ENTRY(cpu_arm926_switch_mm)
 #endif
 	mcr	p15, 0, ip, c7, c5, 0		@ invalidate I cache
 	mcr	p15, 0, ip, c7, c10, 4		@ drain WB
+#ifdef CONFIG_ARM_FCSE
+.LCnoflush:
+#endif
 	mcr	p15, 0, r0, c2, c0, 0		@ load page table pointer
 	mcr	p15, 0, ip, c8, c7, 0		@ invalidate I & D TLBs
 #endif
diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S
index 2dd8527..c101e5d 100644
--- a/arch/arm/mm/proc-xscale.S
+++ b/arch/arm/mm/proc-xscale.S
@@ -417,9 +417,16 @@ ENTRY(cpu_xscale_dcache_clean_area)
  */
 	.align	5
 ENTRY(cpu_xscale_switch_mm)
+#ifdef CONFIG_ARM_FCSE
+	cmp	r2, #0
+	beq	.LCnoflush
+#endif
 	clean_d_cache r1, r2
 	mcr	p15, 0, ip, c7, c5, 0		@ Invalidate I cache & BTB
 	mcr	p15, 0, ip, c7, c10, 4		@ Drain Write (& Fill) Buffer
+#ifdef CONFIG_ARM_FCSE
+.LCnoflush:
+#endif
 	mcr	p15, 0, r0, c2, c0, 0		@ load page table pointer
 	mcr	p15, 0, ip, c8, c7, 0		@ invalidate I & D TLBs
 	cpwait_ret lr, ip
diff --git a/arch/arm/plat-s3c24xx/irq.c b/arch/arm/plat-s3c24xx/irq.c
index ae2c5d7..cb1482e 100644
--- a/arch/arm/plat-s3c24xx/irq.c
+++ b/arch/arm/plat-s3c24xx/irq.c
@@ -3,6 +3,8 @@
  * Copyright (c) 2003,2004 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
+ * Copyright (C) 2006, 2007 Sebastian Smolorz <ssmolorz@emlix.com>, emlix GmbH
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
@@ -48,6 +50,9 @@
  *
  *   25-Jul-2005  Ben Dooks
  *		  Split the S3C2440 IRQ code to separate file
+ *
+ *   30-Oct-2006  Sebastian Smolorz
+ *		  Added Adeos/I-pipe support
 */
 
 #include <linux/init.h>
@@ -55,6 +60,7 @@
 #include <linux/interrupt.h>
 #include <linux/ioport.h>
 #include <linux/sysdev.h>
+#include <linux/ipipe.h>
 
 #include <asm/hardware.h>
 #include <asm/irq.h>
@@ -69,6 +75,14 @@
 #include <asm/plat-s3c24xx/pm.h>
 #include <asm/plat-s3c24xx/irq.h>
 
+#ifdef CONFIG_IPIPE
+#ifdef CONFIG_CPU_S3C2440
+extern void __ipipe_s3c_irq_demux_wdtac97(unsigned int irq,
+					  struct pt_regs *regs);
+extern void __ipipe_s3c_irq_demux_cam(unsigned int irq, struct pt_regs *regs);
+#endif /* CONFIG_CPU_S3C2440 */
+#endif /* CONFIG_IPIPE */
+
 /* wakeup irq control */
 
 #ifdef CONFIG_PM
@@ -183,6 +197,9 @@ struct irq_chip s3c_irq_level_chip = {
 	.name		= "s3c-level",
 	.ack		= s3c_irq_maskack,
 	.mask		= s3c_irq_mask,
+#ifdef CONFIG_IPIPE
+	.mask_ack       = s3c_irq_maskack,
+#endif /* CONFIG_IPIPE */
 	.unmask		= s3c_irq_unmask,
 	.set_wake	= s3c_irq_wake
 };
@@ -379,6 +396,9 @@ static struct irq_chip s3c_irq_uart0 = {
 	.mask		= s3c_irq_uart0_mask,
 	.unmask		= s3c_irq_uart0_unmask,
 	.ack		= s3c_irq_uart0_ack,
+#ifdef CONFIG_IPIPE
+	.mask_ack       = s3c_irq_uart0_ack,
+#endif /* CONFIG_IPIPE */
 };
 
 /* UART1 */
@@ -406,6 +426,9 @@ static struct irq_chip s3c_irq_uart1 = {
 	.mask		= s3c_irq_uart1_mask,
 	.unmask		= s3c_irq_uart1_unmask,
 	.ack		= s3c_irq_uart1_ack,
+#ifdef CONFIG_IPIPE
+	.mask_ack	= s3c_irq_uart1_ack,
+#endif /* CONFIG_IPIPE */
 };
 
 /* UART2 */
@@ -433,6 +456,9 @@ static struct irq_chip s3c_irq_uart2 = {
 	.mask		= s3c_irq_uart2_mask,
 	.unmask		= s3c_irq_uart2_unmask,
 	.ack		= s3c_irq_uart2_ack,
+#ifdef CONFIG_IPIPE
+	.mask_ack	= s3c_irq_uart2_ack,
+#endif /* CONFIG_IPIPE */
 };
 
 /* ADC and Touchscreen */
@@ -652,6 +678,121 @@ int s3c24xx_irq_resume(struct sys_device *dev)
 #define s3c24xx_irq_resume  NULL
 #endif
 
+#ifdef CONFIG_IPIPE
+static void __ipipe_s3c_irq_demux_uart(unsigned int start,
+					unsigned int subsrc,
+					struct pt_regs *regs)
+{
+	unsigned int offset = start - IRQ_S3CUART_RX0;
+
+	subsrc >>= offset;
+	subsrc &= 7;
+
+	if (subsrc != 0) {
+		if (subsrc & 1) {
+			__ipipe_handle_irq(start, regs);
+			return;
+		}
+		if (subsrc & 2) {
+			__ipipe_handle_irq(start+1, regs);
+			return;
+		}
+		if (subsrc & 4) {
+			__ipipe_handle_irq(start+2, regs);
+			return;
+		}
+	}
+}
+
+static void __ipipe_s3c_irq_demux_adc(unsigned int subsrc,
+					struct pt_regs *regs)
+{
+	subsrc >>= 9;
+	subsrc &= 3;
+
+	if (subsrc != 0) {
+		if (subsrc & 1) {
+			__ipipe_handle_irq(IRQ_TC, regs);
+			return;
+		}
+		if (subsrc & 2) {
+			__ipipe_handle_irq(IRQ_ADC, regs);
+			return;
+		}
+	}
+}
+
+static void __ipipe_s3c_irq_demux_extint(unsigned long mask,
+						struct pt_regs *regs)
+{
+	unsigned int irq;
+	unsigned long eintpnd = __raw_readl(S3C24XX_EINTPEND);
+	unsigned long eintmsk = __raw_readl(S3C24XX_EINTMASK);
+
+	while ((eintpnd &= ~eintmsk & mask)) {
+		irq = __ffs(eintpnd);
+
+		irq += (IRQ_EINT4 - 4);
+
+		__ipipe_handle_irq(irq, regs);
+
+		eintpnd = __raw_readl(S3C24XX_EINTPEND);
+		eintmsk = __raw_readl(S3C24XX_EINTMASK);
+	}
+}
+
+void __ipipe_mach_demux_irq(unsigned irq, struct pt_regs *regs)
+{
+	unsigned int subsrc, submsk;
+
+	/* read the current pending interrupts, and the mask
+	 * for what it is available */
+	for(;;) {
+		subsrc = __raw_readl(S3C2410_SUBSRCPND);
+		submsk = __raw_readl(S3C2410_INTSUBMSK);
+
+		subsrc &= ~submsk;
+
+		if (!subsrc)
+			break;
+
+		switch (irq) {
+		case IRQ_UART0:
+			__ipipe_s3c_irq_demux_uart(IRQ_S3CUART_RX0,
+						   subsrc, regs);
+			break;
+		case IRQ_UART1:
+			__ipipe_s3c_irq_demux_uart(IRQ_S3CUART_RX1,
+						   subsrc, regs);
+			break;
+		case IRQ_UART2:
+			__ipipe_s3c_irq_demux_uart(IRQ_S3CUART_RX2,
+						   subsrc, regs);
+			break;
+		case IRQ_ADCPARENT:
+			__ipipe_s3c_irq_demux_adc(subsrc, regs);
+			break;
+		case IRQ_EINT4t7:
+			__ipipe_s3c_irq_demux_extint(0xff, regs);
+			break;
+		case IRQ_EINT8t23:
+			__ipipe_s3c_irq_demux_extint(0xffffff00, regs);
+			break;
+#ifdef CONFIG_CPU_S3C2440
+		case IRQ_WDT:
+			__ipipe_s3c_irq_demux_wdtac97(subsrc, regs);
+			break;
+#endif /* CONFIG_CPU_S3C2440 */
+#ifdef CONFIG_CPU_S3C244X
+		case IRQ_CAM:
+			__ipipe_s3c_irq_demux_cam(subsrc, regs);
+			break;
+#endif /* CONFIG_CPU_S3C244X */
+		}
+	}
+}
+#endif /* CONFIG_IPIPE */
+
 /* s3c24xx_init_irq
  *
  * Initialise S3C2410 IRQ system
diff --git a/arch/arm/plat-s3c24xx/s3c244x-irq.c b/arch/arm/plat-s3c24xx/s3c244x-irq.c
index 2dbb260..04320b4 100644
--- a/arch/arm/plat-s3c24xx/s3c244x-irq.c
+++ b/arch/arm/plat-s3c24xx/s3c244x-irq.c
@@ -3,6 +3,8 @@
  * Copyright (c) 2003,2004 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
+ * Copyright (C) 2007 Sebastian Smolorz <ssmolorz@emlix.com>, emlix GmbH
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
@@ -92,8 +94,30 @@ static struct irq_chip s3c_irq_cam = {
 	.mask	    = s3c_irq_cam_mask,
 	.unmask	    = s3c_irq_cam_unmask,
 	.ack	    = s3c_irq_cam_ack,
+#ifdef CONFIG_IPIPE
+	.mask_ack   = s3c_irq_cam_ack,
+#endif /* CONFIG_IPIPE */
 };
 
+#ifdef CONFIG_IPIPE
+void __ipipe_s3c_irq_demux_cam(unsigned int subsrc, struct pt_regs *regs)
+{
+	subsrc >>= 11;
+	subsrc &= 3;
+
+	if (subsrc != 0) {
+		if (subsrc & 1) {
+			__ipipe_handle_irq(IRQ_S3C2440_CAM_C, regs);
+			return;
+		}
+		if (subsrc & 2) {
+			__ipipe_handle_irq(IRQ_S3C2440_CAM_P, regs);
+			return;
+		}
+	}
+}
+#endif /* CONFIG_IPIPE */
+
 static int s3c244x_irq_add(struct sys_device *sysdev)
 {
 	unsigned int irqno;
diff --git a/arch/arm/plat-s3c24xx/time.c b/arch/arm/plat-s3c24xx/time.c
index 766473b..c111a49 100644
--- a/arch/arm/plat-s3c24xx/time.c
+++ b/arch/arm/plat-s3c24xx/time.c
@@ -3,6 +3,8 @@
  * Copyright (C) 2003-2005 Simtec Electronics
  *	Ben Dooks, <ben@simtec.co.uk>
  *
+ * Copyright (C) 2006, 2007 Sebastian Smolorz <ssmolorz@emlix.com>, emlix GmbH
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
@@ -25,6 +27,7 @@
 #include <linux/irq.h>
 #include <linux/err.h>
 #include <linux/clk.h>
+#include <linux/module.h>
 
 #include <asm/system.h>
 #include <asm/leds.h>
@@ -40,7 +43,6 @@
 #include <asm/plat-s3c24xx/clock.h>
 #include <asm/plat-s3c24xx/cpu.h>
 
-static unsigned long timer_startval;
 static unsigned long timer_usec_ticks;
 
 #define TIMER_USEC_SHIFT 16
@@ -55,6 +57,39 @@ static unsigned long timer_usec_ticks;
  * Original patch by Dimitry Andric, updated by Ben Dooks
 */
 
+static unsigned long last_free_running_tcnt = 0;
+static unsigned long free_running_tcon = 0;
+static unsigned long timer_lxlost = 0;
+
+#ifdef CONFIG_IPIPE
+
+#ifdef CONFIG_NO_IDLE_HZ
+#error "dynamic tick timer not yet supported with IPIPE"
+#endif /* CONFIG_NO_IDLE_HZ */
+
+unsigned int __ipipe_mach_ticks_per_jiffy;
+EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy);
+
+int __ipipe_mach_timerint = IRQ_TIMER4;
+EXPORT_SYMBOL(__ipipe_mach_timerint);
+
+static unsigned long long *tsc;
+static unsigned *last_cnt;
+static unsigned long timer_ackval = 1UL << (IRQ_TIMER4 - IRQ_EINT0);
+static IPIPE_DEFINE_SPINLOCK(timer_lock);
+
+int __ipipe_mach_timerstolen = 0;
+EXPORT_SYMBOL(__ipipe_mach_timerstolen);
+
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info)
+{
+	info->type = IPIPE_TSC_TYPE_DECREMENTER;
+	info->u.dec.counter = (unsigned *)0x51000038;
+	info->u.dec.mask = 0xffff;
+	info->u.dec.last_cnt = last_cnt;
+	info->u.dec.tsc = tsc;
+}
+#endif /* CONFIG_IPIPE */
 
 /* timer_mask_usec_ticks
  *
@@ -85,44 +120,47 @@ static inline unsigned long timer_ticks_to_usec(unsigned long ticks)
 	return res >> TIMER_USEC_SHIFT;
 }
 
-/***
- * Returns microsecond  since last clock interrupt.  Note that interrupts
- * will have been disabled by do_gettimeoffset()
- * IRQs are disabled before entering here from do_gettimeofday()
- */
-
-#define SRCPND_TIMER4 (1<<(IRQ_TIMER4 - IRQ_EINT0))
 
-static unsigned long s3c2410_gettimeoffset (void)
+static inline unsigned long timer_freerunning_getvalue(void)
 {
-	unsigned long tdone;
-	unsigned long irqpend;
-	unsigned long tval;
-
-	/* work out how many ticks have gone since last timer interrupt */
-
-        tval =  __raw_readl(S3C2410_TCNTO(4));
-	tdone = timer_startval - tval;
+	return __raw_readl(S3C2410_TCNTO(3));
+}
 
-	/* check to see if there is an interrupt pending */
+static inline unsigned long timer_freerunning_getticksoffset(unsigned long tval)
+{
+	long tdone;
 
-	irqpend = __raw_readl(S3C2410_SRCPND);
-	if (irqpend & SRCPND_TIMER4) {
-		/* re-read the timer, and try and fix up for the missed
-		 * interrupt. Note, the interrupt may go off before the
-		 * timer has re-loaded from wrapping.
-		 */
+	tdone =  last_free_running_tcnt - tval;
+	if (tdone < 0)
+		tdone += 0x10000;
 
-		tval =  __raw_readl(S3C2410_TCNTO(4));
-		tdone = timer_startval - tval;
+	return tdone;
+}
 
-		if (tval != 0)
-			tdone += timer_startval;
-	}
+static inline unsigned long getticksoffset(void)
+{
+	return timer_freerunning_getticksoffset(timer_freerunning_getvalue());
+}
 
-	return timer_ticks_to_usec(tdone);
+#ifdef CONFIG_IPIPE
+static inline unsigned long getticksoffset_tscupdate(void)
+{
+	unsigned long tval;
+	unsigned long ticks;
+
+	tval = timer_freerunning_getvalue();
+	ticks = timer_freerunning_getticksoffset(tval);
+	last_free_running_tcnt = tval;
+	*tsc += ticks;
+	*last_cnt = last_free_running_tcnt;
+	return ticks;
 }
+#endif /* CONFIG_IPIPE */
 
+static unsigned long s3c2410_gettimeoffset (void)
+{
+	return timer_ticks_to_usec(timer_lxlost + getticksoffset());
+}
 
 /*
  * IRQ handler for the timer
@@ -130,6 +168,13 @@ static unsigned long s3c2410_gettimeoffset (void)
 static irqreturn_t
 s3c2410_timer_interrupt(int irq, void *dev_id)
 {
+#ifdef CONFIG_IPIPE
+	timer_lxlost = 0;
+
+	if (!__ipipe_mach_timerstolen)
+		getticksoffset_tscupdate();
+#endif /* CONFIG_IPIPE */
+
 	timer_tick();
 	return IRQ_HANDLED;
 }
@@ -147,10 +192,10 @@ static struct irqaction s3c2410_timer_irq = {
 	machine_is_osiris() )
 
 /*
- * Set up timer interrupt, and return the current time in seconds.
+ * Set up timer interrupt.
  *
- * Currently we only use timer4, as it is the only timer which has no
- * other function that can be exploited externally
+ * Currently we use timer4 as event timer and timer3 as tick counter which
+ * permanently counts ticks without interrupt generation.
  */
 static void s3c2410_timer_setup (void)
 {
@@ -158,6 +203,7 @@ static void s3c2410_timer_setup (void)
 	unsigned long tcnt;
 	unsigned long tcfg1;
 	unsigned long tcfg0;
+	unsigned long intmask;
 
 	tcnt = 0xffff;  /* default value for tcnt */
 
@@ -174,8 +220,8 @@ static void s3c2410_timer_setup (void)
 		timer_usec_ticks = timer_mask_usec_ticks(1, 12000000);
 		tcnt = 12000000 / HZ;
 
-		tcfg1 &= ~S3C2410_TCFG1_MUX4_MASK;
-		tcfg1 |= S3C2410_TCFG1_MUX4_TCLK1;
+		tcfg1 &= ~(S3C2410_TCFG1_MUX4_MASK | S3C2410_TCFG1_MUX3_MASK);
+		tcfg1 |= (S3C2410_TCFG1_MUX4_TCLK1 | S3C2410_TCFG1_MUX3_TCLK1);
 	} else {
 		unsigned long pclk;
 		struct clk *clk;
@@ -203,8 +249,8 @@ static void s3c2410_timer_setup (void)
 
 		timer_usec_ticks = timer_mask_usec_ticks(6, pclk);
 
-		tcfg1 &= ~S3C2410_TCFG1_MUX4_MASK;
-		tcfg1 |= S3C2410_TCFG1_MUX4_DIV2;
+		tcfg1 &= ~(S3C2410_TCFG1_MUX4_MASK | S3C2410_TCFG1_MUX3_MASK);
+		tcfg1 |= (S3C2410_TCFG1_MUX4_DIV2 | S3C2410_TCFG1_MUX3_DIV2);
 
 		tcfg0 &= ~S3C2410_TCFG_PRESCALER1_MASK;
 		tcfg0 |= ((6 - 1) / 2) << S3C2410_TCFG_PRESCALER1_SHIFT;
@@ -212,6 +258,14 @@ static void s3c2410_timer_setup (void)
 		tcnt = (pclk / 6) / HZ;
 	}
 
+#ifdef CONFIG_IPIPE
+	tsc = (unsigned long long *)__ipipe_tsc_area;
+	last_cnt = (unsigned *)(tsc + 1);		/* means: +8 bytes */
+	barrier();
+
+	__ipipe_mach_ticks_per_jiffy = tcnt;
+#endif /* CONFIG_IPIPE */
+
 	/* timers reload after counting zero, so reduce the count by 1 */
 
 	tcnt--;
@@ -228,23 +282,37 @@ static void s3c2410_timer_setup (void)
 	__raw_writel(tcfg1, S3C2410_TCFG1);
 	__raw_writel(tcfg0, S3C2410_TCFG0);
 
-	timer_startval = tcnt;
-	__raw_writel(tcnt, S3C2410_TCNTB(4));
-
-	/* ensure timer is stopped... */
+	/* ensure timers are stopped... */
+	tcon &= ~(0x3f<<17);
+	__raw_writel(tcon, S3C2410_TCON);
 
-	tcon &= ~(7<<20);
-	tcon |= S3C2410_TCON_T4RELOAD;
-	tcon |= S3C2410_TCON_T4MANUALUPD;
+	/* Mask timer3 interrupt. */
+	intmask = __raw_readl(S3C2410_INTMSK);
+	intmask |= 1UL << (IRQ_TIMER3 - IRQ_EINT0);
+	__raw_writel(intmask, S3C2410_INTMSK);
 
-	__raw_writel(tcon, S3C2410_TCON);
+	/* Set timer values */
 	__raw_writel(tcnt, S3C2410_TCNTB(4));
 	__raw_writel(tcnt, S3C2410_TCMPB(4));
+	__raw_writel(0xffff, S3C2410_TCNTB(3));
+	__raw_writel(0xffff, S3C2410_TCMPB(3));
+
+	/* Set base tcon value for later programming of timer 4 by Xenomai. */
+	free_running_tcon = tcon |  S3C2410_TCON_T3RELOAD | S3C2410_TCON_T3START;
+
+	/* Set auto reloads for both timers. */
+	tcon |= S3C2410_TCON_T3RELOAD | S3C2410_TCON_T4RELOAD;
 
-	/* start the timer running */
-	tcon |= S3C2410_TCON_T4START;
-	tcon &= ~S3C2410_TCON_T4MANUALUPD;
+	/* Manual update */
+	__raw_writel(tcon | S3C2410_TCON_T3MANUALUPD
+			  | S3C2410_TCON_T4MANUALUPD, S3C2410_TCON);
+
+	tcon |= S3C2410_TCON_T3START | S3C2410_TCON_T4START;
+	/* Start timers.*/
 	__raw_writel(tcon, S3C2410_TCON);
+
+	/* Save start value of timer 3 as begining of first period. */
+	last_free_running_tcnt = 0xffff;
 }
 
 static void __init s3c2410_timer_init (void)
@@ -258,3 +326,58 @@ struct sys_timer s3c24xx_timer = {
 	.offset		= s3c2410_gettimeoffset,
 	.resume		= s3c2410_timer_setup
 };
+
+#ifdef CONFIG_IPIPE
+void __ipipe_mach_acktimer(void)
+{
+	__raw_writel(timer_ackval, S3C2410_SRCPND);
+	__raw_writel(timer_ackval, S3C2410_INTPND);
+}
+
+notrace unsigned long long __ipipe_mach_get_tsc(void)
+{
+	unsigned long long result;
+	unsigned long flags;
+
+	local_irq_save_hw_notrace(flags);
+	spin_lock(&timer_lock);
+	result = *tsc + getticksoffset();
+	spin_unlock(&timer_lock);
+	local_irq_restore_hw_notrace(flags);
+	return result;
+}
+EXPORT_SYMBOL(__ipipe_mach_get_tsc);
+
+static inline void set_dec(unsigned long reload)
+{
+	__raw_writel(reload, S3C2410_TCNTB(4));
+	/* Manual update */
+	__raw_writel(free_running_tcon | S3C2410_TCON_T4MANUALUPD, S3C2410_TCON);
+	/* Start timer */
+	__raw_writel(free_running_tcon | S3C2410_TCON_T4START, S3C2410_TCON);
+}
+
+void __ipipe_mach_set_dec(unsigned long reload)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&timer_lock, flags);
+	timer_lxlost += getticksoffset_tscupdate();
+	set_dec(reload);
+	spin_unlock_irqrestore(&timer_lock, flags);
+}
+EXPORT_SYMBOL(__ipipe_mach_set_dec);
+
+void __ipipe_mach_release_timer(void)
+{
+	free_running_tcon |= S3C2410_TCON_T4RELOAD;
+	__ipipe_mach_set_dec(__ipipe_mach_ticks_per_jiffy - 1);
+	free_running_tcon &= ~S3C2410_TCON_T4RELOAD;
+}
+EXPORT_SYMBOL(__ipipe_mach_release_timer);
+
+unsigned long __ipipe_mach_get_dec(void)
+{
+	return __raw_readl(S3C2410_TCNTO(4));
+}
+#endif /* CONFIG_IPIPE */
diff --git a/arch/arm/vfp/vfphw.S b/arch/arm/vfp/vfphw.S
index 353f9e5..07d1be8 100644
--- a/arch/arm/vfp/vfphw.S
+++ b/arch/arm/vfp/vfphw.S
@@ -19,7 +19,7 @@
 #include "../kernel/entry-header.S"
 
 	.macro	DBGSTR, str
-#ifdef DEBUG
+#if defined(DEBUG) && !defined(CONFIG_IPIPE)
 	stmfd	sp!, {r0-r3, ip, lr}
 	add	r0, pc, #4
 	bl	printk
@@ -31,7 +31,7 @@
 	.endm
 
 	.macro  DBGSTR1, str, arg
-#ifdef DEBUG
+#if defined(DEBUG) && !defined(CONFIG_IPIPE)
 	stmfd	sp!, {r0-r3, ip, lr}
 	mov	r1, \arg
 	add	r0, pc, #4
@@ -44,7 +44,7 @@
 	.endm
 
 	.macro  DBGSTR3, str, arg1, arg2, arg3
-#ifdef DEBUG
+#if defined(DEBUG) && !defined(CONFIG_IPIPE)
 	stmfd	sp!, {r0-r3, ip, lr}
 	mov	r3, \arg3
 	mov	r2, \arg2
@@ -72,6 +72,7 @@
 vfp_support_entry:
 	DBGSTR3	"instr %08x pc %08x state %p", r0, r2, r10
 
+	disable_irq_cond
 	VFPFMRX	r1, FPEXC		@ Is the VFP enabled?
 	DBGSTR1	"fpexc %08x", r1
 	tst	r1, #FPEXC_EN
@@ -130,6 +131,7 @@ check_for_exception:
 					@ out before setting an FPEXC that
 					@ stops us reading stuff
 	VFPFMXR	FPEXC, r1		@ restore FPEXC last
+	enable_irq_cond
 	sub	r2, r2, #4
 	str	r2, [sp, #S_PC]		@ retry the instruction
 	mov	pc, r9			@ we think we have handled things
@@ -149,10 +151,12 @@ look_for_VFP_exceptions:
 	@ Fall into hand on to next handler - appropriate coproc instr
 	@ not recognised by VFP
 
+	enable_irq_cond
 	DBGSTR	"not VFP"
 	mov	pc, lr
 
 process_exception:
+	enable_irq_cond
 	DBGSTR	"bounce"
 	mov	r2, sp			@ nothing stacked - regdump is at TOS
 	mov	lr, r9			@ setup for a return to the user code.
diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c
index 32455c6..4aff12f 100644
--- a/arch/arm/vfp/vfpmodule.c
+++ b/arch/arm/vfp/vfpmodule.c
@@ -42,11 +42,14 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
 {
 	struct thread_info *thread = v;
 	union vfp_state *vfp;
+	unsigned long flags;
 	__u32 cpu = thread->cpu;
 
 	if (likely(cmd == THREAD_NOTIFY_SWITCH)) {
-		u32 fpexc = fmrx(FPEXC);
+		u32 fpexc;
 
+		local_irq_save_hw_cond(flags);
+		fpexc = fmrx(FPEXC);
 #ifdef CONFIG_SMP
 		/*
 		 * On SMP, if VFP is enabled, save the old state in
@@ -71,6 +74,7 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
 		 * old state.
 		 */
 		fmxr(FPEXC, fpexc & ~FPEXC_EN);
+		local_irq_restore_hw_cond(flags);
 		return NOTIFY_DONE;
 	}
 
@@ -87,12 +91,15 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
 		/*
 		 * Disable VFP to ensure we initialise it first.
 		 */
+		local_irq_save_hw_cond(flags);
 		fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
-	}
+	} else
+		local_irq_save_hw_cond(flags);
 
 	/* flush and release case: Per-thread VFP cleanup. */
 	if (last_VFP_context[cpu] == vfp)
 		last_VFP_context[cpu] = NULL;
+	local_irq_restore_hw_cond(flags);
 
 	return NOTIFY_DONE;
 }
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
index 636af28..41e0823 100644
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -32,7 +32,7 @@ config ATMEL_TCLIB
 
 config ATMEL_TCB_CLKSRC
 	bool "TC Block Clocksource"
-	depends on ATMEL_TCLIB && GENERIC_TIME
+	depends on ATMEL_TCLIB && GENERIC_TIME && !IPIPE
 	default y
 	help
 	  Select this to get a high precision clocksource based on a
diff --git a/drivers/pci/htirq.c b/drivers/pci/htirq.c
index 279c940..daabf90 100644
--- a/drivers/pci/htirq.c
+++ b/drivers/pci/htirq.c
@@ -21,7 +21,7 @@
  * With multiple simultaneous hypertransport irq devices it might pay
  * to make this more fine grained.  But start with simple, stupid, and correct.
  */
-static DEFINE_SPINLOCK(ht_irq_lock);
+static IPIPE_DEFINE_SPINLOCK(ht_irq_lock);
 
 struct ht_irq_cfg {
 	struct pci_dev *dev;
diff --git a/drivers/serial/8250.c b/drivers/serial/8250.c
index be95e55..eb76082 100644
--- a/drivers/serial/8250.c
+++ b/drivers/serial/8250.c
@@ -2609,6 +2609,51 @@ static int serial8250_console_early_setup(void)
 	return serial8250_find_port_for_earlycon();
 }
 
+#ifdef CONFIG_IPIPE_DEBUG
+
+#include <stdarg.h>
+
+void __ipipe_serial_debug(const char *fmt, ...)
+{
+        struct uart_8250_port *up = &serial8250_ports[0];
+        unsigned int ier, count;
+        unsigned long flags;
+        char buf[128];
+        va_list ap;
+
+        va_start(ap, fmt);
+        vsprintf(buf, fmt, ap);
+        va_end(ap);
+        count = strlen(buf);
+
+        touch_nmi_watchdog();
+
+        local_irq_save_hw(flags);
+
+        /*
+         *      First save the IER then disable the interrupts
+        */
+        ier = serial_in(up, UART_IER);
+
+        if (up->capabilities & UART_CAP_UUE)
+                serial_out(up, UART_IER, UART_IER_UUE);
+        else
+                serial_out(up, UART_IER, 0);
+
+        uart_console_write(&up->port, buf, count, serial8250_console_putchar);
+
+        /*
+         *      Finally, wait for transmitter to become empty
+         *      and restore the IER
+         */
+        wait_for_xmitr(up, BOTH_EMPTY);
+        serial_out(up, UART_IER, ier);
+
+        local_irq_restore_hw(flags);
+}
+
+#endif
+
 static struct uart_driver serial8250_reg;
 static struct console serial8250_console = {
 	.name		= "ttyS",
diff --git a/include/asm-arm/arch-at91/hardware.h b/include/asm-arm/arch-at91/hardware.h
index 2c826d8..887c226 100644
--- a/include/asm-arm/arch-at91/hardware.h
+++ b/include/asm-arm/arch-at91/hardware.h
@@ -61,6 +61,25 @@
 #define AT91_VA_BASE_SYS	AT91_IO_P2V(AT91_BASE_SYS)
 #define AT91_VA_BASE_EMAC	AT91_IO_P2V(AT91RM9200_BASE_EMAC)
 
+#ifdef CONFIG_IPIPE
+#if defined(CONFIG_ARCH_AT91RM9200)
+#define AT91_BASE_TCB0 AT91RM9200_BASE_TCB0
+#elif defined(CONFIG_ARCH_AT91SAM9260)
+#define AT91_BASE_TCB0 AT91SAM9260_BASE_TCB0
+#elif defined(CONFIG_ARCH_AT91SAM9261)
+#define AT91_BASE_TCB0 AT91SAM9261_BASE_TCB0
+#elif defined(CONFIG_ARCH_AT91SAM9263)
+#define AT91_BASE_TCB0 AT91SAM9263_BASE_TCB0
+#elif defined(CONFIG_ARCH_AT91SAM9RL)
+#define AT91_BASE_TCB0 AT91SAM9RL_BASE_TCB0
+#elif defined(CONFIG_ARCH_AT91X40)
+#define AT91_BASE_TCB0 (AT91_BASE_SYS + AT91_TC)
+#else
+#error "AT91 processor unsupported by Adeos"
+#endif
+#define AT91_VA_BASE_TCB0 AT91_IO_P2V(AT91_BASE_TCB0)
+#endif
+
  /* Internal SRAM is mapped below the IO devices */
 #define AT91_SRAM_MAX		SZ_1M
 #define AT91_VIRT_BASE		(AT91_IO_VIRT_BASE - AT91_SRAM_MAX)
diff --git a/include/asm-arm/arch-at91/irqs.h b/include/asm-arm/arch-at91/irqs.h
index 70b1216..8ecf341 100644
--- a/include/asm-arm/arch-at91/irqs.h
+++ b/include/asm-arm/arch-at91/irqs.h
@@ -45,4 +45,35 @@
 /* FIQ is AIC source 0. */
 #define FIQ_START AT91_ID_FIQ
 
+#if defined(CONFIG_IPIPE) && !defined(__ASSEMBLY__)
+extern unsigned __ipipe_at91_gpio_banks;
+
+#if defined(CONFIG_ARCH_AT91RM9200)
+#define __ipipe_mach_irq_mux_p(irq)					\
+	((unsigned) (irq - AT91RM9200_ID_PIOA) < __ipipe_at91_gpio_banks)
+
+#elif defined(CONFIG_ARCH_AT91SAM9260)
+#define __ipipe_mach_irq_mux_p(irq)					\
+	((unsigned) (irq - AT91SAM9260_ID_PIOA) < __ipipe_at91_gpio_banks)
+
+#elif defined(CONFIG_ARCH_AT91SAM9261)
+#define __ipipe_mach_irq_mux_p(irq)					\
+	((unsigned) (irq - AT91SAM9261_ID_PIOA) < __ipipe_at91_gpio_banks)
+
+#elif defined(CONFIG_ARCH_AT91SAM9263)
+#define __ipipe_mach_irq_mux_p(irq)					\
+	((unsigned) (irq - AT91SAM9263_ID_PIOA) < __ipipe_at91_gpio_banks)
+
+#elif defined(CONFIG_ARCH_AT91SAM9RL)
+#define __ipipe_mach_irq_mux_p(irq)					\
+	((unsigned) (irq - AT91SAM9RL_ID_PIOA) < __ipipe_at91_gpio_banks)
+
+#elif defined(CONFIG_ARCH_AT91X40)
+#define __ipipe_mach_irq_mux_p(irq)					\
+	((unsigned) (irq - AT91X40_ID_PIOA) < __ipipe_at91_gpio_banks)
+
+#endif /* CONFIG_ARCH_AT91X40 */
+
+#endif /* CONFIG_IPIPE && !__ASSEMBLY__ */
+
 #endif
diff --git a/include/asm-arm/arch-at91/timex.h b/include/asm-arm/arch-at91/timex.h
index f1933b0..afd3f9a 100644
--- a/include/asm-arm/arch-at91/timex.h
+++ b/include/asm-arm/arch-at91/timex.h
@@ -52,6 +52,6 @@
 #define AT91X40_MASTER_CLOCK	40000000
 #define CLOCK_TICK_RATE		(AT91X40_MASTER_CLOCK)
 
-#endif
+#endif /* arch specific */
 
 #endif
diff --git a/include/asm-arm/arch-imx/imx-regs.h b/include/asm-arm/arch-imx/imx-regs.h
index fb9de27..724694c 100644
--- a/include/asm-arm/arch-imx/imx-regs.h
+++ b/include/asm-arm/arch-imx/imx-regs.h
@@ -347,7 +347,16 @@
 #define IMX_INTDISNUM      __REG(IMX_AITC_BASE+0x0c)
 #define IMX_INTENABLEH     __REG(IMX_AITC_BASE+0x10)
 #define IMX_INTENABLEL     __REG(IMX_AITC_BASE+0x14)
-
+#define IMX_PRIO7          __REG(IMX_AITC_BASE+0x20)
+#define IMX_PRIO6          __REG(IMX_AITC_BASE+0x24)
+#define IMX_PRIO5          __REG(IMX_AITC_BASE+0x28)
+#define IMX_PRIO4          __REG(IMX_AITC_BASE+0x2C)
+#define IMX_PRIO3          __REG(IMX_AITC_BASE+0x30)
+#define IMX_PRIO2          __REG(IMX_AITC_BASE+0x34)
+#define IMX_PRIO1          __REG(IMX_AITC_BASE+0x38)
+#define IMX_PRIO0          __REG(IMX_AITC_BASE+0x3C)
+
+#define IMX_PRIO(x)	   __REG2(IMX_AITC_BASE+0x20,((7-((x)>>3)) <<2) ) 
 /*
  *  General purpose timers
  */
@@ -417,6 +426,7 @@
 #define PCR_BPIX_8      (3<<25)
 #define PCR_BPIX_12     (4<<25)
 #define PCR_BPIX_16     (4<<25)
+#define PCR_BPIX_MASK   (7<<25)
 #define PCR_PIXPOL      (1<<24)
 #define PCR_FLMPOL      (1<<23)
 #define PCR_LPPOL       (1<<22)
diff --git a/include/asm-arm/arch-imx/irqs.h b/include/asm-arm/arch-imx/irqs.h
index f195542..799ae66 100644
--- a/include/asm-arm/arch-imx/irqs.h
+++ b/include/asm-arm/arch-imx/irqs.h
@@ -113,4 +113,17 @@
 
 #define NR_IRQS (IRQ_GPIOD(32) + 1)
 #define IRQ_GPIO(x)
+
+#ifdef CONFIG_IPIPE
+#define __ipipe_irqbit(irq) (1ULL << (irq))
+
+#define __ipipe_muxed_irqmask (__ipipe_irqbit(GPIO_INT_PORTA) | \
+                               __ipipe_irqbit(GPIO_INT_PORTB) | \
+                               __ipipe_irqbit(GPIO_INT_PORTC) | \
+                               __ipipe_irqbit(GPIO_INT_PORTD))
+
+#define __ipipe_mach_irq_mux_p(irq) (__ipipe_irqbit(irq) \
+                                     & __ipipe_muxed_irqmask)
+#endif /* CONFIG_IPIPE */         
+
 #endif
diff --git a/include/asm-arm/arch-integrator/entry-macro.S b/include/asm-arm/arch-integrator/entry-macro.S
index 491af1a..495262b 100644
--- a/include/asm-arm/arch-integrator/entry-macro.S
+++ b/include/asm-arm/arch-integrator/entry-macro.S
@@ -28,7 +28,11 @@
 		teq	\irqstat, #0
 		ldreq	\irqstat, [\base, #(INTEGRATOR_HDR_IC_OFFSET+IRQ_STATUS)]
 		moveq	\irqnr, #IRQ_CIC_START
-
+#ifdef CONFIG_IPIPE
+		tst	\irqstat, #0x00000040			@ check IRQ_TIMERINT1 first
+		movne	\irqnr, #6
+		bne	1003f
+#endif /* CONFIG_IPIPE */
 1001:		tst	\irqstat, #15
 		bne	1002f
 		add	\irqnr, \irqnr, #4
diff --git a/include/asm-arm/arch-integrator/irqs.h b/include/asm-arm/arch-integrator/irqs.h
index ba7b3af..55513e8 100644
--- a/include/asm-arm/arch-integrator/irqs.h
+++ b/include/asm-arm/arch-integrator/irqs.h
@@ -80,3 +80,6 @@
 
 #define NR_IRQS                         47
 
+#ifdef CONFIG_IPIPE
+#define __ipipe_mach_irq_mux_p(irq) ((irq) == IRQ_CP_CPPLDINT)
+#endif /* CONFIG_IPIPE */
diff --git a/include/asm-arm/arch-integrator/platform.h b/include/asm-arm/arch-integrator/platform.h
index 83c4c1c..dbca380 100644
--- a/include/asm-arm/arch-integrator/platform.h
+++ b/include/asm-arm/arch-integrator/platform.h
@@ -26,7 +26,7 @@
  * 	NOTE: This is a multi-hosted header file for use with uHAL and
  * 	      supported debuggers.
  *
- * 	$Id: platform.s,v 1.32 2000/02/18 10:51:39 asims Exp $
+ * 	$Id: platform.h,v 1.2 2006/02/20 13:54:22 rpm Exp $
  *
  * ***********************************************************************/
 
@@ -436,7 +436,7 @@
  *  Timer definitions
  *
  *  Only use timer 1 & 2
- *  (both run at 24MHz and will need the clock divider set to 16).
+ *  (both run at 1MHZ on /CP and at 24MHz on /AP)
  *
  *  Timer 0 runs at bus frequency and therefore could vary and currently
  *  uHAL can't handle that.
@@ -449,7 +449,12 @@
 
 #define MAX_TIMER                       2
 #define MAX_PERIOD                      699050
+
+#ifdef CONFIG_ARCH_INTEGRATOR_CP
+#define TICKS_PER_uSEC                  1
+#else
 #define TICKS_PER_uSEC                  24
+#endif
 
 /*
  *  These are useconds NOT ticks.
diff --git a/include/asm-arm/arch-integrator/timex.h b/include/asm-arm/arch-integrator/timex.h
index 87a7628..7df2722 100644
--- a/include/asm-arm/arch-integrator/timex.h
+++ b/include/asm-arm/arch-integrator/timex.h
@@ -21,6 +21,6 @@
  */
 
 /*
- * ??
+ * Timer rate
  */
-#define CLOCK_TICK_RATE		(50000000 / 16)
+#define CLOCK_TICK_RATE		(1000000)
diff --git a/include/asm-arm/arch-ixp4xx/irqs.h b/include/asm-arm/arch-ixp4xx/irqs.h
index 1180160..8335c75 100644
--- a/include/asm-arm/arch-ixp4xx/irqs.h
+++ b/include/asm-arm/arch-ixp4xx/irqs.h
@@ -70,6 +70,10 @@
 
 #define	XSCALE_PMU_IRQ		(IRQ_IXP4XX_XSCALE_PMU)
 
+#ifdef CONFIG_IPIPE
+#define __ipipe_mach_irq_mux_p(irq) 0 /* We have no cascaded interrupts. */
+#endif
+
 /*
  * IXDP425 board IRQs
  */
diff --git a/include/asm-arm/arch-pxa/irqs.h b/include/asm-arm/arch-pxa/irqs.h
index b6c8fe3..bbfcec5 100644
--- a/include/asm-arm/arch-pxa/irqs.h
+++ b/include/asm-arm/arch-pxa/irqs.h
@@ -82,6 +82,10 @@
 #define IRQ_TO_GPIO_2_x(i)	((i) - PXA_GPIO_IRQ_BASE)
 #define IRQ_TO_GPIO(i)	(((i) < IRQ_GPIO(2)) ? ((i) - IRQ_GPIO0) : IRQ_TO_GPIO_2_x(i))
 
+#ifdef CONFIG_IPIPE
+#define __ipipe_mach_irq_mux_p(irq) ((irq) == IRQ_GPIO_2_x)
+#endif /* CONFIG_IPIPE */
+
 /*
  * The next 16 interrupts are for board specific purposes.  Since
  * the kernel can only run on one machine at a time, we can re-use
diff --git a/include/asm-arm/arch-s3c2410/irqs.h b/include/asm-arm/arch-s3c2410/irqs.h
index f5435d8..41dcd2f 100644
--- a/include/asm-arm/arch-s3c2410/irqs.h
+++ b/include/asm-arm/arch-s3c2410/irqs.h
@@ -3,6 +3,8 @@
  * Copyright (c) 2003-2005 Simtec Electronics
  *   Ben Dooks <ben@simtec.co.uk>
  *
+ * Copyright (C) 2006, 2007 Sebastian Smolorz <ssmolorz@emlix.com>, emlix GmbH
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
@@ -163,4 +165,38 @@
 /* Our FIQs are routable from IRQ_EINT0 to IRQ_ADCPARENT */
 #define FIQ_START		IRQ_EINT0
 
+#ifdef CONFIG_IPIPE
+#define __ipipe_irqbit(irq)	(1 << ((irq) - S3C2410_CPUIRQ_OFFSET))
+
+#ifdef CONFIG_CPU_S3C2440
+#define __ipipe_muxed_irqmask	(__ipipe_irqbit(IRQ_UART0)	|	\
+				 __ipipe_irqbit(IRQ_UART1)	|	\
+				 __ipipe_irqbit(IRQ_UART2)	|	\
+				 __ipipe_irqbit(IRQ_ADCPARENT)	|	\
+				 __ipipe_irqbit(IRQ_WDT)	|	\
+				 __ipipe_irqbit(IRQ_CAM)	|	\
+				 __ipipe_irqbit(IRQ_EINT4t7)	|	\
+				 __ipipe_irqbit(IRQ_EINT8t23))
+#elif defined CONFIG_CPU_S3C244X
+#define __ipipe_muxed_irqmask	(__ipipe_irqbit(IRQ_UART0)	|	\
+				 __ipipe_irqbit(IRQ_UART1)	|	\
+				 __ipipe_irqbit(IRQ_UART2)	|	\
+				 __ipipe_irqbit(IRQ_ADCPARENT)	|	\
+				 __ipipe_irqbit(IRQ_CAM)	|	\
+				 __ipipe_irqbit(IRQ_EINT4t7)	|	\
+				 __ipipe_irqbit(IRQ_EINT8t23))
+#else
+#define __ipipe_muxed_irqmask	(__ipipe_irqbit(IRQ_UART0)	|	\
+				 __ipipe_irqbit(IRQ_UART1)	|	\
+				 __ipipe_irqbit(IRQ_UART2)	|	\
+				 __ipipe_irqbit(IRQ_ADCPARENT)	|	\
+				 __ipipe_irqbit(IRQ_EINT4t7)	|	\
+				 __ipipe_irqbit(IRQ_EINT8t23))
+#endif
+
+#define __ipipe_mach_irq_mux_p(irq)	((irq) <= IRQ_ADCPARENT  &&	\
+					 (__ipipe_irqbit(irq) &		\
+					  __ipipe_muxed_irqmask))
+#endif /* CONFIG_IPIPE */
+
 #endif /* __ASM_ARCH_IRQ_H */
diff --git a/include/asm-arm/arch-sa1100/irqs.h b/include/asm-arm/arch-sa1100/irqs.h
index 7bf8048..a05927e 100644
--- a/include/asm-arm/arch-sa1100/irqs.h
+++ b/include/asm-arm/arch-sa1100/irqs.h
@@ -144,6 +144,10 @@
 #define IRQ_LOCOMO_SPI_REND	(IRQ_BOARD_END + 20)
 #define IRQ_LOCOMO_SPI_TEND	(IRQ_BOARD_END + 21)
 
+#ifdef CONFIG_IPIPE
+#define __ipipe_mach_irq_mux_p(irq) ((irq) == IRQ_GPIO11_27)
+#endif /* CONFIG_IPIPE */
+
 /*
  * Figure out the MAX IRQ number.
  *
diff --git a/include/asm-arm/assembler.h b/include/asm-arm/assembler.h
index fce8328..4789aa6 100644
--- a/include/asm-arm/assembler.h
+++ b/include/asm-arm/assembler.h
@@ -66,6 +66,18 @@
 	.macro	enable_irq
 	cpsie	i
 	.endm
+
+	.macro  disable_irq_cond
+#ifdef CONFIG_IPIPE
+	cpsid	i
+#endif /* CONFIG_IPIPE */
+	.endm
+
+	.macro  enable_irq_cond
+#ifdef CONFIG_IPIPE
+	cpsie	i
+#endif /* CONFIG_IPIPE */
+	.endm
 #else
 	.macro	disable_irq
 	msr	cpsr_c, #PSR_I_BIT | SVC_MODE
@@ -74,6 +86,18 @@
 	.macro	enable_irq
 	msr	cpsr_c, #SVC_MODE
 	.endm
+
+	.macro	disable_irq_cond
+#ifdef CONFIG_IPIPE
+	msr	cpsr_c, #PSR_I_BIT | SVC_MODE
+#endif /* CONFIG_IPIPE */
+	.endm
+
+	.macro	enable_irq_cond
+#ifdef CONFIG_IPIPE
+	msr	cpsr_c, #SVC_MODE
+#endif /* CONFIG_IPIPE */
+	.endm
 #endif
 
 /*
diff --git a/include/asm-arm/atomic.h b/include/asm-arm/atomic.h
index 3b59f94..16f93e5 100644
--- a/include/asm-arm/atomic.h
+++ b/include/asm-arm/atomic.h
@@ -129,10 +129,10 @@ static inline int atomic_add_return(int i, atomic_t *v)
 	unsigned long flags;
 	int val;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	val = v->counter;
 	v->counter = val += i;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return val;
 }
@@ -142,10 +142,10 @@ static inline int atomic_sub_return(int i, atomic_t *v)
 	unsigned long flags;
 	int val;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	val = v->counter;
 	v->counter = val -= i;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return val;
 }
@@ -155,11 +155,11 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
 	int ret;
 	unsigned long flags;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	ret = v->counter;
 	if (likely(ret == old))
 		v->counter = new;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return ret;
 }
@@ -168,9 +168,9 @@ static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
 {
 	unsigned long flags;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	*addr &= ~mask;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 }
 
 #endif /* __LINUX_ARM_ARCH__ */
diff --git a/include/asm-arm/bitops.h b/include/asm-arm/bitops.h
index 5c60bfc..d487753 100644
--- a/include/asm-arm/bitops.h
+++ b/include/asm-arm/bitops.h
@@ -41,9 +41,9 @@ static inline void ____atomic_set_bit(unsigned int bit, volatile unsigned long *
 
 	p += bit >> 5;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	*p |= mask;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 }
 
 static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long *p)
@@ -53,9 +53,9 @@ static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long
 
 	p += bit >> 5;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	*p &= ~mask;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 }
 
 static inline void ____atomic_change_bit(unsigned int bit, volatile unsigned long *p)
@@ -65,9 +65,9 @@ static inline void ____atomic_change_bit(unsigned int bit, volatile unsigned lon
 
 	p += bit >> 5;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	*p ^= mask;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 }
 
 static inline int
@@ -79,10 +79,10 @@ ____atomic_test_and_set_bit(unsigned int bit, volatile unsigned long *p)
 
 	p += bit >> 5;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	res = *p;
 	*p = res | mask;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return res & mask;
 }
@@ -96,10 +96,10 @@ ____atomic_test_and_clear_bit(unsigned int bit, volatile unsigned long *p)
 
 	p += bit >> 5;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	res = *p;
 	*p = res & ~mask;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return res & mask;
 }
@@ -113,10 +113,10 @@ ____atomic_test_and_change_bit(unsigned int bit, volatile unsigned long *p)
 
 	p += bit >> 5;
 
-	raw_local_irq_save(flags);
+	local_irq_save_hw(flags);
 	res = *p;
 	*p = res ^ mask;
-	raw_local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return res & mask;
 }
diff --git a/include/asm-arm/cacheflush.h b/include/asm-arm/cacheflush.h
index 759a97b..badb18a 100644
--- a/include/asm-arm/cacheflush.h
+++ b/include/asm-arm/cacheflush.h
@@ -15,6 +15,7 @@
 
 #include <asm/glue.h>
 #include <asm/shmparam.h>
+#include <asm/fcse.h>
 
 #define CACHE_COLOUR(vaddr)	((vaddr & (SHMLBA - 1)) >> PAGE_SHIFT)
 
@@ -339,16 +340,20 @@ static inline void flush_cache_mm(struct mm_struct *mm)
 static inline void
 flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
 {
-	if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask))
+	if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) {
+		start = fcse_va_to_mva(vma->vm_mm, start);
+		end = fcse_va_to_mva(vma->vm_mm, end);
 		__cpuc_flush_user_range(start & PAGE_MASK, PAGE_ALIGN(end),
 					vma->vm_flags);
+	}
 }
 
 static inline void
 flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn)
 {
 	if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) {
-		unsigned long addr = user_addr & PAGE_MASK;
+		unsigned long addr;
+		addr = fcse_va_to_mva(vma->vm_mm, user_addr) & PAGE_MASK;
 		__cpuc_flush_user_range(addr, addr + PAGE_SIZE, vma->vm_flags);
 	}
 }
@@ -379,8 +384,14 @@ extern void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
  * Harvard caches are synchronised for the user space address range.
  * This is used for the ARM private sys_cacheflush system call.
  */
-#define flush_cache_user_range(vma,start,end) \
-	__cpuc_coherent_user_range((start) & PAGE_MASK, PAGE_ALIGN(end))
+#define flush_cache_user_range(vma, start, end)				\
+	({								\
+		struct mm_struct *_mm = (vma)->vm_mm;			\
+		unsigned long _start, _end;				\
+		_start = fcse_va_to_mva(_mm, start) & PAGE_MASK;	\
+		_end = PAGE_ALIGN(fcse_va_to_mva(_mm, end));		\
+		__cpuc_coherent_user_range(_start, _end);		\
+	})
 
 /*
  * Perform necessary cache operations to ensure that data previously
@@ -417,7 +428,8 @@ static inline void flush_anon_page(struct vm_area_struct *vma,
 	extern void __flush_anon_page(struct vm_area_struct *vma,
 				struct page *, unsigned long);
 	if (PageAnon(page))
-		__flush_anon_page(vma, page, vmaddr);
+		__flush_anon_page(vma, page,
+				  fcse_va_to_mva(vma->vm_mm, vmaddr));
 }
 
 #define flush_dcache_mmap_lock(mapping) \
diff --git a/include/asm-arm/cpu-multi32.h b/include/asm-arm/cpu-multi32.h
index 3479de9..4a77f96 100644
--- a/include/asm-arm/cpu-multi32.h
+++ b/include/asm-arm/cpu-multi32.h
@@ -52,7 +52,8 @@ extern struct processor {
 	/*
 	 * Set the page table
 	 */
-	void (*switch_mm)(unsigned long pgd_phys, struct mm_struct *mm);
+	void (*switch_mm)(unsigned long pgd_phys,
+			  struct mm_struct *mm, unsigned cacheflush);
 	/*
 	 * Set a possibly extended PTE.  Non-extended PTEs should
 	 * ignore 'ext'.
@@ -66,4 +67,4 @@ extern struct processor {
 #define cpu_do_idle()			processor._do_idle()
 #define cpu_dcache_clean_area(addr,sz)	processor.dcache_clean_area(addr,sz)
 #define cpu_set_pte_ext(ptep,pte,ext)	processor.set_pte_ext(ptep,pte,ext)
-#define cpu_do_switch_mm(pgd,mm)	processor.switch_mm(pgd,mm)
+#define cpu_do_switch_mm(pgd, mm, flush) processor.switch_mm(pgd, mm, flush)
diff --git a/include/asm-arm/cpu-single.h b/include/asm-arm/cpu-single.h
index 0b120ee..28da4f9 100644
--- a/include/asm-arm/cpu-single.h
+++ b/include/asm-arm/cpu-single.h
@@ -39,6 +39,7 @@ extern void cpu_proc_init(void);
 extern void cpu_proc_fin(void);
 extern int cpu_do_idle(void);
 extern void cpu_dcache_clean_area(void *, int);
-extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm);
+extern void cpu_do_switch_mm(unsigned long pgd_phys,
+			     struct mm_struct *mm, unsigned cacheflush);
 extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext);
 extern void cpu_reset(unsigned long addr) __attribute__((noreturn));
diff --git a/include/asm-arm/fcse.h b/include/asm-arm/fcse.h
new file mode 100644
index 0000000..59752b3
--- /dev/null
+++ b/include/asm-arm/fcse.h
@@ -0,0 +1,79 @@
+/*
+ * Filename:    include/asm-arm/fcse.h
+ * Description: ARM Process ID (PID) includes for Fast Address Space Switching
+ *              (FASS) in ARM Linux.
+ * Created:     14/10/2001
+ * Changes:     19/02/2002 - Macros added.
+ *              03/08/2007 - Adapted to kernel 2.6.21 (ssm)
+ *              Feb 2008   - Simplified a bit (rco)
+ *
+ * Copyright:   (C) 2001, 2002 Adam Wiggins <awiggins@cse.unsw.edu.au>
+ *              (C) 2007 Sebastian Smolorz <ssm@emlix.com>
+ *              (C) 2008 Richard Cochran
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of teh GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef __ASM_ARM_FCSE_H
+#define __ASM_ARM_FCSE_H
+
+#include <linux/sched.h>
+
+#ifdef CONFIG_ARM_FCSE
+
+#define FCSE_PID_SHIFT 25
+
+/* Size of PID relocation area */
+#define FCSE_PID_TASK_SIZE (1UL << FCSE_PID_SHIFT)
+
+/* Mask to get rid of PID from relocated address */
+#define FCSE_PID_MASK (FCSE_PID_TASK_SIZE - 1)
+
+#define fcse_tlb_mask(mm) ((mm)->context.cpu_tlb_mask)
+#define fcse_cpu_set_vm_mask(cpu, mm) cpu_set(cpu, (mm)->cpu_vm_mask)
+#define fcse_needs_flush(mm) ((mm)->context.mappings_needing_flush)
+
+/* Sets the CPU's PID Register */
+static inline void fcse_pid_set(unsigned long pid)
+{
+	__asm__ __volatile__("mcr p15, 0, %0, c13, c0, 0": /* */: "r" (pid));
+}
+
+/* Returns the state of the CPU's PID Register */
+static inline unsigned long fcse_pid_get(void)
+{
+	unsigned long pid;
+	__asm__ __volatile__("mrc p15, 0, %0, c13, c0, 0" : "=&r" (pid));
+	return pid & ~FCSE_PID_MASK;
+}
+
+static inline unsigned long fcse_mva_to_va(unsigned long mva)
+{
+	unsigned long pid = fcse_pid_get();
+	if (pid && (pid == (mva & ~FCSE_PID_MASK)))
+		return mva & FCSE_PID_MASK;
+	return mva;
+}
+
+static inline unsigned long
+fcse_va_to_mva(struct mm_struct *mm, unsigned long va)
+{
+	if (va < FCSE_PID_TASK_SIZE)
+		return mm->context.pid | va;
+	return va;
+}
+
+int fcse_pid_alloc(void);
+void fcse_pid_free(unsigned pid);
+
+#else /* CONFIG_ARM_FCSE */
+#define fcse_pid_set(pid) do { } while (0)
+#define fcse_mva_to_va(x) (x)
+#define fcse_va_to_mva(mm, x) ({ (void)(mm); (x); })
+#define fcse_tlb_mask(mm) ((mm)->cpu_vm_mask)
+#define fcse_cpu_set_vm_mask(cpu, mm) do { } while (0)
+#define fcse_needs_flush(mm) (1)
+#endif
+
+#endif /* __ASM_ARM_FCSE_H */
diff --git a/include/asm-arm/ipipe.h b/include/asm-arm/ipipe.h
new file mode 100644
index 0000000..0ffe5b5
--- /dev/null
+++ b/include/asm-arm/ipipe.h
@@ -0,0 +1,198 @@
+/* -*- linux-c -*-
+ * include/asm-arm/ipipe.h
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ * Copyright (C) 2005 Stelian Pop.
+ * Copyright (C) 2006-2008 Gilles Chanteperdrix.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __ARM_IPIPE_H
+#define __ARM_IPIPE_H
+
+#ifdef CONFIG_IPIPE
+
+#include <linux/ipipe_percpu.h>
+
+#define IPIPE_ARCH_STRING	"1.11-01"
+#define IPIPE_MAJOR_NUMBER	1
+#define IPIPE_MINOR_NUMBER	11
+#define IPIPE_PATCH_NUMBER	1
+
+#ifdef CONFIG_SMP
+#error "I-pipe/arm: SMP not yet implemented"
+#define ipipe_processor_id()	(current_thread_info()->cpu)
+#else /* !CONFIG_SMP */
+#define ipipe_processor_id()	0
+#endif	/* CONFIG_SMP */
+
+#define smp_processor_id_hw() ipipe_processor_id()
+
+#define prepare_arch_switch(next) ipipe_schedule_notify(current, next)
+
+/* We would need to clear the SYNC flag for the root domain */
+/* over the current processor in SMP mode. */
+#define task_hijacked(p) !ipipe_root_domain_p
+
+extern unsigned long arm_return_addr(int level);
+
+#define BROKEN_BUILTIN_RETURN_ADDRESS
+#define __BUILTIN_RETURN_ADDRESS0 arm_return_addr(0)
+#define __BUILTIN_RETURN_ADDRESS1 arm_return_addr(1)
+
+
+struct ipipe_domain;
+
+#define IPIPE_TSC_TYPE_NONE        0
+#define IPIPE_TSC_TYPE_FREERUNNING 1
+#define IPIPE_TSC_TYPE_DECREMENTER 2
+
+struct __ipipe_tscinfo {
+        unsigned type;
+        union {
+                struct {
+                        unsigned *counter; /* Hw counter physical address */
+                        unsigned mask; /* Significant bits in the hw counter. */
+                        unsigned long long *tsc; /* 64 bits tsc value. */
+                } fr;
+                struct {
+			unsigned *counter; /* Hw counter physical address */
+			unsigned mask; /* Significant bits in the hw counter. */
+			unsigned *last_cnt; /* Counter value when updating
+						tsc value. */
+			unsigned long long *tsc; /* 64 bits tsc value. */
+                } dec;
+        } u;
+};
+
+struct ipipe_sysinfo {
+
+	int ncpus;		/* Number of CPUs on board */
+	u64 cpufreq;		/* CPU frequency (in Hz) */
+
+	/* Arch-dependent block */
+
+	struct {
+		unsigned tmirq;	/* Timer tick IRQ */
+		u64 tmfreq;	/* Timer frequency */
+                struct __ipipe_tscinfo tsc; /* exported data for u.s. tsc */
+	} archdep;
+};
+
+DECLARE_PER_CPU(struct mm_struct *,ipipe_active_mm);
+/* arch specific stuff */
+extern void *__ipipe_tsc_area;
+extern int __ipipe_mach_timerint;
+extern int __ipipe_mach_timerstolen;
+extern unsigned int __ipipe_mach_ticks_per_jiffy;
+extern void __ipipe_mach_acktimer(void);
+extern unsigned long long __ipipe_mach_get_tsc(void);
+extern void __ipipe_mach_set_dec(unsigned long);
+extern void __ipipe_mach_release_timer(void);
+extern unsigned long __ipipe_mach_get_dec(void);
+extern void __ipipe_mach_demux_irq(unsigned irq, struct pt_regs *regs);
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info);
+int __ipipe_check_tickdev(const char *devname);
+
+#define ipipe_read_tsc(t)		do { t = __ipipe_mach_get_tsc(); } while (0)
+#define __ipipe_read_timebase()		__ipipe_mach_get_tsc()
+
+#define ipipe_cpu_freq()	(HZ * __ipipe_mach_ticks_per_jiffy)
+#define ipipe_tsc2ns(t) \
+({ \
+	unsigned long long delta = (t)*1000; \
+	do_div(delta, ipipe_cpu_freq() / 1000000 + 1); \
+	(unsigned long)delta; \
+})
+#define ipipe_tsc2us(t) \
+({ \
+	unsigned long long delta = (t); \
+	do_div(delta, ipipe_cpu_freq() / 1000000 + 1); \
+	(unsigned long)delta; \
+})
+
+/* Private interface -- Internal use only */
+
+#define __ipipe_check_platform()	do { } while(0)
+
+#define __ipipe_init_platform()		do { } while(0)
+
+#define __ipipe_enable_irq(irq)	irq_desc[irq].chip->enable(irq)
+
+#define __ipipe_disable_irq(irq)	irq_desc[irq].chip->disable(irq)
+
+#define __ipipe_hook_critical_ipi(ipd) do { } while(0)
+
+void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq);
+
+void __ipipe_enable_pipeline(void);
+
+void __ipipe_do_critical_sync(unsigned irq,
+			      void *cookie);
+
+DECLARE_PER_CPU(struct pt_regs, __ipipe_tick_regs);
+
+int __ipipe_handle_irq(int irq,
+                       struct pt_regs *regs);
+
+#define ipipe_update_tick_evtdev(evtdev) do { } while (0)
+
+#define __ipipe_tick_irq	__ipipe_mach_timerint
+
+static inline unsigned long __ipipe_ffnz(unsigned long ul)
+{
+	return ffs(ul) - 1;
+}
+
+/* When running handlers, enable hw interrupts for all domains but the
+ * one heading the pipeline, so that IRQs can never be significantly
+ * deferred for the latter. */
+#define __ipipe_run_isr(ipd, irq)                                       \
+do {                                                                    \
+	local_irq_enable_nohead(ipd);                                   \
+	if (ipd == ipipe_root_domain) {                                 \
+                if (likely(!ipipe_virtual_irq_p(irq)))                  \
+                        ((void (*)(unsigned, struct pt_regs *))         \
+                         ipd->irqs[irq].handler) (irq,                  \
+                                                  &__raw_get_cpu_var(__ipipe_tick_regs)); \
+                else {                                                  \
+                        irq_enter();                                    \
+                        ipd->irqs[irq].handler(irq,ipd->irqs[irq].cookie); \
+                        irq_exit();                                     \
+                }                                                       \
+	} else {							\
+		__clear_bit(IPIPE_SYNC_FLAG, &ipipe_cpudom_var(ipd, status)); \
+		ipd->irqs[irq].handler(irq,ipd->irqs[irq].cookie);	\
+		__set_bit(IPIPE_SYNC_FLAG, &ipipe_cpudom_var(ipd, status)); \
+	}								\
+	local_irq_disable_hw();                                         \
+} while(0)
+
+#define __ipipe_syscall_watched_p(p, sc)                                \
+	(((p)->flags & PF_EVNOTIFY) || (unsigned long)sc >= __ARM_NR_BASE + 64)
+
+#else /* !CONFIG_IPIPE */
+
+#define task_hijacked(p)		0
+
+#define ipipe_update_tick_evtdev(evtdev)	do { } while (0)
+
+#define smp_processor_id_hw()		smp_processor_id()
+
+#endif /* CONFIG_IPIPE */
+
+#endif	/* !__ARM_IPIPE_H */
diff --git a/include/asm-arm/ipipe_base.h b/include/asm-arm/ipipe_base.h
new file mode 100644
index 0000000..5b73c9d
--- /dev/null
+++ b/include/asm-arm/ipipe_base.h
@@ -0,0 +1,107 @@
+/* -*- linux-c -*-
+ * include/asm-arm/ipipe_base.h
+ *
+ * Copyright (C) 2007 Gilles Chanteperdrix.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __ARM_IPIPE_BASE_H
+#define __ARM_IPIPE_BASE_H
+
+#include <linux/threads.h>
+#include <asm/irq.h>
+
+#define IPIPE_NR_XIRQS		NR_IRQS
+#define IPIPE_IRQ_ISHIFT	5	/* 25 for 32bits arch. */
+
+/* ARM traps */
+#define IPIPE_TRAP_ACCESS	 0	/* Data or instruction access exception */
+#define IPIPE_TRAP_SECTION	 1	/* Section fault */
+#define IPIPE_TRAP_DABT		 2	/* Generic data abort */
+#define IPIPE_TRAP_UNKNOWN	 3	/* Unknown exception */
+#define IPIPE_TRAP_BREAK	 4	/* Instruction breakpoint */
+#define IPIPE_TRAP_FPU		 5	/* Floating point exception */
+#define IPIPE_TRAP_VFP		 6	/* VFP floating point exception */
+#define IPIPE_TRAP_UNDEFINSTR	 7	/* Undefined instruction */
+#define IPIPE_TRAP_ALIGNMENT	 8	/* Unaligned access exception */
+#define IPIPE_NR_FAULTS		 9
+
+/* Pseudo-vectors used for kernel events */
+#define IPIPE_FIRST_EVENT	IPIPE_NR_FAULTS
+#define IPIPE_EVENT_SYSCALL	(IPIPE_FIRST_EVENT)
+#define IPIPE_EVENT_SCHEDULE	(IPIPE_FIRST_EVENT + 1)
+#define IPIPE_EVENT_SIGWAKE	(IPIPE_FIRST_EVENT + 2)
+#define IPIPE_EVENT_SETSCHED	(IPIPE_FIRST_EVENT + 3)
+#define IPIPE_EVENT_INIT	(IPIPE_FIRST_EVENT + 4)
+#define IPIPE_EVENT_EXIT	(IPIPE_FIRST_EVENT + 5)
+#define IPIPE_EVENT_CLEANUP	(IPIPE_FIRST_EVENT + 6)
+#define IPIPE_LAST_EVENT	IPIPE_EVENT_CLEANUP
+#define IPIPE_NR_EVENTS		(IPIPE_LAST_EVENT + 1)
+
+#ifndef __ASSEMBLY__
+
+#include <asm/irqflags.h>
+
+#ifdef CONFIG_SMP
+#error "SMP not implemented."
+#define __ipipe_root_status ipipe_root_cpudom_var(status)
+
+#else /* !CONFIG_SMP */
+
+#ifdef CONFIG_VFP
+#define __IPIPE_FEATURE_VFP_SAFE 1
+#endif
+
+#if __GNUC__ >= 4
+/* Alias to ipipe_root_cpudom_var(status) */
+extern unsigned long __ipipe_root_status;
+#else
+extern unsigned long *const __ipipe_root_status_addr;
+#define __ipipe_root_status	(*__ipipe_root_status_addr)
+#endif
+
+static inline void __ipipe_stall_root(void)
+{
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
+	__ipipe_root_status |= 1;
+	local_irq_restore_hw(flags);
+}
+
+static inline unsigned __ipipe_test_root(void)
+{
+	return __ipipe_root_status & 1;
+}
+
+static inline unsigned __ipipe_test_and_stall_root(void)
+{
+	unsigned long flags, res;
+
+	local_irq_save_hw(flags);
+	res = __ipipe_root_status;
+	__ipipe_root_status = res | 1;
+	local_irq_restore_hw(flags);
+
+	return res & 1;
+}
+
+#endif	/* CONFIG_SMP */
+
+#endif /* !__ASSEMBLY__ */
+
+#endif /* __ARM_IPIPE_BASE_H */
diff --git a/include/asm-arm/irqflags.h b/include/asm-arm/irqflags.h
index 6d09974..0aa4056 100644
--- a/include/asm-arm/irqflags.h
+++ b/include/asm-arm/irqflags.h
@@ -10,30 +10,30 @@
  */
 #if __LINUX_ARM_ARCH__ >= 6
 
-#define raw_local_irq_save(x)					\
+#define local_irq_save_hw_notrace(x)					\
 	({							\
 	__asm__ __volatile__(					\
-	"mrs	%0, cpsr		@ local_irq_save\n"	\
+	"mrs	%0, cpsr		@ local_irq_save_hw\n"	\
 	"cpsid	i"						\
 	: "=r" (x) : : "memory", "cc");				\
 	})
 
-#define raw_local_irq_enable()  __asm__("cpsie i	@ __sti" : : : "memory", "cc")
-#define raw_local_irq_disable() __asm__("cpsid i	@ __cli" : : : "memory", "cc")
-#define local_fiq_enable()  __asm__("cpsie f	@ __stf" : : : "memory", "cc")
-#define local_fiq_disable() __asm__("cpsid f	@ __clf" : : : "memory", "cc")
+#define local_irq_enable_hw_notrace()  __asm__("cpsie i	@ __sti" : : : "memory", "cc")
+#define local_irq_disable_hw_notrace() __asm__("cpsid i	@ __cli" : : : "memory", "cc")
+#define local_fiq_enable_hw_notrace()  __asm__("cpsie f	@ __stf" : : : "memory", "cc")
+#define local_fiq_disable_hw_notrace() __asm__("cpsid f	@ __clf" : : : "memory", "cc")
 
 #else
 
 /*
  * Save the current interrupt enable state & disable IRQs
  */
-#define raw_local_irq_save(x)					\
+#define local_irq_save_hw_notrace(x)					\
 	({							\
 		unsigned long temp;				\
 		(void) (&temp == &x);				\
 	__asm__ __volatile__(					\
-	"mrs	%0, cpsr		@ local_irq_save\n"	\
+	"mrs	%0, cpsr		@ local_irq_save_hw\n"	\
 "	orr	%1, %0, #128\n"					\
 "	msr	cpsr_c, %1"					\
 	: "=r" (x), "=r" (temp)					\
@@ -44,11 +44,11 @@
 /*
  * Enable IRQs
  */
-#define raw_local_irq_enable()					\
+#define local_irq_enable_hw_notrace()				\
 	({							\
 		unsigned long temp;				\
 	__asm__ __volatile__(					\
-	"mrs	%0, cpsr		@ local_irq_enable\n"	\
+	"mrs	%0, cpsr		@ local_irq_enable_hw\n"	\
 "	bic	%0, %0, #128\n"					\
 "	msr	cpsr_c, %0"					\
 	: "=r" (temp)						\
@@ -59,11 +59,11 @@
 /*
  * Disable IRQs
  */
-#define raw_local_irq_disable()					\
+#define local_irq_disable_hw_notrace()				\
 	({							\
 		unsigned long temp;				\
 	__asm__ __volatile__(					\
-	"mrs	%0, cpsr		@ local_irq_disable\n"	\
+	"mrs	%0, cpsr		@ local_irq_disable_hw\n"	\
 "	orr	%0, %0, #128\n"					\
 "	msr	cpsr_c, %0"					\
 	: "=r" (temp)						\
@@ -74,7 +74,7 @@
 /*
  * Enable FIQs
  */
-#define local_fiq_enable()					\
+#define local_fiq_enable_hw_notrace()				\
 	({							\
 		unsigned long temp;				\
 	__asm__ __volatile__(					\
@@ -89,7 +89,7 @@
 /*
  * Disable FIQs
  */
-#define local_fiq_disable()					\
+#define local_fiq_disable_hw_notrace()				\
 	({							\
 		unsigned long temp;				\
 	__asm__ __volatile__(					\
@@ -106,19 +106,19 @@
 /*
  * Save the current interrupt enable state.
  */
-#define raw_local_save_flags(x)					\
+#define local_save_flags_hw(x)					\
 	({							\
 	__asm__ __volatile__(					\
-	"mrs	%0, cpsr		@ local_save_flags"	\
+	"mrs	%0, cpsr		@ local_save_flags_hw"	\
 	: "=r" (x) : : "memory", "cc");				\
 	})
 
 /*
  * restore saved IRQ & FIQ state
  */
-#define raw_local_irq_restore(x)				\
+#define local_irq_restore_hw_notrace(x)				\
 	__asm__ __volatile__(					\
-	"msr	cpsr_c, %0		@ local_irq_restore\n"	\
+	"msr	cpsr_c, %0		@ local_irq_restore_hw\n"	\
 	:							\
 	: "r" (x)						\
 	: "memory", "cc")
@@ -128,5 +128,99 @@
 	(int)((flags) & PSR_I_BIT);	\
 })
 
+#define irqs_disabled_hw()			\
+({						\
+	unsigned long flags;			\
+	local_save_flags_hw(flags);		\
+	raw_irqs_disabled_flags(flags);		\
+})
+
+static inline unsigned long raw_mangle_irq_bits(int virt, unsigned long real)
+{
+	/* Merge virtual and real interrupt mask bits into a single
+	   32bit word. */
+	return (real & ~(1L << 8)) | ((virt != 0) << 8);
+}
+
+static inline int raw_demangle_irq_bits(unsigned long *x)
+{
+	int virt = (*x & (1 << 8)) != 0;
+	*x &= ~(1L << 8);
+	return virt;
+}
+
+#ifdef CONFIG_IPIPE
+
+void __ipipe_unstall_root(void);
+void __ipipe_restore_root(unsigned long flags);
+
+/* PSR_I_BIT is bit no. 7 and is set if interrupts are _disabled_ */
+#define raw_local_irq_save(flags)		((flags) = __ipipe_test_and_stall_root() << 7)
+#define raw_local_irq_enable()		__ipipe_unstall_root()
+#define raw_local_irq_disable()		__ipipe_stall_root()
+#define local_fiq_enable()		__ipipe_unstall_root()
+#define local_fiq_disable()		__ipipe_stall_root()
+#define raw_local_save_flags(flags)	((flags) = __ipipe_test_root() << 7)
+#define raw_local_irq_restore(flags)	__ipipe_restore_root(flags & (1 << 7))
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+
+#include <linux/ipipe_trace.h>
+
+#define local_irq_disable_hw() do { \
+	if (!irqs_disabled_hw()) { \
+		local_irq_disable_hw_notrace(); \
+		ipipe_trace_begin(0x80000000); \
+	} \
+} while (0)
+#define local_irq_enable_hw() do { \
+	if (irqs_disabled_hw()) { \
+		ipipe_trace_end(0x80000000); \
+		local_irq_enable_hw_notrace(); \
+	} \
+} while (0)
+#define local_irq_save_hw(x) do { \
+	local_save_flags_hw(x); \
+	if (!raw_irqs_disabled_flags(x)) { \
+		local_irq_disable_hw_notrace(); \
+		ipipe_trace_begin(0x80000001); \
+	} \
+} while (0)
+#define local_irq_restore_hw(x) do { \
+	if (!raw_irqs_disabled_flags(x)) \
+		ipipe_trace_end(0x80000001); \
+	local_irq_restore_hw_notrace(x); \
+} while (0)
+
+#else /* !CONFIG_IPIPE_TRACE_IRQSOFF */
+
+#define local_irq_save_hw(flags)	local_irq_save_hw_notrace(flags)
+#define local_irq_enable_hw()		local_irq_enable_hw_notrace()
+#define local_irq_disable_hw()		local_irq_disable_hw_notrace()
+#define local_fiq_enable_hw()		local_fiq_enable_hw_notrace()
+#define local_fiq_disable_hw()		local_fiq_disable_hw_notrace()
+#define local_irq_restore_hw(flags)	local_irq_restore_hw_notrace(flags)
+
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+
+#else /* !CONFIG_IPIPE */
+
+#define raw_local_irq_save(flags)	local_irq_save_hw_notrace(flags)
+#define raw_local_irq_enable()		local_irq_enable_hw_notrace()
+#define raw_local_irq_disable()		local_irq_disable_hw_notrace()
+#define local_fiq_enable()		local_fiq_enable_hw_notrace()
+#define local_fiq_disable()		local_fiq_disable_hw_notrace()
+#define raw_local_save_flags(flags)	local_save_flags_hw(flags)
+#define raw_local_irq_restore(flags)	local_irq_restore_hw_notrace(flags)
+
+#define local_irq_save_hw(flags)	local_irq_save_hw_notrace(flags)
+#define local_irq_enable_hw()		local_irq_enable_hw_notrace()
+#define local_irq_disable_hw()		local_irq_disable_hw_notrace()
+#define local_fiq_enable_hw()		local_fiq_enable_hw_notrace()
+#define local_fiq_disable_hw()		local_fiq_disable_hw_notrace()
+#define local_irq_restore_hw(flags)	local_irq_restore_hw_notrace(flags)
+
+#endif /* CONFIG_IPIPE */
+
 #endif
 #endif
diff --git a/include/asm-arm/memory.h b/include/asm-arm/memory.h
index 9ba4d71..bda4e74 100644
--- a/include/asm-arm/memory.h
+++ b/include/asm-arm/memory.h
@@ -34,14 +34,23 @@
  * TASK_SIZE - the maximum size of a user space task.
  * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area
  */
+#ifdef CONFIG_ARM_FCSE
+#define TASK_SIZE		UL(0x02000000)
+#define TASK_UNMAPPED_BASE	UL(0x01000000)
+#else
 #define TASK_SIZE		UL(0xbf000000)
 #define TASK_UNMAPPED_BASE	UL(0x40000000)
 #endif
+#endif
 
 /*
  * The maximum size of a 26-bit user space task.
  */
+#ifdef CONFIG_ARM_FCSE
+#define TASK_SIZE_26		UL(0x02000000)
+#else
 #define TASK_SIZE_26		UL(0x04000000)
+#endif
 
 /*
  * Page offset: 3GB
diff --git a/include/asm-arm/mmu.h b/include/asm-arm/mmu.h
index 53099d4..b74d736 100644
--- a/include/asm-arm/mmu.h
+++ b/include/asm-arm/mmu.h
@@ -7,6 +7,11 @@ typedef struct {
 #ifdef CONFIG_CPU_HAS_ASID
 	unsigned int id;
 #endif
+#ifdef CONFIG_ARM_FCSE
+	unsigned long pid;
+	unsigned mappings_needing_flush;
+	cpumask_t cpu_tlb_mask;
+#endif
 	unsigned int kvm_seq;
 } mm_context_t;
 
diff --git a/include/asm-arm/mmu_context.h b/include/asm-arm/mmu_context.h
index 6913d02..bcca4a2 100644
--- a/include/asm-arm/mmu_context.h
+++ b/include/asm-arm/mmu_context.h
@@ -17,6 +17,7 @@
 #include <asm/cacheflush.h>
 #include <asm/proc-fns.h>
 #include <asm-generic/mm_hooks.h>
+#include <asm/fcse.h>
 
 void __check_kvm_seq(struct mm_struct *mm);
 
@@ -64,7 +65,15 @@ static inline void check_context(struct mm_struct *mm)
 		__check_kvm_seq(mm);
 }
 
-#define init_new_context(tsk,mm)	0
+static inline int
+init_new_context(struct task_struct *tsk, struct mm_struct *mm)
+{
+#ifdef CONFIG_ARM_FCSE
+	cpus_clear(mm->context.cpu_tlb_mask);
+	mm->context.mappings_needing_flush = 0;
+#endif /* CONFIG_ARM_FCSE */
+	return 0;
+}
 
 #endif
 
@@ -95,13 +104,31 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next,
 	  struct task_struct *tsk)
 {
 #ifdef CONFIG_MMU
-	unsigned int cpu = smp_processor_id();
+	unsigned int cpu = smp_processor_id_hw();
 
-	if (!cpu_test_and_set(cpu, next->cpu_vm_mask) || prev != next) {
+	if (!cpu_test_and_set(cpu, fcse_tlb_mask(next)) || prev != next) {
+		fcse_cpu_set_vm_mask(cpu, next);
 		check_context(next);
-		cpu_switch_mm(next->pgd, next);
-		if (cache_is_vivt())
-			cpu_clear(cpu, prev->cpu_vm_mask);
+#if defined(CONFIG_IPIPE)
+		if (ipipe_current_domain == ipipe_root_domain) {
+			do {
+				per_cpu(ipipe_active_mm, cpu) = NULL; /* mm state is undefined. */
+				barrier();
+				fcse_pid_set(next->context.pid);
+				cpu_switch_mm(next->pgd, next,
+					      fcse_needs_flush(next));
+				barrier();
+				per_cpu(ipipe_active_mm, cpu) = next;
+			} while (test_and_clear_thread_flag(TIF_MMSWITCH_INT));
+		} else
+#endif /* CONFIG_IPIPE */
+			{
+				fcse_pid_set(next->context.pid);
+				cpu_switch_mm(next->pgd, next,
+					      fcse_needs_flush(next));
+			}
+		if (cache_is_vivt() && prev)
+			cpu_clear(cpu, fcse_tlb_mask(prev));
 	}
 #endif
 }
diff --git a/include/asm-arm/pgtable.h b/include/asm-arm/pgtable.h
index 5571c13..ad4a56a 100644
--- a/include/asm-arm/pgtable.h
+++ b/include/asm-arm/pgtable.h
@@ -344,10 +344,14 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd)
 /* to find an entry in a page-table-directory */
 #define pgd_index(addr)		((addr) >> PGDIR_SHIFT)
 
-#define pgd_offset(mm, addr)	((mm)->pgd+pgd_index(addr))
+#define pgd_offset(mm, addr)						\
+	({								\
+		struct mm_struct *_mm = (mm);				\
+		(_mm->pgd + pgd_index(fcse_va_to_mva(_mm, (addr))));	\
+	})
 
 /* to find an entry in a kernel page-table-directory */
-#define pgd_offset_k(addr)	pgd_offset(&init_mm, addr)
+#define pgd_offset_k(addr)	(init_mm.pgd+pgd_index(addr))
 
 /* Find an entry in the second-level page table.. */
 #define pmd_offset(dir, addr)	((pmd_t *)(dir))
diff --git a/include/asm-arm/proc-fns.h b/include/asm-arm/proc-fns.h
index 75ec760..481c431 100644
--- a/include/asm-arm/proc-fns.h
+++ b/include/asm-arm/proc-fns.h
@@ -223,7 +223,8 @@
 
 #ifdef CONFIG_MMU
 
-#define cpu_switch_mm(pgd,mm) cpu_do_switch_mm(virt_to_phys(pgd),mm)
+#define cpu_switch_mm(pgd, mm, cacheflush) \
+	cpu_do_switch_mm(virt_to_phys(pgd), mm, (cacheflush))
 
 #define cpu_get_pgd()	\
 	({						\
diff --git a/include/asm-arm/system.h b/include/asm-arm/system.h
index 514af79..59cf71d 100644
--- a/include/asm-arm/system.h
+++ b/include/asm-arm/system.h
@@ -265,7 +265,9 @@ extern struct task_struct *__switch_to(struct task_struct *, struct thread_info
 
 #define switch_to(prev,next,last)					\
 do {									\
-	last = __switch_to(prev,task_thread_info(prev), task_thread_info(next));	\
+	local_irq_disable_hw_cond();					\
+	last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \
+	local_irq_enable_hw_cond();					\
 } while (0)
 
 #if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110)
@@ -325,17 +327,17 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size
 #error SMP is not supported on this platform
 #endif
 	case 1:
-		raw_local_irq_save(flags);
+		local_irq_save_hw(flags);
 		ret = *(volatile unsigned char *)ptr;
 		*(volatile unsigned char *)ptr = x;
-		raw_local_irq_restore(flags);
+		local_irq_restore_hw(flags);
 		break;
 
 	case 4:
-		raw_local_irq_save(flags);
+		local_irq_save_hw(flags);
 		ret = *(volatile unsigned long *)ptr;
 		*(volatile unsigned long *)ptr = x;
-		raw_local_irq_restore(flags);
+		local_irq_restore_hw(flags);
 		break;
 #else
 	case 1:
diff --git a/include/asm-arm/thread_info.h b/include/asm-arm/thread_info.h
index f5a6647..36e9366 100644
--- a/include/asm-arm/thread_info.h
+++ b/include/asm-arm/thread_info.h
@@ -149,6 +149,9 @@ extern void iwmmxt_task_switch(struct thread_info *);
 #define TIF_USING_IWMMXT	17
 #define TIF_MEMDIE		18
 #define TIF_FREEZE		19
+#ifdef CONFIG_IPIPE
+#define TIF_MMSWITCH_INT        20
+#endif /* CONFIG_IPIPE */
 
 #define _TIF_SIGPENDING		(1 << TIF_SIGPENDING)
 #define _TIF_NEED_RESCHED	(1 << TIF_NEED_RESCHED)
@@ -156,6 +159,9 @@ extern void iwmmxt_task_switch(struct thread_info *);
 #define _TIF_POLLING_NRFLAG	(1 << TIF_POLLING_NRFLAG)
 #define _TIF_USING_IWMMXT	(1 << TIF_USING_IWMMXT)
 #define _TIF_FREEZE		(1 << TIF_FREEZE)
+#ifdef CONFIG_IPIPE
+#define _TIF_MMSWITCH_INT       (1 << TIF_MMSWITCH_INT)
+#endif /* CONFIG_IPIPE */
 
 /*
  * Change these and you break ASM code in entry-common.S
diff --git a/include/asm-arm/tlbflush.h b/include/asm-arm/tlbflush.h
index 8c6bc1b..542bea1 100644
--- a/include/asm-arm/tlbflush.h
+++ b/include/asm-arm/tlbflush.h
@@ -158,6 +158,7 @@
 #ifndef __ASSEMBLY__
 
 #include <linux/sched.h>
+#include <asm/fcse.h>
 
 struct cpu_tlb_fns {
 	void (*flush_user_range)(unsigned long, unsigned long, struct vm_area_struct *);
@@ -292,7 +293,7 @@ static inline void local_flush_tlb_mm(struct mm_struct *mm)
 	if (tlb_flag(TLB_WB))
 		dsb();
 
-	if (cpu_isset(smp_processor_id(), mm->cpu_vm_mask)) {
+	if (cpu_isset(smp_processor_id(), fcse_tlb_mask(mm))) {
 		if (tlb_flag(TLB_V3_FULL))
 			asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (zero) : "cc");
 		if (tlb_flag(TLB_V4_U_FULL))
@@ -325,12 +326,13 @@ local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
 	const int zero = 0;
 	const unsigned int __tlb_flag = __cpu_tlb_flags;
 
-	uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm);
+	uaddr = (fcse_va_to_mva(vma->vm_mm, uaddr) & PAGE_MASK)
+		| ASID(vma->vm_mm);
 
 	if (tlb_flag(TLB_WB))
 		dsb();
 
-	if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) {
+	if (cpu_isset(smp_processor_id(), fcse_tlb_mask(vma->vm_mm))) {
 		if (tlb_flag(TLB_V3_PAGE))
 			asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (uaddr) : "cc");
 		if (tlb_flag(TLB_V4_U_PAGE))
@@ -437,7 +439,15 @@ static inline void clean_pmd_entry(pmd_t *pmd)
 /*
  * Convert calls to our calling convention.
  */
-#define local_flush_tlb_range(vma,start,end)	__cpu_flush_user_tlb_range(start,end,vma)
+#define local_flush_tlb_range(vma, start, end)			\
+	({							\
+		struct mm_struct *_mm = (vma)->vm_mm;		\
+		unsigned long _start, _end;			\
+		_start = fcse_va_to_mva(_mm, start);		\
+		_end = fcse_va_to_mva(_mm, end);		\
+		__cpu_flush_user_tlb_range(_start, _end, vma);	\
+	})
+
 #define local_flush_tlb_kernel_range(s,e)	__cpu_flush_kern_tlb_range(s,e)
 
 #ifndef CONFIG_SMP
diff --git a/include/asm-generic/cmpxchg-local.h b/include/asm-generic/cmpxchg-local.h
index b2ba2fc..4764463 100644
--- a/include/asm-generic/cmpxchg-local.h
+++ b/include/asm-generic/cmpxchg-local.h
@@ -1,6 +1,7 @@
 #ifndef __ASM_GENERIC_CMPXCHG_LOCAL_H
 #define __ASM_GENERIC_CMPXCHG_LOCAL_H
 
+#include <asm/ipipe_base.h>
 #include <linux/types.h>
 
 extern unsigned long wrong_size_cmpxchg(volatile void *ptr);
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index 181006c..bc69d94 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -161,7 +161,22 @@ extern void irq_enter(void);
  */
 extern void irq_exit(void);
 
-#define nmi_enter()		do { lockdep_off(); __irq_enter(); } while (0)
-#define nmi_exit()		do { __irq_exit(); lockdep_on(); } while (0)
+#define nmi_enter() 					\
+	do {						\
+		ipipe_nmi_enter();			\
+		if (ipipe_root_domain_p) {		\
+			lockdep_off();			\
+			__irq_enter();			\
+		}					\
+	} while (0)
+
+#define nmi_exit()					\
+	do {						\
+		if (ipipe_root_domain_p) {		\
+			__irq_exit();			\
+			lockdep_on();			\
+		}					\
+		ipipe_nmi_exit();			\
+	} while (0)
 
 #endif /* LINUX_HARDIRQ_H */
diff --git a/include/linux/ipipe.h b/include/linux/ipipe.h
new file mode 100644
index 0000000..72d19b5
--- /dev/null
+++ b/include/linux/ipipe.h
@@ -0,0 +1,602 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe.h
+ *
+ * Copyright (C) 2002-2007 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_H
+#define __LINUX_IPIPE_H
+
+#include <linux/spinlock.h>
+#include <linux/cache.h>
+#include <linux/percpu.h>
+#include <linux/mutex.h>
+#include <linux/linkage.h>
+#include <linux/ipipe_base.h>
+#include <linux/ipipe_compat.h>
+#include <asm/ipipe.h>
+
+#ifdef CONFIG_IPIPE
+
+/*
+ * Sanity check: IPIPE_VIRQ_BASE depends on CONFIG_NR_CPUS, and if the
+ * latter gets too large, we fail to map the virtual interrupts.
+ */
+#if IPIPE_VIRQ_BASE / BITS_PER_LONG > BITS_PER_LONG
+#error "CONFIG_NR_CPUS is too large, please lower it."
+#endif
+
+#define IPIPE_VERSION_STRING	IPIPE_ARCH_STRING
+#define IPIPE_RELEASE_NUMBER	((IPIPE_MAJOR_NUMBER << 16) | \
+				 (IPIPE_MINOR_NUMBER <<  8) | \
+				 (IPIPE_PATCH_NUMBER))
+
+#ifndef BROKEN_BUILTIN_RETURN_ADDRESS
+#define __BUILTIN_RETURN_ADDRESS0 ((unsigned long)__builtin_return_address(0))
+#define __BUILTIN_RETURN_ADDRESS1 ((unsigned long)__builtin_return_address(1))
+#endif /* !BUILTIN_RETURN_ADDRESS */
+
+#define IPIPE_ROOT_PRIO		100
+#define IPIPE_ROOT_ID		0
+#define IPIPE_ROOT_NPTDKEYS	4	/* Must be <= BITS_PER_LONG */
+
+#define IPIPE_RESET_TIMER	0x1
+#define IPIPE_GRAB_TIMER	0x2
+
+/* Global domain flags */
+#define IPIPE_SPRINTK_FLAG	0	/* Synchronous printk() allowed */
+#define IPIPE_AHEAD_FLAG	1	/* Domain always heads the pipeline */
+
+/* Interrupt control bits */
+#define IPIPE_HANDLE_FLAG	0
+#define IPIPE_PASS_FLAG		1
+#define IPIPE_ENABLE_FLAG	2
+#define IPIPE_DYNAMIC_FLAG	IPIPE_HANDLE_FLAG
+#define IPIPE_STICKY_FLAG	3
+#define IPIPE_SYSTEM_FLAG	4
+#define IPIPE_LOCK_FLAG		5
+#define IPIPE_WIRED_FLAG	6
+#define IPIPE_EXCLUSIVE_FLAG	7
+
+#define IPIPE_HANDLE_MASK	(1 << IPIPE_HANDLE_FLAG)
+#define IPIPE_PASS_MASK		(1 << IPIPE_PASS_FLAG)
+#define IPIPE_ENABLE_MASK	(1 << IPIPE_ENABLE_FLAG)
+#define IPIPE_DYNAMIC_MASK	IPIPE_HANDLE_MASK
+#define IPIPE_STICKY_MASK	(1 << IPIPE_STICKY_FLAG)
+#define IPIPE_SYSTEM_MASK	(1 << IPIPE_SYSTEM_FLAG)
+#define IPIPE_LOCK_MASK		(1 << IPIPE_LOCK_FLAG)
+#define IPIPE_WIRED_MASK	(1 << IPIPE_WIRED_FLAG)
+#define IPIPE_EXCLUSIVE_MASK	(1 << IPIPE_EXCLUSIVE_FLAG)
+
+#define IPIPE_DEFAULT_MASK	(IPIPE_HANDLE_MASK|IPIPE_PASS_MASK)
+#define IPIPE_STDROOT_MASK	(IPIPE_HANDLE_MASK|IPIPE_PASS_MASK|IPIPE_SYSTEM_MASK)
+
+#define IPIPE_EVENT_SELF        0x80000000
+
+#define IPIPE_NR_CPUS		NR_CPUS
+
+#define ipipe_current_domain	ipipe_cpu_var(ipipe_percpu_domain)
+
+#define ipipe_virtual_irq_p(irq)	((irq) >= IPIPE_VIRQ_BASE && \
+					 (irq) < IPIPE_NR_IRQS)
+
+#define IPIPE_SAME_HANDLER	((ipipe_irq_handler_t)(-1))
+
+typedef int (*ipipe_irq_ackfn_t)(unsigned irq);
+
+typedef int (*ipipe_event_handler_t)(unsigned event,
+				     struct ipipe_domain *from,
+				     void *data);
+struct ipipe_domain {
+
+	int slot;			/* Slot number in percpu domain data array. */
+	struct list_head p_link;	/* Link in pipeline */
+	ipipe_event_handler_t evhand[IPIPE_NR_EVENTS]; /* Event handlers. */
+	unsigned long long evself;	/* Self-monitored event bits. */
+
+	struct {
+		unsigned long control;
+		ipipe_irq_ackfn_t acknowledge;
+		ipipe_irq_handler_t handler;
+		void *cookie;
+	} ____cacheline_aligned irqs[IPIPE_NR_IRQS];
+
+	int priority;
+	void *pdd;
+	unsigned long flags;
+	unsigned domid;
+	const char *name;
+	struct mutex mutex;
+};
+
+#define IPIPE_HEAD_PRIORITY	(-1) /* For domains always heading the pipeline */
+
+struct ipipe_domain_attr {
+
+	unsigned domid;		/* Domain identifier -- Magic value set by caller */
+	const char *name;	/* Domain name -- Warning: won't be dup'ed! */
+	int priority;		/* Priority in interrupt pipeline */
+	void (*entry) (void);	/* Domain entry point */
+	void *pdd;		/* Per-domain (opaque) data pointer */
+};
+
+#ifdef CONFIG_SMP
+/* These ops must start and complete on the same CPU: care for
+ * migration. */
+#define set_bit_safe(b, a)						\
+		({ unsigned long __flags;				\
+		local_irq_save_hw_notrace(__flags);			\
+		__set_bit(b, a);					\
+		local_irq_restore_hw_notrace(__flags); })
+#define test_and_set_bit_safe(b, a)					\
+		({ unsigned long __flags, __x;				\
+		local_irq_save_hw_notrace(__flags);			\
+		__x = __test_and_set_bit(b, a);				\
+		local_irq_restore_hw_notrace(__flags); __x; })
+#define clear_bit_safe(b, a)						\
+		({ unsigned long __flags;				\
+		local_irq_save_hw_notrace(__flags);			\
+		__clear_bit(b, a);					\
+		local_irq_restore_hw_notrace(__flags); })
+#else
+#define set_bit_safe(b, a)		set_bit(b, a)
+#define test_and_set_bit_safe(b, a)	test_and_set_bit(b, a)
+#define clear_bit_safe(b, a)		clear_bit(b, a)
+#endif
+
+#define __ipipe_irq_cookie(ipd, irq)		(ipd)->irqs[irq].cookie
+#define __ipipe_irq_handler(ipd, irq)		(ipd)->irqs[irq].handler
+#define __ipipe_cpudata_irq_hits(ipd, cpu, irq)	ipipe_percpudom(ipd, irqall, cpu)[irq]
+
+extern unsigned __ipipe_printk_virq;
+
+extern unsigned long __ipipe_virtual_irq_map;
+
+extern struct list_head __ipipe_pipeline;
+
+extern int __ipipe_event_monitors[];
+
+/* Private interface */
+
+void ipipe_init(void);
+
+#ifdef CONFIG_PROC_FS
+void ipipe_init_proc(void);
+
+#ifdef CONFIG_IPIPE_TRACE
+void __ipipe_init_tracer(void);
+#else /* !CONFIG_IPIPE_TRACE */
+#define __ipipe_init_tracer()       do { } while(0)
+#endif /* CONFIG_IPIPE_TRACE */
+
+#else	/* !CONFIG_PROC_FS */
+#define ipipe_init_proc()	do { } while(0)
+#endif	/* CONFIG_PROC_FS */
+
+void __ipipe_init_stage(struct ipipe_domain *ipd);
+
+void __ipipe_cleanup_domain(struct ipipe_domain *ipd);
+
+void __ipipe_add_domain_proc(struct ipipe_domain *ipd);
+
+void __ipipe_remove_domain_proc(struct ipipe_domain *ipd);
+
+void __ipipe_flush_printk(unsigned irq, void *cookie);
+
+void __ipipe_walk_pipeline(struct list_head *pos);
+
+int __ipipe_schedule_irq(unsigned irq, struct list_head *head);
+
+int __ipipe_dispatch_event(unsigned event, void *data);
+
+int __ipipe_dispatch_wired(struct ipipe_domain *head_domain, unsigned irq);
+
+void __ipipe_sync_stage(unsigned long syncmask);
+
+void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned irq);
+
+void __ipipe_lock_irq(struct ipipe_domain *ipd, int cpu, unsigned irq);
+
+void __ipipe_unlock_irq(struct ipipe_domain *ipd, unsigned irq);
+
+void __ipipe_pin_range_globally(unsigned long start, unsigned long end);
+
+/* Must be called hw IRQs off. */
+static inline void ipipe_irq_lock(unsigned irq)
+{
+	__ipipe_lock_irq(ipipe_current_domain, ipipe_processor_id(), irq);
+}
+
+/* Must be called hw IRQs off. */
+static inline void ipipe_irq_unlock(unsigned irq)
+{
+	__ipipe_unlock_irq(ipipe_current_domain, irq);
+}
+
+#ifndef __ipipe_sync_pipeline
+#define __ipipe_sync_pipeline(syncmask) __ipipe_sync_stage(syncmask)
+#endif
+
+#ifndef __ipipe_run_irqtail
+#define __ipipe_run_irqtail() do { } while(0)
+#endif
+
+#define __ipipe_pipeline_head_p(ipd) (&(ipd)->p_link == __ipipe_pipeline.next)
+
+/*
+ * Keep the following as a macro, so that client code could check for
+ * the support of the invariant pipeline head optimization.
+ */
+#define __ipipe_pipeline_head() list_entry(__ipipe_pipeline.next,struct ipipe_domain,p_link)
+
+#define __ipipe_event_monitored_p(ev) \
+	(__ipipe_event_monitors[ev] > 0 || (ipipe_current_domain->evself & (1LL << ev)))
+
+#ifdef CONFIG_SMP
+
+cpumask_t __ipipe_set_irq_affinity(unsigned irq,
+				   cpumask_t cpumask);
+
+int __ipipe_send_ipi(unsigned ipi,
+			      cpumask_t cpumask);
+
+#endif /* CONFIG_SMP */
+
+#define ipipe_sigwake_notify(p)	\
+do {					\
+	if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_SIGWAKE)) \
+		__ipipe_dispatch_event(IPIPE_EVENT_SIGWAKE,p);		\
+} while(0)
+
+#define ipipe_exit_notify(p)	\
+do {				\
+	if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_EXIT)) \
+		__ipipe_dispatch_event(IPIPE_EVENT_EXIT,p);		\
+} while(0)
+
+#define ipipe_setsched_notify(p)	\
+do {					\
+	if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_SETSCHED)) \
+		__ipipe_dispatch_event(IPIPE_EVENT_SETSCHED,p);		\
+} while(0)
+
+#define ipipe_schedule_notify(prev, next)				\
+do {									\
+	if ((((prev)->flags|(next)->flags) & PF_EVNOTIFY) &&		\
+	    __ipipe_event_monitored_p(IPIPE_EVENT_SCHEDULE))		\
+		__ipipe_dispatch_event(IPIPE_EVENT_SCHEDULE,next);	\
+} while(0)
+
+#define ipipe_trap_notify(ex, regs)		\
+({						\
+	int ret = 0;				\
+	if ((test_bit(IPIPE_NOSTACK_FLAG, &ipipe_this_cpudom_var(status)) || \
+	     ((current)->flags & PF_EVNOTIFY)) &&			\
+	    __ipipe_event_monitored_p(ex))				\
+		ret = __ipipe_dispatch_event(ex, regs);			\
+	ret;								\
+})
+
+static inline void ipipe_init_notify(struct task_struct *p)
+{
+	if (__ipipe_event_monitored_p(IPIPE_EVENT_INIT))
+		__ipipe_dispatch_event(IPIPE_EVENT_INIT,p);
+}
+
+struct mm_struct;
+
+static inline void ipipe_cleanup_notify(struct mm_struct *mm)
+{
+	if (__ipipe_event_monitored_p(IPIPE_EVENT_CLEANUP))
+		__ipipe_dispatch_event(IPIPE_EVENT_CLEANUP,mm);
+}
+
+/* Public interface */
+
+int ipipe_register_domain(struct ipipe_domain *ipd,
+			  struct ipipe_domain_attr *attr);
+
+int ipipe_unregister_domain(struct ipipe_domain *ipd);
+
+void ipipe_suspend_domain(void);
+
+int ipipe_virtualize_irq(struct ipipe_domain *ipd,
+			 unsigned irq,
+			 ipipe_irq_handler_t handler,
+			 void *cookie,
+			 ipipe_irq_ackfn_t acknowledge,
+			 unsigned modemask);
+
+int ipipe_control_irq(unsigned irq,
+		      unsigned clrmask,
+		      unsigned setmask);
+
+unsigned ipipe_alloc_virq(void);
+
+int ipipe_free_virq(unsigned virq);
+
+int ipipe_trigger_irq(unsigned irq);
+
+static inline int ipipe_propagate_irq(unsigned irq)
+{
+	return __ipipe_schedule_irq(irq, ipipe_current_domain->p_link.next);
+}
+
+static inline int ipipe_schedule_irq(unsigned irq)
+{
+	return __ipipe_schedule_irq(irq, &ipipe_current_domain->p_link);
+}
+
+void ipipe_stall_pipeline_from(struct ipipe_domain *ipd);
+
+unsigned long ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd);
+
+void ipipe_unstall_pipeline_from(struct ipipe_domain *ipd);
+
+unsigned long ipipe_test_and_unstall_pipeline_from(struct ipipe_domain *ipd);
+
+void ipipe_restore_pipeline_from(struct ipipe_domain *ipd,
+					  unsigned long x);
+
+static inline unsigned long ipipe_test_pipeline_from(struct ipipe_domain *ipd)
+{
+	return test_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status));
+}
+
+static inline void ipipe_stall_pipeline_head(void)
+{
+	local_irq_disable_hw();
+	__set_bit(IPIPE_STALL_FLAG, &ipipe_head_cpudom_var(status));
+}
+
+static inline unsigned long ipipe_test_and_stall_pipeline_head(void)
+{
+	local_irq_disable_hw();
+	return __test_and_set_bit(IPIPE_STALL_FLAG, &ipipe_head_cpudom_var(status));
+}
+
+void ipipe_unstall_pipeline_head(void);
+
+void __ipipe_restore_pipeline_head(unsigned long x);
+
+static inline void ipipe_restore_pipeline_head(unsigned long x)
+{
+	/* On some archs, __test_and_set_bit() might return different
+	 * truth value than test_bit(), so we test the exclusive OR of
+	 * both statuses, assuming that the lowest bit is always set in
+	 * the truth value (if this is wrong, the failed optimization will
+	 * be caught in __ipipe_restore_pipeline_head() if
+	 * CONFIG_DEBUG_KERNEL is set). */
+	if ((x ^ test_bit(IPIPE_STALL_FLAG, &ipipe_head_cpudom_var(status))) & 1)
+		__ipipe_restore_pipeline_head(x);
+}
+
+#define ipipe_unstall_pipeline() \
+	ipipe_unstall_pipeline_from(ipipe_current_domain)
+
+#define ipipe_test_and_unstall_pipeline() \
+	ipipe_test_and_unstall_pipeline_from(ipipe_current_domain)
+
+#define ipipe_test_pipeline() \
+	ipipe_test_pipeline_from(ipipe_current_domain)
+
+#define ipipe_test_and_stall_pipeline() \
+	ipipe_test_and_stall_pipeline_from(ipipe_current_domain)
+
+#define ipipe_stall_pipeline() \
+	ipipe_stall_pipeline_from(ipipe_current_domain)
+
+#define ipipe_restore_pipeline(x) \
+	ipipe_restore_pipeline_from(ipipe_current_domain, (x))
+
+void ipipe_init_attr(struct ipipe_domain_attr *attr);
+
+int ipipe_get_sysinfo(struct ipipe_sysinfo *sysinfo);
+
+unsigned long ipipe_critical_enter(void (*syncfn) (void));
+
+void ipipe_critical_exit(unsigned long flags);
+
+static inline void ipipe_set_printk_sync(struct ipipe_domain *ipd)
+{
+	set_bit(IPIPE_SPRINTK_FLAG, &ipd->flags);
+}
+
+static inline void ipipe_set_printk_async(struct ipipe_domain *ipd)
+{
+	clear_bit(IPIPE_SPRINTK_FLAG, &ipd->flags);
+}
+
+static inline void ipipe_set_foreign_stack(struct ipipe_domain *ipd)
+{
+	/* Must be called hw interrupts off. */
+	__set_bit(IPIPE_NOSTACK_FLAG, &ipipe_cpudom_var(ipd, status));
+}
+
+static inline void ipipe_clear_foreign_stack(struct ipipe_domain *ipd)
+{
+	/* Must be called hw interrupts off. */
+	__clear_bit(IPIPE_NOSTACK_FLAG, &ipipe_cpudom_var(ipd, status));
+}
+
+#ifndef ipipe_safe_current
+#define ipipe_safe_current()					\
+({								\
+	struct task_struct *p;					\
+	p = test_bit(IPIPE_NOSTACK_FLAG,			\
+		     &ipipe_this_cpudom_var(status)) ? &init_task : current; \
+	p; \
+})
+#endif
+
+ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd,
+					unsigned event,
+					ipipe_event_handler_t handler);
+
+cpumask_t ipipe_set_irq_affinity(unsigned irq,
+				 cpumask_t cpumask);
+
+int ipipe_send_ipi(unsigned ipi,
+			    cpumask_t cpumask);
+
+int ipipe_setscheduler_root(struct task_struct *p,
+			    int policy,
+			    int prio);
+
+int ipipe_reenter_root(struct task_struct *prev,
+		       int policy,
+		       int prio);
+
+int ipipe_alloc_ptdkey(void);
+
+int ipipe_free_ptdkey(int key);
+
+int ipipe_set_ptd(int key,
+			   void *value);
+
+void *ipipe_get_ptd(int key);
+
+int ipipe_disable_ondemand_mappings(struct task_struct *tsk);
+
+#define local_irq_enable_hw_cond()		local_irq_enable_hw()
+#define local_irq_disable_hw_cond()		local_irq_disable_hw()
+#define local_irq_save_hw_cond(flags)		local_irq_save_hw(flags)
+#define local_irq_restore_hw_cond(flags)	local_irq_restore_hw(flags)
+
+#define local_irq_enable_nohead(ipd)			\
+	do {						\
+		if (!__ipipe_pipeline_head_p(ipd))	\
+			local_irq_enable_hw();		\
+	} while(0)
+
+#define local_irq_disable_nohead(ipd)		\
+	do {						\
+		if (!__ipipe_pipeline_head_p(ipd))	\
+			local_irq_disable_hw();		\
+	} while(0)
+
+#define local_irq_save_full(vflags, rflags)		\
+	do {						\
+		local_irq_save(vflags);			\
+		local_irq_save_hw(rflags);		\
+	} while(0)
+
+#define local_irq_restore_full(vflags, rflags)		\
+	do {						\
+		local_irq_restore_hw(rflags);		\
+		local_irq_restore(vflags);		\
+	} while(0)
+
+static inline void local_irq_restore_nosync(unsigned long x)
+{
+	if (x)
+		set_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipipe_root_domain, status));
+	else
+		clear_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipipe_root_domain, status));
+}
+
+#define ipipe_root_domain_p		(ipipe_current_domain == ipipe_root_domain)
+
+#else	/* !CONFIG_IPIPE */
+
+#define ipipe_init()			do { } while(0)
+#define ipipe_suspend_domain()		do { } while(0)
+#define ipipe_sigwake_notify(p)		do { } while(0)
+#define ipipe_setsched_notify(p)	do { } while(0)
+#define ipipe_init_notify(p)		do { } while(0)
+#define ipipe_exit_notify(p)		do { } while(0)
+#define ipipe_cleanup_notify(mm)	do { } while(0)
+#define ipipe_trap_notify(t,r)		0
+#define ipipe_init_proc()		do { } while(0)
+#define __ipipe_pin_range_globally(start, end)	do { } while(0)
+
+#define local_irq_enable_hw_cond()		do { } while(0)
+#define local_irq_disable_hw_cond()		do { } while(0)
+#define local_irq_save_hw_cond(flags)		do { (void)(flags); } while(0)
+#define local_irq_restore_hw_cond(flags)	do { } while(0)
+
+#define ipipe_irq_lock(irq)		do { } while(0)
+#define ipipe_irq_unlock(irq)		do { } while(0)
+
+#define ipipe_root_domain_p		1
+#define ipipe_safe_current		current
+
+#define local_irq_disable_head()	local_irq_disable()
+
+#define local_irq_save_full(vflags, rflags)	do { (void)(vflags); local_irq_save(rflags); } while(0)
+#define local_irq_restore_full(vflags, rflags)	do { (void)(vflags); local_irq_restore(rflags); } while(0)
+#define local_irq_restore_nosync(vflags)	local_irq_restore(vflags)
+
+#endif	/* CONFIG_IPIPE */
+
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+
+#include <linux/cpumask.h>
+#include <asm/system.h>
+
+static inline int ipipe_disable_context_check(int cpu)
+{
+	return xchg(&per_cpu(ipipe_percpu_context_check, cpu), 0);
+}
+
+static inline void ipipe_restore_context_check(int cpu, int old_state)
+{
+	per_cpu(ipipe_percpu_context_check, cpu) = old_state;
+}
+
+static inline void ipipe_context_check_off(void)
+{
+	int cpu;
+	for_each_online_cpu(cpu)
+		per_cpu(ipipe_percpu_context_check, cpu) = 0;
+}
+
+static inline void ipipe_nmi_enter(void)
+{
+	int cpu = ipipe_processor_id();
+
+	per_cpu(ipipe_saved_context_check_state, cpu) =
+		ipipe_disable_context_check(cpu);
+}
+
+static inline void ipipe_nmi_exit(void)
+{
+	int cpu = ipipe_processor_id();
+
+	ipipe_restore_context_check
+		(cpu, per_cpu(ipipe_saved_context_check_state, cpu));
+}
+
+#else	/* !CONFIG_IPIPE_DEBUG_CONTEXT */
+
+static inline int ipipe_disable_context_check(int cpu)
+{
+	return 0;
+}
+
+static inline void ipipe_restore_context_check(int cpu, int old_state) { }
+
+static inline void ipipe_context_check_off(void) { }
+
+static inline void ipipe_nmi_enter(void) { }
+
+static inline void ipipe_nmi_exit(void) { }
+
+#endif	/* !CONFIG_IPIPE_DEBUG_CONTEXT */
+
+#endif	/* !__LINUX_IPIPE_H */
diff --git a/include/linux/ipipe_base.h b/include/linux/ipipe_base.h
new file mode 100644
index 0000000..d359b82
--- /dev/null
+++ b/include/linux/ipipe_base.h
@@ -0,0 +1,88 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_base.h
+ *
+ * Copyright (C) 2002-2007 Philippe Gerum.
+ *               2007 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_BASE_H
+#define __LINUX_IPIPE_BASE_H
+
+#ifdef CONFIG_IPIPE
+
+#include <linux/bitops.h>
+#include <asm/ipipe_base.h>
+
+/* Number of virtual IRQs */
+#define IPIPE_NR_VIRQS		BITS_PER_LONG
+/* First virtual IRQ # */
+#define IPIPE_VIRQ_BASE		(((IPIPE_NR_XIRQS + BITS_PER_LONG - 1) / BITS_PER_LONG) * BITS_PER_LONG)
+/* Total number of IRQ slots */
+#define IPIPE_NR_IRQS		(IPIPE_VIRQ_BASE + IPIPE_NR_VIRQS)
+/* Number of indirect words needed to map the whole IRQ space. */
+#define IPIPE_IRQ_IWORDS	((IPIPE_NR_IRQS + BITS_PER_LONG - 1) / BITS_PER_LONG)
+#define IPIPE_IRQ_IMASK		(BITS_PER_LONG - 1)
+#define IPIPE_IRQMASK_ANY	(~0L)
+#define IPIPE_IRQMASK_VIRT	(IPIPE_IRQMASK_ANY << (IPIPE_VIRQ_BASE / BITS_PER_LONG))
+
+/* Per-cpu pipeline status */
+#define IPIPE_STALL_FLAG	0	/* Stalls a pipeline stage -- guaranteed at bit #0 */
+#define IPIPE_SYNC_FLAG		1	/* The interrupt syncer is running for the domain */
+#define IPIPE_NOSTACK_FLAG	2	/* Domain currently runs on a foreign stack */
+
+#define IPIPE_STALL_MASK	(1L << IPIPE_STALL_FLAG)
+#define IPIPE_SYNC_MASK		(1L << IPIPE_SYNC_FLAG)
+
+typedef void (*ipipe_irq_handler_t)(unsigned irq,
+				    void *cookie);
+
+extern struct ipipe_domain ipipe_root;
+
+#define ipipe_root_domain (&ipipe_root)
+
+void __ipipe_unstall_root(void);
+
+void __ipipe_restore_root(unsigned long x);
+
+#define ipipe_preempt_disable(flags)	local_irq_save_hw(flags)
+#define ipipe_preempt_enable(flags)	local_irq_restore_hw(flags)
+ 
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+void ipipe_check_context(struct ipipe_domain *border_ipd);
+#else /* !CONFIG_IPIPE_DEBUG_CONTEXT */
+static inline void ipipe_check_context(struct ipipe_domain *border_ipd) { }
+#endif /* !CONFIG_IPIPE_DEBUG_CONTEXT */
+
+/* Generic features */
+
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+#define __IPIPE_FEATURE_REQUEST_TICKDEV  1
+#endif
+
+#define __IPIPE_FEATURE_DELAYED_ATOMICSW 1
+
+#else /* !CONFIG_IPIPE */
+#define ipipe_preempt_disable(flags)	do { \
+						preempt_disable(); \
+						(void)(flags); \
+					} while (0)
+#define ipipe_preempt_enable(flags)	preempt_enable()
+#define ipipe_check_context(ipd)	do { } while(0)
+#endif	/* CONFIG_IPIPE */
+
+#endif	/* !__LINUX_IPIPE_BASE_H */
diff --git a/include/linux/ipipe_compat.h b/include/linux/ipipe_compat.h
new file mode 100644
index 0000000..50a245c
--- /dev/null
+++ b/include/linux/ipipe_compat.h
@@ -0,0 +1,54 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_compat.h
+ *
+ * Copyright (C) 2007 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_COMPAT_H
+#define __LINUX_IPIPE_COMPAT_H
+
+#ifdef CONFIG_IPIPE_COMPAT
+/*
+ * OBSOLETE: defined only for backward compatibility. Will be removed
+ * in future releases, please update client code accordingly.
+ */
+
+#ifdef CONFIG_SMP
+#define ipipe_declare_cpuid	int cpuid
+#define ipipe_load_cpuid()	do { \
+					cpuid = ipipe_processor_id();	\
+				} while(0)
+#define ipipe_lock_cpu(flags)	do { \
+					local_irq_save_hw(flags); \
+					cpuid = ipipe_processor_id(); \
+				} while(0)
+#define ipipe_unlock_cpu(flags)	local_irq_restore_hw(flags)
+#define ipipe_get_cpu(flags)	ipipe_lock_cpu(flags)
+#define ipipe_put_cpu(flags)	ipipe_unlock_cpu(flags)
+#else /* !CONFIG_SMP */
+#define ipipe_declare_cpuid	const int cpuid = 0
+#define ipipe_load_cpuid()	do { } while(0)
+#define ipipe_lock_cpu(flags)	local_irq_save_hw(flags)
+#define ipipe_unlock_cpu(flags)	local_irq_restore_hw(flags)
+#define ipipe_get_cpu(flags)	do { (void)(flags); } while(0)
+#define ipipe_put_cpu(flags)	do { } while(0)
+#endif /* CONFIG_SMP */
+
+#endif /* CONFIG_IPIPE_COMPAT */
+
+#endif	/* !__LINUX_IPIPE_COMPAT_H */
diff --git a/include/linux/ipipe_percpu.h b/include/linux/ipipe_percpu.h
new file mode 100644
index 0000000..f1311dd
--- /dev/null
+++ b/include/linux/ipipe_percpu.h
@@ -0,0 +1,76 @@
+/*   -*- linux-c -*-
+ *   include/linux/ipipe_percpu.h
+ *
+ *   Copyright (C) 2007 Philippe Gerum.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ *   USA; either version 2 of the License, or (at your option) any later
+ *   version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the Free Software
+ *   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_PERCPU_H
+#define __LINUX_IPIPE_PERCPU_H
+
+#include <asm/percpu.h>
+#include <asm/ptrace.h>
+
+struct ipipe_domain;
+
+struct ipipe_percpu_domain_data {
+	unsigned long status;	/* <= Must be first in struct. */
+	unsigned long irqpend_himask;
+	unsigned long irqpend_lomask[IPIPE_IRQ_IWORDS];
+	unsigned long irqheld_mask[IPIPE_IRQ_IWORDS];
+	unsigned long irqall[IPIPE_NR_IRQS];
+	u64 evsync;
+};
+
+#ifdef CONFIG_SMP
+#define ipipe_percpudom(ipd, var, cpu)	\
+	(per_cpu(ipipe_percpu_darray, cpu)[(ipd)->slot].var)
+#define ipipe_cpudom_var(ipd, var)	\
+	(__raw_get_cpu_var(ipipe_percpu_darray)[(ipd)->slot].var)
+#else
+DECLARE_PER_CPU(struct ipipe_percpu_domain_data *, ipipe_percpu_daddr[CONFIG_IPIPE_DOMAINS]);
+#define ipipe_percpudom(ipd, var, cpu)	\
+	(per_cpu(ipipe_percpu_daddr, cpu)[(ipd)->slot]->var)
+#define ipipe_cpudom_var(ipd, var)	\
+	(__raw_get_cpu_var(ipipe_percpu_daddr)[(ipd)->slot]->var)
+#endif
+
+#define IPIPE_ROOT_SLOT			0
+#define IPIPE_HEAD_SLOT			(CONFIG_IPIPE_DOMAINS - 1)
+
+DECLARE_PER_CPU(struct ipipe_percpu_domain_data, ipipe_percpu_darray[CONFIG_IPIPE_DOMAINS]);
+
+DECLARE_PER_CPU(struct ipipe_domain *, ipipe_percpu_domain);
+
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+DECLARE_PER_CPU(int, ipipe_percpu_context_check);
+DECLARE_PER_CPU(int, ipipe_saved_context_check_state);
+#endif
+
+#define ipipe_percpu(var, cpu)		per_cpu(var, cpu)
+#define ipipe_cpu_var(var)		__raw_get_cpu_var(var)
+
+#define ipipe_root_cpudom_var(var)	\
+	__raw_get_cpu_var(ipipe_percpu_darray)[IPIPE_ROOT_SLOT].var
+
+#define ipipe_this_cpudom_var(var)	\
+	ipipe_cpudom_var(ipipe_current_domain, var)
+
+#define ipipe_head_cpudom_var(var)	\
+	__raw_get_cpu_var(ipipe_percpu_darray)[IPIPE_HEAD_SLOT].var
+
+#endif	/* !__LINUX_IPIPE_PERCPU_H */
diff --git a/include/linux/ipipe_tickdev.h b/include/linux/ipipe_tickdev.h
new file mode 100644
index 0000000..4a1cb1b
--- /dev/null
+++ b/include/linux/ipipe_tickdev.h
@@ -0,0 +1,58 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_tickdev.h
+ *
+ * Copyright (C) 2007 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_TICKDEV_H
+#define __LINUX_IPIPE_TICKDEV_H
+
+#if defined(CONFIG_IPIPE) && defined(CONFIG_GENERIC_CLOCKEVENTS)
+
+#include <linux/clockchips.h>
+
+struct tick_device;
+
+struct ipipe_tick_device {
+
+	void (*emul_set_mode)(enum clock_event_mode,
+			      struct clock_event_device *cdev);
+	int (*emul_set_tick)(unsigned long delta,
+			     struct clock_event_device *cdev);
+	void (*real_set_mode)(enum clock_event_mode mode,
+			      struct clock_event_device *cdev);
+	int (*real_set_tick)(unsigned long delta,
+			     struct clock_event_device *cdev);
+	struct tick_device *slave;
+	unsigned long real_max_delta_ns;
+	unsigned long real_mult;
+	int real_shift;
+};
+
+int ipipe_request_tickdev(const char *devname,
+			  void (*emumode)(enum clock_event_mode mode,
+					  struct clock_event_device *cdev),
+			  int (*emutick)(unsigned long evt,
+					 struct clock_event_device *cdev),
+			  int cpu, unsigned long *tmfreq);
+
+void ipipe_release_tickdev(int cpu);
+
+#endif /* CONFIG_IPIPE && CONFIG_GENERIC_CLOCKEVENTS */
+
+#endif /* !__LINUX_IPIPE_TICKDEV_H */
diff --git a/include/linux/ipipe_trace.h b/include/linux/ipipe_trace.h
new file mode 100644
index 0000000..815b5ad
--- /dev/null
+++ b/include/linux/ipipe_trace.h
@@ -0,0 +1,70 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_trace.h
+ *
+ * Copyright (C) 2005 Luotao Fu.
+ *               2005-2007 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef _LINUX_IPIPE_TRACE_H
+#define _LINUX_IPIPE_TRACE_H
+
+#ifdef CONFIG_IPIPE_TRACE
+
+#include <linux/types.h>
+
+void ipipe_trace_begin(unsigned long v);
+void ipipe_trace_end(unsigned long v);
+void ipipe_trace_freeze(unsigned long v);
+void ipipe_trace_special(unsigned char special_id, unsigned long v);
+void ipipe_trace_pid(pid_t pid, short prio);
+int ipipe_trace_max_reset(void);
+int ipipe_trace_frozen_reset(void);
+
+#else /* !CONFIG_IPIPE_TRACE */
+
+#define ipipe_trace_begin(v)		do { (void)(v); } while(0)
+#define ipipe_trace_end(v)		do { (void)(v); } while(0)
+#define ipipe_trace_freeze(v)		do { (void)(v); } while(0)
+#define ipipe_trace_special(id, v)	do { (void)(id); (void)(v); } while(0)
+#define ipipe_trace_pid(pid, prio)	do { (void)(pid); (void)(prio); } while(0)
+#define ipipe_trace_max_reset()		do { } while(0)
+#define ipipe_trace_froze_reset()	do { } while(0)
+
+#endif /* !CONFIG_IPIPE_TRACE */
+
+#ifdef CONFIG_IPIPE_TRACE_PANIC
+void ipipe_trace_panic_freeze(void);
+void ipipe_trace_panic_dump(void);
+#else
+static inline void ipipe_trace_panic_freeze(void) { }
+static inline void ipipe_trace_panic_dump(void) { }
+#endif
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+#define ipipe_trace_irq_entry(irq)	ipipe_trace_begin(irq)
+#define ipipe_trace_irq_exit(irq)	ipipe_trace_end(irq)
+#define ipipe_trace_irqsoff()		ipipe_trace_begin(0x80000000UL)
+#define ipipe_trace_irqson()		ipipe_trace_end(0x80000000UL)
+#else
+#define ipipe_trace_irq_entry(irq)	do { (void)(irq);} while(0)
+#define ipipe_trace_irq_exit(irq)	do { (void)(irq);} while(0)
+#define ipipe_trace_irqsoff()		do { } while(0)
+#define ipipe_trace_irqson()		do { } while(0)
+#endif
+
+#endif	/* !__LINUX_IPIPE_TRACE_H */
diff --git a/include/linux/irq.h b/include/linux/irq.h
index 552e0ec..ecd540b 100644
--- a/include/linux/irq.h
+++ b/include/linux/irq.h
@@ -151,6 +151,12 @@ struct irq_chip {
  * @name:		flow handler name for /proc/interrupts output
  */
 struct irq_desc {
+#ifdef CONFIG_IPIPE
+	void			(*ipipe_ack)(unsigned int irq,
+						      struct irq_desc *desc);
+	void			(*ipipe_end)(unsigned int irq,
+						      struct irq_desc *desc);
+#endif /* CONFIG_IPIPE */
 	irq_flow_handler_t	handle_irq;
 	struct irq_chip		*chip;
 	struct msi_desc		*msi_desc;
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index 2e70006..340b471 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -14,6 +14,7 @@
 #include <linux/compiler.h>
 #include <linux/bitops.h>
 #include <linux/log2.h>
+#include <linux/ipipe_base.h>
 #include <asm/byteorder.h>
 #include <asm/bug.h>
 
@@ -112,9 +113,12 @@ struct user;
  */
 #ifdef CONFIG_PREEMPT_VOLUNTARY
 extern int _cond_resched(void);
-# define might_resched() _cond_resched()
+# define might_resched() do { \
+		ipipe_check_context(ipipe_root_domain); \
+		_cond_resched(); \
+	} while (0)
 #else
-# define might_resched() do { } while (0)
+# define might_resched() ipipe_check_context(ipipe_root_domain)
 #endif
 
 #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
diff --git a/include/linux/linkage.h b/include/linux/linkage.h
index 2119610..e57f731 100644
--- a/include/linux/linkage.h
+++ b/include/linux/linkage.h
@@ -89,4 +89,8 @@
 #define ATTRIB_NORET  __attribute__((noreturn))
 #define NORET_AND     noreturn,
 
+#ifndef notrace
+#define notrace		__attribute__((no_instrument_function))
+#endif
+
 #endif
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 586a943..568cd8d 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -109,6 +109,8 @@ extern unsigned int kobjsize(const void *objp);
 #define VM_CAN_NONLINEAR 0x08000000	/* Has ->fault & does nonlinear pages */
 #define VM_MIXEDMAP	0x10000000	/* Can contain "struct page" and pure PFN pages */
 
+#define VM_PINNED	0x20000000	/* Disable faults for the vma */
+
 #ifndef VM_STACK_DEFAULT_FLAGS		/* arch can override this */
 #define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
 #endif
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index 23f0c54..7991b6d 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -9,13 +9,20 @@
 #include <linux/thread_info.h>
 #include <linux/linkage.h>
 #include <linux/list.h>
+#include <linux/ipipe_base.h>
 
 #ifdef CONFIG_DEBUG_PREEMPT
   extern void add_preempt_count(int val);
   extern void sub_preempt_count(int val);
 #else
-# define add_preempt_count(val)	do { preempt_count() += (val); } while (0)
-# define sub_preempt_count(val)	do { preempt_count() -= (val); } while (0)
+# define add_preempt_count(val)	do { \
+	ipipe_check_context(ipipe_root_domain); \
+	preempt_count() += (val); \
+  } while (0)
+# define sub_preempt_count(val)	do { \
+	ipipe_check_context(ipipe_root_domain); \
+	preempt_count() -= (val); \
+  } while (0)
 #endif
 
 #define inc_preempt_count() add_preempt_count(1)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index c5d3f84..2530fef 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -59,6 +59,7 @@ struct sched_param {
 #include <linux/errno.h>
 #include <linux/nodemask.h>
 #include <linux/mm_types.h>
+#include <linux/ipipe.h>
 
 #include <asm/system.h>
 #include <asm/page.h>
@@ -181,6 +182,13 @@ extern unsigned long long time_sync_thresh;
 /* in tsk->state again */
 #define TASK_DEAD		64
 #define TASK_WAKEKILL		128
+#ifdef CONFIG_IPIPE
+#define TASK_ATOMICSWITCH	512
+#define TASK_NOWAKEUP		1024
+#else  /* !CONFIG_IPIPE */
+#define TASK_ATOMICSWITCH	0
+#define TASK_NOWAKEUP		0
+#endif /* CONFIG_IPIPE */
 
 /* Convenience macros for the sake of set_task_state */
 #define TASK_KILLABLE		(TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
@@ -1287,6 +1295,9 @@ struct task_struct {
 #endif
 	atomic_t fs_excl;	/* holding fs exclusive resources */
 	struct rcu_head rcu;
+#ifdef CONFIG_IPIPE
+	void *ptd[IPIPE_ROOT_NPTDKEYS];
+#endif
 
 	/*
 	 * cache last used pipe for splice
@@ -1507,6 +1518,11 @@ static inline void put_task_struct(struct task_struct *t)
 #define PF_MEMPOLICY	0x10000000	/* Non-default NUMA mempolicy */
 #define PF_MUTEX_TESTER	0x20000000	/* Thread belongs to the rt mutex tester */
 #define PF_FREEZER_SKIP	0x40000000	/* Freezer should not count it as freezeable */
+#ifdef CONFIG_IPIPE
+#define PF_EVNOTIFY	0x80000000	/* Notify other domains about internal events */
+#else
+#define PF_EVNOTIFY	0
+#endif /* CONFIG_IPIPE */
 
 /*
  * Only the _current_ task can read/write to tsk->flags, but other
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index d311a09..9a5c98e 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -89,10 +89,14 @@ extern int __lockfunc generic__raw_read_trylock(raw_rwlock_t *lock);
 # include <linux/spinlock_up.h>
 #endif
 
+#undef TYPE_EQUAL
+#define TYPE_EQUAL(lock, type) \
+	__builtin_types_compatible_p(typeof(lock), type *)
+
 #ifdef CONFIG_DEBUG_SPINLOCK
   extern void __spin_lock_init(spinlock_t *lock, const char *name,
 			       struct lock_class_key *key);
-# define spin_lock_init(lock)					\
+# define _spin_lock_init(lock)				\
 do {								\
 	static struct lock_class_key __key;			\
 								\
@@ -100,10 +104,21 @@ do {								\
 } while (0)
 
 #else
-# define spin_lock_init(lock)					\
+# define _spin_lock_init(lock)				\
 	do { *(lock) = SPIN_LOCK_UNLOCKED; } while (0)
 #endif
 
+# define spin_lock_init(lock)						\
+	do {								\
+		if (TYPE_EQUAL((lock), __ipipe_spinlock_t))		\
+			do {						\
+				IPIPE_DEFINE_SPINLOCK(__lock__);	\
+				*((ipipe_spinlock_t *)lock) = __lock__; \
+			} while(0);					\
+		else							\
+			_spin_lock_init((spinlock_t *)lock);		\
+	} while(0)
+
 #ifdef CONFIG_DEBUG_SPINLOCK
   extern void __rwlock_init(rwlock_t *lock, const char *name,
 			    struct lock_class_key *key);
@@ -178,7 +193,86 @@ do {								\
 #define read_trylock(lock)		__cond_lock(lock, _read_trylock(lock))
 #define write_trylock(lock)		__cond_lock(lock, _write_trylock(lock))
 
-#define spin_lock(lock)			_spin_lock(lock)
+#define PICK_SPINOP(op, lock)						\
+do {									\
+	if (TYPE_EQUAL((lock), __ipipe_spinlock_t))			\
+		__raw_spin##op(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
+	else if (TYPE_EQUAL(lock, spinlock_t))				\
+		_spin##op((spinlock_t *)(lock));			\
+} while (0)
+
+#define PICK_SPINOP_RAW(op, lock)					\
+do {									\
+	if (TYPE_EQUAL((lock), __ipipe_spinlock_t))			\
+		__raw_spin##op(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
+	else if (TYPE_EQUAL(lock, spinlock_t))				\
+		__raw_spin##op(&((spinlock_t *)(lock))->raw_lock);	\
+} while (0)
+
+#define PICK_SPINLOCK_IRQ(lock)						\
+do {									\
+	if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) {			\
+		__ipipe_spin_lock_irq(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
+	} else if (TYPE_EQUAL(lock, spinlock_t))			\
+		_spin_lock_irq((spinlock_t *)(lock));			\
+} while (0)
+
+#define PICK_SPINUNLOCK_IRQ(lock)					\
+do {									\
+	if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) {			\
+		__ipipe_spin_unlock_irq(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
+	} else if (TYPE_EQUAL(lock, spinlock_t))			\
+		_spin_unlock_irq((spinlock_t *)(lock));			\
+} while (0)
+
+#define PICK_SPINLOCK_IRQ_RAW(lock)					\
+do {									\
+	if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) {			\
+		__ipipe_spin_lock_irq(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
+	} else if (TYPE_EQUAL(lock, spinlock_t))			\
+		local_irq_disable();					\
+		__raw_spin_lock(&((spinlock_t *)(lock))->raw_lock);	\
+} while (0)
+
+#define PICK_SPINUNLOCK_IRQ_RAW(lock)					\
+do {									\
+	if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) {			\
+		__ipipe_spin_unlock_irq(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
+	} else if (TYPE_EQUAL(lock, spinlock_t))			\
+		__raw_spin_unlock(&((spinlock_t *)(lock))->raw_lock);	\
+		local_irq_enable();					\
+} while (0)
+
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
+extern int __bad_spinlock_type(void);
+
+#define PICK_SPINLOCK_IRQSAVE(lock, flags)				\
+do {									\
+	if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) {			\
+		(flags) = __ipipe_spin_lock_irqsave(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
+	} else if (TYPE_EQUAL(lock, spinlock_t))			\
+		flags = _spin_lock_irqsave((spinlock_t *)(lock));	\
+	else __bad_spinlock_type();					\
+} while (0)
+#else
+#define PICK_SPINLOCK_IRQSAVE(lock, flags)				\
+do {									\
+	if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) {			\
+		(flags) = __ipipe_spin_lock_irqsave(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
+	} else if (TYPE_EQUAL(lock, spinlock_t))			\
+		_spin_lock_irqsave((spinlock_t *)(lock), flags);	\
+} while (0)
+#endif
+
+#define PICK_SPINUNLOCK_IRQRESTORE(lock, flags)				\
+	do {								\
+	if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) {			\
+		__ipipe_spin_unlock_irqrestore(&((__ipipe_spinlock_t *)(lock))->__raw_lock, flags); \
+	} else if (TYPE_EQUAL(lock, spinlock_t))			\
+		_spin_unlock_irqrestore((spinlock_t *)(lock), flags);	\
+} while (0)
+
+#define spin_lock(lock)	PICK_SPINOP(_lock, lock)
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 # define spin_lock_nested(lock, subclass) _spin_lock_nested(lock, subclass)
@@ -191,7 +285,7 @@ do {								\
 
 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 
-#define spin_lock_irqsave(lock, flags)	flags = _spin_lock_irqsave(lock)
+#define spin_lock_irqsave(lock, flags)	PICK_SPINLOCK_IRQSAVE(lock, flags)
 #define read_lock_irqsave(lock, flags)	flags = _read_lock_irqsave(lock)
 #define write_lock_irqsave(lock, flags)	flags = _write_lock_irqsave(lock)
 
@@ -205,7 +299,7 @@ do {								\
 
 #else
 
-#define spin_lock_irqsave(lock, flags)	_spin_lock_irqsave(lock, flags)
+#define spin_lock_irqsave(lock, flags)	PICK_SPINLOCK_IRQSAVE(lock, flags)
 #define read_lock_irqsave(lock, flags)	_read_lock_irqsave(lock, flags)
 #define write_lock_irqsave(lock, flags)	_write_lock_irqsave(lock, flags)
 #define spin_lock_irqsave_nested(lock, flags, subclass)	\
@@ -213,7 +307,7 @@ do {								\
 
 #endif
 
-#define spin_lock_irq(lock)		_spin_lock_irq(lock)
+#define spin_lock_irq(lock)		PICK_SPINLOCK_IRQ(lock)
 #define spin_lock_bh(lock)		_spin_lock_bh(lock)
 
 #define read_lock_irq(lock)		_read_lock_irq(lock)
@@ -227,32 +321,40 @@ do {								\
  */
 #if defined(CONFIG_DEBUG_SPINLOCK) || defined(CONFIG_PREEMPT) || \
 	!defined(CONFIG_SMP)
-# define spin_unlock(lock)		_spin_unlock(lock)
+#define spin_unlock(lock)		PICK_SPINOP(_unlock, lock)
 # define read_unlock(lock)		_read_unlock(lock)
 # define write_unlock(lock)		_write_unlock(lock)
-# define spin_unlock_irq(lock)		_spin_unlock_irq(lock)
-# define read_unlock_irq(lock)		_read_unlock_irq(lock)
-# define write_unlock_irq(lock)		_write_unlock_irq(lock)
+# define spin_unlock_irq(lock)	PICK_SPINUNLOCK_IRQ(lock)
+# define read_unlock_irq(lock)	_read_unlock_irq(lock)
+# define write_unlock_irq(lock)	_write_unlock_irq(lock)
 #else
-# define spin_unlock(lock) \
-    do {__raw_spin_unlock(&(lock)->raw_lock); __release(lock); } while (0)
-# define read_unlock(lock) \
-    do {__raw_read_unlock(&(lock)->raw_lock); __release(lock); } while (0)
-# define write_unlock(lock) \
-    do {__raw_write_unlock(&(lock)->raw_lock); __release(lock); } while (0)
-# define spin_unlock_irq(lock)			\
+# define spin_unlock(lock)			\
 do {						\
-	__raw_spin_unlock(&(lock)->raw_lock);	\
+	PICK_SPINOP_RAW(_unlock, lock); 	\
+	__release(lock);			\
+} while(0)
+# define read_unlock(lock)			\
+do {						\
+	__raw_read_unlock(&(lock)->raw_lock);	\
+	__release(lock);			\
+} while (0)
+# define write_unlock(lock)			\
+do {						\
+	__raw_write_unlock(&(lock)->raw_lock);	\
 	__release(lock);			\
-	local_irq_enable();			\
 } while (0)
-# define read_unlock_irq(lock)			\
+# define spin_unlock_irq(lock)		\
+do {						\
+	PICK_SPINUNLOCK_IRQ_RAW(lock);		\
+	__release(lock);			\
+} while(0)
+# define read_unlock_irq(lock)		\
 do {						\
 	__raw_read_unlock(&(lock)->raw_lock);	\
 	__release(lock);			\
 	local_irq_enable();			\
 } while (0)
-# define write_unlock_irq(lock)			\
+# define write_unlock_irq(lock)		\
 do {						\
 	__raw_write_unlock(&(lock)->raw_lock);	\
 	__release(lock);			\
@@ -260,8 +362,8 @@ do {						\
 } while (0)
 #endif
 
-#define spin_unlock_irqrestore(lock, flags) \
-					_spin_unlock_irqrestore(lock, flags)
+#define spin_unlock_irqrestore(lock, flags)	\
+					PICK_SPINUNLOCK_IRQRESTORE(lock, flags)
 #define spin_unlock_bh(lock)		_spin_unlock_bh(lock)
 
 #define read_unlock_irqrestore(lock, flags) \
@@ -318,4 +420,29 @@ extern int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock);
  */
 #define spin_can_lock(lock)	(!spin_is_locked(lock))
 
+#ifdef CONFIG_IPIPE
+void __ipipe_spin_lock_irq(raw_spinlock_t *lock);
+void __ipipe_spin_unlock_irq(raw_spinlock_t *lock);
+unsigned long __ipipe_spin_lock_irqsave(raw_spinlock_t *lock);
+void __ipipe_spin_unlock_irqrestore(raw_spinlock_t *lock,
+					     unsigned long x);
+void __ipipe_spin_unlock_irqbegin(ipipe_spinlock_t *lock);
+void __ipipe_spin_unlock_irqcomplete(unsigned long x);
+#define spin_lock_irqsave_cond(lock, flags) \
+	spin_lock_irqsave(lock, flags)
+#define spin_unlock_irqrestore_cond(lock, flags) \
+	spin_unlock_irqrestore(lock, flags)
+#else
+#define spin_lock_irqsave_cond(lock, flags) \
+	do { (void)(flags); spin_lock(lock); } while(0)
+#define spin_unlock_irqrestore_cond(lock, flags) \
+	spin_unlock(lock)
+#define __ipipe_spin_lock_irq(lock)		do { } while(0)
+#define __ipipe_spin_unlock_irq(lock)		do { } while(0)
+#define __ipipe_spin_lock_irqsave(lock)		0
+#define __ipipe_spin_unlock_irqrestore(lock, x)	do { (void)(x); } while(0)
+#define __ipipe_spin_unlock_irqbegin(lock)	do { } while(0)
+#define __ipipe_spin_unlock_irqcomplete(x)	do { (void)(x); } while(0)
+#endif
+
 #endif /* __LINUX_SPINLOCK_H */
diff --git a/include/linux/spinlock_types.h b/include/linux/spinlock_types.h
index 68d88f7..f5ce3c4 100644
--- a/include/linux/spinlock_types.h
+++ b/include/linux/spinlock_types.h
@@ -31,6 +31,10 @@ typedef struct {
 #endif
 } spinlock_t;
 
+typedef struct {
+	raw_spinlock_t __raw_lock;
+} __ipipe_spinlock_t;
+
 #define SPINLOCK_MAGIC		0xdead4ead
 
 typedef struct {
@@ -92,9 +96,21 @@ typedef struct {
  * __SPIN_LOCK_UNLOCKED()/__RW_LOCK_UNLOCKED() as appropriate.
  */
 #define SPIN_LOCK_UNLOCKED	__SPIN_LOCK_UNLOCKED(old_style_spin_init)
+#define IPIPE_SPIN_LOCK_UNLOCKED					\
+	(__ipipe_spinlock_t) {	.__raw_lock = __RAW_SPIN_LOCK_UNLOCKED }
 #define RW_LOCK_UNLOCKED	__RW_LOCK_UNLOCKED(old_style_rw_init)
 
 #define DEFINE_SPINLOCK(x)	spinlock_t x = __SPIN_LOCK_UNLOCKED(x)
 #define DEFINE_RWLOCK(x)	rwlock_t x = __RW_LOCK_UNLOCKED(x)
 
+#ifdef CONFIG_IPIPE
+# define ipipe_spinlock_t		__ipipe_spinlock_t
+# define IPIPE_DEFINE_SPINLOCK(x)	ipipe_spinlock_t x = IPIPE_SPIN_LOCK_UNLOCKED
+# define IPIPE_DECLARE_SPINLOCK(x)	extern ipipe_spinlock_t x
+#else
+# define ipipe_spinlock_t		spinlock_t
+# define IPIPE_DEFINE_SPINLOCK(x)	DEFINE_SPINLOCK(x)
+# define IPIPE_DECLARE_SPINLOCK(x)	extern spinlock_t x
+#endif
+
 #endif /* __LINUX_SPINLOCK_TYPES_H */
diff --git a/init/Kconfig b/init/Kconfig
index 6199d11..c1a5fac 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -73,6 +73,7 @@ config INIT_ENV_ARG_LIMIT
 
 config LOCALVERSION
 	string "Local version - append to kernel release"
+	default "-ipipe"
 	help
 	  Append an extra string to the end of your kernel version.
 	  This will show up when you type uname, for example.
diff --git a/init/main.c b/init/main.c
index f7fb200..f910f7c 100644
--- a/init/main.c
+++ b/init/main.c
@@ -547,7 +547,7 @@ asmlinkage void __init start_kernel(void)
 	debug_objects_early_init();
 	cgroup_init_early();
 
-	local_irq_disable();
+	local_irq_disable_hw();
 	early_boot_irqs_off();
 	early_init_irq_lock_class();
 
@@ -603,6 +603,11 @@ asmlinkage void __init start_kernel(void)
 	timekeeping_init();
 	time_init();
 	sched_clock_init();
+ 	/*
+ 	 * We need to wait for the interrupt and time subsystems to be
+ 	 * initialized before enabling the pipeline.
+ 	 */
+  	ipipe_init();
 	profile_init();
 	if (!irqs_disabled())
 		printk("start_kernel(): bug: interrupts were enabled early\n");
@@ -763,6 +768,7 @@ static void __init do_basic_setup(void)
 	usermodehelper_init();
 	driver_init();
 	init_irq_proc();
+ 	ipipe_init_proc();
 	do_initcalls();
 }
 
diff --git a/kernel/Makefile b/kernel/Makefile
index 1c9938a..ba71be6 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -64,6 +64,7 @@ ifeq ($(CONFIG_PREEMPT_RCU),y)
 obj-$(CONFIG_RCU_TRACE) += rcupreempt_trace.o
 endif
 obj-$(CONFIG_RELAY) += relay.o
+obj-$(CONFIG_IPIPE) += ipipe/
 obj-$(CONFIG_SYSCTL) += utsname_sysctl.o
 obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
 obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
diff --git a/kernel/exit.c b/kernel/exit.c
index 8f6185e..ee390ab 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -1042,6 +1042,7 @@ NORET_TYPE void do_exit(long code)
 
 	if (group_dead)
 		acct_process();
+ 	ipipe_exit_notify(tsk);
 	exit_sem(tsk);
 	exit_files(tsk);
 	exit_fs(tsk);
diff --git a/kernel/fork.c b/kernel/fork.c
index 19908b2..5373863 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -432,6 +432,7 @@ void mmput(struct mm_struct *mm)
 	if (atomic_dec_and_test(&mm->mm_users)) {
 		exit_aio(mm);
 		exit_mmap(mm);
+ 		ipipe_cleanup_notify(mm);
 		set_mm_exe_file(mm, NULL);
 		if (!list_empty(&mm->mmlist)) {
 			spin_lock(&mmlist_lock);
@@ -830,7 +831,7 @@ static void copy_flags(unsigned long clone_flags, struct task_struct *p)
 {
 	unsigned long new_flags = p->flags;
 
-	new_flags &= ~PF_SUPERPRIV;
+ 	new_flags &= ~(PF_SUPERPRIV | PF_EVNOTIFY);
 	new_flags |= PF_FORKNOEXEC;
 	if (!(clone_flags & CLONE_PTRACE))
 		p->ptrace = 0;
@@ -1223,6 +1224,9 @@ static struct task_struct *copy_process(unsigned long clone_flags,
 	write_unlock_irq(&tasklist_lock);
 	proc_fork_connector(p);
 	cgroup_post_fork(p);
+#ifdef CONFIG_IPIPE
+	memset(p->ptd, 0, sizeof(p->ptd));
+#endif /* CONFIG_IPIPE */
 	return p;
 
 bad_fork_free_pid:
diff --git a/kernel/ipipe/Kconfig b/kernel/ipipe/Kconfig
new file mode 100644
index 0000000..934d57c
--- /dev/null
+++ b/kernel/ipipe/Kconfig
@@ -0,0 +1,30 @@
+config IPIPE
+	bool "Interrupt pipeline"
+	default y
+	---help---
+	  Activate this option if you want the interrupt pipeline to be
+	  compiled in.
+
+config IPIPE_DOMAINS
+	int "Max domains"
+	depends on IPIPE
+	default 4
+	---help---
+	The maximum number of I-pipe domains to run concurrently.
+
+config IPIPE_COMPAT
+	bool "Maintain code compatibility with older releases"
+	depends on IPIPE
+	default y
+	---help---
+	Activate this option if you want the compatibility code to be
+	defined, so that older I-pipe clients may use obsolete
+	constructs. WARNING: obsolete code will be eventually
+	deprecated in future I-pipe releases, and removed from the
+	compatibility support as time passes. Please fix I-pipe
+	clients to get rid of such uses as soon as possible.
+
+config IPIPE_DELAYED_ATOMICSW
+       bool
+       depends on IPIPE
+       default n
diff --git a/kernel/ipipe/Kconfig.debug b/kernel/ipipe/Kconfig.debug
new file mode 100644
index 0000000..64a692c
--- /dev/null
+++ b/kernel/ipipe/Kconfig.debug
@@ -0,0 +1,88 @@
+config IPIPE_DEBUG
+	bool "I-pipe debugging"
+	depends on IPIPE
+
+config IPIPE_DEBUG_CONTEXT
+	bool "Check for illicit cross-domain calls"
+	depends on IPIPE_DEBUG
+	default y
+	---help---
+	  Enable this feature to arm checkpoints in the kernel that
+	  verify the correct invocation context. On entry of critical
+	  Linux services a warning is issued if the caller is not
+	  running over the root domain.
+
+config IPIPE_TRACE
+	bool "Latency tracing"
+	depends on IPIPE_DEBUG
+	select FRAME_POINTER
+	select KALLSYMS
+	select PROC_FS
+	---help---
+	  Activate this option if you want to use per-function tracing of
+	  the kernel. The tracer will collect data via instrumentation
+	  features like the one below or with the help of explicite calls
+	  of ipipe_trace_xxx(). See include/linux/ipipe_trace.h for the
+	  in-kernel tracing API. The collected data and runtime control
+	  is available via /proc/ipipe/trace/*.
+
+if IPIPE_TRACE
+
+config IPIPE_TRACE_ENABLE
+	bool "Enable tracing on boot"
+	default y
+	---help---
+	  Disable this option if you want to arm the tracer after booting
+	  manually ("echo 1 > /proc/ipipe/tracer/enable"). This can reduce
+	  boot time on slow embedded devices due to the tracer overhead.
+
+config IPIPE_TRACE_MCOUNT
+	bool "Instrument function entries"
+	default y
+	---help---
+	  When enabled, records every kernel function entry in the tracer
+	  log. While this slows down the system noticeably, it provides
+	  the highest level of information about the flow of events.
+	  However, it can be switch off in order to record only explicit
+	  I-pipe trace points.
+
+config IPIPE_TRACE_IRQSOFF
+	bool "Trace IRQs-off times"
+	default y
+	---help---
+	  Activate this option if I-pipe shall trace the longest path
+	  with hard-IRQs switched off.
+
+config IPIPE_TRACE_SHIFT
+	int "Depth of trace log (14 => 16Kpoints, 15 => 32Kpoints)"
+	range 10 18
+	default 14
+	---help---
+	  The number of trace points to hold tracing data for each
+	  trace path, as a power of 2.
+
+config IPIPE_TRACE_VMALLOC
+	bool "Use vmalloc'ed trace buffer"
+	default y if EMBEDDED
+	---help---
+	  Instead of reserving static kernel data, the required buffer
+	  is allocated via vmalloc during boot-up when this option is
+	  enabled. This can help to start systems that are low on memory,
+	  but it slightly degrades overall performance. Try this option
+	  when a traced kernel hangs unexpectedly at boot time.
+
+config IPIPE_TRACE_PANIC
+	bool "Enable panic back traces"
+	default y
+	---help---
+	  Provides services to freeze and dump a back trace on panic
+	  situations. This is used on IPIPE_DEBUG_CONTEXT exceptions
+	  as well as ordinary kernel oopses. You can control the number
+	  of printed back trace points via /proc/ipipe/trace.
+
+config IPIPE_TRACE_ENABLE_VALUE
+	int
+	default 0 if !IPIPE_TRACE_ENABLE
+	default 1 if IPIPE_TRACE_ENABLE
+
+endif
diff --git a/kernel/ipipe/Makefile b/kernel/ipipe/Makefile
new file mode 100644
index 0000000..6257dfa
--- /dev/null
+++ b/kernel/ipipe/Makefile
@@ -0,0 +1,3 @@
+
+obj-$(CONFIG_IPIPE)	+= core.o
+obj-$(CONFIG_IPIPE_TRACE) += tracer.o
diff --git a/kernel/ipipe/core.c b/kernel/ipipe/core.c
new file mode 100644
index 0000000..4187aa4
--- /dev/null
+++ b/kernel/ipipe/core.c
@@ -0,0 +1,1653 @@
+/* -*- linux-c -*-
+ * linux/kernel/ipipe/core.c
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-independent I-PIPE core support.
+ */
+
+#include <linux/version.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/sched.h>
+#include <linux/kallsyms.h>
+#include <linux/interrupt.h>
+#include <linux/bitops.h>
+#include <linux/tick.h>
+#ifdef CONFIG_PROC_FS
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#endif	/* CONFIG_PROC_FS */
+#include <linux/ipipe_trace.h>
+#include <linux/ipipe_tickdev.h>
+#include <linux/irq.h>
+
+static int __ipipe_ptd_key_count;
+
+static unsigned long __ipipe_ptd_key_map;
+
+static unsigned long __ipipe_domain_slot_map;
+
+struct ipipe_domain ipipe_root;
+
+#ifndef CONFIG_SMP
+/*
+ * Create an alias to the unique root status, so that arch-dep code
+ * may get simple and easy access to this percpu variable.  We also
+ * create an array of pointers to the percpu domain data; this tends
+ * to produce a better code when reaching non-root domains. We make
+ * sure that the early boot code would be able to dereference the
+ * pointer to the root domain data safely by statically initializing
+ * its value (local_irq*() routines depend on this).
+ */
+#if __GNUC__ >= 4
+extern unsigned long __ipipe_root_status
+__attribute__((alias(__stringify(__raw_get_cpu_var(ipipe_percpu_darray)))));
+EXPORT_SYMBOL(__ipipe_root_status);
+#else /* __GNUC__ < 4 */
+/*
+ * Work around a GCC 3.x issue making alias symbols unusable as
+ * constant initializers.
+ */
+unsigned long *const __ipipe_root_status_addr =
+	&__raw_get_cpu_var(ipipe_percpu_darray)[IPIPE_ROOT_SLOT].status;
+EXPORT_SYMBOL(__ipipe_root_status_addr);
+#endif /* __GNUC__ < 4 */
+
+DEFINE_PER_CPU(struct ipipe_percpu_domain_data *, ipipe_percpu_daddr[CONFIG_IPIPE_DOMAINS]) =
+{ [IPIPE_ROOT_SLOT] = (struct ipipe_percpu_domain_data *)&__raw_get_cpu_var(ipipe_percpu_darray) };
+EXPORT_PER_CPU_SYMBOL(ipipe_percpu_daddr);
+#endif /* !CONFIG_SMP */
+
+DEFINE_PER_CPU(struct ipipe_percpu_domain_data, ipipe_percpu_darray[CONFIG_IPIPE_DOMAINS]) =
+{ [IPIPE_ROOT_SLOT] = { .status = IPIPE_STALL_MASK } }; /* Root domain stalled on each CPU at startup. */
+
+DEFINE_PER_CPU(struct ipipe_domain *, ipipe_percpu_domain) = { &ipipe_root };
+
+static IPIPE_DEFINE_SPINLOCK(__ipipe_pipelock);
+
+LIST_HEAD(__ipipe_pipeline);
+
+unsigned long __ipipe_virtual_irq_map;
+
+#ifdef CONFIG_PRINTK
+unsigned __ipipe_printk_virq;
+#endif /* CONFIG_PRINTK */
+
+int __ipipe_event_monitors[IPIPE_NR_EVENTS];
+
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+
+DECLARE_PER_CPU(struct tick_device, tick_cpu_device);
+
+static DEFINE_PER_CPU(struct ipipe_tick_device, ipipe_tick_cpu_device);
+
+int ipipe_request_tickdev(const char *devname,
+			  void (*emumode)(enum clock_event_mode mode,
+					  struct clock_event_device *cdev),
+			  int (*emutick)(unsigned long delta,
+					 struct clock_event_device *cdev),
+			  int cpu, unsigned long *tmfreq)
+{
+	struct ipipe_tick_device *itd;
+	struct tick_device *slave;
+	struct clock_event_device *evtdev;
+	unsigned long long freq;
+	unsigned long flags;
+	int status;
+
+	flags = ipipe_critical_enter(NULL);
+
+	itd = &per_cpu(ipipe_tick_cpu_device, cpu);
+
+	if (itd->slave != NULL) {
+		status = -EBUSY;
+		goto out;
+	}
+
+	slave = &per_cpu(tick_cpu_device, cpu);
+
+	if (strcmp(slave->evtdev->name, devname)) {
+		/*
+		 * No conflict so far with the current tick device,
+		 * check whether the requested device is sane and has
+		 * been blessed by the kernel.
+		 */
+		status = __ipipe_check_tickdev(devname) ?
+			CLOCK_EVT_MODE_UNUSED : CLOCK_EVT_MODE_SHUTDOWN;
+		goto out;
+	}
+
+	/*
+	 * Our caller asks for using the same clock event device for
+	 * ticking than we do, let's create a tick emulation device to
+	 * interpose on the set_next_event() method, so that we may
+	 * both manage the device in oneshot mode. Only the tick
+	 * emulation code will actually program the clockchip hardware
+	 * for the next shot, though.
+	 *
+	 * CAUTION: we still have to grab the tick device even when it
+	 * current runs in periodic mode, since the kernel may switch
+	 * to oneshot dynamically (highres/no_hz tick mode).
+	 */
+
+	evtdev = slave->evtdev;
+	status = evtdev->mode;
+
+        if (status == CLOCK_EVT_MODE_SHUTDOWN)
+                goto out;
+
+	itd->slave = slave;
+	itd->emul_set_mode = emumode;
+	itd->emul_set_tick = emutick;
+	itd->real_set_mode = evtdev->set_mode;
+	itd->real_set_tick = evtdev->set_next_event;
+	itd->real_max_delta_ns = evtdev->max_delta_ns;
+	itd->real_mult = evtdev->mult;
+	itd->real_shift = evtdev->shift;
+	freq = (1000000000ULL * evtdev->mult) >> evtdev->shift;
+	*tmfreq = (unsigned long)freq;
+	evtdev->set_mode = emumode;
+	evtdev->set_next_event = emutick;
+	evtdev->max_delta_ns = ULONG_MAX;
+	evtdev->mult = 1;
+	evtdev->shift = 0;
+out:
+	ipipe_critical_exit(flags);
+
+	return status;
+}
+
+void ipipe_release_tickdev(int cpu)
+{
+	struct ipipe_tick_device *itd;
+	struct tick_device *slave;
+	struct clock_event_device *evtdev;
+	unsigned long flags;
+
+	flags = ipipe_critical_enter(NULL);
+
+	itd = &per_cpu(ipipe_tick_cpu_device, cpu);
+
+	if (itd->slave != NULL) {
+		slave = &per_cpu(tick_cpu_device, cpu);
+		evtdev = slave->evtdev;
+		evtdev->set_mode = itd->real_set_mode;
+		evtdev->set_next_event = itd->real_set_tick;
+		evtdev->max_delta_ns = itd->real_max_delta_ns;
+		evtdev->mult = itd->real_mult;
+		evtdev->shift = itd->real_shift;
+		itd->slave = NULL;
+	}
+
+	ipipe_critical_exit(flags);
+}
+
+#endif /* CONFIG_GENERIC_CLOCKEVENTS */
+
+/*
+ * ipipe_init() -- Initialization routine of the IPIPE layer. Called
+ * by the host kernel early during the boot procedure.
+ */
+void __init ipipe_init(void)
+{
+	struct ipipe_domain *ipd = &ipipe_root;
+
+	__ipipe_check_platform();	/* Do platform dependent checks first. */
+
+	/*
+	 * A lightweight registration code for the root domain. We are
+	 * running on the boot CPU, hw interrupts are off, and
+	 * secondary CPUs are still lost in space.
+	 */
+
+	/* Reserve percpu data slot #0 for the root domain. */
+	ipd->slot = 0;
+	set_bit(0, &__ipipe_domain_slot_map);
+
+	ipd->name = "Linux";
+	ipd->domid = IPIPE_ROOT_ID;
+	ipd->priority = IPIPE_ROOT_PRIO;
+
+	__ipipe_init_stage(ipd);
+
+	INIT_LIST_HEAD(&ipd->p_link);
+	list_add_tail(&ipd->p_link, &__ipipe_pipeline);
+
+	__ipipe_init_platform();
+
+#ifdef CONFIG_PRINTK
+	__ipipe_printk_virq = ipipe_alloc_virq();	/* Cannot fail here. */
+	ipd->irqs[__ipipe_printk_virq].handler = &__ipipe_flush_printk;
+	ipd->irqs[__ipipe_printk_virq].cookie = NULL;
+	ipd->irqs[__ipipe_printk_virq].acknowledge = NULL;
+	ipd->irqs[__ipipe_printk_virq].control = IPIPE_HANDLE_MASK;
+#endif /* CONFIG_PRINTK */
+
+	__ipipe_enable_pipeline();
+
+	printk(KERN_INFO "I-pipe %s: pipeline enabled.\n",
+	       IPIPE_VERSION_STRING);
+}
+
+void __ipipe_init_stage(struct ipipe_domain *ipd)
+{
+	int cpu, n;
+
+	for_each_online_cpu(cpu) {
+
+		ipipe_percpudom(ipd, irqpend_himask, cpu) = 0;
+
+		for (n = 0; n < IPIPE_IRQ_IWORDS; n++) {
+			ipipe_percpudom(ipd, irqpend_lomask, cpu)[n] = 0;
+			ipipe_percpudom(ipd, irqheld_mask, cpu)[n] = 0;
+		}
+
+		for (n = 0; n < IPIPE_NR_IRQS; n++)
+			ipipe_percpudom(ipd, irqall, cpu)[n] = 0;
+
+		ipipe_percpudom(ipd, evsync, cpu) = 0;
+	}
+
+	for (n = 0; n < IPIPE_NR_IRQS; n++) {
+		ipd->irqs[n].acknowledge = NULL;
+		ipd->irqs[n].handler = NULL;
+		ipd->irqs[n].control = IPIPE_PASS_MASK;	/* Pass but don't handle */
+	}
+
+	for (n = 0; n < IPIPE_NR_EVENTS; n++)
+		ipd->evhand[n] = NULL;
+
+	ipd->evself = 0LL;
+	mutex_init(&ipd->mutex);
+
+	__ipipe_hook_critical_ipi(ipd);
+}
+
+void __ipipe_cleanup_domain(struct ipipe_domain *ipd)
+{
+	ipipe_unstall_pipeline_from(ipd);
+
+#ifdef CONFIG_SMP
+	{
+		int cpu;
+
+		for_each_online_cpu(cpu) {
+			while (ipipe_percpudom(ipd, irqpend_himask, cpu) != 0)
+				cpu_relax();
+		}
+	}
+#else
+	__raw_get_cpu_var(ipipe_percpu_daddr)[ipd->slot] = NULL;
+#endif
+
+	clear_bit(ipd->slot, &__ipipe_domain_slot_map);
+}
+
+void __ipipe_unstall_root(void)
+{
+#ifndef CONFIG_IPIPE_DEBUG_CONTEXT
+	BUG_ON(!ipipe_root_domain_p);
+#endif
+
+        local_irq_disable_hw();
+
+        __clear_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
+
+        if (unlikely(ipipe_root_cpudom_var(irqpend_himask) != 0))
+                __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+
+        local_irq_enable_hw();
+}
+
+void __ipipe_restore_root(unsigned long x)
+{
+#ifndef CONFIG_IPIPE_DEBUG_CONTEXT
+	BUG_ON(!ipipe_root_domain_p);
+#endif
+
+	if (x)
+		__ipipe_stall_root();
+	else
+		__ipipe_unstall_root();
+}
+
+void ipipe_stall_pipeline_from(struct ipipe_domain *ipd)
+{
+	set_bit_safe(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status));
+
+	if (__ipipe_pipeline_head_p(ipd))
+		local_irq_disable_hw();
+}
+
+unsigned long ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd)
+{
+	unsigned long x;
+
+	x = test_and_set_bit_safe(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status));
+
+	if (__ipipe_pipeline_head_p(ipd))
+		local_irq_disable_hw();
+
+	return x;
+}
+
+/*
+ * ipipe_unstall_pipeline_from() -- Unstall the pipeline and
+ * synchronize pending interrupts for a given domain. See
+ * __ipipe_walk_pipeline() for more information.
+ */
+void ipipe_unstall_pipeline_from(struct ipipe_domain *ipd)
+{
+	struct list_head *pos;
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
+
+	__clear_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status));
+
+	if (ipd == ipipe_current_domain)
+		pos = &ipd->p_link;
+	else
+		pos = __ipipe_pipeline.next;
+
+	__ipipe_walk_pipeline(pos);
+
+	if (likely(__ipipe_pipeline_head_p(ipd)))
+		local_irq_enable_hw();
+	else
+		local_irq_restore_hw(flags);
+}
+
+unsigned long ipipe_test_and_unstall_pipeline_from(struct ipipe_domain *ipd)
+{
+	unsigned long x;
+
+	x = test_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status));
+	ipipe_unstall_pipeline_from(ipd);
+
+	return x;
+}
+
+void ipipe_restore_pipeline_from(struct ipipe_domain *ipd,
+					  unsigned long x)
+{
+	if (x)
+		ipipe_stall_pipeline_from(ipd);
+	else
+		ipipe_unstall_pipeline_from(ipd);
+}
+
+void ipipe_unstall_pipeline_head(void)
+{
+	local_irq_disable_hw();
+
+	__clear_bit(IPIPE_STALL_FLAG, &ipipe_head_cpudom_var(status));
+
+	if (unlikely(ipipe_head_cpudom_var(irqpend_himask) != 0)) {
+		struct ipipe_domain *head_domain = __ipipe_pipeline_head();
+		if (likely(head_domain == ipipe_current_domain))
+			__ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+		else
+			__ipipe_walk_pipeline(&head_domain->p_link);
+        }
+
+	local_irq_enable_hw();
+}
+
+void __ipipe_restore_pipeline_head(unsigned long x)
+{
+	local_irq_disable_hw();
+
+	if (x) {
+#ifdef CONFIG_DEBUG_KERNEL
+		static int warned;
+		if (!warned && test_and_set_bit(IPIPE_STALL_FLAG, &ipipe_head_cpudom_var(status))) {
+			/*
+			 * Already stalled albeit ipipe_restore_pipeline_head()
+			 * should have detected it? Send a warning once.
+			 */
+			warned = 1;
+			printk(KERN_WARNING
+				   "I-pipe: ipipe_restore_pipeline_head() optimization failed.\n");
+			dump_stack();
+		}
+#else /* !CONFIG_DEBUG_KERNEL */
+		set_bit(IPIPE_STALL_FLAG, &ipipe_head_cpudom_var(status));
+#endif /* CONFIG_DEBUG_KERNEL */
+	}
+	else {
+		__clear_bit(IPIPE_STALL_FLAG, &ipipe_head_cpudom_var(status));
+		if (unlikely(ipipe_head_cpudom_var(irqpend_himask) != 0)) {
+			struct ipipe_domain *head_domain = __ipipe_pipeline_head();
+			if (likely(head_domain == ipipe_current_domain))
+				__ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+			else
+				__ipipe_walk_pipeline(&head_domain->p_link);
+		}
+		local_irq_enable_hw();
+	}
+}
+
+void __ipipe_spin_lock_irq(raw_spinlock_t *lock)
+{
+	local_irq_disable_hw();
+	__raw_spin_lock(lock);
+	__set_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
+}
+
+void __ipipe_spin_unlock_irq(raw_spinlock_t *lock)
+{
+	__raw_spin_unlock(lock);
+	__clear_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
+	local_irq_enable_hw();
+}
+
+unsigned long __ipipe_spin_lock_irqsave(raw_spinlock_t *lock)
+{
+	unsigned long flags;
+	int s;
+
+	local_irq_save_hw(flags);
+	__raw_spin_lock(lock);
+	s = __test_and_set_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
+
+	return raw_mangle_irq_bits(s, flags);
+}
+
+void __ipipe_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long x)
+{
+	__raw_spin_unlock(lock);
+	if (!raw_demangle_irq_bits(&x))
+		__clear_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
+	local_irq_restore_hw(x);
+}
+
+void __ipipe_spin_unlock_irqbegin(ipipe_spinlock_t *lock)
+{
+	__raw_spin_unlock(&lock->__raw_lock);
+}
+
+void __ipipe_spin_unlock_irqcomplete(unsigned long x)
+{
+	if (!raw_demangle_irq_bits(&x))
+		__clear_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
+	local_irq_restore_hw(x);
+}
+
+/* Must be called hw IRQs off. */
+void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned irq)
+{
+	int level = irq >> IPIPE_IRQ_ISHIFT, rank = irq & IPIPE_IRQ_IMASK;
+
+	if (likely(!test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) {
+		__set_bit(rank, &ipipe_cpudom_var(ipd, irqpend_lomask)[level]);
+		__set_bit(level,&ipipe_cpudom_var(ipd, irqpend_himask));
+	} else
+		__set_bit(rank, &ipipe_cpudom_var(ipd, irqheld_mask)[level]);
+
+	ipipe_cpudom_var(ipd, irqall)[irq]++;
+}
+
+/* Must be called hw IRQs off. */
+void __ipipe_lock_irq(struct ipipe_domain *ipd, int cpu, unsigned irq)
+{
+	if (likely(!test_and_set_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) {
+		int level = irq >> IPIPE_IRQ_ISHIFT, rank = irq & IPIPE_IRQ_IMASK;
+		if (__test_and_clear_bit(rank, &ipipe_percpudom(ipd, irqpend_lomask, cpu)[level]))
+			__set_bit(rank, &ipipe_cpudom_var(ipd, irqheld_mask)[level]);
+		if (ipipe_percpudom(ipd, irqpend_lomask, cpu)[level] == 0)
+			__clear_bit(level, &ipipe_percpudom(ipd, irqpend_himask, cpu));
+	}
+}
+
+/* Must be called hw IRQs off. */
+void __ipipe_unlock_irq(struct ipipe_domain *ipd, unsigned irq)
+{
+	int cpu;
+
+	if (likely(test_and_clear_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) {
+		int level = irq >> IPIPE_IRQ_ISHIFT, rank = irq & IPIPE_IRQ_IMASK;
+		for_each_online_cpu(cpu) {
+			if (test_and_clear_bit(rank, &ipipe_percpudom(ipd, irqheld_mask, cpu)[level])) {
+				/* We need atomic ops here: */
+				set_bit(rank, &ipipe_percpudom(ipd, irqpend_lomask, cpu)[level]);
+				set_bit(level, &ipipe_percpudom(ipd, irqpend_himask, cpu));
+			}
+		}
+	}
+}
+
+/* __ipipe_walk_pipeline(): Plays interrupts pending in the log. Must
+   be called with local hw interrupts disabled. */
+
+void __ipipe_walk_pipeline(struct list_head *pos)
+{
+	struct ipipe_domain *this_domain = ipipe_current_domain, *next_domain;
+
+	while (pos != &__ipipe_pipeline) {
+
+		next_domain = list_entry(pos, struct ipipe_domain, p_link);
+
+		if (test_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(next_domain, status)))
+			break;	/* Stalled stage -- do not go further. */
+
+		if (ipipe_cpudom_var(next_domain, irqpend_himask) != 0) {
+
+			if (next_domain == this_domain)
+				__ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+			else {
+
+				ipipe_cpudom_var(this_domain, evsync) = 0;
+				ipipe_current_domain = next_domain;
+				ipipe_suspend_domain();	/* Sync stage and propagate interrupts. */
+
+				if (ipipe_current_domain == next_domain)
+					ipipe_current_domain = this_domain;
+				/*
+				 * Otherwise, something changed the current domain under our
+				 * feet recycling the register set; do not override the new
+				 * domain.
+				 */
+
+				if (ipipe_cpudom_var(this_domain, irqpend_himask) != 0 &&
+				    !test_bit(IPIPE_STALL_FLAG,
+					      &ipipe_cpudom_var(this_domain, status)))
+					__ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+			}
+
+			break;
+		} else if (next_domain == this_domain)
+			break;
+
+		pos = next_domain->p_link.next;
+	}
+}
+
+/*
+ * ipipe_suspend_domain() -- Suspend the current domain, switching to
+ * the next one which has pending work down the pipeline.
+ */
+void ipipe_suspend_domain(void)
+{
+	struct ipipe_domain *this_domain, *next_domain;
+	struct list_head *ln;
+	unsigned long flags;
+
+	local_irq_save_hw(flags);
+
+	this_domain = next_domain = ipipe_current_domain;
+
+	__clear_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(this_domain, status));
+
+	if (ipipe_cpudom_var(this_domain, irqpend_himask) != 0)
+		goto sync_stage;
+
+	for (;;) {
+		ln = next_domain->p_link.next;
+
+		if (ln == &__ipipe_pipeline)
+			break;
+
+		next_domain = list_entry(ln, struct ipipe_domain, p_link);
+
+		if (test_bit(IPIPE_STALL_FLAG,
+			     &ipipe_cpudom_var(next_domain, status)) != 0)
+			break;
+
+		if (ipipe_cpudom_var(next_domain, irqpend_himask) == 0)
+			continue;
+
+		ipipe_current_domain = next_domain;
+
+sync_stage:
+		__ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+
+		if (ipipe_current_domain != next_domain)
+			/*
+			 * Something has changed the current domain under our
+			 * feet, recycling the register set; take note.
+			 */
+			this_domain = ipipe_current_domain;
+	}
+
+	ipipe_current_domain = this_domain;
+
+	local_irq_restore_hw(flags);
+}
+
+/* ipipe_alloc_virq() -- Allocate a pipelined virtual/soft interrupt.
+ * Virtual interrupts are handled in exactly the same way than their
+ * hw-generated counterparts wrt pipelining.
+ */
+unsigned ipipe_alloc_virq(void)
+{
+	unsigned long flags, irq = 0;
+	int ipos;
+
+	spin_lock_irqsave(&__ipipe_pipelock, flags);
+
+	if (__ipipe_virtual_irq_map != ~0) {
+		ipos = ffz(__ipipe_virtual_irq_map);
+		set_bit(ipos, &__ipipe_virtual_irq_map);
+		irq = ipos + IPIPE_VIRQ_BASE;
+	}
+
+	spin_unlock_irqrestore(&__ipipe_pipelock, flags);
+
+	return irq;
+}
+
+/* ipipe_virtualize_irq() -- Attach a handler (and optionally a hw
+   acknowledge routine) to an interrupt for a given domain. */
+
+int ipipe_virtualize_irq(struct ipipe_domain *ipd,
+			 unsigned irq,
+			 ipipe_irq_handler_t handler,
+			 void *cookie,
+			 ipipe_irq_ackfn_t acknowledge,
+			 unsigned modemask)
+{
+	unsigned long flags;
+	int err;
+
+	if (irq >= IPIPE_NR_IRQS)
+		return -EINVAL;
+
+	if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK)
+		return -EPERM;
+
+	if (!test_bit(IPIPE_AHEAD_FLAG, &ipd->flags))
+		/* Silently unwire interrupts for non-heading domains. */
+		modemask &= ~IPIPE_WIRED_MASK;
+
+	spin_lock_irqsave(&__ipipe_pipelock, flags);
+
+	if (handler != NULL) {
+		if (handler == IPIPE_SAME_HANDLER) {
+			handler = ipd->irqs[irq].handler;
+			cookie = ipd->irqs[irq].cookie;
+
+			if (handler == NULL) {
+				err = -EINVAL;
+				goto unlock_and_exit;
+			}
+		} else if ((modemask & IPIPE_EXCLUSIVE_MASK) != 0 &&
+			   ipd->irqs[irq].handler != NULL) {
+			err = -EBUSY;
+			goto unlock_and_exit;
+		}
+
+		/* Wired interrupts can only be delivered to domains
+		 * always heading the pipeline, and using dynamic
+		 * propagation. */
+
+		if ((modemask & IPIPE_WIRED_MASK) != 0) {
+			if ((modemask & (IPIPE_PASS_MASK | IPIPE_STICKY_MASK)) != 0) {
+				err = -EINVAL;
+				goto unlock_and_exit;
+			}
+			modemask |= (IPIPE_HANDLE_MASK);
+		}
+
+		if ((modemask & IPIPE_STICKY_MASK) != 0)
+			modemask |= IPIPE_HANDLE_MASK;
+	} else
+		modemask &=
+		    ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK |
+		      IPIPE_EXCLUSIVE_MASK | IPIPE_WIRED_MASK);
+
+	if (acknowledge == NULL && !ipipe_virtual_irq_p(irq))
+		/* Acknowledge handler unspecified for a hw interrupt:
+		   use the Linux-defined handler instead. */
+		acknowledge = ipipe_root_domain->irqs[irq].acknowledge;
+
+	ipd->irqs[irq].handler = handler;
+	ipd->irqs[irq].cookie = cookie;
+	ipd->irqs[irq].acknowledge = acknowledge;
+	ipd->irqs[irq].control = modemask;
+
+	if (irq < NR_IRQS && handler != NULL && !ipipe_virtual_irq_p(irq)) {
+		__ipipe_enable_irqdesc(ipd, irq);
+
+		if ((modemask & IPIPE_ENABLE_MASK) != 0) {
+			if (ipd != ipipe_current_domain) {
+				/* IRQ enable/disable state is domain-sensitive, so we may
+				   not change it for another domain. What is allowed
+				   however is forcing some domain to handle an interrupt
+				   source, by passing the proper 'ipd' descriptor which
+				   thus may be different from ipipe_current_domain. */
+				err = -EPERM;
+				goto unlock_and_exit;
+			}
+			__ipipe_enable_irq(irq);
+		}
+	}
+
+	err = 0;
+
+      unlock_and_exit:
+
+	spin_unlock_irqrestore(&__ipipe_pipelock, flags);
+
+	return err;
+}
+
+/* ipipe_control_irq() -- Change modes of a pipelined interrupt for
+ * the current domain. */
+
+int ipipe_control_irq(unsigned irq, unsigned clrmask, unsigned setmask)
+{
+	struct ipipe_domain *ipd;
+	unsigned long flags;
+
+	if (irq >= IPIPE_NR_IRQS)
+		return -EINVAL;
+
+	ipd = ipipe_current_domain;
+
+	if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK)
+		return -EPERM;
+
+	if (ipd->irqs[irq].handler == NULL)
+		setmask &= ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK);
+
+	if ((setmask & IPIPE_STICKY_MASK) != 0)
+		setmask |= IPIPE_HANDLE_MASK;
+
+	if ((clrmask & (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK)) != 0)	/* If one goes, both go. */
+		clrmask |= (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK);
+
+	spin_lock_irqsave(&__ipipe_pipelock, flags);
+
+	ipd->irqs[irq].control &= ~clrmask;
+	ipd->irqs[irq].control |= setmask;
+
+	if ((setmask & IPIPE_ENABLE_MASK) != 0)
+		__ipipe_enable_irq(irq);
+	else if ((clrmask & IPIPE_ENABLE_MASK) != 0)
+		__ipipe_disable_irq(irq);
+
+	spin_unlock_irqrestore(&__ipipe_pipelock, flags);
+
+	return 0;
+}
+
+/* __ipipe_dispatch_event() -- Low-level event dispatcher. */
+
+int __ipipe_dispatch_event (unsigned event, void *data)
+{
+	struct ipipe_domain *start_domain, *this_domain, *next_domain;
+	ipipe_event_handler_t evhand;
+	struct list_head *pos, *npos;
+	unsigned long flags;
+	int propagate = 1;
+
+	local_irq_save_hw(flags);
+
+	start_domain = this_domain = ipipe_current_domain;
+
+	list_for_each_safe(pos, npos, &__ipipe_pipeline) {
+		/*
+		 * Note: Domain migration may occur while running
+		 * event or interrupt handlers, in which case the
+		 * current register set is going to be recycled for a
+		 * different domain than the initiating one. We do
+		 * care for that, always tracking the current domain
+		 * descriptor upon return from those handlers.
+		 */
+		next_domain = list_entry(pos, struct ipipe_domain, p_link);
+
+		/*
+		 * Keep a cached copy of the handler's address since
+		 * ipipe_catch_event() may clear it under our feet.
+		 */
+		evhand = next_domain->evhand[event];
+
+		if (evhand != NULL) {
+			ipipe_current_domain = next_domain;
+			ipipe_cpudom_var(next_domain, evsync) |= (1LL << event);
+			local_irq_restore_hw(flags);
+			propagate = !evhand(event, start_domain, data);
+			local_irq_save_hw(flags);
+			ipipe_cpudom_var(next_domain, evsync) &= ~(1LL << event);
+			if (ipipe_current_domain != next_domain)
+				this_domain = ipipe_current_domain;
+		}
+
+		if (next_domain != ipipe_root_domain &&	/* NEVER sync the root stage here. */
+		    ipipe_cpudom_var(next_domain, irqpend_himask) != 0 &&
+		    !test_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(next_domain, status))) {
+			ipipe_current_domain = next_domain;
+			__ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+			if (ipipe_current_domain != next_domain)
+				this_domain = ipipe_current_domain;
+		}
+
+		ipipe_current_domain = this_domain;
+
+		if (next_domain == this_domain || !propagate)
+			break;
+	}
+
+	local_irq_restore_hw(flags);
+
+	return !propagate;
+}
+
+/*
+ * __ipipe_dispatch_wired -- Wired interrupt dispatcher. Wired
+ * interrupts are immediately and unconditionally delivered to the
+ * domain heading the pipeline upon receipt, and such domain must have
+ * been registered as an invariant head for the system (priority ==
+ * IPIPE_HEAD_PRIORITY). The motivation for using wired interrupts is
+ * to get an extra-fast dispatching path for those IRQs, by relying on
+ * a straightforward logic based on assumptions that must always be
+ * true for invariant head domains.  The following assumptions are
+ * made when dealing with such interrupts:
+ *
+ * 1- Wired interrupts are purely dynamic, i.e. the decision to
+ * propagate them down the pipeline must be done from the head domain
+ * ISR.
+ * 2- Wired interrupts cannot be shared or sticky.
+ * 3- The root domain cannot be an invariant pipeline head, in
+ * consequence of what the root domain cannot handle wired
+ * interrupts.
+ * 4- Wired interrupts must have a valid acknowledge handler for the
+ * head domain (if needed), and in any case, must not rely on handlers
+ * provided by lower priority domains during the acknowledge cycle
+ * (see __ipipe_handle_irq).
+ *
+ * Called with hw interrupts off.
+ */
+
+int __ipipe_dispatch_wired(struct ipipe_domain *head_domain, unsigned irq)
+{
+	struct ipipe_domain *old;
+
+	if (test_bit(IPIPE_LOCK_FLAG, &head_domain->irqs[irq].control)) {
+		/* If we can't process this IRQ right now, we must
+		 * mark it as held, so that it will get played during
+		 * normal log sync when the corresponding interrupt
+		 * source is eventually unlocked. */
+		ipipe_cpudom_var(head_domain, irqall)[irq]++;
+		__set_bit(irq & IPIPE_IRQ_IMASK, &ipipe_cpudom_var(head_domain, irqheld_mask)[irq >> IPIPE_IRQ_ISHIFT]);
+		return 0;
+	}
+
+	if (test_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(head_domain, status))) {
+		__ipipe_set_irq_pending(head_domain, irq);
+		return 0;
+	}
+
+	old = ipipe_current_domain;
+	ipipe_current_domain = head_domain; /* Switch to the head domain. */
+
+	ipipe_cpudom_var(head_domain, irqall)[irq]++;
+	__set_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(head_domain, status));
+	head_domain->irqs[irq].handler(irq, head_domain->irqs[irq].cookie); /* Call the ISR. */
+	__ipipe_run_irqtail();
+	__clear_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(head_domain, status));
+
+	/* We expect the caller to start a complete pipeline walk upon
+	 * return, so that propagated interrupts will get played. */
+
+	if (ipipe_current_domain == head_domain)
+		ipipe_current_domain = old; /* Back to the preempted domain. */
+
+	return 1;
+}
+
+/*
+ * __ipipe_sync_stage() -- Flush the pending IRQs for the current
+ * domain (and processor). This routine flushes the interrupt log
+ * (see "Optimistic interrupt protection" from D. Stodolsky et al. for
+ * more on the deferred interrupt scheme). Every interrupt that
+ * occurred while the pipeline was stalled gets played. WARNING:
+ * callers on SMP boxen should always check for CPU migration on
+ * return of this routine. One can control the kind of interrupts
+ * which are going to be sync'ed using the syncmask
+ * parameter. IPIPE_IRQMASK_ANY plays them all, IPIPE_IRQMASK_VIRT
+ * plays virtual interrupts only.
+ *
+ * This routine must be called with hw interrupts off.
+ */
+void __ipipe_sync_stage(unsigned long syncmask)
+{
+	unsigned long mask, submask;
+	struct ipipe_domain *ipd;
+	int level, rank, cpu;
+	unsigned irq;
+
+	if (__test_and_set_bit(IPIPE_SYNC_FLAG, &ipipe_this_cpudom_var(status)))
+		return;
+
+	ipd = ipipe_current_domain;
+	cpu = ipipe_processor_id();
+
+	/*
+	 * The policy here is to keep the dispatching code interrupt-free
+	 * by stalling the current stage. If the upper domain handler
+	 * (which we call) wants to re-enable interrupts while in a safe
+	 * portion of the code (e.g. SA_INTERRUPT flag unset for Linux's
+	 * sigaction()), it will have to unstall (then stall again before
+	 * returning to us!) the stage when it sees fit.
+	 */
+	while ((mask = (ipipe_this_cpudom_var(irqpend_himask) & syncmask)) != 0) {
+		level = __ipipe_ffnz(mask);
+
+		while ((submask = ipipe_this_cpudom_var(irqpend_lomask)[level]) != 0) {
+			rank = __ipipe_ffnz(submask);
+			irq = (level << IPIPE_IRQ_ISHIFT) + rank;
+
+			__clear_bit(rank, &ipipe_this_cpudom_var(irqpend_lomask)[level]);
+
+			if (ipipe_this_cpudom_var(irqpend_lomask)[level] == 0)
+				__clear_bit(level, &ipipe_this_cpudom_var(irqpend_himask));
+			/*
+			 * Make sure the compiler will not postpone
+			 * the pending bitmask updates before calling
+			 * the interrupt handling routine. Otherwise,
+			 * those late updates could overwrite any
+			 * change to irqpend_hi/lomask due to a nested
+			 * interrupt, leaving the latter unprocessed
+			 * (seen on mpc836x).
+			 */
+			barrier();
+
+			if (test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
+				continue;
+
+			__set_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
+
+			if (ipd == ipipe_root_domain)
+				trace_hardirqs_off();
+
+			__ipipe_run_isr(ipd, irq);
+#ifdef CONFIG_SMP
+			{
+				int newcpu = ipipe_processor_id();
+
+				if (newcpu != cpu) {	/* Handle CPU migration. */
+					/*
+					 * We expect any domain to clear the SYNC bit each
+					 * time it switches in a new task, so that preemptions
+					 * and/or CPU migrations (in the SMP case) over the
+					 * ISR do not lock out the log syncer for some
+					 * indefinite amount of time. In the Linux case,
+					 * schedule() handles this (see kernel/sched.c). For
+					 * this reason, we don't bother clearing it here for
+					 * the source CPU in the migration handling case,
+					 * since it must have scheduled another task in by
+					 * now.
+					 */
+					__set_bit(IPIPE_SYNC_FLAG, &ipipe_this_cpudom_var(status));
+					cpu = newcpu;
+				}
+			}
+#endif	/* CONFIG_SMP */
+			if (ipd == ipipe_root_domain &&
+			    test_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status)))
+				trace_hardirqs_on();
+
+			__clear_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
+		}
+	}
+
+	__clear_bit(IPIPE_SYNC_FLAG, &ipipe_this_cpudom_var(status));
+}
+
+/* ipipe_register_domain() -- Link a new domain to the pipeline. */
+
+int ipipe_register_domain(struct ipipe_domain *ipd,
+			  struct ipipe_domain_attr *attr)
+{
+	struct ipipe_domain *_ipd;
+	struct list_head *pos = NULL;
+	unsigned long flags;
+
+	if (!ipipe_root_domain_p) {
+		printk(KERN_WARNING
+		       "I-pipe: Only the root domain may register a new domain.\n");
+		return -EPERM;
+	}
+
+	flags = ipipe_critical_enter(NULL);
+
+	if (attr->priority == IPIPE_HEAD_PRIORITY) {
+		if (test_bit(IPIPE_HEAD_SLOT, &__ipipe_domain_slot_map)) {
+			ipipe_critical_exit(flags);
+			return -EAGAIN;	/* Cannot override current head. */
+		}
+		ipd->slot = IPIPE_HEAD_SLOT;
+	} else
+		ipd->slot = ffz(__ipipe_domain_slot_map);
+
+	if (ipd->slot < CONFIG_IPIPE_DOMAINS) {
+		set_bit(ipd->slot, &__ipipe_domain_slot_map);
+		list_for_each(pos, &__ipipe_pipeline) {
+			_ipd = list_entry(pos, struct ipipe_domain, p_link);
+			if (_ipd->domid == attr->domid)
+				break;
+		}
+	}
+
+	ipipe_critical_exit(flags);
+
+	if (pos != &__ipipe_pipeline) {
+		if (ipd->slot < CONFIG_IPIPE_DOMAINS)
+			clear_bit(ipd->slot, &__ipipe_domain_slot_map);
+		return -EBUSY;
+	}
+
+#ifndef CONFIG_SMP
+	/*
+	 * Set up the perdomain pointers for direct access to the
+	 * percpu domain data. This saves a costly multiply each time
+	 * we need to refer to the contents of the percpu domain data
+	 * array.
+	 */
+	__raw_get_cpu_var(ipipe_percpu_daddr)[ipd->slot] = &__raw_get_cpu_var(ipipe_percpu_darray)[ipd->slot];
+#endif
+
+	ipd->name = attr->name;
+	ipd->domid = attr->domid;
+	ipd->pdd = attr->pdd;
+	ipd->flags = 0;
+
+	if (attr->priority == IPIPE_HEAD_PRIORITY) {
+		ipd->priority = INT_MAX;
+		__set_bit(IPIPE_AHEAD_FLAG,&ipd->flags);
+	}
+	else
+		ipd->priority = attr->priority;
+
+	__ipipe_init_stage(ipd);
+
+	INIT_LIST_HEAD(&ipd->p_link);
+
+#ifdef CONFIG_PROC_FS
+	__ipipe_add_domain_proc(ipd);
+#endif /* CONFIG_PROC_FS */
+
+	flags = ipipe_critical_enter(NULL);
+
+	list_for_each(pos, &__ipipe_pipeline) {
+		_ipd = list_entry(pos, struct ipipe_domain, p_link);
+		if (ipd->priority > _ipd->priority)
+			break;
+	}
+
+	list_add_tail(&ipd->p_link, pos);
+
+	ipipe_critical_exit(flags);
+
+	printk(KERN_INFO "I-pipe: Domain %s registered.\n", ipd->name);
+
+	/*
+	 * Finally, allow the new domain to perform its initialization
+	 * chores.
+	 */
+
+	if (attr->entry != NULL) {
+		ipipe_current_domain = ipd;
+		attr->entry();
+		ipipe_current_domain = ipipe_root_domain;
+
+		local_irq_save_hw(flags);
+
+		if (ipipe_root_cpudom_var(irqpend_himask) != 0 &&
+		    !test_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status)))
+			__ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+
+		local_irq_restore_hw(flags);
+	}
+
+	return 0;
+}
+
+/* ipipe_unregister_domain() -- Remove a domain from the pipeline. */
+
+int ipipe_unregister_domain(struct ipipe_domain *ipd)
+{
+	unsigned long flags;
+
+	if (!ipipe_root_domain_p) {
+		printk(KERN_WARNING
+		       "I-pipe: Only the root domain may unregister a domain.\n");
+		return -EPERM;
+	}
+
+	if (ipd == ipipe_root_domain) {
+		printk(KERN_WARNING
+		       "I-pipe: Cannot unregister the root domain.\n");
+		return -EPERM;
+	}
+#ifdef CONFIG_SMP
+	{
+		unsigned irq;
+		int cpu;
+
+		/*
+		 * In the SMP case, wait for the logged events to drain on
+		 * other processors before eventually removing the domain
+		 * from the pipeline.
+		 */
+
+		ipipe_unstall_pipeline_from(ipd);
+
+		flags = ipipe_critical_enter(NULL);
+
+		for (irq = 0; irq < IPIPE_NR_IRQS; irq++) {
+			clear_bit(IPIPE_HANDLE_FLAG, &ipd->irqs[irq].control);
+			clear_bit(IPIPE_STICKY_FLAG, &ipd->irqs[irq].control);
+			set_bit(IPIPE_PASS_FLAG, &ipd->irqs[irq].control);
+		}
+
+		ipipe_critical_exit(flags);
+
+		for_each_online_cpu(cpu) {
+			while (ipipe_percpudom(ipd, irqpend_himask, cpu) > 0)
+				cpu_relax();
+		}
+	}
+#endif	/* CONFIG_SMP */
+
+	mutex_lock(&ipd->mutex);
+
+#ifdef CONFIG_PROC_FS
+	__ipipe_remove_domain_proc(ipd);
+#endif /* CONFIG_PROC_FS */
+
+	/*
+	 * Simply remove the domain from the pipeline and we are almost done.
+	 */
+
+	flags = ipipe_critical_enter(NULL);
+	list_del_init(&ipd->p_link);
+	ipipe_critical_exit(flags);
+
+	__ipipe_cleanup_domain(ipd);
+
+	mutex_unlock(&ipd->mutex);
+
+	printk(KERN_INFO "I-pipe: Domain %s unregistered.\n", ipd->name);
+
+	return 0;
+}
+
+/*
+ * ipipe_propagate_irq() -- Force a given IRQ propagation on behalf of
+ * a running interrupt handler to the next domain down the pipeline.
+ * ipipe_schedule_irq() -- Does almost the same as above, but attempts
+ * to pend the interrupt for the current domain first.
+ */
+int __ipipe_schedule_irq(unsigned irq, struct list_head *head)
+{
+	struct ipipe_domain *ipd;
+	struct list_head *ln;
+	unsigned long flags;
+
+	if (irq >= IPIPE_NR_IRQS ||
+	    (ipipe_virtual_irq_p(irq)
+	     && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)))
+		return -EINVAL;
+
+	local_irq_save_hw(flags);
+
+	ln = head;
+
+	while (ln != &__ipipe_pipeline) {
+
+		ipd = list_entry(ln, struct ipipe_domain, p_link);
+
+		if (test_bit(IPIPE_HANDLE_FLAG, &ipd->irqs[irq].control)) {
+			__ipipe_set_irq_pending(ipd, irq);
+			local_irq_restore_hw(flags);
+			return 1;
+		}
+
+		ln = ipd->p_link.next;
+	}
+
+	local_irq_restore_hw(flags);
+
+	return 0;
+}
+
+/* ipipe_free_virq() -- Release a virtual/soft interrupt. */
+
+int ipipe_free_virq(unsigned virq)
+{
+	if (!ipipe_virtual_irq_p(virq))
+		return -EINVAL;
+
+	clear_bit(virq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map);
+
+	return 0;
+}
+
+void ipipe_init_attr(struct ipipe_domain_attr *attr)
+{
+	attr->name = "anon";
+	attr->domid = 1;
+	attr->entry = NULL;
+	attr->priority = IPIPE_ROOT_PRIO;
+	attr->pdd = NULL;
+}
+
+/*
+ * ipipe_catch_event() -- Interpose or remove an event handler for a
+ * given domain.
+ */
+ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd,
+					unsigned event,
+					ipipe_event_handler_t handler)
+{
+	ipipe_event_handler_t old_handler;
+	unsigned long flags;
+	int self = 0, cpu;
+
+	if (event & IPIPE_EVENT_SELF) {
+		event &= ~IPIPE_EVENT_SELF;
+		self = 1;
+	}
+
+	if (event >= IPIPE_NR_EVENTS)
+		return NULL;
+
+	flags = ipipe_critical_enter(NULL);
+
+	if (!(old_handler = xchg(&ipd->evhand[event],handler)))	{
+		if (handler) {
+			if (self)
+				ipd->evself |= (1LL << event);
+			else
+				__ipipe_event_monitors[event]++;
+		}
+	}
+	else if (!handler) {
+		if (ipd->evself & (1LL << event))
+			ipd->evself &= ~(1LL << event);
+		else
+			__ipipe_event_monitors[event]--;
+	} else if ((ipd->evself & (1LL << event)) && !self) {
+			__ipipe_event_monitors[event]++;
+			ipd->evself &= ~(1LL << event);
+	} else if (!(ipd->evself & (1LL << event)) && self) {
+			__ipipe_event_monitors[event]--;
+			ipd->evself |= (1LL << event);
+	}
+
+	ipipe_critical_exit(flags);
+
+	if (!handler && ipipe_root_domain_p) {
+		/*
+		 * If we cleared a handler on behalf of the root
+		 * domain, we have to wait for any current invocation
+		 * to drain, since our caller might subsequently unmap
+		 * the target domain. To this aim, this code
+		 * synchronizes with __ipipe_dispatch_event(),
+		 * guaranteeing that either the dispatcher sees a null
+		 * handler in which case it discards the invocation
+		 * (which also prevents from entering a livelock), or
+		 * finds a valid handler and calls it. Symmetrically,
+		 * ipipe_catch_event() ensures that the called code
+		 * won't be unmapped under our feet until the event
+		 * synchronization flag is cleared for the given event
+		 * on all CPUs.
+		 */
+
+		for_each_online_cpu(cpu) {
+			while (ipipe_percpudom(ipd, evsync, cpu) & (1LL << event))
+				schedule_timeout_interruptible(HZ / 50);
+		}
+	}
+
+	return old_handler;
+}
+
+cpumask_t ipipe_set_irq_affinity (unsigned irq, cpumask_t cpumask)
+{
+#ifdef CONFIG_SMP
+	if (irq >= IPIPE_NR_XIRQS)
+		/* Allow changing affinity of external IRQs only. */
+		return CPU_MASK_NONE;
+
+	if (num_online_cpus() > 1)
+		return __ipipe_set_irq_affinity(irq,cpumask);
+#endif /* CONFIG_SMP */
+
+	return CPU_MASK_NONE;
+}
+
+int ipipe_send_ipi (unsigned ipi, cpumask_t cpumask)
+
+{
+#ifdef CONFIG_SMP
+	return __ipipe_send_ipi(ipi,cpumask);
+#else /* !CONFIG_SMP */
+	return -EINVAL;
+#endif /* CONFIG_SMP */
+}
+
+int ipipe_alloc_ptdkey (void)
+{
+	unsigned long flags;
+	int key = -1;
+
+	spin_lock_irqsave(&__ipipe_pipelock,flags);
+
+	if (__ipipe_ptd_key_count < IPIPE_ROOT_NPTDKEYS) {
+		key = ffz(__ipipe_ptd_key_map);
+		set_bit(key,&__ipipe_ptd_key_map);
+		__ipipe_ptd_key_count++;
+	}
+
+	spin_unlock_irqrestore(&__ipipe_pipelock,flags);
+
+	return key;
+}
+
+int ipipe_free_ptdkey (int key)
+{
+	unsigned long flags;
+
+	if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+		return -EINVAL;
+
+	spin_lock_irqsave(&__ipipe_pipelock,flags);
+
+	if (test_and_clear_bit(key,&__ipipe_ptd_key_map))
+		__ipipe_ptd_key_count--;
+
+	spin_unlock_irqrestore(&__ipipe_pipelock,flags);
+
+	return 0;
+}
+
+int ipipe_set_ptd (int key, void *value)
+
+{
+	if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+		return -EINVAL;
+
+	current->ptd[key] = value;
+
+	return 0;
+}
+
+void *ipipe_get_ptd (int key)
+
+{
+	if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+		return NULL;
+
+	return current->ptd[key];
+}
+
+#ifdef CONFIG_PROC_FS
+
+struct proc_dir_entry *ipipe_proc_root;
+
+static int __ipipe_version_info_proc(char *page,
+				     char **start,
+				     off_t off, int count, int *eof, void *data)
+{
+	int len = sprintf(page, "%s\n", IPIPE_VERSION_STRING);
+
+	len -= off;
+
+	if (len <= off + count)
+		*eof = 1;
+
+	*start = page + off;
+
+	if(len > count)
+		len = count;
+
+	if(len < 0)
+		len = 0;
+
+	return len;
+}
+
+static int __ipipe_common_info_show(struct seq_file *p, void *data)
+{
+	struct ipipe_domain *ipd = (struct ipipe_domain *)p->private;
+	char handling, stickiness, lockbit, exclusive, virtuality;
+
+	unsigned long ctlbits;
+	unsigned irq;
+
+	seq_printf(p, "       +----- Handling ([A]ccepted, [G]rabbed, [W]ired, [D]iscarded)\n");
+	seq_printf(p, "       |+---- Sticky\n");
+	seq_printf(p, "       ||+--- Locked\n");
+	seq_printf(p, "       |||+-- Exclusive\n");
+	seq_printf(p, "       ||||+- Virtual\n");
+	seq_printf(p, "[IRQ]  |||||\n");
+
+	mutex_lock(&ipd->mutex);
+
+	for (irq = 0; irq < IPIPE_NR_IRQS; irq++) {
+		/* Remember to protect against
+		 * ipipe_virtual_irq/ipipe_control_irq if more fields
+		 * get involved. */
+		ctlbits = ipd->irqs[irq].control;
+
+		if (irq >= IPIPE_NR_XIRQS && !ipipe_virtual_irq_p(irq))
+			/*
+			 * There might be a hole between the last external
+			 * IRQ and the first virtual one; skip it.
+			 */
+			continue;
+
+		if (ipipe_virtual_irq_p(irq)
+		    && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map))
+			/* Non-allocated virtual IRQ; skip it. */
+			continue;
+
+		/*
+		 * Statuses are as follows:
+		 * o "accepted" means handled _and_ passed down the pipeline.
+		 * o "grabbed" means handled, but the interrupt might be
+		 * terminated _or_ passed down the pipeline depending on
+		 * what the domain handler asks for to the I-pipe.
+		 * o "wired" is basically the same as "grabbed", except that
+		 * the interrupt is unconditionally delivered to an invariant
+		 * pipeline head domain.
+		 * o "passed" means unhandled by the domain but passed
+		 * down the pipeline.
+		 * o "discarded" means unhandled and _not_ passed down the
+		 * pipeline. The interrupt merely disappears from the
+		 * current domain down to the end of the pipeline.
+		 */
+		if (ctlbits & IPIPE_HANDLE_MASK) {
+			if (ctlbits & IPIPE_PASS_MASK)
+				handling = 'A';
+			else if (ctlbits & IPIPE_WIRED_MASK)
+				handling = 'W';
+			else
+				handling = 'G';
+		} else if (ctlbits & IPIPE_PASS_MASK)
+			/* Do not output if no major action is taken. */
+			continue;
+		else
+			handling = 'D';
+
+		if (ctlbits & IPIPE_STICKY_MASK)
+			stickiness = 'S';
+		else
+			stickiness = '.';
+
+		if (ctlbits & IPIPE_LOCK_MASK)
+			lockbit = 'L';
+		else
+			lockbit = '.';
+
+		if (ctlbits & IPIPE_EXCLUSIVE_MASK)
+			exclusive = 'X';
+		else
+			exclusive = '.';
+
+		if (ipipe_virtual_irq_p(irq))
+			virtuality = 'V';
+		else
+			virtuality = '.';
+
+		seq_printf(p, " %3u:  %c%c%c%c%c\n",
+			     irq, handling, stickiness, lockbit, exclusive, virtuality);
+	}
+
+	seq_printf(p, "[Domain info]\n");
+
+	seq_printf(p, "id=0x%.8x\n", ipd->domid);
+
+	if (test_bit(IPIPE_AHEAD_FLAG,&ipd->flags))
+		seq_printf(p, "priority=topmost\n");
+	else
+		seq_printf(p, "priority=%d\n", ipd->priority);
+
+	mutex_unlock(&ipd->mutex);
+
+	return 0;
+}
+
+static int __ipipe_common_info_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, __ipipe_common_info_show, PROC_I(inode)->pde->data);
+}
+
+static struct file_operations __ipipe_info_proc_ops = {
+	.owner		= THIS_MODULE,
+	.open		= __ipipe_common_info_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+void __ipipe_add_domain_proc(struct ipipe_domain *ipd)
+{
+	struct proc_dir_entry *e = create_proc_entry(ipd->name, 0444, ipipe_proc_root);
+	if (e) {
+		e->proc_fops = &__ipipe_info_proc_ops;
+		e->data = (void*) ipd;
+	}
+}
+
+void __ipipe_remove_domain_proc(struct ipipe_domain *ipd)
+{
+	remove_proc_entry(ipd->name,ipipe_proc_root);
+}
+
+void __init ipipe_init_proc(void)
+{
+	ipipe_proc_root = create_proc_entry("ipipe",S_IFDIR, 0);
+	create_proc_read_entry("version",0444,ipipe_proc_root,&__ipipe_version_info_proc,NULL);
+	__ipipe_add_domain_proc(ipipe_root_domain);
+
+	__ipipe_init_tracer();
+}
+
+#endif	/* CONFIG_PROC_FS */
+
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
+
+DEFINE_PER_CPU(int, ipipe_percpu_context_check) = { 1 };
+DEFINE_PER_CPU(int, ipipe_saved_context_check_state);
+
+void ipipe_check_context(struct ipipe_domain *border_ipd)
+{
+	/* Note: We don't make the per_cpu access atomic. We assume that code
+	   which temporarily disables the check does this in atomic context
+	   only. */
+	if (likely(ipipe_current_domain->priority <= border_ipd->priority &&
+		   !test_bit(IPIPE_STALL_FLAG,
+			     &ipipe_head_cpudom_var(status))) ||
+	    !per_cpu(ipipe_percpu_context_check, ipipe_processor_id()))
+		return;
+
+	ipipe_context_check_off();
+
+	ipipe_trace_panic_freeze();
+	ipipe_set_printk_sync(ipipe_current_domain);
+
+	if (ipipe_current_domain->priority > border_ipd->priority)
+		printk(KERN_ERR "I-pipe: Detected illicit call from domain "
+				"'%s'\n"
+		       KERN_ERR "        into a service reserved for domain "
+				"'%s' and below.\n",
+		       ipipe_current_domain->name, border_ipd->name);
+	else
+		printk(KERN_ERR "I-pipe: Detected stalled topmost domain, "
+				"probably caused by a bug.\n"
+				"        A critical section may have been "
+				"left unterminated.\n");
+	dump_stack();
+	ipipe_trace_panic_dump();
+}
+
+EXPORT_SYMBOL(ipipe_check_context);
+#endif /* CONFIG_IPIPE_DEBUG_CONTEXT */
+
+EXPORT_SYMBOL(ipipe_virtualize_irq);
+EXPORT_SYMBOL(ipipe_control_irq);
+EXPORT_SYMBOL(ipipe_suspend_domain);
+EXPORT_SYMBOL(ipipe_alloc_virq);
+EXPORT_PER_CPU_SYMBOL(ipipe_percpu_domain);
+EXPORT_PER_CPU_SYMBOL(ipipe_percpu_darray);
+EXPORT_SYMBOL(ipipe_root);
+EXPORT_SYMBOL(ipipe_stall_pipeline_from);
+EXPORT_SYMBOL(ipipe_test_and_stall_pipeline_from);
+EXPORT_SYMBOL(ipipe_unstall_pipeline_from);
+EXPORT_SYMBOL(ipipe_restore_pipeline_from);
+EXPORT_SYMBOL(ipipe_test_and_unstall_pipeline_from);
+EXPORT_SYMBOL(ipipe_unstall_pipeline_head);
+EXPORT_SYMBOL(__ipipe_restore_pipeline_head);
+EXPORT_SYMBOL(__ipipe_unstall_root);
+EXPORT_SYMBOL(__ipipe_restore_root);
+EXPORT_SYMBOL(__ipipe_spin_lock_irq);
+EXPORT_SYMBOL(__ipipe_spin_unlock_irq);
+EXPORT_SYMBOL(__ipipe_spin_lock_irqsave);
+EXPORT_SYMBOL(__ipipe_spin_unlock_irqrestore);
+EXPORT_SYMBOL(__ipipe_pipeline);
+EXPORT_SYMBOL(__ipipe_lock_irq);
+EXPORT_SYMBOL(__ipipe_unlock_irq);
+EXPORT_SYMBOL(ipipe_register_domain);
+EXPORT_SYMBOL(ipipe_unregister_domain);
+EXPORT_SYMBOL(ipipe_free_virq);
+EXPORT_SYMBOL(ipipe_init_attr);
+EXPORT_SYMBOL(ipipe_catch_event);
+EXPORT_SYMBOL(ipipe_alloc_ptdkey);
+EXPORT_SYMBOL(ipipe_free_ptdkey);
+EXPORT_SYMBOL(ipipe_set_ptd);
+EXPORT_SYMBOL(ipipe_get_ptd);
+EXPORT_SYMBOL(ipipe_set_irq_affinity);
+EXPORT_SYMBOL(ipipe_send_ipi);
+EXPORT_SYMBOL(__ipipe_schedule_irq);
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
+EXPORT_SYMBOL(ipipe_request_tickdev);
+EXPORT_SYMBOL(ipipe_release_tickdev);
+#endif
+
+EXPORT_SYMBOL(ipipe_critical_enter);
+EXPORT_SYMBOL(ipipe_critical_exit);
+EXPORT_SYMBOL(ipipe_trigger_irq);
+EXPORT_SYMBOL(ipipe_get_sysinfo);
diff --git a/kernel/ipipe/tracer.c b/kernel/ipipe/tracer.c
new file mode 100644
index 0000000..fc7fe99
--- /dev/null
+++ b/kernel/ipipe/tracer.c
@@ -0,0 +1,1336 @@
+/* -*- linux-c -*-
+ * kernel/ipipe/tracer.c
+ *
+ * Copyright (C) 2005 Luotao Fu.
+ *               2005-2008 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/kallsyms.h>
+#include <linux/seq_file.h>
+#include <linux/proc_fs.h>
+#include <linux/ctype.h>
+#include <linux/vmalloc.h>
+#include <linux/utsrelease.h>
+#include <linux/sched.h>
+#include <linux/ipipe.h>
+#include <asm/uaccess.h>
+
+#define IPIPE_TRACE_PATHS           4 /* <!> Do not lower below 3 */
+#define IPIPE_DEFAULT_ACTIVE        0
+#define IPIPE_DEFAULT_MAX           1
+#define IPIPE_DEFAULT_FROZEN        2
+
+#define IPIPE_TRACE_POINTS          (1 << CONFIG_IPIPE_TRACE_SHIFT)
+#define WRAP_POINT_NO(point)        ((point) & (IPIPE_TRACE_POINTS-1))
+
+#define IPIPE_DEFAULT_PRE_TRACE     10
+#define IPIPE_DEFAULT_POST_TRACE    10
+#define IPIPE_DEFAULT_BACK_TRACE    100
+
+#define IPIPE_DELAY_NOTE            1000  /* in nanoseconds */
+#define IPIPE_DELAY_WARN            10000 /* in nanoseconds */
+
+#define IPIPE_TFLG_NMI_LOCK         0x0001
+#define IPIPE_TFLG_NMI_HIT          0x0002
+#define IPIPE_TFLG_NMI_FREEZE_REQ   0x0004
+
+#define IPIPE_TFLG_HWIRQ_OFF        0x0100
+#define IPIPE_TFLG_FREEZING         0x0200
+#define IPIPE_TFLG_CURRDOM_SHIFT    10   /* bits 10..11: current domain */
+#define IPIPE_TFLG_CURRDOM_MASK     0x0C00
+#define IPIPE_TFLG_DOMSTATE_SHIFT   12   /* bits 12..15: domain stalled? */
+#define IPIPE_TFLG_DOMSTATE_BITS    3
+
+#define IPIPE_TFLG_DOMAIN_STALLED(point, n) \
+	(point->flags & (1 << (n + IPIPE_TFLG_DOMSTATE_SHIFT)))
+#define IPIPE_TFLG_CURRENT_DOMAIN(point) \
+	((point->flags & IPIPE_TFLG_CURRDOM_MASK) >> IPIPE_TFLG_CURRDOM_SHIFT)
+
+struct ipipe_trace_point {
+	short type;
+	short flags;
+	unsigned long eip;
+	unsigned long parent_eip;
+	unsigned long v;
+	unsigned long long timestamp;
+};
+
+struct ipipe_trace_path {
+	volatile int flags;
+	int dump_lock; /* separated from flags due to cross-cpu access */
+	int trace_pos; /* next point to fill */
+	int begin, end; /* finalised path begin and end */
+	int post_trace; /* non-zero when in post-trace phase */
+	unsigned long long length; /* max path length in cycles */
+	unsigned long nmi_saved_eip; /* for deferred requests from NMIs */
+	unsigned long nmi_saved_parent_eip;
+	unsigned long nmi_saved_v;
+	struct ipipe_trace_point point[IPIPE_TRACE_POINTS];
+} ____cacheline_aligned_in_smp;
+
+enum ipipe_trace_type
+{
+	IPIPE_TRACE_FUNC = 0,
+	IPIPE_TRACE_BEGIN,
+	IPIPE_TRACE_END,
+	IPIPE_TRACE_FREEZE,
+	IPIPE_TRACE_SPECIAL,
+	IPIPE_TRACE_PID,
+};
+
+#define IPIPE_TYPE_MASK             0x0007
+#define IPIPE_TYPE_BITS             3
+
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
+static DEFINE_PER_CPU(struct ipipe_trace_path *, trace_path);
+#else /* !CONFIG_IPIPE_TRACE_VMALLOC */
+static DEFINE_PER_CPU(struct ipipe_trace_path, trace_path[IPIPE_TRACE_PATHS]) =
+	{ [0 ... IPIPE_TRACE_PATHS-1] = { .begin = -1, .end = -1 } };
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
+
+int ipipe_trace_enable = 0;
+
+static DEFINE_PER_CPU(int, active_path) = { IPIPE_DEFAULT_ACTIVE };
+static DEFINE_PER_CPU(int, max_path) = { IPIPE_DEFAULT_MAX };
+static DEFINE_PER_CPU(int, frozen_path) = { IPIPE_DEFAULT_FROZEN };
+static IPIPE_DEFINE_SPINLOCK(global_path_lock);
+static int pre_trace = IPIPE_DEFAULT_PRE_TRACE;
+static int post_trace = IPIPE_DEFAULT_POST_TRACE;
+static int back_trace = IPIPE_DEFAULT_BACK_TRACE;
+static int verbose_trace = 1;
+static unsigned long trace_overhead;
+
+static unsigned long trigger_begin;
+static unsigned long trigger_end;
+
+static DEFINE_MUTEX(out_mutex);
+static struct ipipe_trace_path *print_path;
+#ifdef CONFIG_IPIPE_TRACE_PANIC
+static struct ipipe_trace_path *panic_path;
+#endif /* CONFIG_IPIPE_TRACE_PANIC */
+static int print_pre_trace;
+static int print_post_trace;
+
+
+static long __ipipe_signed_tsc2us(long long tsc);
+static void
+__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point);
+static void __ipipe_print_symname(struct seq_file *m, unsigned long eip);
+
+
+static notrace void
+__ipipe_store_domain_states(struct ipipe_trace_point *point)
+{
+	struct ipipe_domain *ipd;
+	struct list_head *pos;
+	int i = 0;
+
+	list_for_each_prev(pos, &__ipipe_pipeline) {
+		ipd = list_entry(pos, struct ipipe_domain, p_link);
+
+		if (test_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status)))
+			point->flags |= 1 << (i + IPIPE_TFLG_DOMSTATE_SHIFT);
+
+		if (ipd == ipipe_current_domain)
+			point->flags |= i << IPIPE_TFLG_CURRDOM_SHIFT;
+
+		if (++i > IPIPE_TFLG_DOMSTATE_BITS)
+			break;
+	}
+}
+
+static notrace int __ipipe_get_free_trace_path(int old, int cpu)
+{
+	int new_active = old;
+	struct ipipe_trace_path *tp;
+
+	do {
+		if (++new_active == IPIPE_TRACE_PATHS)
+			new_active = 0;
+		tp = &per_cpu(trace_path, cpu)[new_active];
+	} while (new_active == per_cpu(max_path, cpu) ||
+	         new_active == per_cpu(frozen_path, cpu) ||
+	         tp->dump_lock);
+
+	return new_active;
+}
+
+static notrace void
+__ipipe_migrate_pre_trace(struct ipipe_trace_path *new_tp,
+                          struct ipipe_trace_path *old_tp, int old_pos)
+{
+	int i;
+
+	new_tp->trace_pos = pre_trace+1;
+
+	for (i = new_tp->trace_pos; i > 0; i--)
+		memcpy(&new_tp->point[WRAP_POINT_NO(new_tp->trace_pos-i)],
+		       &old_tp->point[WRAP_POINT_NO(old_pos-i)],
+		       sizeof(struct ipipe_trace_point));
+
+	/* mark the end (i.e. the point before point[0]) invalid */
+	new_tp->point[IPIPE_TRACE_POINTS-1].eip = 0;
+}
+
+static notrace struct ipipe_trace_path *
+__ipipe_trace_end(int cpu, struct ipipe_trace_path *tp, int pos)
+{
+	struct ipipe_trace_path *old_tp = tp;
+	long active = per_cpu(active_path, cpu);
+	unsigned long long length;
+
+	/* do we have a new worst case? */
+	length = tp->point[tp->end].timestamp -
+	         tp->point[tp->begin].timestamp;
+	if (length > per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)].length) {
+		/* we need protection here against other cpus trying
+		   to start a proc dump */
+		spin_lock(&global_path_lock);
+
+		/* active path holds new worst case */
+		tp->length = length;
+		per_cpu(max_path, cpu) = active;
+
+		/* find next unused trace path */
+		active = __ipipe_get_free_trace_path(active, cpu);
+
+		spin_unlock(&global_path_lock);
+
+		tp = &per_cpu(trace_path, cpu)[active];
+
+		/* migrate last entries for pre-tracing */
+		__ipipe_migrate_pre_trace(tp, old_tp, pos);
+	}
+
+	return tp;
+}
+
+static notrace struct ipipe_trace_path *
+__ipipe_trace_freeze(int cpu, struct ipipe_trace_path *tp, int pos)
+{
+	struct ipipe_trace_path *old_tp = tp;
+	long active = per_cpu(active_path, cpu);
+	int n;
+
+	/* frozen paths have no core (begin=end) */
+	tp->begin = tp->end;
+
+	/* we need protection here against other cpus trying
+	 * to set their frozen path or to start a proc dump */
+	spin_lock(&global_path_lock);
+
+	per_cpu(frozen_path, cpu) = active;
+
+	/* find next unused trace path */
+	active = __ipipe_get_free_trace_path(active, cpu);
+
+	/* check if this is the first frozen path */
+	for_each_possible_cpu(n) {
+		if (n != cpu &&
+		    per_cpu(trace_path, n)[per_cpu(frozen_path, n)].end >= 0)
+			tp->end = -1;
+	}
+
+	spin_unlock(&global_path_lock);
+
+	tp = &per_cpu(trace_path, cpu)[active];
+
+	/* migrate last entries for pre-tracing */
+	__ipipe_migrate_pre_trace(tp, old_tp, pos);
+
+	return tp;
+}
+
+void notrace
+__ipipe_trace(enum ipipe_trace_type type, unsigned long eip,
+              unsigned long parent_eip, unsigned long v)
+{
+	struct ipipe_trace_path *tp, *old_tp;
+	int pos, next_pos, begin;
+	struct ipipe_trace_point *point;
+	unsigned long flags;
+	int cpu;
+
+	local_irq_save_hw_notrace(flags);
+
+	cpu = ipipe_processor_id();
+ restart:
+	tp = old_tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
+
+	/* here starts a race window with NMIs - catched below */
+
+	/* check for NMI recursion */
+	if (unlikely(tp->flags & IPIPE_TFLG_NMI_LOCK)) {
+		tp->flags |= IPIPE_TFLG_NMI_HIT;
+
+		/* first freeze request from NMI context? */
+		if ((type == IPIPE_TRACE_FREEZE) &&
+		    !(tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)) {
+			/* save arguments and mark deferred freezing */
+			tp->flags |= IPIPE_TFLG_NMI_FREEZE_REQ;
+			tp->nmi_saved_eip = eip;
+			tp->nmi_saved_parent_eip = parent_eip;
+			tp->nmi_saved_v = v;
+		}
+		return; /* no need for restoring flags inside IRQ */
+	}
+
+	/* clear NMI events and set lock (atomically per cpu) */
+	tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT |
+	                           IPIPE_TFLG_NMI_FREEZE_REQ))
+	                       | IPIPE_TFLG_NMI_LOCK;
+
+	/* check active_path again - some nasty NMI may have switched
+	 * it meanwhile */
+	if (unlikely(tp !=
+		     &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)])) {
+		/* release lock on wrong path and restart */
+		tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+		/* there is no chance that the NMI got deferred
+		 * => no need to check for pending freeze requests */
+		goto restart;
+	}
+
+	/* get the point buffer */
+	pos = tp->trace_pos;
+	point = &tp->point[pos];
+
+	/* store all trace point data */
+	point->type = type;
+	point->flags = raw_irqs_disabled_flags(flags) ? IPIPE_TFLG_HWIRQ_OFF : 0;
+	point->eip = eip;
+	point->parent_eip = parent_eip;
+	point->v = v;
+	ipipe_read_tsc(point->timestamp);
+
+	__ipipe_store_domain_states(point);
+
+	/* forward to next point buffer */
+	next_pos = WRAP_POINT_NO(pos+1);
+	tp->trace_pos = next_pos;
+
+	/* only mark beginning if we haven't started yet */
+	begin = tp->begin;
+	if (unlikely(type == IPIPE_TRACE_BEGIN) && (begin < 0))
+		tp->begin = pos;
+
+	/* end of critical path, start post-trace if not already started */
+	if (unlikely(type == IPIPE_TRACE_END) &&
+	    (begin >= 0) && !tp->post_trace)
+		tp->post_trace = post_trace + 1;
+
+	/* freeze only if the slot is free and we are not already freezing */
+	if ((unlikely(type == IPIPE_TRACE_FREEZE) ||
+	     (unlikely(eip >= trigger_begin && eip <= trigger_end) &&
+	     type == IPIPE_TRACE_FUNC)) &&
+	    per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)].begin < 0 &&
+	    !(tp->flags & IPIPE_TFLG_FREEZING)) {
+		tp->post_trace = post_trace + 1;
+		tp->flags |= IPIPE_TFLG_FREEZING;
+	}
+
+	/* enforce end of trace in case of overflow */
+	if (unlikely(WRAP_POINT_NO(next_pos + 1) == begin)) {
+		tp->end = pos;
+		goto enforce_end;
+	}
+
+	/* stop tracing this path if we are in post-trace and
+	 *  a) that phase is over now or
+	 *  b) a new TRACE_BEGIN came in but we are not freezing this path */
+	if (unlikely((tp->post_trace > 0) && ((--tp->post_trace == 0) ||
+	             ((type == IPIPE_TRACE_BEGIN) &&
+	              !(tp->flags & IPIPE_TFLG_FREEZING))))) {
+		/* store the path's end (i.e. excluding post-trace) */
+		tp->end = WRAP_POINT_NO(pos - post_trace + tp->post_trace);
+
+ enforce_end:
+		if (tp->flags & IPIPE_TFLG_FREEZING)
+			tp = __ipipe_trace_freeze(cpu, tp, pos);
+		else
+			tp = __ipipe_trace_end(cpu, tp, pos);
+
+		/* reset the active path, maybe already start a new one */
+		tp->begin = (type == IPIPE_TRACE_BEGIN) ?
+			WRAP_POINT_NO(tp->trace_pos - 1) : -1;
+		tp->end = -1;
+		tp->post_trace = 0;
+		tp->flags = 0;
+
+		/* update active_path not earlier to avoid races with NMIs */
+		per_cpu(active_path, cpu) = tp - per_cpu(trace_path, cpu);
+	}
+
+	/* we still have old_tp and point,
+	 * let's reset NMI lock and check for catches */
+	old_tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+	if (unlikely(old_tp->flags & IPIPE_TFLG_NMI_HIT)) {
+		/* well, this late tagging may not immediately be visible for
+		 * other cpus already dumping this path - a minor issue */
+		point->flags |= IPIPE_TFLG_NMI_HIT;
+
+		/* handle deferred freezing from NMI context */
+		if (old_tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)
+			__ipipe_trace(IPIPE_TRACE_FREEZE, old_tp->nmi_saved_eip,
+			              old_tp->nmi_saved_parent_eip,
+			              old_tp->nmi_saved_v);
+	}
+
+	local_irq_restore_hw_notrace(flags);
+}
+
+static unsigned long __ipipe_global_path_lock(void)
+{
+	unsigned long flags;
+	int cpu;
+	struct ipipe_trace_path *tp;
+
+	spin_lock_irqsave(&global_path_lock, flags);
+
+	cpu = ipipe_processor_id();
+ restart:
+	tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
+
+	/* here is small race window with NMIs - catched below */
+
+	/* clear NMI events and set lock (atomically per cpu) */
+	tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT |
+	                           IPIPE_TFLG_NMI_FREEZE_REQ))
+	                       | IPIPE_TFLG_NMI_LOCK;
+
+	/* check active_path again - some nasty NMI may have switched
+	 * it meanwhile */
+	if (tp != &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)]) {
+		/* release lock on wrong path and restart */
+		tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+		/* there is no chance that the NMI got deferred
+		 * => no need to check for pending freeze requests */
+		goto restart;
+	}
+
+	return flags;
+}
+
+static void __ipipe_global_path_unlock(unsigned long flags)
+{
+	int cpu;
+	struct ipipe_trace_path *tp;
+
+	/* release spinlock first - it's not involved in the NMI issue */
+	__ipipe_spin_unlock_irqbegin(&global_path_lock);
+
+	cpu = ipipe_processor_id();
+	tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
+
+	tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+	/* handle deferred freezing from NMI context */
+	if (tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)
+		__ipipe_trace(IPIPE_TRACE_FREEZE, tp->nmi_saved_eip,
+		              tp->nmi_saved_parent_eip, tp->nmi_saved_v);
+
+	/* See __ipipe_spin_lock_irqsave() and friends. */
+	__ipipe_spin_unlock_irqcomplete(flags);
+}
+
+void notrace ipipe_trace_begin(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_BEGIN, __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_begin);
+
+void notrace ipipe_trace_end(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_END, __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_end);
+
+void notrace ipipe_trace_freeze(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_FREEZE, __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_freeze);
+
+void notrace ipipe_trace_special(unsigned char id, unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_SPECIAL | (id << IPIPE_TYPE_BITS),
+	              __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_special);
+
+void notrace ipipe_trace_pid(pid_t pid, short prio)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_PID | (prio << IPIPE_TYPE_BITS),
+	              __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, pid);
+}
+EXPORT_SYMBOL(ipipe_trace_pid);
+
+int ipipe_trace_max_reset(void)
+{
+	int cpu;
+	unsigned long flags;
+	struct ipipe_trace_path *path;
+	int ret = 0;
+
+	flags = __ipipe_global_path_lock();
+
+	for_each_possible_cpu(cpu) {
+		path = &per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)];
+
+		if (path->dump_lock) {
+			ret = -EBUSY;
+			break;
+		}
+
+		path->begin     = -1;
+		path->end       = -1;
+		path->trace_pos = 0;
+		path->length    = 0;
+	}
+
+	__ipipe_global_path_unlock(flags);
+
+	return ret;
+}
+EXPORT_SYMBOL(ipipe_trace_max_reset);
+
+int ipipe_trace_frozen_reset(void)
+{
+	int cpu;
+	unsigned long flags;
+	struct ipipe_trace_path *path;
+	int ret = 0;
+
+	flags = __ipipe_global_path_lock();
+
+	for_each_online_cpu(cpu) {
+		path = &per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)];
+
+		if (path->dump_lock) {
+			ret = -EBUSY;
+			break;
+		}
+
+		path->begin = -1;
+		path->end = -1;
+		path->trace_pos = 0;
+		path->length    = 0;
+	}
+
+	__ipipe_global_path_unlock(flags);
+
+	return ret;
+}
+EXPORT_SYMBOL(ipipe_trace_frozen_reset);
+
+static void
+__ipipe_get_task_info(char *task_info, struct ipipe_trace_point *point,
+                      int trylock)
+{
+	struct task_struct *task = NULL;
+	char buf[8];
+	int i;
+	int locked = 1;
+
+	if (trylock) {
+		if (!read_trylock(&tasklist_lock))
+			locked = 0;
+	} else
+		read_lock(&tasklist_lock);
+
+	if (locked)
+		task = find_task_by_pid((pid_t)point->v);
+
+	if (task)
+		strncpy(task_info, task->comm, 11);
+	else
+		strcpy(task_info, "-<?>-");
+
+	if (locked)
+		read_unlock(&tasklist_lock);
+
+	for (i = strlen(task_info); i < 11; i++)
+		task_info[i] = ' ';
+
+	sprintf(buf, " %d ", point->type >> IPIPE_TYPE_BITS);
+	strcpy(task_info + (11 - strlen(buf)), buf);
+}
+
+#ifdef CONFIG_IPIPE_TRACE_PANIC
+void ipipe_trace_panic_freeze(void)
+{
+	unsigned long flags;
+	int cpu;
+
+	if (!ipipe_trace_enable)
+		return;
+
+	ipipe_trace_enable = 0;
+	local_irq_save_hw_notrace(flags);
+
+	cpu = ipipe_processor_id();
+
+	panic_path = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
+
+	local_irq_restore_hw(flags);
+}
+EXPORT_SYMBOL(ipipe_trace_panic_freeze);
+
+void ipipe_trace_panic_dump(void)
+{
+	int cnt = back_trace;
+	int start, pos;
+	char task_info[12];
+
+	if (!panic_path)
+		return;
+
+	ipipe_context_check_off();
+
+	printk("I-pipe tracer log (%d points):\n", cnt);
+
+	start = pos = WRAP_POINT_NO(panic_path->trace_pos-1);
+
+	while (cnt-- > 0) {
+		struct ipipe_trace_point *point = &panic_path->point[pos];
+		long time;
+		char buf[16];
+		int i;
+
+		printk(" %c",
+		       (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' ');
+
+		for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--)
+			printk("%c",
+			       (IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ?
+				(IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
+					'#' : '+') :
+				(IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
+					'*' : ' '));
+
+		if (!point->eip)
+			printk("-<invalid>-\n");
+		else {
+			__ipipe_trace_point_type(buf, point);
+			printk(buf);
+
+			switch (point->type & IPIPE_TYPE_MASK) {
+				case IPIPE_TRACE_FUNC:
+					printk("           ");
+					break;
+
+				case IPIPE_TRACE_PID:
+					__ipipe_get_task_info(task_info,
+							      point, 1);
+					printk(task_info);
+					break;
+
+				default:
+					printk("0x%08lx ", point->v);
+			}
+
+			time = __ipipe_signed_tsc2us(point->timestamp -
+				panic_path->point[start].timestamp);
+			printk(" %5ld ", time);
+
+			__ipipe_print_symname(NULL, point->eip);
+			printk(" (");
+			__ipipe_print_symname(NULL, point->parent_eip);
+			printk(")\n");
+		}
+		pos = WRAP_POINT_NO(pos - 1);
+	}
+
+	panic_path = NULL;
+}
+EXPORT_SYMBOL(ipipe_trace_panic_dump);
+#endif /* CONFIG_IPIPE_TRACE_PANIC */
+
+
+/* --- /proc output --- */
+
+static notrace int __ipipe_in_critical_trpath(long point_no)
+{
+	return ((WRAP_POINT_NO(point_no-print_path->begin) <
+	         WRAP_POINT_NO(print_path->end-print_path->begin)) ||
+	        ((print_path->end == print_path->begin) &&
+	         (WRAP_POINT_NO(point_no-print_path->end) >
+	          print_post_trace)));
+}
+
+static long __ipipe_signed_tsc2us(long long tsc)
+{
+        unsigned long long abs_tsc;
+        long us;
+
+	/* ipipe_tsc2us works on unsigned => handle sign separately */
+        abs_tsc = (tsc >= 0) ? tsc : -tsc;
+        us = ipipe_tsc2us(abs_tsc);
+        if (tsc < 0)
+                return -us;
+        else
+                return us;
+}
+
+static void
+__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point)
+{
+	switch (point->type & IPIPE_TYPE_MASK) {
+		case IPIPE_TRACE_FUNC:
+			strcpy(buf, "func    ");
+			break;
+
+		case IPIPE_TRACE_BEGIN:
+			strcpy(buf, "begin   ");
+			break;
+
+		case IPIPE_TRACE_END:
+			strcpy(buf, "end     ");
+			break;
+
+		case IPIPE_TRACE_FREEZE:
+			strcpy(buf, "freeze  ");
+			break;
+
+		case IPIPE_TRACE_SPECIAL:
+			sprintf(buf, "(0x%02x)  ",
+				point->type >> IPIPE_TYPE_BITS);
+			break;
+
+		case IPIPE_TRACE_PID:
+			sprintf(buf, "[%5d] ", (pid_t)point->v);
+			break;
+	}
+}
+
+static void
+__ipipe_print_pathmark(struct seq_file *m, struct ipipe_trace_point *point)
+{
+	char mark = ' ';
+	int point_no = point - print_path->point;
+	int i;
+
+	if (print_path->end == point_no)
+		mark = '<';
+	else if (print_path->begin == point_no)
+		mark = '>';
+	else if (__ipipe_in_critical_trpath(point_no))
+		mark = ':';
+	seq_printf(m, "%c%c", mark,
+	           (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' ');
+
+	if (!verbose_trace)
+		return;
+
+	for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--)
+		seq_printf(m, "%c",
+			(IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ?
+			    (IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
+				'#' : '+') :
+			(IPIPE_TFLG_DOMAIN_STALLED(point, i) ? '*' : ' '));
+}
+
+static void
+__ipipe_print_delay(struct seq_file *m, struct ipipe_trace_point *point)
+{
+	unsigned long delay = 0;
+	int next;
+	char *mark = "  ";
+
+	next = WRAP_POINT_NO(point+1 - print_path->point);
+
+	if (next != print_path->trace_pos)
+		delay = ipipe_tsc2ns(print_path->point[next].timestamp -
+		                     point->timestamp);
+
+	if (__ipipe_in_critical_trpath(point - print_path->point)) {
+		if (delay > IPIPE_DELAY_WARN)
+			mark = "! ";
+		else if (delay > IPIPE_DELAY_NOTE)
+			mark = "+ ";
+	}
+	seq_puts(m, mark);
+
+	if (verbose_trace)
+		seq_printf(m, "%3lu.%03lu%c ", delay/1000, delay%1000,
+		           (point->flags & IPIPE_TFLG_NMI_HIT) ? 'N' : ' ');
+	else
+		seq_puts(m, " ");
+}
+
+static void __ipipe_print_symname(struct seq_file *m, unsigned long eip)
+{
+	char namebuf[KSYM_NAME_LEN+1];
+	unsigned long size, offset;
+	const char *sym_name;
+	char *modname;
+
+	sym_name = kallsyms_lookup(eip, &size, &offset, &modname, namebuf);
+
+#ifdef CONFIG_IPIPE_TRACE_PANIC
+	if (!m) {
+		/* panic dump */
+		if (sym_name) {
+			printk("%s+0x%lx", sym_name, offset);
+			if (modname)
+				printk(" [%s]", modname);
+		}
+	} else
+#endif /* CONFIG_IPIPE_TRACE_PANIC */
+	{
+		if (sym_name) {
+			if (verbose_trace) {
+				seq_printf(m, "%s+0x%lx", sym_name, offset);
+				if (modname)
+					seq_printf(m, " [%s]", modname);
+			} else
+				seq_puts(m, sym_name);
+		} else
+			seq_printf(m, "<%08lx>", eip);
+	}
+}
+
+static void __ipipe_print_headline(struct seq_file *m)
+{
+	seq_printf(m, "Calibrated minimum trace-point overhead: %lu.%03lu "
+		   "us\n\n", trace_overhead/1000, trace_overhead%1000);
+
+	if (verbose_trace) {
+		const char *name[4] = { [0 ... 3] = "<unused>" };
+		struct list_head *pos;
+		int i = 0;
+
+		list_for_each_prev(pos, &__ipipe_pipeline) {
+			struct ipipe_domain *ipd =
+				list_entry(pos, struct ipipe_domain, p_link);
+
+			name[i] = ipd->name;
+			if (++i > 3)
+				break;
+		}
+
+		seq_printf(m,
+		           " +----- Hard IRQs ('|': locked)\n"
+		           " |+---- %s\n"
+		           " ||+--- %s\n"
+		           " |||+-- %s\n"
+		           " ||||+- %s%s\n"
+		           " |||||                        +---------- "
+		               "Delay flag ('+': > %d us, '!': > %d us)\n"
+		           " |||||                        |        +- "
+		               "NMI noise ('N')\n"
+		           " |||||                        |        |\n"
+		           "      Type    User Val.   Time    Delay  Function "
+		               "(Parent)\n",
+		           name[3], name[2], name[1], name[0],
+		           name[0] ? " ('*': domain stalled, '+': current, "
+		               "'#': current+stalled)" : "",
+		           IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000);
+	} else
+		seq_printf(m,
+		           " +--------------- Hard IRQs ('|': locked)\n"
+		           " |             +- Delay flag "
+		               "('+': > %d us, '!': > %d us)\n"
+		           " |             |\n"
+		           "  Type     Time   Function (Parent)\n",
+		           IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000);
+}
+
+static void *__ipipe_max_prtrace_start(struct seq_file *m, loff_t *pos)
+{
+	loff_t n = *pos;
+
+	mutex_lock(&out_mutex);
+
+	if (!n) {
+		struct ipipe_trace_path *tp;
+		unsigned long length_usecs;
+		int points, cpu;
+		unsigned long flags;
+
+		/* protect against max_path/frozen_path updates while we
+		 * haven't locked our target path, also avoid recursively
+		 * taking global_path_lock from NMI context */
+		flags = __ipipe_global_path_lock();
+
+		/* find the longest of all per-cpu paths */
+		print_path = NULL;
+		for_each_online_cpu(cpu) {
+			tp = &per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)];
+			if ((print_path == NULL) ||
+			    (tp->length > print_path->length)) {
+				print_path = tp;
+				break;
+			}
+		}
+		print_path->dump_lock = 1;
+
+		__ipipe_global_path_unlock(flags);
+
+		/* does this path actually contain data? */
+		if (print_path->end == print_path->begin)
+			return NULL;
+
+		/* number of points inside the critical path */
+		points = WRAP_POINT_NO(print_path->end-print_path->begin+1);
+
+		/* pre- and post-tracing length, post-trace length was frozen
+		   in __ipipe_trace, pre-trace may have to be reduced due to
+		   buffer overrun */
+		print_pre_trace  = pre_trace;
+		print_post_trace = WRAP_POINT_NO(print_path->trace_pos -
+		                                 print_path->end - 1);
+		if (points+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1)
+			print_pre_trace = IPIPE_TRACE_POINTS - 1 - points -
+				print_post_trace;
+
+		length_usecs = ipipe_tsc2us(print_path->length);
+		seq_printf(m, "I-pipe worst-case tracing service on %s/ipipe-%s\n"
+			"------------------------------------------------------------\n",
+			UTS_RELEASE, IPIPE_ARCH_STRING);
+		seq_printf(m, "CPU: %d, Begin: %lld cycles, Trace Points: "
+			"%d (-%d/+%d), Length: %lu us\n",
+			cpu, print_path->point[print_path->begin].timestamp,
+			points, print_pre_trace, print_post_trace, length_usecs);
+		__ipipe_print_headline(m);
+	}
+
+	/* check if we are inside the trace range */
+	if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 +
+	                       print_pre_trace + print_post_trace))
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin -
+	                                        print_pre_trace + n)];
+}
+
+static void *__ipipe_prtrace_next(struct seq_file *m, void *p, loff_t *pos)
+{
+	loff_t n = ++*pos;
+
+	/* check if we are inside the trace range with the next entry */
+	if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 +
+	                       print_pre_trace + print_post_trace))
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin -
+	                                        print_pre_trace + *pos)];
+}
+
+static void __ipipe_prtrace_stop(struct seq_file *m, void *p)
+{
+	if (print_path)
+		print_path->dump_lock = 0;
+	mutex_unlock(&out_mutex);
+}
+
+static int __ipipe_prtrace_show(struct seq_file *m, void *p)
+{
+	long time;
+	struct ipipe_trace_point *point = p;
+	char buf[16];
+
+	if (!point->eip) {
+		seq_puts(m, "-<invalid>-\n");
+		return 0;
+	}
+
+	__ipipe_print_pathmark(m, point);
+	__ipipe_trace_point_type(buf, point);
+	seq_puts(m, buf);
+	if (verbose_trace)
+		switch (point->type & IPIPE_TYPE_MASK) {
+			case IPIPE_TRACE_FUNC:
+				seq_puts(m, "           ");
+				break;
+
+			case IPIPE_TRACE_PID:
+				__ipipe_get_task_info(buf, point, 0);
+				seq_puts(m, buf);
+				break;
+
+			default:
+				seq_printf(m, "0x%08lx ", point->v);
+		}
+
+	time = __ipipe_signed_tsc2us(point->timestamp -
+		print_path->point[print_path->begin].timestamp);
+	seq_printf(m, "%5ld", time);
+
+	__ipipe_print_delay(m, point);
+	__ipipe_print_symname(m, point->eip);
+	seq_puts(m, " (");
+	__ipipe_print_symname(m, point->parent_eip);
+	seq_puts(m, ")\n");
+
+	return 0;
+}
+
+static struct seq_operations __ipipe_max_ptrace_ops = {
+	.start = __ipipe_max_prtrace_start,
+	.next  = __ipipe_prtrace_next,
+	.stop  = __ipipe_prtrace_stop,
+	.show  = __ipipe_prtrace_show
+};
+
+static int __ipipe_max_prtrace_open(struct inode *inode, struct file *file)
+{
+	return seq_open(file, &__ipipe_max_ptrace_ops);
+}
+
+static ssize_t
+__ipipe_max_reset(struct file *file, const char __user *pbuffer,
+                  size_t count, loff_t *data)
+{
+	mutex_lock(&out_mutex);
+	ipipe_trace_max_reset();
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+struct file_operations __ipipe_max_prtrace_fops = {
+	.open       = __ipipe_max_prtrace_open,
+	.read       = seq_read,
+	.write      = __ipipe_max_reset,
+	.llseek     = seq_lseek,
+	.release    = seq_release,
+};
+
+static void *__ipipe_frozen_prtrace_start(struct seq_file *m, loff_t *pos)
+{
+	loff_t n = *pos;
+
+	mutex_lock(&out_mutex);
+
+	if (!n) {
+		struct ipipe_trace_path *tp;
+		int cpu;
+		unsigned long flags;
+
+		/* protect against max_path/frozen_path updates while we
+		 * haven't locked our target path, also avoid recursively
+		 * taking global_path_lock from NMI context */
+		flags = __ipipe_global_path_lock();
+
+		/* find the first of all per-cpu frozen paths */
+		print_path = NULL;
+		for_each_online_cpu(cpu) {
+			tp = &per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)];
+			if (tp->end >= 0) {
+				print_path = tp;
+				break;
+			}
+		}
+		if (print_path)
+			print_path->dump_lock = 1;
+
+		__ipipe_global_path_unlock(flags);
+
+		if (!print_path)
+			return NULL;
+
+		/* back- and post-tracing length, post-trace length was frozen
+		   in __ipipe_trace, back-trace may have to be reduced due to
+		   buffer overrun */
+		print_pre_trace  = back_trace-1; /* substract freeze point */
+		print_post_trace = WRAP_POINT_NO(print_path->trace_pos -
+		                                 print_path->end - 1);
+		if (1+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1)
+			print_pre_trace = IPIPE_TRACE_POINTS - 2 -
+				print_post_trace;
+
+		seq_printf(m, "I-pipe frozen back-tracing service on %s/ipipe-%s\n"
+			"------------------------------------------------------"
+			"------\n",
+			UTS_RELEASE, IPIPE_ARCH_STRING);
+		seq_printf(m, "CPU: %d, Freeze: %lld cycles, Trace Points: %d (+%d)\n",
+			cpu, print_path->point[print_path->begin].timestamp,
+			print_pre_trace+1, print_post_trace);
+		__ipipe_print_headline(m);
+	}
+
+	/* check if we are inside the trace range */
+	if (n >= print_pre_trace + 1 + print_post_trace)
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin-
+	                                        print_pre_trace+n)];
+}
+
+static struct seq_operations __ipipe_frozen_ptrace_ops = {
+	.start = __ipipe_frozen_prtrace_start,
+	.next  = __ipipe_prtrace_next,
+	.stop  = __ipipe_prtrace_stop,
+	.show  = __ipipe_prtrace_show
+};
+
+static int __ipipe_frozen_prtrace_open(struct inode *inode, struct file *file)
+{
+	return seq_open(file, &__ipipe_frozen_ptrace_ops);
+}
+
+static ssize_t
+__ipipe_frozen_ctrl(struct file *file, const char __user *pbuffer,
+                    size_t count, loff_t *data)
+{
+	char *end, buf[16];
+	int val;
+	int n;
+
+	n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+	if (copy_from_user(buf, pbuffer, n))
+		return -EFAULT;
+
+	buf[n] = '\0';
+	val = simple_strtol(buf, &end, 0);
+
+	if (((*end != '\0') && !isspace(*end)) || (val < 0))
+		return -EINVAL;
+
+	mutex_lock(&out_mutex);
+	ipipe_trace_frozen_reset();
+	if (val > 0)
+		ipipe_trace_freeze(-1);
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+struct file_operations __ipipe_frozen_prtrace_fops = {
+	.open       = __ipipe_frozen_prtrace_open,
+	.read       = seq_read,
+	.write      = __ipipe_frozen_ctrl,
+	.llseek     = seq_lseek,
+	.release    = seq_release,
+};
+
+static int __ipipe_rd_proc_val(char *page, char **start, off_t off,
+                               int count, int *eof, void *data)
+{
+	int len;
+
+	len = sprintf(page, "%u\n", *(int *)data);
+	len -= off;
+	if (len <= off + count)
+		*eof = 1;
+	*start = page + off;
+	if (len > count)
+		len = count;
+	if (len < 0)
+		len = 0;
+
+	return len;
+}
+
+static int __ipipe_wr_proc_val(struct file *file, const char __user *buffer,
+                               unsigned long count, void *data)
+{
+	char *end, buf[16];
+	int val;
+	int n;
+
+	n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+	if (copy_from_user(buf, buffer, n))
+		return -EFAULT;
+
+	buf[n] = '\0';
+	val = simple_strtol(buf, &end, 0);
+
+	if (((*end != '\0') && !isspace(*end)) || (val < 0))
+		return -EINVAL;
+
+	mutex_lock(&out_mutex);
+	*(int *)data = val;
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+static int __ipipe_rd_trigger(char *page, char **start, off_t off, int count,
+			      int *eof, void *data)
+{
+	int len;
+
+	if (!trigger_begin)
+		return 0;
+
+	len = sprint_symbol(page, trigger_begin);
+	page[len++] = '\n';
+
+	len -= off;
+	if (len <= off + count)
+		*eof = 1;
+	*start = page + off;
+	if (len > count)
+		len = count;
+	if (len < 0)
+		len = 0;
+
+	return len;
+}
+
+static int __ipipe_wr_trigger(struct file *file, const char __user *buffer,
+			      unsigned long count, void *data)
+{
+	char buf[KSYM_SYMBOL_LEN];
+	unsigned long begin, end;
+
+	if (count > sizeof(buf) - 1)
+		count = sizeof(buf) - 1;
+	if (copy_from_user(buf, buffer, count))
+		return -EFAULT;
+	buf[count] = 0;
+	if (buf[count-1] == '\n')
+		buf[count-1] = 0;
+
+	begin = kallsyms_lookup_name(buf);
+	if (!begin || !kallsyms_lookup_size_offset(begin, &end, NULL))
+		return -ENOENT;
+	end += begin - 1;
+
+	mutex_lock(&out_mutex);
+	/* invalidate the current range before setting a new one */
+	trigger_end = 0;
+	wmb();
+	ipipe_trace_frozen_reset();
+
+	/* set new range */
+	trigger_begin = begin;
+	wmb();
+	trigger_end = end;
+	mutex_unlock(&out_mutex);
+
+	return count;
+}
+
+extern struct proc_dir_entry *ipipe_proc_root;
+
+static void __init
+__ipipe_create_trace_proc_val(struct proc_dir_entry *trace_dir,
+                              const char *name, int *value_ptr)
+{
+	struct proc_dir_entry *entry;
+
+	entry = create_proc_entry(name, 0644, trace_dir);
+	if (entry) {
+		entry->data = value_ptr;
+		entry->read_proc = __ipipe_rd_proc_val;
+		entry->write_proc = __ipipe_wr_proc_val;
+		entry->owner = THIS_MODULE;
+	}
+}
+
+void __init __ipipe_init_tracer(void)
+{
+	struct proc_dir_entry *trace_dir;
+	struct proc_dir_entry *entry;
+	unsigned long long start, end, min = ULLONG_MAX;
+	int i;
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
+	int cpu, path;
+
+	for_each_possible_cpu(cpu) {
+		struct ipipe_trace_path *tp_buf;
+
+		tp_buf = vmalloc_node(sizeof(struct ipipe_trace_path) *
+				      IPIPE_TRACE_PATHS, cpu_to_node(cpu));
+		if (!tp_buf) {
+			printk(KERN_ERR "I-pipe: "
+			       "insufficient memory for trace buffer.\n");
+			return;
+		}
+		memset(tp_buf, 0,
+		       sizeof(struct ipipe_trace_path) * IPIPE_TRACE_PATHS);
+		for (path = 0; path < IPIPE_TRACE_PATHS; path++) {
+			tp_buf[path].begin = -1;
+			tp_buf[path].end   = -1;
+		}
+		per_cpu(trace_path, cpu) = tp_buf;
+	}
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
+	ipipe_trace_enable = CONFIG_IPIPE_TRACE_ENABLE_VALUE;
+
+	/* Calculate minimum overhead of __ipipe_trace() */
+	local_irq_disable_hw();
+	for (i = 0; i < 100; i++) {
+		ipipe_read_tsc(start);
+		__ipipe_trace(IPIPE_TRACE_FUNC, __BUILTIN_RETURN_ADDRESS0,
+			      __BUILTIN_RETURN_ADDRESS1, 0);
+		ipipe_read_tsc(end);
+
+		end -= start;
+		if (end < min)
+			min = end;
+	}
+	local_irq_enable_hw();
+	trace_overhead = ipipe_tsc2ns(min);
+
+	trace_dir = create_proc_entry("trace", S_IFDIR, ipipe_proc_root);
+
+	entry = create_proc_entry("max", 0644, trace_dir);
+	if (entry)
+		entry->proc_fops = &__ipipe_max_prtrace_fops;
+
+	entry = create_proc_entry("frozen", 0644, trace_dir);
+	if (entry)
+		entry->proc_fops = &__ipipe_frozen_prtrace_fops;
+
+	entry = create_proc_entry("trigger", 0644, trace_dir);
+	if (entry) {
+		entry->read_proc = __ipipe_rd_trigger;
+		entry->write_proc = __ipipe_wr_trigger;
+		entry->owner = THIS_MODULE;
+	}
+
+	__ipipe_create_trace_proc_val(trace_dir, "pre_trace_points",
+	                              &pre_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "post_trace_points",
+	                              &post_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "back_trace_points",
+	                              &back_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "verbose",
+	                              &verbose_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "enable",
+	                              &ipipe_trace_enable);
+}
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index 964964b..47c9722 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -353,7 +353,9 @@ handle_level_irq(unsigned int irq, struct irq_desc *desc)
 	irqreturn_t action_ret;
 
 	spin_lock(&desc->lock);
+#ifndef CONFIG_IPIPE
 	mask_ack_irq(desc, irq);
+#endif /* CONFIG_IPIPE */
 
 	if (unlikely(desc->status & IRQ_INPROGRESS))
 		goto out_unlock;
@@ -430,8 +432,13 @@ handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc)
 
 	spin_lock(&desc->lock);
 	desc->status &= ~IRQ_INPROGRESS;
+#ifdef CONFIG_IPIPE
+	desc->chip->unmask(irq);
+out:
+#else
 out:
 	desc->chip->eoi(irq);
+#endif
 
 	spin_unlock(&desc->lock);
 }
@@ -475,8 +482,10 @@ handle_edge_irq(unsigned int irq, struct irq_desc *desc)
 
 	kstat_cpu(cpu).irqs[irq]++;
 
+#ifndef CONFIG_IPIPE
 	/* Start handling the irq */
 	desc->chip->ack(irq);
+#endif /* CONFIG_IPIPE */
 
 	/* Mark the IRQ currently in progress.*/
 	desc->status |= IRQ_INPROGRESS;
@@ -516,6 +525,85 @@ out_unlock:
 	spin_unlock(&desc->lock);
 }
 
+#ifdef CONFIG_IPIPE
+
+void __ipipe_ack_simple_irq(unsigned irq, struct irq_desc *desc)
+{
+}
+
+void __ipipe_end_simple_irq(unsigned irq, struct irq_desc *desc)
+{
+}
+
+void __ipipe_ack_level_irq(unsigned irq, struct irq_desc *desc)
+{
+	mask_ack_irq(desc, irq);
+}
+
+void __ipipe_end_level_irq(unsigned irq, struct irq_desc *desc)
+{
+	if (desc->chip->unmask)
+		desc->chip->unmask(irq);
+}
+
+void __ipipe_ack_fasteoi_irq(unsigned irq, struct irq_desc *desc)
+{
+	desc->chip->eoi(irq);
+}
+
+void __ipipe_end_fasteoi_irq(unsigned irq, struct irq_desc *desc)
+{
+	/*
+	 * Non-requestable IRQs should not be masked in EOI handler.
+	 */
+	if (!(desc->status & IRQ_NOREQUEST))
+		desc->chip->unmask(irq);
+}
+
+void __ipipe_ack_edge_irq(unsigned irq, struct irq_desc *desc)
+{
+	desc->chip->ack(irq);
+}
+
+void __ipipe_ack_percpu_irq(unsigned irq, struct irq_desc *desc)
+{
+	if (desc->chip->ack)
+		desc->chip->ack(irq);
+}
+
+void __ipipe_end_percpu_irq(unsigned irq, struct irq_desc *desc)
+{
+	if (desc->chip->eoi)
+		desc->chip->eoi(irq);
+}
+
+void __ipipe_end_edge_irq(unsigned irq, struct irq_desc *desc)
+{
+}
+
+void __ipipe_ack_bad_irq(unsigned irq, struct irq_desc *desc)
+{
+	static int done;
+
+	handle_bad_irq(irq, desc);
+
+	if (!done) {
+		printk(KERN_WARNING "%s: unknown flow handler for IRQ %d\n",
+		       __FUNCTION__, irq);
+		done = 1;
+	}
+}
+
+void __ipipe_noack_irq(unsigned irq, struct irq_desc *desc)
+{
+}
+
+void __ipipe_noend_irq(unsigned irq, struct irq_desc *desc)
+{
+}
+
+#endif /* CONFIG_IPIPE */
+
 /**
  *	handle_percpu_IRQ - Per CPU local irq handler
  *	@irq:	the interrupt number
@@ -530,8 +618,10 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc)
 
 	kstat_this_cpu.irqs[irq]++;
 
+#ifndef CONFIG_IPIPE
 	if (desc->chip->ack)
 		desc->chip->ack(irq);
+#endif /* CONFIG_IPIPE */
 
 	action_ret = handle_IRQ_event(irq, desc->action);
 	if (!noirqdebug)
@@ -558,6 +648,30 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
 
 	if (!handle)
 		handle = handle_bad_irq;
+#ifdef CONFIG_IPIPE
+	else if (handle == &handle_simple_irq) {
+		desc->ipipe_ack = &__ipipe_ack_simple_irq;
+		desc->ipipe_end = &__ipipe_end_simple_irq;
+	}
+	else if (handle == &handle_level_irq) {
+		desc->ipipe_ack = &__ipipe_ack_level_irq;
+		desc->ipipe_end = &__ipipe_end_level_irq;
+	}
+	else if (handle == &handle_edge_irq) {
+		desc->ipipe_ack = &__ipipe_ack_edge_irq;
+		desc->ipipe_end = &__ipipe_end_edge_irq;
+	}
+	else if (handle == &handle_fasteoi_irq) {
+		desc->ipipe_ack = &__ipipe_ack_fasteoi_irq;
+		desc->ipipe_end = &__ipipe_end_fasteoi_irq;
+	}
+#ifdef CONFIG_SMP
+	else if (handle == &handle_percpu_irq) {
+		desc->ipipe_ack = &__ipipe_ack_percpu_irq;
+		desc->ipipe_end = &__ipipe_end_percpu_irq;
+	}
+#endif /* CONFIG_SMP */
+#endif /* CONFIG_IPIPE */
 	else if (desc->chip == &no_irq_chip) {
 		printk(KERN_WARNING "Trying to install %sinterrupt handler "
 		       "for IRQ%d\n", is_chained ? "chained " : "", irq);
@@ -569,7 +683,21 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
 		 * dummy_irq_chip for easy transition.
 		 */
 		desc->chip = &dummy_irq_chip;
+#ifdef CONFIG_IPIPE
+		desc->ipipe_ack = &__ipipe_noack_irq;
+		desc->ipipe_end = &__ipipe_noend_irq;
+#endif /* CONFIG_IPIPE */
+	}
+#ifdef CONFIG_IPIPE
+ 	else if (is_chained) {
+ 		desc->ipipe_ack = handle;
+ 		desc->ipipe_end = &__ipipe_noend_irq;
+		handle = &__ipipe_noack_irq;
+ 	} else {
+ 		desc->ipipe_ack = &__ipipe_ack_bad_irq;
+ 		desc->ipipe_end = &__ipipe_noend_irq;
 	}
+#endif /* CONFIG_IPIPE */
 
 	spin_lock_irqsave(&desc->lock, flags);
 
@@ -579,6 +707,10 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
 			mask_ack_irq(desc, irq);
 		desc->status |= IRQ_DISABLED;
 		desc->depth = 1;
+#ifdef CONFIG_IPIPE
+		desc->ipipe_ack = &__ipipe_ack_bad_irq;
+		desc->ipipe_end = &__ipipe_noend_irq;
+#endif /* CONFIG_IPIPE */
 	}
 	desc->handle_irq = handle;
 	desc->name = name;
diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c
index 5fa6198..ed21856 100644
--- a/kernel/irq/handle.c
+++ b/kernel/irq/handle.c
@@ -176,8 +176,10 @@ unsigned int __do_IRQ(unsigned int irq)
 		/*
 		 * No locking required for CPU-local interrupts:
 		 */
+#ifndef CONFIG_IPIPE
 		if (desc->chip->ack)
 			desc->chip->ack(irq);
+#endif
 		if (likely(!(desc->status & IRQ_DISABLED))) {
 			action_ret = handle_IRQ_event(irq, desc->action);
 			if (!noirqdebug)
@@ -188,8 +190,10 @@ unsigned int __do_IRQ(unsigned int irq)
 	}
 
 	spin_lock(&desc->lock);
+#ifndef CONFIG_IPIPE
 	if (desc->chip->ack)
 		desc->chip->ack(irq);
+#endif
 	/*
 	 * REPLAY is when Linux resends an IRQ that was dropped earlier
 	 * WAITING is used by probe to mark irqs that are being tested
diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index 81a4e4a..da942d6 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -2032,7 +2032,7 @@ void trace_hardirqs_on(void)
 	curr->hardirqs_enabled = 1;
 	ip = (unsigned long) __builtin_return_address(0);
 
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !irqs_disabled_hw()))
 		return;
 	if (DEBUG_LOCKS_WARN_ON(current->hardirq_context))
 		return;
@@ -2068,7 +2068,7 @@ void trace_hardirqs_off(void)
 	if (unlikely(!debug_locks || current->lockdep_recursion))
 		return;
 
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !irqs_disabled_hw()))
 		return;
 
 	if (curr->hardirqs_enabled) {
diff --git a/kernel/power/disk.c b/kernel/power/disk.c
index 14a656c..4cd1475 100644
--- a/kernel/power/disk.c
+++ b/kernel/power/disk.c
@@ -194,6 +194,7 @@ static int create_image(int platform_mode)
 		return error;
 
 	local_irq_disable();
+ 	local_irq_disable_hw_cond();
 	/* At this point, device_suspend() has been called, but *not*
 	 * device_power_down(). We *must* call device_power_down() now.
 	 * Otherwise, drivers for some devices (e.g. interrupt controllers)
@@ -226,6 +227,7 @@ static int create_image(int platform_mode)
 	 */
 	device_power_up();
  Enable_irqs:
+ 	local_irq_enable_hw_cond();
 	local_irq_enable();
 	return error;
 }
@@ -301,6 +303,7 @@ static int resume_target_kernel(void)
 	int error;
 
 	local_irq_disable();
+	local_irq_disable_hw_cond();
 	error = device_power_down(PMSG_PRETHAW);
 	if (error) {
 		printk(KERN_ERR "PM: Some devices failed to power down, "
@@ -331,6 +334,7 @@ static int resume_target_kernel(void)
 	touch_softlockup_watchdog();
 	device_power_up();
  Enable_irqs:
+	local_irq_enable_hw_cond();
 	local_irq_enable();
 	return error;
 }
diff --git a/kernel/printk.c b/kernel/printk.c
index e2129e8..8210af9 100644
--- a/kernel/printk.c
+++ b/kernel/printk.c
@@ -581,6 +581,41 @@ static int have_callable_console(void)
 	return 0;
 }
 
+#ifdef CONFIG_IPIPE
+
+static ipipe_spinlock_t __ipipe_printk_lock = IPIPE_SPIN_LOCK_UNLOCKED;
+
+static int __ipipe_printk_fill;
+
+static char __ipipe_printk_buf[__LOG_BUF_LEN];
+
+void __ipipe_flush_printk (unsigned virq, void *cookie)
+{
+	char *p = __ipipe_printk_buf;
+	int len, lmax, out = 0;
+	unsigned long flags;
+
+	goto start;
+
+	do {
+		spin_unlock_irqrestore(&__ipipe_printk_lock, flags);
+ start:
+		lmax = __ipipe_printk_fill;
+		while (out < lmax) {
+			len = strlen(p) + 1;
+			printk("%s",p);
+			p += len;
+			out += len;
+		}
+		spin_lock_irqsave(&__ipipe_printk_lock, flags);
+	}
+	while (__ipipe_printk_fill != lmax);
+
+	__ipipe_printk_fill = 0;
+
+	spin_unlock_irqrestore(&__ipipe_printk_lock, flags);
+}
+
 /**
  * printk - print a kernel message
  * @fmt: format string
@@ -606,6 +641,47 @@ static int have_callable_console(void)
 
 asmlinkage int printk(const char *fmt, ...)
 {
+  	int r, fbytes, oldcount, cs = -1;
+    	unsigned long flags;
+	va_list args;
+
+	va_start(args, fmt);
+
+	if (test_bit(IPIPE_SPRINTK_FLAG,&ipipe_current_domain->flags) ||
+	    oops_in_progress)
+		cs = ipipe_disable_context_check(ipipe_processor_id());
+
+	if (ipipe_current_domain == ipipe_root_domain || cs != -1) {
+		r = vprintk(fmt, args);
+		if (cs != -1)
+			ipipe_restore_context_check(ipipe_processor_id(), cs);
+		goto out;
+	}
+
+	spin_lock_irqsave(&__ipipe_printk_lock, flags);
+
+	oldcount = __ipipe_printk_fill;
+	fbytes = __LOG_BUF_LEN - oldcount;
+
+	if (fbytes > 1)	{
+		r = vscnprintf(__ipipe_printk_buf + __ipipe_printk_fill,
+			       fbytes, fmt, args) + 1; /* account for the null byte */
+		__ipipe_printk_fill += r;
+	} else
+		r = 0;
+
+	spin_unlock_irqrestore(&__ipipe_printk_lock, flags);
+
+	if (oldcount == 0)
+		ipipe_trigger_irq(__ipipe_printk_virq);
+out:
+	va_end(args);
+
+	return r;
+}
+#else /* !CONFIG_IPIPE */
+asmlinkage int printk(const char *fmt, ...)
+{
 	va_list args;
 	int r;
 
@@ -615,6 +691,7 @@ asmlinkage int printk(const char *fmt, ...)
 
 	return r;
 }
+#endif /* CONFIG_IPIPE */
 
 /* cpu currently holding logbuf_lock */
 static volatile unsigned int printk_cpu = UINT_MAX;
diff --git a/kernel/sched.c b/kernel/sched.c
index 4e2f603..effdf82 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -2088,7 +2088,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
 	smp_wmb();
 	rq = task_rq_lock(p, &flags);
 	old_state = p->state;
-	if (!(old_state & state))
+	if (!(old_state & state) || (old_state & (TASK_NOWAKEUP|TASK_ATOMICSWITCH)))
 		goto out;
 
 	if (p->se.on_rq)
@@ -2438,13 +2438,15 @@ asmlinkage void schedule_tail(struct task_struct *prev)
 #endif
 	if (current->set_child_tid)
 		put_user(task_pid_vnr(current), current->set_child_tid);
+
+ 	ipipe_init_notify(current);
 }
 
 /*
  * context_switch - switch to the new MM and the new
  * thread's register state.
  */
-static inline void
+static inline int
 context_switch(struct rq *rq, struct task_struct *prev,
 	       struct task_struct *next)
 {
@@ -2485,12 +2487,25 @@ context_switch(struct rq *rq, struct task_struct *prev,
 	switch_to(prev, next, prev);
 
 	barrier();
+
+#ifdef CONFIG_IPIPE_DELAYED_ATOMICSW
+	current->state &= ~TASK_ATOMICSWITCH;
+#else
+	prev->state &= ~TASK_ATOMICSWITCH;
+#endif
+	if (task_hijacked(prev))
+		return 1;
+
+#undef clrtask
+
 	/*
 	 * this_rq must be evaluated again because prev may have moved
 	 * CPUs since it called schedule(), thus the 'rq' on its stack
 	 * frame will be invalid.
 	 */
 	finish_task_switch(this_rq(), prev);
+
+	return 0;
 }
 
 /*
@@ -4025,6 +4040,7 @@ void scheduler_tick(void)
 
 void __kprobes add_preempt_count(int val)
 {
+	ipipe_check_context(ipipe_root_domain);
 	/*
 	 * Underflow?
 	 */
@@ -4041,6 +4057,7 @@ EXPORT_SYMBOL(add_preempt_count);
 
 void __kprobes sub_preempt_count(int val)
 {
+	ipipe_check_context(ipipe_root_domain);
 	/*
 	 * Underflow?
 	 */
@@ -4152,6 +4169,9 @@ need_resched:
 	rcu_qsctr_inc(cpu);
 	prev = rq->curr;
 	switch_count = &prev->nivcsw;
+ 	if (unlikely(prev->state & TASK_ATOMICSWITCH))
+		/* Pop one disable level -- one still remains. */
+		preempt_enable();
 
 	release_kernel_lock(prev);
 need_resched_nonpreemptible:
@@ -4194,15 +4214,18 @@ need_resched_nonpreemptible:
 		rq->curr = next;
 		++*switch_count;
 
-		context_switch(rq, prev, next); /* unlocks the rq */
+		if (context_switch(rq, prev, next)) /* unlocks the rq unless hijacked */
+			return;
 		/*
 		 * the context switch might have flipped the stack from under
 		 * us, hence refresh the local variables.
 		 */
 		cpu = smp_processor_id();
 		rq = cpu_rq(cpu);
-	} else
+	} else {
+		prev->state &= ~TASK_ATOMICSWITCH;
 		spin_unlock_irq(&rq->lock);
+	}
 
 	hrtick_set(rq);
 
@@ -4855,6 +4878,7 @@ recheck:
 
 	oldprio = p->prio;
 	__setscheduler(rq, p, policy, param->sched_priority);
+  	ipipe_setsched_notify(p);
 
 	if (running)
 		p->sched_class->set_curr_task(rq);
@@ -5171,6 +5195,7 @@ static void __cond_resched(void)
 #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
 	__might_sleep(__FILE__, __LINE__);
 #endif
+	ipipe_check_context(ipipe_root_domain);
 	/*
 	 * The BKS might be reacquired before we have dropped
 	 * PREEMPT_ACTIVE, which could trigger a second
@@ -8867,3 +8892,60 @@ struct cgroup_subsys cpuacct_subsys = {
 	.subsys_id = cpuacct_subsys_id,
 };
 #endif	/* CONFIG_CGROUP_CPUACCT */
+
+#ifdef CONFIG_IPIPE
+
+int ipipe_setscheduler_root (struct task_struct *p, int policy, int prio)
+{
+	const struct sched_class *prev_class = p->sched_class;
+	int oldprio, on_rq, running;
+	unsigned long flags;
+	struct rq *rq;
+
+	spin_lock_irqsave(&p->pi_lock, flags);
+	rq = __task_rq_lock(p);
+	update_rq_clock(rq);
+	on_rq = p->se.on_rq;
+	running = task_running(rq, p);
+
+	if (on_rq)
+		deactivate_task(rq, p, 0);
+	if (running)
+		p->sched_class->put_prev_task(rq, p);
+
+	oldprio = p->prio;
+	__setscheduler(rq, p, policy, prio);
+	ipipe_setsched_notify(p);
+
+	if (running)
+		p->sched_class->set_curr_task(rq);
+	if (on_rq) {
+		activate_task(rq, p, 0);
+		check_class_changed(rq, p, prev_class, oldprio, running);
+	}
+	__task_rq_unlock(rq);
+	spin_unlock_irqrestore(&p->pi_lock, flags);
+
+	rt_mutex_adjust_pi(p);
+
+	return 0;
+}
+
+EXPORT_SYMBOL(ipipe_setscheduler_root);
+
+int ipipe_reenter_root (struct task_struct *prev, int policy, int prio)
+{
+	finish_task_switch(this_rq(), prev);
+
+	(void)reacquire_kernel_lock(current);
+	preempt_enable_no_resched();
+
+	if (current->policy != policy || current->rt_priority != prio)
+		return ipipe_setscheduler_root(current, policy, prio);
+
+	return 0;
+}
+
+EXPORT_SYMBOL(ipipe_reenter_root);
+
+#endif /* CONFIG_IPIPE */
diff --git a/kernel/signal.c b/kernel/signal.c
index 6c0958e..1ccd514 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -495,6 +495,7 @@ void signal_wake_up(struct task_struct *t, int resume)
 	unsigned int mask;
 
 	set_tsk_thread_flag(t, TIF_SIGPENDING);
+	ipipe_sigwake_notify(t); /* TIF_SIGPENDING must be set first. */
 
 	/*
 	 * For SIGKILL, we want to wake it up in the stopped/traced/killable
diff --git a/kernel/spinlock.c b/kernel/spinlock.c
index ae28c82..523eb7a 100644
--- a/kernel/spinlock.c
+++ b/kernel/spinlock.c
@@ -87,7 +87,7 @@ unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
 	 * _raw_spin_lock_flags() code, because lockdep assumes
 	 * that interrupts are not re-enabled during lock-acquire:
 	 */
-#ifdef CONFIG_LOCKDEP
+#if defined(CONFIG_LOCKDEP) || defined(CONFIG_IPIPE)
 	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 #else
 	_raw_spin_lock_flags(lock, &flags);
@@ -304,7 +304,7 @@ unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclas
 	 * _raw_spin_lock_flags() code, because lockdep assumes
 	 * that interrupts are not re-enabled during lock-acquire:
 	 */
-#ifdef CONFIG_LOCKDEP
+#if defined(CONFIG_LOCKDEP) || defined(CONFIG_IPIPE)
 	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 #else
 	_raw_spin_lock_flags(lock, &flags);
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
index 4f38865..b757e8a 100644
--- a/kernel/time/tick-common.c
+++ b/kernel/time/tick-common.c
@@ -165,6 +165,10 @@ static void tick_setup_device(struct tick_device *td,
 
 	td->evtdev = newdev;
 
+	/* I-pipe: derive global tick IRQ from CPU 0 */
+	if (cpu == 0)
+		ipipe_update_tick_evtdev(newdev);
+
 	/*
 	 * When the device is not per cpu, pin the interrupt to the
 	 * current cpu:
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index d2099f4..8102abc 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -125,6 +125,8 @@ config DEBUG_SECTION_MISMATCH
 	  - Enable verbose reporting from modpost to help solving
 	    the section mismatches reported.
 
+source "kernel/ipipe/Kconfig.debug"
+
 config DEBUG_KERNEL
 	bool "Kernel debugging"
 	help
diff --git a/lib/bust_spinlocks.c b/lib/bust_spinlocks.c
index 486da62..0d33232 100644
--- a/lib/bust_spinlocks.c
+++ b/lib/bust_spinlocks.c
@@ -12,16 +12,19 @@
 #include <linux/tty.h>
 #include <linux/wait.h>
 #include <linux/vt_kern.h>
+#include <linux/ipipe_trace.h>
 
 
 void __attribute__((weak)) bust_spinlocks(int yes)
 {
 	if (yes) {
+  		ipipe_trace_panic_freeze();
 		++oops_in_progress;
 	} else {
 #ifdef CONFIG_VT
 		unblank_screen();
 #endif
+ 		ipipe_trace_panic_dump();
 		if (--oops_in_progress == 0)
 			wake_up_klogd();
 	}
diff --git a/lib/ioremap.c b/lib/ioremap.c
index 14c6078..a275469 100644
--- a/lib/ioremap.c
+++ b/lib/ioremap.c
@@ -85,8 +85,8 @@ int ioremap_page_range(unsigned long addr,
 		if (err)
 			break;
 	} while (pgd++, addr = next, addr != end);
-
-	flush_cache_vmap(start, end);
+	__ipipe_pin_range_globally(start, end);
+ 	flush_cache_vmap(start, end);
 
 	return err;
 }
diff --git a/lib/smp_processor_id.c b/lib/smp_processor_id.c
index 6c90fb9..9c022d2 100644
--- a/lib/smp_processor_id.c
+++ b/lib/smp_processor_id.c
@@ -13,10 +13,13 @@ unsigned int debug_smp_processor_id(void)
 	int this_cpu = raw_smp_processor_id();
 	cpumask_t this_mask;
 
+	if (!ipipe_root_domain_p)
+		goto out;
+
 	if (likely(preempt_count))
 		goto out;
 
-	if (irqs_disabled())
+	if (irqs_disabled() || irqs_disabled_hw())
 		goto out;
 
 	/*
diff --git a/lib/spinlock_debug.c b/lib/spinlock_debug.c
index 9c4b025..08f096b 100644
--- a/lib/spinlock_debug.c
+++ b/lib/spinlock_debug.c
@@ -133,6 +133,8 @@ void _raw_spin_lock(spinlock_t *lock)
 	debug_spin_lock_after(lock);
 }
 
+EXPORT_SYMBOL(_raw_spin_lock);
+
 int _raw_spin_trylock(spinlock_t *lock)
 {
 	int ret = __raw_spin_trylock(&lock->raw_lock);
@@ -148,12 +150,16 @@ int _raw_spin_trylock(spinlock_t *lock)
 	return ret;
 }
 
+EXPORT_SYMBOL(_raw_spin_trylock);
+
 void _raw_spin_unlock(spinlock_t *lock)
 {
 	debug_spin_unlock(lock);
 	__raw_spin_unlock(&lock->raw_lock);
 }
 
+EXPORT_SYMBOL(_raw_spin_unlock);
+
 static void rwlock_bug(rwlock_t *lock, const char *msg)
 {
 	if (!debug_locks_off())
@@ -199,6 +205,8 @@ void _raw_read_lock(rwlock_t *lock)
 	__raw_read_lock(&lock->raw_lock);
 }
 
+EXPORT_SYMBOL(_raw_read_lock);
+
 int _raw_read_trylock(rwlock_t *lock)
 {
 	int ret = __raw_read_trylock(&lock->raw_lock);
@@ -212,12 +220,16 @@ int _raw_read_trylock(rwlock_t *lock)
 	return ret;
 }
 
+EXPORT_SYMBOL(_raw_read_trylock);
+
 void _raw_read_unlock(rwlock_t *lock)
 {
 	RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
 	__raw_read_unlock(&lock->raw_lock);
 }
 
+EXPORT_SYMBOL(_raw_read_unlock);
+
 static inline void debug_write_lock_before(rwlock_t *lock)
 {
 	RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
@@ -275,6 +287,8 @@ void _raw_write_lock(rwlock_t *lock)
 	debug_write_lock_after(lock);
 }
 
+EXPORT_SYMBOL(_raw_write_lock);
+
 int _raw_write_trylock(rwlock_t *lock)
 {
 	int ret = __raw_write_trylock(&lock->raw_lock);
@@ -290,8 +304,12 @@ int _raw_write_trylock(rwlock_t *lock)
 	return ret;
 }
 
+EXPORT_SYMBOL(_raw_write_trylock);
+
 void _raw_write_unlock(rwlock_t *lock)
 {
 	debug_write_unlock(lock);
 	__raw_write_unlock(&lock->raw_lock);
 }
+
+EXPORT_SYMBOL(_raw_write_unlock);
diff --git a/mm/memory.c b/mm/memory.c
index 2302d22..ad2d21f 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -51,6 +51,7 @@
 #include <linux/init.h>
 #include <linux/writeback.h>
 #include <linux/memcontrol.h>
+#include <linux/vmalloc.h>
 
 #include <asm/pgalloc.h>
 #include <asm/uaccess.h>
@@ -478,6 +479,32 @@ out:
 	return pfn_to_page(pfn);
 }
 
+static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
+{
+	/*
+	 * If the source page was a PFN mapping, we don't have
+	 * a "struct page" for it. We do a best-effort copy by
+	 * just copying from the original user address. If that
+	 * fails, we just zero-fill it. Live with it.
+	 */
+	if (unlikely(!src)) {
+		void *kaddr = kmap_atomic(dst, KM_USER0);
+		void __user *uaddr = (void __user *)(va & PAGE_MASK);
+
+		/*
+		 * This really shouldn't fail, because the page is there
+		 * in the page tables. But it might just be unreadable,
+		 * in which case we just give up and fill the result with
+		 * zeroes.
+		 */
+		if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
+			memset(kaddr, 0, PAGE_SIZE);
+		kunmap_atomic(kaddr, KM_USER0);
+		flush_dcache_page(dst);
+	} else
+		copy_user_highpage(dst, src, va, vma);
+}
+
 /*
  * copy one vm_area from one task to the other. Assumes the page tables
  * already present in the new task to be cleared in the whole range
@@ -486,8 +513,8 @@ out:
 
 static inline void
 copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
-		pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
-		unsigned long addr, int *rss)
+	     pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
+	     unsigned long addr, int *rss, struct page *uncow_page)
 {
 	unsigned long vm_flags = vma->vm_flags;
 	pte_t pte = *src_pte;
@@ -526,6 +553,21 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 	 * in the parent and the child
 	 */
 	if (is_cow_mapping(vm_flags)) {
+#ifdef CONFIG_IPIPE
+		if (uncow_page) {
+			struct page *old_page = vm_normal_page(vma, addr, pte);
+			cow_user_page(uncow_page, old_page, addr, vma);
+			pte = mk_pte(uncow_page, vma->vm_page_prot);
+
+			if (vm_flags & VM_SHARED)
+				pte = pte_mkclean(pte);
+			pte = pte_mkold(pte);
+
+			page_add_new_anon_rmap(uncow_page, vma, addr);
+			rss[!!PageAnon(uncow_page)]++;
+			goto out_set_pte;
+		}
+#endif /* CONFIG_IPIPE */
 		ptep_set_wrprotect(src_mm, addr, src_pte);
 		pte = pte_wrprotect(pte);
 	}
@@ -556,13 +598,27 @@ static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 	pte_t *src_pte, *dst_pte;
 	spinlock_t *src_ptl, *dst_ptl;
 	int progress = 0;
+	struct page *uncow_page = NULL;
 	int rss[2];
-
+#ifdef CONFIG_IPIPE
+	int do_cow_break = 0;
+again:
+	if (do_cow_break) {
+		uncow_page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
+		if (!uncow_page)
+			return -ENOMEM;
+		do_cow_break = 0;
+	}
+#else
 again:
+#endif
 	rss[1] = rss[0] = 0;
 	dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
-	if (!dst_pte)
+	if (!dst_pte) {
+		if (uncow_page)
+			page_cache_release(uncow_page);
 		return -ENOMEM;
+	}
 	src_pte = pte_offset_map_nested(src_pmd, addr);
 	src_ptl = pte_lockptr(src_mm, src_pmd);
 	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
@@ -583,7 +639,26 @@ again:
 			progress++;
 			continue;
 		}
-		copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, vma, addr, rss);
+#ifdef CONFIG_IPIPE
+		if (likely(uncow_page == NULL) && likely(pte_present(*src_pte))) {
+			if (is_cow_mapping(vma->vm_flags)) {
+				if (((vma->vm_flags|src_mm->def_flags) & (VM_LOCKED|VM_PINNED))
+				    == (VM_LOCKED|VM_PINNED)) {
+				    	arch_leave_lazy_mmu_mode();
+				    	spin_unlock(src_ptl);
+				    	pte_unmap_nested(src_pte);
+				    	add_mm_rss(dst_mm, rss[0], rss[1]);
+				    	pte_unmap_unlock(dst_pte, dst_ptl);
+				    	cond_resched();
+					do_cow_break = 1;
+					goto again;
+				}
+			}
+		}
+#endif
+		copy_one_pte(dst_mm, src_mm, dst_pte,
+			     src_pte, vma, addr, rss, uncow_page);
+		uncow_page = NULL;
 		progress += 8;
 	} while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
 
@@ -1642,32 +1717,6 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
 	return pte;
 }
 
-static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
-{
-	/*
-	 * If the source page was a PFN mapping, we don't have
-	 * a "struct page" for it. We do a best-effort copy by
-	 * just copying from the original user address. If that
-	 * fails, we just zero-fill it. Live with it.
-	 */
-	if (unlikely(!src)) {
-		void *kaddr = kmap_atomic(dst, KM_USER0);
-		void __user *uaddr = (void __user *)(va & PAGE_MASK);
-
-		/*
-		 * This really shouldn't fail, because the page is there
-		 * in the page tables. But it might just be unreadable,
-		 * in which case we just give up and fill the result with
-		 * zeroes.
-		 */
-		if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
-			memset(kaddr, 0, PAGE_SIZE);
-		kunmap_atomic(kaddr, KM_USER0);
-		flush_dcache_page(dst);
-	} else
-		copy_user_highpage(dst, src, va, vma);
-}
-
 /*
  * This routine handles present pages, when users try to write
  * to a shared page. It is done by copying the page to a new address
@@ -2894,3 +2943,111 @@ void print_vma_addr(char *prefix, unsigned long ip)
 	}
 	up_read(&current->mm->mmap_sem);
 }
+
+#ifdef CONFIG_IPIPE
+
+static inline int ipipe_pin_pte_range(struct mm_struct *mm, pmd_t *pmd,
+				      struct vm_area_struct *vma,
+				      unsigned long addr, unsigned long end)
+{
+	spinlock_t *ptl;
+	pte_t *pte;
+
+	do {
+		pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
+		if (!pte)
+			continue;
+
+		if (!pte_present(*pte) || pte_write(*pte)) {
+			pte_unmap_unlock(pte, ptl);
+			continue;
+		}
+
+		if (do_wp_page(mm, vma, addr, pte, pmd, ptl, *pte) == VM_FAULT_OOM)
+			return -ENOMEM;
+	} while (addr += PAGE_SIZE, addr != end);
+	return 0;
+}
+
+static inline int ipipe_pin_pmd_range(struct mm_struct *mm, pud_t *pud,
+				      struct vm_area_struct *vma,
+				      unsigned long addr, unsigned long end)
+{
+	unsigned long next;
+	pmd_t *pmd;
+
+	pmd = pmd_offset(pud, addr);
+	do {
+		next = pmd_addr_end(addr, end);
+		if (pmd_none_or_clear_bad(pmd))
+			continue;
+		if (ipipe_pin_pte_range(mm, pmd, vma, addr, next))
+			return -ENOMEM;
+	} while (pmd++, addr = next, addr != end);
+	return 0;
+}
+
+static inline int ipipe_pin_pud_range(struct mm_struct *mm, pgd_t *pgd,
+				      struct vm_area_struct *vma,
+				      unsigned long addr, unsigned long end)
+{
+	unsigned long next;
+	pud_t *pud;
+
+	pud = pud_offset(pgd, addr);
+	do {
+		next = pud_addr_end(addr, end);
+		if (pud_none_or_clear_bad(pud))
+			continue;
+		if (ipipe_pin_pmd_range(mm, pud, vma, addr, next))
+			return -ENOMEM;
+	} while (pud++, addr = next, addr != end);
+	return 0;
+}
+
+int ipipe_disable_ondemand_mappings(struct task_struct *tsk)
+{
+	unsigned long addr, next, end;
+	struct vm_area_struct *vma;
+	struct mm_struct *mm;
+	int result = 0;
+	pgd_t *pgd;
+
+	mm = get_task_mm(tsk);
+	if (!mm)
+		return -EPERM;
+
+	down_write(&mm->mmap_sem);
+	if (mm->def_flags & VM_PINNED)
+		goto done_mm;
+
+	for (vma = mm->mmap; vma; vma = vma->vm_next) {
+		if (!is_cow_mapping(vma->vm_flags)
+		    || !(vma->vm_flags & VM_WRITE))
+			continue;
+
+		addr = vma->vm_start;
+		end = vma->vm_end;
+
+		pgd = pgd_offset(mm, addr);
+		do {
+			next = pgd_addr_end(addr, end);
+			if (pgd_none_or_clear_bad(pgd))
+				continue;
+			if (ipipe_pin_pud_range(mm, pgd, vma, addr, next)) {
+				result = -ENOMEM;
+				goto done_mm;
+			}
+		} while (pgd++, addr = next, addr != end);
+	}
+	mm->def_flags |= VM_PINNED;
+
+  done_mm:
+	up_write(&mm->mmap_sem);
+	mmput(mm);
+	return result;
+}
+
+EXPORT_SYMBOL(ipipe_disable_ondemand_mappings);
+
+#endif
diff --git a/mm/mlock.c b/mm/mlock.c
index 7b26560..d2a318a 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -173,10 +173,10 @@ asmlinkage long sys_munlock(unsigned long start, size_t len)
 static int do_mlockall(int flags)
 {
 	struct vm_area_struct * vma, * prev = NULL;
-	unsigned int def_flags = 0;
+	unsigned int def_flags = current->mm->def_flags & VM_PINNED;
 
 	if (flags & MCL_FUTURE)
-		def_flags = VM_LOCKED;
+		def_flags |= VM_LOCKED;
 	current->mm->def_flags = def_flags;
 	if (flags == MCL_FUTURE)
 		goto out;
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 6e45b0f..1a1f7c3 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -163,6 +163,7 @@ int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
 		if (err)
 			break;
 	} while (pgd++, addr = next, addr != end);
+	__ipipe_pin_range_globally((unsigned long) area->addr, end);
 	flush_cache_vmap((unsigned long) area->addr, end);
 	return err;
 }
