From 387e1835d1e7c5ea45035c01a74834fcdffad4e7 Mon Sep 17 00:00:00 2001
From: Tomas Hruby <tom@minix3.org>
Date: Wed, 15 Sep 2010 14:10:54 +0000
Subject: [PATCH] SMP - BSP halts APs before shutting down

---
 kernel/arch/i386/apic.c             |  4 ----
 kernel/arch/i386/apic.h             |  1 -
 kernel/arch/i386/apic_asm.S         |  2 +-
 kernel/arch/i386/arch_smp.c         | 37 +++++++++++++++++++----------
 kernel/arch/i386/include/arch_smp.h |  2 ++
 kernel/main.c                       |  2 +-
 kernel/smp.c                        | 11 +++++++--
 kernel/smp.h                        |  2 ++
 8 files changed, 39 insertions(+), 22 deletions(-)

diff --git a/kernel/arch/i386/apic.c b/kernel/arch/i386/apic.c
index a55a389d0..8da7187bd 100644
--- a/kernel/arch/i386/apic.c
+++ b/kernel/arch/i386/apic.c
@@ -398,10 +398,6 @@ PUBLIC void apic_ipi_sched_handler(void)
 {
 }
 
-PUBLIC void apic_ipi_halt_handler(void)
-{
-}
-
 PUBLIC unsigned int apicid(void)
 {
 	return lapic_read(LAPIC_ID);
diff --git a/kernel/arch/i386/apic.h b/kernel/arch/i386/apic.h
index 03f28c2e3..253086568 100644
--- a/kernel/arch/i386/apic.h
+++ b/kernel/arch/i386/apic.h
@@ -167,7 +167,6 @@ _PROTOTYPE(void apic_ipi_sched_intr, (void));
 _PROTOTYPE(void apic_ipi_halt_intr, (void));
 
 _PROTOTYPE(void apic_ipi_sched_handler, (void));
-_PROTOTYPE(void apic_ipi_halt_handler, (void));
 
 #define APIC_IPI_DEST			0
 #define APIC_IPI_SELF			1
diff --git a/kernel/arch/i386/apic_asm.S b/kernel/arch/i386/apic_asm.S
index 1988dd194..21f3817d7 100644
--- a/kernel/arch/i386/apic_asm.S
+++ b/kernel/arch/i386/apic_asm.S
@@ -73,7 +73,7 @@ ENTRY(apic_ipi_sched_intr)
 	lapic_intr(_C_LABEL(apic_ipi_sched_handler))
 
 ENTRY(apic_ipi_halt_intr)
-	lapic_intr(_C_LABEL(apic_ipi_halt_handler))
+	lapic_intr(_C_LABEL(smp_ipi_halt_handler))
 
 #endif /* CONFIG_SMP */
 
diff --git a/kernel/arch/i386/arch_smp.c b/kernel/arch/i386/arch_smp.c
index 26c19b79e..6a3c8fa96 100644
--- a/kernel/arch/i386/arch_smp.c
+++ b/kernel/arch/i386/arch_smp.c
@@ -37,6 +37,7 @@ extern u32_t busclock[CONFIG_MAX_CPUS];
 extern int panicking;
 
 static int ap_cpu_ready;
+static int cpu_down;
 
 /* there can be at most 255 local APIC ids, each fits in 8 bits */
 PRIVATE unsigned char apicid2cpuid[255];
@@ -170,30 +171,30 @@ PUBLIC void smp_halt_cpu (void)
 	NOT_IMPLEMENTED;
 }
 
-PUBLIC void smp_shutdown_aps (void)
+PUBLIC void smp_shutdown_aps(void)
 {
-	u8_t cpu;
+	unsigned cpu;
 	unsigned aid = apicid();
+	unsigned local_cpu = cpuid;
 	
 	if (ncpus == 1)
 		goto exit_shutdown_aps;
+	
+	/* we must let the other cpus enter the kernel mode */
+	BKL_UNLOCK();
 
 	for (cpu = 0; cpu < ncpus; cpu++) {
-		u16_t i;
-		if (!cpu_is_ready(cpu))
-			continue;
-		if ((aid == cpuid2apicid[cpu]) && (aid == bsp_lapic_id))
+		if (cpu == cpuid)
 			continue;
+		cpu_down = -1;
+		barrier();
 		apic_send_ipi(APIC_SMP_CPU_HALT_VECTOR, cpu, APIC_IPI_DEST);
-		/* TODO wait for the cpu to be down */
+		/* wait for the cpu to be down */
+		while (cpu_down != cpu);
+		printf("CPU %d is down\n", cpu);
+		cpu_clear_flag(cpu, CPU_IS_READY);
 	}
 
-	/* Sending INIT to a processor makes it to wait in a halt state */
-	for (cpu = 0; cpu < ncpus; cpu++) {
-		if ((aid == cpuid2apicid[cpu]) && (aid == bsp_lapic_id))
-			continue;
-		apic_send_init_ipi (cpu, 0);
-	}
 exit_shutdown_aps:
 	ioapic_disable_all();
 
@@ -342,3 +343,13 @@ uniproc_fallback:
 	intr_init (INTS_MINIX, 0); /* no auto eoi */
 	printf("WARNING : SMP initialization failed\n");
 }
+	
+PUBLIC void arch_smp_halt_cpu(void)
+{
+	/* say that we are down */
+	cpu_down = cpuid;
+	barrier();
+	/* unlock the BKL and don't continue */
+	BKL_UNLOCK();
+	for(;;);
+}
diff --git a/kernel/arch/i386/include/arch_smp.h b/kernel/arch/i386/include/arch_smp.h
index 4885c0f06..3a013a982 100644
--- a/kernel/arch/i386/include/arch_smp.h
+++ b/kernel/arch/i386/include/arch_smp.h
@@ -23,6 +23,8 @@
 
 extern unsigned char cpuid2apicid[CONFIG_MAX_CPUS];
 
+#define barrier()	do { mfence(); } while(0)
+
 #endif
 
 #endif /* __SMP_X86_H__ */
diff --git a/kernel/main.c b/kernel/main.c
index e24e0748a..8b3de2509 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -374,7 +374,7 @@ PUBLIC void minix_shutdown(timer_t *tp)
    * monitor again
    */
   if (ncpus > 1)
-	  NOT_IMPLEMENTED;
+	  smp_shutdown_aps();
 #endif
   arch_stop_local_timer();
   hw_intr_disable_all();
diff --git a/kernel/smp.c b/kernel/smp.c
index 3f6e9b5ec..2415e4add 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -11,7 +11,7 @@ static volatile unsigned ap_cpus_booted;
 SPINLOCK_DEFINE(big_kernel_lock)
 SPINLOCK_DEFINE(boot_lock)
 
-void wait_for_APs_to_finish_booting(void)
+PUBLIC void wait_for_APs_to_finish_booting(void)
 {
 	/* we must let the other CPUs to run in kernel mode first */
 	BKL_UNLOCK();
@@ -21,7 +21,14 @@ void wait_for_APs_to_finish_booting(void)
 	BKL_LOCK();
 }
 
-void ap_boot_finished(unsigned cpu)
+PUBLIC void ap_boot_finished(unsigned cpu)
 {
 	ap_cpus_booted++;
 }
+
+PUBLIC void smp_ipi_halt_handler(void)
+{
+	arch_stop_local_timer();
+	arch_smp_halt_cpu();
+}
+
diff --git a/kernel/smp.h b/kernel/smp.h
index 01e00c0f7..e9fba1f84 100644
--- a/kernel/smp.h
+++ b/kernel/smp.h
@@ -53,6 +53,8 @@ SPINLOCK_DECLARE(boot_lock)
 	
 _PROTOTYPE(void wait_for_APs_to_finish_booting, (void));
 _PROTOTYPE(void ap_boot_finished, (unsigned cpu));
+_PROTOTYPE(void smp_ipi_halt_handler, (void));
+_PROTOTYPE(void arch_smp_halt_cpu, (void));
 
 #endif /* __ASSEMBLY__ */
 
-- 
2.44.0