supervisor: Add an example on the HSM extension

The HSM extension from SBI v2.0 allows a supervisor to manage the state
of harts on the system. Moreover, it also guarantees that only one hart
makes it into the booting stages of the supervisor, and it's up to the
supervisor to wake up the rest through the HSM extension.

I have added an example which showcases this extension in different
ways, and it finally shuts down the machine.

Signed-off-by: Miquel Sabaté Solà <mikisabate@gmail.com>

8 files changed, 610 insertions, 0 deletions

diff --git a/arch/riscv/supervisor/hsm/Makefile b/arch/riscv/supervisor/hsm/Makefile
new file mode 100644
index 0000000..b799cf1
--- /dev/null
+++ b/arch/riscv/supervisor/hsm/Makefile
@@ -0,0 +1,63 @@
+V =
+ifeq ($(strip $(V)),)
+	E = @echo
+	Q = @
+else
+	E = @\#
+	Q =
+endif
+
+CC    = $(CROSS_COMPILE)gcc$(CC_SUFFIX)
+LD    = $(CROSS_COMPILE)ld
+QEMU ?= qemu-system-riscv64
+
+SRC    = $(filter-out hsm.ld.S, $(wildcard *.S *.c))
+OBJ    = $(patsubst %.c,%.o,$(patsubst %.S,%.o,$(SRC)))
+LINKER = hsm.ld
+KERNEL = hsm
+
+ISA      ?= rv64imafdc_zicntr_zicsr_zifencei_zihpm_zca_zcd_zba_zbb
+ASFLAGS   = -march=$(ISA) -mabi=lp64d -mcmodel=medany -fno-PIE -g
+CCFLAGS   = $(ASFLAGS) -std=gnu17 -nostdinc -nostdlib -O0
+WARNINGS  = -Werror -Wpedantic -Wall -Wextra -Wcast-align -Wcast-qual -Winit-self \
+            -Wmissing-include-dirs -Wredundant-decls -Wshadow -Wsign-conversion \
+            -Wswitch-default -Wundef -Wunreachable-code
+CCFLAGS  += $(WARNINGS)
+LDFLAGS   = -static -melf64lriscv -z noexecstack -T $(LINKER)
+QEMU_FLAGS += -nographic
+
+CPUS ?= 4
+
+.PHONY: hsm
+$(KERNEL): config $(OBJ)
+	$(Q) $(CC) -E $(LINKER).S -Iinclude/ -o $(LINKER)
+
+	$(E) "	LD	" $@
+	$(Q) $(LD) $(LDFLAGS) $(OBJ) -o $(KERNEL)
+
+.PHONY: config
+config:
+	$(Q) rm -f config.h
+	$(Q) echo "#ifndef __CONFIG_H__" >> config.h
+	$(Q) echo "#define __CONFIG_H__" >> config.h
+	$(Q) echo -e "\n#define NR_CPUS $(CPUS)\n" >> config.h
+	$(Q) echo "#if (NR_CPUS < 2 || NR_CPUS > 9)" >> config.h
+	$(Q) echo "	#error \"Use a value of 'CPUS' > 1 and < 10\"" >> config.h
+	$(Q) echo -e "#endif // #if (NR_CPUS < 2 || NR_CPUS > 9)\n" >> config.h
+	$(Q) echo "#endif // __CONFIG_H__" >> config.h
+
+.c.o:
+	$(E) "	CC	" $(basename $@)
+	$(Q) $(CC) $(CCFLAGS) -c $< -o $@
+
+.S.o:
+	$(E) "	CC	" $(basename $@)
+	$(Q) $(CC) $(CCFLAGS) -c $< -o $@
+
+.PHONY: clean
+clean:
+	$(Q) rm -f *.o $(KERNEL) $(LINKER) config.h
+
+.PHONY: qemu
+qemu: clean $(KERNEL)
+	$(Q) $(QEMU) $(QEMU_FLAGS) -smp $(CPUS) -machine virt -kernel $(KERNEL)
diff --git a/arch/riscv/supervisor/hsm/README.md b/arch/riscv/supervisor/hsm/README.md
new file mode 100644
index 0000000..a950152
--- /dev/null
+++ b/arch/riscv/supervisor/hsm/README.md
@@ -0,0 +1,126 @@
+# Messing with the Hart State Management Extension ("HSM") from the SBI
+
+This is a simple kernel that messes with the HSM Extension from the SBI v2.0. In
+general, and according to the RISC-V specification, harts can enter Supervisor
+mode at any time. This makes thing simple both for the RISC-V specification and
+machines implementing the architecture. For the kernel this means that you never
+know which hart is going to start first, and in cases like the Linux kernel, you
+actually want to ensure that only one hart is running for initialization
+purposes before waking up the rest. In order to guarantee this, the Linux kernel
+runs a lottery.
+
+To run a lottery on this context means that the first hart to appear will
+actually take a lock in the kernel, which will block any other "secondary" hart
+that appears later. This lock is going to be released once the "main" hart
+guarantees that the kernel has been initialized up to a point where other harts
+can come in. This lock is implemented with an atomic variable that holds how
+many harts have been seen (hence, the first hart will read a zero value from
+this variable, and the rest will read a non-zero one).
+
+All of that being said, the SBI specification v2.0 comes with a cleaner
+approach: the HSM extension. With this extension a Supervisor will be able to
+manage the state of any hart of the system, which is either `started`, `stopped`
+or `suspended` (plus transition states). Moreover, there is the guarantee that
+only one hart will be initialized with the `started` state, whereas the rest
+will be on `stopped`. This means that a kernel can drop the whole idea of a
+lottery and have the guarantee that only one hart will run on start. Whenever
+the kernel has been initialized to a specific stage, it will be able to change
+the state of the rest of harts to `started`.
+
+## This example
+
+This example uses the HSM extension but it also keeps the idea of a lottery.
+That is, the first hart will "acquire" the lock ("win" the lottery), and will be
+responsible for bringing the rest up. At first this hart will simply print
+information on the system. For example, on a system with four harts you will get
+something like:
+
+```
+Hello, world!
+Hart that won the lottery: 1
+Harts still asleep: 0, 2, 3
+
+How many harts have _participated_ in the lottery? 1
+How many harts have _failed_ in the lottery? 0
+```
+
+In this example, only one hart is available: one that "won" the lottery, and the
+rest have not even appeared. The hart with ID '1' will be responsible for waking
+up '0', '2' and '3'. After making some initial checks, it will wake up the first
+secondary hart (no reason to just wake up one, just doing it this way for the
+show):
+
+```
+Waking up first secondary hart
+How many harts have _participated_ in the lottery? 2
+How many harts have _failed_ in the lottery? 1
+```
+
+Now two harts are available, but the one that woke up is a "secondary" one, so
+it lost the lottery and is stuck in `head.S` on an infinite loop doing nothing.
+After doing that, the main hart will wake up the rest:
+
+```
+Waking up the rest
+How many harts have _participated_ in the lottery? 4
+How many harts have _failed_ in the lottery? 3
+```
+
+Now all four harts are up, but three of them (the "secondary" ones) are simply
+stuck in an infinite loop since they have "lost" the lottery. At this point this
+example does something weird for a kernel: it will return from the main
+function. Doing so allows `head.S` to count the main hart as a "failed" one, and
+it will print some final messages for it:
+
+```
+Goodbye, cruel world!
+How many harts have _participated_ in the lottery? 4
+How many harts have _failed_ in the lottery? 4
+
+THE END
+```
+
+At the point where "THE END" is printed, we will then perform an SBI call to the
+System Reset Extension ("SRST") to gracefully shutdown the system.
+
+## Test
+
+To test this yourself you need a recent enough QEMU which is able to run as a
+RISC-V system, and you need to set the `CROSS_COMPILE` environment variable as
+you would on the Linux Kernel if you are doing this on a non-RISC-V machine.
+After that, you can simply run `make` to build the binary, or simply run `make
+qemu` to both build it and run QEMU.
+
+Notice also that the Makefile accepts the `CPUS` variable, which by default is
+set to 4. Change this variable to something else to get a different number of
+harts on the example (but lesser than 10, since I haven't bothered to account
+for that when printing numerical values). For example, `make qemu CPUS=8`:
+
+``` shell
+Hello, world!
+Hart that won the lottery: 5
+Harts still asleep: 0, 1, 2, 3, 4, 6, 7
+How many harts have _participated_ in the lottery? 1
+How many harts have _failed_ in the lottery? 0
+
+Waking up first secondary hart
+How many harts have _participated_ in the lottery? 2
+How many harts have _failed_ in the lottery? 1
+
+Waking up the rest
+How many harts have _participated_ in the lottery? 8
+How many harts have _failed_ in the lottery? 7
+
+Goodbye, cruel world!
+How many harts have _participated_ in the lottery? 8
+How many harts have _failed_ in the lottery? 8
+
+THE END
+```
+
+## Rationale
+
+This example came after exploring the topic of SMP on
+[fbos](https://github.com/mssola/fbos). Even if in there I did not explore this
+extension further (because of the needs of the project), I decided that I wanted
+a fresh simple kernel to mess with this extension. And hence this project.
diff --git a/arch/riscv/supervisor/hsm/compiler.h b/arch/riscv/supervisor/hsm/compiler.h
new file mode 100644
index 0000000..f86b0da
--- /dev/null
+++ b/arch/riscv/supervisor/hsm/compiler.h
@@ -0,0 +1,12 @@
+#ifndef __COMPILER_H__
+#define __COMPILER_H__
+
+typedef unsigned long uint64_t;
+typedef unsigned long uintptr_t;
+typedef unsigned long size_t;
+
+typedef struct {
+	unsigned int value;
+} atomic_t;
+
+#endif // __COMPILER_H__
diff --git a/arch/riscv/supervisor/hsm/head.S b/arch/riscv/supervisor/hsm/head.S
new file mode 100644
index 0000000..e912b18
--- /dev/null
+++ b/arch/riscv/supervisor/hsm/head.S
@@ -0,0 +1,44 @@
+.global _start
+.section .head.text
+
+_start:
+	// Start by simply disabling interrupts.
+	csrw sie, zero
+	csrw sip, zero
+	fence.i
+
+	// Initialize the 'a4' register, which is going to be used for printing out
+	// the end message by the main hart.
+	li a4, 0
+
+	// Hart lottery! The main hart will win it, the rest will simply jump into
+	// .Lhalt. See hsm.c for more details.
+	la a3, hart_lottery
+	li a2, 1
+	amoadd.w a3, a2, (a3)
+	bnez a3, .Lhalt
+
+	// Setup a dummy stack and call the kernel.
+	la sp, _STACK_PTR
+	call start_kernel
+
+	// We actually expect the main kernel routine to quit. Here we make the
+	// .Lhalt part of the code to print special messages for that case.
+	addi a4, a4, 1
+
+.Lhalt:
+	// Increase the number of hart that lost the lottery.
+	la a3, hart_failed_lottery
+	li a2, 1
+	amoadd.w zero, a2, (a3)
+
+	// If it's the main hart that is bailing out, first print some final
+	// messages.
+	beqz a4, .Lend
+	call print_lottery
+	call end
+
+	// Die Bart, die!
+.Lend:
+	wfi
+	j .Lend
diff --git a/arch/riscv/supervisor/hsm/hsm.c b/arch/riscv/supervisor/hsm/hsm.c
new file mode 100644
index 0000000..0b2daca
--- /dev/null
+++ b/arch/riscv/supervisor/hsm/hsm.c
@@ -0,0 +1,215 @@
+#include "compiler.h"
+#include "config.h"
+#include "sbi.h"
+
+// Implemented in head.S. Here we need the symbol in order to pass the address
+// of the function when starting secondary harts.
+extern void _start(void);
+
+/*
+ * Similar to the Linux kernel and what I did in https://github.com/mssola/fbos,
+ * harts will enter in a lottery when starting. See the README.md file for more
+ * details on this.
+ */
+atomic_t hart_lottery __attribute__((__section__(".sdata")));
+atomic_t hart_failed_lottery __attribute__((__section__(".sdata")));
+
+// Fake a delay so it's easier on the eyes.
+#define SLEEP()                            \
+	for (int i = 0; i < 1000000000; i++) { \
+	}
+
+/*
+ * Printing utilities.
+ */
+
+void printk(const char *const message);
+
+void write(const char *const message, size_t n)
+{
+	struct sbi_ret ret = sbi_ecall2(DBCN_EXT, DBCN_WRITE, n, (unsigned long)message);
+	if (ret.error != SBI_SUCCESS) {
+		for (;;) {
+			;
+		}
+	}
+}
+
+size_t strlen(const char *const message)
+{
+	size_t i = 0;
+
+	for (; message[i] != '\0'; i++)
+		;
+
+	return i;
+}
+
+__attribute__((__noreturn__)) void die(const char *const message)
+{
+	if (message) {
+		printk(message);
+	}
+
+	for (;;)
+		;
+}
+
+void print_digit(uint64_t digit)
+{
+	char buffer[2];
+
+	if (digit > 9) {
+		die("We cannot print numbers with two or more digits :D\n");
+	}
+
+	buffer[0] = '0' + digit;
+	buffer[1] = '\0';
+	write(buffer, 2);
+}
+
+void printk(const char *const message)
+{
+	size_t len = strlen(message);
+	if (!len) {
+		return;
+	}
+
+	write(message, len);
+}
+
+/*
+ * Printing/asserting the stats of the lottery.
+ */
+
+void assert_lottery(uint64_t participated, uint64_t failed)
+{
+	if (hart_lottery.value != participated) {
+		die("Bad number for participating harts\n");
+	}
+	if (hart_failed_lottery.value != failed) {
+		die("Bad number for failed harts\n");
+	}
+}
+
+void print_lottery(void)
+{
+	printk("How many harts have _participated_ in the lottery? ");
+	print_digit(hart_lottery.value);
+	write("\n", 1);
+
+	printk("How many harts have _failed_ in the lottery? ");
+	print_digit(hart_failed_lottery.value);
+	write("\n", 1);
+}
+
+// Called at the very end of the program. It will shutdown the machine.
+void end(void)
+{
+	printk("\nTHE END\n");
+
+	// SBI call to the SRST extension to gracefully shutdown the machine.
+	sbi_ecall2(SRST_EXT, SRST_RESET, SRST_SHUTDOWN, SRST_NO_REASON);
+}
+
+void start_kernel(uint64_t hart_id, uintptr_t opaque)
+{
+	printk("Hello, world!\n");
+
+	/*
+	 * Figure out the harts that are meant to be stopped for now.
+	 */
+
+	unsigned int other_harts[NR_CPUS - 1];
+	struct sbi_ret ret;
+
+	printk("Hart that won the lottery: ");
+	print_digit(hart_id);
+	write("\n", 1);
+
+	printk("Hart still asleep: ");
+	for (unsigned int i = 0, j = 0; i < NR_CPUS; i++) {
+		if (i != hart_id) {
+			print_digit(i);
+			if (j + 1 < NR_CPUS - 1) {
+				write(", ", 2);
+			}
+			other_harts[j++] = i;
+		}
+	}
+	write("\n", 1);
+
+	/*
+	 * ASSERT: all harts in 'other_harts' are marked as stopped by HSM.
+	 */
+
+	for (unsigned int i = 0; i < NR_CPUS - 1; i++) {
+		ret = sbi_ecall1(HSM_EXT, HSM_HART_GET_STATUS, other_harts[i]);
+
+		if (ret.error != SBI_SUCCESS) {
+			die("HSM_HART_GET_STATUS: unexpected error from SBI\n");
+		}
+		if (ret.value != HSM_HART_STOPPED) {
+			die("HSM_HART_GET_STATUS: hart that was supposed to be stopped is not!\n");
+		}
+	}
+
+	// Initial state of the lottery.
+	print_lottery();
+	assert_lottery(1, 0);
+	write("\n", 1);
+
+	/*
+	 * Wake up the first hart.
+	 */
+
+	printk("Waking up first secondary hart\n");
+
+	ret = sbi_ecall3(HSM_EXT, HSM_HART_START, other_harts[0], (uintptr_t)&_start, opaque);
+	if (ret.error != SBI_SUCCESS) {
+		die("HSM_HART_START: unexpected error from SBI\n");
+	}
+
+	SLEEP();
+
+	ret = sbi_ecall1(HSM_EXT, HSM_HART_GET_STATUS, other_harts[0]);
+	if (ret.error != SBI_SUCCESS) {
+		die("HSM_HART_GET_STATUS: unexpected error from SBI\n");
+	}
+	if (ret.value != HSM_HART_STARTED) {
+		die("HSM_HART_GET_STATUS: hart that was supposed to be started is not!\n");
+	}
+
+	// Another hart should have entered the lottery and failed.
+	print_lottery();
+	assert_lottery(2, 1);
+	write("\n", 1);
+
+	/*
+	 * Now wake up the rest.
+	 */
+
+	printk("Waking up the rest\n");
+
+	for (unsigned int i = 1; i < NR_CPUS - 1; i++) {
+		ret = sbi_ecall3(HSM_EXT, HSM_HART_START, other_harts[i], (uintptr_t)&_start, opaque);
+		if (ret.error != SBI_SUCCESS) {
+			die("HSM_HART_START: unexpected error from SBI\n");
+		}
+	}
+
+	SLEEP();
+
+	// All awake, only one good.
+	print_lottery();
+	assert_lottery(NR_CPUS, NR_CPUS - 1);
+	write("\n", 1);
+
+	/*
+	 * Kernels usually don't quit, but let's do it for the heck of it; head.S
+	 * will handle it.
+	 */
+
+	printk("Goodbye, cruel world!\n");
+	SLEEP();
+}
diff --git a/arch/riscv/supervisor/hsm/hsm.ld.S b/arch/riscv/supervisor/hsm/hsm.ld.S
new file mode 100644
index 0000000..db83dc9
--- /dev/null
+++ b/arch/riscv/supervisor/hsm/hsm.ld.S
@@ -0,0 +1,48 @@
+#define PAGE_SIZE 0x1000
+#define STACK_SIZE PAGE_SIZE
+
+#define LOAD_BASE_ADDR 0x80000000
+#define LOAD_BASE_OFFSET 0x00200000
+#define LOAD_OFFSET (LOAD_BASE_ADDR + LOAD_BASE_OFFSET)
+
+OUTPUT_ARCH(riscv)
+ENTRY(_start)
+
+SECTIONS {
+	. = LOAD_OFFSET;
+	_start = .;
+	. = ALIGN(PAGE_SIZE);
+
+	.text : {
+		_text = .;
+		KEEP(*(.head.text))
+		. = ALIGN(PAGE_SIZE);
+    	*(.text)
+	}
+	. = ALIGN(PAGE_SIZE);
+
+	.data : {
+    	*(.data)
+	}
+	. = ALIGN(8);
+
+	.rodata : {
+    	*(.rodata)
+	}
+	. = ALIGN(8);
+
+	.sdata : {
+		*(.sdata*)
+	}
+	. = ALIGN(8);
+
+	.bss : {
+    	*(.bss)
+	}
+
+	. = ALIGN(8);
+	. = . + STACK_SIZE;
+	_STACK_PTR = .;
+
+	_end = .;
+}
diff --git a/arch/riscv/supervisor/hsm/sbi.c b/arch/riscv/supervisor/hsm/sbi.c
new file mode 100644
index 0000000..72c1521
--- /dev/null
+++ b/arch/riscv/supervisor/hsm/sbi.c
@@ -0,0 +1,29 @@
+#include "compiler.h"
+#include "sbi.h"
+
+// Implementation taken from the Linux kernel (6.12).
+struct sbi_ret __sbi_ecall(unsigned long arg0, unsigned long arg1, unsigned long arg2,
+						   unsigned long arg3, unsigned long arg4, unsigned long arg5, int fid,
+						   int ext)
+{
+	struct sbi_ret ret;
+
+	register uintptr_t a0 asm("a0") = (uintptr_t)(arg0);
+	register uintptr_t a1 asm("a1") = (uintptr_t)(arg1);
+	register uintptr_t a2 asm("a2") = (uintptr_t)(arg2);
+	register uintptr_t a3 asm("a3") = (uintptr_t)(arg3);
+	register uintptr_t a4 asm("a4") = (uintptr_t)(arg4);
+	register uintptr_t a5 asm("a5") = (uintptr_t)(arg5);
+	register uintptr_t a6 asm("a6") = (uintptr_t)(fid);
+	register uintptr_t a7 asm("a7") = (uintptr_t)(ext);
+
+	asm volatile("ecall"
+				 : "+r"(a0), "+r"(a1)
+				 : "r"(a2), "r"(a3), "r"(a4), "r"(a5), "r"(a6), "r"(a7)
+				 : "memory");
+
+	ret.error = (long)a0;
+	ret.value = (long)a1;
+
+	return ret;
+}
diff --git a/arch/riscv/supervisor/hsm/sbi.h b/arch/riscv/supervisor/hsm/sbi.h
new file mode 100644
index 0000000..7891f76
--- /dev/null
+++ b/arch/riscv/supervisor/hsm/sbi.h
@@ -0,0 +1,73 @@
+#ifndef __SBI_H__
+#define __SBI_H__
+
+enum sbi_ret_error {
+	SBI_SUCCESS = 0,
+	SBI_ERR_FAILED = -1,
+	SBI_ERR_NOT_SUPPORTED = -2,
+	SBI_ERR_INVALID_PARAM = -3,
+	SBI_ERR_DENIED = -4,
+	SBI_ERR_INVALID_ADDRESS = -5,
+	SBI_ERR_ALREADY_AVAILABLE = -6,
+	SBI_ERR_ALREADY_STARTED = -7,
+	SBI_ERR_ALREADY_STOPPED = -8,
+	SBI_ERR_NO_SHMEM = -9,
+};
+
+enum sbi_ext {
+	HSM_EXT = 0x0048534D,
+	DBCN_EXT = 0x4442434E,
+	SRST_EXT = 0x53525354,
+};
+
+enum hsm_actions {
+	HSM_HART_START = 0x00,
+	HSM_HART_STOP = 0x01,
+	HSM_HART_GET_STATUS = 0x02,
+	HSM_HART_SUSPEND = 0x03,
+};
+
+enum hsm_states {
+	HSM_HART_STARTED = 0,
+	HSM_HART_STOPPED = 1,
+	HSM_HART_START_PENDING = 2,
+	HSM_HART_STOP_PENDING = 3,
+	HSM_HART_SUSPENDED = 4,
+	HSM_HART_SUSPEND_PENDING = 5,
+	HSM_HART_RESUME_PENDING = 6,
+};
+
+enum dbcn_actions {
+	DBCN_WRITE = 0x00,
+};
+
+enum srst_actions {
+	SRST_RESET = 0,
+};
+
+enum srst_type {
+	SRST_SHUTDOWN = 0,
+	SRST_COLD_REBOOT = 1,
+	SRST_WARM_REBOOT = 2,
+};
+
+enum srst_reason {
+	SRST_NO_REASON = 0,
+	SRST_SYS_FAILURE = 1,
+};
+
+struct sbi_ret {
+	long error;
+	long value;
+};
+
+struct sbi_ret __sbi_ecall(unsigned long arg0, unsigned long arg1, unsigned long arg2,
+						   unsigned long arg3, unsigned long arg4, unsigned long arg5, int fid,
+						   int ext);
+
+#define sbi_ecall0(ext, fid) __sbi_ecall(0, 0, 0, 0, 0, 0, fid, ext)
+#define sbi_ecall1(ext, fid, arg0) __sbi_ecall(arg0, 0, 0, 0, 0, 0, fid, ext)
+#define sbi_ecall2(ext, fid, arg0, arg1) __sbi_ecall(arg0, arg1, 0, 0, 0, 0, fid, ext)
+#define sbi_ecall3(ext, fid, arg0, arg1, arg2) __sbi_ecall(arg0, arg1, arg2, 0, 0, 0, fid, ext)
+
+#endif // __SBI_H__