1 files changed, 47 insertions, 12 deletions

diff --git a/freebsd/sys/sys/pcpu.h b/freebsd/sys/sys/pcpu.h
index bfa7f34d..7aad9f2e 100644
--- a/freebsd/sys/sys/pcpu.h
+++ b/freebsd/sys/sys/pcpu.h
@@ -81,7 +81,31 @@ extern uintptr_t dpcpu_off[];
  */
 #define	DPCPU_NAME(n)		pcpu_entry_##n
 #define	DPCPU_DECLARE(t, n)	extern t DPCPU_NAME(n)
-#define	DPCPU_DEFINE(t, n)	t DPCPU_NAME(n) __section(DPCPU_SETNAME) __used
+/* struct _hack is to stop this from being used with the static keyword. */
+#define	DPCPU_DEFINE(t, n)	\
+    struct _hack; t DPCPU_NAME(n) __section(DPCPU_SETNAME) __used
+#if defined(KLD_MODULE) && (defined(__aarch64__) || defined(__riscv))
+/*
+ * On some architectures the compiler will use PC-relative load to
+ * find the address of DPCPU data with the static keyword. We then
+ * use this to find the offset of the data in a per-CPU region.
+ * This works for in the kernel as we can allocate the space ahead
+ * of time, however modules need to allocate a sepatate space and
+ * then use relocations to fix the address of the data. As
+ * PC-relative data doesn't have a relocation there is nothing for
+ * the kernel module linker to fix so data is accessed from the
+ * wrong location.
+ *
+ * This is a workaround until a better solution can be found.
+ *
+ * VNET_DEFINE_STATIC also has the same workaround.
+ */
+#define	DPCPU_DEFINE_STATIC(t, n)	\
+    t DPCPU_NAME(n) __section(DPCPU_SETNAME) __used
+#else
+#define	DPCPU_DEFINE_STATIC(t, n)	\
+    static t DPCPU_NAME(n) __section(DPCPU_SETNAME) __used
+#endif
 
 /*
  * Accessors with a given base.
@@ -189,14 +213,6 @@ struct pcpu {
 #endif /* __rtems__ */
 } __aligned(CACHE_LINE_SIZE);
 
-#ifdef CTASSERT
-/*
- * To minimize memory waste in per-cpu UMA zones, size of struct pcpu
- * should be denominator of PAGE_SIZE.
- */
-CTASSERT((PAGE_SIZE / sizeof(struct pcpu)) * sizeof(struct pcpu) == PAGE_SIZE);
-#endif
-
 #ifdef _KERNEL
 
 STAILQ_HEAD(cpuhead, pcpu);
@@ -211,15 +227,34 @@ extern struct pcpu *cpuid_to_pcpu[];
 #endif
 #define	curvidata	PCPU_GET(vidata)
 
+#ifndef __rtems__
+#define UMA_PCPU_ALLOC_SIZE		PAGE_SIZE
+
+#ifdef CTASSERT
+#if defined(__i386__) || defined(__amd64__)
+/* Required for counters(9) to work on x86. */
+CTASSERT(sizeof(struct pcpu) == UMA_PCPU_ALLOC_SIZE);
+#else
+/*
+ * To minimize memory waste in per-cpu UMA zones, size of struct pcpu
+ * should be denominator of PAGE_SIZE.
+ */
+CTASSERT((PAGE_SIZE / sizeof(struct pcpu)) * sizeof(struct pcpu) == PAGE_SIZE);
+#endif	/* UMA_PCPU_ALLOC_SIZE && x86 */
+#endif	/* CTASSERT */
+#else /* __rtems__ */
+#define	UMA_PCPU_ALLOC_SIZE sizeof(struct pcpu)
+#endif /* __rtems__ */
+
 /* Accessor to elements allocated via UMA_ZONE_PCPU zone. */
 static inline void *
 zpcpu_get(void *base)
 {
 
 #ifndef __rtems__
-	return ((char *)(base) + sizeof(struct pcpu) * curcpu);
+	return ((char *)(base) + UMA_PCPU_ALLOC_SIZE * curcpu);
 #else /* __rtems__ */
-	return ((char *)(base) + sizeof(struct pcpu) * _SMP_Get_current_processor());
+	return ((char *)(base) + UMA_PCPU_ALLOC_SIZE * _SMP_Get_current_processor());
 #endif /* __rtems__ */
 }
 
@@ -227,7 +262,7 @@ static inline void *
 zpcpu_get_cpu(void *base, int cpu)
 {
 
-	return ((char *)(base) + sizeof(struct pcpu) * cpu);
+	return ((char *)(base) + UMA_PCPU_ALLOC_SIZE * cpu);
 }
 
 /*