copperplate/heapobj-pshared: rebase on HEAPMEM algorithm

Address the issue mentioned in [1] regarding the shared memory
allocator, using a variant of the McKusick scheme significantly
improving the performance figures, especially regarding memory release
operations.

As a by-product of this overhaul, the shared memory allocator can now
manage heaps up to (4GB - PAGE_SIZE).

[1] http://www.xenomai.org/pipermail/xenomai/2018-April/038883.html

lib/copperplate/internal.h

@@ -32,46 +32,110 @@
 #include <copperplate/heapobj.h>
 #include <copperplate/tunables.h>
 
-#ifdef CONFIG_XENO_REGISTRY
-#define DEFAULT_REGISTRY_ROOT		CONFIG_XENO_REGISTRY_ROOT
-#else
-#define DEFAULT_REGISTRY_ROOT		NULL
-#endif
+#ifdef CONFIG_XENO_PSHARED
 
-#define HOBJ_PAGE_SHIFT	9 /* 2^9 => 512 bytes */
-#if HOBJ_PAGE_SHIFT > 21
-#error "page size is too large"
-#endif
+#include <boilerplate/shavl.h>
+
+#define SHEAPMEM_PAGE_SHIFT	9 /* 2^9 => 512 bytes */
+#define SHEAPMEM_PAGE_SIZE	(1UL << SHEAPMEM_PAGE_SHIFT)
+#define SHEAPMEM_PAGE_MASK	(~(SHEAPMEM_PAGE_SIZE - 1))
+#define SHEAPMEM_MIN_LOG2	4 /* 16 bytes */
+/*
+ * Use bucketed memory for sizes between 2^SHEAPMEM_MIN_LOG2 and
+ * 2^(SHEAPMEM_PAGE_SHIFT-1).
+ */
+#define SHEAPMEM_MAX		(SHEAPMEM_PAGE_SHIFT - SHEAPMEM_MIN_LOG2)
+#define SHEAPMEM_MIN_ALIGN	(1U << SHEAPMEM_MIN_LOG2)
+/* Max size of an extent (4Gb - SHEAPMEM_PAGE_SIZE). */
+#define SHEAPMEM_MAX_EXTSZ	(4294967295U - SHEAPMEM_PAGE_SIZE + 1)
+/* Bits we need for encoding a page # */
+#define SHEAPMEM_PGENT_BITS      (32 - SHEAPMEM_PAGE_SHIFT)
+
+/* Each page is represented by a page map entry. */
+#define SHEAPMEM_PGMAP_BYTES	sizeof(struct sheapmem_pgentry)
+
+struct sheapmem_pgentry {
+	/* Linkage in bucket list. */
+	unsigned int prev : SHEAPMEM_PGENT_BITS;
+	unsigned int next : SHEAPMEM_PGENT_BITS;
+	/*  page_list or log2. */
+	unsigned int type : 6;
+	/*
+	 * We hold either a spatial map of busy blocks within the page
+	 * for bucketed memory (up to 32 blocks per page), or the
+	 * overall size of the multi-page block if entry.type ==
+	 * page_list.
+	 */
+	union {
+		uint32_t map;
+		uint32_t bsize;
+	};
+};
 
-#define HOBJ_PAGE_SIZE	(1UL << HOBJ_PAGE_SHIFT)
-#define HOBJ_PAGE_MASK	(~(HOBJ_PAGE_SIZE-1))
+/*
+ * A range descriptor is stored at the beginning of the first page of
+ * a range of free pages. sheapmem_range.size is nrpages *
+ * SHEAPMEM_PAGE_SIZE. Ranges are indexed by address and size in AVL
+ * trees.
+ */
+struct sheapmem_range {
+	struct shavlh addr_node;
+	struct shavlh size_node;
+	size_t size;
+};
 
-#define HOBJ_MINLOG2    4	/* 16 bytes */
-/* +1 for holding HOBJ_PAGE_SIZE < x <= HOBJ_PAGE_SIZE * 2 */
-#define HOBJ_MAXLOG2    (HOBJ_PAGE_SHIFT + 1)
-#define HOBJ_NBUCKETS   (HOBJ_MAXLOG2 - HOBJ_MINLOG2 + 1)
-#define HOBJ_MINALIGNSZ (1U << HOBJ_MINLOG2)
+struct sheapmem_extent {
+	struct holder next;
+	memoff_t membase;	/* Base offset of page array */
+	memoff_t memlim;	/* Offset limit of page array */
+	struct shavl addr_tree;
+	struct shavl size_tree;
+	struct sheapmem_pgentry pagemap[0]; /* Start of page entries[] */
+};
 
-#define HOBJ_MAXEXTSZ   (1U << 31) /* 2Gb */
+#define __SHEAPMEM_MAP_SIZE(__nrpages)					\
+	((__nrpages) * SHEAPMEM_PGMAP_BYTES)
+
+#define __SHEAPMEM_ARENA_SIZE(__size)					\
+	(__size +							\
+	 __align_to(sizeof(struct sheapmem_extent) +			\
+		    __SHEAPMEM_MAP_SIZE((__size) >> SHEAPMEM_PAGE_SHIFT),	\
+		    SHEAPMEM_MIN_ALIGN))
+
+/*
+ * Calculate the minimal size of the memory arena needed to contain a
+ * heap of __user_size bytes, including our meta data for managing it.
+ * Usable at build time if __user_size is constant.
+ */
+#define SHEAPMEM_ARENA_SIZE(__user_size)					\
+	__SHEAPMEM_ARENA_SIZE(__align_to(__user_size, SHEAPMEM_PAGE_SIZE))
 
 /*
  * The struct below has to live in shared memory; no direct reference
  * to process local memory in there.
  */
-struct shared_heap {
+struct shared_heap_memory {
 	char name[XNOBJECT_NAME_LEN];
 	pthread_mutex_t lock;
 	struct listobj extents;
-	size_t ubytes;
-	size_t total;
-	size_t maxcont;
+	size_t arena_size;
+	size_t usable_size;
+	size_t used_size;
+	/* Heads of page lists for log2-sized blocks. */
+	uint32_t buckets[SHEAPMEM_MAX];
 	struct sysgroup_memspec memspec;
-	struct {
-		memoff_t freelist;
-		int fcount;
-	} buckets[HOBJ_NBUCKETS];
 };
 
+ssize_t sheapmem_check(struct shared_heap_memory *heap, void *block);
+
+#endif /* CONFIG_XENO_PSHARED */
+
+#ifdef CONFIG_XENO_REGISTRY
+#define DEFAULT_REGISTRY_ROOT		CONFIG_XENO_REGISTRY_ROOT
+#else
+#define DEFAULT_REGISTRY_ROOT		NULL
+#endif
+
 struct corethread_attributes {
 	size_t stacksize;
 	int detachstate;

testsuite/smokey/memory-pshared/pshared.c

@@ -16,7 +16,7 @@ smokey_test_plugin(memory_pshared,
 
 #define MIN_HEAP_SIZE  8192
 #define MAX_HEAP_SIZE  (1024 * 1024 * 2)
-#define RANDOM_ROUNDS  128
+#define RANDOM_ROUNDS  1024
 
 #define PATTERN_HEAP_SIZE  (128*1024)
 #define PATTERN_ROUNDS     128