 @@ -1,14 +1,14 @@
-/*	$NetBSD: subr_kmem.c,v 1.53 2014/06/23 17:43:42 maxv Exp $	*/
+/*	$NetBSD: subr_kmem.c,v 1.54 2014/06/24 07:28:23 maxv Exp $	*/
 /*-
  * Copyright (c) 2009 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Andrew Doran.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -62,28 +62,28 @@
 /*
  * This allocator has some debug features enabled with "option DEBUG" and
  * "option DIAGNOSTIC".
+ *
  * KMEM_POISON
  *	Try to detect modify-after-free bugs.
+ *
  *	Fill freed (in the sense of kmem_free) memory with a garbage pattern.
  *	Check the pattern on allocation.
+ *
  * KMEM_REDZONE
  *	Try to detect overrun bugs.
+ *
- *	Allocate some more bytes for each allocation.
+ *	Add a 2-byte pattern (allocate some more bytes if needed) at the end
- *	The extra bytes are checked by KMEM_POISON on kmem_free.
+ *	of each allocated buffer. Check this pattern on kmem_free.
+ *
  * KMEM_SIZE
  *	Try to detect alloc/free size mismatch bugs.
+ *
  *	Prefix each allocations with a fixed-sized header and record
  *	the exact user-requested allocation size in it.
  *	When freeing, compare it with kmem_free's "size" argument.
+ *
  * KMEM_GUARD
  *	See the below "kmguard" section.
  */
 /*
 @@ -93,27 +93,27 @@
  * See the comment in uvm/uvm_kmguard.c for what kind of bugs it tries to
  * detect.  Even if compiled in, it's disabled by default because it's very
  * expensive.  You can enable it on boot by:
+ *
  * 	boot -d
  * 	db> w kmem_guard_depth 0t30000
  * 	db> c
+ *
  * The default value of kmem_guard_depth is 0, which means disabled.
  * It can be changed by KMEM_GUARD_DEPTH kernel config option.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_kmem.c,v 1.53 2014/06/23 17:43:42 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_kmem.c,v 1.54 2014/06/24 07:28:23 maxv Exp $");
 #include <sys/param.h>
 #include <sys/callback.h>
 #include <sys/kmem.h>
 #include <sys/pool.h>
 #include <sys/debug.h>
 #include <sys/lockdebug.h>
 #include <sys/cpu.h>
 #include <uvm/uvm_extern.h>
 #include <uvm/uvm_map.h>
 #include <uvm/uvm_kmguard.h>
 @@ -189,29 +189,33 @@ static size_t kmem_cache_big_maxidx __re
 #define	KMEM_GUARD
 #endif /* defined(DEBUG) */
 #if defined(KMEM_POISON)
 static int kmem_poison_ctor(void *, void *, int);
 static void kmem_poison_fill(void *, size_t);
 static void kmem_poison_check(void *, size_t);
 #else /* defined(KMEM_POISON) */
 #define	kmem_poison_fill(p, sz)		/* nothing */
 #define	kmem_poison_check(p, sz)	/* nothing */
 #endif /* defined(KMEM_POISON) */
 #if defined(KMEM_REDZONE)
-#define	REDZONE_SIZE	1
+#define	REDZONE_SIZE	2
 static void kmem_redzone_fill(void *p, size_t sz);
 static void kmem_redzone_check(void *p, size_t sz);
 #else /* defined(KMEM_REDZONE) */
 #define	REDZONE_SIZE	0
 #define	kmem_redzone_fill(p, sz)		/* nothing */
 #define	kmem_redzone_check(p, sz)	/* nothing */
 #endif /* defined(KMEM_REDZONE) */
 #if defined(KMEM_SIZE)
 #define	SIZE_SIZE	(MAX(KMEM_ALIGN, sizeof(size_t)))
 static void kmem_size_set(void *, size_t);
 static void kmem_size_check(void *, size_t);
 #else
 #define	SIZE_SIZE	0
 #define	kmem_size_set(p, sz)	/* nothing */
 #define	kmem_size_check(p, sz)	/* nothing */
 #endif
 #if defined(KMEM_GUARD)
 @@ -238,52 +242,61 @@ kmem_intr_alloc(size_t requested_size, k
 	size_t size;
 	pool_cache_t pc;
 	uint8_t *p;
 	KASSERT(requested_size > 0);
 #ifdef KMEM_GUARD
 	if (requested_size <= kmem_guard_size) {
 		return uvm_kmguard_alloc(&kmem_guard, requested_size,
 		    (kmflags & KM_SLEEP) != 0);
+	}
 #endif
 	size = kmem_roundup_size(requested_size);
-	allocsz = size + REDZONE_SIZE + SIZE_SIZE;
+	allocsz = size + SIZE_SIZE;
 #ifdef KMEM_REDZONE
 	if (size - requested_size < REDZONE_SIZE) {
 		/* If there isn't enough space in the page padding,
 		 * allocate two more bytes for the red zone. */
 		allocsz += REDZONE_SIZE;
+	}
 #endif
 	if ((index = ((allocsz -1) >> KMEM_SHIFT))
 	    < kmem_cache_maxidx) {
 		pc = kmem_cache[index];
 	} else if ((index = ((allocsz - 1) >> KMEM_BIG_SHIFT))
             < kmem_cache_big_maxidx) {
 		pc = kmem_cache_big[index];
 	} else {
 		int ret = uvm_km_kmem_alloc(kmem_va_arena,
 		    (vsize_t)round_page(size),
 		    ((kmflags & KM_SLEEP) ? VM_SLEEP : VM_NOSLEEP)
 		     | VM_INSTANTFIT, (vmem_addr_t *)&p);
 		if (ret) {
 			return NULL;
+		}
 		FREECHECK_OUT(&kmem_freecheck, p);
 		return p;
+	}
 	p = pool_cache_get(pc, kmflags);
 	if (__predict_true(p != NULL)) {
 		kmem_poison_check(p, size);
 		FREECHECK_OUT(&kmem_freecheck, p);
 		kmem_size_set(p, requested_size);
 		kmem_redzone_fill(p, requested_size + SIZE_SIZE);
 		return p + SIZE_SIZE;
+	}
 	return p;
+}
 /*
  * kmem_intr_zalloc: allocate zeroed wired memory.
  */
 void *
 kmem_intr_zalloc(size_t size, km_flag_t kmflags)
+{
 @@ -306,48 +319,54 @@ kmem_intr_free(void *p, size_t requested
 	size_t allocsz, index;
 	size_t size;
 	pool_cache_t pc;
 	KASSERT(p != NULL);
 	KASSERT(requested_size > 0);
 #ifdef KMEM_GUARD
 	if (requested_size <= kmem_guard_size) {
 		uvm_kmguard_free(&kmem_guard, requested_size, p);
 		return;
+	}
 #endif
 	size = kmem_roundup_size(requested_size);
-	allocsz = size + REDZONE_SIZE + SIZE_SIZE;
+	allocsz = size + SIZE_SIZE;
 #ifdef KMEM_REDZONE
 	if (size - requested_size < REDZONE_SIZE) {
 		allocsz += REDZONE_SIZE;
+	}
 #endif
 	if ((index = ((allocsz -1) >> KMEM_SHIFT))
 	    < kmem_cache_maxidx) {
 		pc = kmem_cache[index];
 	} else if ((index = ((allocsz - 1) >> KMEM_BIG_SHIFT))
             < kmem_cache_big_maxidx) {
 		pc = kmem_cache_big[index];
 	} else {
 		FREECHECK_IN(&kmem_freecheck, p);
 		uvm_km_kmem_free(kmem_va_arena, (vaddr_t)p,
 		    round_page(size));
 		return;
+	}
 	p = (uint8_t *)p - SIZE_SIZE;
 	kmem_size_check(p, requested_size);
 	kmem_redzone_check(p, requested_size + SIZE_SIZE);
 	FREECHECK_IN(&kmem_freecheck, p);
 	LOCKDEBUG_MEM_CHECK(p, size);
 	kmem_poison_check((uint8_t *)p + SIZE_SIZE + size,
       	    allocsz - (SIZE_SIZE + size));
 	kmem_poison_fill(p, allocsz);
 	pool_cache_put(pc, p);
+}
 /* ---- kmem API */
 /*
  * kmem_alloc: allocate wired memory.
  * => must not be called from interrupt context.
  */
 void *
 @@ -455,40 +474,40 @@ kmem_init(void)
 	kmem_cache_maxidx = kmem_create_caches(kmem_cache_sizes,
 	    kmem_cache, KMEM_MAXSIZE, KMEM_SHIFT, IPL_VM);
        	kmem_cache_big_maxidx = kmem_create_caches(kmem_cache_big_sizes,
 	    kmem_cache_big, PAGE_SIZE, KMEM_BIG_SHIFT, IPL_VM);
+}
 size_t
 kmem_roundup_size(size_t size)
+{
 	return (size + (KMEM_ALIGN - 1)) & ~(KMEM_ALIGN - 1);
+}
-/* ---- debug */
+/* ------------------ DEBUG / DIAGNOSTIC ------------------ */
 #if defined(KMEM_POISON)
 #if defined(KMEM_POISON) || defined(KMEM_REDZONE)
 #if defined(_LP64)
 #define PRIME 0x9e37fffffffc0000UL
 #else /* defined(_LP64) */
 #define PRIME 0x9e3779b1
 #endif /* defined(_LP64) */
 #endif /* defined(KMEM_POISON) || defined(KMEM_REDZONE) */
 #if defined(KMEM_POISON)
 static inline uint8_t
 kmem_poison_pattern(const void *p)
+{
 	return (uint8_t)(((uintptr_t)p) * PRIME
 	   >> ((sizeof(uintptr_t) - sizeof(uint8_t))) * CHAR_BIT);
+}
 static int
 kmem_poison_ctor(void *arg, void *obj, int flag)
+{
 	size_t sz = (size_t)arg;
 	kmem_poison_fill(obj, sz);
 	return 0;
+}
 @@ -515,49 +534,90 @@ kmem_poison_check(void *p, size_t sz)
 	cp = p;
 	ep = cp + sz;
 	while (cp < ep) {
 		const uint8_t expected = kmem_poison_pattern(cp);
 		if (*cp != expected) {
 			panic("%s: %p: 0x%02x != 0x%02x\n",
 			   __func__, cp, *cp, expected);
+		}
 		cp++;
+	}
+}
 #endif /* defined(KMEM_POISON) */
 #if defined(KMEM_SIZE)
 static void
 kmem_size_set(void *p, size_t sz)
+{
 	memcpy(p, &sz, sizeof(sz));
+}
 static void
 kmem_size_check(void *p, size_t sz)
+{
 	size_t psz;
 	memcpy(&psz, p, sizeof(psz));
 	if (psz != sz) {
 		panic("kmem_free(%p, %zu) != allocated size %zu",
 		    (const uint8_t *)p + SIZE_SIZE, sz, psz);
+	}
+}
-#endif	/* defined(KMEM_SIZE) */
+#endif /* defined(KMEM_SIZE) */
 #if defined(KMEM_REDZONE)
 static inline uint8_t
 kmem_redzone_pattern(const void *p)
+{
 	return (uint8_t)(((uintptr_t)p) * PRIME
 	   >> ((sizeof(uintptr_t) - sizeof(uint8_t))) * CHAR_BIT);
+}
 static void
 kmem_redzone_fill(void *p, size_t sz)
+{
 	uint8_t *cp;
 	const uint8_t *ep;
 	cp = (uint8_t *)p + sz;
 	ep = cp + REDZONE_SIZE;
 	while (cp < ep) {
 		*cp = kmem_redzone_pattern(cp);
 		cp++;
+	}
+}
 static void
 kmem_redzone_check(void *p, size_t sz)
+{
 	uint8_t *cp;
 	const uint8_t *ep;
 	cp = (uint8_t *)p + sz;
 	ep = (uint8_t *)p + sz + REDZONE_SIZE;
 	while (cp < ep) {
 		const uint8_t expected = kmem_redzone_pattern(cp);
 		if (*cp != expected) {
 			panic("%s: %p: 0x%02x != 0x%02x\n",
 			   __func__, cp, *cp, expected);
+		}
 		cp++;
+	}
+}
 #endif /* defined(KMEM_REDZONE) */
 /*
  * Used to dynamically allocate string with kmem accordingly to format.
  */
 char *
 kmem_asprintf(const char *fmt, ...)
+{
 	int size __diagused, len;
 	va_list va;
 	char *str;
 	va_start(va, fmt);
 	len = vsnprintf(NULL, 0, fmt, va);