 @@ -1,115 +1,111 @@
-/*	$NetBSD: memchr.S,v 1.2 2009/07/18 18:06:56 dsl Exp $	*/
+/*
  * Written by J.T. Conklin <jtc@acorntoolworks.com>
-/*-
+ * Public domain.
  * Copyright (c) 2009 The NetBSD Foundation, Inc.
  * All rights reserved.
  * This code is derived from software contributed to The NetBSD Foundation
  * by David Laight.
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <machine/asm.h>
 #if defined(LIBC_SCCS)
-	RCSID("$NetBSD: memchr.S,v 1.2 2009/07/18 18:06:56 dsl Exp $")
+	RCSID("$NetBSD: memchr.S,v 1.3 2009/07/19 23:45:29 christos Exp $")
 #endif
 /*
  * The instruction sequences used try to avoid data dependencies
  * between adjacent instructions (to allow parallel execution).
  * The 'imul' for %r9 could be put into the delay following the
  * memory read (ie inside the loop) at no obvious cost - except
  * that the loop is currently exactly 32 bytes - 2 fetch blocks!.
  * I don't think aligning any of the other branch targets is useful.
  */
 ENTRY(memchr)
-	movabsq	$0x0101010101010101,%r8
+	movzbq	%sil,%rcx
 	lea	(%rdi,%rdx),%r10	/* limit of buffer to scan */
 	movzbq	%sil,%rsi	/* mask high bits! */
-	/* 'directpath' imuls can execute 3 at a time ... (amd) */
+	/*
-	imul	%r8,%rsi	/* search byte replicated in word */
+	 * Align to word boundary.
-	imul	$0x80,%r8,%r9	/* 0x8080808080808080 */
+	 * Consider unrolling loop?
-	test	$7,%dil
+	 */
-	jnz	20f		/* jump if misaligned */
+	testq	%rdx,%rdx	/* nbytes == 0? */
-	jmp	1f		/* jump to avoid 4 nops (13 bytes) in gap */
+	je	.Lzero
 .Lalign:
-	_ALIGN_TEXT		/* entire loop now in 32 aligned bytes */
+	testb	$7,%dil
-:
+	je	.Lword_aligned
-	cmpq	%r10,%rdi	/* end of buffer ? */
+	movq	%rdi,%rax
-	jae	30f		/* jump if so */
+	cmpb	(%rdi),%cl
 	je	.Ldone
 	incq	%rdi
 	decq	%rdx
 	jnz	.Lalign
 	jmp	.Lzero
 .Lword_aligned:
 	/* copy char to all bytes in word */
 	movb	%cl,%ch
 	movq	%rcx,%rsi
 	salq	$16,%rcx
 	orq	%rsi,%rcx
 	movq	%rcx,%rsi
 	salq	$32,%rcx
 	orq	%rsi,%rcx
-	movq	(%rdi),%rax	/* value to check */
+	movabsq	$0x0101010101010101,%r8
-:
+	movabsq	$0x8080808080808080,%r9
 	_ALIGN_TEXT
 .Lloop:
 	cmpq	$7,%rdx		/* nbytes > 8 */
 	jbe	.Lbyte
 	movq	(%rdi),%rsi
 	addq	$8,%rdi
-	xorq	%rsi,%rax	/* now looking for zeros */
+	xorq	%rcx,%rsi
-	mov	%rax,%rcx
+	subq	$8,%rdx
-	subq	%r8,%rax	/* x - 0x10 */
+	subq	%r8,%rsi
-	not	%rcx
+	testq	%r9,%rsi
-	andq	%r9,%rax	/* (x - 0x10) & 0x80 */
+	je	.Lloop
 	andq	%rcx,%rax	/* ((x - 0x10) & 0x80) ^ ~x */
-	je	1b		/* jump if not found */
+	/*
 	 * In rare cases, the above loop may exit prematurely. We must
-/* Found byte in word, get its address */
+	 * return to the loop if none of the bytes in the word are
-	bsf	%rax,%rax
+	 * equal to ch.
-	shr	$3,%eax
+	 */
 	lea	-8(%rax,%rdi),%rax
-	cmpq	%r10,%rax	/* need to check not beyond buffer */
+	leaq	-8(%rdi),%rax
-	jae	30f
+	cmpb	-8(%rdi),%cl	/* 1st byte == ch? */
-	rep
+	je	.Ldone
 	ret			/* amd - no ret after jmp */
 	leaq	-7(%rdi),%rax
-/* Input misaligned, read aligned and kill low bits */
+	cmpb	-7(%rdi),%cl	/* 2nd byte == ch? */
-/* (Getting a -1 is surprisingly hard work!) */
+	je	.Ldone
 :
-	xor	%eax,%eax	/* zeros all 64 bits */
+	leaq	-6(%rdi),%rax
-	mov	%dil,%cl	/* misalignment amount 1..7 (+high bits )*/
+	cmpb	-6(%rdi),%cl	/* 3rd byte == ch? */
-	test	%rdx,%rdx	/* zero length, don't read */
+	je	.Ldone
 	jz	30f
 	leaq	-5(%rdi),%rax
-	and	$~7,%dil	/* %rdi now start of word */
+	cmpb	-5(%rdi),%cl	/* 4th byte == ch? */
-	lea	-1(%rax),%r11	/* all 0xff */
+	je	.Ldone
 	and	$7,%cl		/* 1..7 */
 	leaq	-4(%rdi),%rax
-	mov	(%rdi),%rax	/* word containing first byte */
+	cmpb	-4(%rdi),%cl	/* 5th byte == ch? */
-	shl	$3,%cl		/* 8..56 */
+	je	.Ldone
 	test	%r11,%rsi	/* searching for 0xff */
-	jz	25f
+	leaq	-3(%rdi),%rax
 	cmpb	-3(%rdi),%cl	/* 6th byte == ch? */
-	/* Searching for other than 0xff, set low bytes */
+	je	.Ldone
 	shl	%cl,%r11	/* 0xff in high (wanted) bytes */
-	not	%r11		/* 0xff in low (unwanted) bytes */
+	leaq	-2(%rdi),%rax
-	or	%r11,%rax	/* low bytes now set */
+	cmpb	-2(%rdi),%cl	/* 7th byte == ch? */
-	jmp	2b
+	je	.Ldone
-:	/* Searching for 0xff, clear low bytes */
+	leaq	-1(%rdi),%rax
-	shl	%cl,%r11	/* 0xff in high (wanted) bytes */
+	cmpb	-1(%rdi),%cl	/* 7th byte == ch? */
-	and	%r11,%rax	/* low bytes now zero */
+	jne	.Lloop
-	jmp	2b
+	ret
 .Lbyte:
 	testq	%rdx,%rdx
 	je	.Lzero
 .Lbyte_loop:
 	movq	%rdi,%rax
 	cmpb	(%rdi),%cl
 	je	.Ldone
 	incq	%rdi
 	decq	%rdx
 	jnz	.Lbyte_loop
 .Lzero:
 	xorq	%rax,%rax
-/* Not found */
+.Ldone:
 :	xorq	%rax,%rax
 	ret

 @@ -1,153 +1,133 @@
-/*	$NetBSD: strchr.S,v 1.4 2009/07/18 16:40:31 dsl Exp $	*/
+/*
  * Written by J.T. Conklin <jtc@acorntoolworks.com>
-/*-
+ * Public domain.
  * Copyright (c) 2009 The NetBSD Foundation, Inc.
  * All rights reserved.
  * This code is derived from software contributed to The NetBSD Foundation
  * by David Laight.
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 /* See comments in strlen.S about checking words for byte values */
 #include <machine/asm.h>
 #if defined(LIBC_SCCS)
-	RCSID("$NetBSD: strchr.S,v 1.4 2009/07/18 16:40:31 dsl Exp $")
+	RCSID("$NetBSD: strchr.S,v 1.5 2009/07/19 23:45:29 christos Exp $")
 #endif
 /*
  * On entry %rdi is the buffer and the low byte of %rsi (%sil) the
  * character to search for.
  * Registers %rdx, %rcx, %r8-%r11 and %rax are also usable
  */
 /* Uncomment below to get regression test to run this version but
  * have everything else use the trivial one below. */
 /* #define TEST_STRCHR */
 #ifdef TEST_STRCHR
 ENTRY(test_strchr)
 #else
 ENTRY(strchr)
-#endif
+	movzbq	%sil,%rcx
 	movabsq	$0x0101010101010101,%r8
-	movzbq	%sil,%rdx	/* value to search for (c) */
+	/*
-	/* These imul are 'directpath' on athlons, so are fast */
+	 * Align to word boundary.
-	imul	$0x80,%r8,%r9	/* 0x8080808080808080 */
+	 * Consider unrolling loop?
-	imul	%r8,%rdx	/* (c) copied to all bytes */
+	 */
-	test	$7,%dil
+.Lalign:
-	jnz	20f		/* jump if misaligned */
+	testb	$7,%dil
 	je	.Lword_aligned
-	_ALIGN_TEXT		/* one byte nop */
+	movb	(%rdi),%dl
-:
+	cmpb	%cl,%dl
-	movq	(%rdi),%rax	/* bytes to check (x) */
+	je	.Ldone
-:
+	incq	%rdi
-	addq	$8,%rdi
+	testb	%dl,%dl
-	mov	%rax,%r10
+	jne	.Lalign
-	mov	%rax,%r11	/* for 'char' check */
+	jmp	.Lzero
 	not	%r10		/* invert of data (~x) */
 .Lword_aligned:
-	xorq	%rdx,%r11	/* convert 'char' test to one for NUL */
+	/* copy char to all bytes in word */
-	subq	%r8,%rax	/* x - 0x10 */
+	movb	%cl,%ch
-	movq	%r10,%rsi	/* ~x */
+	movq	%rcx,%rdx
-	subq	%r8,%r11	/* (x ^ c) - 0x10 */
+	salq	$16,%rcx
-/*
+	orq	%rdx,%rcx
- * Here we could check ((x - 0x10) | ((x ^ c) - 0x10)) & 0x80
+	movq	%rcx,%rdx
- * and short-circuit the case where no top bits are set, and
+	salq	$32,%rcx
- * we continue the loop.
+	orq	%rdx,%rcx
  * However it needs 3 more clocks that are difficult to interleave
  * in the existing dependency chain ...
  */
 	andq	%r9,%rax	/* (x - 0x10) & 0x80 */
 	xorq	%rdx,%rsi	/* c ^ ~x == ~(c ^ x) */
 	andq	%r9,%r11	/* ((x ^ c) - 0x10) & 0x80 */
 	andq	%r10,%rax	/* (x - 0x10) & 0x80 & ~x */
 	jne	10f		/* jump if string ends */
 	andq	%rsi,%r11	/* ((x ^ c) - 0x10) & 0x80 & ~(x ^ c) */
 	je	1b		/* jump if no match */
 	/* Found char, since LE can use bit scan */
 	bsf	%r11,%r11	/* 7, 15, 23 ... 63 */
 :	shr	$3,%r11		/* 0, 1, 2 .. 7 */
 	lea	-8(%r11,%rdi),%rax
 	ret
-/* End of string, check whether char is before NUL */
+	movabsq	$0x0101010101010101,%r8
-	_ALIGN_TEXT		/* adds three byte nop */
+	movabsq	$0x8080808080808080,%r9
 :
 	bsf	%rax,%rax	/* count to NUL */
 	andq	%rsi,%r11	/* check for char in last 8 bytes */
 	je	11f
 	bsf	%r11,%r11	/* NUL and char - see which was first */
 	cmp	%r11,%rax
 	jae	8b		/* return 'found' if same - searching for NUL */
 :	xor	%eax,%eax	/* char not found */
 	ret
-/* Source misaligned: read aligned word and make low bytes invalid */
+	/* Check whether any byte in the word is equal to ch or 0. */
-/* I (dsl) think a _ALIGN_TEXT here will slow things down! */
+	_ALIGN_TEXT
-:
+.Lloop:
-	xor	%rcx,%rcx
+	movq	(%rdi),%rdx
-	sub	%dil,%cl	/* Convert low address values 1..7 ... */
+	addq	$8,%rdi
-	sbb	%rsi,%rsi	/* carry was set, so %rsi now ~0u! */
+	movq	%rdx,%rsi
-	and	$7,%cl		/* ... to 7..1 */
+	subq	%r8,%rdx
-	and	$~7,%dil	/* move address to start of word */
+	xorq	%rcx,%rsi
-	shl	$3,%cl		/* now 56, 48 ... 16, 8 */
+	subq	%r8,%rsi
-	movq	(%rdi),%rax	/* aligned word containing first data */
+	orq	%rsi,%rdx
-	xor	%rdx,%rsi	/* invert of search pattern (~c) */
+	testq	%r9,%rdx
-	je	22f		/* searching for 0xff */
+	je	.Lloop
 :	shr	%cl,%rsi	/* ~c in low bytes */
-	or	%rsi,%rax	/* set some bits making low bytes invalid */
+	/*
-	jmp	2b
+	 * In rare cases, the above loop may exit prematurely. We must
 	 * return to the loop if none of the bytes in the word match
-/* We are searching for 0xff, so can't use ~pattern for invalid value */
+	 * ch or are equal to 0.
-:
+	 */
 	mov	%r8,%r10	/* 0x01 pattern */
-	lea	(%r8,%r8),%rsi	/* 0x02 - bits gets set (above) */
+	movb	-8(%rdi),%dl
-	not	%r10		/* now 0xfe */
+	cmpb	%cl,%dl		/* 1st byte == ch? */
-	sar	%cl,%r10	/* top bytes 0xff */
+	jne	1f
-	and	%r10,%rax	/* clear lsb from unwanted low bytes */
+	subq	$8,%rdi
-	jmp	21b
+	jmp	.Ldone
 :	testb	%dl,%dl		/* 1st byte == 0? */
 	je	.Lzero
 	movb	-7(%rdi),%dl
 	cmpb	%cl,%dl		/* 2nd byte == ch? */
 	jne	1f
 	subq	$7,%rdi
 	jmp	.Ldone
 :	testb	%dl,%dl		/* 2nd byte == 0? */
 	je	.Lzero
 	movb	-6(%rdi),%dl
 	cmpb	%cl,%dl		/* 3rd byte == ch? */
 	jne	1f
 	subq	$6,%rdi
 	jmp	.Ldone
 :	testb	%dl,%dl		/* 3rd byte == 0? */
 	je	.Lzero
 	movb	-5(%rdi),%dl
 	cmpb	%cl,%dl		/* 4th byte == ch? */
 	jne	1f
 	subq	$5,%rdi
 	jmp	.Ldone
 :	testb	%dl,%dl		/* 4th byte == 0? */
 	je	.Lzero
 	movb	-4(%rdi),%dl
 	cmpb	%cl,%dl		/* 5th byte == ch? */
 	jne	1f
 	subq	$4,%rdi
 	jmp	.Ldone
 :	testb	%dl,%dl		/* 5th byte == 0? */
 	je	.Lzero
 	movb	-3(%rdi),%dl
 	cmpb	%cl,%dl		/* 6th byte == ch? */
 	jne	1f
 	subq	$3,%rdi
 	jmp	.Ldone
 :	testb	%dl,%dl		/* 6th byte == 0? */
 	je	.Lzero
 	movb	-2(%rdi),%dl
 	cmpb	%cl,%dl		/* 7th byte == ch? */
 	jne	1f
 	subq	$2,%rdi
 	jmp	.Ldone
 :	testb	%dl,%dl		/* 7th byte == 0? */
 	je	.Lzero
 	movb	-1(%rdi),%dl
 	cmpb	%cl,%dl		/* 8th byte == ch? */
 	jne	1f
 	subq	$1,%rdi
 	jmp	.Ldone
 :	testb	%dl,%dl		/* 8th byte == 0? */
 	jne	.Lloop
 .Lzero:
 	/* If a ch wasn't found, return 0. */
 	xorq	%rdi,%rdi
-#ifdef TEST_STRCHR
+.Ldone:
-/* Trivial version for bug-fixing above */
+	movq	%rdi,%rax
 ENTRY(strchr)
 	movq	%rsi,%rdx
 	movq	%rdi,%rsi
 :
 	lodsb
 	cmp	%al,%dl
 	je	2f
 	test	%al,%al
 	jne	1b
 	xor	%eax,%eax
 	ret
 :	lea	-1(%rsi),%rax
 	ret
 #endif
 STRONG_ALIAS(index,strchr)

 @@ -1,20 +1,21 @@
 /*
  * Written by J.T. Conklin <jtc@NetBSD.org>.
  * Public domain.
  * Adapted for NetBSD/x86_64 by Frank van der Linden <fvdl@wasabisystems.com>
  */
 #include <machine/asm.h>
 #if defined(LIBC_SCCS)
-	RCSID("$NetBSD: ffs.S,v 1.2 2009/07/18 12:03:31 dsl Exp $")
+	RCSID("$NetBSD: ffs.S,v 1.3 2009/07/19 23:45:29 christos Exp $")
 #endif
 ENTRY(ffs)
 	bsfl	%edi,%eax
-	jz	1f	 		/* ZF is set if all bits are 0 */
+	jz	L1	 		/* ZF is set if all bits are 0 */
 	incl	%eax			/* bits numbered from 1, not 0 */
 	ret
-:	xorl	%eax,%eax		/* clear result */
+	_ALIGN_TEXT
 L1:	xorl	%eax,%eax		/* clear result */
 	ret