Add x86 AES-NI support.

Limited to amd64 for now.  In principle, AES-NI should work in 32-bit
mode, and there may even be some 32-bit-only CPUs that support
AES-NI, but that requires work to adapt the assembly.


(riastradh)

/*	$NetBSD: aes_ni.c,v 1.1 2020/06/29 23:29:40 riastradh Exp $	*/

/*-
 * Copyright (c) 2020 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(1, "$NetBSD: aes_ni.c,v 1.1 2020/06/29 23:29:40 riastradh Exp $");

#include <sys/types.h>
#include <sys/systm.h>

#include <crypto/aes/aes.h>
#include <crypto/aes/arch/x86/aes_ni.h>

#include <x86/cpuvar.h>
#include <x86/fpu.h>
#include <x86/specialreg.h>

static void
aesni_setenckey(struct aesenc *enc, const uint8_t key[static 16],
    uint32_t nrounds)
{

	switch (nrounds) {
	case 10:
		aesni_setenckey128(enc, key);
		break;
	case 12:
		aesni_setenckey192(enc, key);
		break;
	case 14:
		aesni_setenckey256(enc, key);
		break;
	default:
		panic("invalid AES rounds: %u", nrounds);
	}
}

static void
aesni_setenckey_impl(struct aesenc *enc, const uint8_t key[static 16],
    uint32_t nrounds)
{

	fpu_kern_enter();
	aesni_setenckey(enc, key, nrounds);
	fpu_kern_leave();
}

static void
aesni_setdeckey_impl(struct aesdec *dec, const uint8_t key[static 16],
    uint32_t nrounds)
{
	struct aesenc enc;

	fpu_kern_enter();
	aesni_setenckey(&enc, key, nrounds);
	aesni_enctodec(&enc, dec, nrounds);
	fpu_kern_leave();

	explicit_memset(&enc, 0, sizeof enc);
}

static void
aesni_enc_impl(const struct aesenc *enc, const uint8_t in[static 16],
    uint8_t out[static 16], uint32_t nrounds)
{

	fpu_kern_enter();
	aesni_enc(enc, in, out, nrounds);
	fpu_kern_leave();
}

static void
aesni_dec_impl(const struct aesdec *dec, const uint8_t in[static 16],
    uint8_t out[static 16], uint32_t nrounds)
{

	fpu_kern_enter();
	aesni_dec(dec, in, out, nrounds);
	fpu_kern_leave();
}

static void
aesni_cbc_enc_impl(const struct aesenc *enc, const uint8_t in[static 16],
    uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16],
    uint32_t nrounds)
{

	KASSERT(nbytes % 16 == 0);

	fpu_kern_enter();
	aesni_cbc_enc(enc, in, out, nbytes, iv, nrounds);
	fpu_kern_leave();
}

static void
aesni_cbc_dec_impl(const struct aesdec *dec, const uint8_t in[static 16],
    uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16],
    uint32_t nrounds)
{

	KASSERT(nbytes % 16 == 0);

	fpu_kern_enter();

	if (nbytes % 128) {
		aesni_cbc_dec1(dec, in, out, nbytes % 128, iv, nrounds);
		in += nbytes % 128;
		out += nbytes % 128;
		nbytes -= nbytes % 128;
	}

	KASSERT(nbytes % 128 == 0);
	if (nbytes)
		aesni_cbc_dec8(dec, in, out, nbytes, iv, nrounds);

	fpu_kern_leave();
}

static void
aesni_xts_enc_impl(const struct aesenc *enc, const uint8_t in[static 16],
    uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16],
    uint32_t nrounds)
{

	KASSERT(nbytes % 16 == 0);

	fpu_kern_enter();

	if (nbytes % 128) {
		aesni_xts_enc1(enc, in, out, nbytes % 128, iv, nrounds);
		in += nbytes % 128;
		out += nbytes % 128;
		nbytes -= nbytes % 128;
	}

	KASSERT(nbytes % 128 == 0);
	if (nbytes)
		aesni_xts_enc8(enc, in, out, nbytes, iv, nrounds);

	fpu_kern_leave();
}

static void
aesni_xts_dec_impl(const struct aesdec *dec, const uint8_t in[static 16],
    uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16],
    uint32_t nrounds)
{

	KASSERT(nbytes % 16 == 0);

	fpu_kern_enter();

	if (nbytes % 128) {
		aesni_xts_dec1(dec, in, out, nbytes % 128, iv, nrounds);
		in += nbytes % 128;
		out += nbytes % 128;
		nbytes -= nbytes % 128;
	}

	KASSERT(nbytes % 128 == 0);
	if (nbytes)
		aesni_xts_dec8(dec, in, out, nbytes, iv, nrounds);

	fpu_kern_leave();
}

static int
aesni_xts_update_selftest(void)
{
	static const struct {
		uint8_t	in[16], out[16];
	} cases[] = {
		{{1}, {2}},
		{{0,0,0,0x80}, {0,0,0,0,1}},
		{{0,0,0,0,0,0,0,0x80}, {0,0,0,0,0,0,0,0,1}},
		{{0,0,0,0x80,0,0,0,0x80}, {0,0,0,0,1,0,0,0,1}},
		{{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0x80}, {0x87}},
		{{0,0,0,0,0,0,0,0x80,0,0,0,0,0,0,0,0x80},
		 {0x87,0,0,0,0,0,0,0,1}},
		{{0,0,0,0x80,0,0,0,0,0,0,0,0,0,0,0,0x80}, {0x87,0,0,0,1}},
		{{0,0,0,0x80,0,0,0,0x80,0,0,0,0,0,0,0,0x80},
		 {0x87,0,0,0,1,0,0,0,1}},
	};
	unsigned i;
	uint8_t tweak[16];

	for (i = 0; i < sizeof(cases)/sizeof(cases[0]); i++) {
		aesni_xts_update(cases[i].in, tweak);
		if (memcmp(tweak, cases[i].out, 16))
			return -1;
	}

	/* Success!  */
	return 0;
}

static int
aesni_probe(void)
{
	int result = 0;

	/* Verify that the CPU supports AES-NI.  */
	if ((cpu_feature[1] & CPUID2_AES) == 0)
		return -1;

	fpu_kern_enter();

	/* Verify that our XTS tweak update logic works.  */
	if (aesni_xts_update_selftest())
		result = -1;

	fpu_kern_leave();

	return result;
}

struct aes_impl aes_ni_impl = {
	.ai_name = "Intel AES-NI",
	.ai_probe = aesni_probe,
	.ai_setenckey = aesni_setenckey_impl,
	.ai_setdeckey = aesni_setdeckey_impl,
	.ai_enc = aesni_enc_impl,
	.ai_dec = aesni_dec_impl,
	.ai_cbc_enc = aesni_cbc_enc_impl,
	.ai_cbc_dec = aesni_cbc_dec_impl,
	.ai_xts_enc = aesni_xts_enc_impl,
	.ai_xts_dec = aesni_xts_dec_impl,
};

/*	$NetBSD: aes_ni.h,v 1.1 2020/06/29 23:29:40 riastradh Exp $	*/

/*-
 * Copyright (c) 2020 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef	_CRYPTO_AES_ARCH_X86_AES_NI_H
#define	_CRYPTO_AES_ARCH_X86_AES_NI_H

#include <sys/types.h>

#include <crypto/aes/aes.h>

/* Assembly routines */

void	aesni_setenckey128(struct aesenc *, const uint8_t[static 16]);
void	aesni_setenckey192(struct aesenc *, const uint8_t[static 24]);
void	aesni_setenckey256(struct aesenc *, const uint8_t[static 32]);

void	aesni_enctodec(const struct aesenc *, struct aesdec *, uint32_t);

void	aesni_enc(const struct aesenc *, const uint8_t[static 16],
	    uint8_t[static 16], uint32_t);
void	aesni_dec(const struct aesdec *, const uint8_t[static 16],
	    uint8_t[static 16], uint32_t);

void	aesni_cbc_enc(const struct aesenc *, const uint8_t[static 16],
	    uint8_t[static 16], size_t, uint8_t[static 16], uint32_t);
void	aesni_cbc_dec1(const struct aesdec *, const uint8_t[static 16],
	    uint8_t[static 16], size_t, const uint8_t[static 16], uint32_t);
void	aesni_cbc_dec8(const struct aesdec *, const uint8_t[static 128],
	    uint8_t[static 128], size_t, const uint8_t[static 16], uint32_t);

void	aesni_xts_enc1(const struct aesenc *, const uint8_t[static 16],
	    uint8_t[static 16], size_t, uint8_t[static 16], uint32_t);
void	aesni_xts_enc8(const struct aesenc *, const uint8_t[static 128],
	    uint8_t[static 128], size_t, uint8_t[static 16], uint32_t);
void	aesni_xts_dec1(const struct aesdec *, const uint8_t[static 16],
	    uint8_t[static 16], size_t, uint8_t[static 16], uint32_t);
void	aesni_xts_dec8(const struct aesdec *, const uint8_t[static 128],
	    uint8_t[static 128], size_t, uint8_t[static 16], uint32_t);
void	aesni_xts_update(const uint8_t[static 16], uint8_t[static 16]);

extern struct aes_impl aes_ni_impl;

#endif	/* _CRYPTO_AES_ARCH_X86_AES_NI_H */

/*	$NetBSD: aes_ni_64.S,v 1.1 2020/06/29 23:29:40 riastradh Exp $	*/

/*-
 * Copyright (c) 2020 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <machine/asm.h>

/*
 * MOVDQA/MOVDQU are Move Double Quadword (Aligned/Unaligned), defined
 * to operate on integers; MOVAPS/MOVUPS are Move (Aligned/Unaligned)
 * Packed Single, defined to operate on binary32 floats.  They have
 * exactly the same architectural effects (move a 128-bit quantity from
 * memory into an xmm register).
 *
 * In principle, they might have different microarchitectural effects
 * so that MOVAPS/MOVUPS might incur a penalty when the register is
 * later used for integer paths, but in practice they don't.  So we use
 * the one whose instruction encoding is shorter -- MOVAPS/MOVUPS.
 */
#define	movdqa	movaps
#define	movdqu	movups

/*
 * aesni_setenckey128(struct aesenc *enckey@rdi, const uint8_t key[16] @rsi)
 *
 *	Expand a 16-byte AES-128 key into 10 round keys.
 *
 *	Standard ABI calling convention.
 */
ENTRY(aesni_setenckey128)
	movdqu	(%rsi),%xmm0	/* load master key into %xmm0 */
	movdqa	%xmm0,(%rdi)	/* store master key as the first round key */
	lea	0x10(%rdi),%rdi	/* advance %rdi to next round key */
	aeskeygenassist $0x1,%xmm0,%xmm2
	call	aesni_expand128
	aeskeygenassist $0x2,%xmm0,%xmm2
	call	aesni_expand128
	aeskeygenassist $0x4,%xmm0,%xmm2
	call	aesni_expand128
	aeskeygenassist $0x8,%xmm0,%xmm2
	call	aesni_expand128
	aeskeygenassist $0x10,%xmm0,%xmm2
	call	aesni_expand128
	aeskeygenassist $0x20,%xmm0,%xmm2
	call	aesni_expand128
	aeskeygenassist $0x40,%xmm0,%xmm2
	call	aesni_expand128
	aeskeygenassist $0x80,%xmm0,%xmm2
	call	aesni_expand128
	aeskeygenassist $0x1b,%xmm0,%xmm2
	call	aesni_expand128
	aeskeygenassist $0x36,%xmm0,%xmm2
	call	aesni_expand128
	ret
END(aesni_setenckey128)

/*
 * aesni_setenckey192(struct aesenc *enckey@rdi, const uint8_t key[24] @rsi)
 *
 *	Expand a 24-byte AES-192 key into 12 round keys.
 *
 *	Standard ABI calling convention.
 */
ENTRY(aesni_setenckey192)
	movdqu	(%rsi),%xmm0	/* load master key [0:128) into %xmm0 */
	movq	0x10(%rsi),%xmm1 /* load master key [128:192) into %xmm1 */
	movdqa	%xmm0,(%rdi)	/* store master key [0:128) as round key */
	lea	0x10(%rdi),%rdi /* advance %rdi to next round key */
	aeskeygenassist $0x1,%xmm1,%xmm2
	call	aesni_expand192a
	aeskeygenassist $0x2,%xmm0,%xmm2
	call	aesni_expand192b
	aeskeygenassist $0x4,%xmm1,%xmm2
	call	aesni_expand192a
	aeskeygenassist $0x8,%xmm0,%xmm2
	call	aesni_expand192b
	aeskeygenassist $0x10,%xmm1,%xmm2
	call	aesni_expand192a
	aeskeygenassist $0x20,%xmm0,%xmm2
	call	aesni_expand192b
	aeskeygenassist $0x40,%xmm1,%xmm2
	call	aesni_expand192a
	aeskeygenassist $0x80,%xmm0,%xmm2
	call	aesni_expand192b
	ret
END(aesni_setenckey192)

/*
 * aesni_setenckey256(struct aesenc *enckey@rdi, const uint8_t key[32] @rsi)
 *
 *	Expand a 32-byte AES-256 key into 14 round keys.
 *
 *	Standard ABI calling convention.
 */
ENTRY(aesni_setenckey256)
	movdqu	(%rsi),%xmm0	/* load master key [0:128) into %xmm0 */
	movdqu	0x10(%rsi),%xmm1 /* load master key [128:256) into %xmm1 */
	movdqa	%xmm0,(%rdi)	/* store master key [0:128) as round key */
	movdqa	%xmm1,0x10(%rdi) /* store master key [128:256) as round key */
	lea	0x20(%rdi),%rdi	/* advance %rdi to next round key */
	aeskeygenassist $0x1,%xmm1,%xmm2
	call	aesni_expand256a
	aeskeygenassist $0x1,%xmm0,%xmm2
	call	aesni_expand256b
	aeskeygenassist $0x2,%xmm1,%xmm2
	call	aesni_expand256a
	aeskeygenassist $0x2,%xmm0,%xmm2
	call	aesni_expand256b
	aeskeygenassist $0x4,%xmm1,%xmm2
	call	aesni_expand256a
	aeskeygenassist $0x4,%xmm0,%xmm2
	call	aesni_expand256b
	aeskeygenassist $0x8,%xmm1,%xmm2
	call	aesni_expand256a
	aeskeygenassist $0x8,%xmm0,%xmm2
	call	aesni_expand256b
	aeskeygenassist $0x10,%xmm1,%xmm2
	call	aesni_expand256a
	aeskeygenassist $0x10,%xmm0,%xmm2
	call	aesni_expand256b
	aeskeygenassist $0x20,%xmm1,%xmm2
	call	aesni_expand256a
	aeskeygenassist $0x20,%xmm0,%xmm2
	call	aesni_expand256b
	aeskeygenassist $0x40,%xmm1,%xmm2
	call	aesni_expand256a
	ret
END(aesni_setenckey256)

/*
 * aesni_expand128(uint128_t *rkp@rdi, uint128_t prk@xmm0,
 *     uint128_t keygenassist@xmm2)
 *
 *	1. Compute the AES-128 round key using the previous round key.
 *	2. Store it at *rkp.
 *	3. Set %xmm0 to it.
 *	4. Advance %rdi to point at the next round key.
 *
 *	Internal ABI.  On entry:
 *
 *		%rdi = rkp, pointer to round key to compute
 *		%xmm0 = (prk[0], prk[1], prk[2], prk[3])
 *		%xmm2 = (xxx, xxx, xxx, t = Rot(SubWord(prk[3])) ^ RCON)
 *
 *	On exit:
 *
 *		%rdi = &rkp[1], rkp advanced by one round key
 *		%xmm0 = rk, the round key we just computed
 *		%xmm2 = garbage
 *		%xmm4 = garbage
 *		%xmm5 = garbage
 *		%xmm6 = garbage
 *
 *	Note: %xmm1 is preserved (as are %xmm3 and %xmm7 through %xmm15,
 *	and all other registers).
 */
	.text
	_ALIGN_TEXT
	.type	aesni_expand128,@function
aesni_expand128:
	/*
	 * %xmm2 := (%xmm2[3], %xmm2[3], %xmm2[3], %xmm2[3]),
	 * i.e., set each word of %xmm2 to t := Rot(SubWord(prk[3])) ^ RCON.
	 */
	pshufd	$0b11111111,%xmm2,%xmm2

	/*
	 * %xmm4 := (0, prk[0], prk[1], prk[2])
	 * %xmm5 := (0, 0, prk[0], prk[1])
	 * %xmm6 := (0, 0, 0, prk[0])
	 */
	movdqa	%xmm0,%xmm4
	movdqa	%xmm0,%xmm5
	movdqa	%xmm0,%xmm6
	pslldq	$4,%xmm4
	pslldq	$8,%xmm5
	pslldq	$12,%xmm6

	/*
	 * %xmm0 := (rk[0] = t ^ prk[0],
	 *     rk[1] = t ^ prk[0] ^ prk[1],
	 *     rk[2] = t ^ prk[0] ^ prk[1] ^ prk[2],
	 *     rk[3] = t ^ prk[0] ^ prk[1] ^ prk[2] ^ prk[3])
	 */
	pxor	%xmm2,%xmm0
	pxor	%xmm4,%xmm0
	pxor	%xmm5,%xmm0
	pxor	%xmm6,%xmm0

	movdqa	%xmm0,(%rdi)	/* store round key */
	lea	0x10(%rdi),%rdi	/* advance to next round key address */
	ret
END(aesni_expand128)

/*
 * aesni_expand192a(uint128_t *rkp@rdi, uint128_t prk@xmm0,
 *     uint64_t rklo@xmm1, uint128_t keygenassist@xmm2)
 *
 *	Set even-numbered AES-192 round key.
 *
 *	Internal ABI.  On entry:
 *
 *		%rdi = rkp, pointer to two round keys to compute
 *		%xmm0 = (prk[0], prk[1], prk[2], prk[3])
 *		%xmm1 = (rklo[0], rklo[1], xxx, xxx)
 *		%xmm2 = (xxx, t = Rot(SubWord(rklo[1])) ^ RCON, xxx, xxx)
 *
 *	On exit:
 *
 *		%rdi = &rkp[2], rkp advanced by two round keys
 *		%xmm0 = nrk, second round key we just computed
 *		%xmm1 = rk, first round key we just computed
 *		%xmm2 = garbage
 *		%xmm4 = garbage
 *		%xmm5 = garbage
 *		%xmm6 = garbage
 *		%xmm7 = garbage
 */
	.text
	_ALIGN_TEXT
	.type	aesni_expand192a,@function
aesni_expand192a:
	/*
	 * %xmm2 := (%xmm2[1], %xmm2[1], %xmm2[1], %xmm2[1]),
	 * i.e., set each word of %xmm2 to t := Rot(SubWord(rklo[1])) ^ RCON.
	 */
	pshufd	$0b01010101,%xmm2,%xmm2

	/*
	 * We need to compute:
	 *
	 * rk[0] := rklo[0]
	 * rk[1] := rklo[1]
	 * rk[2] := Rot(Sub(rklo[1])) ^ RCON ^ prk[0]
	 * rk[3] := Rot(Sub(rklo[1])) ^ RCON ^ prk[0] ^ prk[1]
	 * nrk[0] := Rot(Sub(rklo[1])) ^ RCON ^ prk[0] ^ prk[1] ^ prk[2]
	 * nrk[1] := Rot(Sub(rklo[1])) ^ RCON ^ prk[0] ^ ... ^ prk[3]
	 * nrk[2] := Rot(Sub(rklo[1])) ^ RCON ^ prk[0] ^ ... ^ prk[3] ^ rklo[0]
	 * nrk[3] := Rot(Sub(rklo[1])) ^ RCON ^ prk[0] ^ ... ^ prk[3] ^ rklo[0]
	 *     ^ rklo[1]
	 */

	/*
	 * %xmm4 := (prk[0], prk[1], prk[2], prk[3])
	 * %xmm5 := (0, prk[0], prk[1], prk[2])
	 * %xmm6 := (0, 0, prk[0], prk[1])
	 * %xmm7 := (0, 0, 0, prk[0])
	 */
	movdqa	%xmm0,%xmm4
	movdqa	%xmm0,%xmm5
	movdqa	%xmm0,%xmm6
	movdqa	%xmm0,%xmm7
	pslldq	$4,%xmm5
	pslldq	$8,%xmm6
	pslldq	$12,%xmm7

	/* %xmm4 := (rk[2], rk[3], nrk[0], nrk[1]) */
	pxor	%xmm2,%xmm4
	pxor	%xmm5,%xmm4
	pxor	%xmm6,%xmm4
	pxor	%xmm7,%xmm4

	/*
	 * At this point, rk is split across %xmm1 (rk[0],rk[1],...) and
	 * %xmm4 (rk[2],rk[3],...); nrk is in %xmm4 (...,nrk[0],nrk[1]);
	 * and we have yet to compute nrk[2] or nrk[3], which requires
	 * rklo[0] and rklo[1] in %xmm1 (rklo[0], rklo[1], ...).  We need
	 * nrk to end up in %xmm0 at the end, so gather rk into %xmm1 and
	 * nrk into %xmm0.
	 */

	/* %xmm0 := (nrk[0], nrk[1], nrk[1], nrk[1]) */
	pshufd	$0b11111110,%xmm4,%xmm0

	/*
	 * %xmm6 := (0, 0, rklo[0], rklo[1])
	 * %xmm7 := (0, 0, 0, rklo[0])
	 */
	movdqa	%xmm1,%xmm6
	movdqa	%xmm1,%xmm7

	pslldq	$8,%xmm6
	pslldq	$12,%xmm7

	/*
	 * %xmm0 := (nrk[0],
	 *     nrk[1],
	 *     nrk[2] = nrk[1] ^ rklo[0],
	 *     nrk[3] = nrk[1] ^ rklo[0] ^ rklo[1])
	 */
	pxor	%xmm6,%xmm0
	pxor	%xmm7,%xmm0

	/* %xmm1 := (rk[0], rk[1], rk[2], rk[3]) */
	shufps	$0b01000100,%xmm4,%xmm1

	movdqa	%xmm1,(%rdi)		/* store round key */
	movdqa	%xmm0,0x10(%rdi)	/* store next round key */
	lea	0x20(%rdi),%rdi		/* advance two round keys */
	ret
END(aesni_expand192a)

/*
 * aesni_expand192b(uint128_t *roundkey@rdi, uint128_t prk@xmm0,
 *     uint128_t keygenassist@xmm2)
 *
 *	Set odd-numbered AES-192 round key.
 *
 *	Internal ABI.  On entry:
 *
 *		%rdi = rkp, pointer to round key to compute
 *		%xmm0 = (prk[0], prk[1], prk[2], prk[3])
 *		%xmm1 = (xxx, xxx, pprk[2], pprk[3])
 *		%xmm2 = (xxx, xxx, xxx, t = Rot(Sub(prk[3])) ^ RCON)
 *
 *	On exit:
 *
 *		%rdi = &rkp[1], rkp advanced by one round key
 *		%xmm0 = rk, the round key we just computed
 *		%xmm1 = (nrk[0], nrk[1], xxx, xxx), half of next round key
 *		%xmm2 = garbage
 *		%xmm4 = garbage
 *		%xmm5 = garbage
 *		%xmm6 = garbage
 *		%xmm7 = garbage
 */
	.text
	_ALIGN_TEXT
	.type	aesni_expand192b,@function
aesni_expand192b:
	/*
	 * %xmm2 := (%xmm2[3], %xmm2[3], %xmm2[3], %xmm2[3]),
	 * i.e., set each word of %xmm2 to t := Rot(Sub(prk[3])) ^ RCON.
	 */
	pshufd	$0b11111111,%xmm2,%xmm2

	/*
	 * We need to compute:
	 *
	 * rk[0] := Rot(Sub(prk[3])) ^ RCON ^ pprk[2]
	 * rk[1] := Rot(Sub(prk[3])) ^ RCON ^ pprk[2] ^ pprk[3]
	 * rk[2] := Rot(Sub(prk[3])) ^ RCON ^ pprk[2] ^ pprk[3] ^ prk[0]
	 * rk[3] := Rot(Sub(prk[3])) ^ RCON ^ pprk[2] ^ pprk[3] ^ prk[0]
	 *     ^ prk[1]
	 * nrk[0] := Rot(Sub(prk[3])) ^ RCON ^ pprk[2] ^ pprk[3] ^ prk[0]
	 *     ^ prk[1] ^ prk[2]
	 * nrk[1] := Rot(Sub(prk[3])) ^ RCON ^ pprk[2] ^ pprk[3] ^ prk[0]
	 *     ^ prk[1] ^ prk[2] ^ prk[3]
	 */

	/* %xmm1 := (pprk[2], pprk[3], prk[0], prk[1]) */
	shufps	$0b01001110,%xmm0,%xmm1

	/*
	 * %xmm5 := (0, pprk[2], pprk[3], prk[0])
	 * %xmm6 := (0, 0, pprk[2], pprk[3])
	 * %xmm7 := (0, 0, 0, pprk[2])
	 */
	movdqa	%xmm1,%xmm5
	movdqa	%xmm1,%xmm6
	movdqa	%xmm1,%xmm7
	pslldq	$4,%xmm5
	pslldq	$8,%xmm6
	pslldq	$12,%xmm7

	/* %xmm1 := (rk[0], rk[1], rk[2], rk[3) */
	pxor	%xmm2,%xmm1
	pxor	%xmm5,%xmm1
	pxor	%xmm6,%xmm1
	pxor	%xmm7,%xmm1

	/* %xmm4 := (prk[2], prk[3], xxx, xxx) */
	pshufd	$0b00001110,%xmm0,%xmm4

	/* %xmm5 := (0, prk[2], xxx, xxx) */
	movdqa	%xmm4,%xmm5
	pslldq	$4,%xmm5

	/* %xmm0 := (rk[0], rk[1], rk[2], rk[3]) */
	movdqa	%xmm1,%xmm0

	/* %xmm1 := (rk[3], rk[3], xxx, xxx) */
	shufps	$0b00001111,%xmm1,%xmm1

	/*
	 * %xmm1 := (nrk[0] = rk[3] ^ prk[2],
	 *     nrk[1] = rk[3] ^ prk[2] ^ prk[3],
	 *     xxx,
	 *     xxx)
	 */
	pxor	%xmm4,%xmm1
	pxor	%xmm5,%xmm1

	movdqa	%xmm0,(%rdi)	/* store round key */
	lea	0x10(%rdi),%rdi	/* advance to next round key address */
	ret
END(aesni_expand192b)

/*
 * aesni_expand256a(uint128_t *rkp@rdi, uint128_t pprk@xmm0,
 *     uint128_t prk@xmm1, uint128_t keygenassist@xmm2)
 *
 *	Set even-numbered AES-256 round key.
 *
 *	Internal ABI.  On entry:
 *
 *		%rdi = rkp, pointer to round key to compute
 *		%xmm0 = (pprk[0], pprk[1], pprk[2], pprk[3])
 *		%xmm1 = (prk[0], prk[1], prk[2], prk[3])
 *		%xmm2 = (xxx, xxx, xxx, t = Rot(SubWord(prk[3])))
 *
 *	On exit:
 *
 *		%rdi = &rkp[1], rkp advanced by one round key
 *		%xmm0 = rk, the round key we just computed
 *		%xmm1 = prk, previous round key, preserved from entry
 *		%xmm2 = garbage
 *		%xmm4 = garbage
 *		%xmm5 = garbage
 *		%xmm6 = garbage
 *
 *	The computation turns out to be the same as for AES-128; the
 *	previous round key does not figure into it, only the
 *	previous-previous round key.
 */
	aesni_expand256a = aesni_expand128

/*
 * aesni_expand256b(uint128_t *rkp@rdi, uint128_t prk@xmm0,
 *     uint128_t pprk@xmm1, uint128_t keygenassist@xmm2)
 *
 *	Set odd-numbered AES-256 round key.
 *
 *	Internal ABI.  On entry:
 *
 *		%rdi = rkp, pointer to round key to compute
 *		%xmm0 = (prk[0], prk[1], prk[2], prk[3])
 *		%xmm1 = (pprk[0], pprk[1], pprk[2], pprk[3])
 *		%xmm2 = (xxx, xxx, t = Sub(prk[3]), xxx)
 *
 *	On exit:
 *
 *		%rdi = &rkp[1], rkp advanced by one round key
 *		%xmm0 = prk, previous round key, preserved from entry
 *		%xmm1 = rk, the round key we just computed
 *		%xmm2 = garbage
 *		%xmm4 = garbage
 *		%xmm5 = garbage
 *		%xmm6 = garbage
 */
	.text
	_ALIGN_TEXT
	.type	aesni_expand256b,@function
aesni_expand256b:
	/*
	 * %xmm2 := (%xmm2[3], %xmm2[3], %xmm2[3], %xmm2[3]),
	 * i.e., set each word of %xmm2 to t := Sub(prk[3]).
	 */
	pshufd	$0b10101010,%xmm2,%xmm2

	/*
	 * %xmm4 := (0, pprk[0], pprk[1], pprk[2])
	 * %xmm5 := (0, 0, pprk[0], pprk[1])
	 * %xmm6 := (0, 0, 0, pprk[0])
	 */
	movdqa	%xmm1,%xmm4
	movdqa	%xmm1,%xmm5
	movdqa	%xmm1,%xmm6
	pslldq	$4,%xmm4
	pslldq	$8,%xmm5
	pslldq	$12,%xmm6

	/*
	 * %xmm0 := (rk[0] = t ^ pprk[0],
	 *     rk[1] = t ^ pprk[0] ^ pprk[1],
	 *     rk[2] = t ^ pprk[0] ^ pprk[1] ^ pprk[2],
	 *     rk[3] = t ^ pprk[0] ^ pprk[1] ^ pprk[2] ^ pprk[3])
	 */
	pxor	%xmm2,%xmm1
	pxor	%xmm4,%xmm1
	pxor	%xmm5,%xmm1
	pxor	%xmm6,%xmm1

	movdqa	%xmm1,(%rdi)	/* store round key */
	lea	0x10(%rdi),%rdi	/* advance to next round key address */
	ret
END(aesni_expand256b)

/*
 * aesni_enctodec(const struct aesenc *enckey@rdi, struct aesdec *deckey@rsi,
 *     uint32_t nrounds@rdx)
 *
 *	Convert AES encryption round keys to AES decryption round keys.
 *	`rounds' must be between 10 and 14.
 *
 *	Standard ABI calling convention.
 */
ENTRY(aesni_enctodec)
	shl	$4,%edx		/* rdx := byte offset of last round key */
	movdqa	(%rdi,%rdx),%xmm0	/* load last round key */
	movdqa	%xmm0,(%rsi)	/* store last round key verbatim */
1:	sub	$0x10,%rdx	/* advance to next round key */
	lea	0x10(%rsi),%rsi
	jz	2f		/* stop if this is the last one */
	movdqa	(%rdi,%rdx),%xmm0	/* load round key */
	aesimc	%xmm0,%xmm0	/* convert encryption to decryption */
	movdqa	%xmm0,(%rsi)	/* store round key */
	jmp	1b
2:	movdqa	(%rdi),%xmm0	/* load first round key */
	movdqa	%xmm0,(%rsi)	/* store first round key verbatim */
	ret
END(aesni_enctodec)

/*
 * aesni_enc(const struct aesenc *enckey@rdi, const uint8_t in[16] @rsi,
 *     uint8_t out[16] @rdx, uint32_t nrounds@ecx)
 *
 *	Encrypt a single block.
 *
 *	Standard ABI calling convention.
 */
ENTRY(aesni_enc)
	movdqu	(%rsi),%xmm0
	call	aesni_enc1
	movdqu	%xmm0,(%rdx)
	ret
END(aesni_enc)

/*
 * aesni_dec(const struct aesdec *deckey@rdi, const uint8_t in[16] @rsi,
 *     uint8_t out[16] @rdx, uint32_t nrounds@ecx)
 *
 *	Decrypt a single block.
 *
 *	Standard ABI calling convention.
 */
ENTRY(aesni_dec)
	movdqu	(%rsi),%xmm0
	call	aesni_dec1
	movdqu	%xmm0,(%rdx)
	ret
END(aesni_dec)

/*
 * aesni_cbc_enc(const struct aesenc *enckey@rdi, const uint8_t *in@rsi,
 *     uint8_t *out@rdx, size_t nbytes@rcx, uint8_t iv[16] @r8,
 *     uint32_t nrounds@r9d)
 *
 *	Encrypt a contiguous sequence of blocks with AES-CBC.
 *
 *	nbytes must be an integral multiple of 16.
 *
 *	Standard ABI calling convention.
 */
ENTRY(aesni_cbc_enc)
	cmp	$0,%rcx
	jz	2f
	mov	%rcx,%r10		/* r10 := nbytes */
	movdqu	(%r8),%xmm0		/* xmm0 := chaining value */
1:	movdqu	(%rsi),%xmm1		/* xmm1 := plaintext block */
	lea	0x10(%rsi),%rsi
	pxor	%xmm1,%xmm0		/* xmm0 := cv ^ ptxt */
	mov	%r9d,%ecx		/* ecx := nrounds */
	call	aesni_enc1		/* xmm0 := ciphertext block */
	movdqu	%xmm0,(%rdx)
	lea	0x10(%rdx),%rdx
	sub	$0x10,%r10
	jnz	1b			/* repeat if r10 is nonzero */
	movdqu	%xmm0,(%r8)		/* store chaining value */
2:	ret
END(aesni_cbc_enc)

/*
 * aesni_cbc_dec1(const struct aesdec *deckey@rdi, const uint8_t *in@rsi,
 *     uint8_t *out@rdx, size_t nbytes@rcx, const uint8_t iv[16] @r8,
 *     uint32_t nrounds@r9)
 *
 *	Decrypt a contiguous sequence of blocks with AES-CBC.
 *
 *	nbytes must be a positive integral multiple of 16.  This routine
 *	is not vectorized; use aesni_cbc_dec8 for >=8 blocks at once.
 *
 *	Standard ABI calling convention.
 */
ENTRY(aesni_cbc_dec1)
	push	%rbp			/* create stack frame uint128[1] */
	mov	%rsp,%rbp
	sub	$0x10,%rsp
	movdqu	(%r8),%xmm8		/* xmm8 := iv */
	movdqa	%xmm8,(%rsp)		/* save iv */
	mov	%rcx,%r10		/* r10 := nbytes */
	movdqu	-0x10(%rsi,%r10),%xmm0	/* xmm0 := last ciphertext block */
	movdqu	%xmm0,(%r8)		/* update iv */
1:	mov	%r9d,%ecx		/* ecx := nrounds */
	call	aesni_dec1		/* xmm0 := cv ^ ptxt */
	sub	$0x10,%r10
	jz	2f			/* first block if r10 is now zero */
	movdqu	-0x10(%rsi,%r10),%xmm8	/* xmm8 := chaining value */
	pxor	%xmm8,%xmm0		/* xmm0 := ptxt */
	movdqu	%xmm0,(%rdx,%r10)	/* store plaintext block */
	movdqa	%xmm8,%xmm0		/* move cv = ciphertext block */
	jmp	1b
2:	pxor	(%rsp),%xmm0		/* xmm0 := ptxt */
	movdqu	%xmm0,(%rdx)		/* store first plaintext block */
	leave
	ret
END(aesni_cbc_dec1)

/*
 * aesni_cbc_dec8(const struct aesdec *deckey@rdi, const uint8_t *in@rsi,
 *     uint8_t *out@rdx, size_t nbytes@rcx, const uint8_t iv[16] @r8,
 *     uint32_t nrounds@r9)
 *
 *	Decrypt a contiguous sequence of 8-block units with AES-CBC.
 *
 *	nbytes must be a positive integral multiple of 128.
 *
 *	Standard ABI calling convention.
 */
ENTRY(aesni_cbc_dec8)
	push	%rbp			/* create stack frame uint128[1] */
	mov	%rsp,%rbp
	sub	$0x10,%rsp
	movdqu	(%r8),%xmm8		/* xmm8 := iv */
	movdqa	%xmm8,(%rsp)		/* save iv */
	mov	%rcx,%r10		/* r10 := nbytes */
	movdqu	-0x10(%rsi,%r10),%xmm7	/* xmm7 := ciphertext block[n-1] */
	movdqu	%xmm7,(%r8)		/* update iv */
1:	movdqu	-0x20(%rsi,%r10),%xmm6	/* xmm6 := ciphertext block[n-2] */
	movdqu	-0x30(%rsi,%r10),%xmm5	/* xmm5 := ciphertext block[n-3] */
	movdqu	-0x40(%rsi,%r10),%xmm4	/* xmm4 := ciphertext block[n-4] */
	movdqu	-0x50(%rsi,%r10),%xmm3	/* xmm3 := ciphertext block[n-5] */
	movdqu	-0x60(%rsi,%r10),%xmm2	/* xmm2 := ciphertext block[n-6] */
	movdqu	-0x70(%rsi,%r10),%xmm1	/* xmm1 := ciphertext block[n-7] */
	movdqu	-0x80(%rsi,%r10),%xmm0	/* xmm0 := ciphertext block[n-8] */
	movdqa	%xmm6,%xmm15		/* xmm[8+i] := cv[i], 0<i<8 */
	movdqa	%xmm5,%xmm14
	movdqa	%xmm4,%xmm13
	movdqa	%xmm3,%xmm12
	movdqa	%xmm2,%xmm11
	movdqa	%xmm1,%xmm10
	movdqa	%xmm0,%xmm9
	mov	%r9d,%ecx		/* ecx := nrounds */
	call	aesni_dec8		/* xmm[i] := cv[i] ^ ptxt[i], 0<=i<8 */
	pxor	%xmm15,%xmm7		/* xmm[i] := ptxt[i], 0<i<8 */
	pxor	%xmm14,%xmm6
	pxor	%xmm13,%xmm5
	pxor	%xmm12,%xmm4
	pxor	%xmm11,%xmm3
	pxor	%xmm10,%xmm2
	pxor	%xmm9,%xmm1
	movdqu	%xmm7,-0x10(%rdx,%r10)	/* store plaintext blocks */
	movdqu	%xmm6,-0x20(%rdx,%r10)
	movdqu	%xmm5,-0x30(%rdx,%r10)
	movdqu	%xmm4,-0x40(%rdx,%r10)
	movdqu	%xmm3,-0x50(%rdx,%r10)
	movdqu	%xmm2,-0x60(%rdx,%r10)
	movdqu	%xmm1,-0x70(%rdx,%r10)
	sub	$0x80,%r10
	jz	2f			/* first block if r10 is now zero */
	movdqu	-0x10(%rsi,%r10),%xmm7	/* xmm7 := cv[0] */
	pxor	%xmm7,%xmm0		/* xmm0 := ptxt[0] */
	movdqu	%xmm0,(%rdx,%r10)	/* store plaintext block */
	jmp	1b
2:	pxor	(%rsp),%xmm0		/* xmm0 := ptxt[0] */
	movdqu	%xmm0,(%rdx)		/* store first plaintext block */
	leave
	ret
END(aesni_cbc_dec8)

/*
 * aesni_xts_enc1(const struct aesenc *enckey@rdi, const uint8_t *in@rsi,
 *     uint8_t *out@rdx, size_t nbytes@rcx, uint8_t tweak[16] @r8,
 *     uint32_t nrounds@r9d)
 *
 *	Encrypt a contiguous sequence of blocks with AES-XTS.
 *
 *	nbytes must be a positive integral multiple of 16.  This routine
 *	is not vectorized; use aesni_xts_enc8 for >=8 blocks at once.
 *
 *	Standard ABI calling convention.
 */
ENTRY(aesni_xts_enc1)
	mov	%rcx,%r10		/* r10 := nbytes */
	movdqu	(%r8),%xmm15		/* xmm15 := tweak */
1:	movdqu	(%rsi),%xmm0		/* xmm0 := ptxt */
	lea	0x10(%rsi),%rsi		/* advance rdi to next block */
	pxor	%xmm15,%xmm0		/* xmm0 := ptxt ^ tweak */
	mov	%r9d,%ecx		/* ecx := nrounds */
	call	aesni_enc1		/* xmm0 := AES(ptxt ^ tweak) */
	pxor	%xmm15,%xmm0		/* xmm0 := AES(ptxt ^ tweak) ^ tweak */
	movdqu	%xmm0,(%rdx)		/* store ciphertext block */
	lea	0x10(%rdx),%rdx		/* advance rsi to next block */
	call	aesni_xts_mulx		/* xmm15 *= x; trash xmm0 */
	sub	$0x10,%r10
	jnz	1b			/* repeat if more blocks */
	movdqu	%xmm15,(%r8)		/* update tweak */
	ret
END(aesni_xts_enc1)

/*
 * aesni_xts_enc8(const struct aesenc *enckey@rdi, const uint8_t *in@rsi,
 *     uint8_t *out@rdx, size_t nbytes@rcx, uint8_t tweak[16] @r8,
 *     uint32_t nrounds@r9d)
 *
 *	Encrypt a contiguous sequence of blocks with AES-XTS.
 *
 *	nbytes must be a positive integral multiple of 128.
 *
 *	Standard ABI calling convention.
 */
ENTRY(aesni_xts_enc8)
	push	%rbp			/* create stack frame uint128[1] */
	mov	%rsp,%rbp
	sub	$0x10,%rsp
	mov	%rcx,%r10		/* r10 := nbytes */
	movdqu	(%r8),%xmm15		/* xmm15 := tweak[0] */
1:	movdqa	%xmm15,%xmm8		/* xmm8 := tweak[0] */
	call	aesni_xts_mulx		/* xmm15 := tweak[1] */
	movdqa	%xmm15,%xmm9		/* xmm9 := tweak[1] */
	call	aesni_xts_mulx		/* xmm15 := tweak[2] */
	movdqa	%xmm15,%xmm10		/* xmm10 := tweak[2] */
	call	aesni_xts_mulx		/* xmm15 := tweak[3] */
	movdqa	%xmm15,%xmm11		/* xmm11 := tweak[3] */
	call	aesni_xts_mulx		/* xmm15 := tweak[4] */
	movdqa	%xmm15,%xmm12		/* xmm12 := tweak[4] */
	call	aesni_xts_mulx		/* xmm15 := tweak[5] */
	movdqa	%xmm15,%xmm13		/* xmm13 := tweak[5] */
	call	aesni_xts_mulx		/* xmm15 := tweak[6] */
	movdqa	%xmm15,%xmm14		/* xmm14 := tweak[6] */
	call	aesni_xts_mulx		/* xmm15 := tweak[7] */
	movdqu	(%rsi),%xmm0		/* xmm[i] := ptxt[i] */
	movdqu	0x10(%rsi),%xmm1
	movdqu	0x20(%rsi),%xmm2
	movdqu	0x30(%rsi),%xmm3
	movdqu	0x40(%rsi),%xmm4
	movdqu	0x50(%rsi),%xmm5
	movdqu	0x60(%rsi),%xmm6
	movdqu	0x70(%rsi),%xmm7
	lea	0x80(%rsi),%rsi		/* advance rsi to next block group */
	movdqa	%xmm8,(%rsp)		/* save tweak[0] */
	pxor	%xmm8,%xmm0		/* xmm[i] := ptxt[i] ^ tweak[i] */
	pxor	%xmm9,%xmm1
	pxor	%xmm10,%xmm2
	pxor	%xmm11,%xmm3
	pxor	%xmm12,%xmm4
	pxor	%xmm13,%xmm5
	pxor	%xmm14,%xmm6
	pxor	%xmm15,%xmm7
	mov	%r9d,%ecx		/* ecx := nrounds */
	call	aesni_enc8		/* xmm[i] := AES(ptxt[i] ^ tweak[i]) */
	pxor	(%rsp),%xmm0		/* xmm[i] := AES(...) ^ tweak[i] */
	pxor	%xmm9,%xmm1
	pxor	%xmm10,%xmm2
	pxor	%xmm11,%xmm3
	pxor	%xmm12,%xmm4
	pxor	%xmm13,%xmm5
	pxor	%xmm14,%xmm6
	pxor	%xmm15,%xmm7
	movdqu	%xmm0,(%rdx)		/* store ciphertext blocks */
	movdqu	%xmm1,0x10(%rdx)
	movdqu	%xmm2,0x20(%rdx)
	movdqu	%xmm3,0x30(%rdx)
	movdqu	%xmm4,0x40(%rdx)
	movdqu	%xmm5,0x50(%rdx)
	movdqu	%xmm6,0x60(%rdx)
	movdqu	%xmm7,0x70(%rdx)
	lea	0x80(%rdx),%rdx		/* advance rdx to next block group */
	call	aesni_xts_mulx		/* xmm15 := tweak[8] */
	sub	$0x80,%r10
	jnz	1b			/* repeat if more block groups */
	movdqu	%xmm15,(%r8)		/* update tweak */
	leave
	ret
END(aesni_xts_enc8)

/*
 * aesni_xts_dec1(const struct aesdec *deckey@rdi, const uint8_t *in@rsi,
 *     uint8_t *out@rdx, size_t nbytes@rcx, uint8_t tweak[16] @r8,
 *     uint32_t nrounds@r9d)
 *
 *	Decrypt a contiguous sequence of blocks with AES-XTS.
 *
 *	nbytes must be a positive integral multiple of 16.  This routine
 *	is not vectorized; use aesni_xts_dec8 for >=8 blocks at once.
 *
 *	Standard ABI calling convention.
 */
ENTRY(aesni_xts_dec1)
	mov	%rcx,%r10		/* r10 := nbytes */
	movdqu	(%r8),%xmm15		/* xmm15 := tweak */
1:	movdqu	(%rsi),%xmm0		/* xmm0 := ctxt */
	lea	0x10(%rsi),%rsi		/* advance rdi to next block */
	pxor	%xmm15,%xmm0		/* xmm0 := ctxt ^ tweak */
	mov	%r9d,%ecx		/* ecx := nrounds */
	call	aesni_dec1		/* xmm0 := AES(ctxt ^ tweak) */
	pxor	%xmm15,%xmm0		/* xmm0 := AES(ctxt ^ tweak) ^ tweak */
	movdqu	%xmm0,(%rdx)		/* store plaintext block */
	lea	0x10(%rdx),%rdx		/* advance rsi to next block */
	call	aesni_xts_mulx		/* xmm15 *= x; trash xmm0 */
	sub	$0x10,%r10
	jnz	1b			/* repeat if more blocks */
	movdqu	%xmm15,(%r8)		/* update tweak */
	ret
END(aesni_xts_dec1)

/*
 * aesni_xts_dec8(const struct aesdec *deckey@rdi, const uint8_t *in@rsi,
 *     uint8_t *out@rdx, size_t nbytes@rcx, uint8_t tweak[16] @r8,
 *     uint32_t nrounds@r9d)
 *
 *	Decrypt a contiguous sequence of blocks with AES-XTS.
 *
 *	nbytes must be a positive integral multiple of 128.
 *
 *	Standard ABI calling convention.
 */
ENTRY(aesni_xts_dec8)
	push	%rbp			/* create stack frame uint128[1] */
	mov	%rsp,%rbp
	sub	$0x10,%rsp
	mov	%rcx,%r10		/* r10 := nbytes */
	movdqu	(%r8),%xmm15		/* xmm15 := tweak[0] */
1:	movdqa	%xmm15,%xmm8		/* xmm8 := tweak[0] */
	call	aesni_xts_mulx		/* xmm15 := tweak[1] */
	movdqa	%xmm15,%xmm9		/* xmm9 := tweak[1] */
	call	aesni_xts_mulx		/* xmm15 := tweak[2] */
	movdqa	%xmm15,%xmm10		/* xmm10 := tweak[2] */
	call	aesni_xts_mulx		/* xmm15 := tweak[3] */
	movdqa	%xmm15,%xmm11		/* xmm11 := tweak[3] */
	call	aesni_xts_mulx		/* xmm51 := tweak[4] */
	movdqa	%xmm15,%xmm12		/* xmm12 := tweak[4] */
	call	aesni_xts_mulx		/* xmm15 := tweak[5] */
	movdqa	%xmm15,%xmm13		/* xmm13 := tweak[5] */
	call	aesni_xts_mulx		/* xmm15 := tweak[6] */
	movdqa	%xmm15,%xmm14		/* xmm14 := tweak[6] */
	call	aesni_xts_mulx		/* xmm15 := tweak[7] */
	movdqu	(%rsi),%xmm0		/* xmm[i] := ptxt[i] */
	movdqu	0x10(%rsi),%xmm1
	movdqu	0x20(%rsi),%xmm2
	movdqu	0x30(%rsi),%xmm3
	movdqu	0x40(%rsi),%xmm4
	movdqu	0x50(%rsi),%xmm5
	movdqu	0x60(%rsi),%xmm6
	movdqu	0x70(%rsi),%xmm7
	lea	0x80(%rsi),%rsi		/* advance rsi to next block group */
	movdqa	%xmm8,(%rsp)		/* save tweak[0] */
	pxor	%xmm8,%xmm0		/* xmm[i] := ptxt[i] ^ tweak[i] */
	pxor	%xmm9,%xmm1
	pxor	%xmm10,%xmm2
	pxor	%xmm11,%xmm3
	pxor	%xmm12,%xmm4
	pxor	%xmm13,%xmm5
	pxor	%xmm14,%xmm6
	pxor	%xmm15,%xmm7
	mov	%r9d,%ecx		/* ecx := nrounds */
	call	aesni_dec8		/* xmm[i] := AES(ptxt[i] ^ tweak[i]) */
	pxor	(%rsp),%xmm0		/* xmm[i] := AES(...) ^ tweak[i] */
	pxor	%xmm9,%xmm1
	pxor	%xmm10,%xmm2
	pxor	%xmm11,%xmm3
	pxor	%xmm12,%xmm4
	pxor	%xmm13,%xmm5
	pxor	%xmm14,%xmm6
	pxor	%xmm15,%xmm7
	movdqu	%xmm0,(%rdx)		/* store ciphertext blocks */
	movdqu	%xmm1,0x10(%rdx)
	movdqu	%xmm2,0x20(%rdx)
	movdqu	%xmm3,0x30(%rdx)
	movdqu	%xmm4,0x40(%rdx)
	movdqu	%xmm5,0x50(%rdx)
	movdqu	%xmm6,0x60(%rdx)
	movdqu	%xmm7,0x70(%rdx)
	lea	0x80(%rdx),%rdx		/* advance rdx to next block group */
	call	aesni_xts_mulx		/* xmm15 := tweak[8] */
	sub	$0x80,%r10
	jnz	1b			/* repeat if more block groups */
	movdqu	%xmm15,(%r8)		/* update tweak */
	leave
	ret
END(aesni_xts_dec8)

/*
 * aesni_xts_mulx(tweak@xmm15)
 *
 *	Multiply xmm15 by x, modulo x^128 + x^7 + x^2 + x + 1, in place.
 *	Uses %xmm0 as temporary.
 */
	.text
	_ALIGN_TEXT
	.type	aesni_xts_mulx,@function
aesni_xts_mulx:
	/*
	 * Simultaneously determine
	 * (a) whether the high bit of the low quadword must be
	 *     shifted into the low bit of the high quadword, and
	 * (b) whether the high bit of the high quadword must be
	 *     carried into x^128 = x^7 + x^2 + x + 1.
	 */
	pxor	%xmm0,%xmm0	/* xmm0 := 0 */
	pcmpgtq	%xmm15,%xmm0	/* xmm0[i] := -1 if 0 > xmm15[i] else 0 */
	pshufd	$0b01001110,%xmm0,%xmm0	/* swap halves of xmm0 */
	pand	xtscarry(%rip),%xmm0	/* copy xtscarry according to mask */
	psllq	$1,%xmm15	/* shift */
	pxor	%xmm0,%xmm15	/* incorporate (a) and (b) */
	ret
END(aesni_xts_mulx)

	.section .rodata
	.align 16
	.type	xtscarry,@object
xtscarry:
	.byte	0x87,0,0,0, 0,0,0,0,  1,0,0,0, 0,0,0,0
END(xtscarry)

/*
 * aesni_xts_update(const uint8_t in[16] @rdi, uint8_t out[16] @rsi)
 *
 *	Update an AES-XTS tweak.
 *
 *	Standard ABI calling convention.
 */
ENTRY(aesni_xts_update)
	movdqu	(%rdi),%xmm15
	call	aesni_xts_mulx
	movdqu	%xmm15,(%rsi)
	ret
END(aesni_xts_update)

/*
 * aesni_enc1(const struct aesenc *enckey@rdi, uint128_t block@xmm0,
 *     uint32_t nrounds@ecx)
 *
 *	Encrypt a single AES block in %xmm0.
 *
 *	Internal ABI.  Uses %rax and %xmm8 as temporaries.  Destroys %ecx.
 */
	.text
	_ALIGN_TEXT
	.type	aesni_enc1,@function
aesni_enc1:
	pxor	(%rdi),%xmm0	/* xor in first round key */
	shl	$4,%ecx		/* ecx := total byte size of round keys */
	lea	0x10(%rdi,%rcx),%rax	/* rax := end of round key array */
	neg	%rcx		/* rcx := byte offset of round key from end */
1:	movdqa	(%rax,%rcx),%xmm8	/* load round key */
	add	$0x10,%rcx
	jz	2f		/* stop if this is the last one */
	aesenc	%xmm8,%xmm0
	jmp	1b
2:	aesenclast %xmm8,%xmm0
	ret
END(aesni_enc1)

/*
 * aesni_enc8(const struct aesenc *enckey@rdi, uint128_t block0@xmm0, ...,
 *     block7@xmm7, uint32_t nrounds@ecx)
 *
 *	Encrypt eight AES blocks in %xmm0 through %xmm7 in parallel.
 *
 *	Internal ABI.  Uses %rax and %xmm8 as temporaries.  Destroys %ecx.
 */
	.text
	_ALIGN_TEXT
	.type	aesni_enc8,@function
aesni_enc8:
	movdqa	(%rdi),%xmm8	/* xor in first round key */
	pxor	%xmm8,%xmm0
	pxor	%xmm8,%xmm1
	pxor	%xmm8,%xmm2
	pxor	%xmm8,%xmm3
	pxor	%xmm8,%xmm4
	pxor	%xmm8,%xmm5
	pxor	%xmm8,%xmm6
	pxor	%xmm8,%xmm7
	shl	$4,%ecx		/* ecx := total byte size of round keys */
	lea	0x10(%rdi,%rcx),%rax	/* rax := end of round key array */
	neg	%rcx		/* rcx := byte offset of round key from end */
1:	movdqa	(%rax,%rcx),%xmm8	/* load round key */
	add	$0x10,%rcx
	jz	2f		/* stop if this is the last one */
	aesenc	%xmm8,%xmm0
	aesenc	%xmm8,%xmm1
	aesenc	%xmm8,%xmm2
	aesenc	%xmm8,%xmm3
	aesenc	%xmm8,%xmm4
	aesenc	%xmm8,%xmm5
	aesenc	%xmm8,%xmm6
	aesenc	%xmm8,%xmm7
	jmp	1b
2:	aesenclast %xmm8,%xmm0
	aesenclast %xmm8,%xmm1
	aesenclast %xmm8,%xmm2
	aesenclast %xmm8,%xmm3
	aesenclast %xmm8,%xmm4
	aesenclast %xmm8,%xmm5
	aesenclast %xmm8,%xmm6
	aesenclast %xmm8,%xmm7
	ret
END(aesni_enc8)

/*
 * aesni_dec1(const struct aesdec *deckey@rdi, uint128_t block@xmm0,
 *     uint32_t nrounds@ecx)
 *
 *	Decrypt a single AES block in %xmm0.
 *
 *	Internal ABI.  Uses %rax and %xmm8 as temporaries.  Destroys %ecx.
 */
	.text
	_ALIGN_TEXT
	.type	aesni_dec1,@function
aesni_dec1:
	pxor	(%rdi),%xmm0	/* xor in first round key */
	shl	$4,%ecx		/* ecx := byte offset of round key */
	lea	0x10(%rdi,%rcx),%rax	/* rax := pointer to round key */
	neg	%rcx		/* rcx := byte offset of round key from end */
1:	movdqa	(%rax,%rcx),%xmm8	/* load round key */
	add	$0x10,%rcx
	jz	2f		/* stop if this is the last one */
	aesdec	%xmm8,%xmm0
	jmp	1b
2:	aesdeclast %xmm8,%xmm0
	ret
END(aesni_dec1)

/*
 * aesni_dec8(const struct aesdec *deckey@rdi, uint128_t block0@xmm0, ...,
 *     block7@xmm7, uint32_t nrounds@ecx)
 *
 *	Decrypt eight AES blocks in %xmm0 through %xmm7 in parallel.
 *
 *	Internal ABI.  Uses %xmm8 as temporary.  Destroys %rcx.
 */
	.text
	_ALIGN_TEXT
	.type	aesni_dec8,@function
aesni_dec8:
	movdqa	(%rdi),%xmm8	/* xor in first round key */
	pxor	%xmm8,%xmm0
	pxor	%xmm8,%xmm1
	pxor	%xmm8,%xmm2
	pxor	%xmm8,%xmm3
	pxor	%xmm8,%xmm4
	pxor	%xmm8,%xmm5
	pxor	%xmm8,%xmm6
	pxor	%xmm8,%xmm7
	shl	$4,%ecx		/* ecx := byte offset of round key */
	lea	0x10(%rdi,%rcx),%rax	/* rax := pointer to round key */
	neg	%rcx		/* rcx := byte offset of round key from end */
1:	movdqa	(%rax,%rcx),%xmm8	/* load round key */
	add	$0x10,%rcx
	jz	2f		/* stop if this is the last one */
	aesdec	%xmm8,%xmm0
	aesdec	%xmm8,%xmm1
	aesdec	%xmm8,%xmm2
	aesdec	%xmm8,%xmm3
	aesdec	%xmm8,%xmm4
	aesdec	%xmm8,%xmm5
	aesdec	%xmm8,%xmm6
	aesdec	%xmm8,%xmm7
	jmp	1b
2:	aesdeclast %xmm8,%xmm0
	aesdeclast %xmm8,%xmm1
	aesdeclast %xmm8,%xmm2
	aesdeclast %xmm8,%xmm3
	aesdeclast %xmm8,%xmm4
	aesdeclast %xmm8,%xmm5
	aesdeclast %xmm8,%xmm6
	aesdeclast %xmm8,%xmm7
	ret
END(aesni_dec8)

#	$NetBSD: files.aesni,v 1.1 2020/06/29 23:29:40 riastradh Exp $

ifdef amd64	# amd64-only for now; i386 left as exercise for reader
file	crypto/aes/arch/x86/aes_ni.c		aes
file	crypto/aes/arch/x86/aes_ni_64.S		aes
endif