arch/arm64/crypto/speck-neon-core.S

// SPDX-License-Identifier: GPL-2.0
/*
 * ARM64 NEON-accelerated implementation of Speck128-XTS and Speck64-XTS
 *
 * Copyright (c) 2018 Google, Inc
 *
 * Author: Eric Biggers <ebiggers@google.com>
 */

#include <linux/linkage.h>

	.text

	// arguments
	ROUND_KEYS	.req	x0	// const {u64,u32} *round_keys
	NROUNDS		.req	w1	// int nrounds
	NROUNDS_X	.req	x1
	DST		.req	x2	// void *dst
	SRC		.req	x3	// const void *src
	NBYTES		.req	w4	// unsigned int nbytes
	TWEAK		.req	x5	// void *tweak

	// registers which hold the data being encrypted/decrypted
	// (underscores avoid a naming collision with ARM64 registers x0-x3)
	X_0		.req	v0
	Y_0		.req	v1
	X_1		.req	v2
	Y_1		.req	v3
	X_2		.req	v4
	Y_2		.req	v5
	X_3		.req	v6
	Y_3		.req	v7

	// the round key, duplicated in all lanes
	ROUND_KEY	.req	v8

	// index vector for tbl-based 8-bit rotates
	ROTATE_TABLE	.req	v9
	ROTATE_TABLE_Q	.req	q9

	// temporary registers
	TMP0		.req	v10
	TMP1		.req	v11
	TMP2		.req	v12
	TMP3		.req	v13

	// multiplication table for updating XTS tweaks
	GFMUL_TABLE	.req	v14
	GFMUL_TABLE_Q	.req	q14

	// next XTS tweak value(s)
	TWEAKV_NEXT	.req	v15

	// XTS tweaks for the blocks currently being encrypted/decrypted
	TWEAKV0		.req	v16
	TWEAKV1		.req	v17
	TWEAKV2		.req	v18
	TWEAKV3		.req	v19
	TWEAKV4		.req	v20
	TWEAKV5		.req	v21
	TWEAKV6		.req	v22
	TWEAKV7		.req	v23

	.align		4
.Lror64_8_table:
	.octa		0x080f0e0d0c0b0a090007060504030201
.Lror32_8_table:
	.octa		0x0c0f0e0d080b0a090407060500030201
.Lrol64_8_table:
	.octa		0x0e0d0c0b0a09080f0605040302010007
.Lrol32_8_table:
	.octa		0x0e0d0c0f0a09080b0605040702010003
.Lgf128mul_table:
	.octa		0x00000000000000870000000000000001
.Lgf64mul_table:
	.octa		0x0000000000000000000000002d361b00

/*
 * _speck_round_128bytes() - Speck encryption round on 128 bytes at a time
 *
 * Do one Speck encryption round on the 128 bytes (8 blocks for Speck128, 16 for
 * Speck64) stored in X0-X3 and Y0-Y3, using the round key stored in all lanes
 * of ROUND_KEY.  'n' is the lane size: 64 for Speck128, or 32 for Speck64.
 * 'lanes' is the lane specifier: "2d" for Speck128 or "4s" for Speck64.
 */
.macro _speck_round_128bytes	n, lanes

	// x = ror(x, 8)
	tbl		X_0.16b, {X_0.16b}, ROTATE_TABLE.16b
	tbl		X_1.16b, {X_1.16b}, ROTATE_TABLE.16b
	tbl		X_2.16b, {X_2.16b}, ROTATE_TABLE.16b
	tbl		X_3.16b, {X_3.16b}, ROTATE_TABLE.16b

	// x += y
	add		X_0.\lanes, X_0.\lanes, Y_0.\lanes
	add		X_1.\lanes, X_1.\lanes, Y_1.\lanes
	add		X_2.\lanes, X_2.\lanes, Y_2.\lanes
	add		X_3.\lanes, X_3.\lanes, Y_3.\lanes

	// x ^= k
	eor		X_0.16b, X_0.16b, ROUND_KEY.16b
	eor		X_1.16b, X_1.16b, ROUND_KEY.16b
	eor		X_2.16b, X_2.16b, ROUND_KEY.16b
	eor		X_3.16b, X_3.16b, ROUND_KEY.16b

	// y = rol(y, 3)
	shl		TMP0.\lanes, Y_0.\lanes, #3
	shl		TMP1.\lanes, Y_1.\lanes, #3
	shl		TMP2.\lanes, Y_2.\lanes, #3
	shl		TMP3.\lanes, Y_3.\lanes, #3
	sri		TMP0.\lanes, Y_0.\lanes, #(\n - 3)
	sri		TMP1.\lanes, Y_1.\lanes, #(\n - 3)
	sri		TMP2.\lanes, Y_2.\lanes, #(\n - 3)
	sri		TMP3.\lanes, Y_3.\lanes, #(\n - 3)

	// y ^= x
	eor		Y_0.16b, TMP0.16b, X_0.16b
	eor		Y_1.16b, TMP1.16b, X_1.16b
	eor		Y_2.16b, TMP2.16b, X_2.16b
	eor		Y_3.16b, TMP3.16b, X_3.16b
.endm

/*
 * _speck_unround_128bytes() - Speck decryption round on 128 bytes at a time
 *
 * This is the inverse of _speck_round_128bytes().
 */
.macro _speck_unround_128bytes	n, lanes

	// y ^= x
	eor		TMP0.16b, Y_0.16b, X_0.16b
	eor		TMP1.16b, Y_1.16b, X_1.16b
	eor		TMP2.16b, Y_2.16b, X_2.16b
	eor		TMP3.16b, Y_3.16b, X_3.16b

	// y = ror(y, 3)
	ushr		Y_0.\lanes, TMP0.\lanes, #3
	ushr		Y_1.\lanes, TMP1.\lanes, #3
	ushr		Y_2.\lanes, TMP2.\lanes, #3
	ushr		Y_3.\lanes, TMP3.\lanes, #3
	sli		Y_0.\lanes, TMP0.\lanes, #(\n - 3)
	sli		Y_1.\lanes, TMP1.\lanes, #(\n - 3)
	sli		Y_2.\lanes, TMP2.\lanes, #(\n - 3)
	sli		Y_3.\lanes, TMP3.\lanes, #(\n - 3)

	// x ^= k
	eor		X_0.16b, X_0.16b, ROUND_KEY.16b
	eor		X_1.16b, X_1.16b, ROUND_KEY.16b
	eor		X_2.16b, X_2.16b, ROUND_KEY.16b
	eor		X_3.16b, X_3.16b, ROUND_KEY.16b

	// x -= y
	sub		X_0.\lanes, X_0.\lanes, Y_0.\lanes
	sub		X_1.\lanes, X_1.\lanes, Y_1.\lanes
	sub		X_2.\lanes, X_2.\lanes, Y_2.\lanes
	sub		X_3.\lanes, X_3.\lanes, Y_3.\lanes

	// x = rol(x, 8)
	tbl		X_0.16b, {X_0.16b}, ROTATE_TABLE.16b
	tbl		X_1.16b, {X_1.16b}, ROTATE_TABLE.16b
	tbl		X_2.16b, {X_2.16b}, ROTATE_TABLE.16b
	tbl		X_3.16b, {X_3.16b}, ROTATE_TABLE.16b
.endm

.macro _next_xts_tweak	next, cur, tmp, n
.if \n == 64
	/*
	 * Calculate the next tweak by multiplying the current one by x,
	 * modulo p(x) = x^128 + x^7 + x^2 + x + 1.
	 */
	sshr		\tmp\().2d, \cur\().2d, #63
	and		\tmp\().16b, \tmp\().16b, GFMUL_TABLE.16b
	shl		\next\().2d, \cur\().2d, #1
	ext		\tmp\().16b, \tmp\().16b, \tmp\().16b, #8
	eor		\next\().16b, \next\().16b, \tmp\().16b
.else
	/*
	 * Calculate the next two tweaks by multiplying the current ones by x^2,
	 * modulo p(x) = x^64 + x^4 + x^3 + x + 1.
	 */
	ushr		\tmp\().2d, \cur\().2d, #62
	shl		\next\().2d, \cur\().2d, #2
	tbl		\tmp\().16b, {GFMUL_TABLE.16b}, \tmp\().16b
	eor		\next\().16b, \next\().16b, \tmp\().16b
.endif
.endm

/*
 * _speck_xts_crypt() - Speck-XTS encryption/decryption
 *
 * Encrypt or decrypt NBYTES bytes of data from the SRC buffer to the DST buffer
 * using Speck-XTS, specifically the variant with a block size of '2n' and round
 * count given by NROUNDS.  The expanded round keys are given in ROUND_KEYS, and
 * the current XTS tweak value is given in TWEAK.  It's assumed that NBYTES is a
 * nonzero multiple of 128.
 */
.macro _speck_xts_crypt	n, lanes, decrypting

	/*
	 * If decrypting, modify the ROUND_KEYS parameter to point to the last
	 * round key rather than the first, since for decryption the round keys
	 * are used in reverse order.
	 */
.if \decrypting
	mov		NROUNDS, NROUNDS	/* zero the high 32 bits */
.if \n == 64
	add		ROUND_KEYS, ROUND_KEYS, NROUNDS_X, lsl #3
	sub		ROUND_KEYS, ROUND_KEYS, #8
.else
	add		ROUND_KEYS, ROUND_KEYS, NROUNDS_X, lsl #2
	sub		ROUND_KEYS, ROUND_KEYS, #4
.endif
.endif

	// Load the index vector for tbl-based 8-bit rotates
.if \decrypting
	ldr		ROTATE_TABLE_Q, .Lrol\n\()_8_table
.else
	ldr		ROTATE_TABLE_Q, .Lror\n\()_8_table
.endif

	// One-time XTS preparation
.if \n == 64
	// Load first tweak
	ld1		{TWEAKV0.16b}, [TWEAK]

	// Load GF(2^128) multiplication table
	ldr		GFMUL_TABLE_Q, .Lgf128mul_table
.else
	// Load first tweak
	ld1		{TWEAKV0.8b}, [TWEAK]

	// Load GF(2^64) multiplication table
	ldr		GFMUL_TABLE_Q, .Lgf64mul_table

	// Calculate second tweak, packing it together with the first
	ushr		TMP0.2d, TWEAKV0.2d, #63
	shl		TMP1.2d, TWEAKV0.2d, #1
	tbl		TMP0.8b, {GFMUL_TABLE.16b}, TMP0.8b
	eor		TMP0.8b, TMP0.8b, TMP1.8b
	mov		TWEAKV0.d[1], TMP0.d[0]
.endif

.Lnext_128bytes_\@:

	// Calculate XTS tweaks for next 128 bytes
	_next_xts_tweak	TWEAKV1, TWEAKV0, TMP0, \n
	_next_xts_tweak	TWEAKV2, TWEAKV1, TMP0, \n
	_next_xts_tweak	TWEAKV3, TWEAKV2, TMP0, \n
	_next_xts_tweak	TWEAKV4, TWEAKV3, TMP0, \n
	_next_xts_tweak	TWEAKV5, TWEAKV4, TMP0, \n
	_next_xts_tweak	TWEAKV6, TWEAKV5, TMP0, \n
	_next_xts_tweak	TWEAKV7, TWEAKV6, TMP0, \n
	_next_xts_tweak	TWEAKV_NEXT, TWEAKV7, TMP0, \n

	// Load the next source blocks into {X,Y}[0-3]
	ld1		{X_0.16b-Y_1.16b}, [SRC], #64
	ld1		{X_2.16b-Y_3.16b}, [SRC], #64

	// XOR the source blocks with their XTS tweaks
	eor		TMP0.16b, X_0.16b, TWEAKV0.16b
	eor		Y_0.16b,  Y_0.16b, TWEAKV1.16b
	eor		TMP1.16b, X_1.16b, TWEAKV2.16b
	eor		Y_1.16b,  Y_1.16b, TWEAKV3.16b
	eor		TMP2.16b, X_2.16b, TWEAKV4.16b
	eor		Y_2.16b,  Y_2.16b, TWEAKV5.16b
	eor		TMP3.16b, X_3.16b, TWEAKV6.16b
	eor		Y_3.16b,  Y_3.16b, TWEAKV7.16b

	/*
	 * De-interleave the 'x' and 'y' elements of each block, i.e. make it so
	 * that the X[0-3] registers contain only the second halves of blocks,
	 * and the Y[0-3] registers contain only the first halves of blocks.
	 * (Speck uses the order (y, x) rather than the more intuitive (x, y).)
	 */
	uzp2		X_0.\lanes, TMP0.\lanes, Y_0.\lanes
	uzp1		Y_0.\lanes, TMP0.\lanes, Y_0.\lanes
	uzp2		X_1.\lanes, TMP1.\lanes, Y_1.\lanes
	uzp1		Y_1.\lanes, TMP1.\lanes, Y_1.\lanes
	uzp2		X_2.\lanes, TMP2.\lanes, Y_2.\lanes
	uzp1		Y_2.\lanes, TMP2.\lanes, Y_2.\lanes
	uzp2		X_3.\lanes, TMP3.\lanes, Y_3.\lanes
	uzp1		Y_3.\lanes, TMP3.\lanes, Y_3.\lanes

	// Do the cipher rounds
	mov		x6, ROUND_KEYS
	mov		w7, NROUNDS
.Lnext_round_\@:
.if \decrypting
	ld1r		{ROUND_KEY.\lanes}, [x6]
	sub		x6, x6, #( \n / 8 )
	_speck_unround_128bytes	\n, \lanes
.else
	ld1r		{ROUND_KEY.\lanes}, [x6], #( \n / 8 )
	_speck_round_128bytes	\n, \lanes
.endif
	subs		w7, w7, #1
	bne		.Lnext_round_\@

	// Re-interleave the 'x' and 'y' elements of each block
	zip1		TMP0.\lanes, Y_0.\lanes, X_0.\lanes
	zip2		Y_0.\lanes,  Y_0.\lanes, X_0.\lanes
	zip1		TMP1.\lanes, Y_1.\lanes, X_1.\lanes
	zip2		Y_1.\lanes,  Y_1.\lanes, X_1.\lanes
	zip1		TMP2.\lanes, Y_2.\lanes, X_2.\lanes
	zip2		Y_2.\lanes,  Y_2.\lanes, X_2.\lanes
	zip1		TMP3.\lanes, Y_3.\lanes, X_3.\lanes
	zip2		Y_3.\lanes,  Y_3.\lanes, X_3.\lanes

	// XOR the encrypted/decrypted blocks with the tweaks calculated earlier
	eor		X_0.16b, TMP0.16b, TWEAKV0.16b
	eor		Y_0.16b, Y_0.16b,  TWEAKV1.16b
	eor		X_1.16b, TMP1.16b, TWEAKV2.16b
	eor		Y_1.16b, Y_1.16b,  TWEAKV3.16b
	eor		X_2.16b, TMP2.16b, TWEAKV4.16b
	eor		Y_2.16b, Y_2.16b,  TWEAKV5.16b
	eor		X_3.16b, TMP3.16b, TWEAKV6.16b
	eor		Y_3.16b, Y_3.16b,  TWEAKV7.16b
	mov		TWEAKV0.16b, TWEAKV_NEXT.16b

	// Store the ciphertext in the destination buffer
	st1		{X_0.16b-Y_1.16b}, [DST], #64
	st1		{X_2.16b-Y_3.16b}, [DST], #64

	// Continue if there are more 128-byte chunks remaining
	subs		NBYTES, NBYTES, #128
	bne		.Lnext_128bytes_\@

	// Store the next tweak and return
.if \n == 64
	st1		{TWEAKV_NEXT.16b}, [TWEAK]
.else
	st1		{TWEAKV_NEXT.8b}, [TWEAK]
.endif
	ret
.endm

ENTRY(speck128_xts_encrypt_neon)
	_speck_xts_crypt	n=64, lanes=2d, decrypting=0
ENDPROC(speck128_xts_encrypt_neon)

ENTRY(speck128_xts_decrypt_neon)
	_speck_xts_crypt	n=64, lanes=2d, decrypting=1
ENDPROC(speck128_xts_decrypt_neon)

ENTRY(speck64_xts_encrypt_neon)
	_speck_xts_crypt	n=32, lanes=4s, decrypting=0
ENDPROC(speck64_xts_encrypt_neon)

ENTRY(speck64_xts_decrypt_neon)
	_speck_xts_crypt	n=32, lanes=4s, decrypting=1
ENDPROC(speck64_xts_decrypt_neon)