arch/arm/crypto/speck-neon-core.S

// SPDX-License-Identifier: GPL-2.0
/*
 * NEON-accelerated implementation of Speck128-XTS and Speck64-XTS
 *
 * Copyright (c) 2018 Google, Inc
 *
 * Author: Eric Biggers <ebiggers@google.com>
 */

#include <linux/linkage.h>

	.text
	.fpu		neon

	// arguments
	ROUND_KEYS	.req	r0	// const {u64,u32} *round_keys
	NROUNDS		.req	r1	// int nrounds
	DST		.req	r2	// void *dst
	SRC		.req	r3	// const void *src
	NBYTES		.req	r4	// unsigned int nbytes
	TWEAK		.req	r5	// void *tweak

	// registers which hold the data being encrypted/decrypted
	X0		.req	q0
	X0_L		.req	d0
	X0_H		.req	d1
	Y0		.req	q1
	Y0_H		.req	d3
	X1		.req	q2
	X1_L		.req	d4
	X1_H		.req	d5
	Y1		.req	q3
	Y1_H		.req	d7
	X2		.req	q4
	X2_L		.req	d8
	X2_H		.req	d9
	Y2		.req	q5
	Y2_H		.req	d11
	X3		.req	q6
	X3_L		.req	d12
	X3_H		.req	d13
	Y3		.req	q7
	Y3_H		.req	d15

	// the round key, duplicated in all lanes
	ROUND_KEY	.req	q8
	ROUND_KEY_L	.req	d16
	ROUND_KEY_H	.req	d17

	// index vector for vtbl-based 8-bit rotates
	ROTATE_TABLE	.req	d18

	// multiplication table for updating XTS tweaks
	GF128MUL_TABLE	.req	d19
	GF64MUL_TABLE	.req	d19

	// current XTS tweak value(s)
	TWEAKV		.req	q10
	TWEAKV_L	.req	d20
	TWEAKV_H	.req	d21

	TMP0		.req	q12
	TMP0_L		.req	d24
	TMP0_H		.req	d25
	TMP1		.req	q13
	TMP2		.req	q14
	TMP3		.req	q15

	.align		4
.Lror64_8_table:
	.byte		1, 2, 3, 4, 5, 6, 7, 0
.Lror32_8_table:
	.byte		1, 2, 3, 0, 5, 6, 7, 4
.Lrol64_8_table:
	.byte		7, 0, 1, 2, 3, 4, 5, 6
.Lrol32_8_table:
	.byte		3, 0, 1, 2, 7, 4, 5, 6
.Lgf128mul_table:
	.byte		0, 0x87
	.fill		14
.Lgf64mul_table:
	.byte		0, 0x1b, (0x1b << 1), (0x1b << 1) ^ 0x1b
	.fill		12

/*
 * _speck_round_128bytes() - Speck encryption round on 128 bytes at a time
 *
 * Do one Speck encryption round on the 128 bytes (8 blocks for Speck128, 16 for
 * Speck64) stored in X0-X3 and Y0-Y3, using the round key stored in all lanes
 * of ROUND_KEY.  'n' is the lane size: 64 for Speck128, or 32 for Speck64.
 *
 * The 8-bit rotates are implemented using vtbl instead of vshr + vsli because
 * the vtbl approach is faster on some processors and the same speed on others.
 */
.macro _speck_round_128bytes	n

	// x = ror(x, 8)
	vtbl.8		X0_L, {X0_L}, ROTATE_TABLE
	vtbl.8		X0_H, {X0_H}, ROTATE_TABLE
	vtbl.8		X1_L, {X1_L}, ROTATE_TABLE
	vtbl.8		X1_H, {X1_H}, ROTATE_TABLE
	vtbl.8		X2_L, {X2_L}, ROTATE_TABLE
	vtbl.8		X2_H, {X2_H}, ROTATE_TABLE
	vtbl.8		X3_L, {X3_L}, ROTATE_TABLE
	vtbl.8		X3_H, {X3_H}, ROTATE_TABLE

	// x += y
	vadd.u\n	X0, Y0
	vadd.u\n	X1, Y1
	vadd.u\n	X2, Y2
	vadd.u\n	X3, Y3

	// x ^= k
	veor		X0, ROUND_KEY
	veor		X1, ROUND_KEY
	veor		X2, ROUND_KEY
	veor		X3, ROUND_KEY

	// y = rol(y, 3)
	vshl.u\n	TMP0, Y0, #3
	vshl.u\n	TMP1, Y1, #3
	vshl.u\n	TMP2, Y2, #3
	vshl.u\n	TMP3, Y3, #3
	vsri.u\n	TMP0, Y0, #(\n - 3)
	vsri.u\n	TMP1, Y1, #(\n - 3)
	vsri.u\n	TMP2, Y2, #(\n - 3)
	vsri.u\n	TMP3, Y3, #(\n - 3)

	// y ^= x
	veor		Y0, TMP0, X0
	veor		Y1, TMP1, X1
	veor		Y2, TMP2, X2
	veor		Y3, TMP3, X3
.endm

/*
 * _speck_unround_128bytes() - Speck decryption round on 128 bytes at a time
 *
 * This is the inverse of _speck_round_128bytes().
 */
.macro _speck_unround_128bytes	n

	// y ^= x
	veor		TMP0, Y0, X0
	veor		TMP1, Y1, X1
	veor		TMP2, Y2, X2
	veor		TMP3, Y3, X3

	// y = ror(y, 3)
	vshr.u\n	Y0, TMP0, #3
	vshr.u\n	Y1, TMP1, #3
	vshr.u\n	Y2, TMP2, #3
	vshr.u\n	Y3, TMP3, #3
	vsli.u\n	Y0, TMP0, #(\n - 3)
	vsli.u\n	Y1, TMP1, #(\n - 3)
	vsli.u\n	Y2, TMP2, #(\n - 3)
	vsli.u\n	Y3, TMP3, #(\n - 3)

	// x ^= k
	veor		X0, ROUND_KEY
	veor		X1, ROUND_KEY
	veor		X2, ROUND_KEY
	veor		X3, ROUND_KEY

	// x -= y
	vsub.u\n	X0, Y0
	vsub.u\n	X1, Y1
	vsub.u\n	X2, Y2
	vsub.u\n	X3, Y3

	// x = rol(x, 8);
	vtbl.8		X0_L, {X0_L}, ROTATE_TABLE
	vtbl.8		X0_H, {X0_H}, ROTATE_TABLE
	vtbl.8		X1_L, {X1_L}, ROTATE_TABLE
	vtbl.8		X1_H, {X1_H}, ROTATE_TABLE
	vtbl.8		X2_L, {X2_L}, ROTATE_TABLE
	vtbl.8		X2_H, {X2_H}, ROTATE_TABLE
	vtbl.8		X3_L, {X3_L}, ROTATE_TABLE
	vtbl.8		X3_H, {X3_H}, ROTATE_TABLE
.endm

.macro _xts128_precrypt_one	dst_reg, tweak_buf, tmp

	// Load the next source block
	vld1.8		{\dst_reg}, [SRC]!

	// Save the current tweak in the tweak buffer
	vst1.8		{TWEAKV}, [\tweak_buf:128]!

	// XOR the next source block with the current tweak
	veor		\dst_reg, TWEAKV

	/*
	 * Calculate the next tweak by multiplying the current one by x,
	 * modulo p(x) = x^128 + x^7 + x^2 + x + 1.
	 */
	vshr.u64	\tmp, TWEAKV, #63
	vshl.u64	TWEAKV, #1
	veor		TWEAKV_H, \tmp\()_L
	vtbl.8		\tmp\()_H, {GF128MUL_TABLE}, \tmp\()_H
	veor		TWEAKV_L, \tmp\()_H
.endm

.macro _xts64_precrypt_two	dst_reg, tweak_buf, tmp

	// Load the next two source blocks
	vld1.8		{\dst_reg}, [SRC]!

	// Save the current two tweaks in the tweak buffer
	vst1.8		{TWEAKV}, [\tweak_buf:128]!

	// XOR the next two source blocks with the current two tweaks
	veor		\dst_reg, TWEAKV

	/*
	 * Calculate the next two tweaks by multiplying the current ones by x^2,
	 * modulo p(x) = x^64 + x^4 + x^3 + x + 1.
	 */
	vshr.u64	\tmp, TWEAKV, #62
	vshl.u64	TWEAKV, #2
	vtbl.8		\tmp\()_L, {GF64MUL_TABLE}, \tmp\()_L
	vtbl.8		\tmp\()_H, {GF64MUL_TABLE}, \tmp\()_H
	veor		TWEAKV, \tmp
.endm

/*
 * _speck_xts_crypt() - Speck-XTS encryption/decryption
 *
 * Encrypt or decrypt NBYTES bytes of data from the SRC buffer to the DST buffer
 * using Speck-XTS, specifically the variant with a block size of '2n' and round
 * count given by NROUNDS.  The expanded round keys are given in ROUND_KEYS, and
 * the current XTS tweak value is given in TWEAK.  It's assumed that NBYTES is a
 * nonzero multiple of 128.
 */
.macro _speck_xts_crypt	n, decrypting
	push		{r4-r7}
	mov		r7, sp

	/*
	 * The first four parameters were passed in registers r0-r3.  Load the
	 * additional parameters, which were passed on the stack.
	 */
	ldr		NBYTES, [sp, #16]
	ldr		TWEAK, [sp, #20]

	/*
	 * If decrypting, modify the ROUND_KEYS parameter to point to the last
	 * round key rather than the first, since for decryption the round keys
	 * are used in reverse order.
	 */
.if \decrypting
.if \n == 64
	add		ROUND_KEYS, ROUND_KEYS, NROUNDS, lsl #3
	sub		ROUND_KEYS, #8
.else
	add		ROUND_KEYS, ROUND_KEYS, NROUNDS, lsl #2
	sub		ROUND_KEYS, #4
.endif
.endif

	// Load the index vector for vtbl-based 8-bit rotates
.if \decrypting
	ldr		r12, =.Lrol\n\()_8_table
.else
	ldr		r12, =.Lror\n\()_8_table
.endif
	vld1.8		{ROTATE_TABLE}, [r12:64]

	// One-time XTS preparation

	/*
	 * Allocate stack space to store 128 bytes worth of tweaks.  For
	 * performance, this space is aligned to a 16-byte boundary so that we
	 * can use the load/store instructions that declare 16-byte alignment.
	 */
	sub		sp, #128
	bic		sp, #0xf

.if \n == 64
	// Load first tweak
	vld1.8		{TWEAKV}, [TWEAK]

	// Load GF(2^128) multiplication table
	ldr		r12, =.Lgf128mul_table
	vld1.8		{GF128MUL_TABLE}, [r12:64]
.else
	// Load first tweak
	vld1.8		{TWEAKV_L}, [TWEAK]

	// Load GF(2^64) multiplication table
	ldr		r12, =.Lgf64mul_table
	vld1.8		{GF64MUL_TABLE}, [r12:64]

	// Calculate second tweak, packing it together with the first
	vshr.u64	TMP0_L, TWEAKV_L, #63
	vtbl.u8		TMP0_L, {GF64MUL_TABLE}, TMP0_L
	vshl.u64	TWEAKV_H, TWEAKV_L, #1
	veor		TWEAKV_H, TMP0_L
.endif

.Lnext_128bytes_\@:

	/*
	 * Load the source blocks into {X,Y}[0-3], XOR them with their XTS tweak
	 * values, and save the tweaks on the stack for later.  Then
	 * de-interleave the 'x' and 'y' elements of each block, i.e. make it so
	 * that the X[0-3] registers contain only the second halves of blocks,
	 * and the Y[0-3] registers contain only the first halves of blocks.
	 * (Speck uses the order (y, x) rather than the more intuitive (x, y).)
	 */
	mov		r12, sp
.if \n == 64
	_xts128_precrypt_one	X0, r12, TMP0
	_xts128_precrypt_one	Y0, r12, TMP0
	_xts128_precrypt_one	X1, r12, TMP0
	_xts128_precrypt_one	Y1, r12, TMP0
	_xts128_precrypt_one	X2, r12, TMP0
	_xts128_precrypt_one	Y2, r12, TMP0
	_xts128_precrypt_one	X3, r12, TMP0
	_xts128_precrypt_one	Y3, r12, TMP0
	vswp		X0_L, Y0_H
	vswp		X1_L, Y1_H
	vswp		X2_L, Y2_H
	vswp		X3_L, Y3_H
.else
	_xts64_precrypt_two	X0, r12, TMP0
	_xts64_precrypt_two	Y0, r12, TMP0
	_xts64_precrypt_two	X1, r12, TMP0
	_xts64_precrypt_two	Y1, r12, TMP0
	_xts64_precrypt_two	X2, r12, TMP0
	_xts64_precrypt_two	Y2, r12, TMP0
	_xts64_precrypt_two	X3, r12, TMP0
	_xts64_precrypt_two	Y3, r12, TMP0
	vuzp.32		Y0, X0
	vuzp.32		Y1, X1
	vuzp.32		Y2, X2
	vuzp.32		Y3, X3
.endif

	// Do the cipher rounds

	mov		r12, ROUND_KEYS
	mov		r6, NROUNDS

.Lnext_round_\@:
.if \decrypting
.if \n == 64
	vld1.64		ROUND_KEY_L, [r12]
	sub		r12, #8
	vmov		ROUND_KEY_H, ROUND_KEY_L
.else
	vld1.32		{ROUND_KEY_L[],ROUND_KEY_H[]}, [r12]
	sub		r12, #4
.endif
	_speck_unround_128bytes	\n
.else
.if \n == 64
	vld1.64		ROUND_KEY_L, [r12]!
	vmov		ROUND_KEY_H, ROUND_KEY_L
.else
	vld1.32		{ROUND_KEY_L[],ROUND_KEY_H[]}, [r12]!
.endif
	_speck_round_128bytes	\n
.endif
	subs		r6, r6, #1
	bne		.Lnext_round_\@

	// Re-interleave the 'x' and 'y' elements of each block
.if \n == 64
	vswp		X0_L, Y0_H
	vswp		X1_L, Y1_H
	vswp		X2_L, Y2_H
	vswp		X3_L, Y3_H
.else
	vzip.32		Y0, X0
	vzip.32		Y1, X1
	vzip.32		Y2, X2
	vzip.32		Y3, X3
.endif

	// XOR the encrypted/decrypted blocks with the tweaks we saved earlier
	mov		r12, sp
	vld1.8		{TMP0, TMP1}, [r12:128]!
	vld1.8		{TMP2, TMP3}, [r12:128]!
	veor		X0, TMP0
	veor		Y0, TMP1
	veor		X1, TMP2
	veor		Y1, TMP3
	vld1.8		{TMP0, TMP1}, [r12:128]!
	vld1.8		{TMP2, TMP3}, [r12:128]!
	veor		X2, TMP0
	veor		Y2, TMP1
	veor		X3, TMP2
	veor		Y3, TMP3

	// Store the ciphertext in the destination buffer
	vst1.8		{X0, Y0}, [DST]!
	vst1.8		{X1, Y1}, [DST]!
	vst1.8		{X2, Y2}, [DST]!
	vst1.8		{X3, Y3}, [DST]!

	// Continue if there are more 128-byte chunks remaining, else return
	subs		NBYTES, #128
	bne		.Lnext_128bytes_\@

	// Store the next tweak
.if \n == 64
	vst1.8		{TWEAKV}, [TWEAK]
.else
	vst1.8		{TWEAKV_L}, [TWEAK]
.endif

	mov		sp, r7
	pop		{r4-r7}
	bx		lr
.endm

ENTRY(speck128_xts_encrypt_neon)
	_speck_xts_crypt	n=64, decrypting=0
ENDPROC(speck128_xts_encrypt_neon)

ENTRY(speck128_xts_decrypt_neon)
	_speck_xts_crypt	n=64, decrypting=1
ENDPROC(speck128_xts_decrypt_neon)

ENTRY(speck64_xts_encrypt_neon)
	_speck_xts_crypt	n=32, decrypting=0
ENDPROC(speck64_xts_encrypt_neon)

ENTRY(speck64_xts_decrypt_neon)
	_speck_xts_crypt	n=32, decrypting=1
ENDPROC(speck64_xts_decrypt_neon)