From 939a54ac073808db15ed411d563dfadb3ef12798 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 22 Apr 2025 08:27:10 -0700 Subject: crypto: mips - move library functions to arch/mips/lib/crypto/ Continue disentangling the crypto library functions from the generic crypto infrastructure by moving the mips ChaCha and Poly1305 library functions into a new directory arch/mips/lib/crypto/ that does not depend on CRYPTO. This mirrors the distinction between crypto/ and lib/crypto/. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/mips/lib/crypto/poly1305-glue.c | 82 ++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 arch/mips/lib/crypto/poly1305-glue.c (limited to 'arch/mips/lib/crypto/poly1305-glue.c') diff --git a/arch/mips/lib/crypto/poly1305-glue.c b/arch/mips/lib/crypto/poly1305-glue.c new file mode 100644 index 000000000000..576e7a58e0b1 --- /dev/null +++ b/arch/mips/lib/crypto/poly1305-glue.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * OpenSSL/Cryptogams accelerated Poly1305 transform for MIPS + * + * Copyright (C) 2019 Linaro Ltd. + */ + +#include +#include +#include +#include + +asmlinkage void poly1305_init_mips(void *state, const u8 *key); +asmlinkage void poly1305_blocks_mips(void *state, const u8 *src, u32 len, u32 hibit); +asmlinkage void poly1305_emit_mips(void *state, u8 *digest, const u32 *nonce); + +void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) +{ + poly1305_init_mips(&dctx->h, key); + dctx->s[0] = get_unaligned_le32(key + 16); + dctx->s[1] = get_unaligned_le32(key + 20); + dctx->s[2] = get_unaligned_le32(key + 24); + dctx->s[3] = get_unaligned_le32(key + 28); + dctx->buflen = 0; +} +EXPORT_SYMBOL(poly1305_init_arch); + +void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, + unsigned int nbytes) +{ + if (unlikely(dctx->buflen)) { + u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen); + + memcpy(dctx->buf + dctx->buflen, src, bytes); + src += bytes; + nbytes -= bytes; + dctx->buflen += bytes; + + if (dctx->buflen == POLY1305_BLOCK_SIZE) { + poly1305_blocks_mips(&dctx->h, dctx->buf, + POLY1305_BLOCK_SIZE, 1); + dctx->buflen = 0; + } + } + + if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { + unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE); + + poly1305_blocks_mips(&dctx->h, src, len, 1); + src += len; + nbytes %= POLY1305_BLOCK_SIZE; + } + + if (unlikely(nbytes)) { + dctx->buflen = nbytes; + memcpy(dctx->buf, src, nbytes); + } +} +EXPORT_SYMBOL(poly1305_update_arch); + +void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) +{ + if (unlikely(dctx->buflen)) { + dctx->buf[dctx->buflen++] = 1; + memset(dctx->buf + dctx->buflen, 0, + POLY1305_BLOCK_SIZE - dctx->buflen); + poly1305_blocks_mips(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); + } + + poly1305_emit_mips(&dctx->h, dst, dctx->s); + *dctx = (struct poly1305_desc_ctx){}; +} +EXPORT_SYMBOL(poly1305_final_arch); + +bool poly1305_is_arch_optimized(void) +{ + return true; +} +EXPORT_SYMBOL(poly1305_is_arch_optimized); + +MODULE_DESCRIPTION("Poly1305 transform (MIPS accelerated"); +MODULE_LICENSE("GPL v2"); -- cgit v1.2.3 From ffe5ca295d757d912b226239df17396c3cd8dbca Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 28 Apr 2025 12:56:14 +0800 Subject: crypto: mips/poly1305 - Add block-only interface Add block-only interface. Signed-off-by: Herbert Xu --- arch/mips/lib/crypto/poly1305-glue.c | 29 ++++++++++++++++++++--------- arch/mips/lib/crypto/poly1305-mips.pl | 12 ++++++------ 2 files changed, 26 insertions(+), 15 deletions(-) (limited to 'arch/mips/lib/crypto/poly1305-glue.c') diff --git a/arch/mips/lib/crypto/poly1305-glue.c b/arch/mips/lib/crypto/poly1305-glue.c index 576e7a58e0b1..2fea4cacfe27 100644 --- a/arch/mips/lib/crypto/poly1305-glue.c +++ b/arch/mips/lib/crypto/poly1305-glue.c @@ -5,23 +5,33 @@ * Copyright (C) 2019 Linaro Ltd. */ -#include +#include #include +#include #include +#include #include -asmlinkage void poly1305_init_mips(void *state, const u8 *key); -asmlinkage void poly1305_blocks_mips(void *state, const u8 *src, u32 len, u32 hibit); -asmlinkage void poly1305_emit_mips(void *state, u8 *digest, const u32 *nonce); +asmlinkage void poly1305_block_init_arch( + struct poly1305_block_state *state, + const u8 raw_key[POLY1305_BLOCK_SIZE]); +EXPORT_SYMBOL_GPL(poly1305_block_init_arch); +asmlinkage void poly1305_blocks_arch(struct poly1305_block_state *state, + const u8 *src, u32 len, u32 hibit); +EXPORT_SYMBOL_GPL(poly1305_blocks_arch); +asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, + u8 digest[POLY1305_DIGEST_SIZE], + const u32 nonce[4]); +EXPORT_SYMBOL_GPL(poly1305_emit_arch); void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) { - poly1305_init_mips(&dctx->h, key); dctx->s[0] = get_unaligned_le32(key + 16); dctx->s[1] = get_unaligned_le32(key + 20); dctx->s[2] = get_unaligned_le32(key + 24); dctx->s[3] = get_unaligned_le32(key + 28); dctx->buflen = 0; + poly1305_block_init_arch(&dctx->state, key); } EXPORT_SYMBOL(poly1305_init_arch); @@ -37,7 +47,7 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, dctx->buflen += bytes; if (dctx->buflen == POLY1305_BLOCK_SIZE) { - poly1305_blocks_mips(&dctx->h, dctx->buf, + poly1305_blocks_arch(&dctx->state, dctx->buf, POLY1305_BLOCK_SIZE, 1); dctx->buflen = 0; } @@ -46,7 +56,7 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE); - poly1305_blocks_mips(&dctx->h, src, len, 1); + poly1305_blocks_arch(&dctx->state, src, len, 1); src += len; nbytes %= POLY1305_BLOCK_SIZE; } @@ -64,10 +74,11 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) dctx->buf[dctx->buflen++] = 1; memset(dctx->buf + dctx->buflen, 0, POLY1305_BLOCK_SIZE - dctx->buflen); - poly1305_blocks_mips(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); + poly1305_blocks_arch(&dctx->state, dctx->buf, + POLY1305_BLOCK_SIZE, 0); } - poly1305_emit_mips(&dctx->h, dst, dctx->s); + poly1305_emit_arch(&dctx->h, dst, dctx->s); *dctx = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL(poly1305_final_arch); diff --git a/arch/mips/lib/crypto/poly1305-mips.pl b/arch/mips/lib/crypto/poly1305-mips.pl index b05bab884ed2..399f10c3e385 100644 --- a/arch/mips/lib/crypto/poly1305-mips.pl +++ b/arch/mips/lib/crypto/poly1305-mips.pl @@ -93,9 +93,9 @@ $code.=<<___; #endif #ifdef __KERNEL__ -# define poly1305_init poly1305_init_mips -# define poly1305_blocks poly1305_blocks_mips -# define poly1305_emit poly1305_emit_mips +# define poly1305_init poly1305_block_init_arch +# define poly1305_blocks poly1305_blocks_arch +# define poly1305_emit poly1305_emit_arch #endif #if defined(__MIPSEB__) && !defined(MIPSEB) @@ -565,9 +565,9 @@ $code.=<<___; #endif #ifdef __KERNEL__ -# define poly1305_init poly1305_init_mips -# define poly1305_blocks poly1305_blocks_mips -# define poly1305_emit poly1305_emit_mips +# define poly1305_init poly1305_block_init_arch +# define poly1305_blocks poly1305_blocks_arch +# define poly1305_emit poly1305_emit_arch #endif #if defined(__MIPSEB__) && !defined(MIPSEB) -- cgit v1.2.3 From 10a6d72ea355b730aa9702da0fd36aef0898a80e Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 28 Apr 2025 12:56:28 +0800 Subject: crypto: lib/poly1305 - Use block-only interface Now that every architecture provides a block function, use that to implement the lib/poly1305 and remove the old per-arch code. Signed-off-by: Herbert Xu --- arch/arm/lib/crypto/poly1305-glue.c | 57 -------------------------- arch/arm64/lib/crypto/poly1305-glue.c | 58 -------------------------- arch/mips/lib/crypto/poly1305-glue.c | 60 --------------------------- arch/powerpc/lib/crypto/poly1305-p10-glue.c | 63 ----------------------------- arch/x86/lib/crypto/poly1305_glue.c | 60 --------------------------- include/crypto/poly1305.h | 53 +++--------------------- lib/crypto/poly1305.c | 39 ++++++++++++------ 7 files changed, 32 insertions(+), 358 deletions(-) (limited to 'arch/mips/lib/crypto/poly1305-glue.c') diff --git a/arch/arm/lib/crypto/poly1305-glue.c b/arch/arm/lib/crypto/poly1305-glue.c index 3ee16048ec7c..91da42b26d9c 100644 --- a/arch/arm/lib/crypto/poly1305-glue.c +++ b/arch/arm/lib/crypto/poly1305-glue.c @@ -12,7 +12,6 @@ #include #include #include -#include #include asmlinkage void poly1305_block_init_arch( @@ -35,17 +34,6 @@ void __weak poly1305_blocks_neon(struct poly1305_block_state *state, static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); -void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) -{ - dctx->s[0] = get_unaligned_le32(key + 16); - dctx->s[1] = get_unaligned_le32(key + 20); - dctx->s[2] = get_unaligned_le32(key + 24); - dctx->s[3] = get_unaligned_le32(key + 28); - dctx->buflen = 0; - poly1305_block_init_arch(&dctx->state, key); -} -EXPORT_SYMBOL(poly1305_init_arch); - void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, unsigned int len, u32 padbit) { @@ -67,51 +55,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, } EXPORT_SYMBOL_GPL(poly1305_blocks_arch); -void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, - unsigned int nbytes) -{ - if (unlikely(dctx->buflen)) { - u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen); - - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - nbytes -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 1); - dctx->buflen = 0; - } - } - - if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { - poly1305_blocks_arch(&dctx->state, src, nbytes, 1); - src += round_down(nbytes, POLY1305_BLOCK_SIZE); - nbytes %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(nbytes)) { - dctx->buflen = nbytes; - memcpy(dctx->buf, src, nbytes); - } -} -EXPORT_SYMBOL(poly1305_update_arch); - -void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) -{ - if (unlikely(dctx->buflen)) { - dctx->buf[dctx->buflen++] = 1; - memset(dctx->buf + dctx->buflen, 0, - POLY1305_BLOCK_SIZE - dctx->buflen); - poly1305_blocks_arch(&dctx->state, dctx->buf, POLY1305_BLOCK_SIZE, 0); - } - - poly1305_emit_arch(&dctx->h, dst, dctx->s); - *dctx = (struct poly1305_desc_ctx){}; -} -EXPORT_SYMBOL(poly1305_final_arch); - bool poly1305_is_arch_optimized(void) { /* We always can use at least the ARM scalar implementation. */ diff --git a/arch/arm64/lib/crypto/poly1305-glue.c b/arch/arm64/lib/crypto/poly1305-glue.c index d66a820e32d5..681c26557336 100644 --- a/arch/arm64/lib/crypto/poly1305-glue.c +++ b/arch/arm64/lib/crypto/poly1305-glue.c @@ -12,7 +12,6 @@ #include #include #include -#include #include asmlinkage void poly1305_block_init_arch( @@ -30,17 +29,6 @@ EXPORT_SYMBOL_GPL(poly1305_emit_arch); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); -void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) -{ - dctx->s[0] = get_unaligned_le32(key + 16); - dctx->s[1] = get_unaligned_le32(key + 20); - dctx->s[2] = get_unaligned_le32(key + 24); - dctx->s[3] = get_unaligned_le32(key + 28); - dctx->buflen = 0; - poly1305_block_init_arch(&dctx->state, key); -} -EXPORT_SYMBOL(poly1305_init_arch); - void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, unsigned int len, u32 padbit) { @@ -61,52 +49,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, } EXPORT_SYMBOL_GPL(poly1305_blocks_arch); -void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, - unsigned int nbytes) -{ - if (unlikely(dctx->buflen)) { - u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen); - - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - nbytes -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 1); - dctx->buflen = 0; - } - } - - if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { - poly1305_blocks_arch(&dctx->state, src, nbytes, 1); - src += round_down(nbytes, POLY1305_BLOCK_SIZE); - nbytes %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(nbytes)) { - dctx->buflen = nbytes; - memcpy(dctx->buf, src, nbytes); - } -} -EXPORT_SYMBOL(poly1305_update_arch); - -void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) -{ - if (unlikely(dctx->buflen)) { - dctx->buf[dctx->buflen++] = 1; - memset(dctx->buf + dctx->buflen, 0, - POLY1305_BLOCK_SIZE - dctx->buflen); - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 0); - } - - poly1305_emit_arch(&dctx->h, dst, dctx->s); - memzero_explicit(dctx, sizeof(*dctx)); -} -EXPORT_SYMBOL(poly1305_final_arch); - bool poly1305_is_arch_optimized(void) { /* We always can use at least the ARM64 scalar implementation. */ diff --git a/arch/mips/lib/crypto/poly1305-glue.c b/arch/mips/lib/crypto/poly1305-glue.c index 2fea4cacfe27..764a38a65200 100644 --- a/arch/mips/lib/crypto/poly1305-glue.c +++ b/arch/mips/lib/crypto/poly1305-glue.c @@ -9,7 +9,6 @@ #include #include #include -#include #include asmlinkage void poly1305_block_init_arch( @@ -24,65 +23,6 @@ asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, const u32 nonce[4]); EXPORT_SYMBOL_GPL(poly1305_emit_arch); -void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) -{ - dctx->s[0] = get_unaligned_le32(key + 16); - dctx->s[1] = get_unaligned_le32(key + 20); - dctx->s[2] = get_unaligned_le32(key + 24); - dctx->s[3] = get_unaligned_le32(key + 28); - dctx->buflen = 0; - poly1305_block_init_arch(&dctx->state, key); -} -EXPORT_SYMBOL(poly1305_init_arch); - -void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, - unsigned int nbytes) -{ - if (unlikely(dctx->buflen)) { - u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen); - - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - nbytes -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 1); - dctx->buflen = 0; - } - } - - if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { - unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE); - - poly1305_blocks_arch(&dctx->state, src, len, 1); - src += len; - nbytes %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(nbytes)) { - dctx->buflen = nbytes; - memcpy(dctx->buf, src, nbytes); - } -} -EXPORT_SYMBOL(poly1305_update_arch); - -void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) -{ - if (unlikely(dctx->buflen)) { - dctx->buf[dctx->buflen++] = 1; - memset(dctx->buf + dctx->buflen, 0, - POLY1305_BLOCK_SIZE - dctx->buflen); - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 0); - } - - poly1305_emit_arch(&dctx->h, dst, dctx->s); - *dctx = (struct poly1305_desc_ctx){}; -} -EXPORT_SYMBOL(poly1305_final_arch); - bool poly1305_is_arch_optimized(void) { return true; diff --git a/arch/powerpc/lib/crypto/poly1305-p10-glue.c b/arch/powerpc/lib/crypto/poly1305-p10-glue.c index 708435beaba6..50ac802220e0 100644 --- a/arch/powerpc/lib/crypto/poly1305-p10-glue.c +++ b/arch/powerpc/lib/crypto/poly1305-p10-glue.c @@ -10,7 +10,6 @@ #include #include #include -#include #include asmlinkage void poly1305_p10le_4blocks(struct poly1305_block_state *state, const u8 *m, u32 mlen); @@ -45,17 +44,6 @@ void poly1305_block_init_arch(struct poly1305_block_state *dctx, } EXPORT_SYMBOL_GPL(poly1305_block_init_arch); -void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) -{ - dctx->s[0] = get_unaligned_le32(key + 16); - dctx->s[1] = get_unaligned_le32(key + 20); - dctx->s[2] = get_unaligned_le32(key + 24); - dctx->s[3] = get_unaligned_le32(key + 28); - dctx->buflen = 0; - poly1305_block_init_arch(&dctx->state, key); -} -EXPORT_SYMBOL(poly1305_init_arch); - void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, unsigned int len, u32 padbit) { @@ -76,57 +64,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, } EXPORT_SYMBOL_GPL(poly1305_blocks_arch); -void poly1305_update_arch(struct poly1305_desc_ctx *dctx, - const u8 *src, unsigned int srclen) -{ - unsigned int bytes; - - if (!static_key_enabled(&have_p10)) - return poly1305_update_generic(dctx, src, srclen); - - if (unlikely(dctx->buflen)) { - bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - srclen -= bytes; - dctx->buflen += bytes; - if (dctx->buflen < POLY1305_BLOCK_SIZE) - return; - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 1); - dctx->buflen = 0; - } - - if (likely(srclen >= POLY1305_BLOCK_SIZE)) { - poly1305_blocks_arch(&dctx->state, src, srclen, 1); - src += srclen - (srclen % POLY1305_BLOCK_SIZE); - srclen %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(srclen)) { - dctx->buflen = srclen; - memcpy(dctx->buf, src, srclen); - } -} -EXPORT_SYMBOL(poly1305_update_arch); - -void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) -{ - if (!static_key_enabled(&have_p10)) - return poly1305_final_generic(dctx, dst); - - if (dctx->buflen) { - dctx->buf[dctx->buflen++] = 1; - memset(dctx->buf + dctx->buflen, 0, - POLY1305_BLOCK_SIZE - dctx->buflen); - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 0); - } - - poly1305_emit_arch(&dctx->h, dst, dctx->s); -} -EXPORT_SYMBOL(poly1305_final_arch); - bool poly1305_is_arch_optimized(void) { return static_key_enabled(&have_p10); diff --git a/arch/x86/lib/crypto/poly1305_glue.c b/arch/x86/lib/crypto/poly1305_glue.c index d98764ec3b47..f799828c5809 100644 --- a/arch/x86/lib/crypto/poly1305_glue.c +++ b/arch/x86/lib/crypto/poly1305_glue.c @@ -10,7 +10,6 @@ #include #include #include -#include #include struct poly1305_arch_internal { @@ -96,65 +95,6 @@ void poly1305_emit_arch(const struct poly1305_state *ctx, } EXPORT_SYMBOL_GPL(poly1305_emit_arch); -void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) -{ - dctx->s[0] = get_unaligned_le32(&key[16]); - dctx->s[1] = get_unaligned_le32(&key[20]); - dctx->s[2] = get_unaligned_le32(&key[24]); - dctx->s[3] = get_unaligned_le32(&key[28]); - dctx->buflen = 0; - poly1305_block_init_arch(&dctx->state, key); -} -EXPORT_SYMBOL(poly1305_init_arch); - -void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, - unsigned int srclen) -{ - unsigned int bytes; - - if (unlikely(dctx->buflen)) { - bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - srclen -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 1); - dctx->buflen = 0; - } - } - - if (likely(srclen >= POLY1305_BLOCK_SIZE)) { - bytes = round_down(srclen, POLY1305_BLOCK_SIZE); - poly1305_blocks_arch(&dctx->state, src, bytes, 1); - src += bytes; - srclen -= bytes; - } - - if (unlikely(srclen)) { - dctx->buflen = srclen; - memcpy(dctx->buf, src, srclen); - } -} -EXPORT_SYMBOL(poly1305_update_arch); - -void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) -{ - if (unlikely(dctx->buflen)) { - dctx->buf[dctx->buflen++] = 1; - memset(dctx->buf + dctx->buflen, 0, - POLY1305_BLOCK_SIZE - dctx->buflen); - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 0); - } - - poly1305_emit_arch(&dctx->h, dst, dctx->s); - memzero_explicit(dctx, sizeof(*dctx)); -} -EXPORT_SYMBOL(poly1305_final_arch); - bool poly1305_is_arch_optimized(void) { return static_key_enabled(&poly1305_use_avx); diff --git a/include/crypto/poly1305.h b/include/crypto/poly1305.h index 027d74842cd5..e54abda8cfe9 100644 --- a/include/crypto/poly1305.h +++ b/include/crypto/poly1305.h @@ -55,55 +55,14 @@ struct poly1305_desc_ctx { unsigned int buflen; /* finalize key */ u32 s[4]; - union { - struct { - struct poly1305_state h; - union { - struct poly1305_key opaque_r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE]; - struct poly1305_core_key core_r; - }; - }; - struct poly1305_block_state state; - }; + struct poly1305_block_state state; }; -void poly1305_init_arch(struct poly1305_desc_ctx *desc, - const u8 key[POLY1305_KEY_SIZE]); -void poly1305_init_generic(struct poly1305_desc_ctx *desc, - const u8 key[POLY1305_KEY_SIZE]); - -static inline void poly1305_init(struct poly1305_desc_ctx *desc, const u8 *key) -{ - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) - poly1305_init_arch(desc, key); - else - poly1305_init_generic(desc, key); -} - -void poly1305_update_arch(struct poly1305_desc_ctx *desc, const u8 *src, - unsigned int nbytes); -void poly1305_update_generic(struct poly1305_desc_ctx *desc, const u8 *src, - unsigned int nbytes); - -static inline void poly1305_update(struct poly1305_desc_ctx *desc, - const u8 *src, unsigned int nbytes) -{ - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) - poly1305_update_arch(desc, src, nbytes); - else - poly1305_update_generic(desc, src, nbytes); -} - -void poly1305_final_arch(struct poly1305_desc_ctx *desc, u8 *digest); -void poly1305_final_generic(struct poly1305_desc_ctx *desc, u8 *digest); - -static inline void poly1305_final(struct poly1305_desc_ctx *desc, u8 *digest) -{ - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) - poly1305_final_arch(desc, digest); - else - poly1305_final_generic(desc, digest); -} +void poly1305_init(struct poly1305_desc_ctx *desc, + const u8 key[POLY1305_KEY_SIZE]); +void poly1305_update(struct poly1305_desc_ctx *desc, + const u8 *src, unsigned int nbytes); +void poly1305_final(struct poly1305_desc_ctx *desc, u8 *digest); #if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305) bool poly1305_is_arch_optimized(void); diff --git a/lib/crypto/poly1305.c b/lib/crypto/poly1305.c index 9fec64a599c1..4c9996864090 100644 --- a/lib/crypto/poly1305.c +++ b/lib/crypto/poly1305.c @@ -22,47 +22,60 @@ void poly1305_block_init_generic(struct poly1305_block_state *desc, } EXPORT_SYMBOL_GPL(poly1305_block_init_generic); -void poly1305_init_generic(struct poly1305_desc_ctx *desc, - const u8 key[POLY1305_KEY_SIZE]) +void poly1305_init(struct poly1305_desc_ctx *desc, + const u8 key[POLY1305_KEY_SIZE]) { desc->s[0] = get_unaligned_le32(key + 16); desc->s[1] = get_unaligned_le32(key + 20); desc->s[2] = get_unaligned_le32(key + 24); desc->s[3] = get_unaligned_le32(key + 28); desc->buflen = 0; - poly1305_block_init_generic(&desc->state, key); + if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) + poly1305_block_init_arch(&desc->state, key); + else + poly1305_block_init_generic(&desc->state, key); } -EXPORT_SYMBOL_GPL(poly1305_init_generic); +EXPORT_SYMBOL(poly1305_init); static inline void poly1305_blocks(struct poly1305_block_state *state, const u8 *src, unsigned int len) { - poly1305_blocks_generic(state, src, len, 1); + if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) + poly1305_blocks_arch(state, src, len, 1); + else + poly1305_blocks_generic(state, src, len, 1); } -void poly1305_update_generic(struct poly1305_desc_ctx *desc, const u8 *src, - unsigned int nbytes) +void poly1305_update(struct poly1305_desc_ctx *desc, + const u8 *src, unsigned int nbytes) { desc->buflen = BLOCK_HASH_UPDATE(poly1305_blocks, &desc->state, src, nbytes, POLY1305_BLOCK_SIZE, desc->buf, desc->buflen); } -EXPORT_SYMBOL_GPL(poly1305_update_generic); +EXPORT_SYMBOL(poly1305_update); -void poly1305_final_generic(struct poly1305_desc_ctx *desc, u8 *dst) +void poly1305_final(struct poly1305_desc_ctx *desc, u8 *dst) { if (unlikely(desc->buflen)) { desc->buf[desc->buflen++] = 1; memset(desc->buf + desc->buflen, 0, POLY1305_BLOCK_SIZE - desc->buflen); - poly1305_blocks_generic(&desc->state, desc->buf, - POLY1305_BLOCK_SIZE, 0); + if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) + poly1305_blocks_arch(&desc->state, desc->buf, + POLY1305_BLOCK_SIZE, 0); + else + poly1305_blocks_generic(&desc->state, desc->buf, + POLY1305_BLOCK_SIZE, 0); } - poly1305_emit_generic(&desc->h, dst, desc->s); + if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) + poly1305_emit_arch(&desc->state.h, dst, desc->s); + else + poly1305_emit_generic(&desc->state.h, dst, desc->s); *desc = (struct poly1305_desc_ctx){}; } -EXPORT_SYMBOL_GPL(poly1305_final_generic); +EXPORT_SYMBOL(poly1305_final); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Martin Willi "); -- cgit v1.2.3