From c47e1f4142a3823b6e963e14db295a8a733804b5 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 17 Mar 2025 16:33:59 +0800 Subject: crypto: scomp - Allocate per-cpu buffer on first use of each CPU Per-cpu buffers can be wasteful when the number of CPUs is large, especially if the buffer itself is likely to never be used. Reduce such wastage by only allocating them on first use of a particular CPU. On start-up allocate a single buffer on the first possible CPU. For every other CPU a work struct will be scheduled on first use to allocate the buffer for that CPU. Until the allocation succeeds simply use the first CPU's buffer which is protected under a spin lock. Signed-off-by: Herbert Xu --- include/crypto/internal/acompress.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/crypto/internal/acompress.h') diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index aaf59f3236fa..2af690819a83 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -37,7 +37,6 @@ * * @reqsize: Context size for (de)compression requests * @base: Common crypto API algorithm data structure - * @stream: Per-cpu memory for algorithm * @calg: Cmonn algorithm data structure shared with scomp */ struct acomp_alg { -- cgit v1.2.3 From 42d9f6c774790d290c175e8775ce9f1366438098 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 19 Mar 2025 14:04:52 +0800 Subject: crypto: acomp - Move scomp stream allocation code into acomp Move the dynamic stream allocation code into acomp and make it available as a helper for acomp algorithms. Signed-off-by: Herbert Xu --- crypto/acompress.c | 112 ++++++++++++++++++++++++++++++ crypto/scompress.c | 133 ++++-------------------------------- include/crypto/internal/acompress.h | 33 +++++++++ include/crypto/internal/scompress.h | 28 +++----- 4 files changed, 166 insertions(+), 140 deletions(-) (limited to 'include/crypto/internal/acompress.h') diff --git a/crypto/acompress.c b/crypto/acompress.c index f7a3fbe5447e..2cf5e9d749cf 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -9,13 +9,18 @@ #include #include +#include #include #include #include #include +#include #include #include +#include +#include #include +#include #include #include "compress.h" @@ -434,5 +439,112 @@ void crypto_unregister_acomps(struct acomp_alg *algs, int count) } EXPORT_SYMBOL_GPL(crypto_unregister_acomps); +static void acomp_stream_workfn(struct work_struct *work) +{ + struct crypto_acomp_streams *s = + container_of(work, struct crypto_acomp_streams, stream_work); + struct crypto_acomp_stream __percpu *streams = s->streams; + int cpu; + + for_each_cpu(cpu, &s->stream_want) { + struct crypto_acomp_stream *ps; + void *ctx; + + ps = per_cpu_ptr(streams, cpu); + if (ps->ctx) + continue; + + ctx = s->alloc_ctx(); + if (IS_ERR(ctx)) + break; + + spin_lock_bh(&ps->lock); + ps->ctx = ctx; + spin_unlock_bh(&ps->lock); + + cpumask_clear_cpu(cpu, &s->stream_want); + } +} + +void crypto_acomp_free_streams(struct crypto_acomp_streams *s) +{ + struct crypto_acomp_stream __percpu *streams = s->streams; + void (*free_ctx)(void *); + int i; + + cancel_work_sync(&s->stream_work); + free_ctx = s->free_ctx; + + for_each_possible_cpu(i) { + struct crypto_acomp_stream *ps = per_cpu_ptr(streams, i); + + if (!ps->ctx) + continue; + + free_ctx(ps->ctx); + } + + free_percpu(streams); +} +EXPORT_SYMBOL_GPL(crypto_acomp_free_streams); + +int crypto_acomp_alloc_streams(struct crypto_acomp_streams *s) +{ + struct crypto_acomp_stream __percpu *streams; + struct crypto_acomp_stream *ps; + unsigned int i; + void *ctx; + + if (s->streams) + return 0; + + streams = alloc_percpu(struct crypto_acomp_stream); + if (!streams) + return -ENOMEM; + + ctx = s->alloc_ctx(); + if (IS_ERR(ctx)) { + free_percpu(streams); + return PTR_ERR(ctx); + } + + i = cpumask_first(cpu_possible_mask); + ps = per_cpu_ptr(streams, i); + ps->ctx = ctx; + + for_each_possible_cpu(i) { + ps = per_cpu_ptr(streams, i); + spin_lock_init(&ps->lock); + } + + s->streams = streams; + + INIT_WORK(&s->stream_work, acomp_stream_workfn); + return 0; +} +EXPORT_SYMBOL_GPL(crypto_acomp_alloc_streams); + +struct crypto_acomp_stream *crypto_acomp_lock_stream_bh( + struct crypto_acomp_streams *s) __acquires(stream) +{ + struct crypto_acomp_stream __percpu *streams = s->streams; + int cpu = raw_smp_processor_id(); + struct crypto_acomp_stream *ps; + + ps = per_cpu_ptr(streams, cpu); + spin_lock_bh(&ps->lock); + if (likely(ps->ctx)) + return ps; + spin_unlock(&ps->lock); + + cpumask_set_cpu(cpu, &s->stream_want); + schedule_work(&s->stream_work); + + ps = per_cpu_ptr(streams, cpumask_first(cpu_possible_mask)); + spin_lock(&ps->lock); + return ps; +} +EXPORT_SYMBOL_GPL(crypto_acomp_lock_stream_bh); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous compression type"); diff --git a/crypto/scompress.c b/crypto/scompress.c index f4e3376ca7d3..5a40605570f5 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -7,7 +7,6 @@ * Author: Giovanni Cabiddu */ -#include #include #include #include @@ -132,91 +131,15 @@ static int crypto_scomp_alloc_scratches(void) return scomp_alloc_scratch(scratch, i); } -static void scomp_free_streams(struct scomp_alg *alg) -{ - struct crypto_acomp_stream __percpu *stream = alg->stream; - int i; - - for_each_possible_cpu(i) { - struct crypto_acomp_stream *ps = per_cpu_ptr(stream, i); - - if (!ps->ctx) - continue; - - alg->free_ctx(ps->ctx); - } - - free_percpu(stream); -} - -static int scomp_alloc_streams(struct scomp_alg *alg) -{ - struct crypto_acomp_stream __percpu *stream; - struct crypto_acomp_stream *ps; - unsigned int i; - void *ctx; - - stream = alloc_percpu(struct crypto_acomp_stream); - if (!stream) - return -ENOMEM; - - ctx = alg->alloc_ctx(); - if (IS_ERR(ctx)) { - free_percpu(stream); - return PTR_ERR(ctx); - } - - i = cpumask_first(cpu_possible_mask); - ps = per_cpu_ptr(stream, i); - ps->ctx = ctx; - - for_each_possible_cpu(i) { - ps = per_cpu_ptr(stream, i); - spin_lock_init(&ps->lock); - } - - alg->stream = stream; - return 0; -} - -static void scomp_stream_workfn(struct work_struct *work) -{ - struct scomp_alg *alg = container_of(work, struct scomp_alg, - stream_work); - struct crypto_acomp_stream __percpu *stream = alg->stream; - int cpu; - - for_each_cpu(cpu, &alg->stream_want) { - struct crypto_acomp_stream *ps; - void *ctx; - - ps = per_cpu_ptr(stream, cpu); - if (ps->ctx) - continue; - - ctx = alg->alloc_ctx(); - if (IS_ERR(ctx)) - break; - - spin_lock_bh(&ps->lock); - ps->ctx = ctx; - spin_unlock_bh(&ps->lock); - - cpumask_clear_cpu(cpu, &alg->stream_want); - } -} - static int crypto_scomp_init_tfm(struct crypto_tfm *tfm) { struct scomp_alg *alg = crypto_scomp_alg(__crypto_scomp_tfm(tfm)); int ret = 0; mutex_lock(&scomp_lock); - if (!alg->stream) { - ret = scomp_alloc_streams(alg); - if (ret) - goto unlock; - } + ret = crypto_acomp_alloc_streams(&alg->streams); + if (ret) + goto unlock; if (!scomp_scratch_users) { ret = crypto_scomp_alloc_scratches(); if (ret) @@ -229,13 +152,13 @@ unlock: return ret; } -static struct scomp_scratch *scomp_lock_scratch_bh(void) __acquires(scratch) +static struct scomp_scratch *scomp_lock_scratch(void) __acquires(scratch) { int cpu = raw_smp_processor_id(); struct scomp_scratch *scratch; scratch = per_cpu_ptr(&scomp_scratch, cpu); - spin_lock_bh(&scratch->lock); + spin_lock(&scratch->lock); if (likely(scratch->src)) return scratch; spin_unlock(&scratch->lock); @@ -248,39 +171,10 @@ static struct scomp_scratch *scomp_lock_scratch_bh(void) __acquires(scratch) return scratch; } -static inline void scomp_unlock_scratch_bh(struct scomp_scratch *scratch) +static inline void scomp_unlock_scratch(struct scomp_scratch *scratch) __releases(scratch) { - spin_unlock_bh(&scratch->lock); -} - -static struct crypto_acomp_stream *scomp_lock_stream(struct crypto_scomp *tfm) - __acquires(stream) -{ - struct scomp_alg *alg = crypto_scomp_alg(tfm); - struct crypto_acomp_stream __percpu *stream; - int cpu = raw_smp_processor_id(); - struct crypto_acomp_stream *ps; - - stream = alg->stream; - ps = per_cpu_ptr(stream, cpu); - spin_lock(&ps->lock); - if (likely(ps->ctx)) - return ps; - spin_unlock(&ps->lock); - - cpumask_set_cpu(cpu, &alg->stream_want); - schedule_work(&alg->stream_work); - - ps = per_cpu_ptr(stream, cpumask_first(cpu_possible_mask)); - spin_lock(&ps->lock); - return ps; -} - -static inline void scomp_unlock_stream(struct crypto_acomp_stream *stream) - __releases(stream) -{ - spin_unlock(&stream->lock); + spin_unlock(&scratch->lock); } static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) @@ -306,7 +200,8 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) if (!req->dst || !dlen) return -EINVAL; - scratch = scomp_lock_scratch_bh(); + stream = crypto_acomp_lock_stream_bh(&crypto_scomp_alg(scomp)->streams); + scratch = scomp_lock_scratch(); if (acomp_request_src_isvirt(req)) src = req->svirt; @@ -367,7 +262,6 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) dlen = min(dlen, max); } - stream = scomp_lock_stream(scomp); if (dir) ret = crypto_scomp_compress(scomp, src, slen, dst, &dlen, stream->ctx); @@ -378,8 +272,8 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) if (dst == scratch->dst) memcpy_to_sglist(req->dst, 0, dst, dlen); - scomp_unlock_stream(stream); - scomp_unlock_scratch_bh(scratch); + scomp_unlock_scratch(scratch); + crypto_acomp_unlock_stream_bh(stream); req->dlen = dlen; @@ -466,8 +360,7 @@ static void crypto_scomp_destroy(struct crypto_alg *alg) { struct scomp_alg *scomp = __crypto_scomp_alg(alg); - cancel_work_sync(&scomp->stream_work); - scomp_free_streams(scomp); + crypto_acomp_free_streams(&scomp->streams); } static const struct crypto_type crypto_scomp_type = { @@ -493,8 +386,6 @@ static void scomp_prepare_alg(struct scomp_alg *alg) comp_prepare_alg(&alg->calg); base->cra_flags |= CRYPTO_ALG_REQ_CHAIN; - - INIT_WORK(&alg->stream_work, scomp_stream_workfn); } int crypto_register_scomp(struct scomp_alg *alg) diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 2af690819a83..ee5eff19eaf4 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -11,6 +11,10 @@ #include #include +#include +#include +#include +#include #define ACOMP_REQUEST_ON_STACK(name, tfm) \ char __##name##_req[sizeof(struct acomp_req) + \ @@ -53,6 +57,24 @@ struct acomp_alg { }; }; +struct crypto_acomp_stream { + spinlock_t lock; + void *ctx; +}; + +struct crypto_acomp_streams { + /* These must come first because of struct scomp_alg. */ + void *(*alloc_ctx)(void); + union { + void (*free_ctx)(void *); + void (*cfree_ctx)(const void *); + }; + + struct crypto_acomp_stream __percpu *streams; + struct work_struct stream_work; + cpumask_t stream_want; +}; + /* * Transform internal helpers. */ @@ -157,4 +179,15 @@ static inline bool crypto_acomp_req_chain(struct crypto_acomp *tfm) return crypto_tfm_req_chain(&tfm->base); } +void crypto_acomp_free_streams(struct crypto_acomp_streams *s); +int crypto_acomp_alloc_streams(struct crypto_acomp_streams *s); + +struct crypto_acomp_stream *crypto_acomp_lock_stream_bh( + struct crypto_acomp_streams *s) __acquires(stream); + +static inline void crypto_acomp_unlock_stream_bh( + struct crypto_acomp_stream *stream) __releases(stream) +{ + spin_unlock_bh(&stream->lock); +} #endif diff --git a/include/crypto/internal/scompress.h b/include/crypto/internal/scompress.h index fd74e656ffd2..533d6c16a491 100644 --- a/include/crypto/internal/scompress.h +++ b/include/crypto/internal/scompress.h @@ -9,22 +9,12 @@ #ifndef _CRYPTO_SCOMP_INT_H #define _CRYPTO_SCOMP_INT_H -#include -#include -#include -#include - -struct acomp_req; +#include struct crypto_scomp { struct crypto_tfm base; }; -struct crypto_acomp_stream { - spinlock_t lock; - void *ctx; -}; - /** * struct scomp_alg - synchronous compression algorithm * @@ -33,14 +23,10 @@ struct crypto_acomp_stream { * @compress: Function performs a compress operation * @decompress: Function performs a de-compress operation * @base: Common crypto API algorithm data structure - * @stream: Per-cpu memory for algorithm - * @stream_work: Work struct to allocate stream memmory - * @stream_want: CPU mask for allocating stream memory + * @streams: Per-cpu memory for algorithm * @calg: Cmonn algorithm data structure shared with acomp */ struct scomp_alg { - void *(*alloc_ctx)(void); - void (*free_ctx)(void *ctx); int (*compress)(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx); @@ -48,9 +34,13 @@ struct scomp_alg { unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx); - struct crypto_acomp_stream __percpu *stream; - struct work_struct stream_work; - cpumask_t stream_want; + union { + struct { + void *(*alloc_ctx)(void); + void (*free_ctx)(void *ctx); + }; + struct crypto_acomp_streams streams; + }; union { struct COMP_ALG_COMMON; -- cgit v1.2.3 From 9c8cf582626ef56632006212b385cfbb6c54f094 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 19 Mar 2025 14:04:54 +0800 Subject: crypto: acomp - Add acomp_walk Add acomp_walk which is similar to skcipher_walk but tailored for acomp. Signed-off-by: Herbert Xu --- crypto/acompress.c | 116 ++++++++++++++++++++++++++++++++++++ include/crypto/internal/acompress.h | 44 ++++++++++++++ 2 files changed, 160 insertions(+) (limited to 'include/crypto/internal/acompress.h') diff --git a/crypto/acompress.c b/crypto/acompress.c index 2cf5e9d749cf..70bc1fd044e4 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -15,6 +16,8 @@ #include #include #include +#include +#include #include #include #include @@ -27,6 +30,14 @@ struct crypto_scomp; +enum { + ACOMP_WALK_SLEEP = 1 << 0, + ACOMP_WALK_SRC_LINEAR = 1 << 1, + ACOMP_WALK_SRC_FOLIO = 1 << 2, + ACOMP_WALK_DST_LINEAR = 1 << 3, + ACOMP_WALK_DST_FOLIO = 1 << 4, +}; + static const struct crypto_type crypto_acomp_type; static void acomp_reqchain_done(void *data, int err); @@ -546,5 +557,110 @@ struct crypto_acomp_stream *crypto_acomp_lock_stream_bh( } EXPORT_SYMBOL_GPL(crypto_acomp_lock_stream_bh); +void acomp_walk_done_src(struct acomp_walk *walk, int used) +{ + walk->slen -= used; + if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) + scatterwalk_advance(&walk->in, used); + else + scatterwalk_done_src(&walk->in, used); + + if ((walk->flags & ACOMP_WALK_SLEEP)) + cond_resched(); +} +EXPORT_SYMBOL_GPL(acomp_walk_done_src); + +void acomp_walk_done_dst(struct acomp_walk *walk, int used) +{ + walk->dlen -= used; + if ((walk->flags & ACOMP_WALK_DST_LINEAR)) + scatterwalk_advance(&walk->out, used); + else + scatterwalk_done_dst(&walk->out, used); + + if ((walk->flags & ACOMP_WALK_SLEEP)) + cond_resched(); +} +EXPORT_SYMBOL_GPL(acomp_walk_done_dst); + +int acomp_walk_next_src(struct acomp_walk *walk) +{ + unsigned int slen = walk->slen; + unsigned int max = UINT_MAX; + + if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP)) + max = PAGE_SIZE; + if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) { + walk->in.__addr = (void *)(((u8 *)walk->in.sg) + + walk->in.offset); + return min(slen, max); + } + + return slen ? scatterwalk_next(&walk->in, slen) : 0; +} +EXPORT_SYMBOL_GPL(acomp_walk_next_src); + +int acomp_walk_next_dst(struct acomp_walk *walk) +{ + unsigned int dlen = walk->dlen; + unsigned int max = UINT_MAX; + + if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP)) + max = PAGE_SIZE; + if ((walk->flags & ACOMP_WALK_DST_LINEAR)) { + walk->out.__addr = (void *)(((u8 *)walk->out.sg) + + walk->out.offset); + return min(dlen, max); + } + + return dlen ? scatterwalk_next(&walk->out, dlen) : 0; +} +EXPORT_SYMBOL_GPL(acomp_walk_next_dst); + +int acomp_walk_virt(struct acomp_walk *__restrict walk, + struct acomp_req *__restrict req) +{ + struct scatterlist *src = req->src; + struct scatterlist *dst = req->dst; + + walk->slen = req->slen; + walk->dlen = req->dlen; + + if (!walk->slen || !walk->dlen) + return -EINVAL; + + walk->flags = 0; + if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP)) + walk->flags |= ACOMP_WALK_SLEEP; + if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT)) + walk->flags |= ACOMP_WALK_SRC_LINEAR; + else if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_FOLIO)) { + src = &req->chain.ssg; + sg_init_table(src, 1); + sg_set_folio(src, req->sfolio, walk->slen, req->soff); + } + if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_VIRT)) + walk->flags |= ACOMP_WALK_DST_LINEAR; + else if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_FOLIO)) { + dst = &req->chain.dsg; + sg_init_table(dst, 1); + sg_set_folio(dst, req->dfolio, walk->dlen, req->doff); + } + + if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) { + walk->in.sg = (void *)req->svirt; + walk->in.offset = 0; + } else + scatterwalk_start(&walk->in, src); + if ((walk->flags & ACOMP_WALK_DST_LINEAR)) { + walk->out.sg = (void *)req->dvirt; + walk->out.offset = 0; + } else + scatterwalk_start(&walk->out, dst); + + return 0; +} +EXPORT_SYMBOL_GPL(acomp_walk_virt); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous compression type"); diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index ee5eff19eaf4..fbbff9a8a2d9 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -75,6 +76,37 @@ struct crypto_acomp_streams { cpumask_t stream_want; }; +struct acomp_walk { + union { + /* Virtual address of the source. */ + struct { + struct { + const void *const addr; + } virt; + } src; + + /* Private field for the API, do not use. */ + struct scatter_walk in; + }; + + union { + /* Virtual address of the destination. */ + struct { + struct { + void *const addr; + } virt; + } dst; + + /* Private field for the API, do not use. */ + struct scatter_walk out; + }; + + unsigned int slen; + unsigned int dlen; + + int flags; +}; + /* * Transform internal helpers. */ @@ -190,4 +222,16 @@ static inline void crypto_acomp_unlock_stream_bh( { spin_unlock_bh(&stream->lock); } + +void acomp_walk_done_src(struct acomp_walk *walk, int used); +void acomp_walk_done_dst(struct acomp_walk *walk, int used); +int acomp_walk_next_src(struct acomp_walk *walk); +int acomp_walk_next_dst(struct acomp_walk *walk); +int acomp_walk_virt(struct acomp_walk *__restrict walk, + struct acomp_req *__restrict req); + +static inline bool acomp_walk_more_src(const struct acomp_walk *walk, int cur) +{ + return walk->slen != cur; +} #endif -- cgit v1.2.3 From 64929fe8c0a43508eee952cf57903a61c52601e7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 13:36:55 +0800 Subject: crypto: acomp - Remove request chaining Request chaining requires the user to do too much book keeping. Remove it from acomp. Signed-off-by: Herbert Xu --- crypto/acompress.c | 117 ++++++++++-------------------------- crypto/scompress.c | 18 +----- include/crypto/acompress.h | 14 ----- include/crypto/internal/acompress.h | 5 -- 4 files changed, 35 insertions(+), 119 deletions(-) (limited to 'include/crypto/internal/acompress.h') diff --git a/crypto/acompress.c b/crypto/acompress.c index 5d0b8b8b84f6..b682a88781f0 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -177,7 +177,6 @@ static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt) state->data = req->base.data; req->base.complete = cplt; req->base.data = state; - state->req0 = req; } static void acomp_restore_req(struct acomp_req *req) @@ -188,23 +187,20 @@ static void acomp_restore_req(struct acomp_req *req) req->base.data = state->data; } -static void acomp_reqchain_virt(struct acomp_req_chain *state, int err) +static void acomp_reqchain_virt(struct acomp_req *req) { - struct acomp_req *req = state->cur; + struct acomp_req_chain *state = &req->chain; unsigned int slen = req->slen; unsigned int dlen = req->dlen; - req->base.err = err; - state = &req->chain; - if (state->flags & CRYPTO_ACOMP_REQ_SRC_VIRT) acomp_request_set_src_dma(req, state->src, slen); else if (state->flags & CRYPTO_ACOMP_REQ_SRC_FOLIO) - acomp_request_set_src_folio(req, state->sfolio, state->soff, slen); + acomp_request_set_src_folio(req, state->sfolio, req->soff, slen); if (state->flags & CRYPTO_ACOMP_REQ_DST_VIRT) acomp_request_set_dst_dma(req, state->dst, dlen); else if (state->flags & CRYPTO_ACOMP_REQ_DST_FOLIO) - acomp_request_set_dst_folio(req, state->dfolio, state->doff, dlen); + acomp_request_set_dst_folio(req, state->dfolio, req->doff, dlen); } static void acomp_virt_to_sg(struct acomp_req *req) @@ -229,7 +225,6 @@ static void acomp_virt_to_sg(struct acomp_req *req) size_t off = req->soff; state->sfolio = folio; - state->soff = off; sg_init_table(&state->ssg, 1); sg_set_page(&state->ssg, folio_page(folio, off / PAGE_SIZE), slen, off % PAGE_SIZE); @@ -249,7 +244,6 @@ static void acomp_virt_to_sg(struct acomp_req *req) size_t off = req->doff; state->dfolio = folio; - state->doff = off; sg_init_table(&state->dsg, 1); sg_set_page(&state->dsg, folio_page(folio, off / PAGE_SIZE), dlen, off % PAGE_SIZE); @@ -257,8 +251,7 @@ static void acomp_virt_to_sg(struct acomp_req *req) } } -static int acomp_do_nondma(struct acomp_req_chain *state, - struct acomp_req *req) +static int acomp_do_nondma(struct acomp_req *req, bool comp) { u32 keep = CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | @@ -275,7 +268,7 @@ static int acomp_do_nondma(struct acomp_req_chain *state, fbreq->slen = req->slen; fbreq->dlen = req->dlen; - if (state->op == crypto_acomp_reqtfm(req)->compress) + if (comp) err = crypto_acomp_compress(fbreq); else err = crypto_acomp_decompress(fbreq); @@ -284,114 +277,70 @@ static int acomp_do_nondma(struct acomp_req_chain *state, return err; } -static int acomp_do_one_req(struct acomp_req_chain *state, - struct acomp_req *req) +static int acomp_do_one_req(struct acomp_req *req, bool comp) { - state->cur = req; - if (acomp_request_isnondma(req)) - return acomp_do_nondma(state, req); + return acomp_do_nondma(req, comp); acomp_virt_to_sg(req); - return state->op(req); + return comp ? crypto_acomp_reqtfm(req)->compress(req) : + crypto_acomp_reqtfm(req)->decompress(req); } -static int acomp_reqchain_finish(struct acomp_req *req0, int err, u32 mask) +static int acomp_reqchain_finish(struct acomp_req *req, int err) { - struct acomp_req_chain *state = req0->base.data; - struct acomp_req *req = state->cur; - struct acomp_req *n; - - acomp_reqchain_virt(state, err); - - if (req != req0) - list_add_tail(&req->base.list, &req0->base.list); - - list_for_each_entry_safe(req, n, &state->head, base.list) { - list_del_init(&req->base.list); - - req->base.flags &= mask; - req->base.complete = acomp_reqchain_done; - req->base.data = state; - - err = acomp_do_one_req(state, req); - - if (err == -EINPROGRESS) { - if (!list_empty(&state->head)) - err = -EBUSY; - goto out; - } - - if (err == -EBUSY) - goto out; - - acomp_reqchain_virt(state, err); - list_add_tail(&req->base.list, &req0->base.list); - } - - acomp_restore_req(req0); - -out: + acomp_reqchain_virt(req); + acomp_restore_req(req); return err; } static void acomp_reqchain_done(void *data, int err) { - struct acomp_req_chain *state = data; - crypto_completion_t compl = state->compl; + struct acomp_req *req = data; + crypto_completion_t compl; - data = state->data; + compl = req->chain.compl; + data = req->chain.data; - if (err == -EINPROGRESS) { - if (!list_empty(&state->head)) - return; + if (err == -EINPROGRESS) goto notify; - } - err = acomp_reqchain_finish(state->req0, err, - CRYPTO_TFM_REQ_MAY_BACKLOG); - if (err == -EBUSY) - return; + err = acomp_reqchain_finish(req, err); notify: compl(data, err); } -static int acomp_do_req_chain(struct acomp_req *req, - int (*op)(struct acomp_req *req)) +static int acomp_do_req_chain(struct acomp_req *req, bool comp) { - struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); - struct acomp_req_chain *state; int err; - if (crypto_acomp_req_chain(tfm) || - (!acomp_request_chained(req) && acomp_request_issg(req))) - return op(req); - acomp_save_req(req, acomp_reqchain_done); - state = req->base.data; - state->op = op; - state->src = NULL; - INIT_LIST_HEAD(&state->head); - list_splice_init(&req->base.list, &state->head); - - err = acomp_do_one_req(state, req); + err = acomp_do_one_req(req, comp); if (err == -EBUSY || err == -EINPROGRESS) - return -EBUSY; + return err; - return acomp_reqchain_finish(req, err, ~0); + return acomp_reqchain_finish(req, err); } int crypto_acomp_compress(struct acomp_req *req) { - return acomp_do_req_chain(req, crypto_acomp_reqtfm(req)->compress); + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + + if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) + crypto_acomp_reqtfm(req)->compress(req); + return acomp_do_req_chain(req, true); } EXPORT_SYMBOL_GPL(crypto_acomp_compress); int crypto_acomp_decompress(struct acomp_req *req) { - return acomp_do_req_chain(req, crypto_acomp_reqtfm(req)->decompress); + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + + if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) + crypto_acomp_reqtfm(req)->decompress(req); + return acomp_do_req_chain(req, false); } EXPORT_SYMBOL_GPL(crypto_acomp_decompress); diff --git a/crypto/scompress.c b/crypto/scompress.c index 4a2b3933aa95..7ade3f2fee7e 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -284,28 +284,14 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) return ret; } -static int scomp_acomp_chain(struct acomp_req *req, int dir) -{ - struct acomp_req *r2; - int err; - - err = scomp_acomp_comp_decomp(req, dir); - req->base.err = err; - - list_for_each_entry(r2, &req->base.list, base.list) - r2->base.err = scomp_acomp_comp_decomp(r2, dir); - - return err; -} - static int scomp_acomp_compress(struct acomp_req *req) { - return scomp_acomp_chain(req, 1); + return scomp_acomp_comp_decomp(req, 1); } static int scomp_acomp_decompress(struct acomp_req *req) { - return scomp_acomp_chain(req, 0); + return scomp_acomp_comp_decomp(req, 0); } static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm) diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index f0e01ff77d92..080e134df35c 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -52,10 +52,6 @@ struct acomp_req; struct folio; struct acomp_req_chain { - struct list_head head; - struct acomp_req *req0; - struct acomp_req *cur; - int (*op)(struct acomp_req *req); crypto_completion_t compl; void *data; struct scatterlist ssg; @@ -68,8 +64,6 @@ struct acomp_req_chain { u8 *dst; struct folio *dfolio; }; - size_t soff; - size_t doff; u32 flags; }; @@ -343,8 +337,6 @@ static inline void acomp_request_set_callback(struct acomp_req *req, req->base.data = data; req->base.flags &= keep; req->base.flags |= flgs & ~keep; - - crypto_reqchain_init(&req->base); } /** @@ -552,12 +544,6 @@ static inline void acomp_request_set_dst_folio(struct acomp_req *req, req->base.flags |= CRYPTO_ACOMP_REQ_DST_FOLIO; } -static inline void acomp_request_chain(struct acomp_req *req, - struct acomp_req *head) -{ - crypto_request_chain(&req->base, &head->base); -} - /** * crypto_acomp_compress() -- Invoke asynchronous compress operation * diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index fbbff9a8a2d9..960cdd1f3a57 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -151,11 +151,6 @@ void crypto_unregister_acomp(struct acomp_alg *alg); int crypto_register_acomps(struct acomp_alg *algs, int count); void crypto_unregister_acomps(struct acomp_alg *algs, int count); -static inline bool acomp_request_chained(struct acomp_req *req) -{ - return crypto_request_chained(&req->base); -} - static inline bool acomp_request_issg(struct acomp_req *req) { return !(req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | -- cgit v1.2.3 From b04b395f7a29ed28d3cb27a7b39ac67dfb959fa0 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:02:53 +0800 Subject: crypto: acomp - Use request flag helpers and add acomp_request_flags Use the newly added request flag helpers to manage the request flags. Also add acomp_request_flags which lets bottom-level users to access the request flags without the bits private to the acomp API. Signed-off-by: Herbert Xu --- include/crypto/acompress.h | 27 ++++++++++++++++----------- include/crypto/internal/acompress.h | 6 ++++++ 2 files changed, 22 insertions(+), 11 deletions(-) (limited to 'include/crypto/internal/acompress.h') diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index 080e134df35c..f383a4008854 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -38,6 +38,12 @@ /* Set this bit if destination is a folio. */ #define CRYPTO_ACOMP_REQ_DST_FOLIO 0x00000040 +/* Private flags that should not be touched by the user. */ +#define CRYPTO_ACOMP_REQ_PRIVATE \ + (CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | \ + CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA | \ + CRYPTO_ACOMP_REQ_SRC_FOLIO | CRYPTO_ACOMP_REQ_DST_FOLIO) + #define CRYPTO_ACOMP_DST_MAX 131072 #define MAX_SYNC_COMP_REQSIZE 0 @@ -201,7 +207,7 @@ static inline unsigned int crypto_acomp_reqsize(struct crypto_acomp *tfm) static inline void acomp_request_set_tfm(struct acomp_req *req, struct crypto_acomp *tfm) { - req->base.tfm = crypto_acomp_tfm(tfm); + crypto_request_set_tfm(&req->base, crypto_acomp_tfm(tfm)); } static inline bool acomp_is_async(struct crypto_acomp *tfm) @@ -298,6 +304,11 @@ static inline void *acomp_request_extra(struct acomp_req *req) return (void *)((char *)req + len); } +static inline bool acomp_req_on_stack(struct acomp_req *req) +{ + return crypto_req_on_stack(&req->base); +} + /** * acomp_request_free() -- zeroize and free asynchronous (de)compression * request as well as the output buffer if allocated @@ -307,7 +318,7 @@ static inline void *acomp_request_extra(struct acomp_req *req) */ static inline void acomp_request_free(struct acomp_req *req) { - if (!req || (req->base.flags & CRYPTO_TFM_REQ_ON_STACK)) + if (!req || acomp_req_on_stack(req)) return; kfree_sensitive(req); } @@ -328,15 +339,9 @@ static inline void acomp_request_set_callback(struct acomp_req *req, crypto_completion_t cmpl, void *data) { - u32 keep = CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | - CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA | - CRYPTO_ACOMP_REQ_SRC_FOLIO | CRYPTO_ACOMP_REQ_DST_FOLIO | - CRYPTO_TFM_REQ_ON_STACK; - - req->base.complete = cmpl; - req->base.data = data; - req->base.flags &= keep; - req->base.flags |= flgs & ~keep; + flgs &= ~CRYPTO_ACOMP_REQ_PRIVATE; + flgs |= req->base.flags & CRYPTO_ACOMP_REQ_PRIVATE; + crypto_request_set_callback(&req->base, flgs, cmpl, data); } /** diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 960cdd1f3a57..5483ca5b46ad 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -229,4 +229,10 @@ static inline bool acomp_walk_more_src(const struct acomp_walk *walk, int cur) { return walk->slen != cur; } + +static inline u32 acomp_request_flags(struct acomp_req *req) +{ + return crypto_request_flags(&req->base) & ~CRYPTO_ACOMP_REQ_PRIVATE; +} + #endif -- cgit v1.2.3 From 05fa2c6e87da31eab150cdaca6697cd1de122ec7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:02:55 +0800 Subject: crypto: acomp - Add ACOMP_FBREQ_ON_STACK Add a helper to create an on-stack fallback request from a given request. Use this helper in acomp_do_nondma. Signed-off-by: Herbert Xu --- crypto/acompress.c | 14 +------------- include/crypto/internal/acompress.h | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 13 deletions(-) (limited to 'include/crypto/internal/acompress.h') diff --git a/crypto/acompress.c b/crypto/acompress.c index b682a88781f0..f343b1a4b1d1 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -253,21 +253,9 @@ static void acomp_virt_to_sg(struct acomp_req *req) static int acomp_do_nondma(struct acomp_req *req, bool comp) { - u32 keep = CRYPTO_ACOMP_REQ_SRC_VIRT | - CRYPTO_ACOMP_REQ_SRC_NONDMA | - CRYPTO_ACOMP_REQ_DST_VIRT | - CRYPTO_ACOMP_REQ_DST_NONDMA; - ACOMP_REQUEST_ON_STACK(fbreq, crypto_acomp_reqtfm(req)); + ACOMP_FBREQ_ON_STACK(fbreq, req); int err; - acomp_request_set_callback(fbreq, req->base.flags, NULL, NULL); - fbreq->base.flags &= ~keep; - fbreq->base.flags |= req->base.flags & keep; - fbreq->src = req->src; - fbreq->dst = req->dst; - fbreq->slen = req->slen; - fbreq->dlen = req->dlen; - if (comp) err = crypto_acomp_compress(fbreq); else diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 5483ca5b46ad..8840fd2c1db5 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -23,6 +23,12 @@ struct acomp_req *name = acomp_request_on_stack_init( \ __##name##_req, (tfm), 0, true) +#define ACOMP_FBREQ_ON_STACK(name, req) \ + char __##name##_req[sizeof(struct acomp_req) + \ + MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ + struct acomp_req *name = acomp_fbreq_on_stack_init( \ + __##name##_req, (req)) + /** * struct acomp_alg - asynchronous compression algorithm * @@ -235,4 +241,24 @@ static inline u32 acomp_request_flags(struct acomp_req *req) return crypto_request_flags(&req->base) & ~CRYPTO_ACOMP_REQ_PRIVATE; } +static inline struct acomp_req *acomp_fbreq_on_stack_init( + char *buf, struct acomp_req *old) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(old); + struct acomp_req *req; + + req = acomp_request_on_stack_init(buf, tfm, 0, true); + acomp_request_set_callback(req, acomp_request_flags(old), NULL, NULL); + req->base.flags &= ~CRYPTO_ACOMP_REQ_PRIVATE; + req->base.flags |= old->base.flags & CRYPTO_ACOMP_REQ_PRIVATE; + req->src = old->src; + req->dst = old->dst; + req->slen = old->slen; + req->dlen = old->dlen; + req->soff = old->soff; + req->doff = old->doff; + + return req; +} + #endif -- cgit v1.2.3 From 097c432caaa6d91f87732fe991cb08139e31101a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:03:00 +0800 Subject: crypto: acomp - Add ACOMP_REQUEST_CLONE Add a new helper ACOMP_REQUEST_CLONE that will transform a stack request into a dynamically allocated one if possible, and otherwise switch it over to the sycnrhonous fallback transform. The intended usage is: ACOMP_STACK_ON_REQUEST(req, tfm); ... err = crypto_acomp_compress(req); /* The request cannot complete synchronously. */ if (err == -EAGAIN) { /* This will not fail. */ req = ACOMP_REQUEST_CLONE(req, gfp); /* Redo operation. */ err = crypto_acomp_compress(req); } Signed-off-by: Herbert Xu --- crypto/acompress.c | 23 +++++++++++++++++++++++ include/crypto/acompress.h | 30 ++++++++++++++++++++++++++---- include/crypto/internal/acompress.h | 11 +++-------- 3 files changed, 52 insertions(+), 12 deletions(-) (limited to 'include/crypto/internal/acompress.h') diff --git a/crypto/acompress.c b/crypto/acompress.c index f343b1a4b1d1..530b9bfd03a5 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -316,6 +316,8 @@ int crypto_acomp_compress(struct acomp_req *req) { struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + if (acomp_req_on_stack(req) && acomp_is_async(tfm)) + return -EAGAIN; if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) crypto_acomp_reqtfm(req)->compress(req); return acomp_do_req_chain(req, true); @@ -326,6 +328,8 @@ int crypto_acomp_decompress(struct acomp_req *req) { struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + if (acomp_req_on_stack(req) && acomp_is_async(tfm)) + return -EAGAIN; if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) crypto_acomp_reqtfm(req)->decompress(req); return acomp_do_req_chain(req, false); @@ -603,5 +607,24 @@ int acomp_walk_virt(struct acomp_walk *__restrict walk, } EXPORT_SYMBOL_GPL(acomp_walk_virt); +struct acomp_req *acomp_request_clone(struct acomp_req *req, + size_t total, gfp_t gfp) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + struct acomp_req *nreq; + + nreq = kmalloc(total, gfp); + if (!nreq) { + acomp_request_set_tfm(req, tfm->fb); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + return req; + } + + memcpy(nreq, req, total); + acomp_request_set_tfm(req, tfm); + return req; +} +EXPORT_SYMBOL_GPL(acomp_request_clone); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous compression type"); diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index f383a4008854..93cee67c27c0 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -49,10 +49,19 @@ #define MAX_SYNC_COMP_REQSIZE 0 #define ACOMP_REQUEST_ALLOC(name, tfm, gfp) \ + char __##name##_req[sizeof(struct acomp_req) + \ + MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ + struct acomp_req *name = acomp_request_alloc_init( \ + __##name##_req, (tfm), (gfp)) + +#define ACOMP_REQUEST_ON_STACK(name, tfm) \ char __##name##_req[sizeof(struct acomp_req) + \ MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ struct acomp_req *name = acomp_request_on_stack_init( \ - __##name##_req, (tfm), (gfp), false) + __##name##_req, (tfm)) + +#define ACOMP_REQUEST_CLONE(name, gfp) \ + acomp_request_clone(name, sizeof(__##name##_req), gfp) struct acomp_req; struct folio; @@ -571,12 +580,12 @@ int crypto_acomp_compress(struct acomp_req *req); */ int crypto_acomp_decompress(struct acomp_req *req); -static inline struct acomp_req *acomp_request_on_stack_init( - char *buf, struct crypto_acomp *tfm, gfp_t gfp, bool stackonly) +static inline struct acomp_req *acomp_request_alloc_init( + char *buf, struct crypto_acomp *tfm, gfp_t gfp) { struct acomp_req *req; - if (!stackonly && (req = acomp_request_alloc(tfm, gfp))) + if ((req = acomp_request_alloc(tfm, gfp))) return req; req = (void *)buf; @@ -586,4 +595,17 @@ static inline struct acomp_req *acomp_request_on_stack_init( return req; } +static inline struct acomp_req *acomp_request_on_stack_init( + char *buf, struct crypto_acomp *tfm) +{ + struct acomp_req *req = (void *)buf; + + acomp_request_set_tfm(req, tfm); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + return req; +} + +struct acomp_req *acomp_request_clone(struct acomp_req *req, + size_t total, gfp_t gfp); + #endif diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 8840fd2c1db5..b51d66633935 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -17,12 +17,6 @@ #include #include -#define ACOMP_REQUEST_ON_STACK(name, tfm) \ - char __##name##_req[sizeof(struct acomp_req) + \ - MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ - struct acomp_req *name = acomp_request_on_stack_init( \ - __##name##_req, (tfm), 0, true) - #define ACOMP_FBREQ_ON_STACK(name, req) \ char __##name##_req[sizeof(struct acomp_req) + \ MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ @@ -245,9 +239,10 @@ static inline struct acomp_req *acomp_fbreq_on_stack_init( char *buf, struct acomp_req *old) { struct crypto_acomp *tfm = crypto_acomp_reqtfm(old); - struct acomp_req *req; + struct acomp_req *req = (void *)buf; - req = acomp_request_on_stack_init(buf, tfm, 0, true); + acomp_request_set_tfm(req, tfm->fb); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; acomp_request_set_callback(req, acomp_request_flags(old), NULL, NULL); req->base.flags &= ~CRYPTO_ACOMP_REQ_PRIVATE; req->base.flags |= old->base.flags & CRYPTO_ACOMP_REQ_PRIVATE; -- cgit v1.2.3 From 5f3437e9c89eec7bbf0ee5f582894d41f57528bc Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:05:27 +0800 Subject: crypto: acomp - Simplify folio handling Rather than storing the folio as is and handling it later, convert it to a scatterlist right away. Signed-off-by: Herbert Xu --- crypto/acompress.c | 42 ++-------------------------------- crypto/scompress.c | 10 ++------- include/crypto/acompress.h | 45 ++++++------------------------------- include/crypto/internal/acompress.h | 16 +------------ 4 files changed, 12 insertions(+), 101 deletions(-) (limited to 'include/crypto/internal/acompress.h') diff --git a/crypto/acompress.c b/crypto/acompress.c index 530b9bfd03a5..674325ecefbc 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -33,9 +33,7 @@ struct crypto_scomp; enum { ACOMP_WALK_SLEEP = 1 << 0, ACOMP_WALK_SRC_LINEAR = 1 << 1, - ACOMP_WALK_SRC_FOLIO = 1 << 2, - ACOMP_WALK_DST_LINEAR = 1 << 3, - ACOMP_WALK_DST_FOLIO = 1 << 4, + ACOMP_WALK_DST_LINEAR = 1 << 2, }; static const struct crypto_type crypto_acomp_type; @@ -195,12 +193,8 @@ static void acomp_reqchain_virt(struct acomp_req *req) if (state->flags & CRYPTO_ACOMP_REQ_SRC_VIRT) acomp_request_set_src_dma(req, state->src, slen); - else if (state->flags & CRYPTO_ACOMP_REQ_SRC_FOLIO) - acomp_request_set_src_folio(req, state->sfolio, req->soff, slen); if (state->flags & CRYPTO_ACOMP_REQ_DST_VIRT) acomp_request_set_dst_dma(req, state->dst, dlen); - else if (state->flags & CRYPTO_ACOMP_REQ_DST_FOLIO) - acomp_request_set_dst_folio(req, state->dfolio, req->doff, dlen); } static void acomp_virt_to_sg(struct acomp_req *req) @@ -208,9 +202,7 @@ static void acomp_virt_to_sg(struct acomp_req *req) struct acomp_req_chain *state = &req->chain; state->flags = req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | - CRYPTO_ACOMP_REQ_DST_VIRT | - CRYPTO_ACOMP_REQ_SRC_FOLIO | - CRYPTO_ACOMP_REQ_DST_FOLIO); + CRYPTO_ACOMP_REQ_DST_VIRT); if (acomp_request_src_isvirt(req)) { unsigned int slen = req->slen; @@ -219,16 +211,6 @@ static void acomp_virt_to_sg(struct acomp_req *req) state->src = svirt; sg_init_one(&state->ssg, svirt, slen); acomp_request_set_src_sg(req, &state->ssg, slen); - } else if (acomp_request_src_isfolio(req)) { - struct folio *folio = req->sfolio; - unsigned int slen = req->slen; - size_t off = req->soff; - - state->sfolio = folio; - sg_init_table(&state->ssg, 1); - sg_set_page(&state->ssg, folio_page(folio, off / PAGE_SIZE), - slen, off % PAGE_SIZE); - acomp_request_set_src_sg(req, &state->ssg, slen); } if (acomp_request_dst_isvirt(req)) { @@ -238,16 +220,6 @@ static void acomp_virt_to_sg(struct acomp_req *req) state->dst = dvirt; sg_init_one(&state->dsg, dvirt, dlen); acomp_request_set_dst_sg(req, &state->dsg, dlen); - } else if (acomp_request_dst_isfolio(req)) { - struct folio *folio = req->dfolio; - unsigned int dlen = req->dlen; - size_t off = req->doff; - - state->dfolio = folio; - sg_init_table(&state->dsg, 1); - sg_set_page(&state->dsg, folio_page(folio, off / PAGE_SIZE), - dlen, off % PAGE_SIZE); - acomp_request_set_src_sg(req, &state->dsg, dlen); } } @@ -579,18 +551,8 @@ int acomp_walk_virt(struct acomp_walk *__restrict walk, walk->flags |= ACOMP_WALK_SLEEP; if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT)) walk->flags |= ACOMP_WALK_SRC_LINEAR; - else if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_FOLIO)) { - src = &req->chain.ssg; - sg_init_table(src, 1); - sg_set_folio(src, req->sfolio, walk->slen, req->soff); - } if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_VIRT)) walk->flags |= ACOMP_WALK_DST_LINEAR; - else if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_FOLIO)) { - dst = &req->chain.dsg; - sg_init_table(dst, 1); - sg_set_folio(dst, req->dfolio, walk->dlen, req->doff); - } if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) { walk->in.sg = (void *)req->svirt; diff --git a/crypto/scompress.c b/crypto/scompress.c index 7ade3f2fee7e..c330b81bc5a6 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -193,10 +193,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) if (dst_isvirt) dst = req->dvirt; else { - if (acomp_request_dst_isfolio(req)) { - dpage = folio_page(req->dfolio, 0); - doff = req->doff; - } else if (dlen <= req->dst->length) { + if (dlen <= req->dst->length) { dpage = sg_page(req->dst); doff = req->dst->offset; } else @@ -218,10 +215,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) else { src = NULL; do { - if (acomp_request_src_isfolio(req)) { - spage = folio_page(req->sfolio, 0); - soff = req->soff; - } else if (slen <= req->src->length) { + if (slen <= req->src->length) { spage = sg_page(req->src); soff = req->src->offset; } else diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index 96ec0090a855..1b30290d6380 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -32,17 +32,10 @@ /* Set this bit for if virtual address destination cannot be used for DMA. */ #define CRYPTO_ACOMP_REQ_DST_NONDMA 0x00000010 -/* Set this bit if source is a folio. */ -#define CRYPTO_ACOMP_REQ_SRC_FOLIO 0x00000020 - -/* Set this bit if destination is a folio. */ -#define CRYPTO_ACOMP_REQ_DST_FOLIO 0x00000040 - /* Private flags that should not be touched by the user. */ #define CRYPTO_ACOMP_REQ_PRIVATE \ (CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | \ - CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA | \ - CRYPTO_ACOMP_REQ_SRC_FOLIO | CRYPTO_ACOMP_REQ_DST_FOLIO) + CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA) #define CRYPTO_ACOMP_DST_MAX 131072 @@ -84,10 +77,6 @@ struct acomp_req_chain { * @dst: Destination scatterlist * @svirt: Source virtual address * @dvirt: Destination virtual address - * @sfolio: Source folio - * @soff: Source folio offset - * @dfolio: Destination folio - * @doff: Destination folio offset * @slen: Size of the input buffer * @dlen: Size of the output buffer and number of bytes produced * @chain: Private API code data, do not use @@ -98,15 +87,11 @@ struct acomp_req { union { struct scatterlist *src; const u8 *svirt; - struct folio *sfolio; }; union { struct scatterlist *dst; u8 *dvirt; - struct folio *dfolio; }; - size_t soff; - size_t doff; unsigned int slen; unsigned int dlen; @@ -373,8 +358,6 @@ static inline void acomp_request_set_params(struct acomp_req *req, req->base.flags &= ~(CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | - CRYPTO_ACOMP_REQ_SRC_FOLIO | - CRYPTO_ACOMP_REQ_DST_FOLIO | CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA); } @@ -397,7 +380,6 @@ static inline void acomp_request_set_src_sg(struct acomp_req *req, req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_VIRT; - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; } /** @@ -417,7 +399,6 @@ static inline void acomp_request_set_src_dma(struct acomp_req *req, req->slen = slen; req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; } @@ -438,7 +419,6 @@ static inline void acomp_request_set_src_nondma(struct acomp_req *req, req->svirt = src; req->slen = slen; - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_SRC_NONDMA; req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; } @@ -457,13 +437,9 @@ static inline void acomp_request_set_src_folio(struct acomp_req *req, struct folio *folio, size_t off, unsigned int len) { - req->sfolio = folio; - req->soff = off; - req->slen = len; - - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_VIRT; - req->base.flags |= CRYPTO_ACOMP_REQ_SRC_FOLIO; + sg_init_table(&req->chain.ssg, 1); + sg_set_folio(&req->chain.ssg, folio, len, off); + acomp_request_set_src_sg(req, &req->chain.ssg, len); } /** @@ -484,7 +460,6 @@ static inline void acomp_request_set_dst_sg(struct acomp_req *req, req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_VIRT; - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; } /** @@ -504,7 +479,6 @@ static inline void acomp_request_set_dst_dma(struct acomp_req *req, req->dlen = dlen; req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; } @@ -524,7 +498,6 @@ static inline void acomp_request_set_dst_nondma(struct acomp_req *req, req->dvirt = dst; req->dlen = dlen; - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_DST_NONDMA; req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; } @@ -543,13 +516,9 @@ static inline void acomp_request_set_dst_folio(struct acomp_req *req, struct folio *folio, size_t off, unsigned int len) { - req->dfolio = folio; - req->doff = off; - req->dlen = len; - - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_VIRT; - req->base.flags |= CRYPTO_ACOMP_REQ_DST_FOLIO; + sg_init_table(&req->chain.dsg, 1); + sg_set_folio(&req->chain.dsg, folio, len, off); + acomp_request_set_dst_sg(req, &req->chain.dsg, len); } /** diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index b51d66633935..d6d53c7696fd 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -154,9 +154,7 @@ void crypto_unregister_acomps(struct acomp_alg *algs, int count); static inline bool acomp_request_issg(struct acomp_req *req) { return !(req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | - CRYPTO_ACOMP_REQ_DST_VIRT | - CRYPTO_ACOMP_REQ_SRC_FOLIO | - CRYPTO_ACOMP_REQ_DST_FOLIO)); + CRYPTO_ACOMP_REQ_DST_VIRT)); } static inline bool acomp_request_src_isvirt(struct acomp_req *req) @@ -191,16 +189,6 @@ static inline bool acomp_request_isnondma(struct acomp_req *req) CRYPTO_ACOMP_REQ_DST_NONDMA); } -static inline bool acomp_request_src_isfolio(struct acomp_req *req) -{ - return req->base.flags & CRYPTO_ACOMP_REQ_SRC_FOLIO; -} - -static inline bool acomp_request_dst_isfolio(struct acomp_req *req) -{ - return req->base.flags & CRYPTO_ACOMP_REQ_DST_FOLIO; -} - static inline bool crypto_acomp_req_chain(struct crypto_acomp *tfm) { return crypto_tfm_req_chain(&tfm->base); @@ -250,8 +238,6 @@ static inline struct acomp_req *acomp_fbreq_on_stack_init( req->dst = old->dst; req->slen = old->slen; req->dlen = old->dlen; - req->soff = old->soff; - req->doff = old->doff; return req; } -- cgit v1.2.3 From 300e6d6e9ebf2dcc0ca3eab68ed65207e947926c Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:21:09 +0800 Subject: crypto: acomp - Remove reqsize field Remove the type-specific reqsize field in favour of the common one. Signed-off-by: Herbert Xu --- crypto/acompress.c | 2 +- include/crypto/internal/acompress.h | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) (limited to 'include/crypto/internal/acompress.h') diff --git a/crypto/acompress.c b/crypto/acompress.c index 36b52a72a43b..d5605b0ad266 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -109,7 +109,7 @@ static int crypto_acomp_init_tfm(struct crypto_tfm *tfm) acomp->compress = alg->compress; acomp->decompress = alg->decompress; - acomp->reqsize = alg->base.cra_reqsize ?: alg->reqsize; + acomp->reqsize = alg->base.cra_reqsize; acomp->base.exit = crypto_acomp_exit_tfm; diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index d6d53c7696fd..0f3ad65be2d9 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -40,7 +40,6 @@ * counterpart to @init, used to remove various changes set in * @init. * - * @reqsize: Context size for (de)compression requests * @base: Common crypto API algorithm data structure * @calg: Cmonn algorithm data structure shared with scomp */ @@ -50,8 +49,6 @@ struct acomp_alg { int (*init)(struct crypto_acomp *tfm); void (*exit)(struct crypto_acomp *tfm); - unsigned int reqsize; - union { struct COMP_ALG_COMMON; struct comp_alg_common calg; -- cgit v1.2.3 From ddd0855fa3c3f1de020ab3aeddce15fe38e116f7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 15 Apr 2025 17:23:19 +0800 Subject: crypto: deflate - Make the acomp walk atomic Add an atomic flag to the acomp walk and use that in deflate. Due to the use of a per-cpu context, it is impossible to sleep during the walk in deflate. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-lkp/202504151654.4c3b6393-lkp@intel.com Fixes: 08cabc7d3c86 ("crypto: deflate - Convert to acomp") Signed-off-by: Herbert Xu --- crypto/acompress.c | 4 ++-- crypto/deflate.c | 4 ++-- include/crypto/internal/acompress.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include/crypto/internal/acompress.h') diff --git a/crypto/acompress.c b/crypto/acompress.c index 606d09a7fbfd..b0f9192f6b2e 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -536,7 +536,7 @@ int acomp_walk_next_dst(struct acomp_walk *walk) EXPORT_SYMBOL_GPL(acomp_walk_next_dst); int acomp_walk_virt(struct acomp_walk *__restrict walk, - struct acomp_req *__restrict req) + struct acomp_req *__restrict req, bool atomic) { struct scatterlist *src = req->src; struct scatterlist *dst = req->dst; @@ -548,7 +548,7 @@ int acomp_walk_virt(struct acomp_walk *__restrict walk, return -EINVAL; walk->flags = 0; - if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP)) + if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) && !atomic) walk->flags |= ACOMP_WALK_SLEEP; if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT)) walk->flags |= ACOMP_WALK_SRC_LINEAR; diff --git a/crypto/deflate.c b/crypto/deflate.c index 57d7af4dfdfb..0d2b64d96d6e 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -60,7 +60,7 @@ static int deflate_compress_one(struct acomp_req *req, struct acomp_walk walk; int ret; - ret = acomp_walk_virt(&walk, req); + ret = acomp_walk_virt(&walk, req, true); if (ret) return ret; @@ -140,7 +140,7 @@ static int deflate_decompress_one(struct acomp_req *req, struct acomp_walk walk; int ret; - ret = acomp_walk_virt(&walk, req); + ret = acomp_walk_virt(&walk, req, true); if (ret) return ret; diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 0f3ad65be2d9..7eda32619024 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -208,7 +208,7 @@ void acomp_walk_done_dst(struct acomp_walk *walk, int used); int acomp_walk_next_src(struct acomp_walk *walk); int acomp_walk_next_dst(struct acomp_walk *walk); int acomp_walk_virt(struct acomp_walk *__restrict walk, - struct acomp_req *__restrict req); + struct acomp_req *__restrict req, bool atomic); static inline bool acomp_walk_more_src(const struct acomp_walk *walk, int cur) { -- cgit v1.2.3 From 19da081a28c95fe9b03ce952a2bf4a6f6bf5112c Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 23 Apr 2025 17:22:28 +0800 Subject: crypto: api - Add crypto_request_clone and fb Add a helper to clone crypto requests and eliminate code duplication. Use kmemdup in the helper. Also add an fb field to crypto_tfm. This also happens to fix the existing implementations which were buggy. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202504230118.1CxUaUoX-lkp@intel.com/ Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202504230004.c7mrY0C6-lkp@intel.com/ Signed-off-by: Herbert Xu --- crypto/acompress.c | 29 +++-------------------------- crypto/ahash.c | 32 +++++--------------------------- crypto/api.c | 18 ++++++++++++++++++ include/crypto/acompress.h | 9 ++++++--- include/crypto/hash.h | 9 ++++++--- include/crypto/internal/acompress.h | 7 ++++++- include/crypto/internal/hash.h | 7 ++++++- include/linux/crypto.h | 11 ++++++++--- 8 files changed, 58 insertions(+), 64 deletions(-) (limited to 'include/crypto/internal/acompress.h') diff --git a/crypto/acompress.c b/crypto/acompress.c index 4c665c6fb5d6..9dea76ed4513 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -11,15 +11,13 @@ #include #include #include -#include +#include #include #include -#include #include #include #include #include -#include #include #include #include @@ -79,7 +77,7 @@ static void crypto_acomp_exit_tfm(struct crypto_tfm *tfm) alg->exit(acomp); if (acomp_is_async(acomp)) - crypto_free_acomp(acomp->fb); + crypto_free_acomp(crypto_acomp_fb(acomp)); } static int crypto_acomp_init_tfm(struct crypto_tfm *tfm) @@ -89,8 +87,6 @@ static int crypto_acomp_init_tfm(struct crypto_tfm *tfm) struct crypto_acomp *fb = NULL; int err; - acomp->fb = acomp; - if (tfm->__crt_alg->cra_type != &crypto_acomp_type) return crypto_init_scomp_ops_async(tfm); @@ -104,7 +100,7 @@ static int crypto_acomp_init_tfm(struct crypto_tfm *tfm) if (crypto_acomp_reqsize(fb) > MAX_SYNC_COMP_REQSIZE) goto out_free_fb; - acomp->fb = fb; + tfm->fb = crypto_acomp_tfm(fb); } acomp->compress = alg->compress; @@ -570,24 +566,5 @@ int acomp_walk_virt(struct acomp_walk *__restrict walk, } EXPORT_SYMBOL_GPL(acomp_walk_virt); -struct acomp_req *acomp_request_clone(struct acomp_req *req, - size_t total, gfp_t gfp) -{ - struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); - struct acomp_req *nreq; - - nreq = kmalloc(total, gfp); - if (!nreq) { - acomp_request_set_tfm(req, tfm->fb); - req->base.flags = CRYPTO_TFM_REQ_ON_STACK; - return req; - } - - memcpy(nreq, req, total); - acomp_request_set_tfm(req, tfm); - return req; -} -EXPORT_SYMBOL_GPL(acomp_request_clone); - MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous compression type"); diff --git a/crypto/ahash.c b/crypto/ahash.c index 7a74092323b9..9b813f7b9177 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -12,13 +12,11 @@ * Copyright (c) 2008 Loc Ho */ -#include #include #include #include #include #include -#include #include #include #include @@ -301,7 +299,8 @@ int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, err = alg->setkey(tfm, key, keylen); if (!err && ahash_is_async(tfm)) - err = crypto_ahash_setkey(tfm->fb, key, keylen); + err = crypto_ahash_setkey(crypto_ahash_fb(tfm), + key, keylen); if (unlikely(err)) { ahash_set_needkey(tfm, alg); return err; @@ -732,7 +731,7 @@ static void crypto_ahash_exit_tfm(struct crypto_tfm *tfm) tfm->__crt_alg->cra_exit(tfm); if (ahash_is_async(hash)) - crypto_free_ahash(hash->fb); + crypto_free_ahash(crypto_ahash_fb(hash)); } static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) @@ -745,8 +744,6 @@ static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) crypto_ahash_set_statesize(hash, alg->halg.statesize); crypto_ahash_set_reqsize(hash, crypto_tfm_alg_reqsize(tfm)); - hash->fb = hash; - if (tfm->__crt_alg->cra_type == &crypto_shash_type) return crypto_init_ahash_using_shash(tfm); @@ -756,7 +753,7 @@ static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) if (IS_ERR(fb)) return PTR_ERR(fb); - hash->fb = fb; + tfm->fb = crypto_ahash_tfm(fb); } ahash_set_needkey(hash, alg); @@ -1036,7 +1033,7 @@ EXPORT_SYMBOL_GPL(ahash_request_free); int crypto_hash_digest(struct crypto_ahash *tfm, const u8 *data, unsigned int len, u8 *out) { - HASH_REQUEST_ON_STACK(req, tfm->fb); + HASH_REQUEST_ON_STACK(req, crypto_ahash_fb(tfm)); int err; ahash_request_set_callback(req, 0, NULL, NULL); @@ -1049,24 +1046,5 @@ int crypto_hash_digest(struct crypto_ahash *tfm, const u8 *data, } EXPORT_SYMBOL_GPL(crypto_hash_digest); -struct ahash_request *ahash_request_clone(struct ahash_request *req, - size_t total, gfp_t gfp) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ahash_request *nreq; - - nreq = kmalloc(total, gfp); - if (!nreq) { - ahash_request_set_tfm(req, tfm->fb); - req->base.flags = CRYPTO_TFM_REQ_ON_STACK; - return req; - } - - memcpy(nreq, req, total); - ahash_request_set_tfm(req, tfm); - return req; -} -EXPORT_SYMBOL_GPL(ahash_request_clone); - MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous cryptographic hash type"); diff --git a/crypto/api.c b/crypto/api.c index e427cc5662b5..172e82f79c69 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -528,6 +528,7 @@ void *crypto_create_tfm_node(struct crypto_alg *alg, goto out; tfm = (struct crypto_tfm *)(mem + frontend->tfmsize); + tfm->fb = tfm; err = frontend->init_tfm(tfm); if (err) @@ -712,5 +713,22 @@ void crypto_destroy_alg(struct crypto_alg *alg) } EXPORT_SYMBOL_GPL(crypto_destroy_alg); +struct crypto_async_request *crypto_request_clone( + struct crypto_async_request *req, size_t total, gfp_t gfp) +{ + struct crypto_tfm *tfm = req->tfm; + struct crypto_async_request *nreq; + + nreq = kmemdup(req, total, gfp); + if (!nreq) { + req->tfm = tfm->fb; + return req; + } + + nreq->flags &= ~CRYPTO_TFM_REQ_ON_STACK; + return nreq; +} +EXPORT_SYMBOL_GPL(crypto_request_clone); + MODULE_DESCRIPTION("Cryptographic core API"); MODULE_LICENSE("GPL"); diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index 1b30290d6380..933c48a4855b 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -114,7 +114,6 @@ struct crypto_acomp { int (*compress)(struct acomp_req *req); int (*decompress)(struct acomp_req *req); unsigned int reqsize; - struct crypto_acomp *fb; struct crypto_tfm base; }; @@ -553,7 +552,11 @@ static inline struct acomp_req *acomp_request_on_stack_init( return req; } -struct acomp_req *acomp_request_clone(struct acomp_req *req, - size_t total, gfp_t gfp); +static inline struct acomp_req *acomp_request_clone(struct acomp_req *req, + size_t total, gfp_t gfp) +{ + return container_of(crypto_request_clone(&req->base, total, gfp), + struct acomp_req, base); +} #endif diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 5f87d1040a7c..68813a83443b 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -246,7 +246,6 @@ struct crypto_ahash { bool using_shash; /* Underlying algorithm is shash, not ahash */ unsigned int statesize; unsigned int reqsize; - struct crypto_ahash *fb; struct crypto_tfm base; }; @@ -1035,7 +1034,11 @@ static inline struct ahash_request *ahash_request_on_stack_init( return req; } -struct ahash_request *ahash_request_clone(struct ahash_request *req, - size_t total, gfp_t gfp); +static inline struct ahash_request *ahash_request_clone( + struct ahash_request *req, size_t total, gfp_t gfp) +{ + return container_of(crypto_request_clone(&req->base, total, gfp), + struct ahash_request, base); +} #endif /* _CRYPTO_HASH_H */ diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 7eda32619024..6550dad18e0f 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -220,13 +220,18 @@ static inline u32 acomp_request_flags(struct acomp_req *req) return crypto_request_flags(&req->base) & ~CRYPTO_ACOMP_REQ_PRIVATE; } +static inline struct crypto_acomp *crypto_acomp_fb(struct crypto_acomp *tfm) +{ + return __crypto_acomp_tfm(crypto_acomp_tfm(tfm)->fb); +} + static inline struct acomp_req *acomp_fbreq_on_stack_init( char *buf, struct acomp_req *old) { struct crypto_acomp *tfm = crypto_acomp_reqtfm(old); struct acomp_req *req = (void *)buf; - acomp_request_set_tfm(req, tfm->fb); + acomp_request_set_tfm(req, crypto_acomp_fb(tfm)); req->base.flags = CRYPTO_TFM_REQ_ON_STACK; acomp_request_set_callback(req, acomp_request_flags(old), NULL, NULL); req->base.flags &= ~CRYPTO_ACOMP_REQ_PRIVATE; diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 1e80dd084a23..0bc0fefc9b3c 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -272,13 +272,18 @@ static inline bool crypto_ahash_req_chain(struct crypto_ahash *tfm) return crypto_tfm_req_chain(&tfm->base); } +static inline struct crypto_ahash *crypto_ahash_fb(struct crypto_ahash *tfm) +{ + return __crypto_ahash_cast(crypto_ahash_tfm(tfm)->fb); +} + static inline struct ahash_request *ahash_fbreq_on_stack_init( char *buf, struct ahash_request *old) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(old); struct ahash_request *req = (void *)buf; - ahash_request_set_tfm(req, tfm->fb); + ahash_request_set_tfm(req, crypto_ahash_fb(tfm)); req->base.flags = CRYPTO_TFM_REQ_ON_STACK; ahash_request_set_callback(req, ahash_request_flags(old), NULL, NULL); req->base.flags &= ~CRYPTO_AHASH_REQ_PRIVATE; diff --git a/include/linux/crypto.h b/include/linux/crypto.h index f691ce01745e..fe75320ff9a3 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include @@ -411,9 +411,11 @@ struct crypto_tfm { u32 crt_flags; int node; - + + struct crypto_tfm *fb; + void (*exit)(struct crypto_tfm *tfm); - + struct crypto_alg *__crt_alg; void *__crt_ctx[] CRYPTO_MINALIGN_ATTR; @@ -509,5 +511,8 @@ static inline void crypto_request_set_tfm(struct crypto_async_request *req, req->flags &= ~CRYPTO_TFM_REQ_ON_STACK; } +struct crypto_async_request *crypto_request_clone( + struct crypto_async_request *req, size_t total, gfp_t gfp); + #endif /* _LINUX_CRYPTO_H */ -- cgit v1.2.3 From b75fa20c127eb736b0ac9b30be051f526a2316a9 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 25 Apr 2025 11:05:29 +0800 Subject: crypto: api - Add crypto_stack_request_init and initialise flags fully Add a helper to initialise crypto stack requests and use it for ahash and acomp. Make sure that the flags field is initialised fully in the helper to silence false-positive warnings from the compiler. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202504250751.mdy28Ibr-lkp@intel.com/ Signed-off-by: Herbert Xu --- include/crypto/acompress.h | 3 +-- include/crypto/hash.h | 3 +-- include/crypto/internal/acompress.h | 4 ++-- include/crypto/internal/hash.h | 4 ++-- include/linux/crypto.h | 8 ++++++++ 5 files changed, 14 insertions(+), 8 deletions(-) (limited to 'include/crypto/internal/acompress.h') diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index 933c48a4855b..f1812e92d3e3 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -547,8 +547,7 @@ static inline struct acomp_req *acomp_request_on_stack_init( { struct acomp_req *req = (void *)buf; - acomp_request_set_tfm(req, tfm); - req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + crypto_stack_request_init(&req->base, crypto_acomp_tfm(tfm)); return req; } diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 68813a83443b..c2497c300a28 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -1029,8 +1029,7 @@ static inline struct ahash_request *ahash_request_on_stack_init( { struct ahash_request *req = (void *)buf; - ahash_request_set_tfm(req, tfm); - req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + crypto_stack_request_init(&req->base, crypto_ahash_tfm(tfm)); return req; } diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 6550dad18e0f..b72bb7a6a2b2 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -231,8 +231,8 @@ static inline struct acomp_req *acomp_fbreq_on_stack_init( struct crypto_acomp *tfm = crypto_acomp_reqtfm(old); struct acomp_req *req = (void *)buf; - acomp_request_set_tfm(req, crypto_acomp_fb(tfm)); - req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + crypto_stack_request_init(&req->base, + crypto_acomp_tfm(crypto_acomp_fb(tfm))); acomp_request_set_callback(req, acomp_request_flags(old), NULL, NULL); req->base.flags &= ~CRYPTO_ACOMP_REQ_PRIVATE; req->base.flags |= old->base.flags & CRYPTO_ACOMP_REQ_PRIVATE; diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 0bc0fefc9b3c..33d45275f5bd 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -283,8 +283,8 @@ static inline struct ahash_request *ahash_fbreq_on_stack_init( struct crypto_ahash *tfm = crypto_ahash_reqtfm(old); struct ahash_request *req = (void *)buf; - ahash_request_set_tfm(req, crypto_ahash_fb(tfm)); - req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + crypto_stack_request_init(&req->base, + crypto_ahash_tfm(crypto_ahash_fb(tfm))); ahash_request_set_callback(req, ahash_request_flags(old), NULL, NULL); req->base.flags &= ~CRYPTO_AHASH_REQ_PRIVATE; req->base.flags |= old->base.flags & CRYPTO_AHASH_REQ_PRIVATE; diff --git a/include/linux/crypto.h b/include/linux/crypto.h index fe75320ff9a3..b8d875b11193 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -514,5 +514,13 @@ static inline void crypto_request_set_tfm(struct crypto_async_request *req, struct crypto_async_request *crypto_request_clone( struct crypto_async_request *req, size_t total, gfp_t gfp); +static inline void crypto_stack_request_init(struct crypto_async_request *req, + struct crypto_tfm *tfm) +{ + req->flags = 0; + crypto_request_set_tfm(req, tfm); + req->flags |= CRYPTO_TFM_REQ_ON_STACK; +} + #endif /* _LINUX_CRYPTO_H */ -- cgit v1.2.3 From 8fd17374be8f220c26bec2b482cabf51ebbaed80 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 1 May 2025 20:37:32 +0800 Subject: crypto: api - Rename CRYPTO_ALG_REQ_CHAIN to CRYPTO_ALG_REQ_VIRT As chaining has been removed, all that remains of REQ_CHAIN is just virtual address support. Rename it before the reintroduction of batching creates confusion. Signed-off-by: Herbert Xu --- crypto/acompress.c | 4 ++-- crypto/ahash.c | 6 +++--- crypto/deflate.c | 2 +- crypto/scompress.c | 2 +- include/crypto/algapi.h | 4 ++-- include/crypto/internal/acompress.h | 4 ++-- include/crypto/internal/hash.h | 4 ++-- include/linux/crypto.h | 4 ++-- 8 files changed, 15 insertions(+), 15 deletions(-) (limited to 'include/crypto/internal/acompress.h') diff --git a/crypto/acompress.c b/crypto/acompress.c index 6ecbfc49bfa8..be28cbfd22e3 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -287,7 +287,7 @@ int crypto_acomp_compress(struct acomp_req *req) if (acomp_req_on_stack(req) && acomp_is_async(tfm)) return -EAGAIN; - if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) + if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req)) return crypto_acomp_reqtfm(req)->compress(req); return acomp_do_req_chain(req, true); } @@ -299,7 +299,7 @@ int crypto_acomp_decompress(struct acomp_req *req) if (acomp_req_on_stack(req) && acomp_is_async(tfm)) return -EAGAIN; - if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) + if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req)) return crypto_acomp_reqtfm(req)->decompress(req); return acomp_do_req_chain(req, false); } diff --git a/crypto/ahash.c b/crypto/ahash.c index cc9885d5cfd2..57c131a13067 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -407,7 +407,7 @@ static int ahash_do_req_chain(struct ahash_request *req, u8 *page = NULL; int err; - if (crypto_ahash_req_chain(tfm) || + if (crypto_ahash_req_virt(tfm) || !update || !ahash_request_isvirt(req)) return op(req); @@ -550,7 +550,7 @@ int crypto_ahash_finup(struct ahash_request *req) if (ahash_req_on_stack(req) && ahash_is_async(tfm)) return -EAGAIN; if (!crypto_ahash_alg(tfm)->finup || - (!crypto_ahash_req_chain(tfm) && ahash_request_isvirt(req))) + (!crypto_ahash_req_virt(tfm) && ahash_request_isvirt(req))) return ahash_def_finup(req); return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->finup); } @@ -622,7 +622,7 @@ int crypto_ahash_digest(struct ahash_request *req) return shash_ahash_digest(req, prepare_shash_desc(req, tfm)); if (ahash_req_on_stack(req) && ahash_is_async(tfm)) return -EAGAIN; - if (!crypto_ahash_req_chain(tfm) && ahash_request_isvirt(req)) + if (!crypto_ahash_req_virt(tfm) && ahash_request_isvirt(req)) return ahash_def_digest(req); if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; diff --git a/crypto/deflate.c b/crypto/deflate.c index 7eb1a5c44ee0..fe8e4ad0fee1 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -228,7 +228,7 @@ static struct acomp_alg acomp = { .init = deflate_init, .base.cra_name = "deflate", .base.cra_driver_name = "deflate-generic", - .base.cra_flags = CRYPTO_ALG_REQ_CHAIN, + .base.cra_flags = CRYPTO_ALG_REQ_VIRT, .base.cra_module = THIS_MODULE, }; diff --git a/crypto/scompress.c b/crypto/scompress.c index 15148c58d648..c651e7f2197a 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -355,7 +355,7 @@ static void scomp_prepare_alg(struct scomp_alg *alg) comp_prepare_alg(&alg->calg); - base->cra_flags |= CRYPTO_ALG_REQ_CHAIN; + base->cra_flags |= CRYPTO_ALG_REQ_VIRT; } int crypto_register_scomp(struct scomp_alg *alg) diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index f5f730969d72..423e57eca351 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -255,9 +255,9 @@ static inline u32 crypto_tfm_alg_type(struct crypto_tfm *tfm) return tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK; } -static inline bool crypto_tfm_req_chain(struct crypto_tfm *tfm) +static inline bool crypto_tfm_req_virt(struct crypto_tfm *tfm) { - return tfm->__crt_alg->cra_flags & CRYPTO_ALG_REQ_CHAIN; + return tfm->__crt_alg->cra_flags & CRYPTO_ALG_REQ_VIRT; } static inline u32 crypto_request_flags(struct crypto_async_request *req) diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index b72bb7a6a2b2..ffffd88bbbad 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -186,9 +186,9 @@ static inline bool acomp_request_isnondma(struct acomp_req *req) CRYPTO_ACOMP_REQ_DST_NONDMA); } -static inline bool crypto_acomp_req_chain(struct crypto_acomp *tfm) +static inline bool crypto_acomp_req_virt(struct crypto_acomp *tfm) { - return crypto_tfm_req_chain(&tfm->base); + return crypto_tfm_req_virt(&tfm->base); } void crypto_acomp_free_streams(struct crypto_acomp_streams *s); diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 33d45275f5bd..e911f32f46dc 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -267,9 +267,9 @@ static inline bool ahash_request_isvirt(struct ahash_request *req) return req->base.flags & CRYPTO_AHASH_REQ_VIRT; } -static inline bool crypto_ahash_req_chain(struct crypto_ahash *tfm) +static inline bool crypto_ahash_req_virt(struct crypto_ahash *tfm) { - return crypto_tfm_req_chain(&tfm->base); + return crypto_tfm_req_virt(&tfm->base); } static inline struct crypto_ahash *crypto_ahash_fb(struct crypto_ahash *tfm) diff --git a/include/linux/crypto.h b/include/linux/crypto.h index b8d875b11193..b50f1954d1bb 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -133,8 +133,8 @@ */ #define CRYPTO_ALG_FIPS_INTERNAL 0x00020000 -/* Set if the algorithm supports request chains and virtual addresses. */ -#define CRYPTO_ALG_REQ_CHAIN 0x00040000 +/* Set if the algorithm supports virtual addresses. */ +#define CRYPTO_ALG_REQ_VIRT 0x00040000 /* The high bits 0xff000000 are reserved for type-specific flags. */ -- cgit v1.2.3