diff --git a/inc/ocf_mngt.h b/inc/ocf_mngt.h index 3c4e5450..fa60be24 100644 --- a/inc/ocf_mngt.h +++ b/inc/ocf_mngt.h @@ -610,7 +610,6 @@ void ocf_mngt_cache_standby_load(ocf_cache_t cache, /** * @brief Completion callback of cache standby detach operation * - * @param[in] cache Cache handle * @param[in] priv Callback context * @param[in] error Error code (zero on success) */ diff --git a/src/metadata/metadata_superblock.h b/src/metadata/metadata_superblock.h index e629157d..cc84aadc 100644 --- a/src/metadata/metadata_superblock.h +++ b/src/metadata/metadata_superblock.h @@ -1,5 +1,7 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -42,8 +44,8 @@ struct ocf_superblock_config { ocf_cache_line_size_t line_size; uint32_t core_count; - unsigned long valid_core_bitmap[(OCF_CORE_MAX / - (sizeof(unsigned long) * 8)) + 1]; + unsigned long valid_core_bitmap[OCF_DIV_ROUND_UP_STATIC(OCF_CORE_MAX, + sizeof(unsigned long) * 8)]; bool cleaner_disabled; ocf_cleaning_t cleaning_policy_type; diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index beab576c..2f5e4cdb 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -872,25 +872,29 @@ static int _ocf_mngt_init_new_cache(struct ocf_cache_mngt_init_params *params, goto alloc_err; } - result = env_refcnt_init(&cache->refcnt.cache, "cache", sizeof("cache")); + result = env_refcnt_init(&cache->refcnt.cache, + "cache", sizeof("cache")); if (result) goto lock_init_err; - result = env_refcnt_init(&cache->refcnt.dirty, "dirty", sizeof("dirty")); + result = env_refcnt_init(&cache->refcnt.dirty, + "dirty", sizeof("dirty")); if (result) goto dirty_refcnt_err; - result = env_refcnt_init(&cache->refcnt.metadata, "metadata", sizeof("metadata")); + result = env_refcnt_init(&cache->refcnt.metadata, + "metadata", sizeof("metadata")); if (result) goto metadata_refcnt_err; - result = env_refcnt_init(&cache->refcnt.d2c, "d2c", sizeof("d2c")); + result = env_refcnt_init(&cache->refcnt.d2c, + "d2c", sizeof("d2c")); if (result) goto d2c_refcnt_err; - for (i = 0; i < OCF_USER_IO_CLASS_MAX; i++) { + for (i = 0; i < OCF_USER_IO_CLASS_MAX; i++) { result = env_refcnt_init(&cache->user_parts[i].cleaning.counter, - "cleaning", sizeof("cleaning")); + "cleaning", sizeof("cleaning")); if (result) goto cleaning_refcnt_err; env_atomic_set(&cache->user_parts[i].cleaning.cleaner_running, 0); @@ -2001,7 +2005,7 @@ static void _ocf_mngt_attach_switch_to_pt(ocf_pipeline_t pipeline, env_refcnt_unfreeze(&cache->refcnt.metadata); env_refcnt_freeze(&cache->refcnt.d2c); - ocf_mngt_continue_pipeline_on_zero_refcnt(&cache->refcnt.d2c, pipeline); + ocf_pipeline_continue_on_zero_refcnt(pipeline, &cache->refcnt.d2c); } static void _ocf_mngt_attach_handle_error( @@ -2150,7 +2154,7 @@ static void ocf_mngt_cache_stop_wait_metadata_io(ocf_pipeline_t pipeline, struct env_refcnt *refcnt = &context->cache->refcnt.metadata; env_refcnt_freeze(refcnt); - ocf_mngt_continue_pipeline_on_zero_refcnt(refcnt, context->pipeline); + ocf_pipeline_continue_on_zero_refcnt(context->pipeline, refcnt); } static void ocf_mngt_cache_stop_check_dirty(ocf_pipeline_t pipeline, @@ -2601,7 +2605,7 @@ static void _ocf_mngt_standby_detach_wait_metadata_io(ocf_pipeline_t pipeline, struct env_refcnt *refcnt = &context->cache->refcnt.metadata; env_refcnt_freeze(refcnt); - ocf_mngt_continue_pipeline_on_zero_refcnt(refcnt, context->pipeline); + ocf_pipeline_continue_on_zero_refcnt(context->pipeline, refcnt); } static void _ocf_mngt_activate_set_cache_device(ocf_pipeline_t pipeline, @@ -3114,7 +3118,7 @@ static int _ocf_mngt_cache_validate_cfg(struct ocf_mngt_cache_config *cfg) if (!ocf_cache_line_size_is_valid(cfg->cache_line_size)) return -OCF_ERR_INVALID_CACHE_LINE_SIZE; - if (cfg->backfill.queue_unblock_size > cfg->backfill.max_queue_size ) + if (cfg->backfill.queue_unblock_size > cfg->backfill.max_queue_size) return -OCF_ERR_INVAL; return 0; @@ -3798,7 +3802,7 @@ static void ocf_mngt_cache_detach_stop_cache_io(ocf_pipeline_t pipeline, struct env_refcnt *refcnt = &context->cache->refcnt.metadata; env_refcnt_freeze(refcnt); - ocf_mngt_continue_pipeline_on_zero_refcnt(refcnt, context->pipeline); + ocf_pipeline_continue_on_zero_refcnt(context->pipeline, refcnt); } static void ocf_mngt_cache_detach_composite_invalidate_cmpl(ocf_cache_t cache, @@ -4168,13 +4172,6 @@ struct ocf_pipeline_properties ocf_mngt_detach_composite_pipeline_properties = { }, }; -static void _ocf_mngt_begin_detach_complete(void *priv) -{ - struct ocf_mngt_cache_unplug_context *context = priv; - - ocf_pipeline_next(context->pipeline); -} - void ocf_mngt_cache_detach_composite(ocf_cache_t cache, ocf_mngt_cache_detach_end_t cmpl, ocf_uuid_t target_uuid, void *priv) @@ -4223,6 +4220,6 @@ void ocf_mngt_cache_detach_composite(ocf_cache_t cache, /* prevent dirty io */ env_refcnt_freeze(&cache->refcnt.dirty); - env_refcnt_register_zero_cb(&cache->refcnt.dirty, - _ocf_mngt_begin_detach_complete, context); + ocf_pipeline_continue_on_zero_refcnt(context->pipeline, + &cache->refcnt.dirty); } diff --git a/src/mngt/ocf_mngt_common.c b/src/mngt/ocf_mngt_common.c index 1733e20e..f8714c89 100644 --- a/src/mngt/ocf_mngt_common.c +++ b/src/mngt/ocf_mngt_common.c @@ -1,6 +1,7 @@ /* * Copyright(c) 2012-2021 Intel Corporation * Copyright(c) 2024-2025 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -515,16 +516,3 @@ int ocf_mngt_cache_visit_reverse(ocf_ctx_t ocf_ctx, return result; } - -static void _ocf_mngt_continue_pipeline_on_zero_refcnt_cb(void *priv) -{ - ocf_pipeline_next((ocf_pipeline_t)priv); -} - -void ocf_mngt_continue_pipeline_on_zero_refcnt(struct env_refcnt *refcnt, - ocf_pipeline_t pipeline) -{ - env_refcnt_register_zero_cb(refcnt, - _ocf_mngt_continue_pipeline_on_zero_refcnt_cb, - pipeline); -} diff --git a/src/mngt/ocf_mngt_common.h b/src/mngt/ocf_mngt_common.h index 4e0adc43..9889306d 100644 --- a/src/mngt/ocf_mngt_common.h +++ b/src/mngt/ocf_mngt_common.h @@ -1,6 +1,7 @@ /* * Copyright(c) 2012-2021 Intel Corporation * Copyright(c) 2025 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -37,7 +38,4 @@ bool ocf_mngt_cache_is_locked(ocf_cache_t cache); void __set_cleaning_policy(ocf_cache_t cache, ocf_cleaning_t new_cleaning_policy); -void ocf_mngt_continue_pipeline_on_zero_refcnt(struct env_refcnt *refcnt, - ocf_pipeline_t pipeline); - #endif /* __OCF_MNGT_COMMON_H__ */ diff --git a/src/mngt/ocf_mngt_flush.c b/src/mngt/ocf_mngt_flush.c index e96f86c6..31dd4cef 100644 --- a/src/mngt/ocf_mngt_flush.c +++ b/src/mngt/ocf_mngt_flush.c @@ -9,6 +9,7 @@ #include "ocf_env_refcnt.h" #include "ocf_mngt_flush_priv.h" #include "ocf_mngt_common.h" +#include "ocf_mngt_flush_priv.h" #include "../ocf_priv.h" #include "../metadata/metadata.h" #include "../cleaning/cleaning.h" @@ -106,8 +107,8 @@ static void _ocf_mngt_begin_flush(ocf_pipeline_t pipeline, void *priv, env_refcnt_freeze(&cache->refcnt.dirty); context->flags.dirty_freeze = true; - ocf_mngt_continue_pipeline_on_zero_refcnt(&cache->refcnt.dirty, - context->pipeline); + ocf_pipeline_continue_on_zero_refcnt(context->pipeline, + &cache->refcnt.dirty); } static void _ocf_mngt_begin_cache_lines_invalidate(ocf_pipeline_t pipeline, @@ -118,7 +119,7 @@ static void _ocf_mngt_begin_cache_lines_invalidate(ocf_pipeline_t pipeline, env_refcnt_freeze(refcnt); context->flags.metadata_freeze = true; - ocf_mngt_continue_pipeline_on_zero_refcnt(refcnt, context->pipeline); + ocf_pipeline_continue_on_zero_refcnt(context->pipeline, refcnt); } bool ocf_mngt_core_is_dirty(ocf_core_t core) diff --git a/src/ocf_core.c b/src/ocf_core.c index ffba9b82..5065377a 100644 --- a/src/ocf_core.c +++ b/src/ocf_core.c @@ -487,15 +487,8 @@ static void *ocf_core_io_allocator_new(ocf_io_allocator_t allocator, uint64_t addr, uint32_t bytes, uint32_t dir) { ocf_core_t core = ocf_volume_to_core(volume); - struct ocf_request *req; - req = ocf_req_new(queue, core, addr, bytes, dir); - if (!req) - return NULL; - - req->core = ocf_volume_to_core(volume); - - return req; + return ocf_req_new(queue, core, addr, bytes, dir); } static void ocf_core_io_allocator_del(ocf_io_allocator_t allocator, void *obj) diff --git a/src/ocf_def_priv.h b/src/ocf_def_priv.h index 4236ce5f..c4c249cc 100644 --- a/src/ocf_def_priv.h +++ b/src/ocf_def_priv.h @@ -1,6 +1,7 @@ /* * Copyright(c) 2012-2021 Intel Corporation * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -16,6 +17,9 @@ #define BYTES_TO_PAGES(x) ((((uint64_t)x) + (PAGE_SIZE - 1)) / PAGE_SIZE) #define PAGES_TO_BYTES(x) (((uint64_t)x) * PAGE_SIZE) +#define OCF_DIV_ROUND_UP_STATIC(n, d) \ + (((n) + (d) - 1) / (d)) + #define OCF_DIV_ROUND_UP(x, y) \ ({ \ __typeof__ (x) __x = (x); \ diff --git a/src/ocf_lru.c b/src/ocf_lru.c index 7e7ad2cd..1f8e6e72 100644 --- a/src/ocf_lru.c +++ b/src/ocf_lru.c @@ -363,7 +363,7 @@ static inline void lru_iter_eviction_init(struct ocf_lru_iter *iter, req); } -static inline uint32_t _lru_next_lru_list(struct ocf_lru_iter *iter) +static inline void _lru_iter_advance_list(struct ocf_lru_iter *iter) { unsigned increment; @@ -371,8 +371,6 @@ static inline uint32_t _lru_next_lru_list(struct ocf_lru_iter *iter) iter->next_avail_lru = ocf_rotate_right(iter->next_avail_lru, increment, OCF_NUM_LRU_LISTS); iter->lru_idx = (iter->lru_idx + increment) % OCF_NUM_LRU_LISTS; - - return iter->lru_idx; } static inline bool _lru_lru_is_empty(struct ocf_lru_iter *iter) @@ -380,12 +378,12 @@ static inline bool _lru_lru_is_empty(struct ocf_lru_iter *iter) return !(iter->next_avail_lru & (1ULL << (OCF_NUM_LRU_LISTS - 1))); } -static inline uint32_t _lru_next_lru(struct ocf_lru_iter *iter) +static inline uint32_t lru_iter_advance(struct ocf_lru_iter *iter) { iter->lru_element_idx++; if ((iter->lru_element_idx >= OCF_LRU_MAX_LRU_ELEMENT_IDX) || _lru_lru_is_empty(iter)) { - _lru_next_lru_list(iter); + _lru_iter_advance_list(iter); iter->lru_element_idx = 0; } return iter->lru_idx; @@ -523,7 +521,7 @@ static inline ocf_cache_line_t lru_req_next_cline(struct ocf_request *req, lru_get_cline_list(cache, cline), cline); } - _lru_next_lru(iter); + lru_iter_advance(iter); ret = cline; line_unlock_wr: @@ -554,7 +552,8 @@ static inline ocf_cache_line_t lru_iter_eviction_next(struct ocf_lru_iter *iter, struct ocf_lru_list *list; do { - curr_lru = _lru_next_lru(iter); + lru_iter_advance(iter); + curr_lru = iter->lru_idx; ocf_metadata_lru_lock(&cache->metadata.lock, curr_lru); @@ -606,7 +605,8 @@ static inline ocf_cache_line_t lru_iter_free_next(struct ocf_lru_iter *iter, ENV_BUG_ON(dst_part == free); do { - curr_lru = _lru_next_lru(iter); + lru_iter_advance(iter); + curr_lru = iter->lru_idx; ocf_metadata_lru_lock(&cache->metadata.lock, curr_lru); @@ -642,7 +642,8 @@ static inline ocf_cache_line_t lru_iter_cleaning_next(struct ocf_lru_iter *iter) ocf_cache_line_t cline; do { - curr_lru = _lru_next_lru(iter); + lru_iter_advance(iter); + curr_lru = iter->lru_idx; cline = iter->curr_cline[curr_lru]; while (cline != end_marker && ! ocf_cache_line_try_lock_rd( diff --git a/src/ocf_queue.c b/src/ocf_queue.c index bc7cbb3b..3055eae2 100644 --- a/src/ocf_queue.c +++ b/src/ocf_queue.c @@ -1,6 +1,7 @@ /* * Copyright(c) 2012-2022 Intel Corporation * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ #include "ocf/ocf.h" @@ -288,7 +289,6 @@ void ocf_queue_push_req(struct ocf_request *req, uint flags) void ocf_queue_push_req_cb(struct ocf_request *req, ocf_req_cb req_cb, uint flags) { - req->error = 0; /* Please explain why!!! */ req->engine_handler = req_cb; ocf_queue_push_req(req, flags); } diff --git a/src/ocf_request.c b/src/ocf_request.c index 27418fde..f81211aa 100644 --- a/src/ocf_request.c +++ b/src/ocf_request.c @@ -240,7 +240,7 @@ struct ocf_request *ocf_req_new(ocf_queue_t queue, ocf_core_t core, OCF_DEBUG_TRACE(cache); - ocf_req_init(req, cache, queue, NULL, addr, bytes, rw); + ocf_req_init(req, cache, queue, core, addr, bytes, rw); req->core_line_first = core_line_first; req->core_line_last = core_line_last; diff --git a/src/utils/utils_pipeline.c b/src/utils/utils_pipeline.c index 81bb0de7..a364b216 100644 --- a/src/utils/utils_pipeline.c +++ b/src/utils/utils_pipeline.c @@ -1,10 +1,12 @@ /* * Copyright(c) 2019-2022 Intel Corporation * Copyright(c) 2023-2025 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ #include "ocf/ocf.h" +#include "ocf_env_refcnt.h" #include "../engine/cache_engine.h" #include "../engine/engine_common.h" #include "../ocf_request.h" @@ -161,3 +163,15 @@ void ocf_pipeline_finish(ocf_pipeline_t pipeline, int error) pipeline->error = error; ocf_queue_push_req(pipeline->req, OCF_QUEUE_PRIO_HIGH); } + +static void _ocf_pipeline_continue_on_zero_refcnt_cb(void *priv) +{ + ocf_pipeline_next((ocf_pipeline_t)priv); +} + +void ocf_pipeline_continue_on_zero_refcnt(ocf_pipeline_t pipeline, + struct env_refcnt *refcnt) +{ + env_refcnt_register_zero_cb(refcnt, + _ocf_pipeline_continue_on_zero_refcnt_cb, pipeline); +} diff --git a/src/utils/utils_pipeline.h b/src/utils/utils_pipeline.h index 2174b030..9bd85081 100644 --- a/src/utils/utils_pipeline.h +++ b/src/utils/utils_pipeline.h @@ -1,6 +1,7 @@ /* * Copyright(c) 2019-2022 Intel Corporation * Copyright(c) 2024-2025 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -183,6 +184,9 @@ void ocf_pipeline_rollback(ocf_pipeline_t pipeline, int error); void ocf_pipeline_finish(ocf_pipeline_t pipeline, int error); +void ocf_pipeline_continue_on_zero_refcnt(ocf_pipeline_t pipeline, + struct env_refcnt *refcnt); + #define OCF_PL_NEXT_RET(pipeline) ({ \ ocf_pipeline_next(pipeline); \ return; \