Merge branch 'master' into sdl

Author: Adam Rutkowski

Diff for: doc/requirements/disable_cleaner

@@ -0,0 +1,67 @@
+---
+group: disable_cleaner
+---
+
+Disabling cleaner is a feature that is intented to be used with read cache
+(Write-Thourgh and Write-Around modes), where no dirty data is produced, so
+the cleaning is not necessary. Its major benefit is metadata memory footprint
+reduction, which is result of not allocating "cleaning" metatada section,
+that constitutes about 20% of total metadata capacity.
+
+While it is possible to use disable_cleaner option Write-Back and Write-Only
+modes, it may result in performance degradation due to necessity to perform
+cleaning on eviction.
+
+--------------------------------------------------------------------------------
+--------------------------------------------------------------------------------
+title: Setting cleaner_disabled mode
+id: set_cleaner_disabled
+---
+
+It shall be possible to select cleaner_disabled mode during cache attach
+operation by setting apropirate field in attach config.
+
+--------------------------------------------------------------------------------
+--------------------------------------------------------------------------------
+title: Loading cache cleaner_disabled mode
+id: load_cleaner_disabled
+---
+
+The cleaner_disabled setting should be preserved in cache metadata, i.e. on the
+cache load/recovery the setting should be restored to the value it had before
+the cache stop.
+
+--------------------------------------------------------------------------------
+--------------------------------------------------------------------------------
+title: Metadata "cleaning" section allocation
+id: cleaning_section_alocation
+---
+
+When disable_cleaner option is selected the "cleaning" section in OCF metadata
+shall not be allocated neither in DRAM nor on the cache drive.
+
+--------------------------------------------------------------------------------
+--------------------------------------------------------------------------------
+title: Starting with NOP cleaning policy
+id: starting_with_nop_policy
+---
+
+When attaching/loading cache with disable_cleaner option selected, the cleaning
+policy shall always be NOP.
+
+--------------------------------------------------------------------------------
+--------------------------------------------------------------------------------
+title: NOP cleaning policy enforcement
+id: nop_enforcement
+---
+
+When disable_cleaner option is selected it shall not be possible to change the
+cleaning policy to other than NOP.
+
+--------------------------------------------------------------------------------
+--------------------------------------------------------------------------------
+title: Default setting
+id: default_setting
+---
+
+By default the disable_cleaner option shall not be selected.

Diff for: env/posix/ocf_env.h

@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2019-2021 Intel Corporation
+ * Copyright(c) 2019-2022 Intel Corporation
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -67,11 +67,13 @@ typedef uint64_t sector_t;
 #define ENV_MEM_ATOMIC	0
 
 /* DEBUGING */
+void env_stack_trace(void);
+
 #define ENV_WARN(cond, fmt...)		printf(fmt)
 #define ENV_WARN_ON(cond)		;
 #define ENV_WARN_ONCE(cond, fmt...)	ENV_WARN(cond, fmt)
 
-#define ENV_BUG()			assert(0)
+#define ENV_BUG()			do {env_stack_trace(); assert(0);} while(0)
 #define ENV_BUG_ON(cond)		do { if (cond) ENV_BUG(); } while (0)
 #define ENV_BUILD_BUG_ON(cond)		_Static_assert(!(cond), "static "\
 					"assertion failure")

Diff for: inc/ocf_err.h

@@ -155,7 +155,10 @@ typedef enum {
 	/** Operation only allowed in standby mode **/
 	OCF_ERR_CACHE_NOT_STANDBY,
 
-	OCF_ERR_MAX = OCF_ERR_CACHE_NOT_STANDBY,
+	/** Operation not allowed when cleaner is disabled **/
+	OCF_ERR_CLEANER_DISABLED,
+
+	OCF_ERR_MAX = OCF_ERR_CLEANER_DISABLED,
 
 } ocf_error_t;
 

Diff for: inc/ocf_mngt.h

@@ -424,6 +424,21 @@ struct ocf_mngt_cache_attach_config {
 	 * @brief If set, cache device will be discarded on cache start
 	 */
 	bool discard_on_start;
+
+	/**
+	 * @brief If set, asynchronous cleaning will be disabled
+	 *
+	 * Has similar effect to setting cleaning policy to NOP, but
+	 * additionally prevents allocating "cleaning" metadata section,
+	 * which can reduce memory footprint by up to 20%.
+	 *
+	 * When this option is selected, any attempt to change the cleaning
+	 * policy will fail.
+	 *
+	 * @note This option is meaningful only with ocf_mngt_cache_attach().
+	 *       When used with ocf_mngt_cache_load() it's ignored.
+	 */
+	bool disable_cleaner;
 };
 
 /**
@@ -441,6 +456,7 @@ static inline void ocf_mngt_cache_attach_config_set_default(
 	cfg->open_cores = true;
 	cfg->force = false;
 	cfg->discard_on_start = true;
+	cfg->disable_cleaner = false;
 	cfg->device.perform_test = true;
 	cfg->device.volume_params = NULL;
 }

Diff for: src/cleaning/acp.c

@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2012-2021 Intel Corporation
+ * Copyright(c) 2012-2022 Intel Corporation
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -363,7 +363,7 @@ static int ocf_acp_recovery_handle(ocf_parallelize_t parallelize,
 	ocf_core_id_t core_id;
 	uint32_t step = 0;
 
-	portion = DIV_ROUND_UP((uint64_t)entries, shards_cnt);
+	portion = OCF_DIV_ROUND_UP((uint64_t)entries, shards_cnt);
 	begin = portion*shard_id;
 	end = OCF_MIN(portion*(shard_id + 1), entries);
 

Diff for: src/cleaning/alru.c

@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2012-2021 Intel Corporation
+ * Copyright(c) 2012-2022 Intel Corporation
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -516,7 +516,7 @@ static int ocf_alru_recovery_handle(ocf_parallelize_t parallelize,
 	uint32_t step = 0;
 	int i;
 
-	portion = DIV_ROUND_UP((uint64_t)entries, shards_cnt);
+	portion = OCF_DIV_ROUND_UP((uint64_t)entries, shards_cnt);
 	begin = portion*shard_id;
 	end = OCF_MIN((uint64_t)portion*(shard_id + 1), entries);
 

Diff for: src/concurrency/ocf_cache_line_concurrency.c

@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2012-2021 Intel Corporation
+ * Copyright(c) 2012-2022 Intel Corporation
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -141,9 +141,17 @@ static int ocf_cl_lock_line_slow(struct ocf_alock *alock,
 	return ret;
 }
 
+static uint32_t ocf_cl_lock_get_entries_count(struct ocf_alock *alock,
+		struct ocf_request *req)
+{
+	return req->core_line_count;
+}
+
 static struct ocf_alock_lock_cbs ocf_cline_conc_cbs = {
 		.lock_entries_fast = ocf_cl_lock_line_fast,
-		.lock_entries_slow = ocf_cl_lock_line_slow
+		.lock_entries_slow = ocf_cl_lock_line_slow,
+		.get_entries_count = ocf_cl_lock_get_entries_count
+
 };
 
 bool ocf_cache_line_try_lock_rd(struct ocf_alock *alock,

Diff for: src/concurrency/ocf_mio_concurrency.c

@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2021 Intel Corporation
+ * Copyright(c) 2021-2022 Intel Corporation
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -98,9 +98,16 @@ static int ocf_mio_lock_slow(struct ocf_alock *alock,
 	return ret;
 }
 
+static uint32_t ocf_mio_lock_get_entries_count(struct ocf_alock *alock,
+		struct ocf_request *req)
+{
+	return req->core_line_count;
+}
+
 static struct ocf_alock_lock_cbs ocf_mio_conc_cbs = {
 		.lock_entries_fast = ocf_mio_lock_fast,
-		.lock_entries_slow = ocf_mio_lock_slow
+		.lock_entries_slow = ocf_mio_lock_slow,
+		.get_entries_count = ocf_mio_lock_get_entries_count
 };
 
 int ocf_mio_async_lock(struct ocf_alock *alock,

Diff for: src/concurrency/ocf_pio_concurrency.c

@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2021 Intel Corporation
+ * Copyright(c) 2021-2022 Intel Corporation
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -68,7 +68,7 @@ static int ocf_pio_lock_fast(struct ocf_alock *alock,
 
 	for (i = 0; i < req->core_line_count; i++) {
 		entry = ocf_pio_lock_get_entry(alock, req, i);
-		if (entry == OUT_OF_RANGE)
+		if (unlikely(entry == OUT_OF_RANGE))
 			continue;
 
 		ENV_BUG_ON(ocf_alock_is_index_locked(alock, req, i));
@@ -87,7 +87,7 @@ static int ocf_pio_lock_fast(struct ocf_alock *alock,
 	/* Request is not locked, discard acquired locks */
 	for (; i >= 0; i--) {
 		entry = ocf_pio_lock_get_entry(alock, req, i);
-		if (entry == OUT_OF_RANGE)
+		if (unlikely(entry == OUT_OF_RANGE))
 			continue;
 
 		if (ocf_alock_is_index_locked(alock, req, i)) {
@@ -109,7 +109,7 @@ static int ocf_pio_lock_slow(struct ocf_alock *alock,
 
 	for (i = 0; i < req->core_line_count; i++) {
 		entry = ocf_pio_lock_get_entry(alock, req, i);
-		if (entry == OUT_OF_RANGE)
+		if (unlikely(entry == OUT_OF_RANGE))
 			continue;
 
 		ENV_BUG_ON(ocf_alock_is_index_locked(alock, req, i));
@@ -127,7 +127,7 @@ static int ocf_pio_lock_slow(struct ocf_alock *alock,
 err:
 	for (; i >= 0; i--) {
 		entry = ocf_pio_lock_get_entry(alock, req, i);
-		if (entry == OUT_OF_RANGE)
+		if (unlikely(entry == OUT_OF_RANGE))
 			continue;
 
 		ocf_alock_waitlist_remove_entry(alock, req, i, entry, OCF_WRITE);
@@ -136,9 +136,26 @@ static int ocf_pio_lock_slow(struct ocf_alock *alock,
 	return ret;
 }
 
+static uint32_t ocf_pio_lock_get_entries_count(struct ocf_alock *alock,
+		struct ocf_request *req)
+{
+	uint32_t i, count = 0;
+	ocf_cache_line_t entry;
+
+	for (i = 0; i < req->core_line_count; i++) {
+		entry = ocf_pio_lock_get_entry(alock, req, i);
+		if (unlikely(entry == OUT_OF_RANGE))
+			continue;
+		count++;
+	}
+
+	return count;
+}
+
 static struct ocf_alock_lock_cbs ocf_pio_conc_cbs = {
 		.lock_entries_fast = ocf_pio_lock_fast,
-		.lock_entries_slow = ocf_pio_lock_slow
+		.lock_entries_slow = ocf_pio_lock_slow,
+		.get_entries_count = ocf_pio_lock_get_entries_count
 };
 
 int ocf_pio_async_lock(struct ocf_alock *alock, struct ocf_request *req,
@@ -157,7 +174,7 @@ void ocf_pio_async_unlock(struct ocf_alock *alock, struct ocf_request *req)
 			continue;
 
 		entry = ocf_pio_lock_get_entry(alock, req, i);
-		if (entry == OUT_OF_RANGE)
+		if (unlikely(entry == OUT_OF_RANGE))
 			continue;
 
 		ocf_alock_unlock_one_wr(alock, entry);

Diff for: src/metadata/metadata.c

@@ -638,7 +638,8 @@ static void ocf_metadata_flush_unlock_collision_page(
  * Initialize hash metadata interface
  */
 int ocf_metadata_init_variable_size(struct ocf_cache *cache,
-		uint64_t device_size, ocf_cache_line_size_t line_size)
+		uint64_t device_size, ocf_cache_line_size_t line_size,
+		bool cleaner_disabled)
 {
 	int result = 0;
 	uint32_t i = 0;
@@ -688,6 +689,12 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache,
 			raw->raw_type = metadata_raw_type_atomic;
 		}
 
+		if (i == metadata_segment_cleaning && cleaner_disabled) {
+			raw->entry_size = 0;
+			raw->entries_in_page = 1;
+			continue;
+		}
+
 		/* Entry size configuration */
 		raw->entry_size
 			= ocf_metadata_get_element_size(i, line_size);
@@ -715,6 +722,9 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache,
 	 */
 	for (i = metadata_segment_variable_size_start;
 			i < metadata_segment_max; i++) {
+		if (i == metadata_segment_cleaning && cleaner_disabled)
+			continue;
+
 		if (i == metadata_segment_collision) {
 			lock_page =
 				ocf_metadata_flush_lock_collision_page;
@@ -779,6 +789,7 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache,
 
 	cache->conf_meta->cachelines = ctrl->cachelines;
 	cache->conf_meta->line_size = line_size;
+	cache->conf_meta->cleaner_disabled = cleaner_disabled;
 
 	ocf_metadata_raw_info(cache, ctrl);
 
@@ -1620,6 +1631,7 @@ static void ocf_metadata_load_properties_cmpl(
 	properties.shutdown_status = superblock->clean_shutdown;
 	properties.dirty_flushed = superblock->dirty_flushed;
 	properties.cache_name = superblock->name;
+	properties.cleaner_disabled = superblock->cleaner_disabled;
 
 	OCF_CMPL_RET(priv, 0, &properties);
 }

Diff for: src/metadata/metadata.h

@@ -41,10 +41,12 @@ int ocf_metadata_init(struct ocf_cache *cache,
  * @param cache - Cache instance
  * @param device_size - Device size in bytes
  * @param cache_line_size Cache line size
+ * @param cleaner_disabled Cleaner is disabled
  * @return 0 - Operation success otherwise failure
  */
 int ocf_metadata_init_variable_size(struct ocf_cache *cache,
-		uint64_t device_size, ocf_cache_line_size_t cache_line_size);
+		uint64_t device_size, ocf_cache_line_size_t line_size,
+		bool cleaner_disabled);
 
 /**
  * @brief Initialize collision table
@@ -201,6 +203,7 @@ struct ocf_metadata_load_properties {
 	ocf_cache_mode_t cache_mode;
 	ocf_cache_line_size_t line_size;
 	char *cache_name;
+	bool cleaner_disabled;
 };
 
 typedef void (*ocf_metadata_load_properties_end_t)(void *priv, int error,

Diff for: src/metadata/metadata_cleaning_policy.c

@@ -18,6 +18,8 @@ ocf_metadata_get_cleaning_policy(struct ocf_cache *cache,
 	struct ocf_metadata_ctrl *ctrl
 		= (struct ocf_metadata_ctrl *) cache->metadata.priv;
 
+	ENV_BUG_ON(cache->conf_meta->cleaner_disabled);
+
 	return ocf_metadata_raw_wr_access(cache,
 			&(ctrl->raw_desc[metadata_segment_cleaning]), line);
 }