DAOS-17535 chk: misc improvements for CR logic

Include the followings:

1. When create CHK IV namespace, make the secondary group to be same as
   the primary group. Otherwise, CHK logic may hit DER_NONEXIST trouble
   when communicate via IV.

2. Integrate CHK IV namespace create and destroy API, cleanup related
   logic, redefine the version.

3. Get ranks list and IV namespace version from CHK leader when rejoin.
   Adjust CHK_REJOIN RPC for related changes.

4. Remove unsupported functionality for checking the specified 'phase'.

5. Add new test for case of lost some engine(s) before start checker.

Test-tag: recovery

Signed-off-by: Fan Yong <fan.yong@hpe.com>

Author: Nasf-Fan

src/chk/chk_common.c

@@ -1073,8 +1073,7 @@ chk_policy_refresh(uint32_t policy_nr, struct chk_policy *policies, struct chk_p
 }
 
 int
-chk_prop_prepare(d_rank_t leader, uint32_t flags, int phase,
-		 uint32_t policy_nr, struct chk_policy *policies,
+chk_prop_prepare(d_rank_t leader, uint32_t flags, uint32_t policy_nr, struct chk_policy *policies,
 		 d_rank_list_t *ranks, struct chk_property *prop)
 {
 	int rc = 0;
@@ -1086,11 +1085,8 @@ chk_prop_prepare(d_rank_t leader, uint32_t flags, int phase,
 		prop->cp_flags &= ~CHK__CHECK_FLAG__CF_FAILOUT;
 	if (flags & CHK__CHECK_FLAG__CF_NO_AUTO)
 		prop->cp_flags &= ~CHK__CHECK_FLAG__CF_AUTO;
-	prop->cp_flags |= flags & ~(CHK__CHECK_FLAG__CF_RESET |
-				    CHK__CHECK_FLAG__CF_ORPHAN_POOL |
-				    CHK__CHECK_FLAG__CF_NO_FAILOUT |
-				    CHK__CHECK_FLAG__CF_NO_AUTO);
-	prop->cp_phase = phase;
+	prop->cp_flags |= flags & ~(CHK__CHECK_FLAG__CF_RESET | CHK__CHECK_FLAG__CF_ORPHAN_POOL |
+				    CHK__CHECK_FLAG__CF_NO_FAILOUT | CHK__CHECK_FLAG__CF_NO_AUTO);
 	if (ranks != NULL)
 		prop->cp_rank_nr = ranks->rl_nr;
 
@@ -1301,10 +1297,7 @@ chk_ins_fini(struct chk_instance **p_ins)
 		return;
 
 	ins->ci_inited = 0;
-	chk_iv_ns_cleanup(&ins->ci_iv_ns);
-
-	if (ins->ci_iv_group != NULL)
-		crt_group_secondary_destroy(ins->ci_iv_group);
+	chk_iv_ns_destroy(ins);
 
 	d_rank_list_free(ins->ci_ranks);
 	D_ASSERT(d_list_empty(&ins->ci_dead_ranks));

src/chk/chk_engine.c

@@ -2045,9 +2045,8 @@ chk_engine_sched(void *args)
 
 static int
 chk_engine_start_prep(struct chk_instance *ins, uint32_t rank_nr, d_rank_t *ranks,
-		      uint32_t policy_nr, struct chk_policy *policies, int pool_nr,
-		      uuid_t pools[], uint64_t gen, int phase, uint32_t api_flags,
-		      d_rank_t leader, uint32_t flags)
+		      uint32_t policy_nr, struct chk_policy *policies, int pool_nr, uuid_t pools[],
+		      uint64_t gen, uint32_t api_flags, d_rank_t leader, uint32_t flags)
 {
 	struct chk_traverse_pools_args	 ctpa = { 0 };
 	struct chk_bookmark		*cbk = &ins->ci_bk;
@@ -2134,8 +2133,7 @@ chk_engine_start_prep(struct chk_instance *ins, uint32_t rank_nr, d_rank_t *rank
 
 init:
 	if (!chk_is_on_leader(gen, leader, true)) {
-		rc = chk_prop_prepare(leader, api_flags, phase, policy_nr, policies, rank_list,
-				      prop);
+		rc = chk_prop_prepare(leader, api_flags, policy_nr, policies, rank_list, prop);
 		if (rc != 0)
 			goto out;
 
@@ -2263,16 +2261,15 @@ chk_engine_pool_filter(uuid_t uuid, void *arg, int *phase)
 int
 chk_engine_start(uint64_t gen, uint32_t rank_nr, d_rank_t *ranks, uint32_t policy_nr,
 		 struct chk_policy *policies, int pool_nr, uuid_t pools[], uint32_t api_flags,
-		 int phase, d_rank_t leader, uint32_t flags, uuid_t iv_uuid,
+		 uint32_t ns_ver, d_rank_t leader, uint32_t flags, uuid_t iv_uuid,
 		 struct ds_pool_clues *clues)
 {
-	struct chk_instance		*ins = chk_engine;
-	struct chk_bookmark		*cbk = &ins->ci_bk;
-	struct umem_attr		 uma = { 0 };
-	char				 uuid_str[DAOS_UUID_STR_SIZE];
-	d_rank_t			 myrank = dss_self_rank();
-	int				 rc;
-	int				 rc1;
+	struct chk_instance *ins    = chk_engine;
+	struct chk_bookmark *cbk    = &ins->ci_bk;
+	struct umem_attr     uma    = {0};
+	d_rank_t             myrank = dss_self_rank();
+	int                  rc;
+	int                  rc1;
 
 	rc = chk_ins_can_start(ins);
 	if (rc != 0)
@@ -2294,12 +2291,7 @@ chk_engine_start(uint64_t gen, uint32_t rank_nr, d_rank_t *ranks, uint32_t polic
 	if (ins->ci_sched != ABT_THREAD_NULL)
 		ABT_thread_free(&ins->ci_sched);
 
-	chk_iv_ns_cleanup(&ins->ci_iv_ns);
-
-	if (ins->ci_iv_group != NULL) {
-		crt_group_secondary_destroy(ins->ci_iv_group);
-		ins->ci_iv_group = NULL;
-	}
+	chk_iv_ns_destroy(ins);
 
 	uma.uma_id = UMEM_CLASS_VMEM;
 
@@ -2313,27 +2305,20 @@ chk_engine_start(uint64_t gen, uint32_t rank_nr, d_rank_t *ranks, uint32_t polic
 	if (rc != 0)
 		goto out_tree;
 
-	rc = chk_engine_start_prep(ins, rank_nr, ranks, policy_nr, policies,
-				   pool_nr, pools, gen, phase, api_flags, leader, flags);
+	rc = chk_engine_start_prep(ins, rank_nr, ranks, policy_nr, policies, pool_nr, pools, gen,
+				   api_flags, leader, flags);
 	if (rc != 0)
 		goto out_tree;
 
 	if (chk_is_on_leader(gen, leader, true)) {
 		ins->ci_iv_ns = chk_leader_get_iv_ns();
-		if (unlikely(ins->ci_iv_ns == NULL))
-			goto out_tree;
+		D_ASSERT(ins->ci_iv_ns != NULL);
+
+		ins->ci_ns_ver = ns_ver;
 	} else {
-		uuid_unparse_lower(iv_uuid, uuid_str);
-		rc = crt_group_secondary_create(uuid_str, NULL, ins->ci_ranks, &ins->ci_iv_group);
+		rc = chk_iv_ns_create(ins, iv_uuid, leader, ns_ver);
 		if (rc != 0)
 			goto out_tree;
-
-		rc = ds_iv_ns_create(dss_get_module_info()->dmi_ctx, iv_uuid, ins->ci_iv_group,
-				     &ins->ci_iv_id, &ins->ci_iv_ns);
-		if (rc != 0)
-			goto out_group;
-
-		ds_iv_ns_update(ins->ci_iv_ns, leader, ins->ci_iv_ns->iv_master_term + 1);
 	}
 
 	uuid_copy(cbk->cb_iv_uuid, iv_uuid);
@@ -2366,30 +2351,26 @@ chk_engine_start(uint64_t gen, uint32_t rank_nr, d_rank_t *ranks, uint32_t polic
 			D_WARN(DF_ENGINE" failed to update engine bookmark: "DF_RC"\n",
 			       DP_ENGINE(ins), DP_RC(rc1));
 	}
-	chk_iv_ns_cleanup(&ins->ci_iv_ns);
-out_group:
-	if (ins->ci_iv_group != NULL) {
-		crt_group_secondary_destroy(ins->ci_iv_group);
-		ins->ci_iv_group = NULL;
-	}
+	chk_iv_ns_destroy(ins);
 out_tree:
 	chk_destroy_pending_tree(ins);
 	chk_destroy_pool_tree(ins);
 out_done:
 	ins->ci_starting = 0;
 out_log:
 	if (rc >= 0) {
-		D_INFO(DF_ENGINE " %s on rank %u with api_flags %x, phase %d, leader %u, "
-		       "flags %x, iv "DF_UUIDF": rc %d\n",
+		D_INFO(DF_ENGINE " %s on rank %u with api_flags %x, ns_ver %d, leader %u, "
+				 "flags %x, iv " DF_UUIDF ": rc %d\n",
 		       DP_ENGINE(ins), chk_is_ins_reset(ins, api_flags) ? "start" : "resume",
-		       myrank, api_flags, phase, leader, flags, DP_UUID(iv_uuid), rc);
+		       myrank, api_flags, ns_ver, leader, flags, DP_UUID(iv_uuid), rc);
 
 		chk_ranks_dump(ins->ci_ranks->rl_nr, ins->ci_ranks->rl_ranks);
 		chk_pools_dump(&ins->ci_pool_list, pool_nr, pools);
 	} else {
-		D_ERROR(DF_ENGINE" failed to start on rank %u with %d pools, api_flags %x, "
-			"phase %d, leader %u, flags %x, gen "DF_X64", iv "DF_UUIDF": "DF_RC"\n",
-			DP_ENGINE(ins), myrank, pool_nr, api_flags, phase, leader, flags, gen,
+		D_ERROR(DF_ENGINE " failed to start on rank %u with %d pools, api_flags %x, "
+				  "ns_ver %d, leader %u, flags %x, gen " DF_X64 ", iv " DF_UUIDF
+				  ": " DF_RC "\n",
+			DP_ENGINE(ins), myrank, pool_nr, api_flags, ns_ver, leader, flags, gen,
 			DP_UUID(iv_uuid), DP_RC(rc));
 	}
 
@@ -2413,7 +2394,7 @@ chk_engine_stop(uint64_t gen, int pool_nr, uuid_t pools[], uint32_t *flags)
 	if (cbk->cb_magic != CHK_BK_MAGIC_ENGINE)
 		D_GOTO(log, rc = -DER_NOTAPPLICABLE);
 
-	if (ins->ci_starting)
+	if (ins->ci_starting || ins->ci_rejoining)
 		D_GOTO(log, rc = -DER_BUSY);
 
 	if (ins->ci_stopping || ins->ci_sched_exiting)
@@ -2642,32 +2623,42 @@ chk_engine_query(uint64_t gen, int pool_nr, uuid_t pools[], uint32_t *ins_status
 int
 chk_engine_mark_rank_dead(uint64_t gen, d_rank_t rank, uint32_t version)
 {
-	struct chk_instance	*ins = chk_engine;
-	struct chk_property	*prop = &ins->ci_prop;
-	struct chk_bookmark	*cbk = &ins->ci_bk;
-	d_rank_list_t		*rank_list = NULL;
-	int			 rc = 0;
+	struct chk_instance *ins  = chk_engine;
+	struct chk_property *prop = &ins->ci_prop;
+	struct chk_bookmark *cbk  = &ins->ci_bk;
+	int                  rc   = 0;
 
 	if (cbk->cb_gen != gen)
 		D_GOTO(out, rc = -DER_NOTAPPLICABLE);
 
-	rc = chk_prop_fetch(prop, &rank_list);
-	if (rc != 0)
-		goto out;
+	/* For check engine on the leader, reload rank list that has been refreshed by leader. */
+	if (chk_is_on_leader(cbk->cb_gen, prop->cp_leader, true)) {
+		d_rank_list_free(ins->ci_ranks);
+		ins->ci_ranks = NULL;
+	}
 
-	D_ASSERT(rank_list != NULL);
+	if (ins->ci_ranks == NULL) {
+		rc = chk_prop_fetch(prop, &ins->ci_ranks);
+		if (rc != 0)
+			goto out;
 
-	/* For check engine on the leader, related rank has already been marked as "dead". */
-	if (chk_is_on_leader(cbk->cb_gen, prop->cp_leader, true))
-		goto group;
+		/* For check engine on the leader, it's done. */
+		if (chk_is_on_leader(cbk->cb_gen, prop->cp_leader, true)) {
+			ins->ci_ns_ver = version;
+			goto out;
+		}
+	}
+
+	if (unlikely(ins->ci_ranks == NULL))
+		D_GOTO(out, rc = -DER_NOTAPPLICABLE);
 
-	if (!chk_remove_rank_from_list(rank_list, rank))
+	if (!chk_remove_rank_from_list(ins->ci_ranks, rank))
 		D_GOTO(out, rc = -DER_NOTAPPLICABLE);
 
 	prop->cp_rank_nr--;
-	rc = chk_prop_update(prop, rank_list);
-	if (rc != 0)
-		goto out;
+	rc = chk_prop_update(prop, ins->ci_ranks);
+	if (rc == 0)
+		rc = chk_iv_ns_update(ins, version);
 
 	/*
 	 * NOTE: If the rank dead before DAOS check start, then subsequent check start will
@@ -2688,19 +2679,7 @@ chk_engine_mark_rank_dead(uint64_t gen, d_rank_t rank, uint32_t version)
 	 *	 sometime later as the DAOS check going.
 	 */
 
-group:
-	if (ins->ci_iv_group != NULL)
-		rc = crt_group_secondary_modify(ins->ci_iv_group, rank_list, rank_list,
-						CRT_GROUP_MOD_OP_REPLACE, version);
-
 out:
-	if (rc == 0) {
-		d_rank_list_free(ins->ci_ranks);
-		ins->ci_ranks = rank_list;
-		rank_list = NULL;
-	}
-
-	d_rank_list_free(rank_list);
 	if (rc != -DER_NOTAPPLICABLE)
 		D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO,
 			 DF_ENGINE" on rank %u mark rank %u as dead with gen "
@@ -3364,19 +3343,19 @@ chk_engine_notify(struct chk_iv *iv)
 void
 chk_engine_rejoin(void *args)
 {
-	struct chk_instance		*ins = chk_engine;
-	struct chk_property		*prop = &ins->ci_prop;
-	struct chk_bookmark		*cbk = &ins->ci_bk;
-	uuid_t				*pools = NULL;
-	struct chk_iv			 iv = { 0 };
-	struct umem_attr		 uma = { 0 };
-	char				 uuid_str[DAOS_UUID_STR_SIZE];
-	d_rank_t			 myrank = dss_self_rank();
-	uint32_t			 pool_nr = 0;
-	uint32_t			 flags = 0;
-	int				 rc = 0;
-	int				 rc1;
-	bool				 need_join = false;
+	struct chk_instance *ins     = chk_engine;
+	struct chk_property *prop    = &ins->ci_prop;
+	struct chk_bookmark *cbk     = &ins->ci_bk;
+	d_rank_list_t       *ranks   = NULL;
+	uuid_t              *pools   = NULL;
+	struct chk_iv        iv      = {0};
+	struct umem_attr     uma     = {0};
+	d_rank_t             myrank  = dss_self_rank();
+	uint32_t             pool_nr = 0;
+	uint32_t             flags   = 0;
+	int                  rc      = 0;
+	int                  rc1;
+	bool                 need_join = false;
 
 	if (cbk->cb_magic != CHK_BK_MAGIC_ENGINE)
 		goto out_log;
@@ -3385,7 +3364,7 @@ chk_engine_rejoin(void *args)
 	    cbk->cb_ins_status != CHK__CHECK_INST_STATUS__CIS_PAUSED)
 		goto out_log;
 
-	/* We do NOT support leader (and its associated engine ) to rejoin former check instance. */
+	/* We do NOT support leader (and its associated engine) to rejoin former check instance. */
 	if (chk_is_on_leader(cbk->cb_gen, prop->cp_leader, true))
 		goto out_log;
 
@@ -3420,22 +3399,10 @@ chk_engine_rejoin(void *args)
 	if (rc != 0)
 		goto out_tree;
 
-	uuid_unparse_lower(cbk->cb_iv_uuid, uuid_str);
-	rc = crt_group_secondary_create(uuid_str, NULL, ins->ci_ranks, &ins->ci_iv_group);
-	if (rc != 0)
-		goto out_tree;
-
-	rc = ds_iv_ns_create(dss_get_module_info()->dmi_ctx, cbk->cb_iv_uuid, ins->ci_iv_group,
-			     &ins->ci_iv_id, &ins->ci_iv_ns);
-	if (rc != 0)
-		goto out_group;
-
-	ds_iv_ns_update(ins->ci_iv_ns, prop->cp_leader, ins->ci_iv_ns->iv_master_term + 1);
-
 again:
 	/* Ask leader whether this engine can rejoin or not. */
 	rc = chk_rejoin_remote(prop->cp_leader, cbk->cb_gen, myrank, cbk->cb_iv_uuid, &flags,
-			       &pool_nr, &pools);
+			       &ins->ci_ns_ver, &pool_nr, &pools, &ranks);
 	if (rc != 0) {
 		if ((rc == -DER_OOG || rc == -DER_GRPVER) && !ins->ci_pause) {
 			D_INFO(DF_ENGINE" Someone is not ready %d, let's rejoin after 1 sec\n",
@@ -3445,14 +3412,22 @@ chk_engine_rejoin(void *args)
 				goto again;
 		}
 
-		goto out_iv;
+		goto out_tree;
 	}
 
-	if (pool_nr == 0) {
+	if (ranks == NULL || pool_nr == 0) {
 		need_join = false;
-		D_GOTO(out_iv, rc = 1);
+		D_GOTO(out_tree, rc = 1);
 	}
 
+	d_rank_list_free(ins->ci_ranks);
+	ins->ci_ranks = ranks;
+	ranks         = NULL;
+
+	rc = chk_iv_ns_create(ins, cbk->cb_iv_uuid, prop->cp_leader, ins->ci_ns_ver);
+	if (rc != 0)
+		goto out_tree;
+
 	rc = chk_pools_load_list(ins, cbk->cb_gen, 0, pool_nr, pools, NULL);
 	if (rc != 0)
 		goto out_notify;
@@ -3496,17 +3471,13 @@ chk_engine_rejoin(void *args)
 	D_CDEBUG(rc1 != 0, DLOG_ERR, DLOG_INFO,
 		 DF_ENGINE" on rank %u notify leader for its exit, status %u: rc1 = %d\n",
 		 DP_ENGINE(ins), myrank, cbk->cb_ins_status, rc1);
-out_iv:
-	chk_iv_ns_cleanup(&ins->ci_iv_ns);
-out_group:
-	if (ins->ci_iv_group != NULL) {
-		crt_group_secondary_destroy(ins->ci_iv_group);
-		ins->ci_iv_group = NULL;
-	}
+	chk_iv_ns_destroy(ins);
 out_tree:
 	chk_destroy_pending_tree(ins);
 	chk_destroy_pool_tree(ins);
 out_log:
+	d_rank_list_free(ranks);
+	D_FREE(pools);
 	if (need_join)
 		D_CDEBUG(rc < 0, DLOG_ERR, DLOG_INFO,
 			 DF_ENGINE" rejoin on rank %u with iv "DF_UUIDF": "DF_RC"\n",