Skip to content

Restore DataNode removal cleanup after child procedure failures #18947

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
CRZbulabula merged 1 commit into from
Oct 20, 2026
Merged

Restore DataNode removal cleanup after child procedure failures #18947

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 12 additions & 13 deletions ...c/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodesProcedure.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,19 +158,16 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, RemoveDataNodeState
return Flow.NO_MORE_STATE;
}
} catch (Exception e) {
if (isRollbackSupported(state)) {
setFailure(new ProcedureException(ProcedureMessages.REMOVE_DATA_NODE_FAILED + state));
// Rollback support permits failure cleanup; transient execution errors still retry here.
LOG.error(
ProcedureMessages.LOG_RETRIEVABLE_ERROR_TRYING_REMOVE_DATA_NODE_ARG_STATE_ARG_4EFEB850,
removedDataNodes,
state,
e);
if (getCycles() > RETRY_THRESHOLD) {
setFailure(new ProcedureException(ProcedureMessages.STATE_STUCK_AT + state));
} else {
LOG.error(
ProcedureMessages.LOG_RETRIEVABLE_ERROR_TRYING_REMOVE_DATA_NODE_ARG_STATE_ARG_4EFEB850,
removedDataNodes,
state,
e);
if (getCycles() > RETRY_THRESHOLD) {
setFailure(new ProcedureException(ProcedureMessages.STATE_STUCK_AT + state));
} else {
setNextState(state);
}
setNextState(state);
}
}
return Flow.HAS_MORE_STATE;
Expand Down Expand Up @@ -291,7 +288,9 @@ protected long getRollbackRetryTimeout() {

@Override
protected boolean isRollbackSupported(RemoveDataNodeState state) {
return false;
// A failed migration child must still trigger node cleanup. This restores node statuses and
// stops drained nodes without undoing completed region migrations.
return state == RemoveDataNodeState.STOP_DATA_NODE;
}

/**
Expand Down
2 changes: 2 additions & 0 deletions ...t/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -420,12 +420,14 @@ public void testProcedureExecutorCanActuallyRetryCompensationCommitFailure() thr
failStatusWrite = true;
TestProcedure procedure = procedure(NodeStatus.Running, RemoveDataNodeState.STOP_DATA_NODE);
procedure.step(env);
assertFalse("A transient cleanup failure must remain retryable", procedure.isFailed());
assertEquals(
"The state-machine executor must retain the failed compensation step",
RemoveDataNodeState.STOP_DATA_NODE,
procedure.state());
failStatusWrite = false;
procedure.step(env);
assertFalse("Successful cleanup retry must not leave a failure", procedure.isFailed());
assertStates(FIRST, NodeStatus.Running, NodeStatus.Running, null);
}

Expand Down
Loading