@Test
    public void testChangePeersChaosApplyTasks() throws Exception {
        // start cluster
        final List<PeerId> peers = new ArrayList<>();
        peers.add(new PeerId("127.0.0.1", TestUtils.INIT_PORT));
        final TestCluster cluster = new TestCluster("testChangePeersChaosApplyTasks", this.dataPath, peers, 1000);
        assertTrue(cluster.start(peers.get(0).getEndpoint(), false, 100000));
        // start other peers
        for (int i = 1; i < 10; i++) {
            final PeerId peer = new PeerId("127.0.0.1", TestUtils.INIT_PORT + i);
            peers.add(peer);
            assertTrue(cluster.start(peer.getEndpoint(), true, 100000));
        }

        final int threads = 3;
        final List<ChangeArg> args = new ArrayList<>();
        final List<Future<?>> futures = new ArrayList<>();
        final CountDownLatch latch = new CountDownLatch(threads);
        for (int t = 0; t < threads; t++) {
            final ChangeArg arg = new ChangeArg(cluster, peers, false, true);
            args.add(arg);
            futures.add(startChangePeersThread(arg));

            TestUtils.runInThread(() -> {
                try {
                    for (int i = 0; i < 5000;) {
                        cluster.waitLeader();
                        final Node leader = cluster.getLeader();
                        if (leader == null) {
                            continue;
                        }
                        final SynchronizedClosure done = new SynchronizedClosure();
                        final Task task = new Task(ByteBuffer.wrap(("hello" + i).getBytes()), done);
                        leader.apply(task);
                        final Status status = done.await();
                        if (status.isOk()) {
                            if (++i % 100 == 0) {
                                System.out.println("Progress:" + i);
                            }
                        } else {
                            assertEquals(RaftError.EPERM, status.getRaftError());
                        }
                    }
                } catch (final Exception e) {
                    e.printStackTrace();
                } finally {
                    latch.countDown();
                }
            });
        }

        latch.await();
        for (final ChangeArg arg : args) {
            arg.stop = true;
        }
		for (final Future<?> future : futures) {
			try {
				future.get(20, TimeUnit.SECONDS);
			} catch (TimeoutException e) {
				LOG.warn("Timeout while waiting for future completion in testChangePeersChaosApplyTasks", e);
			}
		}

        cluster.waitLeader();
        final SynchronizedClosure done = new SynchronizedClosure();
        final Node leader = cluster.getLeader();
        leader.changePeers(new Configuration(peers), done);
        try {
        	 Status status = done.await();
     	     assertTrue(status.getErrorMsg(), status.isOk());
             cluster.ensureSame();
             assertEquals(10, cluster.getFsms().size());
            for (final MockStateMachine fsm : cluster.getFsms()) {
                final int logSize = fsm.getLogs().size();
                assertTrue("logSize= " + logSize, logSize >= 5000 * threads);
            }
        } finally {
            cluster.stopAll();
        }
    }
mock state machine that fails to load snapshot.
@author boyan (boyan@alibaba-inc.com)

2018-Apr-23 11:45:29 AM
jraft-core/src/test/java/com/alipay/sofa/jraft/core/NodeTest.java