private void reconfigure(ZookeeperServerConfig newConfig) {
        Instant reconfigTriggered = Instant.now();
        String newServers = servers(newConfig);
        log.log(Level.INFO, "Will reconfigure ZooKeeper cluster." +
                            "\nServers in active config:" + servers(activeConfig) +
                            "\nServers in new config:" + newServers);
        String connectionSpec = vespaZooKeeperAdmin.localConnectionSpec(activeConfig);
        Instant now = Instant.now();
        // For reconfig to succeed, the current and resulting ensembles must have a majority. When an ensemble grows and
        // the joining servers outnumber the existing ones, we have to wait for enough of them to start to have a majority.
        Instant end = now.plus(timeout);
        // Loop reconfiguring since we might need to wait until another reconfiguration is finished before we can succeed
        for (int attempt = 1; ; attempt++) {
            try {
                Instant reconfigStarted = Instant.now();
                vespaZooKeeperAdmin.reconfigure(connectionSpec, newServers);
                Instant reconfigEnded = Instant.now();
                log.log(Level.INFO, "Reconfiguration completed in " +
                                    Duration.between(reconfigTriggered, reconfigEnded) +
                                    ", after " + attempt + " attempt(s). ZooKeeper reconfig call took " +
                                    Duration.between(reconfigStarted, reconfigEnded));
                activeConfig = newConfig;
                return;
            } catch (ReconfigException e) {
                Duration delay = backoff.delay(attempt);
                now = Instant.now();
                if (now.isBefore(end)) {
                    log.log(Level.INFO, "Reconfiguration attempt " + attempt + " failed. Retrying in " + delay +
                                           ", time left " + Duration.between(now, end) + ": " + Exceptions.toMessageString(e));
                    sleeper.sleep(delay);
                }
                else {
                    log.log(Level.SEVERE, "Reconfiguration attempt " + attempt + " failed, and was failing for " +
                                          timeout + "; giving up now: " + Exceptions.toMessageString(e));
                    shutdown();
                    if (haltOnFailure)
                        Process.logAndDie("Reconfiguration did not complete within timeout " + timeout + ". Forcing container shutdown.");
                    else
                        throw e;
                }
            }
        }
    }
@Test
    public void testReconfigure() {
        ZookeeperServerConfig initialConfig = createConfig(3, true);
        reconfigurer.startOrReconfigure(initialConfig);
        assertSame(initialConfig, reconfigurer.activeConfig());

        // Cluster grows
        ZookeeperServerConfig nextConfig = createConfig(5, true);
        reconfigurer.startOrReconfigure(nextConfig);
        assertEquals("node1:2181", reconfigurer.connectionSpec());
        assertEquals("server.0=node0:2182:2183;2181,server.1=node1:2182:2183;2181,server.2=node2:2182:2183;2181,server.3=node3:2182:2183;2181,server.4=node4:2182:2183;2181",
                     reconfigurer.servers());
        assertEquals(2, reconfigurer.reconfigurations());
        assertSame(nextConfig, reconfigurer.activeConfig());

        // Cluster shrinks
        nextConfig = createConfig(3, true);
        reconfigurer.startOrReconfigure(nextConfig);
        assertEquals(3, reconfigurer.reconfigurations());
        assertEquals("node1:2181", reconfigurer.connectionSpec());
        assertEquals("server.0=node0:2182:2183;2181,server.1=node1:2182:2183;2181,server.2=node2:2182:2183;2181",
                     reconfigurer.servers());
        assertSame(nextConfig, reconfigurer.activeConfig());

        // Cluster loses node1, but node3 joins. Indices are shuffled.
        nextConfig = createConfig(3, true, 1);
        reconfigurer.startOrReconfigure(nextConfig);
        assertEquals(4, reconfigurer.reconfigurations());
        assertEquals("server.0=node0:2182:2183;2181,server.1=node2:2182:2183;2181,server.2=node3:2182:2183;2181",
                     reconfigurer.servers());
        assertSame(nextConfig, reconfigurer.activeConfig());
    }