@Test
  public void inconsistentReplicaCapacityTest() throws Exception {
    assumeTrue(listenCrossColo && !fullAutoCompatible);
    clusterManager.close();
    metricRegistry = new MetricRegistry();
    String staticClusterName = "TestOnly";
    File tempDir = Files.createTempDirectory("helixClusterManagerTest").toFile();
    tempDir.deleteOnExit();
    String tempDirPath = tempDir.getAbsolutePath();
    String testHardwareLayoutPath = tempDirPath + File.separator + "hardwareLayoutTest.json";
    String testPartitionLayoutPath = tempDirPath + File.separator + "partitionLayoutTest.json";
    String testZkLayoutPath = tempDirPath + File.separator + "zkLayoutPath.json";

    // initialize test hardware layout and partition layout, create mock helix cluster for testing.
    TestHardwareLayout testHardwareLayout1 = constructInitialHardwareLayoutJSON(staticClusterName);
    TestPartitionLayout testPartitionLayout1 = constructInitialPartitionLayoutJSON(testHardwareLayout1, 3, localDc);
    JSONObject zkJson = constructZkLayoutJSON(dcsToZkInfo.values());
    Utils.writeJsonObjectToFile(zkJson, testZkLayoutPath);
    Utils.writeJsonObjectToFile(testHardwareLayout1.getHardwareLayout().toJSONObject(), testHardwareLayoutPath);
    Utils.writeJsonObjectToFile(testPartitionLayout1.getPartitionLayout().toJSONObject(), testPartitionLayoutPath);
    MockHelixCluster testCluster =
        new MockHelixCluster("AmbryTest-", testHardwareLayoutPath, testPartitionLayoutPath, testZkLayoutPath, localDc,
            useAggregatedView, 100, fullAutoCompatible ? 10000 : -1);

    List<DataNode> initialNodes = testHardwareLayout1.getAllExistingDataNodes();
    Partition partitionToTest = (Partition) testPartitionLayout1.getPartitionLayout().getPartitions(null).get(0);

    // add a new node into cluster
    testHardwareLayout1.addNewDataNodes(1);
    Utils.writeJsonObjectToFile(testHardwareLayout1.getHardwareLayout().toJSONObject(), testHardwareLayoutPath);
    DataNode newAddedNode =
        testHardwareLayout1.getAllExistingDataNodes().stream().filter(n -> !initialNodes.contains(n)).findAny().get();
    // add a new replica on new node for partitionToTest
    Disk diskOnNewNode = newAddedNode.getDisks().get(0);
    // deliberately change capacity of partition to ensure new replica picks new capacity
    partitionToTest.replicaCapacityInBytes += 1;
    partitionToTest.addReplica(new Replica(partitionToTest, diskOnNewNode, testHardwareLayout1.clusterMapConfig));
    Utils.writeJsonObjectToFile(testPartitionLayout1.getPartitionLayout().toJSONObject(), testPartitionLayoutPath);
    testCluster.upgradeWithNewHardwareLayout(testHardwareLayoutPath);
    testCluster.upgradeWithNewPartitionLayout(testPartitionLayoutPath,
        HelixBootstrapUpgradeUtil.HelixAdminOperation.BootstrapCluster);

    // reset hardware/partition layout, this also resets replica capacity of partitionToTest. However, it won't touch
    // instanceConfig of new added node because it is not in hardware layout. So, replica on new added node still has
    // larger capacity. We use this particular replica to mock inconsistent replica capacity case.
    // Note that instanceConfig of new node is still kept in cluster because upgrading cluster didn't force remove
    // instanceConfig that not present in static clustermap.
    testHardwareLayout1 = constructInitialHardwareLayoutJSON(staticClusterName);
    testPartitionLayout1 = constructInitialPartitionLayoutJSON(testHardwareLayout1, 3, localDc);
    Utils.writeJsonObjectToFile(testHardwareLayout1.getHardwareLayout().toJSONObject(), testHardwareLayoutPath);
    Utils.writeJsonObjectToFile(testPartitionLayout1.getPartitionLayout().toJSONObject(), testPartitionLayoutPath);
    testCluster.upgradeWithNewHardwareLayout(testHardwareLayoutPath);
    testCluster.upgradeWithNewPartitionLayout(testPartitionLayoutPath,
        HelixBootstrapUpgradeUtil.HelixAdminOperation.BootstrapCluster);

    Properties props = new Properties();
    props.setProperty("clustermap.host.name", hostname);
    props.setProperty("clustermap.cluster.name", "AmbryTest-" + staticClusterName);
    props.setProperty("clustermap.aggregated.view.cluster.name", "AmbryTest-" + staticClusterName);
    props.setProperty("clustermap.use.aggregated.view", Boolean.toString(useAggregatedView));
    props.setProperty("clustermap.datacenter.name", localDc);
    props.setProperty("clustermap.port", Integer.toString(portNum));
    props.setProperty("clustermap.dcs.zk.connect.strings", zkJson.toString(2));
    props.setProperty("clustermap.current.xid", Long.toString(CURRENT_XID));
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(new VerifiableProperties(props));
    // instantiate HelixClusterManager and its initialization should fail because validation on replica capacity cannot
    // succeed (The aforementioned replica has larger capacity than its peers)
    try {
      new HelixClusterManager(clusterMapConfig, selfInstanceName,
          new MockHelixManagerFactory(testCluster, null, null, useAggregatedView), metricRegistry);
      fail("Initialization should fail due to inconsistent replica capacity");
    } catch (IOException e) {
      // expected
    }
  }
Test the case where replicas from same partition have different capacities (which should block the startup)
@throws Exception
ambry-clustermap/src/test/java/com/github/ambry/clustermap/HelixClusterManagerTest.java