@Test
  public void testAddNewPartitionsAndDataNodesBeingFullAutoCompatible() throws Exception {
    assumeTrue(fullAutoCompatible);
    Map<String, Map<String, Set<String>>> instancesUnderResourcesInDc = new HashMap<>();
    Map<String, Map<String, Set<Integer>>> partitionsUnderResourcesInDc = new HashMap<>();
    int basePort = 18088;
    int basePartitionId = 0;
    int dataNode = 6;
    int numPartition = 200;
    testHardwareLayout = constructInitialHardwareLayoutJSON(CLUSTER_NAME_IN_STATIC_CLUSTER_MAP, dataNode, basePort);
    testPartitionLayout = constructInitialPartitionLayoutJSON(testHardwareLayout, numPartition, null);
    testHardwareLayout.getHardwareLayout().getDatacenters().forEach(dc -> {
      instancesUnderResourcesInDc.put(dc.getName(), new HashMap<>());
      partitionsUnderResourcesInDc.put(dc.getName(), new HashMap<>());
    });

    // Test 1: Add initial partition to helix
    long expectedResourceCount = 1;
    writeBootstrapOrUpgrade(expectedResourceCount, false, dataNode);
    addExpectedPartitionAndInstanceToResource(testPartitionLayout, FULL_AUTO_COMPATIBLE_RESOURCE_NAME_START_NUMBER,
        instancesUnderResourcesInDc, partitionsUnderResourcesInDc);
    verifyResourceInFullAuto(testHardwareLayout.getHardwareLayout(), instancesUnderResourcesInDc,
        partitionsUnderResourcesInDc);
    verifyDataNodeConfigs();
    // Use a smaller number to be max instances in a resource, it will fail
    try {
      writeBootstrapOrUpgrade(expectedResourceCount, false, 1);
      fail("The number or hosts in a resource is larger than 1");
    } catch (Exception e) {
    }

    // Test 2: Add a new group of partitions and instances.
    basePort += dataNode * 2; // it has two datacenter
    basePartitionId += numPartition;
    TestHardwareLayout secondTestHardwareLayout =
        constructInitialHardwareLayoutJSON(CLUSTER_NAME_IN_STATIC_CLUSTER_MAP, dataNode, basePort);
    TestPartitionLayout secondTestPartitionLayout =
        constructInitialPartitionLayoutJSON(secondTestHardwareLayout, numPartition, null, basePartitionId);

    // This two layouts have different set of datanodes, and different set of partitions. Merge this two layouts, and
    // we should create two different resources for these two layouts
    testPartitionLayout.merge(secondTestPartitionLayout);
    expectedResourceCount = 2;
    writeBootstrapOrUpgrade(expectedResourceCount, false, dataNode);
    addExpectedPartitionAndInstanceToResource(secondTestPartitionLayout,
        FULL_AUTO_COMPATIBLE_RESOURCE_NAME_START_NUMBER + 1, instancesUnderResourcesInDc, partitionsUnderResourcesInDc);
    verifyResourceInFullAuto(testHardwareLayout.getHardwareLayout(), instancesUnderResourcesInDc,
        partitionsUnderResourcesInDc);
    verifyDataNodeConfigs();

    // Test 3: Create another set of data nodes and partitions, but this time, only half the size as last time.
    basePort += dataNode * 2; // it has two datacenter
    basePartitionId += numPartition;
    TestHardwareLayout thirdTestHardwareLayout =
        constructInitialHardwareLayoutJSON(CLUSTER_NAME_IN_STATIC_CLUSTER_MAP, dataNode / 2, basePort);
    TestPartitionLayout thirdTestPartitionLayout =
        constructInitialPartitionLayoutJSON(thirdTestHardwareLayout, numPartition / 2, null, basePartitionId);

    testPartitionLayout.merge(thirdTestPartitionLayout);
    expectedResourceCount = 3;
    writeBootstrapOrUpgrade(expectedResourceCount, false, dataNode);
    addExpectedPartitionAndInstanceToResource(thirdTestPartitionLayout,
        FULL_AUTO_COMPATIBLE_RESOURCE_NAME_START_NUMBER + 2, instancesUnderResourcesInDc, partitionsUnderResourcesInDc);
    verifyResourceInFullAuto(testHardwareLayout.getHardwareLayout(), instancesUnderResourcesInDc,
        partitionsUnderResourcesInDc);
    verifyDataNodeConfigs();

    // Test 4: Create another set of data nodes with only the half size, and add a partition that has replicas from third
    // and fourth layout
    basePort += dataNode; // it has two datacenter
    basePartitionId += numPartition / 2;
    TestHardwareLayout fourthTestHardwareLayout =
        constructInitialHardwareLayoutJSON(CLUSTER_NAME_IN_STATIC_CLUSTER_MAP, dataNode / 2, basePort);
    TestPartitionLayout fourthTestPartitionLayout =
        constructInitialPartitionLayoutJSON(fourthTestHardwareLayout, numPartition / 2, null, basePartitionId);
    testPartitionLayout.merge(fourthTestPartitionLayout);
    List<DataNode> dataNodesForNewPartition = thirdTestHardwareLayout.getIndependentDataNodes(1);
    dataNodesForNewPartition.addAll(fourthTestHardwareLayout.getIndependentDataNodes(2));
    testPartitionLayout.addNewPartition(testHardwareLayout, dataNodesForNewPartition, DEFAULT_PARTITION_CLASS);
    expectedResourceCount = 3;
    writeBootstrapOrUpgrade(expectedResourceCount, false, dataNode);
    addExpectedPartitionAndInstanceToResource(fourthTestPartitionLayout,
        FULL_AUTO_COMPATIBLE_RESOURCE_NAME_START_NUMBER + 2, instancesUnderResourcesInDc, partitionsUnderResourcesInDc);
    for (Datacenter dc : testHardwareLayout.getHardwareLayout().getDatacenters()) {
      String resourceName = String.valueOf(FULL_AUTO_COMPATIBLE_RESOURCE_NAME_START_NUMBER + 2);
      partitionsUnderResourcesInDc.get(dc.getName()).get(resourceName).add(testPartitionLayout.getPartitionCount() - 1);
    }
    verifyResourceInFullAuto(testHardwareLayout.getHardwareLayout(), instancesUnderResourcesInDc,
        partitionsUnderResourcesInDc);
    verifyDataNodeConfigs();

    // Test 5: Adding a new partition from fourth layout
    basePartitionId += numPartition / 2 + 1;
    List<DataNode> finalDataNodes = new ArrayList<>(fourthTestHardwareLayout.getIndependentDataNodes(3));

    testPartitionLayout.addNewPartition(testHardwareLayout, finalDataNodes, DEFAULT_PARTITION_CLASS);
    writeBootstrapOrUpgrade(expectedResourceCount, false, dataNode);
    for (Datacenter dc : testHardwareLayout.getHardwareLayout().getDatacenters()) {
      String resourceName = String.valueOf(FULL_AUTO_COMPATIBLE_RESOURCE_NAME_START_NUMBER + 2);
      partitionsUnderResourcesInDc.get(dc.getName()).get(resourceName).add(testPartitionLayout.getPartitionCount() - 1);
    }
    verifyResourceInFullAuto(testHardwareLayout.getHardwareLayout(), instancesUnderResourcesInDc,
        partitionsUnderResourcesInDc);
    verifyDataNodeConfigs();

    // Test 6: Add several groups together
    basePort += dataNode;
    basePartitionId += 1;
    int numGroups = 4;
    for (int i = 0; i < numGroups; i++) {
      int numNewDataNode = i == numGroups - 1 ? dataNode / 2 : dataNode;
      int numNewPartition = i == numGroups - 1 ? numPartition / 2 : numPartition;
      TestHardwareLayout newHardwareLayout =
          constructInitialHardwareLayoutJSON(CLUSTER_NAME_IN_STATIC_CLUSTER_MAP, numNewDataNode, basePort);
      TestPartitionLayout newPartitionLayout =
          constructInitialPartitionLayoutJSON(newHardwareLayout, numNewPartition, null, basePartitionId);
      testPartitionLayout.merge(newPartitionLayout);
      addExpectedPartitionAndInstanceToResource(newPartitionLayout,
          FULL_AUTO_COMPATIBLE_RESOURCE_NAME_START_NUMBER + 3 + i, instancesUnderResourcesInDc,
          partitionsUnderResourcesInDc);
      basePort += numNewDataNode * 2;
      basePartitionId += numNewPartition;
    }
    expectedResourceCount += 4;
    writeBootstrapOrUpgrade(expectedResourceCount, false, dataNode);
    verifyResourceInFullAuto(testHardwareLayout.getHardwareLayout(), instancesUnderResourcesInDc,
        partitionsUnderResourcesInDc);
    verifyDataNodeConfigs();

    // Test 7: Replace a data node with new one
    String dcName = activeDcSet.iterator().next();
    DataNode randomDataNode = testHardwareLayout.getRandomDataNodeFromDc(dcName);
    DataNode newDataNode = testPartitionLayout.replaceDataNodeWithNewOne(randomDataNode);
    System.out.println(
        "Replace data node " + randomDataNode.getPort() + " with new data node " + newDataNode.getPort());
    writeBootstrapOrUpgrade(expectedResourceCount, true, dataNode);
    // update instancesUnderResourcesInDc
    Map<String, Set<String>> instancesUnderResource = instancesUnderResourcesInDc.get(dcName);
    for (Map.Entry<String, Set<String>> entry : instancesUnderResource.entrySet()) {
      if (entry.getValue().contains(ClusterMapUtils.getInstanceName(randomDataNode))) {
        entry.getValue().remove(ClusterMapUtils.getInstanceName(randomDataNode));
        entry.getValue().add(ClusterMapUtils.getInstanceName(newDataNode));
        break;
      }
    }
    verifyResourceInFullAuto(testHardwareLayout.getHardwareLayout(), instancesUnderResourcesInDc,
        partitionsUnderResourcesInDc);
    verifyDataNodeConfigs();
  }
Test adding partitions and instances in FULL_AUTO compatible way.
@throws Exception
ambry-tools/src/integration-test/java/com/github/ambry/clustermap/HelixBootstrapUpgradeToolTest.java