@Test
  public void getReplicaIdsByStateWithDuplicateResourceNameTest() throws Exception {
    assumeTrue(useAggregatedView);
    // create a HelixClusterManager with DynamicClusterChangeHandler
    Properties properties = new Properties();
    properties.putAll(props);
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(new VerifiableProperties(properties));
    HelixClusterManager helixClusterManager =
        new HelixClusterManager(clusterMapConfig, selfInstanceName, helixManagerFactory, new MetricRegistry());

    List<String> resourceNames = helixCluster.getResources(localDc);
    // there should be only ont resource name, which is 0
    String oldResourceName = resourceNames.get(0);

    // Test case 1: only old resource exists in both ideal state and the external view
    IdealState oldIdealState = helixCluster.getResourceIdealState(oldResourceName, localDc);
    for (String partitionName : oldIdealState.getPartitionSet()) {
      List<AmbryReplica> replicas = helixClusterManager.getManagerQueryHelper()
          .getReplicaIdsByState(new AmbryPartition(Long.parseLong(partitionName), null, null), ReplicaState.STANDBY,
              localDc);
      assertFalse(replicas.isEmpty());
      assertEquals(0, helixClusterManager.helixClusterManagerMetrics.resourceNameMismatchCount.getCount());
    }

    // Test case 2: create new resource in ideal state, but not external view
    String newResourceName = String.valueOf(Integer.valueOf(oldResourceName) + 10000);

    IdealState newIdealState = new IdealState(newResourceName);
    Map<String, List<String>> newPartitions = new HashMap<>();
    for (Map.Entry<String, List<String>> entry : oldIdealState.getPreferenceLists().entrySet()) {
      newPartitions.put(entry.getKey(), new ArrayList<>(entry.getValue()));
    }
    newIdealState.setPreferenceLists(newPartitions);
    // this will only add resource to ideal state
    helixCluster.addNewResource(newResourceName, newIdealState, localDc);

    // Local dc's partition's resource should be updated to newResource name
    // But the aggregated external view didn't get updated, so the replicas are still in the old resources
    Map<String, Set<String>> partitionToResourceAfterUpdate =
        (helixClusterManager).getPartitionToResourceMapByDC().get(localDc);
    long mismatchCount = 0;
    int iteration = 0;
    for (String partitionName : newPartitions.keySet()) {
      assertTrue(partitionToResourceAfterUpdate.get(partitionName).contains(newResourceName));
      assertFalse(partitionToResourceAfterUpdate.get(partitionName).contains(oldResourceName));
      // even number, choose standby, odd number, choose error.
      // error state should return empty from both resource.
      ReplicaState state = iteration % 2 == 0 ? ReplicaState.STANDBY : ReplicaState.ERROR;
      List<AmbryReplica> replicas = helixClusterManager.getManagerQueryHelper()
          .getReplicaIdsByState(new AmbryPartition(Long.parseLong(partitionName), null, null), state, localDc);
      if (state == ReplicaState.ERROR) {
        assertTrue(replicas.isEmpty());
      } else {
        assertFalse(replicas.isEmpty());
      }
      assertEquals(mismatchCount + 1,
          helixClusterManager.helixClusterManagerMetrics.resourceNameMismatchCount.getCount());
      mismatchCount = helixClusterManager.helixClusterManagerMetrics.resourceNameMismatchCount.getCount();
      iteration++;
    }

    // Test case 3: new resource are both in ideal state and external view
    MockHelixAdmin localMockHelixAdmin = helixCluster.getHelixAdminFromDc(localDc);
    // Use local mock helix admin to add this resource again so that external view would be updated
    localMockHelixAdmin.addResource(clusterNameStatic, newResourceName, newIdealState);
    localMockHelixAdmin.triggerRoutingTableNotification();
    // Sleep 1 second so the routing table notification can be received.
    Thread.sleep(1000);

    for (String partitionName : newPartitions.keySet()) {
      List<AmbryReplica> replicas = helixClusterManager.getManagerQueryHelper()
          .getReplicaIdsByState(new AmbryPartition(Long.parseLong(partitionName), null, null), ReplicaState.STANDBY,
              localDc);
      assertFalse(replicas.isEmpty());
      assertEquals(mismatchCount, helixClusterManager.helixClusterManagerMetrics.resourceNameMismatchCount.getCount());
    }

    // Test case 4: remove the old resource
    helixCluster.removeResourceIdealState(oldResourceName, localDc);
    localMockHelixAdmin.triggerRoutingTableNotification();
    // Sleep 1 second so the routing table notification can be received.
    Thread.sleep(1000);

    for (String partitionName : newPartitions.keySet()) {
      List<AmbryReplica> replicas = helixClusterManager.getManagerQueryHelper()
          .getReplicaIdsByState(new AmbryPartition(Long.parseLong(partitionName), null, null), ReplicaState.STANDBY,
              localDc);
      assertFalse(replicas.isEmpty());
      assertEquals(mismatchCount, helixClusterManager.helixClusterManagerMetrics.resourceNameMismatchCount.getCount());
    }

    // Test 5: the other datacenter going through the same process
    IdealState oldIdealStateInRemote = helixCluster.getResourceIdealState(oldResourceName, remoteDc);
    IdealState newIdealStateInRemote = new IdealState(newResourceName);
    Map<String, List<String>> newPartitionsInRemote = new HashMap<>();
    for (Map.Entry<String, List<String>> entry : oldIdealStateInRemote.getPreferenceLists().entrySet()) {
      newPartitionsInRemote.put(entry.getKey(), new ArrayList<>(entry.getValue()));
    }
    newIdealStateInRemote.setPreferenceLists(newPartitionsInRemote);
    // this will only add resource to ideal state
    helixCluster.addNewResource(newResourceName, newIdealStateInRemote, remoteDc);
    Map<String, Set<String>> partitionToResourceAfterUpdateInRemote =
        (helixClusterManager).getPartitionToResourceMapByDC().get(remoteDc);
    iteration = 0;
    for (String partitionName : newPartitionsInRemote.keySet()) {
      assertTrue(partitionToResourceAfterUpdateInRemote.get(partitionName).contains(newResourceName));
      assertFalse(partitionToResourceAfterUpdateInRemote.get(partitionName).contains(oldResourceName));
      ReplicaState state = iteration % 2 == 0 ? ReplicaState.STANDBY : ReplicaState.ERROR;
      List<AmbryReplica> replicas = helixClusterManager.getManagerQueryHelper()
          .getReplicaIdsByState(new AmbryPartition(Long.parseLong(partitionName), null, null), state, remoteDc);
      if (state == ReplicaState.ERROR) {
        assertTrue(replicas.isEmpty());
      } else {
        assertFalse(replicas.isEmpty());
      }
      assertEquals(mismatchCount + 1,
          helixClusterManager.helixClusterManagerMetrics.resourceNameMismatchCount.getCount());
      mismatchCount = helixClusterManager.helixClusterManagerMetrics.resourceNameMismatchCount.getCount();
    }

    MockHelixAdmin remoteMockHelixAdmin = helixCluster.getHelixAdminFromDc(remoteDc);
    remoteMockHelixAdmin.addResource(clusterNameStatic, newResourceName, newIdealStateInRemote);
    // Trigger routing table notification from local helix admin
    localMockHelixAdmin.triggerRoutingTableNotification();
    // Sleep 1 second so the routing table notification can be received.
    Thread.sleep(1000);
    for (String partitionName : newPartitionsInRemote.keySet()) {
      List<AmbryReplica> replicas = helixClusterManager.getManagerQueryHelper()
          .getReplicaIdsByState(new AmbryPartition(Long.parseLong(partitionName), null, null), ReplicaState.STANDBY,
              remoteDc);
      assertFalse(replicas.isEmpty());
      assertEquals(mismatchCount, helixClusterManager.helixClusterManagerMetrics.resourceNameMismatchCount.getCount());
    }

    helixClusterManager.close();
  }
Test when updates of IDEALSTATE haven't reached to aggregated external view.
@throws Exception
ambry-clustermap/src/test/java/com/github/ambry/clustermap/ClusterChangeHandlerTest.java