protected void checkRemoveFromClusterNodeLabelsOfQueue(
      Collection<String> labelsToRemove) throws IOException {
    // Check if label to remove doesn't existed or null/empty, will throw
    // exception if any of labels to remove doesn't meet requirement
    for (String label : labelsToRemove) {
      label = normalizeLabel(label);

      // check if any queue contains this label
      for (Entry<String, Queue> entry : queueCollections.entrySet()) {
        String queueName = entry.getKey();
        Set<String> queueLabels = entry.getValue().accessibleNodeLabels;
        if (queueLabels.contains(label)) {
          throw new IOException("Cannot remove label=" + label
              + ", because queue=" + queueName + " is using this label. "
              + "Please remove label on queue before remove the label");
        }
      }
    }
  }
@Test(timeout = 60000)
  public void testcheckRemoveFromClusterNodeLabelsOfQueue() throws Exception {
    lmgr = new RMNodeLabelsManager();
    Configuration conf = new Configuration();
    File tempDir = File.createTempFile("nlb", ".tmp");
    tempDir.delete();
    tempDir.mkdirs();
    tempDir.deleteOnExit();
    conf.set(YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR,
        tempDir.getAbsolutePath());
    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
    conf.set(YarnConfiguration.RM_SCHEDULER,
        "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler");
    Configuration withQueueLabels = getConfigurationWithQueueLabels(conf);
    MockRM rm = initRM(conf);
    lmgr.addToCluserNodeLabels(toSet(NodeLabel.newInstance("x", false)));
    lmgr.removeFromClusterNodeLabels(Arrays.asList(new String[] { "x" }));
    lmgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("x"));
    rm.stop();
    class TestRMLabelManger extends RMNodeLabelsManager {
      @Override
      protected void checkRemoveFromClusterNodeLabelsOfQueue(
          Collection<String> labelsToRemove) throws IOException {
        checkQueueCall = true;
        // Do nothing
      }
    }
    lmgr = new TestRMLabelManger();
    MockRM rm2 = initRM(withQueueLabels);
    Assert.assertFalse(
        "checkRemoveFromClusterNodeLabelsOfQueue should not be called"
            + "on recovery",
        checkQueueCall);
    lmgr.removeFromClusterNodeLabels(Arrays.asList(new String[] { "x" }));
    Assert
        .assertTrue("checkRemoveFromClusterNodeLabelsOfQueue should be called "
            + "since its not recovery", checkQueueCall);
    rm2.stop();
  }