private void testEnsureHeadlessIdentities(boolean createAmbariIdentities, boolean ambariServerPrincipalAsService) throws Exception {
    String clusterName = "c1";
    String realm = "EXAMPLE.COM";
    String ambariServerHostname = StageUtils.getHostName();
    String ambariServerPrincipalName;
    String ambariServerKeytabFilePath = new File("ambari.server.keytab").getAbsolutePath();
    KerberosPrincipalType ambariServerPrincipalType;
    String ambariServerPrincipalNameExpected;

    if (ambariServerPrincipalAsService) {
      ambariServerPrincipalName = "ambari-server${principal_suffix}/_HOST@${realm}";
      ambariServerPrincipalType = KerberosPrincipalType.SERVICE;
      ambariServerPrincipalNameExpected = String.format("ambari-server-%s/%s@%s", clusterName, ambariServerHostname, realm);
    } else {
      ambariServerPrincipalName = "ambari-server${principal_suffix}@${realm}";
      ambariServerPrincipalType = KerberosPrincipalType.USER;
      ambariServerPrincipalNameExpected = String.format("ambari-server-%s@%s", clusterName, realm);
    }

    Map<String, String> propertiesKrb5Conf = new HashMap<>();

    Map<String, String> propertiesKerberosEnv = new HashMap<>();
    propertiesKerberosEnv.put(KerberosHelper.DEFAULT_REALM, realm);
    propertiesKerberosEnv.put(KerberosHelper.KDC_TYPE, "mit-kdc");
    propertiesKerberosEnv.put("password_length", "20");
    propertiesKerberosEnv.put("password_min_lowercase_letters", "1");
    propertiesKerberosEnv.put("password_min_uppercase_letters", "1");
    propertiesKerberosEnv.put("password_min_digits", "1");
    propertiesKerberosEnv.put("password_min_punctuation", "0");
    propertiesKerberosEnv.put("password_min_whitespace", "0");
    propertiesKerberosEnv.put(KerberosHelper.CREATE_AMBARI_PRINCIPAL, (createAmbariIdentities) ? "true" : "false");

    Config configKrb5Conf = createMock(Config.class);
    expect(configKrb5Conf.getProperties()).andReturn(propertiesKrb5Conf).times(1);

    Config configKerberosEnv = createMock(Config.class);
    expect(configKerberosEnv.getProperties()).andReturn(propertiesKerberosEnv).times(1);

    Host host1 = createMockHost("host1");
    Host host2 = createMockHost("host3");
    Host host3 = createMockHost("host2");

    Map<String, ServiceComponentHost> service1Component1HostMap = new HashMap<>();
    service1Component1HostMap.put("host1", createMockServiceComponentHost(State.INSTALLED));

    Map<String, ServiceComponentHost> service2Component1HostMap = new HashMap<>();
    service2Component1HostMap.put("host2", createMockServiceComponentHost(State.INSTALLED));

    Map<String, ServiceComponent> service1ComponentMap = new HashMap<>();
    service1ComponentMap.put("COMPONENT11", createMockComponent("COMPONENT11", true, service1Component1HostMap));

    Map<String, ServiceComponent> service2ComponentMap = new HashMap<>();
    service2ComponentMap.put("COMPONENT21", createMockComponent("COMPONENT21", true, service2Component1HostMap));

    Service service1 = createMockService("SERVICE1", service1ComponentMap);
    Service service2 = createMockService("SERVICE2", service2ComponentMap);

    Map<String, Service> servicesMap = new HashMap<>();
    servicesMap.put("SERVICE1", service1);
    servicesMap.put("SERVICE2", service2);

    Cluster cluster = createMockCluster(clusterName, Arrays.asList(host1, host2, host3), SecurityType.KERBEROS, configKrb5Conf, configKerberosEnv);
    expect(cluster.getServices()).andReturn(servicesMap).anyTimes();
    expect(cluster.getDesiredStackVersion()).andReturn(new StackId("HDP-2.2")).anyTimes();

    Map<String, String> kerberosDescriptorProperties = new HashMap<>();
    kerberosDescriptorProperties.put("additional_realms", "");
    kerberosDescriptorProperties.put("keytab_dir", "/etc/security/keytabs");
    kerberosDescriptorProperties.put(KerberosHelper.DEFAULT_REALM, "${kerberos-env/realm}");
    kerberosDescriptorProperties.put("principal_suffix", "-${cluster_name|toLower()}");

    ArrayList<KerberosIdentityDescriptor> service1Component1Identities = new ArrayList<>();
    service1Component1Identities.add(createMockIdentityDescriptor(
        "s1c1_1.user",
        createMockPrincipalDescriptor("s1c1_1@${realm}", KerberosPrincipalType.USER, "s1c1", null),
        createMockKeytabDescriptor("s1c1_1.user.keytab", null)
    ));
    service1Component1Identities.add(createMockIdentityDescriptor(
        "s1c1_1.service",
        createMockPrincipalDescriptor("s1c1_1/_HOST@${realm}", KerberosPrincipalType.SERVICE, "s1c1", null),
        createMockKeytabDescriptor("s1c1_1.service.keytab", null)
    ));

    HashMap<String, KerberosComponentDescriptor> service1ComponentDescriptorMap = new HashMap<>();
    service1ComponentDescriptorMap.put("COMPONENT11", createMockComponentDescriptor("COMPONENT11", service1Component1Identities, null));

    List<KerberosIdentityDescriptor> service1Identities = new ArrayList<>();
    service1Identities.add(createMockIdentityDescriptor(
        "s1_1.user",
        createMockPrincipalDescriptor("s1_1@${realm}", KerberosPrincipalType.USER, "s1", null),
        createMockKeytabDescriptor("s1_1.user.keytab", null)
    ));
    service1Identities.add(createMockIdentityDescriptor(
        "s1_1.service",
        createMockPrincipalDescriptor("s1/_HOST@${realm}", KerberosPrincipalType.SERVICE, "s1", null),
        createMockKeytabDescriptor("s1.service.keytab", null)
    ));

    KerberosServiceDescriptor service1KerberosDescriptor = createMockServiceDescriptor("SERVICE1", service1ComponentDescriptorMap, service1Identities, false);

    ArrayList<KerberosIdentityDescriptor> service2Component1Identities = new ArrayList<>();
    service2Component1Identities.add(createMockIdentityDescriptor(
        "s2_1.user",
        createMockPrincipalDescriptor("s2_1@${realm}", KerberosPrincipalType.USER, "s2", null),
        createMockKeytabDescriptor("s2_1.user.keytab", null)
    ));
    service2Component1Identities.add(createMockIdentityDescriptor(
        "s2c1_1.service",
        createMockPrincipalDescriptor("s2c1_1/_HOST@${realm}", KerberosPrincipalType.SERVICE, "s2c1", null),
        createMockKeytabDescriptor("s2c1_1.service.keytab", null)
    ));

    HashMap<String, KerberosComponentDescriptor> service2ComponentDescriptorMap = new HashMap<>();
    service2ComponentDescriptorMap.put("COMPONENT21", createMockComponentDescriptor("COMPONENT21", service2Component1Identities, null));

    KerberosServiceDescriptor service2KerberosDescriptor = createMockServiceDescriptor("SERVICE2", service2ComponentDescriptorMap, null, false);

    KerberosDescriptor kerberosDescriptor = createMock(KerberosDescriptor.class);
    expect(kerberosDescriptor.getProperties()).andReturn(kerberosDescriptorProperties);
    expect(kerberosDescriptor.getService("SERVICE1")).andReturn(service1KerberosDescriptor).times(1);
    expect(kerberosDescriptor.getService("SERVICE2")).andReturn(service2KerberosDescriptor).times(1);

    Capture<ResolvedKerberosPrincipal> spnegoPrincipalCapture = newCapture(CaptureType.LAST);
    Capture<ResolvedKerberosPrincipal> ambariPrincipalCapture = newCapture(CaptureType.LAST);
    String spnegoPrincipalNameExpected = String.format("HTTP/%s@%s", ambariServerHostname, realm);
    if (createAmbariIdentities) {
      ArrayList<KerberosIdentityDescriptor> ambarServerComponent1Identities = new ArrayList<>();
      ambarServerComponent1Identities.add(createMockIdentityDescriptor(
          KerberosHelper.AMBARI_SERVER_KERBEROS_IDENTITY_NAME,
          createMockPrincipalDescriptor(ambariServerPrincipalName, ambariServerPrincipalType, "ambari", null),
          createMockKeytabDescriptor(ambariServerKeytabFilePath, null)));

      ambarServerComponent1Identities.add(createMockIdentityDescriptor(
          "spnego",
          createMockPrincipalDescriptor("HTTP/_HOST@${realm}", KerberosPrincipalType.SERVICE, null, null),
          createMockKeytabDescriptor("spnego.service.keytab", null)));

      KerberosComponentDescriptor ambariServerComponentKerberosDescriptor = createMockComponentDescriptor(RootComponent.AMBARI_SERVER.name(), ambarServerComponent1Identities, null);

      HashMap<String, KerberosComponentDescriptor> ambariServerComponentDescriptorMap = new HashMap<>();
      ambariServerComponentDescriptorMap.put(RootComponent.AMBARI_SERVER.name(), ambariServerComponentKerberosDescriptor);

      KerberosServiceDescriptor ambariServiceKerberosDescriptor = createMockServiceDescriptor(RootService.AMBARI.name(), ambariServerComponentDescriptorMap, null, false);
      expect(ambariServiceKerberosDescriptor.getComponent(RootComponent.AMBARI_SERVER.name())).andReturn(ambariServerComponentKerberosDescriptor).once();

      expect(kerberosDescriptor.getService(RootService.AMBARI.name())).andReturn(ambariServiceKerberosDescriptor).once();

      ConfigureAmbariIdentitiesServerAction configureAmbariIdentitiesServerAction = injector.getInstance(ConfigureAmbariIdentitiesServerAction.class);

      expect(configureAmbariIdentitiesServerAction.installAmbariServerIdentity(capture(ambariPrincipalCapture), anyString(), eq(ambariServerKeytabFilePath),
          eq("user1"), eq("rw"), eq("groupA"), eq("r"), (ActionLog) eq(null)))
          .andReturn(true)
          .once();
      expect(configureAmbariIdentitiesServerAction.installAmbariServerIdentity(capture(spnegoPrincipalCapture), anyString(), eq("spnego.service.keytab"),
          eq("user1"), eq("rw"), eq("groupA"), eq("r"), (ActionLog) eq(null)))
          .andReturn(true)
          .once();

      configureAmbariIdentitiesServerAction.configureJAAS(ambariServerPrincipalNameExpected, ambariServerKeytabFilePath, null);
      expectLastCall().once();
    }

    setupKerberosDescriptor(kerberosDescriptor);

    Map<String, Map<String, String>> existingConfigurations = new HashMap<>();
    existingConfigurations.put("kerberos-env", propertiesKerberosEnv);

    Set<String> services = new HashSet<String>() {
      {
        add("SERVICE1");
        add("SERVICE2");
      }
    };

    Capture<? extends String> capturePrincipal = newCapture(CaptureType.ALL);
    Capture<? extends String> capturePrincipalForKeytab = newCapture(CaptureType.ALL);

    CreatePrincipalsServerAction createPrincipalsServerAction = injector.getInstance(CreatePrincipalsServerAction.class);
    expect(createPrincipalsServerAction.createPrincipal(capture(capturePrincipal), eq(false), EasyMock.anyObject(), anyObject(KerberosOperationHandler.class), eq(false), isNull(ActionLog.class)))
        .andReturn(new CreatePrincipalsServerAction.CreatePrincipalResult("anything", "password", 1))
        .times(3);

    if (createAmbariIdentities) {
      if (ambariServerPrincipalAsService) {
        expect(createPrincipalsServerAction.createPrincipal(capture(capturePrincipal), eq(true), EasyMock.anyObject(), anyObject(KerberosOperationHandler.class), eq(false), isNull(ActionLog.class)))
            .andReturn(new CreatePrincipalsServerAction.CreatePrincipalResult("anything", "password", 1))
            .times(2);
      } else {
        expect(createPrincipalsServerAction.createPrincipal(capture(capturePrincipal), eq(true), EasyMock.anyObject(), anyObject(KerberosOperationHandler.class), eq(false), isNull(ActionLog.class)))
            .andReturn(new CreatePrincipalsServerAction.CreatePrincipalResult("anything", "password", 1))
            .times(1);
        expect(createPrincipalsServerAction.createPrincipal(capture(capturePrincipal), eq(false), EasyMock.anyObject(), anyObject(KerberosOperationHandler.class), eq(false), isNull(ActionLog.class)))
            .andReturn(new CreatePrincipalsServerAction.CreatePrincipalResult("anything", "password", 1))
            .times(1);
      }
    }

    CreateKeytabFilesServerAction createKeytabFilesServerAction = injector.getInstance(CreateKeytabFilesServerAction.class);
    expect(createKeytabFilesServerAction.createKeytab(capture(capturePrincipalForKeytab),anyObject(), eq("password"), eq(1), anyObject(KerberosOperationHandler.class), eq(true), eq(true), isNull(ActionLog.class)))
        .andReturn(new Keytab())
        .times(createAmbariIdentities ? 5 : 3);

    replayAll();

    AmbariMetaInfo ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class);
    ambariMetaInfo.init();

    CredentialStoreService credentialStoreService = injector.getInstance(CredentialStoreService.class);
    credentialStoreService.setCredential(cluster.getClusterName(), KerberosHelper.KDC_ADMINISTRATOR_CREDENTIAL_ALIAS,
        new PrincipalKeyCredential("principal", "password"), CredentialStoreType.TEMPORARY);

    KerberosHelper kerberosHelper = injector.getInstance(KerberosHelper.class);
    kerberosHelper.ensureHeadlessIdentities(cluster, existingConfigurations, services);

    verifyAll();

    if (createAmbariIdentities) {
      assertEquals(ambariPrincipalCapture.getValue().getPrincipal(), ambariServerPrincipalNameExpected);
      assertEquals(spnegoPrincipalCapture.getValue().getPrincipal(), spnegoPrincipalNameExpected);
    }

    List<? extends String> capturedPrincipals = capturePrincipal.getValues();
    assertEquals(createAmbariIdentities ? 5 : 3, capturedPrincipals.size());
    assertTrue(capturedPrincipals.contains("s1_1@EXAMPLE.COM"));
    assertTrue(capturedPrincipals.contains("s1c1_1@EXAMPLE.COM"));
    assertTrue(capturedPrincipals.contains("s2_1@EXAMPLE.COM"));

    List<? extends String> capturedPrincipalsForKeytab = capturePrincipalForKeytab.getValues();
    assertEquals(createAmbariIdentities ? 5 : 3, capturedPrincipalsForKeytab.size());
    assertTrue(capturedPrincipalsForKeytab.contains("s1_1@EXAMPLE.COM"));
    assertTrue(capturedPrincipalsForKeytab.contains("s1c1_1@EXAMPLE.COM"));
    assertTrue(capturedPrincipalsForKeytab.contains("s2_1@EXAMPLE.COM"));

    if (createAmbariIdentities) {
      String spnegoPrincipalName = String.format("HTTP/%s@EXAMPLE.COM", ambariServerHostname);

      assertTrue(capturedPrincipals.contains(ambariServerPrincipalNameExpected));
      assertTrue(capturedPrincipals.contains(spnegoPrincipalName));

      assertTrue(capturedPrincipalsForKeytab.contains(ambariServerPrincipalNameExpected));
      assertTrue(capturedPrincipalsForKeytab.contains(spnegoPrincipalName));
    }
  }
Tests that when regenerating keytabs for an upgrade, that the retry allowed
boolean is set on the tasks created.

@throws Exception
ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java