@Test
  public void testIllegalPropertyCombos() throws Exception {
    int port = getRandomAvailableTCPPort();

    Properties props = new Properties();
    props.setProperty(MCAST_PORT, "26753");
    props.setProperty(ConfigurationProperties.LOCATORS, getIPLiteral() + "[" + port + "]");
    props.setProperty(ConfigurationProperties.SECURITY_PEER_AUTH_INIT,
        UserPasswordAuthInit.class.getName() + ".create");
    props.setProperty(ConfigurationProperties.ENABLE_CLUSTER_CONFIGURATION, "false");

    try {
      Locator.startLocatorAndDS(port, null, null, props);
      fail("Expected an IllegalArgumentException while starting locator");

    } catch (IllegalArgumentException ex) {
      // success
    }

    // Also try setting the authenticator
    props = new Properties();
    props.setProperty(MCAST_PORT, "26753");
    props.setProperty(LOCATORS, getIPLiteral() + "[" + port + "]");
    props.setProperty(SECURITY_PEER_AUTHENTICATOR,
        LdapUserAuthenticator.class.getName() + ".create");
    props.setProperty(ENABLE_CLUSTER_CONFIGURATION, "false");

    try {
      Locator.startLocatorAndDS(port, null, null, props);
      fail("Expected an IllegalArgumentException while starting locator");

    } catch (IllegalArgumentException expected) {
      // success
    }

    props = new Properties();
    props.setProperty(MCAST_PORT, "26753");
    props.setProperty(SECURITY_PEER_AUTH_INIT, UserPasswordAuthInit.class.getName() + ".create");

    try {
      getSystem(props);
      fail("Expected an IllegalArgumentException while connection to DS");

    } catch (IllegalArgumentException expected) {
      // success
    }

    // Also try setting the authenticator
    props = new Properties();
    props.setProperty(MCAST_PORT, "26753");
    props.setProperty(SECURITY_PEER_AUTHENTICATOR,
        LdapUserAuthenticator.class.getName() + ".create");

    try {
      getSystem(props);
      fail("Expected an IllegalArgumentException while connection to DS");

    } catch (IllegalArgumentException expected) {
      // success
    }
  }
Check that mcast-port setting for discovery or with locator are incompatible with security
geode-core/src/distributedTest/java/org/apache/geode/security/P2PAuthenticationDUnitTest.java