private void recoverSingleKey(LocalRegion r, final @NotNull Object keys,
      InterestResultPolicy policy,
      final @NotNull InterestType interestType, Connection recoveredConnection, boolean isDurable,
      boolean receiveValues,
      boolean isFirstNewConnection) {
    r.startRegisterInterest();
    try {
      // Remove all matching values from local cache
      if (isFirstNewConnection) { // only if this recoveredEP
        // becomes primaryEndpoint
        r.clearKeysOfInterest(keys, interestType, policy);
        if (logger.isDebugEnabled()) {
          logger.debug(
              "{}.recoverSingleRegion :Endpoint recovered is primary so clearing the keys of interest starting kind={} region={}: {}",
              this, InterestType.getString(interestType), r.getFullPath(), keys);
        }
      }
      // Register interest, get new values back
      List<Object> serverKeys;
      if (policy != InterestResultPolicy.KEYS_VALUES) {
        serverKeys = r.getServerProxy().registerInterestOn(recoveredConnection, keys, interestType,
            policy, isDurable, !receiveValues, r.getAttributes().getDataPolicy());
        // Restore keys based on server's response
        if (isFirstNewConnection) {
          // only if this recoveredEP becomes primaryEndpoint
          r.refreshEntriesFromServerKeys(recoveredConnection, serverKeys, policy);
        }
      } else {
        if (!isFirstNewConnection) {
          // InterestResultPolicy.KEYS_VALUES now fetches values in
          // RegisterInterestOp's response itself and in this case
          // refreshEntriesFromServerKeys(...) does not explicitly fetch values
          // but only updates keys-values to region. To not fetch values, we
          // need to use policy NONE or KEYS.
          r.getServerProxy().registerInterestOn(recoveredConnection, keys,
              interestType, InterestResultPolicy.NONE, isDurable, !receiveValues,
              r.getAttributes().getDataPolicy());
        } else {
          serverKeys =
              r.getServerProxy().registerInterestOn(recoveredConnection, keys, interestType, policy,
                  isDurable, !receiveValues, r.getAttributes().getDataPolicy());
          r.refreshEntriesFromServerKeys(recoveredConnection, serverKeys, policy);
        }
      }
    } finally {
      r.finishRegisterInterest();
    }
  }
Find a new primary, adding any failed servers we encounter to the excluded servers list

First we try to make a backup server the primary, but if run out of backup servers we will try
to find a new server.
geode-core/src/main/java/org/apache/geode/cache/client/internal/QueueManagerImpl.java