@VisibleForTesting
  public void recoverPrimary(Set<ServerLocation> excludedServers) {
    if (pool.getPoolOrCacheCancelInProgress() != null) {
      return;
    }
    final boolean isDebugEnabled = logger.isDebugEnabled();
    if (!isPrimaryRecoveryNeeded(queueConnections)) {
      if (isDebugEnabled) {
        logger.debug("Primary recovery not needed");
      }
      return;
    }

    if (isDebugEnabled) {
      logger.debug(
          "SubscriptionManager redundancy satisfier - primary endpoint has been lost. Attempting to recover");
    }

    if (printRecoveringPrimary) {
      logger.info(
          "SubscriptionManager redundancy satisfier - primary endpoint has been lost. Attempting to recover.");
      printRecoveringPrimary = false;
    }

    QueueConnectionImpl newPrimary = null;
    while (newPrimary == null && pool.getPoolOrCacheCancelInProgress() == null) {
      List<Connection> backups = queueConnections.getBackups();
      newPrimary = promoteBackupToPrimary(backups);
      // Hitesh now lets say that server crashed
      if (newPrimary == null) {
        // could not find a backup to promote
        break;
      }
      if (!addToConnectionList(newPrimary, true)) {
        synchronized (lock) {
          // make sure we don't retry the same backup server
          queueConnections = queueConnections.removeConnection(newPrimary);
        }
        newPrimary = null;
      }
    }

    if (newPrimary != null) {
      if (isDebugEnabled) {
        logger.debug(
            "SubscriptionManager redundancy satisfier - Switched backup server to primary: {}",
            newPrimary.getEndpoint());
      }
      if (PoolImpl.AFTER_PRIMARY_RECOVERED_CALLBACK_FLAG) {
        ClientServerObserver bo = ClientServerObserverHolder.getInstance();
        bo.afterPrimaryRecovered(newPrimary.getServer());
      }

      // new primary from back up server was found, alert affected cqs if necessary
      cqsConnected();
      printPrimaryNotFoundError = true;
      printRecoveringPrimary = true;
      return;
    }

    while (newPrimary == null) {
      newPrimary = createNewPrimary(excludedServers);
      if (newPrimary == null) {
        // could not find a new primary to create
        break;
      }

      if (!addToConnectionList(newPrimary, true)) {
        excludedServers.add(newPrimary.getServer());
        newPrimary = null;
      }

      if (newPrimary != null) {
        if (isDebugEnabled) {
          logger.debug(
              "SubscriptionManager redundancy satisfier - Non backup server was made primary. Recovering interest {}",
              newPrimary.getEndpoint());
        }

        if (!recoverInterest(newPrimary, true)) {
          excludedServers.add(newPrimary.getServer());
          newPrimary = null;
        }

        markQueueAsReadyForEvents(newPrimary);

        // New primary queue was found from a non backup, alert the affected cqs
        cqsConnected();
      }

      if (newPrimary != null && PoolImpl.AFTER_PRIMARY_RECOVERED_CALLBACK_FLAG) {
        ClientServerObserver bo = ClientServerObserverHolder.getInstance();
        bo.afterPrimaryRecovered(newPrimary.getServer());
      }
      printPrimaryNotFoundError = true;
      printRecoveringPrimary = true;
      return;
    }
    // No primary queue was found, alert the affected cqs
    cqsDisconnected();
    if (isDebugEnabled) {
      logger.debug("SubscriptionManager redundancy satisfier - Could not recover a new primary");
    }
    synchronized (lock) {
      queueConnections = queueConnections.setPrimaryDiscoveryFailed(null);
      lock.notifyAll();
    }
  }
Find a new primary, adding any failed servers we encounter to the excluded servers list

First we try to make a backup server the primary, but if run out of backup servers we will try
to find a new server.
geode-core/src/main/java/org/apache/geode/cache/client/internal/QueueManagerImpl.java