@Test
  public void testConcurrentLoad() throws Throwable {
    final String name = getUniqueName() + "Region";
    final String objectName = "theKey";
    final Integer value = 44;
    final String exceptionString = "causing first cache-load to fail";

    remoteLoaderInvoked = false;
    loaderInvoked = false;

    vm0.invoke("create region " + name + " in vm0", new CacheSerializableRunnable() {
      @Override
      public void run2() {
        remoteLoaderInvoked = false;
        loaderInvoked = false;
        RegionFactory<Object, Object> factory = getCache().createRegionFactory();
        factory.setScope(Scope.DISTRIBUTED_ACK);
        factory.setConcurrencyChecksEnabled(true);
        factory.setCacheLoader(new CacheLoader<Object, Object>() {
          boolean firstInvocation = true;

          @Override
          public synchronized Object load(LoaderHelper helper) {
            System.out.println("invoked cache loader for " + helper.getKey());
            loaderInvoked = true;
            loaderInvokedLatch.countDown();
            if (firstInvocation) {
              firstInvocation = false;
              try {
                // wait for both threads to be ready for the exception to be thrown
                System.out.println("waiting for vm0t2 to be ready before throwing exception");
                readyForExceptionLatch.await(30, TimeUnit.SECONDS);
                // give the second thread time to get into loader code
                Thread.sleep(5000);
              } catch (InterruptedException e) {
                fail("interrupted");
              }
              System.out.println("throwing exception");
              throw new RuntimeException(exceptionString);
            }
            System.out.println("returning value=" + value);
            return value;
          }

          @Override
          public void close() {}
        });

        Region<Object, Object> region = createRegion(name, factory);
        region.create(objectName, null);
        IgnoredException.addIgnoredException(exceptionString);
      }
    });

    AsyncInvocation<Void> async1 = null;
    try {
      async1 = vm0.invokeAsync("Concurrently invoke the remote loader on the same key - t1",
          new CacheSerializableRunnable() {
            @Override
            public void run2() {
              Region<Object, Object> region = getCache().getRegion("root" + SEPARATOR + name);

              logger.info("t1 is invoking get(" + objectName + ")");
              try {
                logger.info("t1 retrieved value " + region.get(objectName));
                fail("first load should have triggered an exception");
              } catch (RuntimeException e) {
                if (!e.getMessage().contains(exceptionString)) {
                  throw e;
                }
              }
            }
          });

      vm0.invoke(
          "Concurrently invoke the loader on the same key - t2", new CacheSerializableRunnable() {
            @Override
            public void run2() {
              final Region<Object, Object> region = getCache().getRegion("root" + SEPARATOR + name);
              final Object[] valueHolder = new Object[1];

              // wait for vm1 to cause the loader to be invoked
              logger.info("t2 is waiting for loader to be invoked by t1");
              try {
                loaderInvokedLatch.await(30, TimeUnit.SECONDS);
              } catch (InterruptedException e) {
                fail("interrupted");
              }
              await().until(() -> loaderInvoked);

              Thread t = new Thread("invoke get()") {
                @Override
                public void run() {
                  try {
                    valueHolder[0] = region.get(objectName);
                  } catch (RuntimeException e) {
                    valueHolder[0] = e;
                  }
                }
              };

              t.setDaemon(true);
              t.start();
              try {
                // let the thread get to the point of blocking on vm1's Future
                // in LocalRegion.nonTxnFindObject()
                Thread.sleep(5000);
              } catch (InterruptedException e) {
                fail("interrupted");
              }

              readyForExceptionLatch.countDown();
              try {
                t.join(30000);
              } catch (InterruptedException e) {
                fail("interrupted");
              }
              if (t.isAlive()) {
                t.interrupt();
                fail("get() operation blocked for too long - test needs some work");
              }

              logger.info("t2 is invoking get(" + objectName + ")");
              Object value = valueHolder[0];
              if (value instanceof RuntimeException) {
                if (((Exception) value).getMessage().contains(exceptionString)) {
                  fail("second load should not have thrown an exception");
                } else {
                  throw (RuntimeException) value;
                }
              } else {
                logger.info("t2 retrieved value " + value);
                assertThat(value).isNotNull();
              }
            }
          });
    } finally {
      if (async1 != null) {
        async1.get();
      }
    }
  }
This test is for a bug in which a cache loader threw an exception that caused the wrong value
to be put in a Future in nonTxnFindObject. This in turn caused a concurrent search for the
object to not invoke the loader a second time.
<p>
VM0 is used to create a cache and a region having a loader that simulates the conditions that
caused the bug. One async thread then does a get() which invokes the loader. Another async
thread does a get() which reaches nonTxnFindObject and blocks waiting for the first thread's
load to complete. The loader then throws an exception that is sent back to the first thread.
The second thread should then cause the loader to be invoked again, and this time the loader
will return a value. Both threads then validate that they received the expected result.
geode-core/src/distributedTest/java/org/apache/geode/cache30/SearchAndLoadDUnitTest.java