public void testRetryAfterFailedCleanup(boolean delayEnabled) throws Exception {
    HiveConf.setBoolVar(conf, HIVE_COMPACTOR_DELAYED_CLEANUP_ENABLED, delayEnabled);
    HiveConf.setTimeVar(conf, HIVE_COMPACTOR_CLEANER_RETENTION_TIME, 2, TimeUnit.SECONDS);
    MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.HIVE_COMPACTOR_CLEANER_MAX_RETRY_ATTEMPTS, 3);
    MetastoreConf.setTimeVar(conf, MetastoreConf.ConfVars.HIVE_COMPACTOR_CLEANER_RETRY_RETENTION_TIME, 100, TimeUnit.MILLISECONDS);
    String errorMessage = "No cleanup here!";

    //Prevent cleaner from marking the compaction as cleaned
    TxnStore mockedHandler = spy(txnHandler);
    MetadataCache metadataCache = new MetadataCache(true);
    FSRemover fsRemover = new FSRemover(conf, ReplChangeManager.getInstance(conf), metadataCache);
    List<TaskHandler> taskHandlers = TaskHandlerFactory.getInstance()
            .getHandlers(conf, mockedHandler, metadataCache, false, fsRemover);
    doThrow(new RuntimeException(errorMessage)).when(mockedHandler).markCleaned(nullable(CompactionInfo.class));

    Table t = newTable("default", "retry_test", false);

    addBaseFile(t, null, 20L, 20);
    addDeltaFile(t, null, 21L, 22L, 2);
    addDeltaFile(t, null, 23L, 24L, 2);
    burnThroughTransactions("default", "retry_test", 25);

    CompactionRequest rqst = new CompactionRequest("default", "retry_test", CompactionType.MAJOR);
    long compactTxn = compactInTxn(rqst);
    addBaseFile(t, null, 25L, 25, compactTxn);

    //delayed start retention time
    long retentionTime = delayEnabled ? conf.getTimeVar(HIVE_COMPACTOR_CLEANER_RETENTION_TIME, TimeUnit.MILLISECONDS) : 0;
    //retry retention time
    long retryRetentionTime = getTimeVar(conf, HIVE_COMPACTOR_CLEANER_RETRY_RETENTION_TIME, TimeUnit.MILLISECONDS);
    // Sleep 100ms longer than the actual retention to make sure the compaction will be picked by the cleaner
    Thread.sleep(retentionTime + 100);

    for (int i = 1; i < 4; i++) {
      Cleaner cleaner = new Cleaner();
      cleaner.setConf(conf);
      cleaner.init(new AtomicBoolean(true));
      cleaner.setCleanupHandlers(taskHandlers);

      ReflectionUtil.setInAllFields(cleaner, "txnHandler", mockedHandler);

      cleaner.run();

      // Sleep 100ms longer than the actual retry retention to make sure the compaction will be picked up again by the cleaner
      Thread.sleep(retryRetentionTime * (long) Math.pow(2, i - 1) + 100);

      // Check retry attempts updated
      Optional<CompactionInfo> compactionByTxnId = txnHandler.getCompactionByTxnId(compactTxn);
      Assert.assertTrue("Expected compactionInfo, but got nothing returned", compactionByTxnId.isPresent());
      CompactionInfo ci = compactionByTxnId.get();

      // Check if state is still 'ready for cleaning'
      Assert.assertEquals(String.format("Expected 'r' (ready for cleaning) state, but got: '%c'", ci.state), 'r', ci.state);
      // Check if error message was set correctly
      Assert.assertEquals(String.format("Expected error message: '%s', but got '%s'", errorMessage, ci.errorMessage),
          errorMessage, ci.errorMessage);
      // Check if retentionTime was set correctly
      int cleanAttempts = (int)(Math.log(ci.retryRetention / retryRetentionTime) / Math.log(2)) + 1;
      Assert.assertEquals(String.format("Expected %d clean attempts, but got %d", i, cleanAttempts), i, cleanAttempts);
    }

    //Do a final run to reach the maximum retry attempts, so the state finally should be set to failed
    Cleaner cleaner = new Cleaner();
    cleaner.setConf(conf);
    cleaner.init(new AtomicBoolean(true));
    cleaner.setCleanupHandlers(taskHandlers);
    ReflectionUtil.setInAllFields(cleaner, "txnHandler", mockedHandler);

    cleaner.run();

    ShowCompactResponse scr = txnHandler.showCompact(new ShowCompactRequest());
    Assert.assertEquals(String.format("Expected %d CompactionInfo, but got %d", 1, scr.getCompactsSize()),
        1, scr.getCompactsSize());
    ShowCompactResponseElement scre = scr.getCompacts().get(0);
    //The state finally should be set to failed.
    Assert.assertEquals(String.format("Expected '%s' state, but got '%s'", "failed", scre.getState()),
        "failed", scre.getState());
    Assert.assertEquals(String.format("Expected error message: '%s', but got '%s'", errorMessage, scre.getErrorMessage()),
        errorMessage, scre.getErrorMessage());
  }
Tests for the compactor Cleaner thread
ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java