@Override
  public boolean accept(Path path) {

    if (engineContext == null) {
      this.engineContext = new HoodieLocalEngineContext(this.conf);
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("Checking acceptance for path " + path);
    }
    Path folder = null;
    try {
      if (storage == null) {
        storage = new HoodieHadoopStorage(convertToStoragePath(path), conf);
      }

      // Assumes path is a file
      folder = path.getParent(); // get the immediate parent.
      // Try to use the caches.
      if (nonHoodiePathCache.contains(folder.toString())) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("Accepting non-hoodie path from cache: " + path);
        }
        return true;
      }

      if (hoodiePathCache.containsKey(folder.toString())) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(String.format("%s Hoodie path checked against cache, accept => %s \n", path,
              hoodiePathCache.get(folder.toString()).contains(path)));
        }
        return hoodiePathCache.get(folder.toString()).contains(path);
      }

      // Skip all files that are descendants of .hoodie in its path.
      String filePath = path.toString();
      if (filePath.contains("/" + HoodieTableMetaClient.METAFOLDER_NAME + "/")
          || filePath.endsWith("/" + HoodieTableMetaClient.METAFOLDER_NAME)) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(String.format("Skipping Hoodie Metadata file  %s \n", filePath));
        }
        return false;
      }

      // Perform actual checking.
      Path baseDir;
      StoragePath storagePath = convertToStoragePath(folder);
      if (HoodiePartitionMetadata.hasPartitionMetadata(storage, storagePath)) {
        HoodiePartitionMetadata metadata = new HoodiePartitionMetadata(storage, storagePath);
        metadata.readFromFS();
        baseDir = HoodieHiveUtils.getNthParent(folder, metadata.getPartitionDepth());
      } else {
        baseDir = safeGetParentsParent(folder);
      }

      if (baseDir != null) {
        // Check whether baseDir in nonHoodiePathCache
        if (nonHoodiePathCache.contains(baseDir.toString())) {
          if (LOG.isDebugEnabled()) {
            LOG.debug("Accepting non-hoodie path from cache: " + path);
          }
          return true;
        }
        HoodieTableFileSystemView fsView = null;
        try {
          HoodieTableMetaClient metaClient = metaClientCache.get(baseDir.toString());
          if (null == metaClient) {
            metaClient = HoodieTableMetaClient.builder()
                .setConf(storage.getConf().newInstance()).setBasePath(baseDir.toString())
                .setLoadActiveTimelineOnLoad(true).build();
            metaClientCache.put(baseDir.toString(), metaClient);
          }

          final Configuration conf = getConf();
          final String timestampAsOf = conf.get(TIMESTAMP_AS_OF.key());
          if (nonEmpty(timestampAsOf)) {
            validateTimestampAsOf(metaClient, timestampAsOf);

            // Build FileSystemViewManager with specified time, it's necessary to set this config when you may
            // access old version files. For example, in spark side, using "hoodie.datasource.read.paths"
            // which contains old version files, if not specify this value, these files will be filtered.
            fsView = FileSystemViewManager.createInMemoryFileSystemViewWithTimeline(engineContext,
                metaClient, HoodieInputFormatUtils.buildMetadataConfig(conf),
                metaClient.getActiveTimeline().filterCompletedInstants().findInstantsBeforeOrEquals(timestampAsOf));
          } else {
            fsView = FileSystemViewManager.createInMemoryFileSystemView(engineContext,
                metaClient, HoodieInputFormatUtils.buildMetadataConfig(conf));
          }
          String partition = HadoopFSUtils.getRelativePartitionPath(new Path(metaClient.getBasePath().toString()), folder);
          List<HoodieBaseFile> latestFiles = fsView.getLatestBaseFiles(partition).collect(Collectors.toList());
          // populate the cache
          if (!hoodiePathCache.containsKey(folder.toString())) {
            hoodiePathCache.put(folder.toString(), new HashSet<>());
          }
          LOG.info("Based on hoodie metadata from base path: " + baseDir.toString() + ", caching " + latestFiles.size()
              + " files under " + folder);
          for (HoodieBaseFile lfile : latestFiles) {
            hoodiePathCache.get(folder.toString()).add(new Path(lfile.getPath()));
          }

          // accept the path, if its among the latest files.
          if (LOG.isDebugEnabled()) {
            LOG.debug(String.format("%s checked after cache population, accept => %s \n", path,
                hoodiePathCache.get(folder.toString()).contains(path)));
          }
          return hoodiePathCache.get(folder.toString()).contains(path);
        } catch (TableNotFoundException e) {
          // Non-hoodie path, accept it.
          if (LOG.isDebugEnabled()) {
            LOG.debug(String.format("(1) Caching non-hoodie path under %s with basePath %s \n", folder.toString(), baseDir.toString()));
          }
          nonHoodiePathCache.add(folder.toString());
          nonHoodiePathCache.add(baseDir.toString());
          return true;
        } finally {
          if (fsView != null) {
            fsView.close();
          }
        }
      } else {
        // files is at < 3 level depth in FS tree, can't be hoodie dataset
        if (LOG.isDebugEnabled()) {
          LOG.debug(String.format("(2) Caching non-hoodie path under %s \n", folder.toString()));
        }
        nonHoodiePathCache.add(folder.toString());
        return true;
      }
    } catch (Exception e) {
      String msg = "Error checking path :" + path + ", under folder: " + folder;
      LOG.error(msg, e);
      throw new HoodieException(msg, e);
    }
  }
@Test
  public void testNonHoodiePaths() throws IOException {
    java.nio.file.Path path1 = Paths.get(basePath, "nonhoodiefolder");
    Files.createDirectories(path1);
    assertTrue(pathFilter.accept(new Path(path1.toUri())));

    java.nio.file.Path path2 = Paths.get(basePath, "nonhoodiefolder/somefile");
    Files.createFile(path2);
    assertTrue(pathFilter.accept(new Path(path2.toUri())));
    assertEquals(2, pathFilter.nonHoodiePathCache.size(), "NonHoodiePathCache size should be 2");
  }