private long evaluateComparatorWithHistogram(ColStatistics cs, long currNumRows, String colTypeLowerCase,
        String boundValue, boolean upperBound, boolean closedBound) {
      final KllFloatsSketch kll = KllFloatsSketch.heapify(Memory.wrap(cs.getHistogram()));

      if (kll.getN() == 0) {
        return 0;
      }

      try {
        final float value = extractFloatFromLiteralValue(colTypeLowerCase, boundValue);

        // kll ignores null values (i.e., kll.getN() + numNulls = currNumRows), we therefore need to use kll.getN()
        // instead of currNumRows since the CDF is expressed as a fraction of kll.getN(), not currNumRows
        if (upperBound) {
          return Math.round(kll.getN() * (closedBound ?
              lessThanOrEqualSelectivity(kll, value) : lessThanSelectivity(kll, value)));
        } else {
          return Math.round(kll.getN() * (closedBound ?
              greaterThanOrEqualSelectivity(kll, value) : greaterThanSelectivity(kll, value)));
        }
      } catch (RuntimeException e) {
        LOG.debug("Selectivity computation using histogram failed to parse the boundary value ({}), "
            + ", using the generic computation strategy", boundValue, e);
        return currNumRows / 3;
      }
    }
removes all values which are outside of the scope of the column
ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java