@Override
    public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlus rootSchema) {
      PerfLogger perfLogger = SessionState.getPerfLogger();
      perfLogger.perfLogBegin(this.getClass().getName(), PerfLogger.PLAN_GENERATION);
      RelNode calcitePlan;
      subqueryId = -1;

      /*
       * recreate cluster, so that it picks up the additional traitDef
       */
      RelOptPlanner planner = createPlanner(conf, statsSource, ctx.isExplainPlan());
      final RexBuilder rexBuilder = new RexBuilder(new HiveTypeFactory());
      final RelOptCluster optCluster = RelOptCluster.create(planner, rexBuilder);

      this.cluster = optCluster;
      this.relOptSchema = relOptSchema;
      this.functionHelper = new HiveFunctionHelper(rexBuilder);

      // 1. Gen Calcite Plan
      perfLogger.perfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
      try {
        calcitePlan = genLogicalPlan(getQB(), true, null, null);
        // freeze the names in the hash map for objects that are only interested
        // in the parsed tables in the original query.
        tabNameToTabObject.markParsingCompleted();
        // if it is to create view, we do not use table alias
        resultSchema = convertRowSchemaToResultSetSchema(relToHiveRR.get(calcitePlan),
            (forViewCreation || getQB().isMaterializedView()) ? false : HiveConf.getBoolVar(conf,
                HiveConf.ConfVars.HIVE_RESULTSET_USE_UNIQUE_COLUMN_NAMES));
      } catch (SemanticException e) {
        semanticException = e;
        throw new RuntimeException(e);
      }

      if (LOG.isDebugEnabled()) {
        LOG.debug("Initial CBO Plan:\n" + RelOptUtil.toString(calcitePlan));
      }
      perfLogger.perfLogEnd(this.getClass().getName(), PerfLogger.PLAN_GENERATION);

      perfLogger.perfLogBegin(this.getClass().getName(), PerfLogger.MV_REWRITE_FIELD_TRIMMER);
      // Create executor
      RexExecutor executorProvider = new HiveRexExecutorImpl();
      calcitePlan.getCluster().getPlanner().setExecutor(executorProvider);

      // Create and set MD provider
      HiveDefaultRelMetadataProvider mdProvider = new HiveDefaultRelMetadataProvider(conf, HIVE_REL_NODE_CLASSES);
      RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(mdProvider.getMetadataProvider()));
      optCluster.setMetadataQuerySupplier(HiveRelMetadataQuery::new);
      optCluster.invalidateMetadataQuery();

      calcitePlan = applyMaterializedViewRewritingByText(
              ast, calcitePlan, optCluster, mdProvider.getMetadataProvider());

      // We need to get the ColumnAccessInfo and viewToTableSchema for views.
      if (conf.getBoolVar(ConfVars.HIVE_STATS_COLLECT_SCANCOLS) || !skipAuthorization()) {
        HiveRelFieldTrimmer.get()
            .trim(HiveRelFactories.HIVE_BUILDER.create(optCluster, null), calcitePlan, this.columnAccessInfo,
                this.viewProjectToTableSchema);
      }
      perfLogger.perfLogEnd(this.getClass().getName(), PerfLogger.MV_REWRITE_FIELD_TRIMMER);

      //Remove subquery
      perfLogger.perfLogBegin(this.getClass().getName(), PerfLogger.REMOVING_SUBQUERY);
      if (LOG.isDebugEnabled()) {
        LOG.debug("Plan before removing subquery:\n" + RelOptUtil.toString(calcitePlan));
      }
      calcitePlan = removeSubqueries(calcitePlan, mdProvider.getMetadataProvider());
      if (LOG.isDebugEnabled()) {
        LOG.debug("Plan after removing subquery:\n" + RelOptUtil.toString(calcitePlan));
      }
      perfLogger.perfLogEnd(this.getClass().getName(), PerfLogger.REMOVING_SUBQUERY);
      perfLogger.perfLogBegin(this.getClass().getName(), PerfLogger.DECORRELATION);
      calcitePlan = HiveRelDecorrelator.decorrelateQuery(calcitePlan);
      if (LOG.isDebugEnabled()) {
        LOG.debug("Plan after decorrelation:\n" + RelOptUtil.toString(calcitePlan));
      }
      perfLogger.perfLogEnd(this.getClass().getName(), PerfLogger.DECORRELATION);

      // Validate query materialization for query results caching. This check needs
      // to occur before constant folding, which may remove some function calls
      // from the query plan.
      // In addition, if it is a materialized view creation and we are enabling it
      // for rewriting, it should pass all checks done for query results caching
      // and on top of that we should check that it only contains operators that
      // are supported by the rewriting algorithm.
      perfLogger.perfLogBegin(this.getClass().getName(), PerfLogger.VALIDATE_QUERY_MATERIALIZATION);
      HiveRelOptMaterializationValidator materializationValidator = new HiveRelOptMaterializationValidator();
      materializationValidator.validate(calcitePlan);
      setInvalidResultCacheReason(
          materializationValidator.getResultCacheInvalidReason());
      setMaterializationValidationResult(
          materializationValidator.getAutomaticRewritingValidationResult());
      perfLogger.perfLogEnd(this.getClass().getName(), PerfLogger.VALIDATE_QUERY_MATERIALIZATION);

      // 2. Apply pre-join order optimizations
      perfLogger.perfLogBegin(this.getClass().getName(), PerfLogger.PREJOIN_ORDERING);
      calcitePlan = applyPreJoinOrderingTransforms(calcitePlan, mdProvider.getMetadataProvider(), executorProvider);
      if (LOG.isDebugEnabled()) {
        LOG.debug("Plan after pre-join transformations:\n" + RelOptUtil.toString(calcitePlan));
      }
      perfLogger.perfLogEnd(this.getClass().getName(), PerfLogger.PREJOIN_ORDERING);
      // 3. Materialized view based rewriting
      // We disable it for CTAS and MV creation queries (trying to avoid any problem
      // due to data freshness)
      perfLogger.perfLogBegin(this.getClass().getName(), PerfLogger.MV_REWRITING);
      if (conf.getBoolVar(ConfVars.HIVE_MATERIALIZED_VIEW_ENABLE_AUTO_REWRITING) &&
              !getQB().isMaterializedView() && !ctx.isLoadingMaterializedView() && !getQB().isCTAS() &&
               getQB().hasTableDefined() &&
              !forViewCreation) {
        calcitePlan =
            applyMaterializedViewRewriting(planner, calcitePlan, mdProvider.getMetadataProvider(), executorProvider);
        if (LOG.isDebugEnabled()) {
          LOG.debug("Plan after view-based rewriting:\n" + RelOptUtil.toString(calcitePlan));
        }
      }
      perfLogger.perfLogEnd(this.getClass().getName(), PerfLogger.MV_REWRITING);

      // 4. Apply join order optimizations: reordering MST algorithm
      //    If join optimizations failed because of missing stats, we continue with
      //    the rest of optimizations
      perfLogger.perfLogBegin(this.getClass().getName(), PerfLogger.JOIN_REORDERING);
      if (profilesCBO.contains(ExtendedCBOProfile.JOIN_REORDERING)) {
        calcitePlan = applyJoinOrderingTransform(calcitePlan, mdProvider.getMetadataProvider(), executorProvider);
        if (LOG.isDebugEnabled()) {
          LOG.debug("Plan after join transformations:\n" + RelOptUtil.toString(calcitePlan));
        }
      } else {
        disableSemJoinReordering = false;
      }
      perfLogger.perfLogEnd(this.getClass().getName(), PerfLogger.JOIN_REORDERING);

      // 5. Apply post-join order optimizations
      perfLogger.perfLogBegin(this.getClass().getName(), PerfLogger.POSTJOIN_ORDERING);
      calcitePlan = applyPostJoinOrderingTransform(calcitePlan, mdProvider.getMetadataProvider(), executorProvider);
      perfLogger.perfLogEnd(this.getClass().getName(), PerfLogger.POSTJOIN_ORDERING);
      // Perform the CTE rewriting near the end of CBO transformations to avoid interference of the new HiveTableSpool 
      // operator with other rules (especially those related to constant folding and branch pruning).
      if (!forViewCreation) {
        calcitePlan = applyCteRewriting(planner, calcitePlan, mdProvider.getMetadataProvider(), executorProvider);
        if (LOG.isDebugEnabled()) {
          LOG.debug("Plan after CTE rewriting:\n{}", RelOptUtil.toString(calcitePlan));
        }
      }
      calcitePlan = applySearchExpandTransforms(calcitePlan, mdProvider.getMetadataProvider(), executorProvider);

      perfLogger.perfLogBegin(this.getClass().getName(), PerfLogger.HIVE_SORT_PREDICATES);
      if (conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_SORT_PREDS_WITH_STATS)) {
        calcitePlan = calcitePlan.accept(new HiveFilterSortPredicates(noColsMissingStats));
      }
      perfLogger.perfLogEnd(this.getClass().getName(), PerfLogger.HIVE_SORT_PREDICATES);
      if (LOG.isDebugEnabled()) {
        LOG.debug("Plan after post-join transformations:\n" + RelOptUtil.toString(calcitePlan));
      }
      return calcitePlan;
    }
Code responsible for Calcite plan generation and optimization.
ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java