@Override
  protected void doAlteration(Table table, Partition partition) throws HiveException {
    StorageDescriptor sd = getStorageDescriptor(table, partition);
    // if orc table, restrict changing the file format as it can break schema evolution
    if (AlterTableUtils.isSchemaEvolutionEnabled(table, context.getConf()) &&
        sd.getInputFormat().equals(OrcInputFormat.class.getName())
        && !desc.getInputFormat().equals(OrcInputFormat.class.getName())) {
      throw new HiveException(ErrorMsg.CANNOT_CHANGE_FILEFORMAT, "ORC", desc.getDbTableName());
    }

    sd.setInputFormat(desc.getInputFormat());
    sd.setOutputFormat(desc.getOutputFormat());
    if (desc.getSerdeName() != null) {
      sd.getSerdeInfo().setSerializationLib(desc.getSerdeName());
    }
  }
Operation process of setting the file format.
ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/set/fileformat/AlterTableSetFileFormatOperation.java