private void launchHorovodExec() throws IOException {
        LOG.info("Launching horovod exec in Application Master");
        StringBuilder commandBuilder = new StringBuilder();
        StringBuilder ldLibraryPath = new StringBuilder();

        String mpiExtraLdLibraryPath = conf.get(HboxConfiguration.HBOX_HOROVOD_EXTRA_LD_LIBRARY_PATH);
        if (mpiExtraLdLibraryPath != null) {
            ldLibraryPath.append(mpiExtraLdLibraryPath);
            LOG.info("add " + ldLibraryPath + " to LD_LIBRARY_PATH");
        }
        if (conf.getBoolean(
                HboxConfiguration.HBOX_MPI_INSTALL_DIR_ENABLE, HboxConfiguration.DEFAULT_HBOX_MPI_INSTALL_DIR_ENABLE)) {
            String mpiInstallDir =
                    conf.get(HboxConfiguration.HBOX_MPI_INSTALL_DIR, HboxConfiguration.DEFAULT_HBOX_MPI_INSTALL_DIR);
            commandBuilder
                    .append(mpiInstallDir)
                    .append(File.separator)
                    .append("bin")
                    .append(File.separator);
            ldLibraryPath
                    .append(":")
                    .append(mpiInstallDir)
                    .append(File.separator)
                    .append("lib");
        }
        int processPerWorker = conf.getInt(
                HboxConfiguration.HBOX_HOROVOD_PROCESS_NUM_PER_WORKER,
                HboxConfiguration.DEDAULT_HBOX_HOROVOD_PROCESS_NUM_PER_WORKER);
        commandBuilder
                .append("mpirun -np ")
                .append(workerNum * processPerWorker)
                .append(" -H ");
        ldLibraryPath.append(":").append(System.getenv("LD_LIBRARY_PATH"));
        for (Container container : acquiredWorkerContainers) {
            if (processPerWorker == 1)
                commandBuilder.append(container.getNodeId().getHost()).append(",");
            else
                commandBuilder
                        .append(container.getNodeId().getHost())
                        .append(":")
                        .append(processPerWorker)
                        .append(",");
        }
        commandBuilder.deleteCharAt(commandBuilder.length() - 1);
        String horovodConfig = readHorovodConfig();
        if (horovodConfig.trim().equals("")) commandBuilder.append(" ");
        else commandBuilder.append(" ").append(readHorovodConfig()).append(" ");
        commandBuilder.append(
                "-bind-to none -map-by slot -x NCCL_DEBUG=INFO -x LD_LIBRARY_PATH -x PATH -mca pml ob1 -mca btl ^openib -mca btl_tcp_if_include \"10.0.0.0/8\" ");
        commandBuilder.append(String.join(" ", hboxCommandArgs));

        List<String> env = new ArrayList<>(20);
        Map<String, String> userEnv = new HashMap<>();
        if (conf.get(HboxConfiguration.HBOX_CONTAINER_ENV) != null) {
            String[] containerEnv = StringUtils.split(conf.get(HboxConfiguration.HBOX_CONTAINER_ENV), "|");
            for (String envPair : containerEnv) {
                String[] userEnvPair = StringUtils.split(envPair, "=");
                if (userEnvPair.length != 2) {
                    LOG.error(envPair + " is not correct");
                } else {
                    env.add(envPair);
                    userEnv.put(userEnvPair[0], userEnvPair[1]);
                }
            }
        }
        env.add("PWD=" + mpiExecDir);
        if (userEnv.containsKey("PATH")) {
            env.add("PATH=" + userEnv.get("PATH") + System.getProperty("path.separator") + System.getenv("PATH"));
        } else {
            env.add("PATH=" + System.getenv("PATH"));
        }
        if (userEnv.containsKey("LD_LIBRARY_PATH")) {
            env.add("LD_LIBRARY_PATH=" + userEnv.get("LD_LIBRARY_PATH") + System.getProperty("path.separator")
                    + ldLibraryPath.toString());
        } else {
            env.add("LD_LIBRARY_PATH=" + ldLibraryPath.toString());
        }

        File mpiExec = new File(mpiExecDir);
        LOG.info("Executing horovod exec command: " + commandBuilder.toString());
        LOG.info("Horovod mpi exec Process run in: " + mpiExec.toString());
        mpiExecProcess =
                processLaunch.exec(commandBuilder.toString(), env.toArray(new String[env.size()]), this.envs, mpiExec);

        Thread stdinThread = new Thread(new Runnable() {
            @Override
            public void run() {
                try {
                    BufferedReader reader;
                    reader = new BufferedReader(new InputStreamReader(mpiExecProcess.getInputStream()));
                    String mpiExecOutput;
                    while ((mpiExecOutput = reader.readLine()) != null) {
                        if (mpiExecOutput.startsWith("command")) {
                            LOG.info("Container horovod Command " + mpiExecOutput);
                            appendMessage(new Message(LogType.STDERR, mpiExecOutput));
                            // get orted command
                            mpiContainerCommand =
                                    mpiExecOutput.replaceFirst("command:", "").replace("--daemonize", "");
                        } else {
                            LOG.info(mpiExecOutput);
                            appendMessage(new Message(LogType.STDOUT, mpiExecOutput));
                        }
                    }
                } catch (Exception e) {
                    LOG.warn("Error in horovod exec process stdinThread");
                }
            }
        });
        stdinThread.start();

        Thread stderrThread = new Thread(new Runnable() {
            @Override
            public void run() {
                try {
                    BufferedReader reader;
                    reader = new BufferedReader(new InputStreamReader(mpiExecProcess.getErrorStream()));
                    String mpiExecStderr;
                    while ((mpiExecStderr = reader.readLine()) != null) {
                        LOG.info(mpiExecStderr);
                        appendMessage(new Message(LogType.STDERR, mpiExecStderr));
                    }
                } catch (Exception e) {
                    LOG.warn("Error in mpi exec process stderrThread");
                }
            }
        });
        stderrThread.start();
    }
Extract orted command from mpiexec
With option -mca plm_rsh_agent /bin/echo -mca plm_base_verbose 1
For openmpinossh version, sample command syntax is "command:-mca ess env -mca orte_ess_jobid "<jobid>" -mca orte_ess_vpid <template> -mca orte_ess_num_procs 31 --hnp-uri "<jobid>.0;tcp://xx.xx.xx.xx:xxxx"
For openmpi-3.1.4 version, sample command syntax is "/usr/bin/ssh <template>  orted -mca ess "env" -mca ess_base_jobid "<jobid>" -mca ess_base_vpid "<template>" -mca ess_base_num_procs "2" -mca orte_node_regex "qt[x:x,x]ss@0(2)" -mca orte_hnp_uri "<jobid>.0;tcp://xx.xx.xx.xx:xxxx" -mca plm_base_verbose "1" -mca plm "rsh" -mca pmix "^s1,s2,cray,isolated""
@param mpiExecOutput
@return
core/src/main/java/net/qihoo/hbox/AM/ApplicationMaster.java