private void launchMpiExec() throws IOException {
        LOG.info("Launching mpiexec in Application Master");

        final List<String> mpiexecArgs = new ArrayList<>(20);
        final StringBuilder mpiexec = new StringBuilder();
        final StringBuilder nodes = new StringBuilder();
        final StringBuilder ldLibraryPath = new StringBuilder();
        final String mpiExtraLdLibraryPath = conf.get(HboxConfiguration.HBOX_MPI_EXTRA_LD_LIBRARY_PATH);

        String mpiInstallDir = Paths.get(conf.get(
                        HboxConfiguration.HBOX_MPI_INSTALL_DIR, HboxConfiguration.DEFAULT_HBOX_MPI_INSTALL_DIR))
                .toAbsolutePath()
                .toString();

        if (mpiExtraLdLibraryPath != null) {
            ldLibraryPath.append(mpiExtraLdLibraryPath).append(":");
            LOG.info("add " + ldLibraryPath + " to LD_LIBRARY_PATH");
        }
        ldLibraryPath.append(mpiInstallDir).append(File.separator).append("lib");
        if (null != System.getenv("LD_LIBRARY_PATH")) {
            ldLibraryPath.append(":").append(System.getenv("LD_LIBRARY_PATH"));
        }

        mpiexec.append(mpiInstallDir).append(File.separator).append("bin").append(File.separator);
        mpiexec.append("mpiexec");

        // get mpi version
        Process mpiexecVersion = null;
        String mpiexecVersionDesc = null;
        try {
            mpiexecVersion = new ProcessBuilder(
                            "env",
                            "OPAL_PREFIX=" + mpiInstallDir,
                            "LD_LIBRARY_PATH=" + ldLibraryPath.toString(),
                            mpiexec.toString(),
                            "-V")
                    .redirectErrorStream(true)
                    .start();
            if (null != mpiexecVersion) {
                mpiexecVersion.getOutputStream().close();
                final long startMs = System.currentTimeMillis();
                try (final BufferedReader reader =
                        new BufferedReader(new InputStreamReader(mpiexecVersion.getInputStream()))) {
                    while (mpiexecVersion.isAlive() && System.currentTimeMillis() - startMs < 1000) {
                        if (reader.ready()) {
                            mpiexecVersionDesc = reader.readLine();
                            if (null != mpiexecVersionDesc) {
                                break;
                            }
                        }
                        Utilities.sleep(50);
                    }
                    if (null == mpiexecVersionDesc) {
                        mpiexecVersionDesc = reader.readLine();
                    }
                }
            }
        } catch (final Throwable e) {
            LOG.warn("cannot read the mpiexec version", e);
        } finally {
            if (null != mpiexecVersion) {
                mpiexecVersion.destroy();
                try {
                    if (!mpiexecVersion.waitFor(1, TimeUnit.SECONDS)) {
                        mpiexecVersion.destroyForcibly();
                    }
                    if (0 != mpiexecVersion.exitValue()) {
                        LOG.warn("mpiexec -V fail" + (null == mpiexecVersionDesc ? "" : ": " + mpiexecVersionDesc));
                        mpiexecVersionDesc = null;
                    }
                } catch (final InterruptedException ignore) {
                    Thread.currentThread().interrupt();
                    mpiexecVersion.destroyForcibly();
                } catch (final IllegalThreadStateException e) {
                    LOG.warn("cannot read the mpiexec version", e);
                }
                mpiexecVersion.getInputStream().close();
                mpiexecVersion = null;
            }
            if (null != mpiexecVersionDesc) {
                Span.current().setAttribute("mpi.runtime.desc", mpiexecVersionDesc);
            }
        }

        mpiexecArgs.add(mpiexec.toString());
        mpiexecArgs.add("--host");
        for (Container container : acquiredWorkerContainers) {
            nodes.append(container.getNodeId().getHost()).append(",");
        }
        nodes.deleteCharAt(nodes.length() - 1);
        mpiexecArgs.add(nodes.toString());

        // mpiexecArgs is run directly on AM(not by a shell), so no escaping is needed.
        // The user command appended is transfered from mpiexec to the orted process on the work
        // container as an argv array. To redirect stdout/stderr of the user command as a subprocess
        // of orted, we can insert a shell like this:
        //   /bin/sh -xc 'exec "$@" 1>>stdout 2>>stderr' -- hboxCommandArgs
        // The worker command is not processed by yarn nm service, so <LOG_DIR> cannot be replaced by yarn.
        // This command is also executed directly by execv syscall, so add them here as raw string.
        //
        // The first bash snippet run on workers:
        // * unset PMIX_INSTALL_PREFIX, avoid introduce the openmpi prefix path on AM
        // * fix PATH, remove the openmpi prefix of AM which is appended to the worker when OPAL_PREFIX of AM is set
        // * fix LD_LIBRARY_PATH, also remove the openmpi prefix of AM
        // * redirect stderr to both mpi-stderr and normal stderr (which is send back to mpiexec by orted)
        mpiexecArgs.add("/bin/bash");
        mpiexecArgs.add("-ec");
        if (conf.getBoolean(
                HboxConfiguration.HBOX_AGG_ALL_MPI_STDERR, HboxConfiguration.DEFAULT_HBOX_AGG_ALL_MPI_STDERR)) {
            mpiexecArgs.add(String.format(
                    "unset PMIX_INSTALL_PREFIX; LD_LIBRARY_PATH=\"${LD_LIBRARY_PATH#*:}\" PATH=\"${PATH#*:}\" exec 2> >(tee \"$%s/%s\" | sed \"s/^/[rank $((%s-1)) stderr] /\" >&2) \"$@\"",
                    HboxConstants.Environment.HBOX_CONTAINER_LOG_DIR,
                    HboxConstants.MPI_STD_ERR_FILE,
                    HboxConstants.Environment.HBOX_TF_INDEX));
        } else {
            mpiexecArgs.add(String.format(
                    "unset PMIX_INSTALL_PREFIX; (( %s == 1 )) && exec 2> >(tee \"$%s/%s\" >&2) || exec 2>\"$%s/%s\"; LD_LIBRARY_PATH=\"${LD_LIBRARY_PATH#*:}\" PATH=\"${PATH#*:}\" exec \"$@\"",
                    HboxConstants.Environment.HBOX_TF_INDEX,
                    HboxConstants.Environment.HBOX_CONTAINER_LOG_DIR,
                    HboxConstants.MPI_STD_ERR_FILE,
                    HboxConstants.Environment.HBOX_CONTAINER_LOG_DIR,
                    HboxConstants.MPI_STD_ERR_FILE));
        }
        mpiexecArgs.add("--");

        // The second bash snippet run on workers:
        // * for the 1st rank, redirected stdout to both mpi-stdout and normal stdout
        //   (which is send back to mpiexec by orted)
        // * for other ranks, stdout is only redirected to mpi-stdout
        mpiexecArgs.add("/bin/bash");
        mpiexecArgs.add("-ec");
        mpiexecArgs.add(String.format(
                "(( %s != 1 )) && exec >\"$%s/%s\" \"$@\"; exec > >(tee \"$%s/%s\") \"$@\"",
                HboxConstants.Environment.HBOX_TF_INDEX,
                HboxConstants.Environment.HBOX_CONTAINER_LOG_DIR,
                HboxConstants.MPI_STD_OUT_FILE,
                HboxConstants.Environment.HBOX_CONTAINER_LOG_DIR,
                HboxConstants.MPI_STD_OUT_FILE));
        mpiexecArgs.add("--");

        for (final String arg : hboxCommandArgs) {
            mpiexecArgs.add(arg);
        }

        List<String> envLists = new ArrayList<>();

        envLists.add("PATH=" + System.getenv("PATH"));
        envLists.add("PWD=" + mpiExecDir);
        envLists.add("LD_LIBRARY_PATH=" + ldLibraryPath.toString());

        // MPI related options
        // relocated openmpi install dir
        envLists.add("OPAL_PREFIX=" + mpiInstallDir);
        // bind-to none option
        envLists.add("OMPI_MCA_hwloc_base_binding_policy=none");
        // -mca plm_rsh_agent /bin/true
        envLists.add("OMPI_MCA_plm_rsh_agent=/bin/true");
        // -mca plm_base_verbose 1
        envLists.add("OMPI_MCA_plm_base_verbose=1");
        // --oversubscribe
        envLists.add("OMPI_MCA_rmaps_base_oversubscribe=1");
        // --pernode
        envLists.add("OMPI_MCA_rmaps_ppr_pernode=1");

        // append all hbox.am.env prefix config to exec env list
        conf.forEach(entry -> {
            final String key = entry.getKey();
            final String value = entry.getValue();
            if (null != key && null != value) {
                if (key.startsWith(HboxConstants.AM_ENV_PREFIX)) {
                    final String envName =
                            key.substring(HboxConstants.AM_ENV_PREFIX.length()).trim();
                    if (!envName.isEmpty()) {
                        envLists.add(envName + "=" + value);
                    }
                }
            }
        });

        LOG.info("Executing mpi exec command: " + String.join(" ", mpiexecArgs));
        LOG.info("Mpi exec Process run in: " + mpiExecDir);
        mpiExecProcess = processLaunch.exec(
                mpiexecArgs.toArray(new String[0]),
                envLists.toArray(new String[envLists.size()]),
                this.envs,
                new File(mpiExecDir));

        Thread stdinThread = new Thread(new Runnable() {
            @Override
            public void run() {
                try {
                    BufferedReader reader;
                    reader = new BufferedReader(new InputStreamReader(mpiExecProcess.getInputStream()));
                    String mpiExecOutput;
                    while ((mpiExecOutput = reader.readLine()) != null) {
                        LOG.info(mpiExecOutput);
                        appendMessage(new Message(LogType.STDOUT, mpiExecOutput));
                        extractMpiRTECommand(mpiExecOutput, amContainerStdOut);
                    }
                } catch (Exception e) {
                    LOG.warn("Error in mpi exec process stdinThread");
                }
            }
        });
        stdinThread.start();

        Thread stderrThread = new Thread(new Runnable() {
            @Override
            public void run() {
                try {
                    BufferedReader reader;
                    reader = new BufferedReader(new InputStreamReader(mpiExecProcess.getErrorStream()));
                    String mpiExecStderr;
                    while ((mpiExecStderr = reader.readLine()) != null) {
                        LOG.info(mpiExecStderr);
                        appendMessage(new Message(LogType.STDERR, mpiExecStderr));
                        extractMpiRTECommand(mpiExecStderr, null);
                        if (conf.getBoolean(
                                HboxConfiguration.HBOX_CONTAINER_RUNNING_LOG_ENABLE,
                                HboxConfiguration.DEFAULT_HBOX_CONTAINER_RUNNING_LOG_ENABLE)) {
                            amContainerStdErr.append(mpiExecStderr);
                        }
                    }
                } catch (Exception e) {
                    LOG.warn("Error in mpi exec process stderrThread");
                }
            }
        });
        stderrThread.start();
    }
Application Master launch "mpiexec" process locally for mpi app
core/src/main/java/net/qihoo/hbox/AM/ApplicationMaster.java