How can I write large output to Process getOutputStream?

Question

I am trying to execute a command (eg. ps -ef | grep apache) using ProcessBuilder and Process. The code works as long as the output of 'ps -ef' is small. But if the output is too big, the program hangs. Is there a way to fix this? Here is my code based on [http://www.javaworld.com/javaworld/jw-12-2000/jw-1229-traps.html]

#### Program.java ####
    import java.io.BufferedOutputStream;
    import java.io.File;
    import java.io.IOException;
    import java.io.OutputStreamWriter;
    import java.io.PrintWriter;

    import java.util.ArrayList;
    import java.util.Collections;
    import java.util.List;
    import java.util.Map;

    public class Program {

        private List<String> command;

        public Program(String commandString) throws IOException {
            this(commandString, null);
        }

        public List<String> getCommand() {
            return this.command;
        }

        private void setCommand(String filename, String location, String commandString, List<String> parameters) throws IOException {
            if(filename != null) {
                commandString = new File(location, filename).getCanonicalPath();
            }

            this.command = 
                Collections.synchronizedList(new ArrayList<String>());

            this.command.add(commandString);
            if (parameters != null) {
                for (String arg: parameters) {
                    command.add(arg);
                }
            }
        }

        public String[] run() throws IOException, InterruptedException {
            return this.run(null);
        }

        public String[] run(String input) throws IOException, InterruptedException {
            ProcessBuilder processBuilder = new ProcessBuilder(this.command);

            List<String> commandList = processBuilder.command();

            Process process = processBuilder.start();
            if(input != null) {
                PrintWriter writer = new PrintWriter(new OutputStreamWriter(new BufferedOutputStream(process.getOutputStream())), true);
                writer.println(input);
                writer.flush();
                writer.close();
            }
            process.getOutputStream().close();
            Gobbler outGobbler = new Gobbler(process.getInputStream());
            Gobbler errGobbler = new Gobbler(process.getErrorStream());

            Thread outThread = new Thread(outGobbler);
            Thread errThread  = new Thread(errGobbler);

            outThread.start();
            errThread.start();

            outThread.join();
            errThread.join();

            int exitVal = process.waitFor();
            System.out.println("PROCESS WAIT FOR: " + exitVal);

            List<String> output = outGobbler.getOuput();

            return output.toArray(new String[output.size()]);
        }
    }



#### CommandExecutor.java ####

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;

public class CommandExecutor {

    public List<List<Object>> programs;

    public static void main(String[] args) {

        try {
            CommandExecutor ce = new CommandExecutor(args[0]);
            String output = ce.run();
            System.out.println("Command: " + args[0]);
            System.out.println("Output: " + output);
        } catch (IOException e) {
            // TODO Auto-generated catch block
            System.out.println(e.getLocalizedMessage());
            e.printStackTrace();
        } catch (InterruptedException ie) {
            // TODO Auto-generated catch block
            System.out.println(ie.getLocalizedMessage());
            ie.printStackTrace();
        }

    }

    public CommandExecutor(String command) throws IOException {
        this.setPrograms(command);
    }

    private void setPrograms(String command) throws IOException {
        this.programs = new ArrayList<List<Object>>();

        //String cmdstring = "";
        String[] commands = command.split("\\s*;\\s*");
        for(String c: commands) {
            //String subcmdstr = "";
            String file = null;
            String[] chainedCommands = c.split("\\s*\\|\\s*");
            String lastCmd = chainedCommands[chainedCommands.length-1];
            String[] fileCmd = lastCmd.split("\\s*>\\s*");
            if(fileCmd.length > 1) {
                chainedCommands[chainedCommands.length-1] = fileCmd[0];
                file = fileCmd[1];
            }
            List<Object> l = new ArrayList<Object>();
            for(String p: chainedCommands) {
                /*if(subcmdstr.equals("")) {
                    subcmdstr = p;
                }
                else {
                    subcmdstr += " redirects to " + p;
                }*/
                String[] cmdparams = p.split(" ");
                String cmd = cmdparams[0];
                List<String> params = new ArrayList<String>();
                for(int j = 1; j < cmdparams.length; j++) {
                    params.add(cmdparams[j]);
                }
                Program prog = new Program(cmd, params);
                l.add(prog);
            }
            if(file != null) {
                //subcmdstr += " redirects to file: " + file;
                l.add(file);
            }
            this.programs.add(l);
            //cmdstring += "new command: " + subcmdstr + "\n";
        }
        //System.out.println("Actual Command: " + command);
        //System.out.println("Command String:\n" + cmdstring);
    }

    public String run() throws IOException, InterruptedException {
        String output = "";

        for(List<Object> l: this.programs) {
            String[] out = new String[0];
            int count = 0;
            boolean filenotfound = true;
            for(Object o: l) {
                if(o instanceof Program) {
                    Program p = (Program) o;
                    if(count == 0) {
                        out = p.run();
                    }
                    else {
                        out = p.run(CommandExecutor.arrayToString(out));
                    }
                }
                else if(o instanceof String) {
                    PrintWriter f = new PrintWriter(new File((String)o));
                    f.print(CommandExecutor.arrayToString(out));
                    f.close();
                    filenotfound = false;
                }
                count++;
            }
            if(filenotfound) {
                output += CommandExecutor.arrayToString(out);
            }
        }

        return output;
    }

    public static String arrayToString(String[] strArray) {
        String str = "";
        for(String s: strArray) {
            str += s;
        }
        return str;
    }
}

Thanks,

Quadir

Answer 1

Ok, I got it working. Below is the code, given a list of commands, it pipes the output of one command to the next.

/* 
 ####### PipeRedirection.java 
*/

import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;

public class PipeRedirection {

    public static void main(String[] args) throws FileNotFoundException {

        if(args.length < 2) {
            System.err.println("Need at least two arguments");
            System.exit(1);
        }

        try {
            String input = null;
            for(int i = 0; i < args.length; i++) {

                String[] commandList = args[i].split(" ");

                ProcessBuilder pb = new ProcessBuilder(commandList);
                //pb.redirectErrorStream(true);
                Process p = pb.start();

                if(input != null) {
                    PrintWriter writer = new PrintWriter(new OutputStreamWriter(new BufferedOutputStream(p.getOutputStream())), true);
                    writer.println(input);
                    writer.flush();
                    writer.close();
                }

                InputProcess.Gobbler outGobbler = new InputProcess.Gobbler(p.getInputStream());
                InputProcess.Gobbler errGobbler = new InputProcess.Gobbler(p.getErrorStream());
                Thread outThread = new Thread(outGobbler);
                Thread errThread = new Thread(errGobbler);
                outThread.start();
                errThread.start();

                outThread.join();
                errThread.join();

                int exitVal = p.waitFor();
                System.out.println("\n****************************");
                System.out.println("Command: " + args[i]);
                System.out.println("Exit Value = " + exitVal);
                List<String> output = outGobbler.getOuput();
                input = "";
                for(String o: output) {
                    input += o;
                }
            }
            System.out.println("Final Output:");
            System.out.println(input);

        } catch (IOException ioe) {
            // TODO Auto-generated catch block
            System.err.println(ioe.getLocalizedMessage());
            ioe.printStackTrace();
        } catch (InterruptedException ie) {
            // TODO Auto-generated catch block
            System.err.println(ie.getLocalizedMessage());
            ie.printStackTrace();
        }

    }


    public static class Gobbler implements Runnable {
        private BufferedReader reader;
        private List<String> output;

        public Gobbler(InputStream inputStream) {
            this.reader = new BufferedReader(new InputStreamReader(inputStream));
        }

        public void run() {
            String line;
            this.output = new ArrayList<String>();
            try {
                while((line = this.reader.readLine()) != null) {
                    this.output.add(line + "\n");
                }
                this.reader.close();
            }
            catch (IOException e) {
                // TODO
                System.err.println("ERROR: " + e.getMessage());
            }
        }

        public List<String> getOuput() {
            return this.output;
        }
    }
}

Answer 2

Don't print it as a String but give the CommandExecuter an optional OutputStream (in your Case you pass System.out as the argument) and write it to that stream.

In your current program the Main Method will execute the program and won't print anything (hang) until your run method returns something.