Node.js WriteStream Unknown Encoding error

Question

I've created a module that parses NGinx logs, and now i am writing a command tool that uses it. My problem is that I allow a whole directory to be parsed, which is not a problem on the reading then parsing side, since I have a pool that qeueus up the reads and parsing, however, on the command line tool, I am allowing to rewrite the logs in a different format, JSON for now, - ok I will cut to the chase, I have writen this Writer object that will keep references of all the WriteStreams (wstreams[readFilePath] (I know I am using the readFilePath, this is just a key to look up), and there is also a global reference of all the reading streams through the module exposed object Parser.rstreams[readFilePath]

// creating a writer to handle the data buffering from the parser's readstreams
writer = {
wstreams: {},
append: function(data, wfile, rfile){
    console.log(JSON.stringify(this.wstreams[rfile]));
    if(this.wstreams[rfile] 
        && (this.wstreams[rfile].write(data, wfile) == false) // <-- crashing here 
        && parser.rstreams[rfile]
        && parser.rstreams[rfile].pause){
            console.log('Pausing: ' + rfile);
            parser.rstreams[rfile].pause();
    }
},
addStream: function(wfile, rfile){
    var wstream = fs.createWriteStream(wfile, {'flags': 'w', 'encoding':'utf8', 'mode': '0666'});
    console.log("stream added: " + wfile + " r: " + rfile);
    this.wstreams[rfile] = wstream;
    this.wstreams[rfile].on('drain', function(){
        if(parser.rstreams[rfile]
            && parser.rstreams[rfile].readable 
            && parser.rstreams[rfile].resume){
                console.log('Drained: ' + rfile);
                parser.rstreams[rfile].resume();
        }
    });
  }
}

When the a writeStream is trying to write data, it's throwing an Unknown Encoding exception, which makes no sens since it defaults to utf8, and even if I pass the optional encoding, it will do the same thing, I tried ut8, utf-8, and ascii

{"path":"/Users/akhoury/code/rk/ginx/bin/here.json","fd":8,"writable":true,"flags":"w","encoding":"utf8","mode":"0666","bytesWritten":0,"busy":false,"_queue":[],"_events":{}}
[GINX][ERROR][uncaughtException] Error: Unknown encoding
[GINX-DEBUG] Exiting - 0 {file:cursor} record(s) stored in /Users/akhoury/code/rk/ginx/tmp/stored.cursors

/Users/akhoury/code/rk/ginx/lib/ginx.js:453
throw err;
      ^
Error: Unknown encoding
at Buffer.write (buffer.js:382:13)
at new Buffer (buffer.js:261:26)
at WriteStream.write (fs.js:1548:12)
at Object.writer.append (/Users/akhoury/code/rk/ginx/bin/ginx.js:95:38)
at /Users/akhoury/code/rk/ginx/bin/ginx.js:152:16
at Ginx.eval [as hardParseLine] (eval at generateParseLine (/Users/akhoury/code/rk/ginx/lib/ginx.js:59:21))
at streamData (/Users/akhoury/code/rk/ginx/lib/ginx.js:179:13)
at Ginx.parseFile.fs.stat.stream.on.streamEnd.cursor (/Users/akhoury/code/rk/ginx/lib/ginx.js:346:28)
at EventEmitter.emit (events.js:93:17)
at ReadStream._emitData (fs.js:1365:10)

I even JSON.stringify the stream to see what's in it, and it looks fine.

I looked at the source of buffer.js and it doesn't make sense, that error should occur when the encoding is none of the allowed list https://github.com/joyent/node/blob/master/lib/buffer.js :50's

then I have a loop that will read the directory, if directory, and then write.addStream(outputfile, inputfile)

if (stats.isDirectory()) {
fs.mkdir(output, function () {
    fs.readdir(input, function (err, files) {
        if (err) error(err);
        files.forEach(function (wfile) {
            wfile = path.join(output, file);
            rfile = path.join(input, file);
            console.log("W:"+ wfile + " R: " + rfile);
            //prepend the JSON openings for each new file before we go on.
            if (isNewFile(rfile)) {
                fs.writeFileSync(wfile, "{[", 'utf8');
            }
            writer.addStream(wfile, rfile); // <-- adding the stream to writer here
        });
        processDirectory(input, output);
    });
});
} else if (stats.isFile()) {
if (isNewFile(input)) {
    fs.writeFile(output, "{[", 'utf8', function () {
        writer.addStream(output, input);
        processFile(input, output);
    });
} else {
    writer.addStream(output, input);
    processFile(input, output);
  }
}

then in processFile and processDirectory, everytime I receive a rowCallback, which means a line has been parsed, I use writer.append

// process file parsing to JSON output
function processFile(input, ouput) {
parser.parseFile(input,

function (err, row) {
    if (err) error(err);
    writer.append(ifLastRow(row), output, row.__file);
},

function (err, rfile) {
    if (err) error(err);
    //close the JSON array
    writer.append("]}", output, file);
});

}

// process directory parsing to JSON outputs
function processDirectory(input, output) {
parser.parseDir(input,

function (err, row) {
    if (err) error(err);
    var fname = row.__fname;
    writer.append(ifLastRow(row), path.join(output, fname), row.__file);
},

function (err, rfile) {
    if (err) error(err);
    var wfile = path.join(output, rfile.substring(rfile.lastIndexOf(path.sep) + 1));
    //close the JSON array
    writer.append("]}", wfile, rfile);
},

function (err, filesCount) {
    if (err) error(err);
 });
}

Can anyone see what am I doing wrong here? am I creating the stream the wrong way?

I know that's a lot to read but I didn't want to be too general. Thank you.

Answer 1

The problem is you are passing a filename as the second parameter to stream.write(), but the second param to .write() is an optional encoding (see previous link).

The error is because it is trying to use the filename as an encoding, which is an Unknown encoding. If data is a buffer, then let it determine the encoding based on the buffer.

A write stream is tied to a file, so you don't need to pass the filename on each write. Try changing:

    && (this.wstreams[rfile].write(data, wfile) == false)

To:

    && (this.wstreams[rfile].write(data) == false)