Why does one file object flush, but the other one doesn't?

Question

I wanted a file object that flushes out straight to file as data is being written, and wrote this:

class FlushingFileObject(file):
    def write(self,*args,**kwargs):
        return_val= file.write(self,*args,**kwargs)
        self.flush()
        return return_val

    def writelines(self,*args,**kwargs):
        return_val= file.writelines(self,*args,**kwargs)
        self.flush()
        return return_val

but interestingly it doesn't flush as I write to it, so I tried a few things including this:

class FlushingFileObject(object):
    def __init__(self,*args,**kwargs):
        self.file_object= file(*args,**kwargs)

    def __getattr__(self, item):
        return getattr(self.file_object,item)

    def write(self,*args,**kwargs):
        return_val= self.file_object.write(*args,**kwargs)
        self.file_object.flush()
        return return_val

    def writelines(self,*args,**kwargs):
        return_val= self.file_object.writelines(*args,**kwargs)
        self.file_object.flush()
        return return_val

which does flush.

Why doesn't subclassing file work in this instance?

Answer 1

Great question.

This happens because Python optimizes calls to write on file objects by bypassing the Python-level write method and calling fputs directly.

To see this in action, consider:

$ cat file_subclass.py
import sys
class FileSubclass(file):
    def write(self, *a, **kw):
        raise Exception("write called!")
    writelines = write
sys.stdout = FileSubclass("/dev/null", "w")
print "foo"
sys.stderr.write("print succeeded!\n")
$ python print_magic.py
print succeeded!

The write method was never called!

Now, when the object isn't a subclass of file, things work as expected:

$ cat object_subclass.py
import sys
class ObjectSubclass(object):
    def __init__(self):
        pass
    def write(self, *a, **kw):
        raise Exception("write called!")
    writelines = write
sys.stdout = ObjectSubclass()
print "foo"
sys.stderr.write("print succeeded!\n")
$ python object_subclass.py
Traceback (most recent call last):
  File "x.py", line 13, in <module>
    print "foo"
  File "x.py", line 8, in write
    raise Exception("write called!")
Exception: write called!

Digging through the Python source a bit, it looks like the culprit is the PyFile_WriteString function, called by the print statement, which checks to see whether the object being written to is an instance of file, and if it is, bypasses the object's methods and calls fputs directly:

int
PyFile_WriteString(const char *s, PyObject *f)
{

    if (f == NULL) {
        /* … snip … */
    }
    else if (PyFile_Check(f)) { //-- `isinstance(f, file)`
        PyFileObject *fobj = (PyFileObject *) f;
        FILE *fp = PyFile_AsFile(f);
        if (fp == NULL) {
            err_closed();
            return -1;
        }
        FILE_BEGIN_ALLOW_THREADS(fobj)
        fputs(s, fp); //-- fputs, bypassing the Python object entirely
        FILE_END_ALLOW_THREADS(fobj)
        return 0;
    }
    else if (!PyErr_Occurred()) {
        PyObject *v = PyString_FromString(s);
        int err;
        if (v == NULL)
            return -1;
        err = PyFile_WriteObject(v, f, Py_PRINT_RAW);
        Py_DECREF(v);
        return err;
    }
    else
        return -1;
}