Ensemble of custom and pre-trained model gives run-time error

Question

I'm attempting to create an ensemble of a custom CNN and pre-trained VGG16 for a medical image classification task using Keras with Tensorflow backend (TF: 1.9.0 and Keras: 2.1.6). The code is as given below:

#load libraries
from keras.models import Model, Input
from keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D, Activation, Average, Dense
from load_data import load_resized_training_data, load_resized_validation_data
from load_data import load_resized_test_data
from keras.losses import categorical_crossentropy
from keras.callbacks import ModelCheckpoint, TensorBoard
from keras.optimizers import Adam
from keras.applications.vgg16 import VGG16
import numpy as np
################################################################################
#load data
batch_size = 8
num_epochs = 1
img_rows= 224
img_cols = 224
num_channels = 3
num_classes = 2

X_train, Y_train = load_resized_training_data(img_rows, img_cols)
X_valid, Y_valid = load_resized_validation_data(img_rows, img_cols)
X_test, Y_test = load_resized_test_data(img_rows, img_cols)
print(X_train.shape, Y_train.shape, X_valid.shape, Y_valid.shape,X_test.shape, Y_test.shape)
X_train = X_train.astype('float32')
X_valid = X_valid.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_valid /= 255
X_test /= 255 
###############################################################################  
'''Since the two models work with the data of the same shape, 
it makes sense to define a single input layer that will be used by every model.'''
input_shape = X_train[0,:,:,:].shape
print(input_shape) # 224, 224, 3
model_input = Input(shape=input_shape)
print(model_input) # Tensor("input_1:0", shape=(?, 224, 224, 3), dtype=float32)

###############################################################################
'''define the first model: a simple sequential model in the form of functional api'''

x = Conv2D(16, kernel_size=(3, 3), activation='relu')(model_input)
x = MaxPooling2D((2, 2))(x)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = MaxPooling2D((2, 2))(x)
x = Conv2D(64, (3, 3), activation='relu')(x)
x = MaxPooling2D((2, 2))(x)
x = Conv2D(128, (3, 3), activation='relu')(x)
x = MaxPooling2D((2, 2))(x)
x = Conv2D(256, (3, 3), activation='relu')(x)
x = MaxPooling2D((2, 2))(x)
x = Conv2D(512, (3, 3), activation='relu')(x)
x = Conv2D(2, (1, 1))(x)
x = GlobalAveragePooling2D()(x)
x = Activation(activation='softmax')(x)

custom_model = Model(inputs=model_input, outputs=x, name='custom_cnn')
###############################################################################
def compile_and_train(model, num_epochs): 

    model.compile(loss=categorical_crossentropy, optimizer=Adam(), metrics=['acc']) 
    filepath = 'weights/' + model.name + '.{epoch:02d}-{val_acc:.2f}.hdf5'
    checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_weights_only=True, save_best_only=True, mode='auto', period=1)
    tensor_board = TensorBoard(log_dir='logs/', histogram_freq=0, batch_size=batch_size)
    history = model.fit(X_train, Y_train, batch_size=batch_size, epochs=num_epochs, verbose=1, callbacks=[checkpoint, tensor_board], validation_data=(X_valid, Y_valid))
    return history

#compile and train the model
_ = compile_and_train(custom_model, num_epochs=num_epochs)
###############################################################################
def evaluate_error(model):
    pred = model.predict(X_test, batch_size = batch_size)
    pred = np.argmax(pred, axis=1)
    pred = np.expand_dims(pred, axis=1) # make same shape as y_test
    error = np.sum(np.not_equal(pred, Y_test)) / Y_test.shape[0]  

    return error

evaluate_error(custom_model)
###############################################################################
'''second model is a pretrained vgg16 model initialized with imagenet weights and going to be trained from the first layer'''

vgg16_model = VGG16(weights='imagenet', include_top=False, input_shape=(img_rows, img_cols, 3))
x = vgg16_model.output
x = GlobalAveragePooling2D()(x)
predictions = Dense(num_classes, activation='softmax')(x)
vgg16_custom_model = Model(inputs=vgg16_model.input, outputs=predictions, name='vgg16_cnn')

#compile and train the model
_ = compile_and_train(vgg16_custom_model, num_epochs=num_epochs)

#Evaluate the model by calculating the error on the test set
evaluate_error(vgg16_custom_model)
###############################################################################
custom_model.load_weights('weights/custom_cnn.01-0.60.hdf5')
vgg16_custom_model.load_weights('weights/vgg16_cnn.01-0.50.hdf5')

models = [custom_model, vgg16_custom_model]
###############################################################################
def ensemble(models, model_input):

    outputs = [model.outputs[0] for model in models]
    y = Average()(outputs)

    model = Model(inputs=model_input, outputs=y, name='ensemble')

    return model

ensemble_model = ensemble(models, model_input)
evaluate_error(ensemble_model)
###############################################################################

The code works all fine until the creation of the ensemble definition. When defining the ensemble I get the following error:

RuntimeError: Graph disconnected: cannot obtain value for tensor Tensor("input_7:0", shape=(?, 224, 224, 3), dtype=float32) at layer "input_7". The following previous layers were accessed without issue: []

I am not sure whether I can pass model_input to the pre-trained VGG16 in this manner. How the ensemble definition needs to be modified?

Answer 1

The problem is that the input of VGG model is not being fed by the input layer of ensemble_model. To resolve this issue, you need to modify the definition of ensemble_model and create a new input layer which is then passed to the two models:

def ensemble(models):
    input_img = Input(shape=input_shape)

    outputs = [model(input_img) for model in models] # get the output of model given the input image
    y = Average()(outputs)

    model = Model(inputs=input_img, outputs=y, name='ensemble')
    return model

ensemble_model = ensemble(models)