finding accuracy of tensorflow model

Question

I was trying to find the accuracy after training this simple linear model with sigmoid function:

import numpy as np
import tensorflow as tf
import _pickle as cPickle

with open("var_x.txt", "rb") as fp:   # Unpickling
    var_x = cPickle.load(fp)

with open("var_y.txt", "rb") as fp:   # Unpickling
    var_y = cPickle.load(fp)

with open("var_x_test.txt", "rb") as fp:   # Unpickling
    var_x_test = cPickle.load(fp)

with open("var_y_test.txt", "rb") as fp:   # Unpickling
    var_y_test = cPickle.load(fp)

def model_fn(features, labels, mode):
  # Build a linear model and predict values
  W = tf.get_variable("W", [4], dtype=tf.float64)
  b = tf.get_variable("b", [1], dtype=tf.float64)
  y = tf.sigmoid( tf.reduce_sum(W*features['x']) + b)
  if mode == tf.estimator.ModeKeys.PREDICT:
    return tf.estimator.EstimatorSpec(mode=mode, predictions=y)

  loss = tf.reduce_sum(tf.square(y - labels))

  global_step = tf.train.get_global_step()
  optimizer = tf.train.GradientDescentOptimizer(0.01)
  train = tf.group(optimizer.minimize(loss),
                   tf.assign_add(global_step, 1))

  return tf.estimator.EstimatorSpec(
      mode=mode,
      predictions=y,
      loss=loss,
      train_op=train)

estimator = tf.estimator.Estimator(model_fn=model_fn)

x_train = np.array(var_x)
y_train = np.array(var_y)
x_test = np.array(var_x_test)
y_test = np.array(var_y_test)

input_fn = tf.estimator.inputs.numpy_input_fn(
    {"x": x_train}, y_train, batch_size=4, num_epochs=60, shuffle=True)

estimator.train(input_fn=input_fn, steps=1000)

test_input_fn= tf.estimator.inputs.numpy_input_fn(
    x ={"x":np.array(x_test)},
    y=np.array(y_test),
    num_epochs=1,
    shuffle=False
    )

accuracy_score = estimator.evaluate(input_fn=test_input_fn["accuracy"])

print(accuracy_score)

But the dictionary doesn't have an "accuracy" key. How do I find it? Also, how do I use tensorboard to track the accuracy after each step?

Thank you in advance, the tensorflow tutorial is very bad at explaining.

Answer 1

test_results = {}

test_results['model'] = model.evaluate(
    test_features, test_labels, verbose=0)

print(f" Accuracy: {test_results}")

Answer 2

You need to create the accuracy yourself in model_fn using tf.metrics.accuracy and pass it to eval_metric_ops that will be returned by the function.

def model_fn(features, labels, mode):
    # define model...
    y = tf.nn.sigmoid(...)
    predictions = tf.cast(y > 0.5, tf.int64)
    eval_metric_ops = {'accuracy': tf.metrics.accuracy(labels, predictions)}
    #...
    return tf.estimator.EstimatorSpec(mode=mode, train_op=train_op, 
        loss=loss, eval_metric_ops=eval_metric_ops)

Then the output of estimator.evaluate() will contain an accuracy key that will hold the accuracy computed on the validation set.

metrics = estimator.evaluate(test_input_fn)
print(metrics['accuracy'])