Basic neural net in tensorflow

Question

I've been trying to implement a basic neural network in tensorflow, the input is just random data of ones/zeros in (x,y,z), however I want my network to output a 1 whenever x = 1 and output a 0 otherwise.

This is my network code

import tensorflow as tf
import numpy as np

x_data = np.array([[0,0,1],
         [0,1,1],
         [1,0,0],
         [0,1,0],
         [1,1,1],
         [0,1,1],
         [1,1,1]])

x_test = np.array([[1,1,1], [0,1,0], [0,0,0]])
y_data = np.array([0,0,1,0,1,0,1])


iters = 1000
learning_rate = 0.1
weights = {
'w1': tf.Variable(tf.random_normal([3, 5])),
'w2': tf.Variable(tf.random_normal([5, 1])),
}
bias = {
'b1': tf.Variable(tf.random_normal([5])),
'b2': tf.Variable(tf.random_normal([1])),
}

def predict(x, weights, bias):
    l1 = tf.add(tf.matmul(x, weights['w1']), bias['b1'])
    l1 = tf.nn.sigmoid(l1)
    out = tf.add(tf.matmul(l1, weights['w2']), bias['b2'])
    return out


x = tf.placeholder(tf.float32, shape=(None,3))
y = tf.placeholder(tf.float32, shape=(None))

pred = predict(x, weights, bias)

cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost)

init = tf.global_variables_initializer()

# graph
with tf.Session() as sess:
sess.run(init)

for i in range(0, iters):
    _, c = sess.run([optimizer, cost], feed_dict={x: x_data, y: y_data})
    if i % 100 == 0:
        print("cost: " + str(c))

print(sess.run(weights['w1']))
print(sess.run(pred, feed_dict={x: x_test}))

Which outputs

[-0.37119362]
[-0.23264697]
[-0.14701667]

However my test data should output [1,0,0], I'm really not sure whats wrong here. I've tried playing with the hyperparameters and looking on stackoverflow. I've also tried using softmax_cross_entropy as a cost function, although it gives me an error saying the logits is not the same shape as labels.

Does anyone know why this isn't outputting what I'm expecting?

Answer 1

First, you need to pass through an activation function (ie tf.nn.sigmoid) before the output.

Make sure that tf.nn.sigmoid_cross_entropy_with_logits takes the logits (before the sigmoid activation).

Also you had a shape issue with your input y_data that was (7) instead of (7, 1)

Here is a working version of your code:

import tensorflow as tf
import numpy as np

x_data = np.array([[0,0,1],
         [0,1,1],
         [1,0,0],
         [0,1,0],
         [1,1,1],
         [0,1,1],
         [1,1,1]])

x_test = np.array([[1,1,1], [0,1,0], [0,0,0]])
y_data = np.array([[0],[0],[1],[0],[1],[0],[1]])


iters = 1000
learning_rate = 0.1
weights = {
'w1': tf.Variable(tf.random_normal([3, 5])),
'w2': tf.Variable(tf.random_normal([5, 1])),
}
bias = {
'b1': tf.Variable(tf.random_normal([5])),
'b2': tf.Variable(tf.random_normal([1])),
}

def predict(x, weights, bias):
    l1 = tf.add(tf.matmul(x, weights['w1']), bias['b1'])
    l1 = tf.nn.sigmoid(l1)    
    out = tf.add(tf.matmul(l1, weights['w2']), bias['b2'])
    return out


x = tf.placeholder(tf.float32, shape=(None,3))
y = tf.placeholder(tf.float32, shape=(None,1))

pred = predict(x, weights, bias)
pred_postactivation = tf.nn.sigmoid(pred)

cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost)

init = tf.global_variables_initializer()

# graph
with tf.Session() as sess:
    sess.run(init)

    for i in range(0, iters):
        _, c = sess.run([optimizer, cost], feed_dict={x: x_data, y: y_data})
        if i % 100 == 0:
            print("cost: " + str(c))

    print(sess.run(weights['w1']))
    print(sess.run(pred_postactivation, feed_dict={x: x_test}))

Which outputs:

cost: 1.23954
cost: 0.583582
cost: 0.455403
cost: 0.327644
cost: 0.230051
cost: 0.165296
cost: 0.123712
cost: 0.0962315
cost: 0.0772587
cost: 0.0636141
[[ 0.94488049  0.78105074  0.81608331  1.75763154 -4.47565413]
 [-2.61545444  0.26020721  0.151407    1.33066297  1.00578034]
 [-1.2027328   0.05413296 -0.13530347 -0.39841765  0.16014417]]
[[ 0.92521071]
 [ 0.05481482]
 [ 0.07227208]]