tf.constant and tf.placeholder behave differently

Question

I want to wrap the tf.metrics around a Sonnet module for measuring performance of each batch, and the following is the work I have done:

import tensorflow as tf
import sonnet as snt

class Metrics(snt.AbstractModule):
    def __init__(self, indicator, summaries = None, name = "metrics"):
        super(Metrics, self).__init__(name = name)
        self._indicator = indicator
        self._summaries = summaries

    def _build(self, labels, logits):
        if self._indicator == "accuracy":
            metric, metric_update = tf.metrics.accuracy(labels, logits)
            with tf.control_dependencies([metric_update]):
                outputs = tf.identity(metric)
        elif self._indicator == "precision":
            metric, metric_update = tf.metrics.precision(labels, logits)
            with tf.control_dependencies([metric_update]):
                outputs = tf.identity(metric)
        elif self._indicator == "recall":
            metric, metric_update = tf.metrics.recall(labels, logits)
            with tf.control_dependencies([metric_update]):
                outputs = tf.identity(metric)
        elif self._indicator == "f1_score":
            metric_recall, metric_update_recall = tf.metrics.recall(labels, logits)
            metric_precision, metric_update_precision = tf.metrics.precision(labels, logits)
            with tf.control_dependencies([metric_update_recall, metric_update_precision]):
                outputs = 2.0 / (1.0 / metric_recall + 1.0 / metric_precision)
        else:
            raise ValueError("unsupported metrics")

        if type(self._summaries) == list:
            self._summaries.append(tf.summary.scalar(self._indicator, outputs))

        return outputs

However, when I want to test the module, the following code works:

def test3():
    import numpy as np

    labels = tf.constant([1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], tf.int32)
    logits = tf.constant([1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], tf.int32)

    metrics = Metrics("accuracy")
    accuracy = metrics(labels, logits)

    metrics2 = Metrics("f1_score")
    f1_score = metrics2(labels, logits)

    writer = tf.summary.FileWriter("utils-const", tf.get_default_graph())
    with tf.Session() as sess:
        sess.run([tf.global_variables_initializer(), tf.local_variables_initializer()])

        accu, f1 = sess.run([accuracy, f1_score])
        print(accu)
        print(f1)

    writer.close()

However the following code does NOT work:

def test4():
    from tensorflow.python import debug as tf_debug
    import numpy as np

    tf_labels = tf.placeholder(dtype=tf.int32, shape=[None])
    tf_logits = tf.placeholder(dtype=tf.int32, shape=[None])

    labels = np.array([1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], np.int32)
    logits = np.array([1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], np.int32)

    metrics = Metrics("accuracy")
    accuracy = metrics(tf_labels, tf_logits)

    metrics2 = Metrics("f1_score")
    f1_score = metrics2(tf_labels, tf_logits)

    writer = tf.summary.FileWriter("utils-feed", tf.get_default_graph())
    with tf.Session() as sess:
        sess.run([tf.global_variables_initializer(), tf.local_variables_initializer()])

        sess = tf_debug.LocalCLIDebugWrapperSession(sess)

        accu, f1 = sess.run([accuracy, f1_score], feed_dict = {tf_labels: labels, tf_logits: logits})
        print(accu)
        print(f1)

    writer.close()

The output of test3() is correct, 0.88. The output of test4() is wrong, 0.0. However, they should be equivalent.

Anyone has any idea?

Answer 1

Are you sure it is not the tf.constant version that fails? I find tf.metrics having a weird behavior in combination with tf.constant:

import tensorflow as tf

a = tf.constant(1.)
mean_a, mean_a_uop = tf.metrics.mean(a)
with tf.control_dependencies([mean_a_uop]):
  mean_a = tf.identity(mean_a)

sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
tf.local_variables_initializer().run()

for _ in range(10):
  print(sess.run(mean_a))

returns, when run on the GPU,

0.0
2.0
1.5
1.3333334
1.25
1.2
1.1666666
1.1428572
1.125
1.1111112

instead of 1s. It looks as if the count is lagging by one. (I am assuming the first value would be inf but is zero due to some conditions on count). A placeholder version of this code is running as expected on the other hand.

On the CPU, the behavior is even weirder, as the output is non-deterministic. Example of output:

0.0
1.0
1.0
0.75
1.0
1.0
0.85714287
0.875
1.0
0.9

Looks like a bug you could log on tensorflow's github repo. (Note that using running metrics on constants is less than useful -- but it is still a bug).

EDIT Now I also stumbled on weird examples with a tf.placeholder, it seems that tf.metrics has a bug that is unfortunately not limited to its use with tf.constants.