Robin on Linux

Testing performance of Tensorflow’s fixed-point-quantization on x86_64 cpu

Robin Dong 2018-06-22 No Comments

Google has published their quantization method on this paper. It use int8 to run feed-forward but float32 for back-propagation, since back-propagation need more accurate to accumulate gradients. I got a question right after reading the paper: why all the performance test works are on platform of mobile-phone (ARM architecture)? The quantization consequences of model in google’s method doesn’t only need addition and multiplication of int8 numbers, but also bit-shift operations. The AVX instruments set in Intel x86_64 architecture could accelerate MAC (Multiplication, Addition and aCcumulation), but couldn’t boost bit-shift operations.

To verify my suspicion, I wrote a model with ResNet-50 (float32) to classify CIFAR-100 dataset. After running a few epochs, I evaluate the speed of inference by using my ‘eval.py’. The result is:

Time: 5.58819s

1	Time: 5.58819s

Then, I follow these steps to add tf.contrib.quantize.create_training_graph() and tf.contrib.quantize.create_eval_graph() into my code. This time, the speed of inference is:

Time: 6.23221s

1	Time: 6.23221s

A little bit of disappointment. Using quantized (int8) version of model could not accelerate processing speed of x86 CPU. May be we need to find other more powerful quantization algorithm.

Appendix:

# eval.py
from input_data import Cifar100Data

import tensorflow as tf
import numpy as np

import resnet_v2
import argparse
import time
import sys

EVAL_SAMPLES = 10000
BATCH_SIZE = 10000

MODEL_PATH = './models/'
MODEL_NAME = 'cifar_resnet_50'

def cnn_part(images):
    print(images.shape)
    ivg, _ = resnet_v2.resnet_v2_50(images, 100)
    return ivg

def main(_):
    with tf.device('/cpu:0'):
        images = tf.placeholder(tf.float32, [BATCH_SIZE, 32, 32, 3])
        labels = tf.placeholder(tf.int64, [BATCH_SIZE])

    with tf.contrib.slim.arg_scope([tf.contrib.slim.conv2d],
                        weights_initializer = tf.truncated_normal_initializer(mean = 0, stddev = 0.1)):
        image_vector = cnn_part(images)

    loss = tf.losses.sparse_softmax_cross_entropy(labels = labels, logits = image_vector)
    loss = tf.reduce_mean(loss)
    opt = tf.train.AdamOptimizer(1e-3)
    train_op = tf.contrib.slim.learning.create_train_op(loss, opt)

    correct_prediction = tf.equal(tf.argmax(image_vector, 1), labels)
    correct_prediction = tf.cast(correct_prediction, tf.float32)
    accuracy = tf.reduce_mean(correct_prediction)

    data = Cifar100Data('/disk3/cifar/cifar-100-python/test')

    saver = tf.train.Saver()
    with tf.Session() as sess:
        with tf.gfile.FastGFile('./models/cifar_resnet_50_quant.pb') as fl:
            graph_def = tf.GraphDef()
            graph_def.ParseFromString(fl.read())
        tf.import_graph_def(graph_def, name = '')

        saver.restore(sess, MODEL_PATH + MODEL_NAME + '-' + str(FLAGS.epoch))

        batch = data.next_batch(BATCH_SIZE)

        for i in range(3):
            begin = time.time()
            res = sess.run(accuracy, feed_dict = {images: batch[0], labels: batch[1]})
            print("Time: %gs" % (time.time() - begin))
            print(res)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--epoch', type=str,
                        default='8',
                        help='Epoch of checkpoint for evaluation')
    FLAGS, unparsed = parser.parse_known_args()

    tf.app.run(main = main, argv = [sys.argv[0]] + unparsed)

# eval.py

from input_data import Cifar100Data

import tensorflow as tf

import numpy as np

import resnet_v2

import argparse

import time

import sys

EVAL_SAMPLES = 10000

BATCH_SIZE = 10000

MODEL_PATH = './models/'

MODEL_NAME = 'cifar_resnet_50'

def cnn_part(images):

print(images.shape)

ivg, _ = resnet_v2.resnet_v2_50(images, 100)

return ivg

def main(_):

with tf.device('/cpu:0'):

images = tf.placeholder(tf.float32, [BATCH_SIZE, 32, 32, 3])

labels = tf.placeholder(tf.int64, [BATCH_SIZE])

with tf.contrib.slim.arg_scope([tf.contrib.slim.conv2d],

weights_initializer = tf.truncated_normal_initializer(mean = 0, stddev = 0.1)):

image_vector = cnn_part(images)

loss = tf.losses.sparse_softmax_cross_entropy(labels = labels, logits = image_vector)

loss = tf.reduce_mean(loss)

opt = tf.train.AdamOptimizer(1e-3)

train_op = tf.contrib.slim.learning.create_train_op(loss, opt)

correct_prediction = tf.equal(tf.argmax(image_vector, 1), labels)

correct_prediction = tf.cast(correct_prediction, tf.float32)

accuracy = tf.reduce_mean(correct_prediction)

data = Cifar100Data('/disk3/cifar/cifar-100-python/test')

saver = tf.train.Saver()

with tf.Session() as sess:

with tf.gfile.FastGFile('./models/cifar_resnet_50_quant.pb') as fl:

graph_def = tf.GraphDef()

graph_def.ParseFromString(fl.read())

tf.import_graph_def(graph_def, name = '')

saver.restore(sess, MODEL_PATH + MODEL_NAME + '-' + str(FLAGS.epoch))

batch = data.next_batch(BATCH_SIZE)

for i in range(3):

begin = time.time()

res = sess.run(accuracy, feed_dict = {images: batch[0], labels: batch[1]})

print("Time: %gs" % (time.time() - begin))

print(res)

if __name__ == '__main__':

parser = argparse.ArgumentParser()

parser.add_argument('--epoch', type=str,

default='8',

help='Epoch of checkpoint for evaluation')

FLAGS, unparsed = parser.parse_known_args()

tf.app.run(main = main, argv = [sys.argv[0]] + unparsed)

Some tips about LaTeX

Robin Dong 2018-06-14 No Comments

1. After running ‘bibtex paper’, it reports

Too many commas in name 1 of "J.Chen, R.Monga, S.Bengio, R.Jozefowicz" for entry Revisit_SGD

1	Too many commas in name 1 of "J.Chen, R.Monga, S.Bengio, R.Jozefowicz" for entry Revisit_SGD

This is because we need to use ‘and’ to replace commas. After changing them

# Change 'author = "J.Chen, R.Monga, S.Bengio, R.Jozefowicz"' to
author = "J.Chen and R.Monga and S.Bengio and R.Jozefowicz"

1 2	# Change 'author = "J.Chen, R.Monga, S.Bengio, R.Jozefowicz"' to author = "J.Chen and R.Monga and S.Bengio and R.Jozefowicz"

The errors disappeared.

2. How to extend space between two rows in a table?
Answer:

\begingroup
\setlength{\tabcolsep}{10pt} % Default value: 6pt
\renewcommand{\arraystretch}{1.5} % Default value: 1
\begin{tabular}{ c c c }
First Row & -6 & -5 \\
Second Row & 4 & 10\\
Third Row & 20 & 30\\
Fourth Row & 100 & -30\\
\end{tabular}
\endgroup

\begingroup

\setlength{\tabcolsep}{10pt} % Default value: 6pt

\renewcommand{\arraystretch}{1.5} % Default value: 1

\begin{tabular}{ c c c }

First Row & -6 & -5 \\

Second Row & 4 & 10\\

Third Row & 20 & 30\\

Fourth Row & 100 & -30\\

\end{tabular}

\endgroup

3. Problem: Can’t upload .bib file in arXiv.org
Answer: run ‘pdflatex paper’ to generate paper.aux from paper.tex, and then run ‘bibtex paper’ to convert paper.bib to paper.bbl. Now we could upload .bbl file to arXiv.

4. Problem: When select ‘Tools’–>’Check Spelling…’ in texStudio, it report “No dictionary Available”.
Answer: Download english dictionary from https://extensions.openoffice.org/en/download/1471, change suffix from ‘oxt’ to ‘zip’ and unzip it. In ‘preferences’ of texStudio, set dictionary path to the unzip directory. (ref)

After solved all these problems, I eventually submit my paper here: https://arxiv.org/abs/1806.03925

Use pandas and matplotlib to draw line chart

Robin Dong 2018-06-08 No Comments

I have two CSV files. Their content looks like:

3,7578.8374
4,4911.78
5,4922.014
6,3158.1414
7,2656.271
8,2520.162
9,1659.447
10,2295.329
...

3,7578.8374

4,4911.78

5,4922.014

6,3158.1414

7,2656.271

8,2520.162

9,1659.447

10,2295.329

...

The simplest way to load and draw them is by using pandas and matplotlib.

import matplotlib.pyplot as plt
import pandas as pd

one = pd.read_csv('./my1.csv', names = ['step', 'loss'])
two = pd.read_csv('./my2.csv', names = ['step', 'loss'])

plt.plot(one['step'], one['loss'], label = 'my1')
plt.plot(two['step'], two['loss'], label = 'my2')

plt.ylabel("Loss")
plt.xlabel("Training step")
plt.legend(prop = {'size': 10})      # Set size of legend to be smaller

plt.show()

import matplotlib.pyplot as plt

import pandas as pd

one = pd.read_csv('./my1.csv', names = ['step', 'loss'])

two = pd.read_csv('./my2.csv', names = ['step', 'loss'])

plt.plot(one['step'], one['loss'], label = 'my1')

plt.plot(two['step'], two['loss'], label = 'my2')

plt.ylabel("Loss")

plt.xlabel("Training step")

plt.legend(prop = {'size': 10}) # Set size of legend to be smaller

plt.show()

The figure draw out by this snippet is shown below:

Hard training works in deep learning

Robin Dong 2018-06-01 No Comments

This week, I was trying to train two deep-learning models. They are different from my previous training job: they are really hard to converge to a small ‘loss’.

The first model is about bird image classification. Previously we wrote a modified Resnet-50 model by using MXNet and could use it to reach 78% evaluation-accuracy. But after we rewrote the same model by using Tensorflow, it could only reach 50% evaluation-accuracy, which seems very weird. The first thing that in my mind is that it’s a regularization problem, so I randomly pad/crop and rotate the training images:

  image = tf.image.resize_image_with_crop_or_pad(image, IMAGE_HEIGHT + 80, IMAGE_WIDTH + 80)
  image = tf.contrib.image.rotate(image, tf.random_uniform([1], minval = -math.pi / 3.0, maxval = math.pi / 3.0))
  image = tf.random_crop(image, [IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_CHANNELS])

image = tf.image.resize_image_with_crop_or_pad(image, IMAGE_HEIGHT + 80, IMAGE_WIDTH + 80)

image = tf.contrib.image.rotate(image, tf.random_uniform([1], minval = -math.pi / 3.0, maxval = math.pi / 3.0))

image = tf.random_crop(image, [IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_CHANNELS])

By data augmentation, the evaluation accuracy rise to about 60%, but still far from the result of MXNet.
Then I change the optimizer from AdamOptimizer to GradientDescentOptimizer, since my colleague tell me the AdamOptimizer is too powerful that it tends to cause overfit. And I also add ‘weight_decay’ for my Resnet-50 model. This time, the evaluation accuracy shrived to 76%. The affection of ‘weight_decay’ is significantly positive.

The second model is about object detection. We just use the example of Tensorflow’s model library. It includes many cutting-edge models to implement object detection. I just want to try SSD(Single Shot Detection) on MobileNetV2:

python object_detection/train.py \
  --logtostderr \
  --pipeline_config_path=/disk3/donghao/models/research/object_detection/samples/configs/ssd_mobilenet_v2_coco.config \
  --train_dir=/disk3/donghao/myckpt/ \
  --num_clones=2

python object_detection/train.py \

--logtostderr \

--pipeline_config_path=/disk3/donghao/models/research/object_detection/samples/configs/ssd_mobilenet_v2_coco.config \

--train_dir=/disk3/donghao/myckpt/ \

--num_clones=2

The loss is rapidly reducing from hundreds to twelve, but stay at eleven for a very long time. The loss looks like will stay here forever. Then I begin to adjust hyper-parameters. After testing several learning rates and optimizers, the results doesn’t change at all.
Eventually, I noticed that the loss doesn’t stay forever, it WILL REDUCE AT LAST. For some tasks such as classification, its loss will converge significantly. But for other tasks such as object detection, its loss will shrink at extremely slow speed. Use AdamOptimizer and small learning rate is a better choice for this type of task.

To check abnormal loss value when training a new model

Robin Dong 2018-05-25 No Comments

Yesterday I wrote a Tensorflow program to train CIFAR100 dataset with Resnet-50 model. But when the training begin, I saw the ‘loss’ of classification is abnormally big and didn’t reduce at all:

loss[2.6032338e+25]
loss[2.5617402e+25]
loss[3.3851871e+25]
loss[3.092054e+25] 
...

loss[2.6032338e+25]

loss[2.5617402e+25]

loss[3.3851871e+25]

loss[3.092054e+25]

...

Firstly, I thought the code for processing dataset may be wrong. But after print out the data in console, the loading input data seems all right. Then I print all the value of tensors right after initialization of model. And these value seems correct either.
Without other choices, I began to check the initializer in Tensorflow code:

    with slim.arg_scope([slim.conv2d],
                        weights_initializer = tf.truncated_normal_initializer(mean = 0, stddev = 0.1)):
      img_inf, _ = resnet_v2.resnet_v2_50(image, NUM_CLASSES)

with slim.arg_scope([slim.conv2d],

weights_initializer = tf.truncated_normal_initializer(mean = 0, stddev = 0.1)):

img_inf, _ = resnet_v2.resnet_v2_50(image, NUM_CLASSES)

If the loss is too big, maybe I could decrease the initial value of tensors in model? Then I change ‘mean’ from ‘0’ to ‘0.1’ for ‘slim.conv2d’:

                        weights_initializer = tf.truncated_normal_initializer(mean = 0.001, stddev = 1)):

1	weights_initializer = tf.truncated_normal_initializer(mean = 0.001, stddev = 1)):

But the loss seems more crazy:

loss[1.1468245e+29]
loss[1.6610325e+29]
loss[1.1840615e+29]
...

loss[1.1468245e+29]

loss[1.6610325e+29]

loss[1.1840615e+29]

...

I have to change ‘mean’ and ‘stddev’ again:

                        weights_initializer = tf.truncated_normal_initializer(mean = 0.001, stddev = 1)):

1	weights_initializer = tf.truncated_normal_initializer(mean = 0.001, stddev = 1)):

This time, the loss seems correct now.

loss[1215.8724]
loss[1023.67676]
loss[583.6274]
...

loss[1215.8724]

loss[1023.67676]

loss[583.6274]

...

This is the first time I saw that initialized value could make the training accuracy so different.

An example for running operation before fetching data in Tensorflow

Robin Dong 2018-05-18 No Comments

In tensorflow, what should we do if we want run something before fetching data (such as, using queue in tensorflow)? Here is an example tested by myself:

import tensorflow as tf
from tensorflow.python.framework import ops

def my_func(x):
    print("hello")
    return x

queue = tf.FIFOQueue(10, "float32")
init  = queue.enqueue_many(([1, 2, 3],))

my_op = tf.py_func(my_func, [1.0], tf.float32)

with ops.control_dependencies([my_op]):
    inc = queue.enqueue([1],)

with tf.Session() as sess:
    sess.run(init)
    print("Before Enqueue")
    sess.run(inc)
    print("After Enqueue")

import tensorflow as tf

from tensorflow.python.framework import ops

def my_func(x):

print("hello")

return x

queue = tf.FIFOQueue(10, "float32")

init = queue.enqueue_many(([1, 2, 3],))

my_op = tf.py_func(my_func, [1.0], tf.float32)

with ops.control_dependencies([my_op]):

inc = queue.enqueue([1],)

with tf.Session() as sess:

sess.run(init)

print("Before Enqueue")

sess.run(inc)

print("After Enqueue")

It will print

Before Enqueue
hello
After Enqueue

Before Enqueue

hello

After Enqueue

Successfully, we add an operation before enqueue a item into queue.

Why my model doesn’t converge?

Robin Dong 2018-05-11 No Comments

To use Resnet-50 to run CIFAR100 dataset, I wrote a program by using Tensorflow. But when running it, the loss seems keeping in about 4.5~4.6 forever:

step: 199, loss: 4.61291, accuracy: 0
step: 200, loss: 4.60952, accuracy: 0
step: 201, loss: 4.60763, accuracy: 0
step: 202, loss: 4.62495, accuracy: 0
step: 203, loss: 4.62312, accuracy: 0
step: 204, loss: 4.60703, accuracy: 0
step: 205, loss: 4.60947, accuracy: 0
step: 206, loss: 4.59816, accuracy: 0
step: 207, loss: 4.62643, accuracy: 0
step: 208, loss: 4.59422, accuracy: 0
...

step: 199, loss: 4.61291, accuracy: 0

step: 200, loss: 4.60952, accuracy: 0

step: 201, loss: 4.60763, accuracy: 0

step: 202, loss: 4.62495, accuracy: 0

step: 203, loss: 4.62312, accuracy: 0

step: 204, loss: 4.60703, accuracy: 0

step: 205, loss: 4.60947, accuracy: 0

step: 206, loss: 4.59816, accuracy: 0

step: 207, loss: 4.62643, accuracy: 0

step: 208, loss: 4.59422, accuracy: 0

...

After changed models (from Resnet to fully-connect-net), optimizers (from AdamOptimizer to AdagradOptimizer), and even learning rate (from 1e-3 to even 1e-7), the phenomena didn’t change at all.
Finally, I checked the loss and the output vector step by step, and found that the problem is not in model but dataset code:

    def next_batch(self, batch_size = 64):
        images = []
        labels = []
        for i in range(self.pos, self.pos + batch_size):
            image = self.data['data'][self.pos]
            image = image.reshape(3, 32, 32)
            image = image.transpose(1, 2, 0)
            image = image.astype(np.float32) / 255.0
            images.append(image)
            label = self.data['fine_labels'][self.pos]
            labels.append(label)

        if (self.pos + batch_size) >= CIFAR100_TRAIN_SAMPLES:
            self.pos = 0
        else:
            self.pos = self.pos + batch_size

        return [images, labels]

def next_batch(self, batch_size = 64):

images = []

labels = []

for i in range(self.pos, self.pos + batch_size):

image = self.data['data'][self.pos]

image = image.reshape(3, 32, 32)

image = image.transpose(1, 2, 0)

image = image.astype(np.float32) / 255.0

images.append(image)

label = self.data['fine_labels'][self.pos]

labels.append(label)

if (self.pos + batch_size) >= CIFAR100_TRAIN_SAMPLES:

self.pos = 0

else:

self.pos = self.pos + batch_size

return [images, labels]

Every batch of data have the same pictures and same labels! Than’t why the model didn’t converge. I should have used ‘i’ instead of ‘self.pos’ as index to fetch data and labels.

So in DeepLearning area, problems comes not only from models and hyper-parameters, but also dataset, or faulty codes…

Problem about using slim.batch_norm() of Tensorflow (second episode)

Robin Dong 2018-05-04 No Comments

In previous article, I have found out the reason. But how to resolve it on Multi-GPU-Training is still a question. As the suggestion of this issue in github, I tried two way to fix the problem:

First, rewrite my Averaging-Gradients-Training to learn tf.slim.create_train_op():

...
def create_train_grads(total_loss, optimizer):
  update_ops = set(ops.get_collection(ops.GraphKeys.UPDATE_OPS))
  with ops.control_dependencies(update_ops):
    barrier = control_flow_ops.no_op(name='update_barrier')
  total_loss = control_flow_ops.with_dependencies([barrier], total_loss)

  variables_to_train = tf_variables.trainable_variables()

  grads = optimizer.compute_gradients(total_loss, variables_to_train)
  return grads
...
          cross_entropy = tf.reduce_mean(cross_entropy)

          tf.get_variable_scope().reuse_variables()

          grads = create_train_grads(cross_entropy, opt)
          tower_grads.append(grads)
...
  grads = average_gradients(tower_grads)
  grad_updates = opt.apply_gradients(grads)

  with ops.name_scope('train_op'):
    # Ensure the train_tensor computes grad_updates.
    train_op = control_flow_ops.with_dependencies([grad_updates], cross_entropy)

  # Add the operation used for training to the 'train_op' collection
  train_ops = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
  if train_op not in train_ops:
    train_ops.append(train_op)

...

def create_train_grads(total_loss, optimizer):

update_ops = set(ops.get_collection(ops.GraphKeys.UPDATE_OPS))

with ops.control_dependencies(update_ops):

barrier = control_flow_ops.no_op(name='update_barrier')

total_loss = control_flow_ops.with_dependencies([barrier], total_loss)

variables_to_train = tf_variables.trainable_variables()

grads = optimizer.compute_gradients(total_loss, variables_to_train)

return grads

...

cross_entropy = tf.reduce_mean(cross_entropy)

tf.get_variable_scope().reuse_variables()

grads = create_train_grads(cross_entropy, opt)

tower_grads.append(grads)

...

grads = average_gradients(tower_grads)

grad_updates = opt.apply_gradients(grads)

with ops.name_scope('train_op'):

# Ensure the train_tensor computes grad_updates.

train_op = control_flow_ops.with_dependencies([grad_updates], cross_entropy)

# Add the operation used for training to the 'train_op' collection

train_ops = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)

if train_op not in train_ops:

train_ops.append(train_op)

But unfortunately, this didn’t work at all. The inference result was still a mess.

Then, another way, I use Asynchronous-Gradient-Training and tf.slim.create_train_op():

...
          cross_entropy = tf.reduce_mean(cross_entropy)

          train_op = tf.contrib.slim.learning.create_train_op(cross_entropy, opt)
          tower_ops.append(train_op)
...
  train_step = tf.group(*tower_ops)

...

cross_entropy = tf.reduce_mean(cross_entropy)

train_op = tf.contrib.slim.learning.create_train_op(cross_entropy, opt)

tower_ops.append(train_op)

...

train_step = tf.group(*tower_ops)

Now the inference works very well! And the training speed become a little bit faster than Averaging-Gradients-Training, for the Averaging Operation needs to wait multi gradients from multi GPUs.

Problem about using slim.batch_norm() of Tensorflow

Robin Dong 2018-04-27 No Comments

After using resnet_v2_50 in tensorflow/models, I found that the inference result is totally incorrect, though the training accuracy looks very well.
Firstly, I suspected the regularization of samples:

  image = tf.image.resize_image_with_crop_or_pad(image, IMAGE_HEIGHT + 66, IMAGE_WIDTH + 66)
  image = tf.random_crop(image, [IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_CHANNELS])
  image = tf.image.random_flip_left_right(image)

image = tf.image.resize_image_with_crop_or_pad(image, IMAGE_HEIGHT + 66, IMAGE_WIDTH + 66)

image = tf.random_crop(image, [IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_CHANNELS])

image = tf.image.random_flip_left_right(image)

Indeed I had extended the image to a too big size. But after I changing padding size to ’10’, the inference accuracy was still incorrect.
Then I checked the code about importing data:

# To avoid various formats of picture, I encode all image to 'jpeg' and write them as TFRecord
img = cv2.imread(file_name)                                          
raw_image = cv2.imencode('.jpeg', img)[1].tostring()
....
# When importing data from TFRecord
image = tf.image.decode_image(image)

# To avoid various formats of picture, I encode all image to 'jpeg' and write them as TFRecord

img = cv2.imread(file_name)

raw_image = cv2.imencode('.jpeg', img)[1].tostring()

....

# When importing data from TFRecord

image = tf.image.decode_image(image)

and changed my inference code as the data importing routines. But the problem still existed.

About one week past. Finally, I found this issue in Github. It explains all my questions: the cause is the slim.batch_norm(). After I adding these code to my program (learning from slim.create_train_op()):

update_ops = set(ops.get_collection(ops.GraphKeys.UPDATE_OPS))
with ops.control_dependencies(update_ops):
  barrier = control_flow_ops.no_op(name='update_barrier')
total_loss = control_flow_ops.with_dependencies([barrier], total_loss)
grads = optimizer.compute_gradients(total_loss)
...

update_ops = set(ops.get_collection(ops.GraphKeys.UPDATE_OPS))

with ops.control_dependencies(update_ops):

barrier = control_flow_ops.no_op(name='update_barrier')

total_loss = control_flow_ops.with_dependencies([barrier], total_loss)

grads = optimizer.compute_gradients(total_loss)

...

The inference accuracy is — still low. Without other choice, I removed all slim.batch_norm() in resnet_v2.py, and at this time inference accuracy becomes the same with training accuracy.
Looks problem partly been solved, but I still need to find out why sli.batch_norm() doesn’t work well in inference …

Experiment for distributed Tensorflow

Robin Dong 2018-04-11 No Comments

Here is my experimental code for distributed Tensorflow, which is learned from the example.

import tensorflow as tf
import argparse
import time

FLAGS = None

def main():
    print(tf.__version__)

    cluster_spec = tf.train.ClusterSpec({
        'worker': ['localhost:1829'],
        'ps': ['localhost:1057'],
        })
            
    if FLAGS.ps: 
        server = tf.train.Server(cluster_spec, job_name = 'ps', task_index = 0)
        server.join()
    else:   
        server = tf.train.Server(cluster_spec, job_name = 'worker', task_index = FLAGS.worker)
        print(server.target)
            
        with tf.device('/job:ps/task:0'):
            init = tf.constant_initializer([0])
            c = tf.get_variable('myc', shape = [], initializer = init)

        res = tf.add(c, 1)
        train_op = tf.assign(c, res)
    
        with tf.Session(target = server.target) as sess:
            c.initializer.run()
            while True:
                res = sess.run(train_op)
                print(res)
                time.sleep(1)
...

import tensorflow as tf

import argparse

import time

FLAGS = None

def main():

print(tf.__version__)

cluster_spec = tf.train.ClusterSpec({

'worker': ['localhost:1829'],

'ps': ['localhost:1057'],

})

if FLAGS.ps:

server = tf.train.Server(cluster_spec, job_name = 'ps', task_index = 0)

server.join()

else:

server = tf.train.Server(cluster_spec, job_name = 'worker', task_index = FLAGS.worker)

print(server.target)

with tf.device('/job:ps/task:0'):

init = tf.constant_initializer([0])

c = tf.get_variable('myc', shape = [], initializer = init)

res = tf.add(c, 1)

train_op = tf.assign(c, res)

with tf.Session(target = server.target) as sess:

c.initializer.run()

while True:

res = sess.run(train_op)

print(res)

time.sleep(1)

...

The important thing is that we need to use tf.assign() to push Variable back to Parameter Server. The operation ‘tf.add’ was about to run on the task0 of worker in this example. But if we deploy more complicated application by many tasks, things became weird: a pipeline operation sometimes even runs on ‘ps’ role! The official solution to this problem is using ‘tf.train.replica_device_setter()’， which will automatically deploy Variables to parameter servers and Operations (many replicas) to many workers. What did ‘tf.train.replica_device_setter()’ do? Let’s see the backbone code of its implementation:

def replica_device_setter(ps_tasks=0, ps_device="/job:ps",
                          worker_device="/job:worker", merge_devices=True,
                          cluster=None, ps_ops=None, ps_strategy=None):
...
  if ps_ops is None:
    # TODO(sherrym): Variables in the LOCAL_VARIABLES collection should not be
    # placed in the parameter server.
    ps_ops = ["Variable", "VariableV2", "VarHandleOp"]

  if not merge_devices:
    logging.warning(
        "DEPRECATION: It is recommended to set merge_devices=true in "
        "replica_device_setter")
  if ps_strategy is None:
    ps_strategy = _RoundRobinStrategy(ps_tasks)
  if not six.callable(ps_strategy):
    raise TypeError("ps_strategy must be callable")
  chooser = _ReplicaDeviceChooser(
      ps_tasks, ps_device, worker_device, merge_devices, ps_ops, ps_strategy)
  return chooser.device_function

def replica_device_setter(ps_tasks=0, ps_device="/job:ps",

worker_device="/job:worker", merge_devices=True,

cluster=None, ps_ops=None, ps_strategy=None):

...

if ps_ops is None:

# TODO(sherrym): Variables in the LOCAL_VARIABLES collection should not be

# placed in the parameter server.

ps_ops = ["Variable", "VariableV2", "VarHandleOp"]

if not merge_devices:

logging.warning(

"DEPRECATION: It is recommended to set merge_devices=true in "

"replica_device_setter")

if ps_strategy is None:

ps_strategy = _RoundRobinStrategy(ps_tasks)

if not six.callable(ps_strategy):

raise TypeError("ps_strategy must be callable")

chooser = _ReplicaDeviceChooser(

ps_tasks, ps_device, worker_device, merge_devices, ps_ops, ps_strategy)

return chooser.device_function

All the Variables will be counted as ‘ps_ops’, and the deploy strategy for Operations will be replication, for it’s called ‘_ReplicaDeviceChooser’.

def device_function(self, op):
...
    node_def = op if isinstance(op, node_def_pb2.NodeDef) else op.node_def
    if self._ps_tasks and self._ps_device and node_def.op in self._ps_ops:
      ps_device = pydev.DeviceSpec.from_string(self._ps_device)
    
      current_job, ps_job = current_device.job, ps_device.job
      if ps_job and (not current_job or current_job == ps_job):
        ps_device.task = self._ps_strategy(op)
  
      ps_device.merge_from(current_device)
      return ps_device.to_string()
        
    worker_device = pydev.DeviceSpec.from_string(self._worker_device or "")
    worker_device.merge_from(current_device)
    return worker_device.to_string()

def device_function(self, op):

...

node_def = op if isinstance(op, node_def_pb2.NodeDef) else op.node_def

if self._ps_tasks and self._ps_device and node_def.op in self._ps_ops:

ps_device = pydev.DeviceSpec.from_string(self._ps_device)

current_job, ps_job = current_device.job, ps_device.job

if ps_job and (not current_job or current_job == ps_job):

ps_device.task = self._ps_strategy(op)

ps_device.merge_from(current_device)

return ps_device.to_string()

worker_device = pydev.DeviceSpec.from_string(self._worker_device or "")

worker_device.merge_from(current_device)

return worker_device.to_string()

All the ‘op’ in ‘self._ps_ops’ will be put into ‘ps_device’.