diff --git a/RELEASE.md b/RELEASE.md index f9930e0630b38c..c78673777d6676 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,10 @@ +# Release 1.3.1 + +NOTE: TensorFlow 1.3.1 is a GitHub only release. The latest exported binaries are still version 1.3.0. + +## Bug Fixes and Other Changes +* Fixing the hash mismatch errors when building from source. + # Release 1.3.0 See also [TensorBoard 0.1.4](https://github.com/tensorflow/tensorboard/releases/tag/0.1.4) release notes. diff --git a/WORKSPACE b/WORKSPACE index 74ce13f4e88710..eb6f26b30358e0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -2,11 +2,11 @@ workspace(name = "org_tensorflow") http_archive( name = "io_bazel_rules_closure", - sha256 = "bc41b80486413aaa551860fc37471dbc0666e1dbb5236fb6177cb83b0c105846", - strip_prefix = "rules_closure-dec425a4ff3faf09a56c85d082e4eed05d8ce38f", + sha256 = "110fe68753413777944b473c25eed6368c4a0487cee23a7bac1b13cc49d3e257", + strip_prefix = "rules_closure-4af89ef1db659eb41f110df189b67d4cf14073e1", urls = [ - "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz", # 2017-06-02 - "https://github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz", + "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz", + "https://github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz", # 2017-08-28 ], ) diff --git a/tensorflow/contrib/cmake/external/gemmlowp.cmake b/tensorflow/contrib/cmake/external/gemmlowp.cmake index eee61ffd57bf60..c41ce43f29bcb1 100644 --- a/tensorflow/contrib/cmake/external/gemmlowp.cmake +++ b/tensorflow/contrib/cmake/external/gemmlowp.cmake @@ -14,7 +14,7 @@ # ============================================================================== include (ExternalProject) -set(gemmlowp_URL http://github.com/google/gemmlowp/archive/a6f29d8ac48d63293f845f2253eccbf86bc28321.tar.gz) +set(gemmlowp_URL http://mirror.bazel.build/github.com/google/gemmlowp/archive/a6f29d8ac48d63293f845f2253eccbf86bc28321.tar.gz) set(gemmlowp_HASH SHA256=75d40ea8e68b0d1644f052fffe8f14a410b2a73d40ccb859a95c0578d194ec26) set(gemmlowp_BUILD ${CMAKE_CURRENT_BINARY_DIR}/gemmlowp/src/gemmlowp) set(gemmlowp_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/gemmlowp/src/gemmlowp) diff --git a/tensorflow/contrib/legacy_seq2seq/python/ops/seq2seq.py b/tensorflow/contrib/legacy_seq2seq/python/ops/seq2seq.py index 23b4a73b23d28e..137523324830b3 100644 --- a/tensorflow/contrib/legacy_seq2seq/python/ops/seq2seq.py +++ b/tensorflow/contrib/legacy_seq2seq/python/ops/seq2seq.py @@ -311,10 +311,10 @@ def embedding_rnn_seq2seq(encoder_inputs, """Embedding RNN sequence-to-sequence model. This model first embeds encoder_inputs by a newly created embedding (of shape - [num_encoder_symbols x input_size]). Then it runs an RNN to encode + [num_encoder_symbols x embedding_size]). Then it runs an RNN to encode embedded encoder_inputs into a state vector. Next, it embeds decoder_inputs by another newly created embedding (of shape [num_decoder_symbols x - input_size]). Then it runs RNN decoder, initialized with the last + embedding_size]). Then it runs RNN decoder, initialized with the last encoder state, on embedded decoder_inputs. Args: diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 264d5d5952b5c3..0c862c739efd8f 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -65,7 +65,6 @@ config_setting( # will also need appropriate -mavx*, as required by specific op you use. name = "xsmm_backward", values = { - "define": "tensorflow_xsmm=1", "define": "tensorflow_xsmm_backward=1", }, ) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index ccb861c93a5d28..a9c0a657e37b40 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -20,7 +20,7 @@ limitations under the License. #define TF_MAJOR_VERSION 1 #define TF_MINOR_VERSION 3 -#define TF_PATCH_VERSION 0 +#define TF_PATCH_VERSION 1 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", // "-beta", "-rc", "-rc.1") diff --git a/tensorflow/docs_src/programmers_guide/meta_graph.md b/tensorflow/docs_src/api_guides/python/meta_graph.md similarity index 100% rename from tensorflow/docs_src/programmers_guide/meta_graph.md rename to tensorflow/docs_src/api_guides/python/meta_graph.md diff --git a/tensorflow/docs_src/programmers_guide/reading_data.md b/tensorflow/docs_src/api_guides/python/reading_data.md similarity index 93% rename from tensorflow/docs_src/programmers_guide/reading_data.md rename to tensorflow/docs_src/api_guides/python/reading_data.md index a7d9372053462b..ff8b4f1aa748c1 100644 --- a/tensorflow/docs_src/programmers_guide/reading_data.md +++ b/tensorflow/docs_src/api_guides/python/reading_data.md @@ -1,6 +1,9 @@ # Reading data -There are three main methods of getting data into a TensorFlow program: +Note: The preferred way to feed data into a tensorflow program is using the +@{$datasets$Datasets API}. + +There are three other methods of getting data into a TensorFlow program: * Feeding: Python code provides the data when running each step. * Reading from files: an input pipeline reads the data from files @@ -19,6 +22,9 @@ graph. Supply feed data through the `feed_dict` argument to a run() or eval() call that initiates computation. +Note: "Feeding" is the least efficient way to feed data into a tensorflow +program and should only be used for small experiments and debugging. + ```python with tf.Session(): input = tf.placeholder(tf.float32) @@ -51,6 +57,9 @@ A typical pipeline for reading records from files has the following stages: 7. *Optional* preprocessing 8. Example queue +Note: This section discusses implementing input pipelines useing the +queue-based APIs which can be cleanly replaced by the ${$datasets$Dataset API}. + ### Filenames, shuffling, and epoch limits For the list of filenames, use either a constant string Tensor (like @@ -405,7 +414,8 @@ This is only used for small data sets that can be loaded entirely in memory. There are two approaches: * Store the data in a constant. -* Store the data in a variable, that you initialize and then never change. +* Store the data in a variable, that you initialize (or assign to) and then + never change. Using a constant is a bit simpler, but uses more memory (since the constant is stored inline in the graph data structure, which may be duplicated a few times). @@ -461,19 +471,31 @@ You can compare these with the `fully_connected_feed` and ## Multiple input pipelines Commonly you will want to train on one dataset and evaluate (or "eval") on -another. One way to do this is to actually have two separate processes: +another. One way to do this is to actually have two separate graphs and +sessions, maybe in separate processes: * The training process reads training input data and periodically writes checkpoint files with all the trained variables. * The evaluation process restores the checkpoint files into an inference model that reads validation input data. -This is what is done in -@{$deep_cnn#save-and-restore-checkpoints$the example CIFAR-10 model}. This has a couple of benefits: +This is what is done @{tf.estimator$estimators} and manually in +@{$deep_cnn#save-and-restore-checkpoints$the example CIFAR-10 model}. +This has a couple of benefits: * The eval is performed on a single snapshot of the trained variables. * You can perform the eval even after training has completed and exited. You can have the train and eval in the same graph in the same process, and share -their trained variables. See -@{$variables$the shared variables tutorial}. +their trained variables or layers. See @{$variables$the shared variables tutorial}. + +To support the single-graph approach +@{$programmers_guide/datasets$Datasets} also supplies +@{$programmers_guide/datasets#creating_an_iterator$advanced iterator types} that +that allow the user to change the input pipeline without rebuilding the graph or +session. + +Note: Regardless of the implementation, many +operations (like ${tf.layers.batch_normalization}, and @{tf.layers.dropout}) +need to know if they are in training or evaluation mode, and you must be +careful to set this apropriately if you change the data source. diff --git a/tensorflow/docs_src/extend/adding_an_op.md b/tensorflow/docs_src/extend/adding_an_op.md index e8441d51c2da30..09e5bff42b9d17 100644 --- a/tensorflow/docs_src/extend/adding_an_op.md +++ b/tensorflow/docs_src/extend/adding_an_op.md @@ -155,7 +155,7 @@ REGISTER_KERNEL_BUILDER(Name("ZeroOut").Device(DEVICE_CPU), ZeroOutOp); ### Multi-threaded CPU kernels To write a multi-threaded CPU kernel, the Shard function in -[`work_sharder.h`](https://www.tensorflow.org/code/tensorflow/core/framework/work_sharder.h) +[`work_sharder.h`](https://www.tensorflow.org/code/tensorflow/core/util/work_sharder.h) can be used. This function shards a computation function across the threads configured to be used for intra-op threading (see intra_op_parallelism_threads in @@ -528,7 +528,7 @@ REGISTER\_OP("ZeroOut") (Note that the set of [attribute types](#attr-types) is different from the -@{$dims_types$tensor types} used for inputs and outputs.) +@{tf.DType$tensor types} used for inputs and outputs.) Your kernel can then access this attr in its constructor via the `context` parameter: @@ -603,7 +603,7 @@ define an attr with constraints, you can use the following ``s: * `{, }`: The value is of type `type`, and must be one of `` or ``, where `` and `` are supported - @{$dims_types#data-types$tensor types}. You don't specify + @{tf.DType$tensor types}. You don't specify that the type of the attr is `type`. This is implied when you have a list of types in `{...}`. For example, in this case the attr `t` is a type that must be an `int32`, a `float`, or a `bool`: @@ -685,7 +685,8 @@ REGISTER_OP("AttrDefaultExampleForAllTypes") .Attr("l_int: list(int) = [2, 3, 5, 7]"); ``` -Note in particular that the values of type `type` use @{$dims_types#data-types$the `DT_*` names for the types}. +Note in particular that the values of type `type` +use @{tf.DType$the `DT_*` names for the types}. #### Polymorphism {#polymorphism} @@ -1015,7 +1016,7 @@ expressions: `string`). This specifies a single tensor of the given type. See - @{$dims_types#data-types$the list of supported Tensor types}. + @{tf.DType$the list of supported Tensor types}. ```c++ REGISTER_OP("BuiltInTypesExample") @@ -1058,7 +1059,7 @@ expressions: * For a sequence of tensors with the same type: ` * `, where `` is the name of an [Attr](#attrs) with type `int`. The `` can either be - @{$dims_types#data-types$a specific type like `int32` or `float`}, + @{tf.DType$a specific type like `int32` or `float`}, or the name of an attr with type `type`. As an example of the first, this op accepts a list of `int32` tensors: diff --git a/tensorflow/docs_src/get_started/embedding_viz.md b/tensorflow/docs_src/get_started/embedding_viz.md deleted file mode 100644 index 84245b11bea455..00000000000000 --- a/tensorflow/docs_src/get_started/embedding_viz.md +++ /dev/null @@ -1,287 +0,0 @@ -# TensorBoard: Embedding Visualization - -Embeddings are ubiquitous in machine learning, appearing in recommender systems, -NLP, and many other applications. Indeed, in the context of TensorFlow, it's -natural to view tensors (or slices of tensors) as points in space, so almost any -TensorFlow system will naturally give rise to various embeddings. - -TensorBoard has a built-in visualizer, called the Embedding Projector, -for interactive visualization and analysis of high-dimensional data like -embeddings. The embedding projector will read the embeddings from your model -checkpoint file. Although it's most useful for embeddings, it will load any 2D -tensor, including your training weights. - -To learn more about embeddings and how to train them, see the -@{$word2vec$Vector Representations of Words} tutorial. -If you are interested in embeddings of images, check out -[this article](http://colah.github.io/posts/2014-10-Visualizing-MNIST/) for -interesting visualizations of MNIST images. On the other hand, if you are -interested in word embeddings, -[this article](http://colah.github.io/posts/2015-01-Visualizing-Representations/) -gives a good introduction. - - - -By default, the Embedding Projector projects the high-dimensional data into 3 -dimensions using -[principal component analysis](https://en.wikipedia.org/wiki/Principal_component_analysis). -For a visual explanation of PCA, see -[this article](http://setosa.io/ev/principal-component-analysis/). Another -very useful projection you can use is -[t-SNE](https://en.wikipedia.org/wiki/T-distributed_stochastic_neighbor_embedding). -We talk about more t-SNE later in the tutorial. - -If you are working with an embedding, you'll probably want to attach -labels/images to the data points. You can do this by generating a -[metadata file](#metadata) containing the labels for each point and configuring -the projector either by using our Python API, or manually constructing and -saving a -[projector_config.pbtxt](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tensorboard/plugins/projector/projector_config.proto) -in the same directory as your checkpoint file. - -## Setup - -For in depth information on how to run TensorBoard and make sure you are -logging all the necessary information, -see @{$summaries_and_tensorboard$TensorBoard: Visualizing Learning}. - -To visualize your embeddings, there are 3 things you need to do: - -1) Setup a 2D tensor that holds your embedding(s). - -```python -embedding_var = tf.Variable(....) -``` - -2) Periodically save your model variables in a checkpoint in -LOG_DIR. - -```python -saver = tf.train.Saver() -saver.save(session, os.path.join(LOG_DIR, "model.ckpt"), step) -``` - -3) (Optional) Associate metadata with your embedding. - -If you have any metadata (labels, images) associated with your embedding, you -can tell TensorBoard about it either by directly storing a -[projector_config.pbtxt](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tensorboard/plugins/projector/projector_config.proto) -in the LOG_DIR, or use our python API. - -For instance, the following projector_config.ptxt associates the -word_embedding tensor with metadata stored in $LOG_DIR/metadata.tsv: - -``` -embeddings { - tensor_name: 'word_embedding' - metadata_path: '$LOG_DIR/metadata.tsv' -} -``` - -The same config can be produced programmatically using the following code snippet: - -```python -from tensorflow.contrib.tensorboard.plugins import projector - -# Create randomly initialized embedding weights which will be trained. -N = 10000 # Number of items (vocab size). -D = 200 # Dimensionality of the embedding. -embedding_var = tf.Variable(tf.random_normal([N,D]), name='word_embedding') - -# Format: tensorflow/tensorboard/plugins/projector/projector_config.proto -config = projector.ProjectorConfig() - -# You can add multiple embeddings. Here we add only one. -embedding = config.embeddings.add() -embedding.tensor_name = embedding_var.name -# Link this tensor to its metadata file (e.g. labels). -embedding.metadata_path = os.path.join(LOG_DIR, 'metadata.tsv') - -# Use the same LOG_DIR where you stored your checkpoint. -summary_writer = tf.summary.FileWriter(LOG_DIR) - -# The next line writes a projector_config.pbtxt in the LOG_DIR. TensorBoard will -# read this file during startup. -projector.visualize_embeddings(summary_writer, config) -``` - -After running your model and training your embeddings, run TensorBoard and point -it to the LOG_DIR of the job. - -```python -tensorboard --logdir=LOG_DIR -``` - -Then click on the *Embeddings* tab on the top pane -and select the appropriate run (if there are more than one run). - - -## Metadata -Usually embeddings have metadata associated with it (e.g. labels, images). The -metadata should be stored in a separate file outside of the model checkpoint -since the metadata is not a trainable parameter of the model. The format should -be a [TSV file](https://en.wikipedia.org/wiki/Tab-separated_values) -(tab characters shown in red) with the first line containing column headers -(shown in bold) and subsequent lines contain the metadata values: - - -Word\tFrequency
- Airplane\t345
- Car\t241
- ... - - -There is no explicit key shared with the main data file; instead, the order in -the metadata file is assumed to match the order in the embedding tensor. In -other words, the first line is the header information and the (i+1)-th line in -the metadata file corresponds to the i-th row of the embedding tensor stored in -the checkpoint. - -Note: If the TSV metadata file has only a single column, then we don’t expect a -header row, and assume each row is the label of the embedding. We include this -exception because it matches the commonly-used "vocab file" format. - -### Images -If you have images associated with your embeddings, you will need to -produce a single image consisting of small thumbnails of each data point. -This is known as the -[sprite image](https://www.google.com/webhp#q=what+is+a+sprite+image). -The sprite should have the same number of rows and columns with thumbnails -stored in row-first order: the first data point placed in the top left and the -last data point in the bottom right: - - - - - - - - - - - - - - - - - -
012
345
67
- -Note in the example above that the last row doesn't have to be filled. For a -concrete example of a sprite, see -[this sprite image](https://www.tensorflow.org/images/mnist_10k_sprite.png) of 10,000 MNIST digits -(100x100). - -Note: We currently support sprites up to 8192px X 8192px. - -After constructing the sprite, you need to tell the Embedding Projector where -to find it: - - -```python -embedding.sprite.image_path = PATH_TO_SPRITE_IMAGE -# Specify the width and height of a single thumbnail. -embedding.sprite.single_image_dim.extend([w, h]) -``` - -## Interaction - -The Embedding Projector has three panels: - -1. *Data panel* on the top left, where you can choose the run, the embedding - tensor and data columns to color and label points by. -2. *Projections panel* on the bottom left, where you choose the type of - projection (e.g. PCA, t-SNE). -3. *Inspector panel* on the right side, where you can search for particular - points and see a list of nearest neighbors. - -### Projections -The Embedding Projector has three methods of reducing the dimensionality of a -data set: two linear and one nonlinear. Each method can be used to create either -a two- or three-dimensional view. - -**Principal Component Analysis** A straightforward technique for reducing -dimensions is Principal Component Analysis (PCA). The Embedding Projector -computes the top 10 principal components. The menu lets you project those -components onto any combination of two or three. PCA is a linear projection, -often effective at examining global geometry. - -**t-SNE** A popular non-linear dimensionality reduction technique is t-SNE. -The Embedding Projector offers both two- and three-dimensional t-SNE views. -Layout is performed client-side animating every step of the algorithm. Because -t-SNE often preserves some local structure, it is useful for exploring local -neighborhoods and finding clusters. Although extremely useful for visualizing -high-dimensional data, t-SNE plots can sometimes be mysterious or misleading. -See this [great article](http://distill.pub/2016/misread-tsne/) for how to use -t-SNE effectively. - -**Custom** You can also construct specialized linear projections based on text -searches for finding meaningful directions in space. To define a projection -axis, enter two search strings or regular expressions. The program computes the -centroids of the sets of points whose labels match these searches, and uses the -difference vector between centroids as a projection axis. - -### Navigation - -To explore a data set, you can navigate the views in either a 2D or a 3D mode, -zooming, rotating, and panning using natural click-and-drag gestures. -Clicking on a point causes the right pane to show an explicit textual list of -nearest neighbors, along with distances to the current point. The -nearest-neighbor points themselves are highlighted on the projection. - -Zooming into the cluster gives some information, but it is sometimes more -helpful to restrict the view to a subset of points and perform projections only -on those points. To do so, you can select points in multiple ways: - -1. After clicking on a point, its nearest neighbors are also selected. -2. After a search, the points matching the query are selected. -3. Enabling selection, clicking on a point and dragging defines a selection - sphere. - -After selecting a set of points, you can isolate those points for -further analysis on their own with the "Isolate Points" button in the Inspector -pane on the right hand side. - - -![Selection of nearest neighbors](https://www.tensorflow.org/images/embedding-nearest-points.png "Selection of nearest neighbors") -*Selection of the nearest neighbors of “important” in a word embedding dataset.* - -The combination of filtering with custom projection can be powerful. Below, we filtered -the 100 nearest neighbors of “politics” and projected them onto the -“best” - “worst” vector as an x axis. The y axis is random. - -You can see that on the right side we have “ideas”, “science”, “perspective”, -“journalism” while on the left we have “crisis”, “violence” and “conflict”. - - - - - - - - - - -
- Custom controls panel - - Custom projection -
- Custom projection controls. - - Custom projection of neighbors of "politics" onto "best" - "worst" vector. -
- -### Collaborative Features - -To share your findings, you can use the bookmark panel in the bottom right -corner and save the current state (including computed coordinates of any -projection) as a small file. The Projector can then be pointed to a set of one -or more of these files, producing the panel below. Other users can then walk -through a sequence of bookmarks. - -Bookmark panel diff --git a/tensorflow/docs_src/get_started/estimator.md b/tensorflow/docs_src/get_started/estimator.md index a55454f8af362c..4f3a438d17d20a 100644 --- a/tensorflow/docs_src/get_started/estimator.md +++ b/tensorflow/docs_src/get_started/estimator.md @@ -273,9 +273,7 @@ Then, the code creates a `DNNClassifier` model using the following arguments: containing 10, 20, and 10 neurons, respectively. * `n_classes=3`. Three target classes, representing the three Iris species. * `model_dir=/tmp/iris_model`. The directory in which TensorFlow will save - checkpoint data during model training. For more on logging and monitoring - with TensorFlow, see - @{$monitors$Logging and Monitoring Basics with tf.estimator}. + checkpoint data and TensorBoard summaries during model training. ## Describe the training input pipeline {#train-input} @@ -315,9 +313,7 @@ classifier.train(input_fn=train_input_fn, steps=1000) However, if you're looking to track the model while it trains, you'll likely want to instead use a TensorFlow @{tf.train.SessionRunHook$`SessionRunHook`} -to perform logging operations. See the tutorial -@{$monitors$Logging and Monitoring Basics with tf.estimator} -for more on this topic. +to perform logging operations. ## Evaluate Model Accuracy {#evaluate-accuracy} diff --git a/tensorflow/docs_src/get_started/index.md b/tensorflow/docs_src/get_started/index.md index 0d302ec3830f23..003fac1a287688 100644 --- a/tensorflow/docs_src/get_started/index.md +++ b/tensorflow/docs_src/get_started/index.md @@ -22,19 +22,14 @@ To learn about the high-level API, read the following guides: * @{$get_started/estimator$tf.estimator Quickstart}, which introduces this API. - * @{$get_started/input_fn$Building Input Functions with tf.contrib.learn}, + * @{$get_started/input_fn$Building Input Functions}, which takes you into a somewhat more sophisticated use of this API. - * @{$get_started/monitors$Logging and Monitoring Basics with tf.contrib.learn}, - which explains how to audit the progress of model training. TensorBoard is a utility to visualize different aspects of machine learning. The following guides explain how to use TensorBoard: * @{$get_started/summaries_and_tensorboard$TensorBoard: Visualizing Learning}, which gets you started. - * @{$get_started/embedding_viz$TensorBoard: Embedding Visualization}, which - demonstrates how to view and interact with high-dimensional data, such as - embeddings. * @{$get_started/graph_viz$TensorBoard: Graph Visualization}, which explains how to visualize the computational graph. Graph visualization is typically more useful for programmers using the low-level API. diff --git a/tensorflow/docs_src/get_started/input_fn.md b/tensorflow/docs_src/get_started/input_fn.md index 422f45c586aa58..7706c07b1d940f 100644 --- a/tensorflow/docs_src/get_started/input_fn.md +++ b/tensorflow/docs_src/get_started/input_fn.md @@ -249,7 +249,7 @@ here](https://www.tensorflow.org/code/tensorflow/examples/tutorials/input_fn/bos ### Importing the Housing Data -To start, set up your imports (including `pandas` and `tensorflow`) and @{$monitors#enabling-logging-with-tensorflow$set logging verbosity} to +To start, set up your imports (including `pandas` and `tensorflow`) and set logging verbosity to `INFO` for more detailed log output: ```python diff --git a/tensorflow/docs_src/get_started/leftnav_files b/tensorflow/docs_src/get_started/leftnav_files index 656727fbfe0942..bb67eaddda369c 100644 --- a/tensorflow/docs_src/get_started/leftnav_files +++ b/tensorflow/docs_src/get_started/leftnav_files @@ -5,8 +5,6 @@ mnist/pros.md mnist/mechanics.md estimator.md input_fn.md -monitors.md summaries_and_tensorboard.md -embedding_viz.md graph_viz.md tensorboard_histograms.md diff --git a/tensorflow/docs_src/get_started/monitors.md b/tensorflow/docs_src/get_started/monitors.md deleted file mode 100644 index 5606e95365812a..00000000000000 --- a/tensorflow/docs_src/get_started/monitors.md +++ /dev/null @@ -1,406 +0,0 @@ -# Logging and Monitoring Basics with tf.contrib.learn - -When training a model, it’s often valuable to track and evaluate progress in -real time. In this tutorial, you’ll learn how to use TensorFlow’s logging -capabilities and the `Monitor` API to audit the in-progress training of a neural -network classifier for categorizing irises. This tutorial builds on the code -developed in @{$estimator$tf.estimator Quickstart} so if you -haven't yet completed that tutorial, you may want to explore it first, -especially if you're looking for an intro/refresher on tf.contrib.learn basics. - -## Setup {#setup} - -For this tutorial, you'll be building upon the following code from -@{$estimator$tf.estimator Quickstart}: - -```python -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -import numpy as np -import tensorflow as tf - -# Data sets -IRIS_TRAINING = os.path.join(os.path.dirname(__file__), "iris_training.csv") -IRIS_TEST = os.path.join(os.path.dirname(__file__), "iris_test.csv") - -def main(unused_argv): - # Load datasets. - training_set = tf.contrib.learn.datasets.base.load_csv_with_header( - filename=IRIS_TRAINING, target_dtype=np.int, features_dtype=np.float32) - test_set = tf.contrib.learn.datasets.base.load_csv_with_header( - filename=IRIS_TEST, target_dtype=np.int, features_dtype=np.float32) - - # Specify that all features have real-value data - feature_columns = [tf.contrib.layers.real_valued_column("", dimension=4)] - - # Build 3 layer DNN with 10, 20, 10 units respectively. - classifier = tf.contrib.learn.DNNClassifier(feature_columns=feature_columns, - hidden_units=[10, 20, 10], - n_classes=3, - model_dir="/tmp/iris_model") - - # Fit model. - classifier.fit(x=training_set.data, - y=training_set.target, - steps=2000) - - # Evaluate accuracy. - accuracy_score = classifier.evaluate(x=test_set.data, - y=test_set.target)["accuracy"] - print('Accuracy: {0:f}'.format(accuracy_score)) - - # Classify two new flower samples. - new_samples = np.array( - [[6.4, 3.2, 4.5, 1.5], [5.8, 3.1, 5.0, 1.7]], dtype=float) - y = list(classifier.predict(new_samples, as_iterable=True)) - print('Predictions: {}'.format(str(y))) - -if __name__ == "__main__": - tf.app.run() -``` - -Copy the above code into a file, and download the corresponding -[training](http://download.tensorflow.org/data/iris_training.csv) and -[test](http://download.tensorflow.org/data/iris_test.csv) data sets to the same -directory. - -In the following sections, you'll progressively make updates to the above code -to add logging and monitoring capabilities. Final code incorporating all updates -is [available for download -here](https://www.tensorflow.org/code/tensorflow/examples/tutorials/monitors/iris_monitors.py). - -## Overview - -The @{$estimator$tf.estimator Quickstart tutorial} walked through -how to implement a neural net classifier to categorize iris examples into one of -three species. - -But when [the code](#setup) from this tutorial is run, the output contains no -logging tracking how model training is progressing—only the results of the -`print` statements that were included: - -```none -Accuracy: 0.933333 -Predictions: [1 2] -``` - -Without any logging, model training feels like a bit of a black box; you can't -see what's happening as TensorFlow steps through gradient descent, get a sense -of whether the model is converging appropriately, or audit to determine whether -[early stopping](https://en.wikipedia.org/wiki/Early_stopping) might be -appropriate. - -One way to address this problem would be to split model training into multiple -`fit` calls with smaller numbers of steps in order to evaluate accuracy more -progressively. However, this is not recommended practice, as it greatly slows -down model training. Fortunately, tf.contrib.learn offers another solution: a -@{tf.contrib.learn.monitors$Monitor API} designed to help -you log metrics and evaluate your model while training is in progress. In the -following sections, you'll learn how to enable logging in TensorFlow, set up a -ValidationMonitor to do streaming evaluations, and visualize your metrics using -TensorBoard. - -## Enabling Logging with TensorFlow - -TensorFlow uses five different levels for log messages. In order of ascending -severity, they are `DEBUG`, `INFO`, `WARN`, `ERROR`, and `FATAL`. When you -configure logging at any of these levels, TensorFlow will output all log -messages corresponding to that level and all levels of higher severity. For -example, if you set a logging level of `ERROR`, you'll get log output containing -`ERROR` and `FATAL` messages, and if you set a level of `DEBUG`, you'll get log -messages from all five levels. - -By default, TensorFlow is configured at a logging level of `WARN`, but when -tracking model training, you'll want to adjust the level to `INFO`, which will -provide additional feedback as `fit` operations are in progress. - -Add the following line to the beginning of your code (right after your -`import`s): - -```python -tf.logging.set_verbosity(tf.logging.INFO) -``` - -Now when you run the code, you'll see additional log output like the following: - -```none -INFO:tensorflow:loss = 1.18812, step = 1 -INFO:tensorflow:loss = 0.210323, step = 101 -INFO:tensorflow:loss = 0.109025, step = 201 -``` - -With `INFO`-level logging, tf.contrib.learn automatically outputs [training-loss -metrics](https://en.wikipedia.org/wiki/Loss_function) to stderr after every 100 -steps. - -## Configuring a ValidationMonitor for Streaming Evaluation - -Logging training loss is helpful to get a sense whether your model is -converging, but what if you want further insight into what's happening during -training? tf.contrib.learn provides several high-level `Monitor`s you can attach -to your `fit` operations to further track metrics and/or debug lower-level -TensorFlow operations during model training, including: - -Monitor | Description -------------------- | ----------- -`CaptureVariable` | Saves a specified variable's values into a collection at every _n_ steps of training -`PrintTensor` | Logs a specified tensor's values at every _n_ steps of training -`SummarySaver` | Saves @{tf.Summary} [protocol buffers](https://developers.google.com/protocol-buffers/) for a given tensor using a @{tf.summary.FileWriter} at every _n_ steps of training -`ValidationMonitor` | Logs a specified set of evaluation metrics at every _n_ steps of training, and, if desired, implements early stopping under certain conditions - -### Evaluating Every *N* Steps - -For the iris neural network classifier, while logging training loss, you might -also want to simultaneously evaluate against test data to see how well the model -is generalizing. You can accomplish this by configuring a `ValidationMonitor` -with the test data (`test_set.data` and `test_set.target`), and setting how -often to evaluate with `every_n_steps`. The default value of `every_n_steps` is -`100`; here, set `every_n_steps` to `50` to evaluate after every 50 steps of -model training: - -```python -validation_monitor = tf.contrib.learn.monitors.ValidationMonitor( - test_set.data, - test_set.target, - every_n_steps=50) -``` - -Place this code right before the line instantiating the `classifier`. - -`ValidationMonitor`s rely on saved checkpoints to perform evaluation operations, -so you'll want to modify instantiation of the `classifier` to add a -@{tf.contrib.learn.RunConfig} that includes -`save_checkpoints_secs`, which specifies how many seconds should elapse between -checkpoint saves during training. Because the iris data set is quite small, and -thus trains quickly, it makes sense to set `save_checkpoints_secs` to 1 (saving -a checkpoint every second) to ensure a sufficient number of checkpoints: - -```python -classifier = tf.contrib.learn.DNNClassifier( - feature_columns=feature_columns, - hidden_units=[10, 20, 10], - n_classes=3, - model_dir="/tmp/iris_model", - config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1)) -``` - -NOTE: The `model_dir` parameter specifies an explicit directory -(`/tmp/iris_model`) for model data to be stored; this directory path will be -easier to reference later on than an autogenerated one. Each time you run the -code, any existing data in `/tmp/iris_model` will be loaded, and model training -will continue where it left off in the last run (e.g., running the script twice -in succession will execute 4000 steps during training—2000 during each -`fit` operation). To start over model training from scratch, delete -`/tmp/iris_model` before running the code. - -Finally, to attach your `validation_monitor`, update the `fit` call to include a -`monitors` param, which takes a list of all monitors to run during model -training: - -```python -classifier.fit(x=training_set.data, - y=training_set.target, - steps=2000, - monitors=[validation_monitor]) -``` - -Now, when you rerun the code, you should see validation metrics in your log -output, e.g.: - -```none -INFO:tensorflow:Validation (step 50): loss = 1.71139, global_step = 0, accuracy = 0.266667 -... -INFO:tensorflow:Validation (step 300): loss = 0.0714158, global_step = 268, accuracy = 0.966667 -... -INFO:tensorflow:Validation (step 1750): loss = 0.0574449, global_step = 1729, accuracy = 0.966667 -``` - -### Customizing the Evaluation Metrics with MetricSpec - -By default, if no evaluation metrics are specified, `ValidationMonitor` will log -both [loss](https://en.wikipedia.org/wiki/Loss_function) and accuracy, but you -can customize the list of metrics that will be run every 50 steps. To specify -the exact metrics you'd like to run in each evaluation pass, you can add a -`metrics` param to the `ValidationMonitor` constructor. `metrics` takes a dict -of key/value pairs, where each key is the name you'd like logged for the metric, -and the corresponding value is a -[`MetricSpec`](https://www.tensorflow.org/code/tensorflow/contrib/learn/python/learn/metric_spec.py) -object. - -The `MetricSpec` constructor accepts four parameters: - -* `metric_fn`. The function that calculates and returns the value of a metric. - This can be a predefined function available in the - @{tf.contrib.metrics} module, such as - @{tf.contrib.metrics.streaming_precision} or - @{tf.contrib.metrics.streaming_recall}. - - Alternatively, you can define your own custom metric function, which must - take `predictions` and `labels` tensors as arguments (a `weights` argument - can also optionally be supplied). The function must return the value of the - metric in one of two formats: - - * A single tensor - * A pair of ops `(value_op, update_op)`, where `value_op` returns the - metric value and `update_op` performs a corresponding operation to - update internal model state. - -* `prediction_key`. The key of the tensor containing the predictions returned - by the model. This argument may be omitted if the model returns either a - single tensor or a dict with a single entry. For a `DNNClassifier` model, - class predictions will be returned in a tensor with the key - @{tf.contrib.learn.PredictionKey.CLASSES}. - -* `label_key`. The key of the tensor containing the labels returned by the - model, as specified by the model's @{$input_fn$`input_fn`}. As - with `prediction_key`, this argument may be omitted if the `input_fn` - returns either a single tensor or a dict with a single entry. In the iris - example in this tutorial, the `DNNClassifier` does not have an `input_fn` - (`x`,`y` data is passed directly to `fit`), so it's not necessary to provide - a `label_key`. - -* `weights_key`. *Optional*. The key of the tensor (returned by the - @{$input_fn$`input_fn`}) containing weights inputs for the - `metric_fn`. - -The following code creates a `validation_metrics` dict that defines three -metrics to log during model evaluation: - -* `"accuracy"`, using @{tf.contrib.metrics.streaming_accuracy} - as the `metric_fn` -* `"precision"`, using @{tf.contrib.metrics.streaming_precision} - as the `metric_fn` -* `"recall"`, using @{tf.contrib.metrics.streaming_recall} - as the `metric_fn` - -```python -validation_metrics = { - "accuracy": - tf.contrib.learn.MetricSpec( - metric_fn=tf.contrib.metrics.streaming_accuracy, - prediction_key=tf.contrib.learn.PredictionKey.CLASSES), - "precision": - tf.contrib.learn.MetricSpec( - metric_fn=tf.contrib.metrics.streaming_precision, - prediction_key=tf.contrib.learn.PredictionKey.CLASSES), - "recall": - tf.contrib.learn.MetricSpec( - metric_fn=tf.contrib.metrics.streaming_recall, - prediction_key=tf.contrib.learn.PredictionKey.CLASSES) -} -``` - -Add the above code before the `ValidationMonitor` constructor. Then revise the -`ValidationMonitor` constructor as follows to add a `metrics` parameter to log -the accuracy, precision, and recall metrics specified in `validation_metrics` -(loss is always logged, and doesn't need to be explicitly specified): - -```python -validation_monitor = tf.contrib.learn.monitors.ValidationMonitor( - test_set.data, - test_set.target, - every_n_steps=50, - metrics=validation_metrics) -``` - -Rerun the code, and you should see precision and recall included in your log -output, e.g.: - -```none -INFO:tensorflow:Validation (step 50): recall = 0.0, loss = 1.20626, global_step = 1, precision = 0.0, accuracy = 0.266667 -... -INFO:tensorflow:Validation (step 600): recall = 1.0, loss = 0.0530696, global_step = 571, precision = 1.0, accuracy = 0.966667 -... -INFO:tensorflow:Validation (step 1500): recall = 1.0, loss = 0.0617403, global_step = 1452, precision = 1.0, accuracy = 0.966667 -``` - -### Early Stopping with ValidationMonitor - -Note that in the above log output, by step 600, the model has already achieved -precision and recall rates of 1.0. This raises the question as to whether model -training could benefit from -[early stopping](https://en.wikipedia.org/wiki/Early_stopping). - -In addition to logging eval metrics, `ValidationMonitor`s make it easy to -implement early stopping when specified conditions are met, via three params: - -| Param | Description | -| -------------------------------- | ----------------------------------------- | -| `early_stopping_metric` | Metric that triggers early stopping | -: : (e.g., loss or accuracy) under conditions : -: : specified in `early_stopping_rounds` and : -: : `early_stopping_metric_minimize`. Default : -: : is `"loss"`. : -| `early_stopping_metric_minimize` | `True` if desired model behavior is to | -: : minimize the value of : -: : `early_stopping_metric`; `False` if : -: : desired model behavior is to maximize the : -: : value of `early_stopping_metric`. Default : -: : is `True`. : -| `early_stopping_rounds` | Sets a number of steps during which if | -: : the `early_stopping_metric` does not : -: : decrease (if : -: : `early_stopping_metric_minimize` is : -: : `True`) or increase (if : -: : `early_stopping_metric_minimize` is : -: : `False`), training will be stopped. : -: : Default is `None`, which means early : -: : stopping will never occur. : - -Make the following revision to the `ValidationMonitor` constructor, which -specifies that if loss (`early_stopping_metric="loss"`) does not decrease -(`early_stopping_metric_minimize=True`) over a period of 200 steps -(`early_stopping_rounds=200`), model training will stop immediately at that -point, and not complete the full 2000 steps specified in `fit`: - -```python -validation_monitor = tf.contrib.learn.monitors.ValidationMonitor( - test_set.data, - test_set.target, - every_n_steps=50, - metrics=validation_metrics, - early_stopping_metric="loss", - early_stopping_metric_minimize=True, - early_stopping_rounds=200) -``` - -Rerun the code to see if model training stops early: - -```none -... -INFO:tensorflow:Validation (step 1150): recall = 1.0, loss = 0.056436, global_step = 1119, precision = 1.0, accuracy = 0.966667 -INFO:tensorflow:Stopping. Best step: 800 with loss = 0.048313818872. -``` - -Indeed, here training stops at step 1150, indicating that for the past 200 -steps, loss did not decrease, and that overall, step 800 produced the smallest -loss value against the test data set. This suggests that additional calibration -of hyperparameters by decreasing the step count might further improve the model. - -## Visualizing Log Data with TensorBoard - -Reading through the log produced by `ValidationMonitor` provides plenty of raw -data on model performance during training, but it may also be helpful to see -visualizations of this data to get further insight into trends—for -example, how accuracy is changing over step count. You can use TensorBoard (a -separate program packaged with TensorFlow) to plot graphs like this by setting -the `logdir` command-line argument to the directory where you saved your model -training data (here, `/tmp/iris_model`). Run the following on your command line: - -
$ tensorboard --logdir=/tmp/iris_model/
-Starting TensorBoard 39 on port 6006
- -Then navigate to `http://0.0.0.0:`*``* in your browser, where -*``* is the port specified in the command-line output (here, -`6006`). - -If you click on the accuracy field, you'll see an image like the following, -which shows accuracy plotted against step count: - -![Accuracy over step count in TensorBoard](https://www.tensorflow.org/images/validation_monitor_tensorboard_accuracy.png "Accuracy over step count in TensorBoard") - -For more on using TensorBoard, see @{$summaries_and_tensorboard$TensorBoard: Visualizing Learning} and @{$graph_viz$TensorBoard: Graph Visualization}. diff --git a/tensorflow/docs_src/get_started/summaries_and_tensorboard.md b/tensorflow/docs_src/get_started/summaries_and_tensorboard.md index 45d43e7a6e76ef..ece8fbf43c3c99 100644 --- a/tensorflow/docs_src/get_started/summaries_and_tensorboard.md +++ b/tensorflow/docs_src/get_started/summaries_and_tensorboard.md @@ -17,7 +17,7 @@ TensorBoard is fully configured, it looks like this: This tutorial is intended to get you started with simple TensorBoard usage. -There are other resources available as well! The [TensorBoard README](https://www.tensorflow.org/code/tensorflow/tensorboard/README.md) +There are other resources available as well! The [TensorBoard's GitHub](https://github.com/tensorflow/tensorboard) has a lot more information on TensorBoard usage, including tips & tricks, and debugging information. @@ -216,5 +216,4 @@ corner. Each tab represents a set of serialized data that can be visualized. For in depth information on how to use the *graph* tab to visualize your graph, see @{$graph_viz$TensorBoard: Graph Visualization}. -For more usage information on TensorBoard in general, see the [TensorBoard -README](https://www.tensorflow.org/code/tensorflow/tensorboard/README.md). +For more usage information on TensorBoard in general, see the [TensorBoard's GitHub](https://github.com/tensorflow/tensorboard). diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index 2b13d4fcb6925b..78c64102a20409 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -33,7 +33,7 @@ must be installed on your system: `LD_LIBRARY_PATH` environment variable as described in the NVIDIA documentation. * The NVIDIA drivers associated with CUDA Toolkit 8.0. - * cuDNN v5.1. For details, see + * cuDNN v6. For details, see [NVIDIA's documentation](https://developer.nvidia.com/cudnn). Ensure that you create the `CUDA_HOME` environment variable as described in the NVIDIA documentation. @@ -55,7 +55,7 @@ TensorFlow with GPU support, but only if you do the following: * Install TensorFlow from sources as documented in @{$install_sources$Installing TensorFlow from Sources}. * Install or upgrade to at least the following NVIDIA versions: - * CUDA toolkit 7.0 or greater + * CUDA toolkit 8.0 or greater * cuDNN v3 or greater * GPU card with CUDA Compute Capability 3.0 or higher. @@ -167,7 +167,7 @@ Take the following steps to install TensorFlow with Virtualenv: Python version, and GPU support. Find the appropriate value for tfBinaryURL for your system [here](#the_url_of_the_tensorflow_python_package). For example, if you - are installing TensorFlow for Linux, Python 2.7, and CPU-only support, + are installing TensorFlow for Linux, Python 3.4, and CPU-only support, issue the following command to install TensorFlow in the active virtualenv environment: @@ -272,7 +272,7 @@ take the following steps: Python version, and GPU support. Find the appropriate value for tfBinaryURL [here](#the_url_of_the_tensorflow_python_package). For example, to - install TensorFlow for Linux, Python 2.7, and CPU-only support, issue + install TensorFlow for Linux, Python 3.4, and CPU-only support, issue the following command: 
@@ -460,7 +460,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
      where tfBinaryURL is the
      [URL of the TensorFlow Python package](#the_url_of_the_tensorflow_python_package).
      For example, the following command installs the CPU-only version of
-     TensorFlow for Python 2.7:
+     TensorFlow for Python 3.4:
 
      
      (tensorflow)$ pip install --ignore-installed --upgrade \
diff --git a/tensorflow/docs_src/install/install_windows.md b/tensorflow/docs_src/install/install_windows.md
index a06ab88046ed4f..8e319f0fb640f3 100644
--- a/tensorflow/docs_src/install/install_windows.md
+++ b/tensorflow/docs_src/install/install_windows.md
@@ -29,7 +29,7 @@ installed on your system:
     Ensure that you append the relevant Cuda pathnames to the `%PATH%`
     environment variable as described in the NVIDIA documentation.
   * The NVIDIA drivers associated with CUDA Toolkit 8.0.
-  * cuDNN v5.1. For details, see
+  * cuDNN v6 or v6.1. For details, see
     [NVIDIA's documentation](https://developer.nvidia.com/cudnn).
     Note that cuDNN is typically installed in a different location from the
     other CUDA DLLs. Ensure that you add the directory where you installed
@@ -40,7 +40,7 @@ installed on your system:
 
 If you have a different version of one of the preceding packages, please
 change to the specified versions.  In particular, the cuDNN version
-must match exactly: TensorFlow will not load if it cannot find `cuDNN64_5.dll`.
+must match exactly: TensorFlow will not load if it cannot find `cuDNN64_6.dll`.
 To use a different version of cuDNN, you must build from source.
 
 ## Determine how to install TensorFlow
@@ -71,13 +71,14 @@ Use that package at your own risk.
 
 ## Installing with native pip
 
-If the following version of Python is not installed on your machine,
+If neither of the following versions of Python is installed on your machine,
 install it now:
 
   * [Python 3.5.x 64-bit from python.org](https://www.python.org/downloads/release/python-352/)
+  * [Python 3.6.x 64-bit from python.org](https://www.python.org/downloads/release/python-362/)
 
-TensorFlow only supports version 3.5.x of Python on Windows.
-Note that Python 3.5.x comes with the pip3 package manager, which is the
+-TensorFlow supports Python 3.5.x and 3.6.x on Windows.
+Note that Python 3 comes with the pip3 package manager, which is the
 program you'll use to install TensorFlow.
 
 To install TensorFlow, start a terminal. Then issue the appropriate
@@ -115,12 +116,12 @@ Take the following steps to install TensorFlow in an Anaconda environment:
      environment. To install the CPU-only version of TensorFlow, enter the
      following command:
 
-     
(tensorflow)C:\> pip install --ignore-installed --upgrade https://storage.googleapis.com/tensorflow/windows/cpu/tensorflow-1.3.0-cp35-cp35m-win_amd64.whl 

+     
(tensorflow)C:\> pip install --ignore-installed --upgrade tensorflow 

 
      To install the GPU version of TensorFlow, enter the following command
      (on a single line):
 
-     
(tensorflow)C:\> pip install --ignore-installed --upgrade https://storage.googleapis.com/tensorflow/windows/gpu/tensorflow_gpu-1.3.0-cp35-cp35m-win_amd64.whl 

+     
(tensorflow)C:\> pip install --ignore-installed --upgrade tensorflow-gpu 

 
 ## Validate your installation
 
@@ -152,6 +153,9 @@ TensorFlow}.
 If the system outputs an error message instead of a greeting, see [Common
 installation problems](#common_installation_problems).
 
+There is also a helpful [script](https://gist.github.com/mrry/ee5dbcfdd045fa48a27d56664411d41c)
+for Windows TensorFlow installation issues.
+
 ## Common installation problems
 
 We are relying on Stack Overflow to document TensorFlow installation problems
diff --git a/tensorflow/docs_src/performance/performance_guide.md b/tensorflow/docs_src/performance/performance_guide.md
index a5508ac23e5bd5..bf69b7e6fc24a4 100644
--- a/tensorflow/docs_src/performance/performance_guide.md
+++ b/tensorflow/docs_src/performance/performance_guide.md
@@ -1,43 +1,182 @@
 # Performance Guide
 
-This guide contains a collection of best practices for optimizing your
-TensorFlow code. The best practices apply to both new and experienced
-Tensorflow users.  As a complement to the best practices in this document, the
-@{$performance_models$High-Performance Models} document links to example code
-and details for creating models that scale on a variety of hardware.
-
-## Best Practices
-While optimizing implementations of different types of models can be different,
-the topics below cover best practices to get the most performance from
-TensorFlow. Although these suggestions focus on image-based models, we will
-regularly add tips for all kinds of models. The following list highlights key
-best practices:
-
-*   Build and install from source
-*   Utilize queues for reading data
-*   Preprocessing on the CPU
-*   Use `NCHW` image data format
-*   Place shared parameters on the GPU
-*   Use fused batch norm
-
-The following sections detail the preceding suggestions.
-
-### Build and install from source
-
-To install the most optimized version of TensorFlow, build and install
-TensorFlow from source by following [Installing TensorFlow from Source](../install/install_sources).
-Building from source with compiler optimizations for the target hardware and
-ensuring the latest CUDA platform and cuDNN libraries are installed results in
-the highest performing installs.
-
-For the most stable experience, build from the [latest release](https://github.com/tensorflow/tensorflow/releases)
-branch. To get the latest performance changes and accept some stability risk,
-build from [master](https://github.com/tensorflow/tensorflow).
-
-If there is a need to build TensorFlow on a platform that has different hardware
-than the target, then cross-compile with the highest optimizations for the target
-platform.  The following command is an example of telling `bazel` to compile for
-a specific platform:
+This guide contains a collection of best practices for optimizing TensorFlow
+code. The guide is divided into a few sections:
+
+*   [General best practices](#general_best_practices) covers topics that are
+    common across a variety of model types and hardware.
+*   [Optimizing for GPU](#optimizing_for_gpu) details tips specifically relevant
+    to GPUs.
+*   [Optimizing for CPU](#optimizing_for_cpu) details CPU specific information.
+
+## General best practices
+
+The sections below cover best practices that are relevant to a variety of
+hardware and models. The best practices section is broken down into the
+following sections:
+
+*   [Input pipeline optimizations](#input-pipeline-optimization)
+*   [Data formats](#data-formats)
+*   [Common fused Ops](#common-fused-ops)
+*   [Building and installing from source](#building-and-installing-from-source)
+
+### Input pipeline optimization
+
+Typical models retrieve data from disk and preprocess it before sending the data
+through the network. For example, models that process JPEG images will follow
+this flow: load image from disk, decode JPEG into a tensor, crop and pad,
+possibly flip and distort, and then batch. This flow is referred to as the input
+pipeline. As GPUs and other hardware accelerators get faster, preprocessing of
+data can be a bottleneck.
+
+Determining if the input pipeline is the bottleneck can be complicated. One of
+the most straightforward methods is to reduce the model to a single operation
+(trivial model) after the input pipeline and measure the examples per second. If
+the difference in examples per second for the full model and the trivial model
+is minimal then the input pipeline is likely a bottleneck. Below are some other
+approaches to identifying issues:
+
+*   Check if a GPU is underutilized by running `watch -n 2 nvidia-smi`. If GPU
+    utilization is not approaching 80-100%, then the input pipeline may be the
+    bottleneck.
+*   Generate a timeline and look for large blocks of white space (waiting). An
+    example of generating a timeline exists as part of the @{$jit$XLA JIT}
+    tutorial.
+*   Check CPU usage. It is possible to have an optimized input pipeline and lack
+    the CPU cycles to process the pipeline.
+*   Estimate the throughput needed and verify the disk used is capable of that
+    level of throughput. Some cloud solutions have network attached disks that
+    start as low as 50 MB/sec, which is slower than spinning disks (150 MB/sec),
+    SATA SSDs (500 MB/sec), and PCIe SSDs (2,000+ MB/sec).
+
+#### Preprocessing on the CPU
+
+Placing input pipeline operations on the CPU can significantly improve
+performance. Utilizing the CPU for the input pipeline frees the GPU to focus on
+training. To ensure preprocessing is on the CPU, wrap the preprocessing
+operations as shown below:
+
+```python
+with tf.device('/cpu:0'):
+  # function to get and process images or data.
+  distorted_inputs = load_and_distort_images()
+```
+
+If using `tf.estimator.Estimator` the input function is automatically placed on
+the CPU.
+
+#### Using the Dataset API
+
+The @{$datasets$Dataset API} is replacing `queue_runner` as the recommended API
+for building input pipelines. The API was added to contrib as part of TensorFlow
+1.2 and will move to core in the near future. This
+[ResNet example](https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10_estimator/cifar10_main.py)
+([arXiv:1512.03385](https://arxiv.org/abs/1512.03385))
+training CIFAR-10 illustrates the use of the Dataset API along with
+`tf.estimator.Estimator`. The Dataset API utilizes C++ multi-threading and has a
+much lower overhead than the Python-based `queue_runner` that is limited by
+Python's multi-threading performance.
+
+While feeding data using a `feed_dict` offers a high level of flexibility, in
+most instances using `feed_dict` does not scale optimally. However, in instances
+where only a single GPU is being used the difference can be negligible. Using
+the Dataset API is still strongly recommended. Try to avoid the following:
+
+```python
+# feed_dict often results in suboptimal performance when using large inputs.
+sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})
+```
+
+#### Use large files
+
+Reading large numbers of small files significantly impacts I/O performance.
+One approach to get maximum I/O throughput is to preprocess input data into
+larger (~100MB) `TFRecord` files. For smaller data sets (200MB-1GB), the best
+approach is often to load the entire data set into memory. The document
+[Downloading and converting to TFRecord format](https://github.com/tensorflow/models/tree/master/slim#Data)
+includes information and scripts for creating `TFRecords` and this
+[script](https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10_estimator/generate_cifar10_tfrecords.py)
+converts the CIFAR-10 data set into `TFRecords`.
+
+### Data formats
+
+Data formats refers to the structure of the Tensor passed to a given Op. The
+discussion below is specifically about 4D Tensors representing images. In
+TensorFlow the parts of the 4D tensor are often referred to by the following
+letters:
+
+*   N refers to the number of images in a batch.
+*   H refers to the number of pixels in the vertical (height) dimension.
+*   W refers to the number of pixels in the horizontal (width) dimension.
+*   C refers to the channels. For example, 1 for black and white or grayscale
+    and 3 for RGB.
+
+Within TensorFlow there are two naming conventions representing the two most
+common data formats:
+
+*   `NCHW` or `channels_first`
+*   `NHWC` or `channels_last`
+
+`NHWC` is the TensorFlow default and `NCHW` is the optimal format to use when
+training on NVIDIA GPUs using [cuDNN](https://developer.nvidia.com/cudnn).
+
+The best practice is to build models that work with both data formats. This
+simplifies training on GPUs and then running inference on CPUs. If TensorFlow is
+compiled with the [Intel MKL](#tensorflow_with_intel_mkl-dnn) optimizations,
+many operations, especially those related to CNN based models, will be optimized
+and support `NCHW`. If not using the MKL, some operations are not supported on
+CPU when using `NCHW`.
+
+The brief history of these two formats is that TensorFlow started by using
+`NHWC` because it was a little faster on CPUs. In the long term, we are working
+on tools to auto rewrite graphs to make switching between the formats
+transparent and take advantages of micro optimizations where a GPU Op may be
+faster using `NHWC` than the normally most efficient `NCHW`.
+
+### Common fused Ops
+
+Fused Ops combine multiple operations into a single kernel for improved
+performance. There are many fused Ops within TensorFlow and @{$xla$XLA} will
+create fused Ops when possible to automatically improve performance. Collected
+below are select fused Ops that can greatly improve performance and may be
+overlooked.
+
+#### Fused batch norm
+
+Fused batch norm combines the multiple operations needed to do batch
+normalization into a single kernel. Batch norm is an expensive process that for
+some models makes up a large percentage of the operation time. Using fused batch
+norm can result in a 12%-30% speedup.
+
+There are two commonly used batch norms and both support fusing. The core
+@{tf.layers.batch_normalization} added fused starting in TensorFlow 1.3.
+
+```python
+bn = tf.layers.batch_normalization(
+    input_layer, fused=True, data_format='NCHW')
+```
+
+The contrib @{tf.contrib.layers.batch_norm} method has had fused as an option
+since before TensorFlow 1.0.
+
+```python
+bn = tf.contrib.layers.batch_norm(input_layer, fused=True, data_format='NCHW')
+```
+
+### Building and installing from source
+
+The default TensorFlow binaries target the broadest range of hardware to make
+TensorFlow accessible to everyone. If using CPUs for training or inference, it
+is recommended to compile TensorFlow with all of the optimizations available for
+the CPU in use. Speedups for training and inference on CPU are documented below
+in [Comparing compiler optimizations](#comparing-compiler-optimizations).
+
+To install the most optimized version of TensorFlow,
+@{$install_sources$build and install} from source. If there is a need to build
+TensorFlow on a platform that has different hardware than the target, then
+cross-compile with the highest optimizations for the target platform. The
+following command is an example of using `bazel` to compile for a specific
+platform:
 
 ```python
 # This command optimizes for Intel’s Broadwell processor
@@ -47,106 +186,467 @@ bazel build -c opt --copt=-march="broadwell" --config=cuda //tensorflow/tools/pi
 
 #### Environment, build, and install tips
 
-*   Compile with the highest level of compute the [GPU
-    supports](http://developer.nvidia.com/cuda-gpus), e.g. P100: 6.0, Titan X
-    (pascal): 6.2, Titan X (maxwell): 5.2, and K80: 3.7.
-*   Install the latest CUDA platform and cuDNN libraries.
-*   Make sure to use a version of gcc that supports all of the optimizations of
-    the target CPU. The recommended minimum gcc version is 4.8.3.  On OS X upgrade
-    to the latest Xcode version and use the version of clang that comes with Xcode.
-*   TensorFlow checks on startup whether it has been compiled with the
-    optimizations available on the CPU. If the optimizations are not included,
-    TensorFlow will emit warnings, e.g. AVX, AVX2, and FMA instructions not
-    included.
-
-### Utilize queues for reading data
-
-One common cause of poor performance is underutilizing GPUs, or essentially
-"starving" them of data by not setting up an efficient pipeline. Make sure to
-set up an input pipeline to utilize queues and stream data effectively. Review
-the @{$reading_data#reading_from_files$Reading Data guide} for implementation
-details. One way to identify a "starved" GPU is to generate and review
-timelines. A detailed tutorial for timelines does not exist, but a quick example
-of generating a timeline exists as part of the @{$jit$XLA JIT} tutorial. Another
-simple way to check if a GPU is underutilized is to run `watch nvidia-smi`, and
-if GPU utilization is not approaching 100% then the GPU is not getting data fast
-enough.
-
-Unless for a special circumstance or for example code, do not feed data
-into the session from Python variables, e.g. `dictionary`.
+*   `./configure` asks which compute capability to include in the build. This
+    does not impact overall performance but does impact initial startup. After
+    running TensorFlow once, the compiled kernels are cached by CUDA. If using
+    a docker container, the data is not cached and the penalty is paid each time
+    TensorFlow starts. The best practice is to include the
+    [compute capabilities](http://developer.nvidia.com/cuda-gpus)
+    of the GPUs that will be used, e.g. P100: 6.0, Titan X (Pascal): 6.1, Titan
+    X (Maxwell): 5.2, and K80: 3.7.
+*   Use a version of gcc that supports all of the optimizations of the target
+    CPU. The recommended minimum gcc version is 4.8.3. On OS X, upgrade to the
+    latest Xcode version and use the version of clang that comes with Xcode.
+*   Install the latest stable CUDA platform and cuDNN libraries supported by
+    TensorFlow.
+
+## Optimizing for GPU
+
+This section contains GPU-specific tips that are not covered in the
+[General best practices](#general-best-practices). Obtaining optimal performance
+on multi-GPUs is a challenge. A common approach is to use data parallelism.
+Scaling through the use of data parallelism involves making multiple copies of
+the model, which are referred to as "towers", and then placing one tower on each
+of the GPUs. Each tower operates on a different mini-batch of data and then
+updates variables, also known as parameters, that need to be shared between
+each of the towers. How each tower gets the updated variables and how the
+gradients are applied has an impact on the performance, scaling, and convergence
+of the model.  The rest of this section provides an overview of variable
+placement and the towering of a model on multiple GPUs.
+@{$performance_models$High-Performance Models} gets into more details regarding
+more complex methods that can be used to share and update variables between
+towers.
+
+The best approach to handling variable updates depends on the model, hardware,
+and even how the hardware has been configured. An example of this, is that two
+systems can be built with NVIDIA Tesla P100s but one may be using PCIe and the
+other [NVLink](http://www.nvidia.com/object/nvlink.html). In that scenario, the
+optimal solution for each system may be different. For real world examples, read
+the @{$benchmarks$benchmark} page which details the settings that were optimal
+for a variety of platforms. Below is a summary of what was learned from
+benchmarking various platforms and configurations:
+
+*   **Tesla K80**: If the GPUs are on the same PCI Express root complex and are
+    able to use [NVIDIA GPUDirect](https://developer.nvidia.com/gpudirect) Peer
+    to Peer, then placing the variables equally across the GPUs used for
+    training is the best approach. If the GPUs cannot use GPUDirect, then
+    placing the variables on the CPU is the best option.
+
+*   **Titan X (Maxwell and Pascal), M40, P100, and similar**: For models like
+    ResNet and InceptionV3, placing variables on the CPU is the optimal setting,
+    but for models with a lot of variables like AlexNet and VGG, using GPUs with
+    `NCCL` is better.
+
+A common approach to managing where variables are placed, is to create a method
+to determine where each Op is to be placed and use that method in place of a
+specific device name when calling `with tf.device():`. Consider a scenario where
+a model is being trained on 2 GPUs and the variables are to be placed on the
+CPU. There would be a loop for creating and placing the "towers" on each of the
+2 GPUs. A custom device placement method would be created that watches for Ops
+of type `Variable`, `VariableV2`, and `VarHandleOp` and indicates that they are
+to be placed on the CPU. All other Ops would be placed on the target GPU.
+The building of the graph would proceed as follows:
+
+*   On the first loop a "tower" of the model would be created for `gpu:0`.
+    During the placement of the Ops, the custom device placement method would
+    indicate that variables are to be placed on `cpu:0` and all other Ops on
+    `gpu:0`.
+
+*   On the second loop, `reuse` is set to `True` to indicate that variables are
+    to be reused and then the "tower" is created on `gpu:1`. During the
+    placement of the Ops associated with the "tower", the variables that were
+    placed on `cpu:0` are reused and all other Ops are created and placed on
+    `gpu:1`.
+
+The final result is all of the variables are placed on the CPU with each GPU
+having a copy of all of the computational Ops associated with the model.
+
+The code snippet below illustrates two different approaches for variable
+placement: one is placing variables on the CPU; the other is placing variables
+equally across the GPUs.
 
 ```python
-# Using feed_dict often results in suboptimal performance when using large inputs.
-sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})
-```
 
-### Preprocessing on the CPU
+class GpuParamServerDeviceSetter(object):
+  """Used with tf.device() to place variables on the least loaded GPU.
+
+    A common use for this class is to pass a list of GPU devices, e.g. ['gpu:0',
+    'gpu:1','gpu:2'], as ps_devices.  When each variable is placed, it will be
+    placed on the least loaded gpu. All other Ops, which will be the computation
+    Ops, will be placed on the worker_device.
+  """
+
+  def __init__(self, worker_device, ps_devices):
+    """Initializer for GpuParamServerDeviceSetter.
+    Args:
+      worker_device: the device to use for computation Ops.
+      ps_devices: a list of devices to use for Variable Ops. Each variable is
+      assigned to the least loaded device.
+    """
+    self.ps_devices = ps_devices
+    self.worker_device = worker_device
+    self.ps_sizes = [0] * len(self.ps_devices)
+
+  def __call__(self, op):
+    if op.device:
+      return op.device
+    if op.type not in ['Variable', 'VariableV2', 'VarHandleOp']:
+      return self.worker_device
+
+    # Gets the least loaded ps_device
+    device_index, _ = min(enumerate(self.ps_sizes), key=operator.itemgetter(1))
+    device_name = self.ps_devices[device_index]
+    var_size = op.outputs[0].get_shape().num_elements()
+    self.ps_sizes[device_index] += var_size
+
+    return device_name
+
+def _create_device_setter(is_cpu_ps, worker, num_gpus):
+  """Create device setter object."""
+  if is_cpu_ps:
+    # tf.train.replica_device_setter supports placing variables on the CPU, all
+    # on one GPU, or on ps_servers defined in a cluster_spec.
+    return tf.train.replica_device_setter(
+        worker_device=worker, ps_device='/cpu:0', ps_tasks=1)
+  else:
+    gpus = ['/gpu:%d' % i for i in range(num_gpus)]
+    return ParamServerDeviceSetter(worker, gpus)
+
+# The method below is a modified snippet from the full example.
+def _resnet_model_fn():
+    # When set to False, variables are placed on the least loaded GPU. If set
+    # to True, the variables will be placed on the CPU.
+    is_cpu_ps = False
+
+    # Loops over the number of GPUs and creates a copy ("tower") of the model on
+    # each GPU.
+    for i in range(num_gpus):
+      worker = '/gpu:%d' % i
+      # Creates a device setter used to determine where Ops are to be placed.
+      device_setter = _create_device_setter(is_cpu_ps, worker, FLAGS.num_gpus)
+      # Creates variables on the first loop.  On subsequent loops reuse is set
+      # to True, which results in the "towers" sharing variables.
+      with tf.variable_scope('resnet', reuse=bool(i != 0)):
+        with tf.name_scope('tower_%d' % i) as name_scope:
+          # tf.device calls the device_setter for each Op that is created.
+          # device_setter returns the device the Op is to be placed on.
+          with tf.device(device_setter):
+            # Creates the "tower".
+            _tower_fn(is_training, weight_decay, tower_features[i],
+                      tower_labels[i], tower_losses, tower_gradvars,
+                      tower_preds, False)
 
-Placing preprocessing operations on the CPU can significantly improve
-performance.  When preprocessing occurs on the GPU the flow of data is
-CPU -> GPU (preprocessing) -> CPU -> GPU (training).  The data is bounced back
-and forth between the CPU and GPU.  When preprocessing is placed on the CPU,
-the data flow is CPU (preprocessing) -> GPU (training).  Another benefit is
-preprocessing on the CPU frees GPU time to focus on training.
+```
 
-Placing preprocessing on the CPU can result in a 6X+ increase in samples/sec
-processed, which could lead to training in 1/6th of the time.  To ensure
-preprocessing is on the CPU, wrap the preprocessing operations as shown below:
+In the near future the above code will be for illustration purposes only as
+there will be easy to use high level methods to support a wide range of popular
+approaches. This
+[example](https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10_estimator)
+will continue to get updated as the API expands and evolves to address multi-GPU
+scenarios.
+
+## Optimizing for CPU
+
+CPUs, which includes Intel® Xeon Phi™, achieve optimal performance when
+TensorFlow is @{$install_sources$built from source} with all of the instructions
+supported by the target CPU.
+
+Beyond using the latest instruction sets, Intel® has added support for the
+Intel® Math Kernel Library for Deep Neural Networks (Intel® MKL-DNN) to
+TensorFlow. While the name is not completely accurate, these optimizations are
+often simply referred to as 'MKL' or 'TensorFlow with MKL'. [TensorFlow
+with Intel® MKL-DNN](#tensorflow_with_intel_mkl_dnn) contains details on the
+MKL optimizations.
+
+The two configurations listed below are used to optimize CPU performance by
+adjusting the thread pools.
+
+*   `intra_op_parallelism_threads`: Nodes that can use multiple threads to
+    parallelize their execution will schedule the individual pieces into this
+    pool.
+*   `inter_op_parallelism_threads`: All ready nodes are scheduled in this pool.
+
+These configurations are set via the `tf.ConfigProto` and passed to `tf.Session`
+in the `config` attribute as shown in the snippet below.  For both configuration
+options, if they are unset or set to 0, will default to the number of logical
+CPU cores. Testing has shown that the default is effective for systems ranging
+from one CPU with 4 cores to multiple CPUs with 70+ combined logical cores.
+A common alternative optimization is to set the number of threads in both pools
+equal to the number of physical cores rather than logical cores.
 
 ```python
-with tf.device('/cpu:0'):
-  # function to get and process images or data.
-  distorted_inputs = load_and_distort_images()
+
+  config = tf.ConfigProto()
+  config.intra_op_parallelism_threads = 44
+  config.inter_op_parallelism_threads = 44
+  tf.session(config=config)
+
 ```
 
-### Use large files
+The [Comparing compiler optimizations](#comparing-compiler-optimizations)
+section contains the results of tests that used different compiler
+optimizations.
 
-Under some circumstances, both the CPU and GPU can be starved for data by the
-I/O system. If you are using many small files to form your input data set, you
-may be limited by the speed of your filesystem. If your training loop runs
-faster when using SSDs vs HDDs for storing your input data, you could could be
-I/O bottlenecked.
+### TensorFlow with Intel® MKL DNN
 
-If this is the case, you should pre-process your input data, creating a few
-large TFRecord files.
+Intel® has added optimizations to TensorFlow for Intel® Xeon® and Intel® Xeon
+Phi™ though the use of Intel® Math Kernel Library for Deep Neural Networks
+(Intel® MKL-DNN) optimized primitives. The optimizations also provide speedups
+for the consumer line of processors, e.g. i5 and i7 Intel processors. The Intel
+published paper
+[TensorFlow* Optimizations on Modern Intel® Architecture](https://software.intel.com/en-us/articles/tensorflow-optimizations-on-modern-intel-architecture)
+contains additional details on the implementation.
 
-### Use NCHW image data format
+> Note: MKL was added as of TensorFlow 1.2 and currently only works on Linux. It
+> also does not work when also using `--config=cuda`.
 
-Image data format refers to the representation of batches of images. TensorFlow
-supports `NHWC` (TensorFlow default) and `NCHW` (cuDNN default). N refers to the
-number of images in a batch, H refers to the number of pixels in the vertical
-dimension, W refers to the number of pixels in the horizontal dimension, and C
-refers to the channels (e.g. 1 for black and white, 3 for RGB, etc.) Although
-cuDNN can operate on both formats, it is faster to operate in its default
-format.
+In addition to providing significant performance improvements for training CNN
+based models, compiling with the MKL creates a binary that is optimized for AVX
+and AVX2. The result is a single binary that is optimized and compatible with
+most modern (post-2011) processors.
 
-The best practice is to build models that work with both `NCHW` and `NHWC` as it
-is common to train using `NCHW` on GPU, and then do inference with NHWC on CPU.
+TensorFlow can be compiled with the MKL optimizations using the following
+commands that depending on the version of the TensorFlow source used.
 
-There are edge cases where `NCHW` can be slower on GPU than `NHWC`. One
-[case](https://github.com/tensorflow/tensorflow/issues/7551#issuecomment-280421351)
-is using non-fused batch norm on WRN-16-4 without dropout. In that case using
-fused batch norm, which is also recommended, is the optimal solution.
+For TensorFlow source versions after 1.3.0:
+
+```bash
+./configure
+# Pick the desired options
+bazel build --config=mkl -c opt //tensorflow/tools/pip_package:build_pip_package
+
+```
 
-The very brief history of these two formats is that TensorFlow started by using
-`NHWC` because it was a little faster on CPUs. Then the TensorFlow team
-discovered that `NCHW` performs better when using the NVIDIA cuDNN library.  The
-current recommendation is that users support both formats in their models. In
-the long term, we plan to rewrite graphs to make switching between the formats
-transparent.
+For TensorFlow versions 1.2.0 through 1.3.0:
 
-### Use fused batch norm
+```bash
+./configure
+Do you wish to build TensorFlow with MKL support? [y/N] Y
+Do you wish to download MKL LIB from the web? [Y/n] Y
+# Select the defaults for the rest of the options.
 
-When using batch norm
-@{tf.contrib.layers.batch_norm} set the attribute `fused=True`:
+bazel build --config=mkl --copt="-DEIGEN_USE_VML" -c opt //tensorflow/tools/pip_package:build_pip_package
+
+```
+
+#### Tuning MKL for the best performance
+
+This section details the different configurations and environment variables that
+can be used to tune the MKL to get optimal performance. Before tweaking various
+environment variables make sure the model is using the `NCHW` (`channels_first`)
+[data format](#data-formats). The MKL is optimized for `NCHW` and Intel is
+working to get near performance parity when using `NHWC`.
+
+MKL uses the following environment variables to tune performance:
+
+*   KMP_BLOCKTIME - Sets the time, in milliseconds, that a thread should wait,
+    after completing the execution of a parallel region, before sleeping.
+*   KMP_AFFINITY - Enables the run-time library to bind threads to physical
+    processing units.
+*   KMP_SETTINGS - Enables (true) or disables (false) the printing of OpenMP*
+    run-time library environment variables during program execution.
+*   OMP_NUM_THREADS - Specifies the number of threads to use.
+
+More details on the KMP variables are on
+[Intel's](https://software.intel.com/en-us/node/522775) site and the OMP
+variables on
+[gnu.org](https://gcc.gnu.org/onlinedocs/libgomp/Environment-Variables.html)
+
+While there can be substantial gains from adjusting the environment variables,
+which is discussed below, the simplified advice is to set the
+`inter_op_parallelism_threads` equal to the number of physical CPUs and to set
+the following environment variables:
+
+*   KMP_BLOCKTIME=0
+*   KMP_AFFINITY=granularity=fine,verbose,compact,1,0
+
+Example setting MKL variables with command-line arguments:
+
+```bash
+KMP_BLOCKTIME=0 KMP_AFFINITY=granularity=fine,verbose,compact,1,0 \
+KMP_SETTINGS=1 python your_python_script.py
+```
+
+Example setting MKL variables with python `os.environ`:
 
 ```python
-bn = tf.contrib.layers.batch_norm(
-          input_layer, fused=True, data_format='NCHW'
-          scope=scope, **kwargs)
+os.environ["KMP_BLOCKTIME"] = str(FLAGS.kmp_blocktime)
+os.environ["KMP_SETTINGS"] = str(FLAGS.kmp_settings)
+os.environ["KMP_AFFINITY"]= FLAGS.kmp_affinity
+if FLAGS.num_intra_threads > 0:
+  os.environ["OMP_NUM_THREADS"]= str(FLAGS.num_intra_threads)
+
 ```
 
-The non-fused batch norm does computations using several individual Ops. Fused
-batch norm combines the individual operations into a single kernel, which runs
-faster.
+There are models and hardware platforms that benefit from different settings.
+Each variable that impacts performance is discussed below.
+
+*   **KMP_BLOCKTIME**: The MKL default is 200ms, which was not optimal in our
+    testing. 0 (0ms) was a good default for CNN based models that were tested.
+    The best performance for AlexNex was achieved at 30ms and both GoogleNet and
+    VGG11 performed best set at 1ms.
+
+*   **KMP_AFFINITY**: The recommended setting is
+    `granularity=fine,verbose,compact,1,0`.
+
+*   **OMP_NUM_THREADS**: This defaults to the number of physical cores.
+    Adjusting this parameter beyond matching the number of cores can have an
+    impact when using Intel® Xeon Phi™ (Knights Landing) for some models. See
+    [TensorFlow* Optimizations on Modern Intel® Architecture](https://software.intel.com/en-us/articles/tensorflow-optimizations-on-modern-intel-architecture)
+    for optimal settings.
+
+*   **intra_op_parallelism_threads**: Setting this equal to the number of
+    physical cores is recommended. Setting the value to 0, which is the default
+    and will result in the value being set to the number of logical cores, is an
+    option to try for some architectures.  This value and `OMP_NUM_THREADS`
+    should be equal.
+
+*   **inter_op_parallelism_threads**: Setting this equal to the number of
+    sockets is recommended. Setting the value to 0, which is the default,
+    results in the value being set to the number of logical cores.
+
+### Comparing compiler optimizations
+
+Collected below are performance results running training and inference on
+different types of CPUs on different platforms with various compiler
+optimizations.  The models used were ResNet-50
+([arXiv:1512.03385](https://arxiv.org/abs/1512.03385)) and
+InceptionV3 ([arXiv:1512.00567](https://arxiv.org/abs/1512.00567)).
+
+For each test, when the MKL optimization was used the environment variable
+KMP_BLOCKTIME was set to 0 (0ms) and KMP_AFFINITY to
+`granularity=fine,verbose,compact,1,0`.
+
+#### Inference InceptionV3
+
+**Environment**
+
+*   Instance Type: AWS EC2 m4.xlarge
+*   CPU: Intel(R) Xeon(R) CPU E5-2686 v4 @ 2.30GHz (Broadwell)
+*   Dataset: ImageNet
+*   TensorFlow Version: 1.2.0 RC2
+*   Test Script: [tf_cnn_benchmarks.py](https://github.com/tensorflow/benchmarks/blob/mkl_experiment/scripts/tf_cnn_benchmarks/tf_cnn_benchmarks.py)
+
+**Batch Size: 1**
+
+Command executed for the MKL test:
+
+```bash
+python tf_cnn_benchmarks.py --forward_only=True --device=cpu --mkl=True \
+--kmp_blocktime=0 --nodistortions --model=inception3 --data_format=NCHW \
+--batch_size=1 --num_inter_threads=1 --num_intra_threads=4 \
+--data_dir=
+```
+
+| Optimization | Data Format | Images/Sec   | Intra threads | Inter Threads |
+:              :             : (step time)  :               :               :
+| ------------ | ----------- | ------------ | ------------- | ------------- |
+| AVX2         | NHWC        | 6.8 (147ms)  | 4             | 0             |
+| MKL          | NCHW        | 6.6 (151ms)  | 4             | 1             |
+| MKL          | NHWC        | 5.95 (168ms) | 4             | 1             |
+| AVX          | NHWC        | 4.7 (211ms)  | 4             | 0             |
+| SSE3         | NHWC        | 2.7 (370ms)  | 4             | 0             |
+
+**Batch Size: 32**
+
+Command executed for the MKL test:
+
+```bash
+python tf_cnn_benchmarks.py --forward_only=True --device=cpu --mkl=True \
+--kmp_blocktime=0 --nodistortions --model=inception3 --data_format=NCHW \
+--batch_size=32 --num_inter_threads=1 --num_intra_threads=4 \
+--data_dir=
+```
+
+| Optimization | Data Format | Images/Sec    | Intra threads | Inter Threads |
+:              :             : (step time)   :               :               :
+| ------------ | ----------- | ------------- | ------------- | ------------- |
+| MKL          | NCHW        | 10.24         | 4             | 1             |
+:              :             : (3125ms)      :               :               :
+| MKL          | NHWC        | 8.9 (3595ms)  | 4             | 1             |
+| AVX2         | NHWC        | 7.3 (4383ms)  | 4             | 0             |
+| AVX          | NHWC        | 5.1 (6275ms)  | 4             | 0             |
+| SSE3         | NHWC        | 2.8 (11428ms) | 4             | 0             |
+
+#### Inference ResNet-50
+
+**Environment**
+
+*   Instance Type: AWS EC2 m4.xlarge
+*   CPU: Intel(R) Xeon(R) CPU E5-2686 v4 @ 2.30GHz (Broadwell)
+*   Dataset: ImageNet
+*   TensorFlow Version: 1.2.0 RC2
+*   Test Script: [tf_cnn_benchmarks.py](https://github.com/tensorflow/benchmarks/blob/mkl_experiment/scripts/tf_cnn_benchmarks/tf_cnn_benchmarks.py)
+
+**Batch Size: 1**
+
+Command executed for the MKL test:
+
+```bash
+python tf_cnn_benchmarks.py --forward_only=True --device=cpu --mkl=True \
+--kmp_blocktime=0 --nodistortions --model=resnet50 --data_format=NCHW \
+--batch_size=1 --num_inter_threads=1 --num_intra_threads=4 \
+--data_dir=
+```
+
+| Optimization | Data Format | Images/Sec   | Intra threads | Inter Threads |
+:              :             : (step time)  :               :               :
+| ------------ | ----------- | ------------ | ------------- | ------------- |
+| AVX2         | NHWC        | 6.8 (147ms)  | 4             | 0             |
+| MKL          | NCHW        | 6.6 (151ms)  | 4             | 1             |
+| MKL          | NHWC        | 5.95 (168ms) | 4             | 1             |
+| AVX          | NHWC        | 4.7 (211ms)  | 4             | 0             |
+| SSE3         | NHWC        | 2.7 (370ms)  | 4             | 0             |
+
+**Batch Size: 32**
+
+Command executed for the MKL test:
+
+```bash
+python tf_cnn_benchmarks.py --forward_only=True --device=cpu --mkl=True \
+--kmp_blocktime=0 --nodistortions --model=resnet50 --data_format=NCHW \
+--batch_size=32 --num_inter_threads=1 --num_intra_threads=4 \
+--data_dir=
+```
+
+| Optimization | Data Format | Images/Sec    | Intra threads | Inter Threads |
+:              :             : (step time)   :               :               :
+| ------------ | ----------- | ------------- | ------------- | ------------- |
+| MKL          | NCHW        | 10.24         | 4             | 1             |
+:              :             : (3125ms)      :               :               :
+| MKL          | NHWC        | 8.9 (3595ms)  | 4             | 1             |
+| AVX2         | NHWC        | 7.3 (4383ms)  | 4             | 0             |
+| AVX          | NHWC        | 5.1 (6275ms)  | 4             | 0             |
+| SSE3         | NHWC        | 2.8 (11428ms) | 4             | 0             |
+
+#### Training InceptionV3
+
+**Environment**
+
+*   Instance Type: Dedicated AWS EC2 r4.16xlarge (Broadwell)
+*   CPU: Intel Xeon E5-2686 v4 (Broadwell) Processors
+*   Dataset: ImageNet
+*   TensorFlow Version: 1.2.0 RC2
+*   Test Script: [tf_cnn_benchmarks.py](https://github.com/tensorflow/benchmarks/blob/mkl_experiment/scripts/tf_cnn_benchmarks/tf_cnn_benchmarks.py)
+
+Command executed for MKL test:
+
+```bash
+python tf_cnn_benchmarks.py --device=cpu --mkl=True --kmp_blocktime=0 \
+--nodistortions --model=resnet50 --data_format=NCHW --batch_size=32 \
+--num_inter_threads=2 --num_intra_threads=36 \
+--data_dir=
+```
 
+Optimization | Data Format | Images/Sec | Intra threads | Inter Threads
+------------ | ----------- | ---------- | ------------- | -------------
+MKL          | NCHW        | 20.8       | 36            | 2
+AVX2         | NHWC        | 6.2        | 36            | 0
+AVX          | NHWC        | 5.7        | 36            | 0
+SSE3         | NHWC        | 4.3        | 36            | 0
+
+ResNet and [AlexNet](http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf)
+were also run on this configuration but in an ad hoc manner. There were not
+enough runs executed to publish a coherent table of results. The incomplete
+results strongly indicated the final result would be similar to the table above
+with MKL providing significant 3x+ gains over AVX2.
diff --git a/tensorflow/docs_src/programmers_guide/datasets.md b/tensorflow/docs_src/programmers_guide/datasets.md
index 68ed4bcd47c525..968c743a797bb7 100644
--- a/tensorflow/docs_src/programmers_guide/datasets.md
+++ b/tensorflow/docs_src/programmers_guide/datasets.md
@@ -1,4 +1,4 @@
-# Using the `Dataset` API for TensorFlow Input Pipelines
+# Importing Data
 
 The `Dataset` API enables you to build complex input pipelines from
 simple, reusable pieces. For example, the pipeline for an image model might
@@ -735,7 +735,7 @@ def dataset_input_fn():
 
     return {"image_data": image, "date_time": parsed["date_time"]}, label
 
-  # Use `Dataset.map()` to build a pair of a feature dictionary and a label 
+  # Use `Dataset.map()` to build a pair of a feature dictionary and a label
   # tensor for each example.
   dataset = dataset.map(parser)
   dataset = dataset.shuffle(buffer_size=10000)
diff --git a/tensorflow/docs_src/programmers_guide/dims_types.md b/tensorflow/docs_src/programmers_guide/dims_types.md
deleted file mode 100644
index 65b748d56ecf03..00000000000000
--- a/tensorflow/docs_src/programmers_guide/dims_types.md
+++ /dev/null
@@ -1,69 +0,0 @@
-# Tensor Ranks, Shapes, and Types
-
-TensorFlow programs use a tensor data structure to represent all data. You can
-think of a TensorFlow tensor as an n-dimensional array or list.
-A tensor has a static type and dynamic dimensions. Only tensors may be passed
-between nodes in the computation graph.
-
-## Rank
-
-In the TensorFlow system, tensors are described by a unit of dimensionality
-known as *rank*. Tensor rank is not the same as matrix rank. Tensor rank
-(sometimes referred to as *order* or *degree* or *n-dimension*) is the number
-of dimensions of the tensor. For example, the following tensor (defined as a
-Python list) has a rank of 2:
-
-    t = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
-
-A rank two tensor is what we typically think of as a matrix, a rank one tensor
-is a vector. For a rank two tensor you can access any element with the syntax
-`t[i, j]`.  For a rank three tensor you would need to address an element with
-`t[i, j, k]`.
-
-Rank | Math entity | Python example
---- | --- | ---
-0 | Scalar (magnitude only) | `s = 483`
-1 | Vector (magnitude and direction) | `v = [1.1, 2.2, 3.3]`
-2 | Matrix (table of numbers) | `m = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]`
-3 | 3-Tensor (cube of numbers) | `t = [[[2], [4], [6]], [[8], [10], [12]], [[14], [16], [18]]]`
-n | n-Tensor (you get the idea) | `....`
-
-## Shape
-
-The TensorFlow documentation uses three notational conventions to describe
-tensor dimensionality: rank, shape, and dimension number. The following table
-shows how these relate to one another:
-
-Rank | Shape | Dimension number | Example
---- | --- | --- | ---
-0 | [] | 0-D | A 0-D tensor.  A scalar.
-1 | [D0] | 1-D | A 1-D tensor with shape [5].
-2 | [D0, D1] | 2-D | A 2-D tensor with shape [3, 4].
-3 | [D0, D1, D2] | 3-D | A 3-D tensor with shape [1, 4, 3].
-n | [D0, D1, ... Dn-1] | n-D | A tensor with shape [D0, D1, ... Dn-1].
-
-Shapes can be represented via Python lists / tuples of ints, or with the
-@{tf.TensorShape}.
-
-## Data types
-
-In addition to dimensionality, Tensors have a data type. You can assign any one
-of the following data types to a tensor:
-
-Data type | Python type | Description
---- | --- | ---
-`DT_FLOAT` | `tf.float32` | 32 bits floating point.
-`DT_DOUBLE` | `tf.float64` | 64 bits floating point.
-`DT_INT8` | `tf.int8` | 8 bits signed integer.
-`DT_INT16` | `tf.int16` | 16 bits signed integer.
-`DT_INT32` | `tf.int32` | 32 bits signed integer.
-`DT_INT64` | `tf.int64` | 64 bits signed integer.
-`DT_UINT8` | `tf.uint8` | 8 bits unsigned integer.
-`DT_UINT16` | `tf.uint16` | 16 bits unsigned integer.
-`DT_STRING` | `tf.string` | Variable length byte arrays.  Each element of a Tensor is a byte array.
-`DT_BOOL` | `tf.bool` | Boolean.
-`DT_COMPLEX64` | `tf.complex64` | Complex number made of two 32 bits floating points: real and imaginary parts.
-`DT_COMPLEX128` | `tf.complex128` | Complex number made of two 64 bits floating points: real and imaginary parts.
-`DT_QINT8` | `tf.qint8` | 8 bits signed integer used in quantized Ops.
-`DT_QINT32` | `tf.qint32` | 32 bits signed integer used in quantized Ops.
-`DT_QUINT8` | `tf.quint8` | 8 bits unsigned integer used in quantized Ops.
diff --git a/tensorflow/docs_src/programmers_guide/estimators.md b/tensorflow/docs_src/programmers_guide/estimators.md
new file mode 100644
index 00000000000000..755bb049c99a71
--- /dev/null
+++ b/tensorflow/docs_src/programmers_guide/estimators.md
@@ -0,0 +1,153 @@
+# Estimators
+
+This document introduces **Estimators**--a high-level TensorFlow API that
+greatly simplifies machine learning programming. Estimators encapsulate
+the following actions:
+
+*   training
+*   evaluation
+*   prediction
+*   export for serving
+
+You may either use the pre-made Estimators we provide or write your
+own custom Estimators.  All Estimators--whether pre-made or custom--are
+classes based on the `tf.estimator.Estimator` class.
+
+Note: TensorFlow also provides an Estimator class at
+`tf.contrib.learn.Estimator`, which you should not use.
+
+
+## Advantages of Estimators
+
+Estimators provide the following benefits:
+
+*   You can run Estimators-based models on a local host or on a
+    distributed multi-server environment without changing your model.
+    Furthermore, you can run Estimators-based models on CPUs, GPUs,
+    or TPUs without recoding your model.
+*   Estimators simplify sharing implementations between model developers.
+*   You can develop a state of the art model with high-level intuitive code,
+    In short, it is generally much easier to create models with Estimators
+    than with the low-level TensorFlow APIs.
+*   Estimators are themselves built on tf.layers, which
+    simplifies customization.
+*   Estimators build the graph for you.  In other words, you don't have to
+    build the graph.
+*   Estimators provide a safe distributed training loop that controls how and
+    when to:
+    *   build the graph
+    *   initialize variables
+    *   start queues
+    *   handle exceptions
+    *   create checkpoint files and recover from failures
+    *   save summaries for TensorBoard
+
+When writing an application with Estimators, you must separate the data input
+pipeline from the model.  This separation simplifies experiments with
+different data sets.
+
+
+## Pre-made Estimators
+
+Pre-made Estimators enable you to work at a much higher conceptual level
+than the base TensorFlow APIs. You no longer have to worry about creating
+the computational graph or sessions since Estimators handle all
+the "plumbing" for you.  That is, pre-made Estimators create and manage
+`Graph` and `Session` objects for you.  Furthermore, pre-made Estimators
+let you experiment with different model architectures by making only minimal
+code changes.  `DNNClassifier`, for example, is a pre-made Estimator class that
+trains classification models through dense, feed-forward neural networks.
+
+
+### Structure of a pre-made Estimators program
+
+A TensorFlow program relying on a pre-made Estimator typically consists
+of the following four steps:
+
+1.  **Write one or more dataset importing functions.** For example, you might
+    create one function to import the training set and another function to
+    import the test set. Each dataset importing function must return two
+    objects:
+
+    *   a dictionary in which the keys are feature column names and the
+        values are Tensors (or SparseTensors) containing the corresponding
+        feature data
+    *   a Tensor containing one or more labels
+
+    For example, the following code illustrates the basic skeleton for
+    an input function:
+
+        def input_fn(dataset):
+           ...  # manipulate dataset, extracting feature names and the label
+           return feature_dict, label
+
+    See @{$datasets$Using the `Dataset` API for TensorFlow Input Pipelines}
+    for full details.)
+
+2.  **Define the feature columns.** Each @{tf.feature_column}
+    identifies a feature name, its type, and any input pre-processing.
+    For example, the following snippet creates three feature
+    columns that hold integer or floating-point data.  The first two
+    feature columns simply identify the feature's name and type. The
+    third feature column also specifies a lambda the program will invoke
+    to scale the raw data:
+
+        # Define three numeric feature columns.
+        population = tf.feature_column.numeric_column('population')
+        crime_rate = tf.feature_column.numeric_column('crime_rate')
+        median_education = tf.feature_column.numeric_column('median_education',
+                            normalizer_fn='lambda x: x - global_education_mean')
+
+3.  **Instantiate the relevant pre-made Estimator.**  For example, here's
+    a sample instantiation of a pre-made Estimator named `LinearClassifier`:
+
+        # Instantiate an estimator, passing the feature columns.
+        estimator = tf.estimator.Estimator.LinearClassifier(
+            feature_columns=[population, crime_rate, median_education],
+            )
+
+4.  **Call a training, evaluation, or inference method.**
+    For example, all Estimators provide a `train` method, which trains a model.
+
+        # my_training_set is the function created in Step 1
+        estimator.train(input_fn=my_training_set, steps=2000)
+
+
+### Benefits of pre-made Estimators
+
+Pre-made Estimators encode best practices, providing the following benefits:
+
+*   Best practices for determining where different parts of the computational
+    graph should run, implementing strategies on a single machine or on a
+    cluster.
+*   Best practices for event (summary) writing and universally useful
+    summaries.
+
+If you don't use pre-made Estimators, you must implement the preceding
+features yourself.
+
+
+## Custom Estimators
+
+The heart of every Estimator--whether pre-made or custom--is its
+**model function**, which is a method that builds graphs for training,
+evaluation, and prediction. When you are using a pre-made Estimator,
+someone else has already implemented the model function. When relying
+on a custom Estimator, you must write the model function yourself. A
+@{$extend/estimators$companion document}
+explains how to write the model function.
+
+
+## Recommended workflow
+
+We recommend the following workflow:
+
+1.  Assuming a suitable pre-made Estimator exists, use it to build your
+    first model and use its results to establish a baseline.
+2.  Build and test your overall pipeline, including the integrity and
+    reliability of your data with this pre-made Estimator.
+3.  If suitable alternative pre-made Estimators are available, run
+    experiments to determine which pre-made Estimator produces the
+    best results.
+4.  Possibly, further improve your model by building your own custom Estimator.
+
diff --git a/tensorflow/docs_src/programmers_guide/faq.md b/tensorflow/docs_src/programmers_guide/faq.md
index 56486a48b7adab..865016dc02d89f 100644
--- a/tensorflow/docs_src/programmers_guide/faq.md
+++ b/tensorflow/docs_src/programmers_guide/faq.md
@@ -53,10 +53,6 @@ TensorFlow assigns operations to devices, and the
 @{$deep_cnn$CIFAR-10 tutorial} for an example model that
 uses multiple GPUs.
 
-#### What are the different types of tensors that are available?
-
-TensorFlow supports a variety of different data types and tensor shapes. See the
-@{$dims_types$ranks, shapes, and types reference} for more details.
 
 ## Running a TensorFlow computation
 
@@ -171,7 +167,8 @@ available. These operations allow you to build sophisticated
 @{$reading_data$input pipelines}, at the cost of making the
 TensorFlow computation somewhat more complicated. See the how-to documentation
 for
-@{$reading_data#creating-threads-to-prefetch-using-queuerunner-objects$using `QueueRunner` objects to drive queues and readers}
+@{$reading_data#creating-threads-to-prefetch-using-queuerunner-objects$using
+`QueueRunner` objects to drive queues and readers}
 for more information on how to use them.
 
 ## Variables
@@ -240,11 +237,6 @@ to encode the batch size as a Python constant, but instead to use a symbolic
 * Use @{tf.reduce_mean} instead
   of `tf.reduce_sum(...) / batch_size`.
 
-* If you use
-  @{$reading_data#feeding$placeholders for feeding input},
-  you can specify a variable batch dimension by creating the placeholder with
-  [`tf.placeholder(..., shape=[None, ...])`](../api_docs/python/io_ops.md#placeholder). The
-  `None` element of the shape corresponds to a variable-sized dimension.
 
 ## TensorBoard
 
@@ -269,36 +261,33 @@ the flag --host=localhost. This should quiet any security warnings.
 
 ## Extending TensorFlow
 
-See also the how-to documentation for
+See the how-to documentation for
 @{$adding_an_op$adding a new operation to TensorFlow}.
 
 #### My data is in a custom format. How do I read it using TensorFlow?
 
-There are two main options for dealing with data in a custom format.
+There are three main options for dealing with data in a custom format.
+
+The easiest option is to write parsing code in Python that transforms the data
+into a numpy array. Then use @{tf.contrib.data.Dataset.from_tensor_slices} to
+create an input pipeline from the in-memory data.
 
-The easier option is to write parsing code in Python that transforms the data
-into a numpy array, then feed a
-@{tf.placeholder} a tensor with
-that data. See the documentation on
-@{$reading_data#feeding$using placeholders for input} for
-more details. This approach is easy to get up and running, but the parsing can
-be a performance bottleneck.
+If your data doesn't fit in memory, try doing the parsing in the Dataset
+pipeline. Start with an appropriate file reader, like
+@{tf.contrib.data.TextLineDataset}. Then convert the dataset by mapping
+@{tf.contrib.data.Dataset.map$mapping} appropriate operations over it.
+Prefer predefined TensorFlow operations such as @{tf.decode_raw},
+@{tf.decode_csv}, @{tf.parse_example}, or @{tf.image.decode_png}.
 
-The more efficient option is to
+If your data is not easily parsable with the built-in TensorFlow operations,
+consider converting it, offline, to a format that is easily parsable, such
+as ${tf.python_io.TFRecordWriter$`TFRecord`} format.
+
+The more efficient method to customize the parsing behavior is to
 @{$adding_an_op$add a new op written in C++} that parses your
-data format. The
-@{$new_data_formats$guide to handling new data formats} has
+data format. The @{$new_data_formats$guide to handling new data formats} has
 more information about the steps for doing this.
 
-#### How do I define an operation that takes a variable number of inputs?
-
-The TensorFlow op registration mechanism allows you to define inputs that are a
-single tensor, a list of tensors with the same type (for example when adding
-together a variable-length list of tensors), or a list of tensors with different
-types (for example when enqueuing a tuple of tensors to a queue).  See the
-how-to documentation for
-@{$adding_an_op#list-inputs-and-outputs$adding an op with a list of inputs or outputs}
-for more details of how to define these different input types.
 
 ## Miscellaneous
 
diff --git a/tensorflow/docs_src/programmers_guide/graphs.md b/tensorflow/docs_src/programmers_guide/graphs.md
index b2313a4a638df7..989018bc8655f9 100644
--- a/tensorflow/docs_src/programmers_guide/graphs.md
+++ b/tensorflow/docs_src/programmers_guide/graphs.md
@@ -44,8 +44,31 @@ programs:
 
 * **Portability.** The dataflow graph is a language-independent representation
   of the code in your model. You can build a dataflow graph in Python, store it
-  in a [SavedModel](TODO), and restore it in a C++ program for low-latency
-  inference.
+  in a @{$saved_model$SavedModel}, and restore it in a C++ program for
+  low-latency inference.
+
+
+## What is a @{tf.Graph}?
+
+A @{tf.Graph} contains two relevant kinds of information:
+
+* **Graph structure.** The nodes and edges of the graph, indicating how
+  individual operations are composed together, but not prescribing how they
+  should be used. The graph structure is like assembly code: inspecting it can
+  convey some useful information, but it does not contain all of the useful
+  context that source code conveys.
+
+* **Graph collections.** TensorFlow provides a general mechanism for storing
+  collections of metadata in a @{tf.Graph}. The @{tf.add_to_collection} function
+  enables you to associate a list of objects with a key (where @{tf.GraphKeys}
+  defines some of the standard keys), and @{tf.get_collection} enables you to
+  look up all objects associated with a key. Many parts of the TensorFlow
+  library use this facility: for example, when you create a @{tf.Variable}, it
+  is added by default to collections representing "global variables" and
+  "trainable variables". When you later come to create a @{tf.train.Saver} or
+  @{tf.train.Optimizer}, the variables in these collections are used as the
+  default arguments.
+
 
 ## Building a @{tf.Graph}
 
@@ -109,7 +132,7 @@ an operation:
   to all operations created in a particular context. The current name scope
   prefix is a `"/"`-delimited list of the names of all active @{tf.name_scope}
   context managers. If a name scope has already been used in the current
-  context, TensorFlow appens `"_1"`, `"_2"`, and so on. For example: 
+  context, TensorFlow appens `"_1"`, `"_2"`, and so on. For example:
 
   ```python
   c_0 = tf.constant(0, name="c")  # => operation named "c"
@@ -296,7 +319,7 @@ described below.
 * **`target`.** If this argument is left empty (the default), the session will
   only use devices in the local machine. However, you may also specify a
   `grpc://` URL to specify the address of a TensorFlow server, which gives the
-  session access to all devices on machines that that server controls. See 
+  session access to all devices on machines that that server controls. See
   @{tf.train.Server} for details of how to create a TensorFlow
   server. For example, in the common **between-graph replication**
   configuration, the @{tf.Session} connects to a @{tf.train.Server} in the same
@@ -419,89 +442,6 @@ with tf.Session() as sess:
   print(metadata.step_stats)
 ```
 
-## `GraphDef` and `MetaGraphDef`
-
-TensorFlow uses a dataflow graph as a portable representation for your
-application. A @{tf.Graph} contains two relevant kinds of information:
-
-* **Graph structure.** The nodes and edges of the graph, indicating how
-  individual operations are composed together, but not prescribing how they
-  should be used. The graph structure is like assembly code: inspecting it can
-  convey some useful information, but it does not contain all of the useful
-  context that source code conveys.
-
-* **Graph collections.** TensorFlow provides a general mechanism for storing
-  collections of metadata in a @{tf.Graph}. The @{tf.add_to_collection} function
-  enables you to associate a list of objects with a key (where @{tf.GraphKeys}
-  defines some of the standard keys), and @{tf.get_collection} enables you to
-  look up all objects associated with a key. Many parts of the TensorFlow
-  library use this facility: for example, when you create a @{tf.Variable}, it
-  is added by default to collections representing "global variables" and
-  "trainable variables". When you later come to create a @{tf.train.Saver} or
-  @{tf.train.Optimizer}, the variables in these collections are used as the
-  default arguments.
-
-A @{tf.Graph} can be saved in two forms:
-
-* @{tf.GraphDef}: This is a low-level representation of the graph structure,
-  containing a description of all of its operations (as @{tf.NodeDef} protocol
-  buffers) and the edges between them. The @{tf.GraphDef} representation is
-  primarily used with low-level APIs, such as the `tensorflow::Session` C++
-  API, and it typically requires additional context (such as the names of
-  particular operations) to make use of it. The @{tf.Graph.as_graph_def} method
-  converts a @{tf.Graph} to a @{tf.GraphDef}.
-
-* `tf.train.MetaGraphDef`: This is a higher-level representation of a dataflow
-  graph, which includes a @{tf.GraphDef}, and information that helps to
-  understand the graph (such as the contents of the graph collections). The
-  @{tf.train.export_meta_graph} function converts a @{tf.Graph} to a
-  `tf.train.MetaGraphDef`. The @{tf.train.Saver.save} method also writes a
-  `tf.train.MetaGraphDef` that can be used in conjunction with the saved
-  checkpoint to restore the state of a training process at the point it was
-  saved.
-
-In most cases, we encourage you to use `tf.train.MetaGraphDef` instead of
-@{tf.GraphDef}. There are cases where a @{tf.GraphDef} can be useful---for
-example, when performing low-level graph modifications using functions like
-@{tf.import_graph_def} or
-the
-[Graph Transform](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/graph_transforms/README.md) tool---but
-`tf.train.MetaGraphDef` is a better building block for high-level applications.
-For example the [SavedModel library](TODO) uses `tf.train.MetaGraphDef` to
-package up a @{tf.Graph} and a set of trained model parameters so that it can be
-used for serving.
-
-If you have a `tf.train.MetaGraphDef`, the @{tf.train.import_meta_graph}
-function will load it into the default graph. Calling this function has two
-main features:
-
-1. It will restore the contents of the graph collections from the original
-   graph. APIs such as @{tf.global_variables} and the default arguments to
-   APIs like @{tf.train.Optimizer.minimize} will work the same way as they
-   did in the original graph.
-
-2. The function returns a @{tf.train.Saver}, which can be used to restore the
-   state (trained parameters, etc.) associated with the graph from a checkpoint.
-   The @{tf.train.latest_checkpoint} function can help to find the latest
-   checkpoint from a particular checkpoint directory.
-
-If you have a @{tf.GraphDef}, the @{tf.import_graph_def} function enables you
-to load the graph into an existing Python @{tf.Graph} object. To make use of the
-imported graph, you must know the names of operations or tensors in the
-@{tf.GraphDef}. The @{tf.import_graph_def} function has two main features to
-help you use the imported graph:
-
-1. You can **rebind** tensors in the imported graph to @{tf.Tensor} objects in
-   the default graph by passing the optional `input_map` argument. For example,
-   `input_map` enables you to take import a graph fragment defined in a
-   @{tf.GraphDef}, and statically connect tensors in the graph you are
-   building to @{tf.placeholder} tensors in that fragment.
-
-2. You can **return** @{tf.Tensor} or @{tf.Operation} objects from the imported
-   graph by passing their names in the `return_elements` list.
-
-In addition, you can use @{tf.device} and @{tf.name_scope} to control the
-device placement and name of the imported nodes.
 
 ## Visualizing your graph
 
@@ -577,7 +517,7 @@ the default graph, which can be useful in more advanced used cases. For example:
 
 * The default graph stores information about every @{tf.Operation} and
   @{tf.Tensor} that was ever added to it. If your program creates a large number
-  of unconnected subgraphs, it may be more efficient to use a different 
+  of unconnected subgraphs, it may be more efficient to use a different
   @{tf.Graph} to build each subgraph, so that unrelated state can be garbage
   collected.
 
diff --git a/tensorflow/docs_src/programmers_guide/index.md b/tensorflow/docs_src/programmers_guide/index.md
index aa2e12504ddc35..eef35d6dcc70ec 100644
--- a/tensorflow/docs_src/programmers_guide/index.md
+++ b/tensorflow/docs_src/programmers_guide/index.md
@@ -1,38 +1,42 @@
 # Programmer's Guide
 
 The documents in this unit dive into the details of writing TensorFlow
-code.  This section begins with the following guides, each of which
-explain a particular aspect of TensorFlow:
-
-  * @{$variables$Variables: Creation, Initialization, Saving, Loading, and
-     Sharing}, which details the mechanics of TensorFlow Variables.
-  * @{$dims_types$Tensor Ranks, Shapes, and Types}, which explains Tensor
-    rank (the number of dimensions), shape (the size of each dimension),
-    and datatypes.
-  * @{$threading_and_queues$Threading and Queues}, which explains TensorFlow's
-    rich queuing system.
-  * @{$reading_data$Reading Data}, which documents three different mechanisms
-    for getting data into a TensorFlow program.
-
-The following guide is helpful when training a complex model over multiple
-days:
-
-  * @{$supervisor$Supervisor: Training Helper for Days-Long Trainings}, which
-    explains how to gracefully handle system crashes during a lengthy training
-    session.
-
-TensorFlow provides a debugger named `tfdbg`, which is documented in the
-following guide:
-
-  * @{$debugger$Debugging TensorFlow Programs},
-    which walks you through the use of `tfdbg` within an application. It covers
-    using `tfdbg` with both the low-level TensorFlow API and the Estimator API.
-
-To learn about the TensorFlow versioning scheme consult:
-
-  * @{$version_compat$The TensorFlow Version Compatibility Guide}, which explains
-TensorFlow's versioning nomenclature and compatibility rules.
-
-We conclude this section with a FAQ about TensorFlow programming:
-
-  * @{$faq$Frequently Asked Questions}
+code.  For TensorFlow 1.3, we revised this document extensively.
+The units are now as follows:
+
+  * @{$programmers_guide/estimators$Estimators}, which introduces a high-level
+    TensorFlow API that greatly simplifies ML programming.
+  * @{$programmers_guide/tensors$Tensors}, which explains how to create,
+    manipulate, and access Tensors--the fundamental object in TensorFlow.
+  * @{$programmers_guide/variables$Variables}, which details how
+    to represent shared, persistent state in your program.
+  * @{$programmers_guide/graphs$Graphs and Sessions}, which explains:
+      * dataflow graphs, which are TensorFlow's representation of computations
+        as dependencies between operations.
+      * sessions, which are TensorFlow's mechanism for running dataflow graphs
+        across one or more local or remote devices.
+    If you are programming with the low-level TensorFlow API, this unit
+    is essential. If you are programming with a high-level TensorFlow API
+    such as Estimators or Keras, the high-level API creates and manages
+    graphs and sessions for you, but understanding graphs and sessions
+    can still be helpful.
+  * @{$programmers_guide/saved_model$Saving and Restoring}, which
+    explains how to save and restore variables and models.
+  * @{$programmers_guide/datasets$Input Pipelines}, which explains how to
+    set up data pipelines to read data sets into your TensorFlow program.
+  * @{$programmers_guide/threading_and_queues$Threading and Queues}, which
+    explains TensorFlow's older system for multi-threaded, queue-based input
+    pipelines. Beginning with TensorFlow 1.2, we recommend using the
+    `tf.contrib.data` module instead, which is documented in the
+    "Input Pipelines" unit.
+  * @{$programmers_guide/embedding$Embeddings}, which introduces the concept
+    of embeddings, provides a simple example of training an embedding in
+    TensorFlow, and explains how to view embeddings with the TensorBoard
+    Embedding Projector.
+  * @{$programmers_guide/debugger$Debugging TensorFlow Programs}, which
+    explains how to use the TensorFlow debugger (tfdbg).
+  * @{$programmers_guide/version_compat$TensorFlow Version Compatibility},
+    which explains backward compatibility guarantees and non-guarantees.
+  * @{$programmers_guide/faq$FAQ}, which contains frequently asked
+    questions about TensorFlow. (We have not revised this document for v1.3,
+    except to remove some obsolete information.)
diff --git a/tensorflow/docs_src/programmers_guide/leftnav_files b/tensorflow/docs_src/programmers_guide/leftnav_files
index 2a58c4647d1c4b..f2642e601496c6 100644
--- a/tensorflow/docs_src/programmers_guide/leftnav_files
+++ b/tensorflow/docs_src/programmers_guide/leftnav_files
@@ -1,15 +1,12 @@
 index.md
+estimators.md
 tensors.md
 variables.md
-dims_types.md
 graphs.md
+saved_model.md
 datasets.md
 threading_and_queues.md
-reading_data.md
 embedding.md
 debugger.md
-supervisor.md
-saved_model.md
-meta_graph.md
 version_compat.md
 faq.md
diff --git a/tensorflow/docs_src/programmers_guide/saved_model.md b/tensorflow/docs_src/programmers_guide/saved_model.md
index 2d90399d32e133..80592d6ff13509 100644
--- a/tensorflow/docs_src/programmers_guide/saved_model.md
+++ b/tensorflow/docs_src/programmers_guide/saved_model.md
@@ -150,7 +150,7 @@ Notes:
    @{tf.variables_initializer} for more information.
 
 *  To inspect the variables in a checkpoint, you can use the
-   [`inspect_checkpoint`](https://cs.corp.google.com/#piper///depot/google3/third_party/tensorflow/python/tools/inspect_checkpoint.py)
+   [`inspect_checkpoint`](https://www.tensorflow.org/code/tensorflow/python/tools/inspect_checkpoint.py)
    library, particularly the `print_tensors_in_checkpoint_file` function.
 
 *  By default, `Saver` uses the value of the @{tf.Variable.name} property
@@ -432,7 +432,7 @@ the same keys.  These `SignatureDef`s differ only in their outputs, as
 provided by the corresponding `ExportOutput` entry.  The inputs are always
 those provided by the `serving_input_receiver_fn`.
 An inference request may specify the head by name.  One head must be named
-using [`signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY`](https://cs.corp.google.com/#piper///depot/google3/third_party/tensorflow/python/saved_model/signature_constants.py)
+using [`signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY`](https://www.tensorflow.org/code/tensorflow/python/saved_model/signature_constants.py)
 indicating which `SignatureDef` will be served when an inference request
 does not specify one.
 
@@ -443,7 +443,7 @@ For local deployment, you can serve your model using
 [TensorFlow Serving](http://github.com/tensorflow/serving), an open-source project that loads a
 SavedModel and exposes it as a [gRPC](http://www.grpc.io/) service.
 
-First, [install TensorFlow Serving](http://github.com/tensorflow/serving).
+First, [install TensorFlow Serving](https://tensorflow.github.io/serving/setup#prerequisites).
 
 Then build and run the local model server, substituting `$export_dir_base` with
 the path to the SavedModel you exported above:
@@ -565,7 +565,7 @@ If you built TensorFlow from source code, you must run the following
 additional command to build `saved_model_cli`:
 
 ```
-$ blaze build third_party/tensorflow/python/tools:saved_model_cli
+$ bazel build tensorflow/python/tools:saved_model_cli
 ```
 
 ### Overview of commands
@@ -879,5 +879,3 @@ of checkpoints and assets:
 
 Each graph is associated with a specific set of tags, which enables
 identification during a load or restore operation.
-
-
diff --git a/tensorflow/docs_src/programmers_guide/supervisor.md b/tensorflow/docs_src/programmers_guide/supervisor.md
deleted file mode 100644
index ec7c91b1472f21..00000000000000
--- a/tensorflow/docs_src/programmers_guide/supervisor.md
+++ /dev/null
@@ -1,402 +0,0 @@
-# Supervisor: Training Helper for Days-Long Trainings.
-
-To train a model with TensorFlow you can simply run a training op a number of
-times and save a checkpoint of the trained parameters when you're done.  This
-works well for small models that can train in a few hours.
-
-Larger models that require days of training, possibly across multiple replicas,
-need a more robust training process that:
-
- * Handles shutdowns and crashes cleanly.
- * Can be resumed after a shutdown or a crash.
- * Can be monitored through TensorBoard.
-
-To be able to resume training after a shutdown or a crash the training process
-must save checkpoints regularly.  On restart, it must look for the most recent
-checkpoint and load it before resuming training.
-
-To be monitored through TensorBoard, the training process must run summary ops
-regularly and append the returned values to an events file as explained in
-@{$summaries_and_tensorboard$TensorBoard: Visualizing Learning}.
-TensorBoard monitors events files and displays graphs reporting training
-progress over time.
-
-The @{tf.train.Supervisor} provides
-a set of services that helps implement a robust training process.
-
-This how-to shows how to use the supervisor directly.  Please also consider
-using one of several frameworks built on top of the supervisor that provide
-richer training loops, and numerous customization options:
-@{$python/contrib.learn$`tf.learn`} is a good choice.
-
-Note that the supervisor is very helpful for training large models, but can
-also be used for smaller models without any penalty.
-
-## Very Simple Scenario
-
-The simplest scenario for using a supervisor is to:
-
- * Create a `Supervisor` object, passing it the path to a directory where to
-   save checkpoints and summaries.
-
- * Ask the supervisor for a session with
-   @{tf.train.Supervisor.managed_session}.
-
- * Use the session to execute a train op, checking at each step if the
-   supervisor requests that the training stops.
-
-```python
-  ...create graph...
-  my_train_op = ...
-
-  sv = tf.train.Supervisor(logdir="/my/training/directory")
-  with sv.managed_session() as sess:
-    for step in range(100000):
-      if sv.should_stop():
-        break
-      sess.run(my_train_op)
-```
-
-### Started Services
-
-In the very simple scenario, the `managed_session()` call starts a few
-services, which run in their own threads, and use the managed session to run
-ops in your graph.
-
-If your graph contains an integer variable named `global_step`, the services
-use its value to measure the number of training steps executed.  See the @{$mechanics#training$MNIST training tutorial} for how to
-create a `global_step` variable.
-
- * _Checkpointing_ service: Saves a copy of the graph variables in the logdir.
-   The checkpoint filename uses the value of the `global_step` variable if one
-   was added to your graph.  Runs every 10 minutes by default.
-
- * _Summary_ service: Runs all the summary ops and appends their output to an
-   @{$summaries_and_tensorboard$events file} in the logdir.  Runs
-   every 2 minutes by default.
-
- * _Step counter_: Counts how many steps have been executed, by looking at
-   changes in the `global_step` variable.  Appends a summary to the events file
-   reporting the number of global steps per second.  The summary tag is
-   "global_step/sec".  This also runs every 2 minutes by default.
-
- * _Queue Runners_: If any @{tf.train.QueueRunner} were added to the
-   graph, the supervisor launches them in their own threads.
-
-All time intervals can be changed when constructing the supervisor object.  See
-the [supervisor reference](#supervisor_reference) for details.
-
-### Checking for Stop
-
-The check for stop in the main training loop is important and necessary.
-
-Exceptions raised in the service threads are reported to the supervisor which
-then sets its `should_stop()` condition to true.  Other service threads notice
-that condition and terminate properly.  The main training loop, within the
-`managed_session()` block, must also check for the stop condition and
-terminate.
-
-Note that `managed_session()` takes care of catching exceptions raised from the
-training loop to report them to the supervisor.  The main loop does not need to
-do anything special about exceptions.  It only needs to check for the stop
-condition.
-
-### Recovery
-
-If the training program shuts down or crashes, its most recent checkpoint and
-event files are left in the logdir.  When you restart the program,
-`managed_session()` restores the graph from the most recent checkpoint and
-resumes training where it stopped.
-
-A new events file is created.  If you start TensorBoard and point it to the
-logdir, it will know how to merge the contents of the two events files and will
-show the training resuming at the last global step from the checkpoint.
-
-## Larger Model Scenario
-
-The very simple scenario is sufficient for most small to medium sized models.
-Larger models may run out memory when the summary service runs: The summary ops
-are run in parallel with the main loop running the train op.  This can cause
-memory usage to peak to up to two times the normal use.
-
-For a larger model you can tell the supervisor to not run the summary service
-and instead run it yourself in your main training loop: pass `summary_op=None`
-when constructing the supervisor.
-
-For example this code runs the summary op every 100 steps in the training loop:
-
-```python
-  ...create graph...
-  my_train_op = ...
-  my_summary_op = tf.summary.merge_all()
-
-  sv = tf.train.Supervisor(logdir="/my/training/directory",
-                     summary_op=None) # Do not run the summary service
-  with sv.managed_session() as sess:
-    for step in range(100000):
-      if sv.should_stop():
-        break
-      if step % 100 == 0:
-        _, summ = sess.run([my_train_op, my_summary_op])
-        sv.summary_computed(sess, summ)
-      else:
-        sess.run(my_train_op)
-```
-
-## Pre-trained Model Scenario
-
-The `managed_session()` call takes care of initializing the model in the
-session.  The model is restored from a checkpoint if one is available,
-or initialized from scratch otherwise.
-
-One common scenario is to initialize the model by loading a "pre-trained"
-checkpoint that was saved while training a usually slightly different model
-using a different dataset.
-
-You can load a pre-trained checkpoint by passing an "init function" to the
-supervisor.  This function is called only if the model needs to be initialized
-from scratch, not when the model can be recovered from a checkpoint from the
-logdir.
-
-To load the pre-trained model, the init function needs a
-@{tf.train.Saver} object, so you should create
-a saver for this purpose.  This is usually a good idea because the new model
-may contain variables that are not present in the pre-trained checkpoint: This
-saver must only restore the pre-trained variables.  If you were using the
-default saver, you could get an error trying to restore all the variables of
-the new model from the pre-trained checkpoint.
-
-```python
-  ...create graph...
-  # Create a saver that restores only the pre-trained variables.
-  pre_train_saver = tf.train.Saver([pre_train_var1, pre_train_var2])
-
-  # Define an init function that loads the pretrained checkpoint.
-  def load_pretrain(sess):
-    pre_train_saver.restore(sess, "")
-
-  # Pass the init function to the supervisor.
-  #
-  # The init function is called _after_ the variables have been initialized
-  # by running the init_op.
-  sv = tf.train.Supervisor(logdir="/my/training/directory",
-                     init_fn=load_pretrain)
-  with sv.managed_session() as sess:
-    # Here sess was either initialized from the pre-trained-checkpoint or
-    # recovered from a checkpoint saved in a previous run of this code.
-    ...
-```
-
-## Running Your Own Services
-
-Supervisor services, such as the checkpointing service, run in threads parallel
-to the main training loop.  You sometimes want to add your own services, for
-example to fetch different sets of summaries on a different schedule than the
-usual summary service.
-
-Use the @{tf.train.Supervisor.loop} method of
-the supervisor for this purpose.  It repeatedly calls a function of your choice
-on a timer until the supervisor stop condition becomes true, so it plays nicely
-with the other services.
-
-Example: Call `my_additional_summaries()` every 20mn:
-
-```python
-
-def my_additional_summaries(sv, sess):
- ...fetch and write summaries, see below...
-
-...
-  sv = tf.train.Supervisor(logdir="/my/training/directory")
-  with sv.managed_session() as sess:
-    # Call my_additional_summaries() every 1200s, or 20mn,
-    # passing (sv, sess) as arguments.
-    sv.loop(1200, my_additional_summaries, args=(sv, sess))
-    ...main training loop...
-```
-
-## Writing Summaries
-
-The supervisor always creates an events file in its logdir, as well as a
-@{tf.summary.FileWriter} to append
-events and summaries to that file.  If you want to write your own summaries it
-is a good idea to append them to that same events file: TensorBoard likes it
-better when only one events file in a directory is being actively appended to.
-
-The supervisor provides a helper function to append summaries:
-@{tf.train.Supervisor.summary_computed}.
-Just pass to the function the output returned by a summary op.  Here is an
-example of using that function to implement `my_additional_summaries()` from the
-previous example:
-
-```python
-def my_additional_summaries(sv, sess):
-  summaries = sess.run(my_additional_summary_op)
-  sv.summary_computed(sess, summaries)
-```
-
-For more advanced usages, the supervisor provides access to its summary writer
-through its
-@{tf.train.Supervisor.summary_writer}
-attribute.
-
-## Supervisor Reference
-
-The [Very Simple Scenario](#very_simple_scenario), and the [Larger Model
-Scenario](#larger_model_scenario) show basic uses of a supervisor.  More
-advanced scenarios can be constructed by using the many options provided by the
-supervisor
-
-### Checkpointing: Where and When.
-
-The `managed_session()` call launches the checkpointing service, which can be
-configured by the following keyword arguments to the `Supervisor()`
-constructor:
-
- * `logdir`: path to a directory where the checkpointing service creates
-   checkpoints.  The directory is created if needed.  Passing `None` disables
-   the checkpointing and the summary services.
-
- * `checkpoint_basename`: Name of the checkpoint files to create, defaults to
-   "model.ckpt".
-
-   If the model contains a scalar integer variable named `global_step`, the
-   value of that variable is appended to the checkpoint filename.
-
-   For example, at global step 1234 the checkpoint filename is
-   "model.ckpt-1234".
-
- * `save_model_secs`: Number of seconds between each checkpoint.  Defaults to
-   600, or 10 minutes.
-
-   When choosing a value, consider how much work you want to lose in case of a
-   crash: you will never lose more than `save_model_secs` seconds of work.
-   Setting this to 0 disables the checkpointing service.
-
- * `saver`: A @{tf.train.Saver} object to use
-   for checkpointing.
-
-   If you do not pass one, the supervisor creates one for you by calling
-   `tf.train.Saver()`, which add ops to save and restore all variables in your model.
-   This is usually what you need.
-
-Example: Use a custom Saver and checkpoint every 30 seconds.
-
-```python
-  ...create graph...
-  my_saver = tf.train.Saver()
-  sv = tf.train.Supervisor(logdir="/my/training/directory",
-                     saver=my_saver,
-                     save_model_secs=30)
-  with sv.managed_session() as sess:
-    ...training loop...
-```
-
-### Summaries: Where and When.
-
-The `managed_session()` call launches the summary service which fetches
-summaries and reports the number of steps executed per second.  It can be
-configured by the following keyword arguments to the `Supervisor()`
-constructor:
-
- * `logdir`: Path to a directory where the summary service creates event files.
-   The directory is created if needed.  Passing `None` disables the summary
-   service as well as the checkpointing services.
-
- * `save_summaries_secs`: Number of seconds between each run of the summary
-   service.  Defaults to 120, or 2 minutes.
-
-   When choosing a value, consider how expensive your summaries are, and how
-   much disk they will occupy.  Pass 0 to disable the summary service.
-
- * `summary_op`: Op to use to fetch the summaries.
-
-   If not specified, the supervisor use the first op in the
-   `tf.GraphKeys.SUMMARY_OP` @{tf.Graph.add_to_collection$graph collection}.  If
-   the collection is empty the supervisor creates an op that aggregates all
-   summaries in the graph using `tf.summary.merge_all()`.
-
-   Passing `None` disables the summary service.
-
- * `global_step`: Tensor to use to count the global step.
-
-   If not specified, the supervisor uses the first tensor in the
-   `tf.GraphKeys.GLOBAL_STEP` @{tf.Graph.add_to_collection$graph collection}.  If
-   the collection is empty, the supervisor looks for a scalar integer variable
-   named `global_step` in the graph.
-
-   If found, the global step tensor is used to measure the number of training
-   steps executed.  Note that your training op is responsible for incrementing
-   the global step value.
-
-### Model Initialization and Recovery
-
-The `managed_session()` call takes care of initializing or recovering a
-session.  It returns a session with a fully initialized model, ready to run
-ops.  If a checkpoint exists in the logdir when `managed_session()` is called,
-the model is initialized by loading that checkpoint, otherwise it is
-initialized by calling an init op and optionally an init function.
-
-When no checkpoint is available, model initialization is controlled by the
-following keyword arguments to the `Supervisor()` constructor:
-
- * `init_op`: Op to run to initialize the model.
-
-   If not specified, the supervisor uses the first op in the
-   `tf.GraphKeys.INIT_OP` collection.  If the collection is empty, the
-   supervisor adds an op to initialize all the variables in the graph by
-   calling `tf.global_variables_initializer()`.
-
-   Pass `None` to not use an init op.
-
- * `init_fn`: Python function to call to initialize the model.
-
-   If specified, called as `init_fn(sess)` where `sess` is the managed session.
-   If an init op is also used, the init function is called _after_ the init op.
-
- * `local_init_op`: An additional op to initialize parts of the graph that are
-   not saved in checkpoints such as tables and
-   @{tf.contrib.framework.local_variable$local variables}. The
-   local init op is run _before_ the init op and the init function.
-
-   If not specified, the supervisor uses the first op in the
-   `tf.GraphKeys.LOCAL_INIT_OP` collection.  If the collection is empty the
-   supervisor adds an op to initialize all the tables and local variables in
-   the graph by calling `tf.tables_initializer()` and
-   `tf.local_variables_initializer()`.
-
-   Pass `None` to not use a local init op.
-
- * `ready_op`: Op to check if the model is initialized.
-
-   After running the local init op, the init op, and the init function, the
-   supervisor verifies that the model is fully initialized by running the ready
-   op.  This is an op that returns an empty string if the model is initialized,
-   or a description of what parts of the model are not initialized if not.
-
-   If not specified, the supervisor uses the first op in the
-   `tf.GraphKeys.READY_OP` collection.  If the collection is empty the
-   supervisor creates a ready op that verifies that all variables are
-   initialized by calling `tf.report_uninitialized_variables()`.
-
-   Pass `None` to disable the ready op.  In that case the model is not
-   checked after initialization.
-
-Checkpoint recovery is controlled by the following keyword arguments to the
-`Supervisor()` constructor:
-
- * `logdir`: Path to a directory in which to look for checkpoints.  The
-  checkpoint service saves a metadata file, named "checkpoint", in the
-  checkpoint directory that indicates the path to the most recent checkpoint.
-
-  This file is in text format. When in a pinch, you can edit it manually to
-  recover from a different checkpoint than the most recent one.
-
- * `ready_op`: (see above).  The ready op is run before and after loading the
-   checkpoint.  The first run checks if the model needs to be initialized and
-   the second run verifies that the model is fully initialized.
-
- * `local_init_op`: (see above).  The local init op is run before running the
-   ready op the first time, to initialize local variables and tables.
-
- * `saver`: (see above).  Saver object used to load the checkpoint.
diff --git a/tensorflow/docs_src/programmers_guide/threading_and_queues.md b/tensorflow/docs_src/programmers_guide/threading_and_queues.md
index 3483c7533cb330..313de178de26c5 100644
--- a/tensorflow/docs_src/programmers_guide/threading_and_queues.md
+++ b/tensorflow/docs_src/programmers_guide/threading_and_queues.md
@@ -51,21 +51,89 @@ and @{tf.RandomShuffleQueue},
 are important TensorFlow objects that aid in computing tensors asynchronously
 in a graph.
 
-For example, a typical input pipeline uses a `RandomShuffleQueue` to
-prepare inputs for training a model:
+For example, a typical queue-based input pipeline uses a `RandomShuffleQueue` to
+prepare inputs for training a model as follows:
 
-* Multiple threads prepare training examples and enqueue them in the queue.
+* Multiple threads prepare training examples and enqueue them.
 * A training thread executes a training op that dequeues mini-batches from the
   queue
 
-This architecture has many benefits, as highlighted in the
-@{$reading_data$Reading data how to}, which also gives an overview of
-functions that simplify the construction of input pipelines.
+We recommend using the @{tf.contrib.data.Dataset.shuffle$`shuffle`}
+and @{tf.contrib.data.Dataset.batch$`batch`} methods of a
+@{tf.contrib.data.Dataset$`Dataset`} to acomplish this. However, if you'd prefer
+to use a queue-based version instead, you can find a full implementation in the
+@{tf.train.shuffle_batch} function.
 
-The TensorFlow `Session` object is multithreaded and thread-safe, so multiple
-threads can
+For demonstration purposes a simplified implementation is given below.
+
+This function takes a source tensor, a capacity, and a batch size as arguments
+and returns a tensor that dequeues a shuffled batch when executed.
+
+``` python
+def simple_shuffle_batch(source, capacity, batch_size=10):
+  # Create a random shuffle queue.
+  queue = tf.RandomShuffleQueue(capacity=capacity,
+                                min_after_dequeue=int(0.9*capacity),
+                                shapes=source.shape, dtypes=source.dtype)
+
+  # Create an op to enqueue one item.
+  enqueue = queue.enqueue(source)
+
+  # Create a queue runner that, when started, will launch 4 threads applying
+  # that enqueue op.
+  num_threads = 4
+  qr = tf.train.QueueRunner(queue, [enqueue] * num_threads)
+
+  # Register the queue runner so it can be found and started by
+  # `tf.train.start_queue_runners` later (the threads are not launched yet).
+  tf.train.add_queue_runner(qr)
+
+  # Create an op to dequeue a batch
+  return queue.dequeue_many(batch_size)
+```
+
+Once started by @{tf.train.start_queue_runners}, or indirectly through
+@{tf.train.MonitoredSession}, the `QueueRunner` will launch the
+threads in the background to fill the queue. Meanwhile the main thread will
+execute the `dequeue_many` op to pull data from it. Note how these ops do not
+depend on each other, except indirectly through the internal state of the queue.
+
+The simplest possible use of this function might be something like this:
+
+``` python
+# create a dataset that counts from 0 to 99
+input = tf.constant(list(range(100)))
+input = tf.contrib.data.Dataset.from_tensor_slices(input)
+input = input.make_one_shot_iterator().get_next()
+
+# Create a slightly shuffled batch from the sorted elements
+get_batch = simple_shuffle_batch(input, capacity=20)
+
+# `MonitoredSession` will start and manage the `QueueRunner` threads.
+with tf.train.MonitoredSession() as sess:
+  # Since the `QueueRunners` have been started, data is available in the
+  # queue, so the `sess.run(get_batch)` call will not hang.
+  while not sess.should_stop():
+    print(sess.run(get_batch))
+```
+
+```
+[ 8 10  7  5  4 13 15 14 25  0]
+[23 29 28 31 33 18 19 11 34 27]
+[12 21 37 39 35 22 44 36 20 46]
+...
+```
+
+For most use cases, the automatic thread startup and management provided
+by @{tf.train.MonitoredSession} is sufficient. In the rare case that it is not,
+TensorFlow provides tools for manually managing your threads and queues.
+
+## Manual Thread Management
+
+As we have seen, the TensorFlow `Session` object is multithreaded and
+thread-safe, so multiple threads can
 easily use the same session and run ops in parallel.  However, it is not always
-easy to implement a Python program that drives threads as described above.  All
+easy to implement a Python program that drives threads as required.  All
 threads must be able to stop together, exceptions must be caught and
 reported, and queues must be properly closed when stopping.
 
@@ -77,7 +145,7 @@ stop together and report exceptions to a program that waits for them to stop.
 The `QueueRunner` class is used to create a number of threads cooperating to
 enqueue tensors in the same queue.
 
-## Coordinator
+### Coordinator
 
 The @{tf.train.Coordinator} class manages background threads in a TensorFlow
 program and helps multiple threads stop together.
@@ -124,7 +192,7 @@ Obviously, the coordinator can manage threads doing very different things.
 They don't have to be all the same as in the example above.  The coordinator
 also has support to capture and report exceptions.  See the @{tf.train.Coordinator} documentation for more details.
 
-## QueueRunner
+### QueueRunner
 
 The @{tf.train.QueueRunner} class creates a number of threads that repeatedly
 run an enqueue op.  These threads can use a coordinator to stop together.  In
@@ -152,7 +220,7 @@ threads to process and enqueue examples.  Create a `Coordinator` and ask the
 queue runner to start its threads with the coordinator.  Write a training loop
 that also uses the coordinator.
 
-```
+```python
 # Create a queue runner that will run 4 threads in parallel to enqueue
 # examples.
 qr = tf.train.QueueRunner(queue, [enqueue_op] * 4)
@@ -164,20 +232,21 @@ coord = tf.train.Coordinator()
 enqueue_threads = qr.create_threads(sess, coord=coord, start=True)
 # Run the training loop, controlling termination with the coordinator.
 for step in xrange(1000000):
-    if coord.should_stop():
-        break
-    sess.run(train_op)
+  if coord.should_stop():
+    break
+  sess.run(train_op)
 # When done, ask the threads to stop.
 coord.request_stop()
 # And wait for them to actually do it.
 coord.join(enqueue_threads)
 ```
 
-## Handling exceptions
+### Handling exceptions
 
 Threads started by queue runners do more than just run the enqueue ops.  They
 also catch and handle exceptions generated by queues, including the
-`tf.errors.OutOfRangeError` exception, which is used to report that a queue was closed.
+`tf.errors.OutOfRangeError` exception, which is used to report that a queue was
+closed.
 
 A training program that uses a coordinator must similarly catch and report
 exceptions in its main loop.
@@ -186,15 +255,15 @@ Here is an improved version of the training loop above.
 
 ```python
 try:
-    for step in xrange(1000000):
-        if coord.should_stop():
-            break
-        sess.run(train_op)
+  for step in xrange(1000000):
+    if coord.should_stop():
+      break
+    sess.run(train_op)
 except Exception, e:
-    # Report exceptions to the coordinator.
-    coord.request_stop(e)
+  # Report exceptions to the coordinator.
+  coord.request_stop(e)
 finally:
-    # Terminate as usual. It is safe to call `coord.request_stop()` twice.
-    coord.request_stop()
-    coord.join(threads)
+  # Terminate as usual. It is safe to call `coord.request_stop()` twice.
+  coord.request_stop()
+  coord.join(threads)
 ```
diff --git a/tensorflow/docs_src/programmers_guide/variables.md b/tensorflow/docs_src/programmers_guide/variables.md
index 2f8e7eef5ed597..dd18760e1dd8e0 100644
--- a/tensorflow/docs_src/programmers_guide/variables.md
+++ b/tensorflow/docs_src/programmers_guide/variables.md
@@ -52,6 +52,8 @@ other_variable = tf.get_variable("other_variable", dtype=tf.int32,
 Note that when the initializer is a `tf.Tensor` you should not specify the
 variable's shape, as the shape of the initializer tensor will be used.
 
+
+
 ### Variable collections
 
 Because disconnected parts of a TensorFlow program might want to create
@@ -223,152 +225,6 @@ with tf.control_dependencies([assignment]):
                       # assign_add operation.
 ```
 
-## Saving and Restoring
-
-The easiest way to save and restore a model is to use a `tf.train.Saver` object.
-The constructor adds `save` and `restore` ops to the graph for all, or a
-specified list, of the variables in the graph.  The `Saver` object provides
-methods to run these ops, specifying paths for the checkpoint files to write to
-or read from.
-
-To restore a model checkpoint without a graph, you must first import the graph
-from the `MetaGraph` file (typical extension is `.meta`). Do this by calling
-@{tf.train.import_meta_graph}, which in turn returns a `Saver` from which one
-can than perform a `restore`.
-
-### Checkpoint Files
-
-TensorFlow saves variables in binary files that, roughly speaking, map variable
-names to tensor values.
-
-When you create a `Saver` object, you can optionally choose names for the
-variables in the checkpoint files.  By default, `Saver` uses the value of the
-@{tf.Variable.name} property for
-each variable.
-
-To inspect the variables in a checkpoint, you can use
-the
-[`inspect_checkpoint`](https://www.tensorflow.org/code/tensorflow/python/tools/inspect_checkpoint.py) library,
-particularly the `print_tensors_in_checkpoint_file` function.
-
-### Saving Variables
-
-Create a `Saver` with `tf.train.Saver()` to manage all variables in the
-model. For example, the following snippet demonstrates how to call the
-`tf.train.Saver.save` method to save variables to a checkpoint file:
-
-```python
-# Create some variables.
-v1 = tf.get_variable("v1", shape=[3], initializer = tf.zeros_initializer)
-v2 = tf.get_variable("v2", shape=[5], initializer = tf.zeros_initializer)
-
-inc_v1 = v1.assign(v1+1)
-dec_v2 = v2.assign(v2-1)
-
-# Add an op to initialize the variables.
-init_op = tf.global_variables_initializer()
-
-# Add ops to save and restore all the variables.
-saver = tf.train.Saver()
-
-# Later, launch the model, initialize the variables, do some work, and save the
-# variables to disk.
-with tf.Session() as sess:
-  sess.run(init_op)
-  # Do some work with the model.
-  inc_v1.op.run()
-  dec_v2.op.run()
-  # Save the variables to disk.
-  save_path = saver.save(sess, "/tmp/model.ckpt")
-  print("Model saved in file: %s" % save_path)
-```
-
-### Restoring Variables
-
-The `tf.train.Saver` object not only saves variables to checkpoint files, it
-also restores variables.  Note that when you restore variables from a file you
-do not have to initialize them beforehand. For example, the following snippet
-demonstrates how to call the `tf.train.Saver.restore` method to restore
-variables from a checkpoint file:
-
-```python
-tf.reset_default_graph()
-
-# Create some variables.
-v1 = tf.get_variable("v1", shape=[3])
-v2 = tf.get_variable("v2", shape=[5])
-
-# Add ops to save and restore all the variables.
-saver = tf.train.Saver()
-
-# Later, launch the model, use the saver to restore variables from disk, and
-# do some work with the model.
-with tf.Session() as sess:
-  # Restore variables from disk.
-  saver.restore(sess, "/tmp/model.ckpt")
-  print("Model restored.")
-  # Check the values of the variables
-  print("v1 : %s" % v1.eval())
-  print("v2 : %s" % v2.eval())
-```
-
-
-
-### Choosing which Variables to Save and Restore
-
-If you do not pass any argument to `tf.train.Saver()`, the saver handles all
-variables in the graph.  Each variable is saved under the name that was passed
-when the variable was created.
-
-It is sometimes useful to explicitly specify names for variables in the
-checkpoint files.  For example, you may have trained a model with a variable
-named `"weights"` whose value you want to restore into a variable named
-`"params"`.
-
-It is also sometimes useful to only save or restore a subset of the variables
-used by a model.  For example, you may have trained a neural net with five
-layers, and you now want to train a new model with six layers that reuses the
-existing weights of the five trained layers. You can use the saver to restore
-the weights of just the first five layers.
-
-You can easily specify the names and variables to save or load by passing to the
-`tf.train.Saver()` constructor either a list of variables (which will be stored 
-under their own names), or a Python dictionary in which keys are the names to 
-use and values are the variables to manage. 
-
-Continuing from the save/restore examples, above:
-
-```python
-tf.reset_default_graph()
-# Create some variables.
-v1 = tf.get_variable("v1", [3], initializer = tf.zeros_initializer)
-v2 = tf.get_variable("v2", [5], initializer = tf.zeros_initializer)
-
-# Add ops to save and restore only `v2` using the name "v2"
-saver = tf.train.Saver({"v2": v2})
-
-# Use the saver object normally after that.
-with tf.Session() as sess:
-  # Initialize v1 since the saver will not.
-  v1.initializer.run()
-  saver.restore(sess, "/tmp/model.ckpt")
-  
-  print("v1 : %s" % v1.eval())
-  print("v2 : %s" % v2.eval())
-
-```
-
-Notes:
-
-*  You can create as many `Saver` objects as you want if you need to save and
-   restore different subsets of the model variables.  The same variable can be
-   listed in multiple saver objects, its value is only changed when the
-   `Saver.restore()` method is run.
-
-*  If you only restore a subset of the model variables at the start of a
-   session, you have to run an initialize op for the other variables.  See
-   @{tf.variables_initializer} for more information.
-
 
 ## Sharing variables
 
diff --git a/tensorflow/docs_src/tutorials/image_retraining.md b/tensorflow/docs_src/tutorials/image_retraining.md
index 898db0e73cbade..b0e715edcb2b03 100644
--- a/tensorflow/docs_src/tutorials/image_retraining.md
+++ b/tensorflow/docs_src/tutorials/image_retraining.md
@@ -137,7 +137,7 @@ Once TensorBoard is running, navigate your web browser to `localhost:6006` to vi
 
 The script will log TensorBoard summaries to `/tmp/retrain_logs` by default. You can change the directory with the `--summaries_dir` flag.
 
-The [TensorBoard README](https://www.tensorflow.org/code/tensorflow/tensorboard/README.md) has a lot more information on TensorBoard usage, including tips & tricks, and debugging information.
+The [TensorBoard's GitHub](https://github.com/tensorflow/tensorboard) has a lot more information on TensorBoard usage, including tips & tricks, and debugging information.
 
 ## Using the Retrained Model
 
@@ -337,7 +337,7 @@ the (much larger) training set.
 
 By default the script uses a pretrained version of the Inception v3 model
 architecture. This is a good place to start because it provides high accuracy
-results, but if you intend to deploy your model on mobile devices or other 
+results, but if you intend to deploy your model on mobile devices or other
 resource-constrained environments you may want to trade off a little accuracy
 for much smaller file sizes or faster speeds. To help with that, the
 [retrain.py script](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/image_retraining/retrain.py)
diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 42839c2b948fec..1bb61bbb097a8e 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -235,7 +235,7 @@ def abs(x, name=None):
   absolute value is computed as \\( \sqrt{a^2 + b^2}\\).  For example:
   ```
   # tensor 'x' is [[-2.25 + 4.75j], [-3.25 + 5.75j]]
-  tf.complex_abs(x) ==> [5.25594902, 6.60492229]
+  tf.abs(x) ==> [5.25594902, 6.60492229]
   ```
 
   Args:
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 83a9313e50a62b..9137b4a0d79582 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -730,13 +730,13 @@ def _py_wrap_cc_impl(ctx):
     fail("Exactly one SWIG source file label must be specified.", "srcs")
   module_name = ctx.attr.module_name
   src = ctx.files.srcs[0]
-  inputs = set([src])
+  inputs = depset([src])
   inputs += ctx.files.swig_includes
   for dep in ctx.attr.deps:
     inputs += dep.cc.transitive_headers
   inputs += ctx.files._swiglib
   inputs += ctx.files.toolchain_deps
-  swig_include_dirs = set(_get_repository_roots(ctx, inputs))
+  swig_include_dirs = depset(_get_repository_roots(ctx, inputs))
   swig_include_dirs += sorted([f.dirname for f in ctx.files._swiglib])
   args = [
       "-c++", "-python", "-module", module_name, "-o", ctx.outputs.cc_out.path,
@@ -753,7 +753,7 @@ def _py_wrap_cc_impl(ctx):
       outputs=outputs,
       mnemonic="PythonSwig",
       progress_message="SWIGing " + src.path)
-  return struct(files=set(outputs))
+  return struct(files=depset(outputs))
 
 
 _py_wrap_cc = rule(
@@ -826,7 +826,7 @@ def _get_repository_roots(ctx, files):
 
 # Bazel rule for collecting the header files that a target depends on.
 def _transitive_hdrs_impl(ctx):
-  outputs = set()
+  outputs = depset()
   for dep in ctx.attr.deps:
     outputs += dep.cc.transitive_headers
   return struct(files=outputs)
@@ -866,10 +866,10 @@ def tf_custom_op_library_additional_deps():
 # tf_collected_deps will be the union of the deps of the current target
 # and the tf_collected_deps of the dependencies of this target.
 def _collect_deps_aspect_impl(target, ctx):
-  alldeps = set()
+  alldeps = depset()
   if hasattr(ctx.rule.attr, "deps"):
     for dep in ctx.rule.attr.deps:
-      alldeps = alldeps | set([dep.label])
+      alldeps = alldeps | depset([dep.label])
       if hasattr(dep, "tf_collected_deps"):
         alldeps = alldeps | dep.tf_collected_deps
   return struct(tf_collected_deps=alldeps)
diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh
index 5bea031b689c71..82b41d4f9d6622 100755
--- a/tensorflow/tools/ci_build/ci_sanity.sh
+++ b/tensorflow/tools/ci_build/ci_sanity.sh
@@ -418,6 +418,10 @@ do_pip_smoke_test() {
     "The pip smoke test failed."
 }
 
+do_code_link_check() {
+  tensorflow/tools/ci_build/code_link_check.sh
+}
+
 do_check_load_py_test() {
   BUILD_CMD="bazel build //tensorflow/tools/pip_package:check_load_py_test"
   ${BUILD_CMD}
@@ -431,8 +435,8 @@ do_check_load_py_test() {
 }
 
 # Supply all sanity step commands and descriptions
-SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_buildifier" "do_bazel_nobuild" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check" "do_pip_smoke_test" "do_check_load_py_test")
-SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test")
+SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_buildifier" "do_bazel_nobuild" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check" "do_pip_smoke_test" "do_check_load_py_test" "do_code_link_check")
+SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links")
 
 INCREMENTAL_FLAG=""
 
diff --git a/tensorflow/tools/ci_build/code_link_check.sh b/tensorflow/tools/ci_build/code_link_check.sh
new file mode 100755
index 00000000000000..e7eaa49d08afb3
--- /dev/null
+++ b/tensorflow/tools/ci_build/code_link_check.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# please run this at root directory of tensorflow
+success=1
+
+for i in `grep -onI https://www.tensorflow.org/code/\[a-zA-Z0-9/._-\]\* -r tensorflow`
+do
+  filename=`echo $i|awk -F: '{print $1}'`
+  linenumber=`echo $i|awk -F: '{print $2}'`
+  target=`echo $i|awk -F: '{print $4}'|tail -c +27`
+
+  # skip files in tensorflow/models
+  if [[ $target == tensorflow_models/* ]] ; then
+    continue
+  fi
+
+  if [ ! -f $target ] && [ ! -d $target ]; then
+    success=0
+    echo Broken link $target at line $linenumber of file $filename
+  fi
+done
+
+if [ $success == 0 ]; then
+  echo Code link check fails.
+  exit 1
+fi
+
+echo Code link check success.
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 2bc97facebf07e..0d28c43afcf9ec 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -29,7 +29,7 @@
 # This version string is semver compatible, but incompatible with pip.
 # For pip, we will remove all '-' characters from this string, and use the
 # result for pip.
-_VERSION = '1.3.0'
+_VERSION = '1.3.1'
 
 REQUIRED_PACKAGES = [
     'numpy >= 1.11.0',
diff --git a/tensorflow/tools/test/performance.bzl b/tensorflow/tools/test/performance.bzl
index 64fff844a70d43..b5c4bbf5a700ae 100644
--- a/tensorflow/tools/test/performance.bzl
+++ b/tensorflow/tools/test/performance.bzl
@@ -21,8 +21,8 @@ def tf_cc_logged_benchmark(
     fail(" ".join(("Target must be a single well-defined test, e.g.,",
                    "//path/to:test. Received: %s" % target)))
 
-  all_tags = list(set(tags) + \
-                  set(["benchmark-test", "local", "manual", "regression-test"]))
+  all_tags = list(depset(tags) + \
+                  depset(["benchmark-test", "local", "manual", "regression-test"]))
 
   tf_py_test(
       name = name,
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 42840a2c2c7dc7..8cb5a1f680ca47 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -196,7 +196,6 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       name = "gemmlowp",
       urls = [
           "http://mirror.bazel.build/github.com/google/gemmlowp/archive/a6f29d8ac48d63293f845f2253eccbf86bc28321.tar.gz",
-          "https://github.com/google/gemmlowp/archive/a6f29d8ac48d63293f845f2253eccbf86bc28321.tar.gz",
       ],
       sha256 = "75d40ea8e68b0d1644f052fffe8f14a410b2a73d40ccb859a95c0578d194ec26",
       strip_prefix = "gemmlowp-a6f29d8ac48d63293f845f2253eccbf86bc28321",
@@ -327,7 +326,6 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   patched_http_archive(
       name = "protobuf",
       urls = [
-          "https://github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz",
           "http://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz",
       ],
       sha256 = "6d43b9d223ce09e5d4ce8b0060cb8a7513577a35a64c7e3dad10f0703bf3ad93",
@@ -485,9 +483,8 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       name = "llvm",
       urls = [
           "http://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/9889fe2290766430b99a2d4fadbc5ba92f8004b6.tar.gz",
-          "https://github.com/llvm-mirror/llvm/archive/9889fe2290766430b99a2d4fadbc5ba92f8004b6.tar.gz",
       ],
-      sha256 = "00fb4a83a4dd1c046b19730a80e2183acc647715b7a8dcc8e808d49ea5530ca8",
+      sha256 = "a8da6d42ac7419e543a27e405f8b660f7b065e9ba981cc9cdcdcecb81af9cc43",
       strip_prefix = "llvm-9889fe2290766430b99a2d4fadbc5ba92f8004b6",
       build_file = str(Label("//third_party/llvm:llvm.BUILD")),
       repository = tf_repo_name,
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index b85e565f362633..721bf1eb7c0e4d 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -117,7 +117,7 @@ def get_cxx_inc_directories(repository_ctx, cc):
   includes_cpp = _get_cxx_inc_directories_impl(repository_ctx, cc, True)
   includes_c = _get_cxx_inc_directories_impl(repository_ctx, cc, False)
 
-  includes_cpp_set = set(includes_cpp)
+  includes_cpp_set = depset(includes_cpp)
   return includes_cpp + [inc for inc in includes_c
                          if inc not in includes_cpp_set]