From 6636d794ed96c62ee7113f11280081a6ef1337a6 Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Mon, 13 Feb 2017 11:52:21 -0800 Subject: [PATCH 01/50] C++ docs: add doxygen group annotations to source code Change: 147373087 --- tensorflow/cc/client/client_session.h | 5 +++++ tensorflow/cc/framework/cc_op_gen.cc | 30 ++++++++++++++++++++++++++- tensorflow/cc/framework/ops.h | 7 +++++++ tensorflow/cc/framework/scope.h | 5 +++++ tensorflow/cc/ops/const_op.h | 5 +++++ tensorflow/core/framework/tensor.h | 1 + tensorflow/core/lib/core/status.h | 2 ++ 7 files changed, 54 insertions(+), 1 deletion(-) diff --git a/tensorflow/cc/client/client_session.h b/tensorflow/cc/client/client_session.h index 28ff3ec9641b24..a6fe0205a0aa87 100644 --- a/tensorflow/cc/client/client_session.h +++ b/tensorflow/cc/client/client_session.h @@ -31,6 +31,9 @@ limitations under the License. namespace tensorflow { +/// @addtogroup core +/// @{ + /// A `ClientSession` object lets the caller drive the evaluation of the /// TensorFlow graph constructed with the C++ API. /// @@ -101,6 +104,8 @@ class ClientSession { TF_DISALLOW_COPY_AND_ASSIGN(ClientSession); }; +/// @} + } // end namespace tensorflow #endif // TENSORFLOW_CC_CLIENT_CLIENT_SESSION_H_ diff --git a/tensorflow/cc/framework/cc_op_gen.cc b/tensorflow/cc/framework/cc_op_gen.cc index 5f85d8c5edfe73..2d52aa18b83088 100644 --- a/tensorflow/cc/framework/cc_op_gen.cc +++ b/tensorflow/cc/framework/cc_op_gen.cc @@ -57,6 +57,16 @@ string GetPath(const string& dot_h_fname) { return result; } +// Converts: some/path/to/file.xx +// to: file +// (note that suffix is removed) +string GetFilename(const string& path) { + size_t slash_pos = path.rfind('/'); + if (slash_pos == path.npos) slash_pos = -1; + size_t dot_pos = path.rfind('.'); + return path.substr(slash_pos + 1, dot_pos - (slash_pos + 1)); +} + // Converts: // cc/ops/gen_foo_ops.h // to: @@ -77,6 +87,17 @@ string ToGuard(const string& path) { return guard; } +// Converts: some_name_xyz +// to: Some Name Xyz +string ToTitle(const string& name) { + string title = name; + for (int i = 0; i < title.size(); ++i) { + if (title[i] == '_') title[i] = ' '; + } + str_util::TitlecaseString(&title, " "); + return title; +} + // Change: Into: // ABC /// ABC // /// @@ -841,6 +862,10 @@ namespace ops { )include", "#include \"", op_header, "\"\n", namespace_begin); + const string filename = GetFilename(dot_h_fname); + const string doxygen = strings::StrCat("/// @defgroup ", filename, " ", + ToTitle(filename), "\n", "/// @{\n\n"); + TF_CHECK_OK(h->Append( strings::StrCat("// This file is MACHINE GENERATED! Do not edit.\n\n" "#ifndef ", @@ -850,6 +875,7 @@ namespace ops { *op_header_guard, "\n\n"))); TF_CHECK_OK(h->Append(header)); TF_CHECK_OK(h->Append(namespace_begin)); + TF_CHECK_OK(h->Append(doxygen)); TF_CHECK_OK(cc->Append(cc_header)); } @@ -860,7 +886,9 @@ void FinishFiles(bool internal, WritableFile* h, WritableFile* cc, } // namespace tensorflow )footer" : - R"footer(} // namespace ops + R"footer(/// @} + +} // namespace ops } // namespace tensorflow )footer"; diff --git a/tensorflow/cc/framework/ops.h b/tensorflow/cc/framework/ops.h index d4f1079c3b2805..c47d30ec3c5068 100644 --- a/tensorflow/cc/framework/ops.h +++ b/tensorflow/cc/framework/ops.h @@ -26,8 +26,13 @@ limitations under the License. namespace tensorflow { +/// @defgroup core Core Tensorflow API + class Output; +/// @addtogroup core +/// @{ + /// Represents a node in the computation graph. class Operation { public: @@ -284,6 +289,8 @@ class InputList { std::vector inputs_; }; +/// @} + } // namespace tensorflow #endif // THIRD_PARTY_TENSORFLOW_CC_FRAMEWORK_OPS_H_ diff --git a/tensorflow/cc/framework/scope.h b/tensorflow/cc/framework/scope.h index 47d1026bb23ddb..2963442ec848b0 100644 --- a/tensorflow/cc/framework/scope.h +++ b/tensorflow/cc/framework/scope.h @@ -33,6 +33,9 @@ class GraphDef; class NodeBuilder; struct CompositeOpScopes; +/// @addtogroup core +/// @{ + /// A `Scope` object represents a set of related TensorFlow ops that have the /// same properties such as a common name prefix. /// @@ -273,6 +276,8 @@ struct CompositeOpScopes { Scope last; }; +/// @} + } // namespace tensorflow #endif // THIRD_PARTY_TENSORFLOW_CC_FRAMEWORK_SCOPE_H_ diff --git a/tensorflow/cc/ops/const_op.h b/tensorflow/cc/ops/const_op.h index 8976a24edc6cc6..e8cb6cf1dd1b18 100644 --- a/tensorflow/cc/ops/const_op.h +++ b/tensorflow/cc/ops/const_op.h @@ -23,6 +23,9 @@ limitations under the License. namespace tensorflow { namespace ops { +/// @defgroup const_op Const Op +/// @{ + Output Const(const Scope& scope, const Input::Initializer& val); NodeBuilder::NodeOut AsNodeOut(const Scope& scope, const Input& inp); @@ -70,6 +73,8 @@ Output Const(const Scope& scope, const std::initializer_list& v, std::vector AsNodeOutList(const Scope& scope, const InputList& inp); +/// }@ + } // namespace ops } // namespace tensorflow diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h index 43e44e7a96e7a9..c2a1c3d172ba28 100644 --- a/tensorflow/core/framework/tensor.h +++ b/tensorflow/core/framework/tensor.h @@ -38,6 +38,7 @@ namespace tensorflow { class TensorBuffer; // Forward declaration. class TensorCApi; +/// @ingroup core /// Represents an n-dimensional array of values. class Tensor { public: diff --git a/tensorflow/core/lib/core/status.h b/tensorflow/core/lib/core/status.h index 734ea91c80f565..0dae2abfe9900a 100644 --- a/tensorflow/core/lib/core/status.h +++ b/tensorflow/core/lib/core/status.h @@ -25,6 +25,7 @@ limitations under the License. namespace tensorflow { +/// @ingroup core /// Denotes success or failure of a call in Tensorflow. class Status { public: @@ -101,6 +102,7 @@ inline bool Status::operator==(const Status& x) const { inline bool Status::operator!=(const Status& x) const { return !(*this == x); } +/// @ingroup core std::ostream& operator<<(std::ostream& os, const Status& x); typedef std::function StatusCallback; From 4fb692b85007bda084eafa8515b72e80a1f8dccc Mon Sep 17 00:00:00 2001 From: Andrew Selle Date: Mon, 13 Feb 2017 13:05:58 -0800 Subject: [PATCH 02/50] Documentation changes to adhere to new doc generator Change: 147382677 --- tensorflow/contrib/linalg/__init__.py | 21 +--- tensorflow/contrib/losses/__init__.py | 2 +- tensorflow/contrib/metrics/__init__.py | 92 +-------------- tensorflow/contrib/opt/__init__.py | 2 +- .../training/moving_average_optimizer.py | 69 +++++------ tensorflow/contrib/rnn/__init__.py | 31 +---- tensorflow/python/ops/image_ops.py | 109 +----------------- 7 files changed, 46 insertions(+), 280 deletions(-) diff --git a/tensorflow/contrib/linalg/__init__.py b/tensorflow/contrib/linalg/__init__.py index 3fe0c5f761bb08..6bea0d3d7e28f1 100644 --- a/tensorflow/contrib/linalg/__init__.py +++ b/tensorflow/contrib/linalg/__init__.py @@ -12,33 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Linear algebra libraries for TensorFlow. - -## `LinearOperator` - -Subclasses of `LinearOperator` provide a access to common methods on a -(batch) matrix, without the need to materialize the matrix. This allows: - -* Matrix free computations -* Different operators to take advantage of special strcture, while providing a - consistent API to users. - -### Base class +"""Linear algebra libraries. See the @{$python/contrib.linalg} guide. @@LinearOperator - -### Individual operators - @@LinearOperatorDiag @@LinearOperatorIdentity @@LinearOperatorScaledIdentity @@LinearOperatorMatrix @@LinearOperatorTriL - -### Transformations and Combinations of operators - +@@LinearOperatorUDVHUpdate @@LinearOperatorComposition - """ from __future__ import absolute_import from __future__ import division diff --git a/tensorflow/contrib/losses/__init__.py b/tensorflow/contrib/losses/__init__.py index a405e11c22bab1..75aa3f99ac44df 100644 --- a/tensorflow/contrib/losses/__init__.py +++ b/tensorflow/contrib/losses/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. # ============================================================================== -"""Ops for building neural network losses.""" +"""Ops for building neural network losses. See @{$python/contrib.losses}.""" from __future__ import absolute_import from __future__ import division diff --git a/tensorflow/contrib/metrics/__init__.py b/tensorflow/contrib/metrics/__init__.py index b5ad8fb8b5d99f..56edcc2f146e83 100644 --- a/tensorflow/contrib/metrics/__init__.py +++ b/tensorflow/contrib/metrics/__init__.py @@ -12,90 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""##Ops for evaluation metrics and summary statistics. +"""Ops for evaluation metrics and summary statistics. -### API - -This module provides functions for computing streaming metrics: metrics computed -on dynamically valued `Tensors`. Each metric declaration returns a -"value_tensor", an idempotent operation that returns the current value of the -metric, and an "update_op", an operation that accumulates the information -from the current value of the `Tensors` being measured as well as returns the -value of the "value_tensor". - -To use any of these metrics, one need only declare the metric, call `update_op` -repeatedly to accumulate data over the desired number of `Tensor` values (often -each one is a single batch) and finally evaluate the value_tensor. For example, -to use the `streaming_mean`: - -```python -value = ... -mean_value, update_op = tf.contrib.metrics.streaming_mean(values) -sess.run(tf.local_variables_initializer()) - -for i in range(number_of_batches): - print('Mean after batch %d: %f' % (i, update_op.eval()) -print('Final Mean: %f' % mean_value.eval()) -``` - -Each metric function adds nodes to the graph that hold the state necessary to -compute the value of the metric as well as a set of operations that actually -perform the computation. Every metric evaluation is composed of three steps - -* Initialization: initializing the metric state. -* Aggregation: updating the values of the metric state. -* Finalization: computing the final metric value. - -In the above example, calling streaming_mean creates a pair of state variables -that will contain (1) the running sum and (2) the count of the number of samples -in the sum. Because the streaming metrics use local variables, -the Initialization stage is performed by running the op returned -by `tf.local_variables_initializer()`. It sets the sum and count variables to -zero. - -Next, Aggregation is performed by examining the current state of `values` -and incrementing the state variables appropriately. This step is executed by -running the `update_op` returned by the metric. - -Finally, finalization is performed by evaluating the "value_tensor" - -In practice, we commonly want to evaluate across many batches and multiple -metrics. To do so, we need only run the metric computation operations multiple -times: - -```python -labels = ... -predictions = ... -accuracy, update_op_acc = tf.contrib.metrics.streaming_accuracy( - labels, predictions) -error, update_op_error = tf.contrib.metrics.streaming_mean_absolute_error( - labels, predictions) - -sess.run(tf.local_variables_initializer()) -for batch in range(num_batches): - sess.run([update_op_acc, update_op_error]) - -accuracy, mean_absolute_error = sess.run([accuracy, mean_absolute_error]) -``` - -Note that when evaluating the same metric multiple times on different inputs, -one must specify the scope of each metric to avoid accumulating the results -together: - -```python -labels = ... -predictions0 = ... -predictions1 = ... - -accuracy0 = tf.contrib.metrics.accuracy(labels, predictions0, name='preds0') -accuracy1 = tf.contrib.metrics.accuracy(labels, predictions1, name='preds1') -``` - -Certain metrics, such as streaming_mean or streaming_accuracy, can be weighted -via a `weights` argument. The `weights` tensor must be the same size as the -labels and predictions tensors and results in a weighted average of the metric. - -## Metric `Ops` +See the @{$python/contrib.metrics} guide. @@streaming_accuracy @@streaming_mean @@ -130,18 +49,11 @@ @@streaming_true_negatives_at_thresholds @@streaming_true_positives @@streaming_true_positives_at_thresholds - @@auc_using_histogram - @@accuracy - @@aggregate_metrics @@aggregate_metric_map - @@confusion_matrix - -## Set `Ops` - @@set_difference @@set_intersection @@set_size diff --git a/tensorflow/contrib/opt/__init__.py b/tensorflow/contrib/opt/__init__.py index 8ef90095965f5e..4883e166bbeb16 100644 --- a/tensorflow/contrib/opt/__init__.py +++ b/tensorflow/contrib/opt/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""opt: A module containing optimization routines.""" +"""A module containing optimization routines.""" from __future__ import absolute_import from __future__ import division diff --git a/tensorflow/contrib/opt/python/training/moving_average_optimizer.py b/tensorflow/contrib/opt/python/training/moving_average_optimizer.py index bc1b24be807713..00cb6bfba92a8f 100644 --- a/tensorflow/contrib/opt/python/training/moving_average_optimizer.py +++ b/tensorflow/contrib/opt/python/training/moving_average_optimizer.py @@ -12,39 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Optimizer that computes a moving average of the variables. - -Empirically it has been found that using the moving average of the trained -parameters of a deep network is better than using its trained parameters -directly. This optimizer allows you to compute this moving average and swap the -variables at save time so that any code outside of the training loop will use by -default the averaged values instead of the original ones. - -Example of usage: - -```python - -// Encapsulate your favorite optimizer (here the momentum one) -// inside the MovingAverageOptimizer. -opt = tf.train.MomentumOptimizer(learning_rate, FLAGS.momentum) -opt = tf.contrib.opt.MovingAverageOptimizer(opt) -// Then create your model and all its variables. -model = build_model() -// Add the training op that optimizes using opt. -// This needs to be called before swapping_saver(). -opt.minimize(cost, var_list) -// Then create your saver like this: -saver = opt.swapping_saver() -// Pass it to your training loop. - slim.learning.train( - model, - ... - saver=saver) -``` - -Note that for evaluation, the normal saver should be used instead of -swapping_saver(). -""" +"""Moving average optimizer.""" + from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -60,7 +29,39 @@ class MovingAverageOptimizer(optimizer.Optimizer): - """Optimizer wrapper that maintains a moving average of parameters.""" + """Optimizer that computes a moving average of the variables. + + Empirically it has been found that using the moving average of the trained + parameters of a deep network is better than using its trained parameters + directly. This optimizer allows you to compute this moving average and swap + the variables at save time so that any code outside of the training loop will + use by default the averaged values instead of the original ones. + + Example of usage: + + ```python + + // Encapsulate your favorite optimizer (here the momentum one) + // inside the MovingAverageOptimizer. + opt = tf.train.MomentumOptimizer(learning_rate, FLAGS.momentum) + opt = tf.contrib.opt.MovingAverageOptimizer(opt) + // Then create your model and all its variables. + model = build_model() + // Add the training op that optimizes using opt. + // This needs to be called before swapping_saver(). + opt.minimize(cost, var_list) + // Then create your saver like this: + saver = opt.swapping_saver() + // Pass it to your training loop. + slim.learning.train( + model, + ... + saver=saver) + ``` + + Note that for evaluation, the normal saver should be used instead of + swapping_saver(). + """ def __init__(self, opt, average_decay=0.9999, num_updates=None, sequential_update=True): diff --git a/tensorflow/contrib/rnn/__init__.py b/tensorflow/contrib/rnn/__init__.py index 56c394c028da59..fee5fc88fbee47 100644 --- a/tensorflow/contrib/rnn/__init__.py +++ b/tensorflow/contrib/rnn/__init__.py @@ -12,57 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Module for constructing RNN Cells and additional RNN operations. - -## Base interface for all RNN Cells +"""RNN Cells and additional RNN operations. See @{$python/contrib.rnn} guide. @@RNNCell - -## RNN Cells for use with TensorFlow's core RNN methods - @@BasicRNNCell @@BasicLSTMCell @@GRUCell @@LSTMCell @@LayerNormBasicLSTMCell - -## Classes storing split `RNNCell` state - @@LSTMStateTuple - -## RNN Cell wrappers (RNNCells that wrap other RNNCells) - @@MultiRNNCell @@LSTMBlockWrapper @@DropoutWrapper @@EmbeddingWrapper @@InputProjectionWrapper @@OutputProjectionWrapper - -### Block RNNCells +@@DeviceWrapper +@@ResidualWrapper @@LSTMBlockCell @@GRUBlockCell - -### Fused RNNCells @@FusedRNNCell @@FusedRNNCellAdaptor @@TimeReversedFusedRNN @@LSTMBlockFusedCell - -### LSTM-like cells @@CoupledInputForgetGateLSTMCell @@TimeFreqLSTMCell @@GridLSTMCell - -### RNNCell wrappers @@AttentionCellWrapper - - -## Recurrent Neural Networks - -TensorFlow provides a number of methods for constructing Recurrent Neural -Networks. - +@@CompiledWrapper @@static_rnn @@static_state_saving_rnn @@static_bidirectional_rnn diff --git a/tensorflow/python/ops/image_ops.py b/tensorflow/python/ops/image_ops.py index f97a240acb7d37..7fe01dfdcefa8a 100644 --- a/tensorflow/python/ops/image_ops.py +++ b/tensorflow/python/ops/image_ops.py @@ -14,157 +14,50 @@ # ============================================================================== # pylint: disable=g-short-docstring-punctuation -"""## Encoding and Decoding - -TensorFlow provides Ops to decode and encode JPEG and PNG formats. Encoded -images are represented by scalar string Tensors, decoded images by 3-D uint8 -tensors of shape `[height, width, channels]`. (PNG also supports uint16.) - -The encode and decode Ops apply to one image at a time. Their input and output -are all of variable size. If you need fixed size images, pass the output of -the decode Ops to one of the cropping and resizing Ops. - -Note: The PNG encode and decode Ops support RGBA, but the conversions Ops -presently only support RGB, HSV, and GrayScale. Presently, the alpha channel has -to be stripped from the image and re-attached using slicing ops. +"""Image processing and decoding ops. See the @{$python/image} guide. @@decode_gif - @@decode_jpeg @@encode_jpeg - @@decode_png @@encode_png - @@decode_image - -## Resizing - -The resizing Ops accept input images as tensors of several types. They always -output resized images as float32 tensors. - -The convenience function [`resize_images()`](#resize_images) supports both 4-D -and 3-D tensors as input and output. 4-D tensors are for batches of images, -3-D tensors for individual images. - -Other resizing Ops only support 4-D batches of images as input: -[`resize_area`](#resize_area), [`resize_bicubic`](#resize_bicubic), -[`resize_bilinear`](#resize_bilinear), -[`resize_nearest_neighbor`](#resize_nearest_neighbor). - -Example: - -```python -# Decode a JPG image and resize it to 299 by 299 using default method. -image = tf.image.decode_jpeg(...) -resized_image = tf.image.resize_images(image, [299, 299]) -``` - @@resize_images - @@resize_area @@resize_bicubic @@resize_bilinear @@resize_nearest_neighbor - -## Cropping - @@resize_image_with_crop_or_pad - @@central_crop @@pad_to_bounding_box @@crop_to_bounding_box @@extract_glimpse - @@crop_and_resize - -## Flipping, Rotating and Transposing - @@flip_up_down @@random_flip_up_down - @@flip_left_right @@random_flip_left_right - @@transpose_image - @@rot90 -## Converting Between Colorspaces. - -Image ops work either on individual images or on batches of images, depending on -the shape of their input Tensor. - -If 3-D, the shape is `[height, width, channels]`, and the Tensor represents one -image. If 4-D, the shape is `[batch_size, height, width, channels]`, and the -Tensor represents `batch_size` images. - -Currently, `channels` can usefully be 1, 2, 3, or 4. Single-channel images are -grayscale, images with 3 channels are encoded as either RGB or HSV. Images -with 2 or 4 channels include an alpha channel, which has to be stripped from the -image before passing the image to most image processing functions (and can be -re-attached later). - -Internally, images are either stored in as one `float32` per channel per pixel -(implicitly, values are assumed to lie in `[0,1)`) or one `uint8` per channel -per pixel (values are assumed to lie in `[0,255]`). - -TensorFlow can convert between images in RGB or HSV. The conversion functions -work only on float images, so you need to convert images in other formats using -[`convert_image_dtype`](#convert-image-dtype). - -Example: - -```python -# Decode an image and convert it to HSV. -rgb_image = tf.image.decode_png(..., channels=3) -rgb_image_float = tf.image.convert_image_dtype(rgb_image, tf.float32) -hsv_image = tf.image.rgb_to_hsv(rgb_image) -``` - @@rgb_to_grayscale @@grayscale_to_rgb - @@hsv_to_rgb @@rgb_to_hsv - @@convert_image_dtype - -## Image Adjustments - -TensorFlow provides functions to adjust images in various ways: brightness, -contrast, hue, and saturation. Each adjustment can be done with predefined -parameters or with random parameters picked from predefined intervals. Random -adjustments are often useful to expand a training set and reduce overfitting. - -If several adjustments are chained it is advisable to minimize the number of -redundant conversions by first converting the images to the most natural data -type and representation (RGB or HSV). - @@adjust_brightness @@random_brightness - @@adjust_contrast @@random_contrast - @@adjust_hue @@random_hue - @@adjust_gamma - @@adjust_saturation @@random_saturation - @@per_image_standardization - -## Working with Bounding Boxes - @@draw_bounding_boxes @@non_max_suppression @@sample_distorted_bounding_box - -## Denoising - @@total_variation """ from __future__ import absolute_import From b6d328e124437b6992cd206bee6cb736ea8594f9 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 13 Feb 2017 13:16:07 -0800 Subject: [PATCH 03/50] Update docs. Change: 147383879 --- tensorflow/python/ops/functional_ops.py | 7 +- tensorflow/python/ops/histogram_ops.py | 2 +- tensorflow/python/ops/io_ops.py | 83 +--------- tensorflow/python/ops/math_ops.py | 94 +---------- tensorflow/python/ops/nn.py | 208 +----------------------- 5 files changed, 6 insertions(+), 388 deletions(-) diff --git a/tensorflow/python/ops/functional_ops.py b/tensorflow/python/ops/functional_ops.py index e8ca9e99e6e3a4..230074b6c93b52 100644 --- a/tensorflow/python/ops/functional_ops.py +++ b/tensorflow/python/ops/functional_ops.py @@ -13,12 +13,7 @@ # limitations under the License. # ============================================================================= -"""Functional operations. - -## Higher Order Operators - -TensorFlow provides several higher order operators to simplify the common -map-reduce programming patterns. +"""Functional operations. See the @{$python/functional_ops} guide. @@map_fn @@foldl diff --git a/tensorflow/python/ops/histogram_ops.py b/tensorflow/python/ops/histogram_ops.py index 5e1f322b2ccc3e..8e84479a0a9389 100644 --- a/tensorflow/python/ops/histogram_ops.py +++ b/tensorflow/python/ops/histogram_ops.py @@ -13,7 +13,7 @@ # limitations under the License. # ============================================================================== # pylint: disable=g-short-docstring-punctuation -"""## Histograms +"""Histograms. Please see @{$python/histogram_ops} guide. @@histogram_fixed_width """ diff --git a/tensorflow/python/ops/io_ops.py b/tensorflow/python/ops/io_ops.py index 0a099ae28c3ef3..157304444b8a4a 100644 --- a/tensorflow/python/ops/io_ops.py +++ b/tensorflow/python/ops/io_ops.py @@ -14,52 +14,19 @@ # ============================================================================== # pylint: disable=line-too-long -"""## Placeholders - -TensorFlow provides a placeholder operation that must be fed with data -on execution. For more info, see the section on [Feeding -data](../../how_tos/reading_data/index.md#feeding). +"""Inputs and Readers. See the @{$python/io_ops} guide. @@placeholder @@placeholder_with_default - -For feeding `SparseTensor`s which are composite type, -there is a convenience function: - @@sparse_placeholder - -## Readers - -TensorFlow provides a set of Reader classes for reading data formats. -For more information on inputs and readers, see [Reading -data](../../how_tos/reading_data/index.md). - @@ReaderBase @@TextLineReader @@WholeFileReader @@IdentityReader @@TFRecordReader @@FixedLengthRecordReader - -## Converting - -TensorFlow provides several operations that you can use to convert various data -formats into tensors. - @@decode_csv @@decode_raw - -- - - - -### Example protocol buffer - -TensorFlow's [recommended format for training -examples](../../how_tos/reading_data/index.md#standard-tensorflow-format) -is serialized `Example` protocol buffers, [described -here](https://www.tensorflow.org/code/tensorflow/core/example/example.proto). -They contain `Features`, [described -here](https://www.tensorflow.org/code/tensorflow/core/example/feature.proto). - @@VarLenFeature @@FixedLenFeature @@FixedLenSequenceFeature @@ -68,71 +35,23 @@ @@parse_single_example @@parse_tensor @@decode_json_example - -## Queues - -TensorFlow provides several implementations of 'Queues', which are -structures within the TensorFlow computation graph to stage pipelines -of tensors together. The following describe the basic Queue interface -and some implementations. To see an example use, see [Threading and -Queues](../../how_tos/threading_and_queues/index.md). - @@QueueBase @@FIFOQueue @@PaddingFIFOQueue @@RandomShuffleQueue @@PriorityQueue - -## Conditional Accumulators - @@ConditionalAccumulatorBase @@ConditionalAccumulator @@SparseConditionalAccumulator - -## Dealing with the filesystem - @@matching_files @@read_file @@write_file - -## Input pipeline - -TensorFlow functions for setting up an input-prefetching pipeline. -Please see the [reading data how-to](../../how_tos/reading_data/index.md) -for context. - -### Beginning of an input pipeline - -The "producer" functions add a queue to the graph and a corresponding -`QueueRunner` for running the subgraph that fills that queue. - @@match_filenames_once @@limit_epochs @@input_producer @@range_input_producer @@slice_input_producer @@string_input_producer - -### Batching at the end of an input pipeline - -These functions add a queue to the graph to assemble a batch of -examples, with possible shuffling. They also add a `QueueRunner` for -running the subgraph that fills that queue. - -Use [`batch`](#batch) or [`batch_join`](#batch_join) for batching -examples that have already been well shuffled. Use -[`shuffle_batch`](#shuffle_batch) or -[`shuffle_batch_join`](#shuffle_batch_join) for examples that would -benefit from additional shuffling. - -Use [`batch`](#batch) or [`shuffle_batch`](#shuffle_batch) if you want a -single thread producing examples to batch, or if you have a -single subgraph producing examples but you want to run it in *N* threads -(where you increase *N* until it can keep the queue full). Use -[`batch_join`](#batch_join) or [`shuffle_batch_join`](#shuffle_batch_join) -if you have *N* different subgraphs producing examples to batch and you -want them run by *N* threads. Use `maybe_*` to enqueue conditionally. - @@batch @@maybe_batch @@batch_join diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py index ed07858c865700..e5c5265a8c58b6 100644 --- a/tensorflow/python/ops/math_ops.py +++ b/tensorflow/python/ops/math_ops.py @@ -12,13 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Note: Elementwise binary operations in TensorFlow follow [numpy-style -broadcasting](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html). - -## Arithmetic Operators - -TensorFlow provides several operations that you can use to add basic arithmetic -operators to your graph. +"""Basic arithmetic operators. See the @{$python/math_ops} guide. @@add @@subtract @@ -35,12 +29,6 @@ @@floormod @@mod @@cross - -## Basic Math Functions - -TensorFlow provides several operations that you can use to add basic -mathematical functions to your graph. - @@add_n @@abs @@negative @@ -77,26 +65,17 @@ @@polygamma @@betainc @@rint - -## Matrix Math Functions - -TensorFlow provides several operations that you can use to add linear algebra -functions on matrices to your graph. - @@diag @@diag_part @@trace @@transpose - @@eye @@matrix_diag @@matrix_diag_part @@matrix_band_part @@matrix_set_diag @@matrix_transpose - @@matmul - @@norm @@matrix_determinant @@matrix_inverse @@ -109,43 +88,17 @@ @@self_adjoint_eig @@self_adjoint_eigvals @@svd - - -## Tensor Math Function - -TensorFlow provides operations that you can use to add tensor functions to your -graph. - @@tensordot - - -## Complex Number Functions - -TensorFlow provides several operations that you can use to add complex number -functions to your graph. - @@complex @@conj @@imag @@real - -## Fourier Transform Functions - -TensorFlow provides several operations that you can use to add discrete -Fourier transform functions to your graph. - @@fft @@ifft @@fft2d @@ifft2d @@fft3d @@ifft3d - -## Reduction - -TensorFlow provides several operations that you can use to perform -common math computations that reduce various dimensions of a tensor. - @@reduce_sum @@reduce_prod @@reduce_min @@ -155,69 +108,26 @@ @@reduce_any @@reduce_logsumexp @@count_nonzero - @@accumulate_n - @@einsum - -## Scan - -TensorFlow provides several operations that you can use to perform scans -(running totals) across one axis of a tensor. - @@cumsum @@cumprod - -## Segmentation - -TensorFlow provides several operations that you can use to perform common -math computations on tensor segments. -Here a segmentation is a partitioning of a tensor along -the first dimension, i.e. it defines a mapping from the first dimension onto -`segment_ids`. The `segment_ids` tensor should be the size of -the first dimension, `d0`, with consecutive IDs in the range `0` to `k`, -where `k [[0 0 0 0] - [5 6 7 8]] -``` - @@segment_sum @@segment_prod @@segment_min @@segment_max @@segment_mean - @@unsorted_segment_sum - +@@unsorted_segment_max @@sparse_segment_sum @@sparse_segment_mean @@sparse_segment_sqrt_n - - -## Sequence Comparison and Indexing - -TensorFlow provides several operations that you can use to add sequence -comparison and index extraction to your graph. You can use these operations to -determine sequence differences and determine the indexes of specific values in -a tensor. - @@argmin @@argmax - @@setdiff1d @@where @@unique - @@edit_distance - @@invert_permutation """ from __future__ import absolute_import diff --git a/tensorflow/python/ops/nn.py b/tensorflow/python/ops/nn.py index 00626eba037fc0..5daac8f63ace38 100644 --- a/tensorflow/python/ops/nn.py +++ b/tensorflow/python/ops/nn.py @@ -14,16 +14,7 @@ # ============================================================================= # pylint: disable=unused-import,g-bad-import-order -"""## Activation Functions. - -The activation ops provide different types of nonlinearities for use in neural -networks. These include smooth nonlinearities (`sigmoid`, `tanh`, `elu`, -`softplus`, and `softsign`), continuous but not everywhere differentiable -functions (`relu`, `relu6`, `crelu` and `relu_x`), and random regularization -(`dropout`). - -All activation ops apply componentwise, and produce a tensor of the same -shape as the input tensor. +"""## Neural network support. See the @{$python/nn} guide. @@relu @@relu6 @@ -35,78 +26,6 @@ @@bias_add @@sigmoid @@tanh - -## Convolution - -The convolution ops sweep a 2-D filter over a batch of images, applying the -filter to each window of each image of the appropriate size. The different -ops trade off between generic vs. specific filters: - -* `conv2d`: Arbitrary filters that can mix channels together. -* `depthwise_conv2d`: Filters that operate on each channel independently. -* `separable_conv2d`: A depthwise spatial filter followed by a pointwise filter. - -Note that although these ops are called "convolution", they are strictly -speaking "cross-correlation" since the filter is combined with an input window -without reversing the filter. For details, see [the properties of -cross-correlation](https://en.wikipedia.org/wiki/Cross-correlation#Properties). - -The filter is applied to image patches of the same size as the filter and -strided according to the `strides` argument. `strides = [1, 1, 1, 1]` applies -the filter to a patch at every offset, `strides = [1, 2, 2, 1]` applies the -filter to every other image patch in each dimension, etc. - -Ignoring channels for the moment, and assume that the 4-D `input` has shape -`[batch, in_height, in_width, ...]` and the 4-D `filter` has shape -`[filter_height, filter_width, ...]`, then the spatial semantics of the -convolution ops are as follows: first, according to the padding scheme chosen -as `'SAME'` or `'VALID'`, the output size and the padding pixels are computed. -For the `'SAME'` padding, the output height and width are computed as: - - out_height = ceil(float(in_height) / float(strides[1])) - out_width = ceil(float(in_width) / float(strides[2])) - -and the padding on the top and left are computed as: - - pad_along_height = max((out_height - 1) * strides[1] + - filter_height - in_height, 0) - pad_along_width = max((out_width - 1) * strides[2] + - filter_width - in_width, 0) - pad_top = pad_along_height // 2 - pad_bottom = pad_along_height - pad_top - pad_left = pad_along_width // 2 - pad_right = pad_along_width - pad_left - - -Note that the division by 2 means that there might be cases when the padding on -both sides (top vs bottom, right vs left) are off by one. In this case, the -bottom and right sides always get the one additional padded pixel. For example, -when `pad_along_height` is 5, we pad 2 pixels at the top and 3 pixels at the -bottom. Note that this is different from existing libraries such as cuDNN and -Caffe, which explicitly specify the number of padded pixels and always pad the -same number of pixels on both sides. - -For the `'VALID`' padding, the output height and width are computed as: - - out_height = ceil(float(in_height - filter_height + 1) / float(strides[1])) - out_width = ceil(float(in_width - filter_width + 1) / float(strides[2])) - -and the padding values are always zero. The output is then computed as - - output[b, i, j, :] = - sum_{di, dj} input[b, strides[1] * i + di - pad_top, - strides[2] * j + dj - pad_left, ...] * - filter[di, dj, ...] - -where any value outside the original input image region are considered zero ( -i.e. we pad zero values around the border of the image). - -Since `input` is 4-D, each `input[b, i, j, :]` is a vector. For `conv2d`, these -vectors are multiplied by the `filter[di, dj, :, :]` matrices to produce new -vectors. For `depthwise_conv_2d`, each scalar component `input[b, i, j, k]` -is multiplied by a vector `filter[di, dj, k]`, and all the vectors are -concatenated. - @@convolution @@conv2d @@depthwise_conv2d @@ -123,22 +42,6 @@ @@conv3d_backprop_filter_v2 @@depthwise_conv2d_native_backprop_filter @@depthwise_conv2d_native_backprop_input - -## Pooling - -The pooling ops sweep a rectangular window over the input tensor, computing a -reduction operation for each window (average, max, or max with argmax). Each -pooling op uses rectangular windows of size `ksize` separated by offset -`strides`. For example, if `strides` is all ones every window is used, if -`strides` is all twos every other window is used in each dimension, etc. - -In detail, the output is - - output[i] = reduce(value[strides * i:strides * i + ksize]) - -where the indices also take into consideration the padding values. Please refer -to the `Convolution` section for details about the padding calculation. - @@avg_pool @@max_pool @@max_pool_with_argmax @@ -147,53 +50,9 @@ @@fractional_avg_pool @@fractional_max_pool @@pool - -## Morphological filtering - -Morphological operators are non-linear filters used in image processing. - -[Greyscale morphological dilation -](https://en.wikipedia.org/wiki/Dilation_(morphology)) -is the max-sum counterpart of standard sum-product convolution: - - output[b, y, x, c] = - max_{dy, dx} input[b, - strides[1] * y + rates[1] * dy, - strides[2] * x + rates[2] * dx, - c] + - filter[dy, dx, c] - -The `filter` is usually called structuring function. Max-pooling is a special -case of greyscale morphological dilation when the filter assumes all-zero -values (a.k.a. flat structuring function). - -[Greyscale morphological erosion -](https://en.wikipedia.org/wiki/Erosion_(morphology)) -is the min-sum counterpart of standard sum-product convolution: - - output[b, y, x, c] = - min_{dy, dx} input[b, - strides[1] * y - rates[1] * dy, - strides[2] * x - rates[2] * dx, - c] - - filter[dy, dx, c] - -Dilation and erosion are dual to each other. The dilation of the input signal -`f` by the structuring signal `g` is equal to the negation of the erosion of -`-f` by the reflected `g`, and vice versa. - -Striding and padding is carried out in exactly the same way as in standard -convolution. Please refer to the `Convolution` section for details. - @@dilation2d @@erosion2d @@with_space_to_batch - -## Normalization - -Normalization is useful to prevent neurons from saturating when inputs may -have varying scale, and to aid generalization. - @@l2_normalize @@local_response_normalization @@sufficient_statistics @@ -203,100 +62,35 @@ @@fused_batch_norm @@batch_normalization @@batch_norm_with_global_normalization - -## Losses - -The loss ops measure error between two tensors, or between a tensor and zero. -These can be used for measuring accuracy of a network in a regression task -or for regularization purposes (weight decay). - @@l2_loss @@log_poisson_loss - -## Classification - -TensorFlow provides several operations that help you perform classification. - @@sigmoid_cross_entropy_with_logits @@softmax @@log_softmax @@softmax_cross_entropy_with_logits @@sparse_softmax_cross_entropy_with_logits @@weighted_cross_entropy_with_logits - -## Embeddings - -TensorFlow provides library support for looking up values in embedding -tensors. - @@embedding_lookup @@embedding_lookup_sparse - -## Recurrent Neural Networks - -TensorFlow provides a number of methods for constructing Recurrent -Neural Networks. Most accept an `RNNCell`-subclassed object -(see the documentation for `tf.contrib.rnn`). - @@dynamic_rnn @@bidirectional_dynamic_rnn @@raw_rnn - -## Connectionist Temporal Classification (CTC) - @@ctc_loss @@ctc_greedy_decoder @@ctc_beam_search_decoder - -## Evaluation - -The evaluation ops are useful for measuring the performance of a network. -They are typically used at evaluation time. - @@top_k @@in_top_k - -## Candidate Sampling - -Do you want to train a multiclass or multilabel model with thousands -or millions of output classes (for example, a language model with a -large vocabulary)? Training with a full Softmax is slow in this case, -since all of the classes are evaluated for every training example. -Candidate Sampling training algorithms can speed up your step times by -only considering a small randomly-chosen subset of contrastive classes -(called candidates) for each batch of training examples. - -See our -[Candidate Sampling Algorithms Reference](../../extras/candidate_sampling.pdf) - -### Sampled Loss Functions - -TensorFlow provides the following sampled loss functions for faster training. - @@nce_loss @@sampled_softmax_loss - -### Candidate Samplers - -TensorFlow provides the following samplers for randomly sampling candidate -classes when using one of the sampled loss functions above. - @@uniform_candidate_sampler @@log_uniform_candidate_sampler @@learned_unigram_candidate_sampler @@fixed_unigram_candidate_sampler - -### Miscellaneous candidate sampling utilities - @@compute_accidental_hits - -### Quantization ops - @@quantized_conv2d @@quantized_relu_x @@quantized_max_pool @@quantized_avg_pool - """ from __future__ import absolute_import from __future__ import division From 2323ada83578da8ac6c374cad4fd5e49dcb1cb90 Mon Sep 17 00:00:00 2001 From: Andrew Selle Date: Mon, 13 Feb 2017 14:30:44 -0800 Subject: [PATCH 04/50] Remove @@__init__ from docstrings. Change: 147393760 --- tensorflow/python/training/adadelta.py | 2 -- tensorflow/python/training/adagrad.py | 6 +++--- tensorflow/python/training/adagrad_da.py | 2 -- tensorflow/python/training/adam.py | 2 -- tensorflow/python/training/ftrl.py | 2 -- 5 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tensorflow/python/training/adadelta.py b/tensorflow/python/training/adadelta.py index 50bb28ebcdf6a9..5edc0cf022f176 100644 --- a/tensorflow/python/training/adadelta.py +++ b/tensorflow/python/training/adadelta.py @@ -29,8 +29,6 @@ class AdadeltaOptimizer(optimizer.Optimizer): See [M. D. Zeiler](http://arxiv.org/abs/1212.5701) ([pdf](http://arxiv.org/pdf/1212.5701v1.pdf)) - - @@__init__ """ def __init__(self, learning_rate=0.001, rho=0.95, epsilon=1e-8, diff --git a/tensorflow/python/training/adagrad.py b/tensorflow/python/training/adagrad.py index 9646c5c228f73a..7ac643975b1a33 100644 --- a/tensorflow/python/training/adagrad.py +++ b/tensorflow/python/training/adagrad.py @@ -28,9 +28,9 @@ class AdagradOptimizer(optimizer.Optimizer): """Optimizer that implements the Adagrad algorithm. - See this [paper](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf). - - @@__init__ + See this [paper](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) + or this + [intro](http://cs.stanford.edu/~ppasupat/a9online/uploads/proximal_notes.pdf). """ def __init__(self, learning_rate, initial_accumulator_value=0.1, diff --git a/tensorflow/python/training/adagrad_da.py b/tensorflow/python/training/adagrad_da.py index 4f1b4689863aa7..90d929370bce4f 100644 --- a/tensorflow/python/training/adagrad_da.py +++ b/tensorflow/python/training/adagrad_da.py @@ -38,8 +38,6 @@ class AdagradDAOptimizer(optimizer.Optimizer): trained model. This optimizer only guarantees sparsity for linear models. Be careful when using AdagradDA for deep networks as it will require careful initialization of the gradient accumulators for it to train. - - @@__init__ """ def __init__(self, diff --git a/tensorflow/python/training/adam.py b/tensorflow/python/training/adam.py index b65ee818333619..e327fac4c7ef71 100644 --- a/tensorflow/python/training/adam.py +++ b/tensorflow/python/training/adam.py @@ -32,8 +32,6 @@ class AdamOptimizer(optimizer.Optimizer): See [Kingma et. al., 2014](http://arxiv.org/abs/1412.6980) ([pdf](http://arxiv.org/pdf/1412.6980.pdf)). - - @@__init__ """ def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8, diff --git a/tensorflow/python/training/ftrl.py b/tensorflow/python/training/ftrl.py index 2bb4864d4a74f8..6818cc97ce6152 100644 --- a/tensorflow/python/training/ftrl.py +++ b/tensorflow/python/training/ftrl.py @@ -30,8 +30,6 @@ class FtrlOptimizer(optimizer.Optimizer): See this [paper]( https://www.eecs.tufts.edu/~dsculley/papers/ad-click-prediction.pdf). - - @@__init__ """ def __init__(self, learning_rate, From 0c930f0619f8a6c9331b66ce9f02639cbe2164d5 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Mon, 13 Feb 2017 15:34:03 -0800 Subject: [PATCH 05/50] Documentation changes to adhere to new doc generator Change: 147402290 --- tensorflow/contrib/copy_graph/__init__.py | 5 +- tensorflow/contrib/crf/__init__.py | 4 +- tensorflow/contrib/distributions/__init__.py | 36 +------ .../distributions/python/ops/bijector.py | 19 +--- tensorflow/contrib/ffmpeg/__init__.py | 18 +--- tensorflow/contrib/framework/__init__.py | 7 +- tensorflow/contrib/graph_editor/__init__.py | 97 +------------------ tensorflow/contrib/integrate/__init__.py | 37 +------ tensorflow/contrib/layers/__init__.py | 40 +------- tensorflow/contrib/learn/__init__.py | 19 +--- .../contrib/learn/python/learn/monitors.py | 57 +---------- 11 files changed, 22 insertions(+), 317 deletions(-) diff --git a/tensorflow/contrib/copy_graph/__init__.py b/tensorflow/contrib/copy_graph/__init__.py index 96dc0d7df2d4b5..30a0aac140b576 100644 --- a/tensorflow/contrib/copy_graph/__init__.py +++ b/tensorflow/contrib/copy_graph/__init__.py @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Functions for copying elements from one graph to another. +"""Functions to copy elements between graphs. +See the @{$python/contrib.copy_graph} guide. """ from __future__ import absolute_import @@ -21,7 +22,9 @@ from __future__ import print_function from tensorflow.contrib.copy_graph.python.util import copy_elements +# pylint: disable=wildcard-import from tensorflow.contrib.copy_graph.python.util.copy_elements import * +# pylint: enable=wildcard-import from tensorflow.python.util.all_util import remove_undocumented diff --git a/tensorflow/contrib/crf/__init__.py b/tensorflow/contrib/crf/__init__.py index 7f7818c845d559..442e62d6eda37f 100644 --- a/tensorflow/contrib/crf/__init__.py +++ b/tensorflow/contrib/crf/__init__.py @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Linear-chain CRF layer. - -## This package provides functions for building a linear-chain CRF layer. +"""Linear-chain CRF layer. See the @{$python/contrib.crf} guide. @@crf_sequence_score @@crf_log_norm diff --git a/tensorflow/contrib/distributions/__init__.py b/tensorflow/contrib/distributions/__init__.py index f822f723eb0ce7..f82bd777129066 100644 --- a/tensorflow/contrib/distributions/__init__.py +++ b/tensorflow/contrib/distributions/__init__.py @@ -14,17 +14,10 @@ # ============================================================================== """Classes representing statistical distributions and ops for working with them. -## Classes for statistical distributions. - -Classes that represent batches of statistical distributions. Each class is -initialized with parameters that define the distributions. - -## Base classes +See the @{$python/contrib.distributions} guide. @@Distribution -## Univariate (scalar) distributions - @@Binomial @@Bernoulli @@BernoulliWithSigmoidP @@ -48,56 +41,37 @@ @@StudentTWithAbsDfSoftplusSigma @@Uniform -## Multivariate distributions - -### Multivariate normal - @@MultivariateNormalDiag @@MultivariateNormalFull @@MultivariateNormalCholesky @@MultivariateNormalDiagPlusVDVT @@MultivariateNormalDiagWithSoftplusStDev -### Other multivariate distributions - @@Dirichlet @@DirichletMultinomial @@Multinomial @@WishartCholesky @@WishartFull -### Multivariate Utilities - @@matrix_diag_transform -## Transformed distributions - @@TransformedDistribution @@QuantizedDistribution -## Mixture Models - @@Mixture -## Posterior inference with conjugate priors. - -Functions that transform conjugate prior/likelihood pairs to distributions -representing the posterior or posterior predictive. - -## Normal likelihood with conjugate prior. - @@normal_conjugates_known_sigma_posterior @@normal_conjugates_known_sigma_predictive -## Kullback-Leibler Divergence - @@kl @@RegisterKL -## Utilities - @@softplus_inverse +@@ExpRelaxedOneHotCategorical +@@OneHotCategorical +@@RelaxedBernoulli +@@RelaxedOneHotCategorical """ from __future__ import absolute_import from __future__ import division diff --git a/tensorflow/contrib/distributions/python/ops/bijector.py b/tensorflow/contrib/distributions/python/ops/bijector.py index 7e92f496773ac3..adafbac1428426 100644 --- a/tensorflow/contrib/distributions/python/ops/bijector.py +++ b/tensorflow/contrib/distributions/python/ops/bijector.py @@ -12,22 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -r"""Bijector Ops. - -An API for invertible, differentiable transformations of random variables. - -## Background - -Differentiable, bijective transformations of continuous random variables alter -the calculations made in the cumulative/probability distribution functions and -sample function. This module provides a standard interface for making these -manipulations. - -For more details and examples, see the `Bijector` docstring. - -To apply a `Bijector`, use `distributions.TransformedDistribution`. - -## Bijectors +r"""Bijector Ops. See the @{$python/contrib.distributions.bijector} guide. @@Affine @@AffineLinearOperator @@ -41,8 +26,8 @@ @@SigmoidCentered @@SoftmaxCentered @@Softplus - """ + from __future__ import absolute_import from __future__ import division from __future__ import print_function diff --git a/tensorflow/contrib/ffmpeg/__init__.py b/tensorflow/contrib/ffmpeg/__init__.py index 59d5aac60315b2..348464a879ed71 100644 --- a/tensorflow/contrib/ffmpeg/__init__.py +++ b/tensorflow/contrib/ffmpeg/__init__.py @@ -13,23 +13,7 @@ # limitations under the License. # ============================================================================== # pylint: disable=g-short-docstring-punctuation -"""## Encoding and decoding audio using FFmpeg - -TensorFlow provides Ops to decode and encode audio files using the -[FFmpeg](https://www.ffmpeg.org/) library. FFmpeg must be -locally [installed](https://ffmpeg.org/download.html) for these Ops to succeed. - -Example: - -```python -from tensorflow.contrib import ffmpeg - -audio_binary = tf.read_file('song.mp3') -waveform = ffmpeg.decode_audio( - audio_binary, file_format='mp3', samples_per_second=44100, channel_count=2) -uncompressed_binary = ffmpeg.encode_audio( - waveform, file_format='wav', samples_per_second=44100) -``` +"""Working with audio using FFmpeg. See the @{$python/contrib.ffmpeg} guide. @@decode_audio @@encode_audio diff --git a/tensorflow/contrib/framework/__init__.py b/tensorflow/contrib/framework/__init__.py index 0765ee33ea2a8f..47963c337057ba 100644 --- a/tensorflow/contrib/framework/__init__.py +++ b/tensorflow/contrib/framework/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. # ============================================================================== -"""Framework utilities. +"""Framework utilities. See the @{$python/contrib.framework} guide. @@assert_same_float_dtype @@assert_scalar @@ -29,18 +29,15 @@ @@with_shape @@with_same_shape -## Deprecation @@deprecated @@deprecated_args @@deprecated_arg_values -## Arg_Scope @@arg_scope @@add_arg_scope @@has_arg_scope @@arg_scoped_arguments -## Variables @@add_model_variable @@assert_global_step @@assert_or_get_global_step @@ -65,8 +62,6 @@ @@VariableDeviceChooser @@zero_initializer -## Checkpoint utilities - @@load_checkpoint @@list_variables @@load_variable diff --git a/tensorflow/contrib/graph_editor/__init__.py b/tensorflow/contrib/graph_editor/__init__.py index 47905cc99279c5..e6bf6dcce80f85 100644 --- a/tensorflow/contrib/graph_editor/__init__.py +++ b/tensorflow/contrib/graph_editor/__init__.py @@ -12,102 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""TensorFlow Graph Editor. - -The TensorFlow Graph Editor library allows for modification of an existing -`tf.Graph` instance in-place. - -The author's github username is [purpledog](https://github.com/purpledog). - -## Library overview - -Appending new nodes is the only graph editing operation allowed by the -TensorFlow core library. The Graph Editor library is an attempt to allow for -other kinds of editing operations, namely, *rerouting* and *transforming*. - -* *rerouting* is a local operation consisting in re-plugging existing tensors - (the edges of the graph). Operations (the nodes) are not modified by this - operation. For example, rerouting can be used to insert an operation adding - noise in place of an existing tensor. -* *transforming* is a global operation consisting in transforming a graph into - another. By default, a transformation is a simple copy but it can be - customized to achieved other goals. For instance, a graph can be transformed - into another one in which noise is added after all the operations of a - specific type. - -**Important: modifying a graph in-place with the Graph Editor must be done -`offline`, that is, without any active sessions.** - -Of course new operations can be appended online but Graph Editor specific -operations like rerouting and transforming can currently only be done offline. - -Here is an example of what you **cannot** do: - -* Build a graph. -* Create a session and run the graph. -* Modify the graph with the Graph Editor. -* Re-run the graph with the `same` previously created session. - -To edit an already running graph, follow these steps: - -* Build a graph. -* Create a session and run the graph. -* Save the graph state and terminate the session -* Modify the graph with the Graph Editor. -* create a new session and restore the graph state -* Re-run the graph with the newly created session. - -Note that this procedure is very costly because a new session must be created -after any modifications. Among other things, it takes time because the entire -graph state must be saved and restored again. - -## Sub-graph - -Most of the functions in the Graph Editor library operate on *sub-graph*. -More precisely, they take as input arguments instances of the SubGraphView class -(or anything which can be converted to it). Doing so allows the same function -to transparently operate on single operations as well as sub-graph of any size. - -A subgraph can be created in several ways: - -* using a list of ops: - -```python -my_sgv = ge.sgv(ops) -``` - -* from a name scope: - -```python -my_sgv = ge.sgv_scope("foo/bar", graph=tf.get_default_graph()) -``` - -* using regular expression: - -```python -my_sgv = ge.sgv("foo/.*/.*read$", graph=tf.get_default_graph()) -``` - -Note that the Graph Editor is meant to manipulate several graphs at the same -time, typically during transform or copy operation. For that reason, -to avoid any confusion, the default graph is never used and the graph on -which to operate must always be given explicitly. This is the reason why -*`graph=tf.get_default_graph()`* is used in the code snippets above. - -## Modules overview - -* util: utility functions. -* select: various selection methods of TensorFlow tensors and operations. -* match: TensorFlow graph matching. Think of this as regular expressions for - graphs (but not quite yet). -* reroute: various ways of rerouting tensors to different consuming ops like - *swap* or *reroute_a2b*. -* subgraph: the SubGraphView class, which enables subgraph manipulations in a - TensorFlow `tf.Graph`. -* edit: various editing functions operating on subgraphs like *detach*, - *connect* or *bypass*. -* transform: the Transformer class, which enables transforming - (or simply copying) a subgraph into another one. +"""TensorFlow Graph Editor. See the @{$python/contrib.graph_editor} guide. """ from __future__ import absolute_import diff --git a/tensorflow/contrib/integrate/__init__.py b/tensorflow/contrib/integrate/__init__.py index c951efd3d9f834..7bcebbaffe7d84 100644 --- a/tensorflow/contrib/integrate/__init__.py +++ b/tensorflow/contrib/integrate/__init__.py @@ -13,42 +13,7 @@ # limitations under the License. # ============================================================================== -"""Integration and ODE solvers for TensorFlow. - -## Example: Lorenz attractor - -We can use `odeint` to solve the -[Lorentz system](https://en.wikipedia.org/wiki/Lorenz_system) of ordinary -differential equations, a prototypical example of chaotic dynamics: - -```python -rho = 28.0 -sigma = 10.0 -beta = 8.0/3.0 - -def lorenz_equation(state, t): - x, y, z = tf.unstack(state) - dx = sigma * (y - x) - dy = x * (rho - z) - y - dz = x * y - beta * z - return tf.stack([dx, dy, dz]) - -init_state = tf.constant([0, 2, 20], dtype=tf.float64) -t = np.linspace(0, 50, num=5000) -tensor_state, tensor_info = tf.contrib.integrate.odeint( - lorenz_equation, init_state, t, full_output=True) - -sess = tf.Session() -state, info = sess.run([tensor_state, tensor_info]) -x, y, z = state.T -plt.plot(x, z) -``` - -
- -
- -## Ops +"""Integration and ODE solvers. See the @{$python/contrib.integrate} guide. @@odeint """ diff --git a/tensorflow/contrib/layers/__init__.py b/tensorflow/contrib/layers/__init__.py index c563b29de90366..5a7b6b7d277858 100644 --- a/tensorflow/contrib/layers/__init__.py +++ b/tensorflow/contrib/layers/__init__.py @@ -14,11 +14,7 @@ # ============================================================================== """Ops for building neural network layers, regularizers, summaries, etc. -## Higher level ops for building neural network layers. - -This package provides several ops that take care of creating variables that are -used internally in a consistent way and provide the building blocks for many -common machine learning algorithms. +See the @{$python/contrib.layers} guide. @@avg_pool2d @@batch_norm @@ -45,57 +41,24 @@ @@unit_norm @@embed_sequence -Aliases for fully_connected which set a default activation function are -available: `relu`, `relu6` and `linear`. - -`stack` operation is also available. It builds a stack of layers by applying -a layer repeatedly. - -## Regularizers - -Regularization can help prevent overfitting. These have the signature -`fn(weights)`. The loss is typically added to -`tf.GraphKeys.REGULARIZATION_LOSSES`. - @@apply_regularization @@l1_regularizer @@l2_regularizer @@sum_regularizer -## Initializers - -Initializers are used to initialize variables with sensible values given their -size, data type, and purpose. - @@xavier_initializer @@xavier_initializer_conv2d @@variance_scaling_initializer -## Optimization - -Optimize weights given a loss. - @@optimize_loss -## Summaries - -Helper functions to summarize specific variables or ops. - @@summarize_activation @@summarize_tensor @@summarize_tensors @@summarize_collection -The layers module defines convenience functions `summarize_variables`, -`summarize_weights` and `summarize_biases`, which set the `collection` argument -of `summarize_collection` to `VARIABLES`, `WEIGHTS` and `BIASES`, respectively. - @@summarize_activations -## Feature columns - -Feature columns provide a mechanism to map data to a model. - @@bucketized_column @@check_feature_columns @@create_feature_spec_for_parsing @@ -118,7 +81,6 @@ @@weighted_sum_from_feature_columns @@infer_real_valued_columns @@sequence_input_from_feature_columns - """ from __future__ import absolute_import diff --git a/tensorflow/contrib/learn/__init__.py b/tensorflow/contrib/learn/__init__.py index 2cc38fbbec7934..c49b7e53d1809a 100644 --- a/tensorflow/contrib/learn/__init__.py +++ b/tensorflow/contrib/learn/__init__.py @@ -14,11 +14,7 @@ # ============================================================================== # TODO(ptucker,ipolosukhin): Improve descriptions. -"""High level API for learning with TensorFlow. - -## Estimators - -Train and evaluate TensorFlow models. +"""High level API for learning. See the @{$python/contrib.learn} guide. @@BaseEstimator @@Estimator @@ -37,15 +33,10 @@ @@LinearRegressor @@LogisticRegressor -## Distributed training utilities @@Experiment @@ExportStrategy @@TaskType -## Graph actions - -Perform various training, evaluation, and inference actions on a graph. - @@NanLossDuringTrainingError @@RunConfig @@evaluate @@ -54,10 +45,6 @@ @@run_n @@train -## Input processing - -Queue and read batched input data. - @@extract_dask_data @@extract_dask_labels @@extract_pandas_data @@ -69,8 +56,8 @@ @@read_batch_features @@read_batch_record_features -Export utilities - +@@InputFnOps +@@ProblemType @@build_parsing_serving_input_fn @@ProblemType """ diff --git a/tensorflow/contrib/learn/python/learn/monitors.py b/tensorflow/contrib/learn/python/learn/monitors.py index d8fe2315da172c..9a57605f9e864f 100644 --- a/tensorflow/contrib/learn/python/learn/monitors.py +++ b/tensorflow/contrib/learn/python/learn/monitors.py @@ -13,62 +13,9 @@ # limitations under the License. # ============================================================================== -"""Monitors allow user instrumentation of the training process. +"""Monitors instrument the training process. -Monitors are useful to track training, report progress, request early -stopping and more. Monitors use the observer pattern and notify at the following -points: - -* when training begins -* before a training step -* after a training step -* when training ends - -Monitors are not intended to be reusable. - -There are a few pre-defined monitors: - -* `CaptureVariable`: saves a variable's values -* `GraphDump`: intended for debug only - saves all tensor values -* `PrintTensor`: outputs one or more tensor values to log -* `SummarySaver`: saves summaries to a summary writer -* `ValidationMonitor`: runs model validation, by periodically calculating eval - metrics on a separate data set; supports optional early stopping - -For more specific needs, you can create custom monitors by extending one of the -following classes: - -* `BaseMonitor`: the base class for all monitors -* `EveryN`: triggers a callback every N training steps - -Example: - -```python - class ExampleMonitor(monitors.BaseMonitor): - def __init__(self): - print 'Init' - - def begin(self, max_steps): - print 'Starting run. Will train until step %d.' % max_steps - - def end(self): - print 'Completed run.' - - def step_begin(self, step): - print 'About to run step %d...' % step - return ['loss_1:0'] - - def step_end(self, step, outputs): - print 'Done running step %d. The value of "loss" tensor: %s' % ( - step, outputs['loss_1:0']) - - linear_regressor = LinearRegressor() - example_monitor = ExampleMonitor() - linear_regressor.fit( - x, y, steps=2, batch_size=1, monitors=[example_monitor]) -``` - -## Ops +See the @{$python/contrib.learn.monitors} guide. @@get_default_monitors @@BaseMonitor From aa6b837779c073082d89f52cadb647225ad8aa42 Mon Sep 17 00:00:00 2001 From: Andrew Selle Date: Mon, 13 Feb 2017 15:34:18 -0800 Subject: [PATCH 06/50] Link docstrings to module guides and remove redundant text. Change: 147402322 --- tensorflow/contrib/training/__init__.py | 38 +------------------------ tensorflow/contrib/util/__init__.py | 4 +-- 2 files changed, 2 insertions(+), 40 deletions(-) diff --git a/tensorflow/contrib/training/__init__.py b/tensorflow/contrib/training/__init__.py index ed975b291327b6..245f36c2fe25ff 100644 --- a/tensorflow/contrib/training/__init__.py +++ b/tensorflow/contrib/training/__init__.py @@ -12,51 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Training and input utilities. - -## Splitting sequence inputs into minibatches with state saving - -Use [`SequenceQueueingStateSaver`](#SequenceQueueingStateSaver) or -its wrapper [`batch_sequences_with_states`](#batch_sequences_with_states) if -you have input data with a dynamic primary time / frame count axis which -you'd like to convert into fixed size segments during minibatching, and would -like to store state in the forward direction across segments of an example. +"""Training and input utilities. See @{$python/contrib.training} guide. @@batch_sequences_with_states @@NextQueuedSequenceBatch @@SequenceQueueingStateSaver - - -## Online data resampling - -To resample data with replacement on a per-example basis, use -['rejection_sample'](#rejection_sample) or -['resample_at_rate'](#resample_at_rate). For `rejection_sample`, provide -a boolean Tensor describing whether to accept or reject. Resulting batch sizes -are always the same. For `resample_at_rate`, provide the desired rate for each -example. Resulting batch sizes may vary. If you wish to specify relative -rates, rather than absolute ones, use ['weighted_resample'](#weighted_resample) -(which also returns the actual resampling rate used for each output example). - -Use ['stratified_sample'](#stratified_sample) to resample without replacement -from the data to achieve a desired mix of class proportions that the Tensorflow -graph sees. For instance, if you have a binary classification dataset that is -99.9% class 1, a common approach is to resample from the data so that the data -is more balanced. - @@rejection_sample @@resample_at_rate @@stratified_sample @@weighted_resample - -## Bucketing - -Use ['bucket'](#bucket) or -['bucket_by_sequence_length'](#bucket_by_sequence_length) to stratify -minibatches into groups ("buckets"). Use `bucket_by_sequence_length` -with the argument `dynamic_pad=True` to receive minibatches of similarly -sized sequences for efficient training via `dynamic_rnn`. - @@bucket @@bucket_by_sequence_length """ diff --git a/tensorflow/contrib/util/__init__.py b/tensorflow/contrib/util/__init__.py index 45efdc20c8066a..a0b845502fa990 100644 --- a/tensorflow/contrib/util/__init__.py +++ b/tensorflow/contrib/util/__init__.py @@ -13,9 +13,7 @@ # limitations under the License. # ============================================================================== -"""Utilities for dealing with Tensors. - -## Miscellaneous Utility Functions +"""Utilities for dealing with Tensors. See @{$python/contrib.util} guide. @@constant_value @@make_tensor_proto From c926b7569d9c4b52314c65de29420ceb25d0e352 Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Mon, 13 Feb 2017 15:53:14 -0800 Subject: [PATCH 07/50] Doc fixit task 2. Change: 147404726 --- tensorflow/python/client/session.py | 14 ------- tensorflow/python/framework/dtypes.py | 16 -------- tensorflow/python/framework/errors_impl.py | 3 -- tensorflow/python/framework/function.py | 3 -- tensorflow/python/framework/ops.py | 38 ------------------- tensorflow/python/framework/sparse_tensor.py | 9 ----- tensorflow/python/framework/tensor_shape.py | 20 ---------- tensorflow/python/lib/io/tf_record.py | 4 -- tensorflow/python/ops/data_flow_ops.py | 19 ---------- tensorflow/python/ops/tensor_array_ops.py | 18 --------- .../python/summary/event_accumulator.py | 11 ------ .../python/summary/event_multiplexer.py | 14 ------- tensorflow/python/summary/summary_iterator.py | 12 ------ .../summary/writer/event_file_writer.py | 6 --- tensorflow/python/summary/writer/writer.py | 20 ---------- 15 files changed, 207 deletions(-) diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py index c1a8191def335b..3f70b6240f2067 100644 --- a/tensorflow/python/client/session.py +++ b/tensorflow/python/client/session.py @@ -1150,17 +1150,6 @@ class Session(BaseSession): sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)) ``` - - @@__init__ - @@run - @@close - - @@graph - - @@as_default - - @@reset - """ def __init__(self, target='', graph=None, config=None): @@ -1285,9 +1274,6 @@ class InteractiveSession(BaseSession): # We can also use 'c.eval()' here. print(c.eval()) ``` - - @@__init__ - @@close """ def __init__(self, target='', graph=None, config=None): diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py index 7cc5854a307b96..7564cbcb058649 100644 --- a/tensorflow/python/framework/dtypes.py +++ b/tensorflow/python/framework/dtypes.py @@ -54,22 +54,6 @@ class DType(object): The `tf.as_dtype()` function converts numpy types and string type names to a `DType` object. - - @@is_compatible_with - @@name - @@base_dtype - @@real_dtype - @@is_bool - @@is_floating - @@is_complex - @@is_integer - @@is_quantized - @@is_unsigned - - @@as_numpy_dtype - @@as_datatype_enum - - @@limits """ def __init__(self, type_enum): diff --git a/tensorflow/python/framework/errors_impl.py b/tensorflow/python/framework/errors_impl.py index 79e0b5f069c391..e73bdefb21adf7 100644 --- a/tensorflow/python/framework/errors_impl.py +++ b/tensorflow/python/framework/errors_impl.py @@ -32,9 +32,6 @@ class OpError(Exception): Whenever possible, the session will raise a more specific subclass of `OpError` from the `tf.errors` module. - - @@op - @@node_def """ def __init__(self, node_def, op, message, error_code): diff --git a/tensorflow/python/framework/function.py b/tensorflow/python/framework/function.py index 1bb87e33e3078c..034bfaf21dfced 100644 --- a/tensorflow/python/framework/function.py +++ b/tensorflow/python/framework/function.py @@ -762,9 +762,6 @@ def MyFunc(x, y): b = tf.Constant([2.0]) c, d = MyFunc(a, b, name='mycall') ``` - - @@__init__ - """ def __init__(self, *input_types, **kwargs): diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 2c5f72ae951986..065b8977f4b6d4 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -230,20 +230,6 @@ class Tensor(_TensorLike): # Execute the graph and store the value that `e` represents in `result`. result = sess.run(e) ``` - - @@dtype - @@name - @@value_index - @@graph - @@op - @@consumers - - @@eval - - @@get_shape - @@shape - @@set_shape - """ # List of Python operators that we allow to override. @@ -1021,17 +1007,6 @@ class IndexedSlices(_TensorLike): Contrast this representation with [`SparseTensor`](../../api_docs/python/sparse_ops.md#SparseTensor), which uses multi-dimensional indices and scalar values. - - @@__init__ - - @@values - @@indices - @@dense_shape - - @@name - @@dtype - @@device - @@op """ def __init__(self, values, indices, dense_shape=None): @@ -1155,19 +1130,6 @@ class Operation(object): be executed by passing it to [`Session.run()`](../../api_docs/python/client.md#Session.run). `op.run()` is a shortcut for calling `tf.get_default_session().run(op)`. - - @@name - @@type - @@inputs - @@control_inputs - @@outputs - @@device - @@graph - - @@run - - @@get_attr - @@traceback """ def __init__(self, node_def, g, inputs=None, output_types=None, diff --git a/tensorflow/python/framework/sparse_tensor.py b/tensorflow/python/framework/sparse_tensor.py index c74077be95f5e7..4f65e433a562ab 100644 --- a/tensorflow/python/framework/sparse_tensor.py +++ b/tensorflow/python/framework/sparse_tensor.py @@ -89,15 +89,6 @@ class SparseTensor(_TensorLike): [0, 0, 2, 0] [0, 0, 0, 0]] ``` - - @@__init__ - @@get_shape - @@indices - @@values - @@dense_shape - @@dtype - @@op - @@graph """ @classmethod diff --git a/tensorflow/python/framework/tensor_shape.py b/tensorflow/python/framework/tensor_shape.py index b2c015c0b63550..5fa3310bb6195d 100644 --- a/tensorflow/python/framework/tensor_shape.py +++ b/tensorflow/python/framework/tensor_shape.py @@ -398,26 +398,6 @@ class TensorShape(object): details of shape functions and how to register them. Alternatively, the shape may be set explicitly using [`Tensor.set_shape()`](../../api_docs/python/framework.md#Tensor.set_shape). - - @@merge_with - @@concatenate - - @@ndims - @@dims - @@as_list - @@as_proto - @@is_compatible_with - @@is_fully_defined - - @@with_rank - @@with_rank_at_least - @@with_rank_at_most - - @@assert_has_rank - @@assert_same_rank - @@assert_is_compatible_with - @@assert_is_fully_defined - """ def __init__(self, dims): diff --git a/tensorflow/python/lib/io/tf_record.py b/tensorflow/python/lib/io/tf_record.py index d02baeb6cd4151..450b89e4c16789 100644 --- a/tensorflow/python/lib/io/tf_record.py +++ b/tensorflow/python/lib/io/tf_record.py @@ -86,10 +86,6 @@ class TFRecordWriter(object): This class implements `__enter__` and `__exit__`, and can be used in `with` blocks like a normal file. - - @@__init__ - @@write - @@close """ # TODO(josh11b): Support appending? diff --git a/tensorflow/python/ops/data_flow_ops.py b/tensorflow/python/ops/data_flow_ops.py index cfa70cf46a2140..a963a176f0d094 100644 --- a/tensorflow/python/ops/data_flow_ops.py +++ b/tensorflow/python/ops/data_flow_ops.py @@ -123,17 +123,6 @@ class QueueBase(object): [`tf.RandomShuffleQueue`](#RandomShuffleQueue) for concrete implementations of this class, and instructions on how to create them. - - @@enqueue - @@enqueue_many - - @@dequeue - @@dequeue_many - - @@size - - @@close - """ def __init__(self, dtypes, shapes, names, queue_ref): @@ -571,8 +560,6 @@ class RandomShuffleQueue(QueueBase): See [`tf.QueueBase`](#QueueBase) for a description of the methods on this class. - - @@__init__ """ def __init__(self, capacity, min_after_dequeue, dtypes, shapes=None, @@ -646,8 +633,6 @@ class FIFOQueue(QueueBase): See [`tf.QueueBase`](#QueueBase) for a description of the methods on this class. - - @@__init__ """ def __init__(self, capacity, dtypes, shapes=None, names=None, @@ -699,8 +684,6 @@ class PaddingFIFOQueue(QueueBase): See [`tf.QueueBase`](#QueueBase) for a description of the methods on this class. - - @@__init__ """ def __init__(self, capacity, dtypes, shapes, names=None, shared_name=None, @@ -763,8 +746,6 @@ class PriorityQueue(QueueBase): See [`tf.QueueBase`](#QueueBase) for a description of the methods on this class. - - @@__init__ """ def __init__(self, capacity, types, shapes=None, names=None, shared_name=None, diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py index 929d090a18489e..1b2b9faad70dde 100644 --- a/tensorflow/python/ops/tensor_array_ops.py +++ b/tensorflow/python/ops/tensor_array_ops.py @@ -59,24 +59,6 @@ class TensorArray(object): This class is meant to be used with dynamic iteration primitives such as `while_loop` and `map_fn`. It supports gradient back-propagation via special "flow" control flow dependencies. - - @@handle - @@flow - @@dtype - - @@read - @@gather - @@stack - @@concat - - @@write - @@scatter - @@unstack - @@split - - @@identity - - @@grad """ def __init__(self, diff --git a/tensorflow/python/summary/event_accumulator.py b/tensorflow/python/summary/event_accumulator.py index 024475cc7601ac..884de523de0c54 100644 --- a/tensorflow/python/summary/event_accumulator.py +++ b/tensorflow/python/summary/event_accumulator.py @@ -125,17 +125,6 @@ class EventAccumulator(object): Histograms, audio, and images are very large, so storing all of them is not recommended. - - @@Reload - @@Tags - @@Scalars - @@Graph - @@MetaGraph - @@RunMetadata - @@Histograms - @@CompressedHistograms - @@Images - @@Audio """ def __init__(self, diff --git a/tensorflow/python/summary/event_multiplexer.py b/tensorflow/python/summary/event_multiplexer.py index c8f81c327db542..be2f48db435576 100644 --- a/tensorflow/python/summary/event_multiplexer.py +++ b/tensorflow/python/summary/event_multiplexer.py @@ -65,20 +65,6 @@ class EventMultiplexer(object): If you would like to watch `/parent/directory/path`, wait for it to be created (if necessary) and then periodically pick up new runs, use `AutoloadingMultiplexer` - - @@__init__ - @@AddRun - @@AddRunsFromDirectory - @@Reload - @@Runs - @@RunPaths - @@Scalars - @@Graph - @@MetaGraph - @@Histograms - @@CompressedHistograms - @@Images - @@Audio """ def __init__(self, diff --git a/tensorflow/python/summary/summary_iterator.py b/tensorflow/python/summary/summary_iterator.py index 490ce141f139f1..4196fd815d98ad 100644 --- a/tensorflow/python/summary/summary_iterator.py +++ b/tensorflow/python/summary/summary_iterator.py @@ -44,18 +44,6 @@ class SummaryWriter(object): file contents asynchronously. This allows a training program to call methods to add data to the file directly from the training loop, without slowing down training. - - @@__init__ - - @@add_summary - @@add_session_log - @@add_event - @@add_graph - @@add_run_metadata - @@get_logdir - - @@flush - @@close """ def __init__(self, logdir, graph=None, max_queue=10, flush_secs=120, diff --git a/tensorflow/python/summary/writer/event_file_writer.py b/tensorflow/python/summary/writer/event_file_writer.py index b3455dc6323475..7142998ce726bf 100644 --- a/tensorflow/python/summary/writer/event_file_writer.py +++ b/tensorflow/python/summary/writer/event_file_writer.py @@ -35,12 +35,6 @@ class EventFileWriter(object): The `EventFileWriter` class creates an event file in the specified directory, and asynchronously writes Event protocol buffers to the file. The Event file is encoded using the tfrecord format, which is similar to RecordIO. - - @@__init__ - - @@add_event - @@flush - @@close """ def __init__(self, logdir, max_queue=10, flush_secs=120): diff --git a/tensorflow/python/summary/writer/writer.py b/tensorflow/python/summary/writer/writer.py index fc90d547dc69e0..51565d72f34c53 100644 --- a/tensorflow/python/summary/writer/writer.py +++ b/tensorflow/python/summary/writer/writer.py @@ -36,14 +36,6 @@ class SummaryToEventTransformer(object): This API basically implements a number of endpoints (add_summary, add_session_log, etc). The endpoints all generate an event protobuf, which is passed to the contained event_writer. - - @@__init__ - - @@add_summary - @@add_session_log - @@add_graph - @@add_meta_graph - @@add_run_metadata """ def __init__(self, event_writer, graph=None, graph_def=None): @@ -247,18 +239,6 @@ class FileWriter(SummaryToEventTransformer): file contents asynchronously. This allows a training program to call methods to add data to the file directly from the training loop, without slowing down training. - - @@__init__ - - @@add_summary - @@add_session_log - @@add_event - @@add_graph - @@add_run_metadata - @@get_logdir - - @@flush - @@close """ def __init__(self, From b780726d65b2caa56842c837b840da9549e7cc86 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 13 Feb 2017 16:01:34 -0800 Subject: [PATCH 08/50] Doc updates for Python IO, string, TensorArray, and summary ops Removes the API guide for TensorArray for the moment, since all of its useful content is in the class docstring. Change: 147405811 --- tensorflow/python/lib/io/python_io.py | 24 ++--------------------- tensorflow/python/ops/string_ops.py | 17 ++-------------- tensorflow/python/ops/tensor_array_ops.py | 4 +--- tensorflow/python/summary/summary.py | 10 +++------- 4 files changed, 8 insertions(+), 47 deletions(-) diff --git a/tensorflow/python/lib/io/python_io.py b/tensorflow/python/lib/io/python_io.py index a4c682aa28a7f5..b92cfe8f801341 100644 --- a/tensorflow/python/lib/io/python_io.py +++ b/tensorflow/python/lib/io/python_io.py @@ -13,34 +13,14 @@ # limitations under the License. # ============================================================================== -"""## Data IO (Python Functions) +"""Python functions for directly manipulating TFRecord-formatted files. -A TFRecords file represents a sequence of (binary) strings. The format is not -random access, so it is suitable for streaming large amounts of data but not -suitable if fast sharding or other non-sequential access is desired. +See the @{$python/python_io} guide. @@TFRecordWriter @@tf_record_iterator @@TFRecordCompressionType @@TFRecordOptions - -- - - - -### TFRecords Format Details - -A TFRecords file contains a sequence of strings with CRC hashes. Each record -has the format - - uint64 length - uint32 masked_crc32_of_length - byte data[length] - uint32 masked_crc32_of_data - -and the records are concatenated together to produce the file. The CRC32s -are [described here](https://en.wikipedia.org/wiki/Cyclic_redundancy_check), -and the mask of a CRC is - - masked_crc = ((crc >> 15) | (crc << 17)) + 0xa282ead8ul """ from __future__ import absolute_import diff --git a/tensorflow/python/ops/string_ops.py b/tensorflow/python/ops/string_ops.py index 169390fbf2941b..12293f03ad39d9 100644 --- a/tensorflow/python/ops/string_ops.py +++ b/tensorflow/python/ops/string_ops.py @@ -13,30 +13,17 @@ # limitations under the License. # ============================================================================== -"""## Hashing +"""Operations for working with string Tensors. -String hashing ops take a string input tensor and map each element to an -integer. +See the @{$python/string_ops} guide. @@string_to_hash_bucket_fast @@string_to_hash_bucket_strong @@string_to_hash_bucket - -## Joining - -String joining ops concatenate elements of input string tensors to produce a new -string tensor. - @@reduce_join @@string_join - -## Splitting - @@string_split @@substr - -## Conversion - @@as_string @@encode_base64 @@decode_base64 diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py index 1b2b9faad70dde..79fb15bdc12114 100644 --- a/tensorflow/python/ops/tensor_array_ops.py +++ b/tensorflow/python/ops/tensor_array_ops.py @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""TensorArray operations. - -## Classes containing dynamically sized arrays of Tensors. +"""TensorArray: a dynamically sized array of Tensors. @@TensorArray """ diff --git a/tensorflow/python/summary/summary.py b/tensorflow/python/summary/summary.py index 3cc0f9cd7fbf94..6b0f5a85e66031 100644 --- a/tensorflow/python/summary/summary.py +++ b/tensorflow/python/summary/summary.py @@ -13,13 +13,12 @@ # limitations under the License. # ============================================================================== -"""## Generation of summaries. +"""Tensor summaries for exporting information about a model. + +See the @{$python/summary} guide. -### Class for writing Summaries @@FileWriter @@FileWriterCache - -### Summary Ops @@tensor_summary @@scalar @@histogram @@ -27,10 +26,7 @@ @@image @@merge @@merge_all - -## Utilities @@get_summary_description - """ from __future__ import absolute_import From 31f30255a36ff66dcb50c8be6798845e8725b8a0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 13 Feb 2017 18:06:46 -0800 Subject: [PATCH 09/50] Update docs. Change: 147419923 --- tensorflow/python/ops/control_flow_ops.py | 23 +--------- tensorflow/python/ops/script_ops.py | 5 +-- tensorflow/python/ops/session_ops.py | 4 +- tensorflow/python/ops/sparse_ops.py | 18 +------- tensorflow/python/ops/state_ops.py | 52 +---------------------- 5 files changed, 6 insertions(+), 96 deletions(-) diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py index e3871431f3b909..e7c34e482ce298 100644 --- a/tensorflow/python/ops/control_flow_ops.py +++ b/tensorflow/python/ops/control_flow_ops.py @@ -13,10 +13,7 @@ # limitations under the License. # ============================================================================== -"""## Control Flow Operations - -TensorFlow provides several operations and classes that you can use to control -the execution of operations and add conditional dependencies to your graph. +"""Control Flow Operations. See the @{python/control_flow_ops} guide. @@identity @@tuple @@ -26,22 +23,10 @@ @@cond @@case @@while_loop - -## Logical Operators - -TensorFlow provides several operations that you can use to add logical operators -to your graph. - @@logical_and @@logical_not @@logical_or @@logical_xor - -## Comparison Operators - -TensorFlow provides several operations that you can use to add comparison -operators to your graph. - @@equal @@not_equal @@less @@ -49,12 +34,6 @@ @@greater @@greater_equal @@where - -## Debugging Operations - -TensorFlow provides several operations that you can use to validate values and -debug your graph. - @@is_finite @@is_inf @@is_nan diff --git a/tensorflow/python/ops/script_ops.py b/tensorflow/python/ops/script_ops.py index b89b76cf55f9c3..7996581e8b3946 100644 --- a/tensorflow/python/ops/script_ops.py +++ b/tensorflow/python/ops/script_ops.py @@ -12,13 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""## Script Language Operators. -TensorFlow provides allows you to wrap python/numpy functions as -TensorFlow operators. +"""Script Language Operators. See the @{python/script_ops} guide. @@py_func - """ # pylint: disable=g-bad-name diff --git a/tensorflow/python/ops/session_ops.py b/tensorflow/python/ops/session_ops.py index fceb4ebf863fbd..ff5d5a2b2f8e7c 100644 --- a/tensorflow/python/ops/session_ops.py +++ b/tensorflow/python/ops/session_ops.py @@ -12,10 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""## Tensor Handle Operations. -TensorFlow provides several operators that allows the user to keep tensors -"in-place" across run calls. +"""Tensor Handle Operations. See the @{python/session_ops} guide. @@get_session_handle @@get_session_tensor diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py index 0fff2d3ba538f5..5a49e57866d26f 100644 --- a/tensorflow/python/ops/sparse_ops.py +++ b/tensorflow/python/ops/sparse_ops.py @@ -14,25 +14,14 @@ # ============================================================================== # pylint: disable=g-short-docstring-punctuation -"""## Sparse Tensor Representation - -TensorFlow supports a `SparseTensor` representation for data that is sparse -in multiple dimensions. Contrast this representation with `IndexedSlices`, -which is efficient for representing tensors that are sparse in their first -dimension, and dense along all other dimensions. +"""Sparse Tensor Representation. See the @{python/sparse_ops} guide. @@SparseTensor @@SparseTensorValue - -## Conversion - @@sparse_to_dense @@sparse_tensor_to_dense @@sparse_to_indicator @@sparse_merge - -## Manipulation - @@sparse_concat @@sparse_reorder @@sparse_reshape @@ -41,18 +30,15 @@ @@sparse_reset_shape @@sparse_fill_empty_rows @@sparse_transpose - -## Reduction @@sparse_reduce_sum @@sparse_reduce_sum_sparse - -## Math Operations @@sparse_add @@sparse_softmax @@sparse_tensor_dense_matmul @@sparse_maximum @@sparse_minimum """ + from __future__ import absolute_import from __future__ import division from __future__ import print_function diff --git a/tensorflow/python/ops/state_ops.py b/tensorflow/python/ops/state_ops.py index ff1d2a6951bf34..a6607b9b3d7ebf 100644 --- a/tensorflow/python/ops/state_ops.py +++ b/tensorflow/python/ops/state_ops.py @@ -13,46 +13,27 @@ # limitations under the License. # ============================================================================== -"""## Variables +"""Variables. See the @{python/state_ops} guide. @@Variable - -## Variable helper functions - -TensorFlow provides a set of functions to help manage the set of variables -collected in the graph. - @@global_variables @@local_variables @@model_variables @@trainable_variables @@moving_average_variables - @@global_variables_initializer @@local_variables_initializer @@variables_initializer @@is_variable_initialized @@report_uninitialized_variables @@assert_variables_initialized - @@assign @@assign_add @@assign_sub - -## Saving and Restoring Variables - @@Saver - @@latest_checkpoint - @@get_checkpoint_state @@update_checkpoint_state - -## Sharing Variables - -TensorFlow provides several classes and operations that you can use to -create variables contingent on certain conditions. - @@get_variable @@get_local_variable @@VariableScope @@ -60,9 +41,7 @@ @@variable_op_scope @@get_variable_scope @@make_template - @@no_regularizer - @@constant_initializer @@random_normal_initializer @@truncated_normal_initializer @@ -71,27 +50,9 @@ @@zeros_initializer @@ones_initializer @@orthogonal_initializer - -## Variable Partitioners for Sharding - @@fixed_size_partitioner @@variable_axis_size_partitioner @@min_max_variable_partitioner - -## Sparse Variable Updates - -The sparse update ops modify a subset of the entries in a dense `Variable`, -either overwriting the entries or adding / subtracting a delta. These are -useful for training embedding models and similar lookup-based networks, since -only a small subset of embedding vectors change in any given step. - -Since a sparse update of a large tensor may be generated automatically during -gradient computation (as in the gradient of -[`tf.gather`](../../api_docs/python/array_ops.md#gather)), -an [`IndexedSlices`](#IndexedSlices) class is provided that encapsulates a set -of sparse indices and values. `IndexedSlices` objects are detected and handled -automatically by the optimizers in most cases. - @@scatter_update @@scatter_add @@scatter_sub @@ -102,25 +63,14 @@ @@scatter_nd_sub @@sparse_mask @@IndexedSlices - -### Read-only Lookup Tables - @@initialize_all_tables @@tables_initializer - - -## Exporting and Importing Meta Graphs - @@export_meta_graph @@import_meta_graph - -# Deprecated functions (removed after 2017-03-02). Please don't use them. - @@all_variables @@initialize_all_variables @@initialize_local_variables @@initialize_variables - """ from __future__ import absolute_import From 6b0a5aa102ec193e37f3f522860182381a3f872b Mon Sep 17 00:00:00 2001 From: Andrew Selle Date: Mon, 13 Feb 2017 18:12:19 -0800 Subject: [PATCH 10/50] Seal ffmpeg interface and fix document generator to use ffmpeg build target. Change: 147420392 --- tensorflow/contrib/ffmpeg/__init__.py | 5 + tensorflow/tools/docs/BUILD | 94 ++++++ tensorflow/tools/docs/generate.py | 408 ++++++++++++++++++++++++++ 3 files changed, 507 insertions(+) create mode 100644 tensorflow/tools/docs/generate.py diff --git a/tensorflow/contrib/ffmpeg/__init__.py b/tensorflow/contrib/ffmpeg/__init__.py index 348464a879ed71..7014d16984282c 100644 --- a/tensorflow/contrib/ffmpeg/__init__.py +++ b/tensorflow/contrib/ffmpeg/__init__.py @@ -25,3 +25,8 @@ from tensorflow.contrib.ffmpeg.ffmpeg_ops import decode_audio from tensorflow.contrib.ffmpeg.ffmpeg_ops import encode_audio + +from tensorflow.python.util.all_util import remove_undocumented + +_allowed_symbols = ['decode_audio', 'encode_audio'] +remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/tools/docs/BUILD b/tensorflow/tools/docs/BUILD index 4c37c291d0419a..a3829482d29c7f 100644 --- a/tensorflow/tools/docs/BUILD +++ b/tensorflow/tools/docs/BUILD @@ -16,6 +16,100 @@ py_binary( deps = ["//tensorflow:tensorflow_py"], ) +py_library( + name = "doc_generator_visitor", + srcs = [ + "doc_generator_visitor.py", + ], + srcs_version = "PY2AND3", +) + +py_test( + name = "doc_generator_visitor_test", + size = "small", + srcs = [ + "doc_generator_visitor_test.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":doc_generator_visitor", + "//tensorflow/python:platform_test", + ], +) + +py_library( + name = "parser", + srcs = [ + "parser.py", + ], + srcs_version = "PY2AND3", +) + +py_test( + name = "parser_test", + size = "small", + srcs = [ + "parser_test.py", + ], + srcs_version = "PY2AND3", + tags = ["manual"], + deps = [ + ":parser", + "//tensorflow/python:platform_test", + ], +) + +py_binary( + name = "generate", + srcs = ["generate.py"], + srcs_version = "PY2AND3", + deps = [ + ":doc_generator_visitor", + ":parser", + ":py_guide_parser", + "//tensorflow:tensorflow_py", + "//tensorflow/contrib/ffmpeg:ffmpeg_ops_py", + "//tensorflow/python/debug:debug_py", + "//tensorflow/tools/common:public_api", + "//tensorflow/tools/common:traverse", + ], +) + +py_test( + name = "generate_test", + size = "small", + srcs = [ + "generate_test.py", + ], + srcs_version = "PY2AND3", + tags = ["manual"], + deps = [ + ":generate", + "//tensorflow/python:platform_test", + ], +) + +py_library( + name = "py_guide_parser", + srcs = [ + "py_guide_parser.py", + ], + srcs_version = "PY2AND3", +) + +py_test( + name = "py_guide_parser_test", + size = "small", + srcs = [ + "py_guide_parser_test.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":py_guide_parser", + "//tensorflow/python:client_testlib", + ], +) + filegroup( name = "doxy_config", srcs = ["tf-doxy_for_md-config"], diff --git a/tensorflow/tools/docs/generate.py b/tensorflow/tools/docs/generate.py new file mode 100644 index 00000000000000..79acf05caa42f3 --- /dev/null +++ b/tensorflow/tools/docs/generate.py @@ -0,0 +1,408 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Generate docs for the TensorFlow Python API.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import inspect +import os +import sys + +import six +import tensorflow as tf + +from tensorflow.python import debug as tf_debug + +from tensorflow.tools.common import public_api +from tensorflow.tools.common import traverse +from tensorflow.tools.docs import doc_generator_visitor +from tensorflow.tools.docs import parser +from tensorflow.tools.docs import py_guide_parser + + +def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree, + reverse_index, doc_index, guide_index): + """Write previously extracted docs to disk. + + Write a docs page for each symbol in `index` to a tree of docs at + `output_dir`. + + Symbols with multiple aliases will have only one page written about them, + which is referenced for all aliases. `duplicate_of` and `duplicates` are used + to determine which docs pages to write. + + Args: + output_dir: Directory to write documentation markdown files to. Will be + created if it doesn't exist. + base_dir: Base directory of the code being documented. This prefix is + stripped from all file paths that are part of the documentation. + duplicate_of: A `dict` mapping fully qualified names to "master" names. This + is used to resolve "@{symbol}" references to the "master" name. + duplicates: A `dict` mapping fully qualified names to a set of all + aliases of this name. This is used to automatically generate a list of all + aliases for each name. + index: A `dict` mapping fully qualified names to the corresponding Python + objects. Used to produce docs for child objects, and to check the validity + of "@{symbol}" references. + tree: A `dict` mapping a fully qualified name to the names of all its + members. Used to populate the members section of a class or module page. + reverse_index: A `dict` mapping object ids to fully qualified names. + doc_index: A `dict` mapping a doc key to a DocInfo. + guide_index: A `dict` mapping symbol name strings to GuideRef. + """ + # Make output_dir. + try: + if not os.path.exists(output_dir): + os.makedirs(output_dir) + except OSError as e: + print('Creating output dir "%s" failed: %s' % (output_dir, e)) + raise + + # Parse and write Markdown pages, resolving cross-links (@{symbol}). + for full_name, py_object in six.iteritems(index): + + if full_name in duplicate_of: + continue + + # Methods and some routines are documented only as part of their class. + if not (inspect.ismodule(py_object) or + inspect.isclass(py_object) or + inspect.isfunction(py_object)): + continue + + print('Writing docs for %s (%r).' % (full_name, py_object)) + + # Generate docs for `py_object`, resolving references. + markdown = parser.generate_markdown(full_name, py_object, + duplicate_of=duplicate_of, + duplicates=duplicates, + index=index, + tree=tree, + reverse_index=reverse_index, + doc_index=doc_index, + guide_index=guide_index, + base_dir=base_dir) + + # TODO(deannarubin): use _tree to generate sidebar information. + + path = os.path.join(output_dir, parser.documentation_path(full_name)) + directory = os.path.dirname(path) + try: + if not os.path.exists(directory): + os.makedirs(directory) + with open(path, 'w') as f: + f.write(markdown) + except OSError as e: + print('Cannot write documentation for %s to %s: %s' % (full_name, + directory, e)) + raise + # TODO(deannarubin): write sidebar file? + + # Write a global index containing all full names with links. + with open(os.path.join(output_dir, 'index.md'), 'w') as f: + f.write(parser.generate_global_index('TensorFlow', index, duplicate_of)) + + +def extract(): + """Extract docs from tf namespace and write them to disk.""" + visitor = doc_generator_visitor.DocGeneratorVisitor('tf') + api_visitor = public_api.PublicAPIVisitor(visitor) + + # Access something in contrib so tf.contrib is properly loaded (it's hidden + # behind lazy loading) + _ = tf.contrib.__name__ + + # Exclude some libaries in contrib from the documentation altogether. + # TODO(wicke): Shrink this list. + api_visitor.do_not_descend_map.update({ + 'contrib': [ + 'compiler', + 'factorization', + 'grid_rnn', + 'labeled_tensor', + 'ndlstm', + 'quantization', + 'session_bundle', + 'slim', + 'solvers', + 'specs', + 'tensor_forest', + 'tensorboard', + 'testing', + 'tfprof', + 'training', + ], + 'contrib.bayesflow': [ + 'entropy', 'monte_carlo', + 'special_math', 'stochastic_gradient_estimators', + 'stochastic_graph', 'stochastic_tensor', + 'stochastic_variables', 'variational_inference' + ], + 'contrib.distributions': ['bijector'], + 'contrib.ffmpeg': ['ffmpeg_ops'], + 'contrib.graph_editor': [ + 'edit', + 'match', + 'reroute', + 'subgraph', + 'transform', + 'select', + 'util' + ], + 'contrib.layers': [ + 'feature_column', + 'summaries' + ], + 'contrib.learn': [ + 'datasets', + 'head', + 'graph_actions', + 'io', + 'models', + 'monitors', + 'ops', + 'preprocessing', + 'utils', + ], + 'contrib.util': ['loader'], + }) + + traverse.traverse(tf, api_visitor) + + # tf_debug is not imported with tf, it's a separate module altogether + visitor.set_root_name('tfdbg') + traverse.traverse(tf_debug, api_visitor) + + return visitor + + +class GetMarkdownTitle(py_guide_parser.PyGuideParser): + """Extract the title from a .md file.""" + + def __init__(self): + self.title = None + py_guide_parser.PyGuideParser.__init__(self) + + def process_title(self, _, title): + if self.title is None: # only use the first title + self.title = title + + +class DocInfo(object): + """A simple struct for holding a doc's url and title.""" + + def __init__(self, url, title): + self.url = url + self.title = title + + +def build_doc_index(src_dir): + """Build an index from a keyword designating a doc to DocInfo objects.""" + doc_index = {} + for dirpath, _, filenames in os.walk(src_dir): + suffix = os.path.relpath(path=dirpath, start=src_dir) + for base_name in filenames: + if not base_name.endswith('.md'): continue + title_parser = GetMarkdownTitle() + title_parser.process(os.path.join(dirpath, base_name)) + key_parts = os.path.join(suffix, base_name[:-3]).split('/') + if key_parts[-1] == 'index': + key_parts = key_parts[:-1] + doc_info = DocInfo(os.path.join(suffix, base_name), title_parser.title) + doc_index[key_parts[-1]] = doc_info + if len(key_parts) > 1: + doc_index['/'.join(key_parts[-2:])] = doc_info + + return doc_index + + +class GuideRef(object): + + def __init__(self, base_name, title, section_title, section_tag): + self.url = 'api_guides/python/' + ( + ('%s#%s' % (base_name, section_tag)) if section_tag else base_name) + self.link_text = (('%s > %s' % (title, section_title)) + if section_title else title) + + def make_md_link(self, url_prefix): + return '[%s](%s%s)' % (self.link_text, url_prefix, self.url) + + +class GenerateGuideIndex(py_guide_parser.PyGuideParser): + """Turn guide files into an index from symbol name to a list of GuideRefs.""" + + def __init__(self): + self.index = {} + py_guide_parser.PyGuideParser.__init__(self) + + def process(self, full_path, base_name): + """Index a file, reading from `full_path`, with `base_name` as the link.""" + self.full_path = full_path + self.base_name = base_name + self.title = None + self.section_title = None + self.section_tag = None + py_guide_parser.PyGuideParser.process(self, full_path) + + def process_title(self, _, title): + if self.title is None: # only use the first title + self.title = title + + def process_section(self, _, section_title, tag): + self.section_title = section_title + self.section_tag = tag + + def process_line(self, _, line): + """Index @{symbol} references as in the current file & section.""" + for match in parser.SYMBOL_REFERENCE_RE.finditer(line): + val = self.index.get(match.group(1), []) + val.append(GuideRef( + self.base_name, self.title, self.section_title, self.section_tag)) + self.index[match.group(1)] = val + + +def build_guide_index(guide_src_dir): + """Return dict: symbol name -> GuideRef from the files in `guide_src_dir`.""" + index_generator = GenerateGuideIndex() + for full_path, base_name in py_guide_parser.md_files_in_dir(guide_src_dir): + index_generator.process(full_path, base_name) + return index_generator.index + + +def write(output_dir, base_dir, doc_index, guide_index, visitor): + """Write documentation for an index in a `DocGeneratorVisitor` to disk. + + This function will create `output_dir` if it doesn't exist, and write + the documentation contained in `visitor`. + + Args: + output_dir: The directory to write documentation to. Must not exist. + base_dir: The base dir of the library `visitor` has traversed. This is used + to compute relative paths for file references. + doc_index: A `dict` mapping a doc key to a DocInfo. + guide_index: A `dict` mapping symbol name strings to GuideRef. + visitor: A `DocGeneratorVisitor` that has traversed a library located at + `base_dir`. + """ + write_docs(output_dir, os.path.abspath(base_dir), + visitor.duplicate_of, visitor.duplicates, + visitor.index, visitor.tree, visitor.reverse_index, + doc_index, guide_index) + + +class UpdateTags(py_guide_parser.PyGuideParser): + """Rewrites a Python guide so that each section has an explicit tag.""" + + def process_section(self, line_number, section_title, tag): + self.replace_line(line_number, '

%s

' % (tag, section_title)) + + +def other_docs(src_dir, output_dir, visitor, doc_index): + """Convert all the files in `src_dir` and write results to `output_dir`.""" + header = '\n' + + # Iterate through all the source files and process them. + tag_updater = UpdateTags() + for dirpath, _, filenames in os.walk(src_dir): + # How to get from `dirpath` to api_docs/python/ + relative_path_to_root = os.path.relpath( + path=os.path.join(src_dir, 'api_docs/python'), start=dirpath) + + # Make the directory under output_dir. + new_dir = os.path.join(output_dir, + os.path.relpath(path=dirpath, start=src_dir)) + try: + if not os.path.exists(new_dir): + os.makedirs(new_dir) + except OSError as e: + print('Creating output dir "%s" failed: %s' % (new_dir, e)) + raise + + for base_name in filenames: + full_in_path = os.path.join(dirpath, base_name) + suffix = os.path.relpath(path=full_in_path, start=src_dir) + full_out_path = os.path.join(output_dir, suffix) + if not base_name.endswith('.md'): + print('Copying non-md file %s...' % suffix) + open(full_out_path, 'w').write(open(full_in_path).read()) + continue + if dirpath.endswith('/api_guides/python'): + print('Processing Python guide %s...' % base_name) + md_string = tag_updater.process(full_in_path) + else: + print('Processing doc %s...' % suffix) + md_string = open(full_in_path).read() + + output = parser.replace_references( + md_string, relative_path_to_root, visitor.duplicate_of, + doc_index=doc_index, index=visitor.index) + with open(full_out_path, 'w') as f: + f.write(header + output) + + print('Done.') + + +def _main(src_dir, output_dir, base_dir): + doc_index = build_doc_index(src_dir) + visitor = extract() + write(os.path.join(output_dir, 'api_docs/python'), base_dir, + doc_index, + build_guide_index(os.path.join(src_dir, 'api_guides/python')), + visitor) + other_docs(src_dir, output_dir, visitor, doc_index) + + +if __name__ == '__main__': + argument_parser = argparse.ArgumentParser() + argument_parser.add_argument( + '--output_dir', + type=str, + default=None, + required=True, + help='Directory to write docs to.' + ) + + argument_parser.add_argument( + '--src_dir', + type=str, + default=None, + required=True, + help='Directory with the source docs.' + ) + + # This doc generator works on the TensorFlow codebase. Since this script lives + # at tensorflow/tools/docs, and all code is defined somewhere inside + # tensorflow/, we can compute the base directory (two levels up), which is + # valid unless we're trying to apply this to a different code base, or are + # moving the script around. + script_dir = os.path.dirname(inspect.getfile(inspect.currentframe())) + default_base_dir = os.path.join(script_dir, '..', '..') + + argument_parser.add_argument( + '--base_dir', + type=str, + default=default_base_dir, + help=('Base directory to to strip from file names referenced in docs. ' + 'Defaults to two directories up from the location of this file.') + ) + + flags, _ = argument_parser.parse_known_args() + _main(flags.src_dir, flags.output_dir, flags.base_dir) + if parser.all_errors: + print('Errors during processing:' + '\n '.join(parser.all_errors)) + sys.exit(1) From d315634779e0e7f7372aea6e1e92a5718c76bdd0 Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Mon, 13 Feb 2017 20:50:03 -0800 Subject: [PATCH 11/50] C++ API: clean up generated Attrs struct comments Old: /// Optional attribute setters for Copy : /// /// TensorName(StringPiece): Defaults to "" /// The name of the input tensor. struct Attrs { Attrs TensorName(StringPiece x) { Attrs ret = *this; ret.tensor_name_ = x; return ret; } StringPiece tensor_name_ = ""; }; New: /// Optional attribute setters for Copy struct Attrs { /// The name of the input tensor. /// /// Defaults to "" Attrs TensorName(StringPiece x) { Attrs ret = *this; ret.tensor_name_ = x; return ret; } StringPiece tensor_name_ = ""; }; Change: 147430152 --- tensorflow/cc/framework/cc_op_gen.cc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tensorflow/cc/framework/cc_op_gen.cc b/tensorflow/cc/framework/cc_op_gen.cc index 2d52aa18b83088..f70bfc4a01fedd 100644 --- a/tensorflow/cc/framework/cc_op_gen.cc +++ b/tensorflow/cc/framework/cc_op_gen.cc @@ -530,8 +530,6 @@ OpInfo::OpInfo(const OpDef& g_op_def, const OpDef& i_op_def, string OpInfo::GetOpAttrStruct() const { string struct_fields; string setters; - string attrs_comment = - strings::StrCat("Optional attribute setters for ", op_name, " :\n\n"); for (int i = 0; i < op_def.attr_size(); ++i) { const auto& attr(op_def.attr(i)); @@ -552,13 +550,15 @@ string OpInfo::GetOpAttrStruct() const { strings::StrCat(camel_case_name, suffix, "(", use_const ? "const " : "", attr_type_name, use_const ? "&" : ""); - strings::StrAppend(&attrs_comment, attr_func_def, "): Defaults to ", - SummarizeAttrValue(attr.default_value()), "\n"); + string attr_comment; if (!attr.description().empty()) { - // TODO(keveman): Word wrap and indent this to handle multi-line - // description. - strings::StrAppend(&attrs_comment, " ", attr.description(), "\n"); + strings::StrAppend(&attr_comment, attr.description(), "\n\n"); } + strings::StrAppend(&attr_comment, "Defaults to ", + SummarizeAttrValue(attr.default_value()), "\n"); + attr_comment = MakeComment(attr_comment, " "); + + strings::StrAppend(&setters, attr_comment); strings::StrAppend(&setters, " Attrs ", attr_func_def, " x) {\n"); strings::StrAppend(&setters, " Attrs ret = *this;\n"); strings::StrAppend(&setters, " ret.", attr.name(), "_ = x;\n"); @@ -573,6 +573,8 @@ string OpInfo::GetOpAttrStruct() const { return ""; } + string attrs_comment = + strings::StrCat("Optional attribute setters for ", op_name, "\n"); string struct_decl = MakeComment(attrs_comment, " "); strings::StrAppend(&struct_decl, " struct Attrs {\n"); strings::StrAppend(&struct_decl, setters, struct_fields); From 11cb90815c75d10d4d53b65dd957a49c87740ce9 Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Mon, 13 Feb 2017 21:39:15 -0800 Subject: [PATCH 12/50] C++ API: include optional attrs in generated class comments + improve core docs Old: /// Copy Op. /// /// Performs CPU-to-CPU or GPU-to-GPU deep-copying of tensor, depending on the /// device on which the tensor is allocated. /// /// Unlike the CopyHost Op, this op does not have HostMemory constraint on its /// input or output. /// /// Arguments: /// * scope: A Scope object /// * input: Input tensor. /// /// Returns: /// * `Output`: Output tensor, deep-copied from input. class Copy {...} New: /// Copy Op. /// /// Performs CPU-to-CPU or GPU-to-GPU deep-copying of tensor, depending on the /// device on which the tensor is allocated. /// /// Unlike the CopyHost Op, this op does not have HostMemory constraint on its /// input or output. /// /// Arguments: /// * scope: A Scope object /// * input: Input tensor. /// /// Optional attributes (see `Attrs`): /// * tensor_name: The name of the input tensor. /// /// Returns: /// * `Output`: Output tensor, deep-copied from input. class Copy {...} Change: 147432712 --- tensorflow/cc/framework/cc_op_gen.cc | 40 +++++++++++++++++++--------- tensorflow/cc/framework/ops.h | 3 +++ 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/tensorflow/cc/framework/cc_op_gen.cc b/tensorflow/cc/framework/cc_op_gen.cc index f70bfc4a01fedd..5e9800d4a6170c 100644 --- a/tensorflow/cc/framework/cc_op_gen.cc +++ b/tensorflow/cc/framework/cc_op_gen.cc @@ -437,6 +437,7 @@ OpInfo::OpInfo(const OpDef& g_op_def, const OpDef& i_op_def, } strings::StrAppend(&comment, "\nArguments:\n* scope: A Scope object\n"); + // Process inputs for (int i = 0; i < op_def.input_arg_size(); ++i) { const auto& arg(op_def.input_arg(i)); arg_types.push_back(strings::StrCat( @@ -451,30 +452,45 @@ OpInfo::OpInfo(const OpDef& g_op_def, const OpDef& i_op_def, arg.description(), "\n"); } } + + // Process attrs + string required_attrs_comment; + string optional_attrs_comment; for (int i = 0; i < op_def.attr_size(); ++i) { const auto& attr(op_def.attr(i)); - // If the attr is going to be inferred or is optional, don't add it as a - // required argument. - if ((inferred_input_attrs.find(attr.name()) != - inferred_input_attrs.end()) || - attr.has_default_value()) { - continue; - } + // Skip inferred arguments + if (inferred_input_attrs.count(attr.name()) > 0) continue; + const auto entry = AttrTypeName(attr.type()); const auto attr_type_name = entry.first; const bool use_const = entry.second; + string attr_name = AvoidCPPKeywords(attr.name()); - arg_types.push_back(strings::StrCat(use_const ? "const " : "", - attr_type_name, use_const ? "&" : "")); - arg_names.push_back(AvoidCPPKeywords(attr.name())); + string attr_comment; if (!attr.description().empty()) { - strings::StrAppend(&comment, "* ", AvoidCPPKeywords(attr.name()), ":\n"); // TODO(keveman): Word wrap and indent this, to handle multi-line // descriptions. - strings::StrAppend(&comment, " ", attr.description(), "\n"); + strings::StrAppend(&attr_comment, "* ", attr_name, ": ", + attr.description(), "\n"); + } + if (attr.has_default_value()) { + strings::StrAppend(&optional_attrs_comment, attr_comment); + } else { + strings::StrAppend(&required_attrs_comment, attr_comment); + arg_types.push_back(strings::StrCat( + use_const ? "const " : "", attr_type_name, use_const ? "&" : "")); + arg_names.push_back(attr_name); } } + strings::StrAppend(&comment, required_attrs_comment); + + if (!optional_attrs_comment.empty()) { + strings::StrAppend(&comment, "\nOptional attributes (see `Attrs`):\n"); + strings::StrAppend(&comment, optional_attrs_comment); + } + + // Process outputs for (int i = 0; i < op_def.output_arg_size(); ++i) { const auto& arg = op_def.output_arg(i); bool is_list = ArgIsList(arg); diff --git a/tensorflow/cc/framework/ops.h b/tensorflow/cc/framework/ops.h index c47d30ec3c5068..889d5db31dd06f 100644 --- a/tensorflow/cc/framework/ops.h +++ b/tensorflow/cc/framework/ops.h @@ -85,6 +85,7 @@ class Output { int64 index_ = 0; }; +/// Hash class that can be used for e.g. storing Outputs in an unordered_map struct OutputHash { std::size_t operator()(const Output& output) const { return Hash64Combine(std::hash()(output.node()), @@ -166,6 +167,7 @@ class Input { /// initializer list is indeed a valid multi-dimensional tensor. Initializer(const std::initializer_list& v); + // START_SKIP_DOXYGEN template ::value> struct RealType { typedef string type; @@ -175,6 +177,7 @@ class Input { struct RealType { typedef T type; }; + // END_SKIP_DOXYGEN TensorProto AsTensorProto() { TensorProto tensor_proto; From 81bfd9b69646ea06c39c69a5c03465897e1c53ae Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 13 Feb 2017 22:08:09 -0800 Subject: [PATCH 13/50] TASK 1. Python module doc strings TASK 2. //training/... Python class doc strings Change: 147434408 --- .../contrib/bayesflow/python/ops/entropy.py | 41 +---------- .../bayesflow/python/ops/monte_carlo.py | 49 +------------ .../bayesflow/python/ops/stochastic_graph.py | 4 +- .../bayesflow/python/ops/stochastic_tensor.py | 16 +---- .../python/ops/variational_inference.py | 3 +- tensorflow/python/client/client_lib.py | 12 +--- tensorflow/python/framework/constant_op.py | 68 +------------------ tensorflow/python/ops/array_ops.py | 22 +----- tensorflow/python/ops/check_ops.py | 2 +- .../python/training/gradient_descent.py | 2 - tensorflow/python/training/momentum.py | 2 - tensorflow/python/training/moving_averages.py | 6 -- tensorflow/python/training/optimizer.py | 9 --- .../python/training/proximal_adagrad.py | 2 - .../training/proximal_gradient_descent.py | 2 - tensorflow/python/training/rmsprop.py | 2 - tensorflow/python/training/saver.py | 11 --- tensorflow/python/training/server_lib.py | 11 --- tensorflow/python/training/supervisor.py | 13 ---- .../training/sync_replicas_optimizer.py | 6 -- 20 files changed, 14 insertions(+), 269 deletions(-) diff --git a/tensorflow/contrib/bayesflow/python/ops/entropy.py b/tensorflow/contrib/bayesflow/python/ops/entropy.py index 56490c390cf9c5..4df6d5a911a371 100644 --- a/tensorflow/contrib/bayesflow/python/ops/entropy.py +++ b/tensorflow/contrib/bayesflow/python/ops/entropy.py @@ -12,51 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -r"""Entropy Ops. - -## Background - -Common Shannon entropy, the Evidence Lower BOund (ELBO), KL divergence, and more -all have information theoretic use and interpretations. They are also often -used in variational inference. This library brings together `Ops` for -estimating them, e.g. using Monte Carlo expectations. - -## Examples - -Example of fitting a variational posterior with the ELBO. - -```python -# We start by assuming knowledge of the log of a joint density p(z, x) over -# latent variable z and fixed measurement x. Since x is fixed, the Python -# function does not take x as an argument. -def log_joint(z): - theta = tf.Variable(0.) # Trainable variable that helps define log_joint. - ... - -# Next, define a Normal distribution with trainable parameters. -q = distributions.Normal(mu=tf.Variable(0.), sigma=tf.Variable(1.)) - -# Now, define a loss function (negative ELBO) that, when minimized, will adjust -# mu, sigma, and theta, increasing the ELBO, which we hope will both reduce the -# KL divergence between q(z) and p(z | x), and increase p(x). Note that we -# cannot guarantee both, but in general we expect both to happen. -elbo = entropy.elbo_ratio(log_p, q, n=10) -loss = -elbo - -# Minimize the loss -train_op = tf.train.GradientDescentOptimizer(0.1).minimize(loss) -tf.global_variables_initializer().run() -for step in range(100): - train_op.run() -``` - -## Ops +"""Support for Entropy Ops. See ${python/contrib.bayesflow.entropy}. @@elbo_ratio @@entropy_shannon @@renyi_ratio @@renyi_alpha - """ from __future__ import absolute_import diff --git a/tensorflow/contrib/bayesflow/python/ops/monte_carlo.py b/tensorflow/contrib/bayesflow/python/ops/monte_carlo.py index 198d755dffceed..a8654bcf312b64 100644 --- a/tensorflow/contrib/bayesflow/python/ops/monte_carlo.py +++ b/tensorflow/contrib/bayesflow/python/ops/monte_carlo.py @@ -12,58 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -r"""Monte Carlo integration and helpers. +"""Monte Carlo integration and helpers. -## Background - -Monte Carlo integration refers to the practice of estimating an expectation with -a sample mean. For example, given random variable `Z in R^k` with density `p`, -the expectation of function `f` can be approximated like: - -``` -E_p[f(Z)] = \int f(z) p(z) dz - ~ S_n - := n^{-1} \sum_{i=1}^n f(z_i), z_i iid samples from p. -``` - -If `E_p[|f(Z)|] < infinity`, then `S_n --> E_p[f(Z)]` by the strong law of large -numbers. If `E_p[f(Z)^2] < infinity`, then `S_n` is asymptotically normal with -variance `Var[f(Z)] / n`. - -Practitioners of Bayesian statistics often find themselves wanting to estimate -`E_p[f(Z)]` when the distribution `p` is known only up to a constant. For -example, the joint distribution `p(z, x)` may be known, but the evidence -`p(x) = \int p(z, x) dz` may be intractable. In that case, a parameterized -distribution family `q_lambda(z)` may be chosen, and the optimal `lambda` is the -one minimizing the KL divergence between `q_lambda(z)` and -`p(z | x)`. We only know `p(z, x)`, but that is sufficient to find `lambda`. - - -## Log-space evaluation and subtracting the maximum. - -Care must be taken when the random variable lives in a high dimensional space. -For example, the naive importance sample estimate `E_q[f(Z) p(Z) / q(Z)]` -involves the ratio of two terms `p(Z) / q(Z)`, each of which must have tails -dropping off faster than `O(|z|^{-(k + 1)})` in order to have finite integral. -This ratio would often be zero or infinity up to numerical precision. - -For that reason, we write - -``` -Log E_q[ f(Z) p(Z) / q(Z) ] - = Log E_q[ exp{Log[f(Z)] + Log[p(Z)] - Log[q(Z)] - C} ] + C, where -C := Max[ Log[f(Z)] + Log[p(Z)] - Log[q(Z)] ]. -``` - -The maximum value of the exponentiated term will be 0.0, and the expectation -can be evaluated in a stable manner. - -## Ops +See the ${@python/contrib.bayesflow.monte_carlo} guide. @@expectation @@expectation_importance_sampler @@expectation_importance_sampler_logspace - """ from __future__ import absolute_import diff --git a/tensorflow/contrib/bayesflow/python/ops/stochastic_graph.py b/tensorflow/contrib/bayesflow/python/ops/stochastic_graph.py index 81c04d8e9b548d..cf09e36ccb6172 100644 --- a/tensorflow/contrib/bayesflow/python/ops/stochastic_graph.py +++ b/tensorflow/contrib/bayesflow/python/ops/stochastic_graph.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Classes and helper functions for Stochastic Computation Graphs. +"""Support for Stochastic Computation Graphs. -## Stochastic Computation Graph Helper Functions +See the ${@python/contrib.bayesflow.stochastic_graph} guide. @@surrogate_loss """ diff --git a/tensorflow/contrib/bayesflow/python/ops/stochastic_tensor.py b/tensorflow/contrib/bayesflow/python/ops/stochastic_tensor.py index 62a583f188e31d..324e4f23be471e 100644 --- a/tensorflow/contrib/bayesflow/python/ops/stochastic_tensor.py +++ b/tensorflow/contrib/bayesflow/python/ops/stochastic_tensor.py @@ -12,26 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Classes and helper functions for creating Stochastic Tensors. +"""Support for creating Stochastic Tensors. -`StochasticTensor` objects wrap `Distribution` objects. Their -values may be samples from the underlying distribution, or the distribution -mean (as governed by `value_type`). These objects provide a `loss` -method for use when sampling from a non-reparameterized distribution. -The `loss`method is used in conjunction with `stochastic_graph.surrogate_loss` -to produce a single differentiable loss in stochastic graphs having -both continuous and discrete stochastic nodes. - -## Stochastic Tensor Classes +See the ${@python/contrib.bayesflow.stochastic_tensor} guide. @@BaseStochasticTensor @@StochasticTensor - -## Stochastic Tensor Value Types - @@MeanValue @@SampleValue - @@value_type @@get_current_value_type """ diff --git a/tensorflow/contrib/bayesflow/python/ops/variational_inference.py b/tensorflow/contrib/bayesflow/python/ops/variational_inference.py index 9c2d8435644d4a..17a86666863fff 100644 --- a/tensorflow/contrib/bayesflow/python/ops/variational_inference.py +++ b/tensorflow/contrib/bayesflow/python/ops/variational_inference.py @@ -14,13 +14,12 @@ # ============================================================================== """Variational inference. -## Ops +See the ${@python/contrib.bayesflow.variational_inference} guide. @@elbo @@elbo_with_log_joint @@ELBOForms @@register_prior - """ from __future__ import absolute_import diff --git a/tensorflow/python/client/client_lib.py b/tensorflow/python/client/client_lib.py index 91f328205b844e..b9ecaa4c851c08 100644 --- a/tensorflow/python/client/client_lib.py +++ b/tensorflow/python/client/client_lib.py @@ -13,20 +13,13 @@ # limitations under the License. # ============================================================================== -"""This library contains classes for launching graphs and executing operations. +"""Support for launching graphs and executing operations. -The [basic usage](../../get_started/index.md#basic-usage) guide has -examples of how a graph is launched in a [`tf.Session`](#Session). - -## Session management +See the @{$python/client} guide. @@Session @@InteractiveSession - @@get_default_session - -## Error classes and convenience functions - @@OpError @@CancelledError @@UnknownError @@ -44,7 +37,6 @@ @@InternalError @@UnavailableError @@DataLossError - @@exception_type_from_error_code @@error_code_from_exception_type @@raise_exception_on_not_ok_status diff --git a/tensorflow/python/framework/constant_op.py b/tensorflow/python/framework/constant_op.py index 3bcc5377797a90..f315cce93a175f 100644 --- a/tensorflow/python/framework/constant_op.py +++ b/tensorflow/python/framework/constant_op.py @@ -13,82 +13,16 @@ # limitations under the License. # ============================================================================== -"""## Constant Value Tensors - -TensorFlow provides several operations that you can use to generate constants. +"""Operations that generate constants. See the ${@python/constant} guide. @@zeros @@zeros_like - @@ones @@ones_like - @@fill - @@constant - -## Sequences - @@linspace - @@range - -## Random Tensors - -TensorFlow has several ops that create random tensors with different -distributions. The random ops are stateful, and create new random values each -time they are evaluated. - -The `seed` keyword argument in these functions acts in conjunction with -the graph-level random seed. Changing either the graph-level seed using -[`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) or the -op-level seed will change the underlying seed of these operations. Setting -neither graph-level nor op-level seed, results in a random seed for all -operations. -See [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) -for details on the interaction between operation-level and graph-level random -seeds. - -### Examples: - -```python -# Create a tensor of shape [2, 3] consisting of random normal values, with mean -# -1 and standard deviation 4. -norm = tf.random_normal([2, 3], mean=-1, stddev=4) - -# Shuffle the first dimension of a tensor -c = tf.constant([[1, 2], [3, 4], [5, 6]]) -shuff = tf.random_shuffle(c) - -# Each time we run these ops, different results are generated -sess = tf.Session() -print(sess.run(norm)) -print(sess.run(norm)) - -# Set an op-level seed to generate repeatable sequences across sessions. -norm = tf.random_normal([2, 3], seed=1234) -sess = tf.Session() -print(sess.run(norm)) -print(sess.run(norm)) -sess = tf.Session() -print(sess.run(norm)) -print(sess.run(norm)) -``` - -Another common use of random values is the initialization of variables. Also see -the [Variables How To](../../how_tos/variables/index.md). - -```python -# Use random uniform values in [0, 1) as the initializer for a variable of shape -# [2, 3]. The default type is float32. -var = tf.Variable(tf.random_uniform([2, 3]), name="var") -init = tf.global_variables_initializer() - -sess = tf.Session() -sess.run(init) -print(sess.run(var)) -``` - @@random_normal @@truncated_normal @@random_uniform diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index c5ad2bdd2bc845..c811866f7256c2 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -13,10 +13,7 @@ # limitations under the License. # ============================================================================== -"""## Casting - -TensorFlow provides several operations that you can use to cast tensor data -types in your graph. +"""Support for manipulating tensors. See the @{$python/array_ops} guide. @@string_to_number @@to_double @@ -27,12 +24,6 @@ @@cast @@bitcast @@saturate_cast - -## Shapes and Shaping - -TensorFlow provides several operations that you can use to determine the shape -of a tensor and change the shape of a tensor. - @@broadcast_dynamic_shape @@broadcast_static_shape @@shape @@ -43,12 +34,6 @@ @@squeeze @@expand_dims @@meshgrid - -## Slicing and Joining - -TensorFlow provides several operations to slice or extract parts of a tensor, -or join multiple tensors together. - @@slice @@strided_slice @@split @@ -83,10 +68,6 @@ @@quantize_v2 @@quantized_concat @@setdiff1d - -## Fake quantization -Operations used to help train for better quantization accuracy. - @@fake_quant_with_min_max_args @@fake_quant_with_min_max_args_gradient @@fake_quant_with_min_max_vars @@ -94,6 +75,7 @@ @@fake_quant_with_min_max_vars_per_channel @@fake_quant_with_min_max_vars_per_channel_gradient """ + from __future__ import absolute_import from __future__ import division from __future__ import print_function diff --git a/tensorflow/python/ops/check_ops.py b/tensorflow/python/ops/check_ops.py index 0e088649cd5d79..ec8f29767ced5f 100644 --- a/tensorflow/python/ops/check_ops.py +++ b/tensorflow/python/ops/check_ops.py @@ -13,7 +13,7 @@ # limitations under the License. # ============================================================================== # pylint: disable=g-short-docstring-punctuation -"""## Asserts and Boolean Checks +"""## Asserts and Boolean Checks. See the @{$python/check_ops} guide. @@assert_negative @@assert_positive diff --git a/tensorflow/python/training/gradient_descent.py b/tensorflow/python/training/gradient_descent.py index da431ecdf8dd97..239c671e8b09eb 100644 --- a/tensorflow/python/training/gradient_descent.py +++ b/tensorflow/python/training/gradient_descent.py @@ -27,8 +27,6 @@ class GradientDescentOptimizer(optimizer.Optimizer): """Optimizer that implements the gradient descent algorithm. - - @@__init__ """ def __init__(self, learning_rate, use_locking=False, name="GradientDescent"): diff --git a/tensorflow/python/training/momentum.py b/tensorflow/python/training/momentum.py index bd9e124cdfa8a3..104204b4065ad4 100644 --- a/tensorflow/python/training/momentum.py +++ b/tensorflow/python/training/momentum.py @@ -26,8 +26,6 @@ class MomentumOptimizer(optimizer.Optimizer): """Optimizer that implements the Momentum algorithm. - - @@__init__ """ def __init__(self, learning_rate, momentum, diff --git a/tensorflow/python/training/moving_averages.py b/tensorflow/python/training/moving_averages.py index a7784103f3e3bf..4b1f668a949f3b 100644 --- a/tensorflow/python/training/moving_averages.py +++ b/tensorflow/python/training/moving_averages.py @@ -282,12 +282,6 @@ class ExponentialMovingAverage(object): saver.restore(...checkpoint filename...) # var0 and var1 now hold the moving average values ``` - - @@__init__ - @@apply - @@average_name - @@average - @@variables_to_restore """ def __init__(self, decay, num_updates=None, zero_debias=False, diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py index ebaab86ec4dfc6..2d9fac645f8d9d 100644 --- a/tensorflow/python/training/optimizer.py +++ b/tensorflow/python/training/optimizer.py @@ -173,12 +173,6 @@ class directly, but instead instantiate one of its subclasses such as opt.apply_gradients(capped_grads_and_vars) ``` - @@__init__ - - @@minimize - @@compute_gradients - @@apply_gradients - ### Gating Gradients Both `minimize()` and `compute_gradients()` accept a `gate_gradients` @@ -212,9 +206,6 @@ class directly, but instead instantiate one of its subclasses such as This can be useful if you want to log debug a training algorithm, report stats about the slots, etc. - - @@get_slot_names - @@get_slot """ # Values for gate_gradients. diff --git a/tensorflow/python/training/proximal_adagrad.py b/tensorflow/python/training/proximal_adagrad.py index 354df153a77ca7..497c6139378cd1 100644 --- a/tensorflow/python/training/proximal_adagrad.py +++ b/tensorflow/python/training/proximal_adagrad.py @@ -29,8 +29,6 @@ class ProximalAdagradOptimizer(optimizer.Optimizer): """Optimizer that implements the Proximal Adagrad algorithm. See this [paper](http://papers.nips.cc/paper/3793-efficient-learning-using-forward-backward-splitting.pdf). - - @@__init__ """ def __init__(self, learning_rate, initial_accumulator_value=0.1, diff --git a/tensorflow/python/training/proximal_gradient_descent.py b/tensorflow/python/training/proximal_gradient_descent.py index aa0e5e3b4f6037..fee357b97e276a 100644 --- a/tensorflow/python/training/proximal_gradient_descent.py +++ b/tensorflow/python/training/proximal_gradient_descent.py @@ -31,8 +31,6 @@ class ProximalGradientDescentOptimizer(optimizer.Optimizer): """Optimizer that implements the proximal gradient descent algorithm. See this [paper](http://papers.nips.cc/paper/3793-efficient-learning-using-forward-backward-splitting.pdf). - - @@__init__ """ def __init__(self, learning_rate, l1_regularization_strength=0.0, diff --git a/tensorflow/python/training/rmsprop.py b/tensorflow/python/training/rmsprop.py index 3a9685135bae84..3a84ff1559a229 100644 --- a/tensorflow/python/training/rmsprop.py +++ b/tensorflow/python/training/rmsprop.py @@ -51,8 +51,6 @@ class RMSPropOptimizer(optimizer.Optimizer): """Optimizer that implements the RMSProp algorithm. See the [paper](http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf). - - @@__init__ """ def __init__(self, diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py index 4884bd282b1956..e563355080cd3a 100644 --- a/tensorflow/python/training/saver.py +++ b/tensorflow/python/training/saver.py @@ -925,17 +925,6 @@ class Saver(object): If you create several savers, you can specify a different filename for the protocol buffer file in the call to `save()`. - - @@__init__ - @@save - @@restore - - Other utility methods. - - @@last_checkpoints - @@set_last_checkpoints_with_time - @@recover_last_checkpoints - @@as_saver_def """ def __init__(self, diff --git a/tensorflow/python/training/server_lib.py b/tensorflow/python/training/server_lib.py index fc8896a12506dc..97cf1ddbc22d20 100644 --- a/tensorflow/python/training/server_lib.py +++ b/tensorflow/python/training/server_lib.py @@ -100,14 +100,6 @@ class Server(object): cluster (specified by a [`tf.train.ClusterSpec`](#ClusterSpec)), and corresponds to a particular task in a named job. The server can communicate with any other server in the same cluster. - - @@__init__ - @@create_local_server - @@target - @@server_def - - @@start - @@join """ def __init__(self, @@ -257,9 +249,6 @@ class ClusterSpec(object): "ps": ["ps0.example.com:2222", "ps1.example.com:2222"]}) ``` - - @@as_cluster_def - @@as_dict """ def __init__(self, cluster): diff --git a/tensorflow/python/training/supervisor.py b/tensorflow/python/training/supervisor.py index aa5081870e2344..bacddd9d826c0a 100644 --- a/tensorflow/python/training/supervisor.py +++ b/tensorflow/python/training/supervisor.py @@ -190,19 +190,6 @@ class Supervisor(object): initialization needs, see how to specify a `local_init_op` when creating the supervisor. You can also use the `SessionManager` directly to create a session and check if it could be initialized automatically. - - @@__init__ - @@managed_session - @@prepare_or_wait_for_session - @@start_standard_services - @@start_queue_runners - @@summary_computed - - @@stop - @@request_stop - @@should_stop - @@stop_on_exception - @@wait_for_stop """ # Value to pass for the 'ready_op', 'init_op', 'summary_op', 'saver', diff --git a/tensorflow/python/training/sync_replicas_optimizer.py b/tensorflow/python/training/sync_replicas_optimizer.py index 0c69b351b68502..928aff64f9c01c 100644 --- a/tensorflow/python/training/sync_replicas_optimizer.py +++ b/tensorflow/python/training/sync_replicas_optimizer.py @@ -123,12 +123,6 @@ class SyncReplicasOptimizer(optimizer.Optimizer): while not mon_sess.should_stop(): mon_sess.run(training_op) ``` - - @@__init__ - @@compute_gradients - @@apply_gradients - @@get_chief_queue_runner - @@get_init_tokens_op """ def __init__(self, From 91c50b6c6fc1eecee5cd45e5aae7dd9046e974cb Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 13 Feb 2017 22:59:47 -0800 Subject: [PATCH 14/50] Drop reference to monitors guide. Change: 147437030 --- tensorflow/contrib/learn/python/learn/monitors.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorflow/contrib/learn/python/learn/monitors.py b/tensorflow/contrib/learn/python/learn/monitors.py index 9a57605f9e864f..8b41f4e6a68aa7 100644 --- a/tensorflow/contrib/learn/python/learn/monitors.py +++ b/tensorflow/contrib/learn/python/learn/monitors.py @@ -15,8 +15,6 @@ """Monitors instrument the training process. -See the @{$python/contrib.learn.monitors} guide. - @@get_default_monitors @@BaseMonitor @@CaptureVariable From f63bb3a836a0eb8502b5719e65c0500317519de5 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Tue, 14 Feb 2017 07:39:49 -0800 Subject: [PATCH 15/50] tfdbg: adjust file paths after Python API seal to avoid import problems Change: 147470630 --- tensorflow/python/debug/BUILD | 16 ++++++++-------- tensorflow/python/debug/__init__.py | 16 ++++++++-------- tensorflow/python/debug/cli/analyzer_cli.py | 2 +- tensorflow/python/debug/cli/analyzer_cli_test.py | 4 ++-- tensorflow/python/debug/cli/offline_analyzer.py | 2 +- tensorflow/python/debug/cli/stepper_cli.py | 2 +- tensorflow/python/debug/cli/stepper_cli_test.py | 2 +- tensorflow/python/debug/lib/__init__.py | 0 tensorflow/python/debug/{ => lib}/debug_data.py | 0 .../python/debug/{ => lib}/debug_data_test.py | 2 +- tensorflow/python/debug/{ => lib}/debug_utils.py | 0 .../python/debug/{ => lib}/debug_utils_test.py | 2 +- .../debug/{ => lib}/session_debug_file_test.py | 6 +++--- .../debug/{ => lib}/session_debug_testlib.py | 4 ++-- tensorflow/python/debug/{ => lib}/stepper.py | 3 ++- .../python/debug/{ => lib}/stepper_test.py | 2 +- .../python/debug/wrappers/dumping_wrapper.py | 2 +- .../debug/wrappers/dumping_wrapper_test.py | 4 ++-- tensorflow/python/debug/wrappers/framework.py | 4 ++-- .../python/debug/wrappers/framework_test.py | 2 +- tensorflow/python/debug/wrappers/hooks.py | 4 ++-- .../python/debug/wrappers/local_cli_wrapper.py | 2 +- 22 files changed, 41 insertions(+), 40 deletions(-) create mode 100644 tensorflow/python/debug/lib/__init__.py rename tensorflow/python/debug/{ => lib}/debug_data.py (100%) rename tensorflow/python/debug/{ => lib}/debug_data_test.py (99%) rename tensorflow/python/debug/{ => lib}/debug_utils.py (100%) rename tensorflow/python/debug/{ => lib}/debug_utils_test.py (99%) rename tensorflow/python/debug/{ => lib}/session_debug_file_test.py (96%) rename tensorflow/python/debug/{ => lib}/session_debug_testlib.py (99%) rename tensorflow/python/debug/{ => lib}/stepper.py (99%) rename tensorflow/python/debug/{ => lib}/stepper_test.py (99%) diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD index 0aa5ce0a60b6fe..4a1a44927ffb72 100644 --- a/tensorflow/python/debug/BUILD +++ b/tensorflow/python/debug/BUILD @@ -31,7 +31,7 @@ py_library( py_library( name = "debug_data", - srcs = ["debug_data.py"], + srcs = ["lib/debug_data.py"], srcs_version = "PY2AND3", deps = [ "//tensorflow/python:framework", @@ -42,13 +42,13 @@ py_library( py_library( name = "debug_utils", - srcs = ["debug_utils.py"], + srcs = ["lib/debug_utils.py"], srcs_version = "PY2AND3", ) py_library( name = "stepper", - srcs = ["stepper.py"], + srcs = ["lib/stepper.py"], srcs_version = "PY2AND3", deps = [ ":debug_data", @@ -291,7 +291,7 @@ py_test( name = "debug_data_test", size = "small", srcs = [ - "debug_data_test.py", + "lib/debug_data_test.py", ], srcs_version = "PY2AND3", deps = [ @@ -306,7 +306,7 @@ py_test( name = "debug_utils_test", size = "small", srcs = [ - "debug_utils_test.py", + "lib/debug_utils_test.py", ], srcs_version = "PY2AND3", deps = [ @@ -326,7 +326,7 @@ cuda_py_test( name = "stepper_test", size = "small", srcs = [ - "stepper_test.py", + "lib/stepper_test.py", ], additional_deps = [ ":stepper", @@ -398,7 +398,7 @@ py_test( py_library( name = "session_debug_testlib", - srcs = ["session_debug_testlib.py"], + srcs = ["lib/session_debug_testlib.py"], srcs_version = "PY2AND3", deps = [ ":debug_data", @@ -423,7 +423,7 @@ py_library( cuda_py_test( name = "session_debug_file_test", size = "small", - srcs = ["session_debug_file_test.py"], + srcs = ["lib/session_debug_file_test.py"], additional_deps = [ ":debug_data", ":debug_utils", diff --git a/tensorflow/python/debug/__init__.py b/tensorflow/python/debug/__init__.py index 2ef44ca6d91bd0..c9c702089b0018 100644 --- a/tensorflow/python/debug/__init__.py +++ b/tensorflow/python/debug/__init__.py @@ -67,14 +67,14 @@ from __future__ import print_function # pylint: disable=unused-imports -from tensorflow.python.debug.debug_data import DebugDumpDir -from tensorflow.python.debug.debug_data import DebugTensorDatum -from tensorflow.python.debug.debug_data import has_inf_or_nan -from tensorflow.python.debug.debug_data import load_tensor_from_event_file - -from tensorflow.python.debug.debug_utils import add_debug_tensor_watch -from tensorflow.python.debug.debug_utils import watch_graph -from tensorflow.python.debug.debug_utils import watch_graph_with_blacklists +from tensorflow.python.debug.lib.debug_data import DebugDumpDir +from tensorflow.python.debug.lib.debug_data import DebugTensorDatum +from tensorflow.python.debug.lib.debug_data import has_inf_or_nan +from tensorflow.python.debug.lib.debug_data import load_tensor_from_event_file + +from tensorflow.python.debug.lib.debug_utils import add_debug_tensor_watch +from tensorflow.python.debug.lib.debug_utils import watch_graph +from tensorflow.python.debug.lib.debug_utils import watch_graph_with_blacklists from tensorflow.python.debug.wrappers.dumping_wrapper import DumpingDebugWrapperSession from tensorflow.python.debug.wrappers.hooks import DumpingDebugHook diff --git a/tensorflow/python/debug/cli/analyzer_cli.py b/tensorflow/python/debug/cli/analyzer_cli.py index 8e609b86e91bed..59db9924429200 100644 --- a/tensorflow/python/debug/cli/analyzer_cli.py +++ b/tensorflow/python/debug/cli/analyzer_cli.py @@ -29,11 +29,11 @@ from six.moves import xrange # pylint: disable=redefined-builtin -from tensorflow.python.debug import debug_data from tensorflow.python.debug.cli import cli_shared from tensorflow.python.debug.cli import command_parser from tensorflow.python.debug.cli import debugger_cli_common from tensorflow.python.debug.cli import ui_factory +from tensorflow.python.debug.lib import debug_data # String constants for the depth-dependent hanging indent at the beginning diff --git a/tensorflow/python/debug/cli/analyzer_cli_test.py b/tensorflow/python/debug/cli/analyzer_cli_test.py index 5d3f1aed2fb934..e981fe4f96803a 100644 --- a/tensorflow/python/debug/cli/analyzer_cli_test.py +++ b/tensorflow/python/debug/cli/analyzer_cli_test.py @@ -25,11 +25,11 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session -from tensorflow.python.debug import debug_data -from tensorflow.python.debug import debug_utils from tensorflow.python.debug.cli import analyzer_cli from tensorflow.python.debug.cli import command_parser from tensorflow.python.debug.cli import debugger_cli_common +from tensorflow.python.debug.lib import debug_data +from tensorflow.python.debug.lib import debug_utils from tensorflow.python.framework import constant_op from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops diff --git a/tensorflow/python/debug/cli/offline_analyzer.py b/tensorflow/python/debug/cli/offline_analyzer.py index 8edfeeb62e9da3..f04dc162830f04 100644 --- a/tensorflow/python/debug/cli/offline_analyzer.py +++ b/tensorflow/python/debug/cli/offline_analyzer.py @@ -22,8 +22,8 @@ # Google-internal import(s). -from tensorflow.python.debug import debug_data from tensorflow.python.debug.cli import analyzer_cli +from tensorflow.python.debug.lib import debug_data from tensorflow.python.platform import app diff --git a/tensorflow/python/debug/cli/stepper_cli.py b/tensorflow/python/debug/cli/stepper_cli.py index f803917daf6357..b3eb50c81a4650 100644 --- a/tensorflow/python/debug/cli/stepper_cli.py +++ b/tensorflow/python/debug/cli/stepper_cli.py @@ -22,11 +22,11 @@ import numpy as np # pylint: disable=unused-import from six.moves import xrange # pylint: disable=redefined-builtin -from tensorflow.python.debug import stepper from tensorflow.python.debug.cli import cli_shared from tensorflow.python.debug.cli import command_parser from tensorflow.python.debug.cli import debugger_cli_common from tensorflow.python.debug.cli import tensor_format +from tensorflow.python.debug.lib import stepper class NodeStepperCLI(object): diff --git a/tensorflow/python/debug/cli/stepper_cli_test.py b/tensorflow/python/debug/cli/stepper_cli_test.py index 78c56697aeb558..cb9bf7d5931a34 100644 --- a/tensorflow/python/debug/cli/stepper_cli_test.py +++ b/tensorflow/python/debug/cli/stepper_cli_test.py @@ -23,8 +23,8 @@ from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.python.client import session -from tensorflow.python.debug import stepper from tensorflow.python.debug.cli import stepper_cli +from tensorflow.python.debug.lib import stepper from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import test_util diff --git a/tensorflow/python/debug/lib/__init__.py b/tensorflow/python/debug/lib/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/tensorflow/python/debug/debug_data.py b/tensorflow/python/debug/lib/debug_data.py similarity index 100% rename from tensorflow/python/debug/debug_data.py rename to tensorflow/python/debug/lib/debug_data.py diff --git a/tensorflow/python/debug/debug_data_test.py b/tensorflow/python/debug/lib/debug_data_test.py similarity index 99% rename from tensorflow/python/debug/debug_data_test.py rename to tensorflow/python/debug/lib/debug_data_test.py index 753b76358b6aab..7ca222785b4e82 100644 --- a/tensorflow/python/debug/debug_data_test.py +++ b/tensorflow/python/debug/lib/debug_data_test.py @@ -23,7 +23,7 @@ import numpy as np -from tensorflow.python.debug import debug_data +from tensorflow.python.debug.lib import debug_data from tensorflow.python.framework import test_util from tensorflow.python.platform import googletest diff --git a/tensorflow/python/debug/debug_utils.py b/tensorflow/python/debug/lib/debug_utils.py similarity index 100% rename from tensorflow/python/debug/debug_utils.py rename to tensorflow/python/debug/lib/debug_utils.py diff --git a/tensorflow/python/debug/debug_utils_test.py b/tensorflow/python/debug/lib/debug_utils_test.py similarity index 99% rename from tensorflow/python/debug/debug_utils_test.py rename to tensorflow/python/debug/lib/debug_utils_test.py index 081c27a6c4f715..49b8711f103456 100644 --- a/tensorflow/python/debug/debug_utils_test.py +++ b/tensorflow/python/debug/lib/debug_utils_test.py @@ -21,7 +21,7 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session -from tensorflow.python.debug import debug_utils +from tensorflow.python.debug.lib import debug_utils from tensorflow.python.framework import constant_op from tensorflow.python.framework import test_util from tensorflow.python.ops import math_ops diff --git a/tensorflow/python/debug/session_debug_file_test.py b/tensorflow/python/debug/lib/session_debug_file_test.py similarity index 96% rename from tensorflow/python/debug/session_debug_file_test.py rename to tensorflow/python/debug/lib/session_debug_file_test.py index 8acd3975b70b3c..dd886828e07bfc 100644 --- a/tensorflow/python/debug/session_debug_file_test.py +++ b/tensorflow/python/debug/lib/session_debug_file_test.py @@ -21,9 +21,9 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session -from tensorflow.python.debug import debug_data -from tensorflow.python.debug import debug_utils -from tensorflow.python.debug import session_debug_testlib +from tensorflow.python.debug.lib import debug_data +from tensorflow.python.debug.lib import debug_utils +from tensorflow.python.debug.lib import session_debug_testlib from tensorflow.python.framework import constant_op from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables diff --git a/tensorflow/python/debug/session_debug_testlib.py b/tensorflow/python/debug/lib/session_debug_testlib.py similarity index 99% rename from tensorflow/python/debug/session_debug_testlib.py rename to tensorflow/python/debug/lib/session_debug_testlib.py index 2b700facd7d666..c9accafaf23c22 100644 --- a/tensorflow/python/debug/session_debug_testlib.py +++ b/tensorflow/python/debug/lib/session_debug_testlib.py @@ -27,8 +27,8 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session -from tensorflow.python.debug import debug_data -from tensorflow.python.debug import debug_utils +from tensorflow.python.debug.lib import debug_data +from tensorflow.python.debug.lib import debug_utils from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors diff --git a/tensorflow/python/debug/stepper.py b/tensorflow/python/debug/lib/stepper.py similarity index 99% rename from tensorflow/python/debug/stepper.py rename to tensorflow/python/debug/lib/stepper.py index fb5e97262733c8..887c2ddc10bad7 100644 --- a/tensorflow/python/debug/stepper.py +++ b/tensorflow/python/debug/lib/stepper.py @@ -22,7 +22,8 @@ import six from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.debug import debug_data +from tensorflow.python.debug.lib import debug_data +from tensorflow.python.debug.lib import debug_utils from tensorflow.python.framework import ops from tensorflow.python.ops import session_ops diff --git a/tensorflow/python/debug/stepper_test.py b/tensorflow/python/debug/lib/stepper_test.py similarity index 99% rename from tensorflow/python/debug/stepper_test.py rename to tensorflow/python/debug/lib/stepper_test.py index 20d79f3ade4661..79c6eb40a57776 100644 --- a/tensorflow/python/debug/stepper_test.py +++ b/tensorflow/python/debug/lib/stepper_test.py @@ -18,7 +18,7 @@ from __future__ import print_function from tensorflow.python.client import session -from tensorflow.python.debug.stepper import NodeStepper +from tensorflow.python.debug.lib.stepper import NodeStepper from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops diff --git a/tensorflow/python/debug/wrappers/dumping_wrapper.py b/tensorflow/python/debug/wrappers/dumping_wrapper.py index dc3f8468c61a21..9f1cde59c5737f 100644 --- a/tensorflow/python/debug/wrappers/dumping_wrapper.py +++ b/tensorflow/python/debug/wrappers/dumping_wrapper.py @@ -23,7 +23,7 @@ # Google-internal import(s). from tensorflow.core.util import event_pb2 -from tensorflow.python.debug import debug_data +from tensorflow.python.debug.lib import debug_data from tensorflow.python.debug.wrappers import framework from tensorflow.python.platform import gfile diff --git a/tensorflow/python/debug/wrappers/dumping_wrapper_test.py b/tensorflow/python/debug/wrappers/dumping_wrapper_test.py index 2be1077e28e48e..e4096b623c79d5 100644 --- a/tensorflow/python/debug/wrappers/dumping_wrapper_test.py +++ b/tensorflow/python/debug/wrappers/dumping_wrapper_test.py @@ -23,8 +23,8 @@ import tempfile from tensorflow.python.client import session -from tensorflow.python.debug import debug_data -from tensorflow.python.debug import stepper +from tensorflow.python.debug.lib import debug_data +from tensorflow.python.debug.lib import stepper from tensorflow.python.debug.wrappers import dumping_wrapper from tensorflow.python.debug.wrappers import hooks from tensorflow.python.framework import constant_op diff --git a/tensorflow/python/debug/wrappers/framework.py b/tensorflow/python/debug/wrappers/framework.py index 145008e9024675..550b897b31a9e9 100644 --- a/tensorflow/python/debug/wrappers/framework.py +++ b/tensorflow/python/debug/wrappers/framework.py @@ -115,8 +115,8 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session -from tensorflow.python.debug import debug_utils -from tensorflow.python.debug import stepper +from tensorflow.python.debug.lib import debug_utils +from tensorflow.python.debug.lib import stepper from tensorflow.python.framework import errors diff --git a/tensorflow/python/debug/wrappers/framework_test.py b/tensorflow/python/debug/wrappers/framework_test.py index d56c7057f66bbe..9e9a90ec5022ec 100644 --- a/tensorflow/python/debug/wrappers/framework_test.py +++ b/tensorflow/python/debug/wrappers/framework_test.py @@ -23,7 +23,7 @@ import numpy as np from tensorflow.python.client import session -from tensorflow.python.debug import debug_data +from tensorflow.python.debug.lib import debug_data from tensorflow.python.debug.wrappers import framework from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes diff --git a/tensorflow/python/debug/wrappers/hooks.py b/tensorflow/python/debug/wrappers/hooks.py index cda2becc6eba5c..850c487022d6f6 100644 --- a/tensorflow/python/debug/wrappers/hooks.py +++ b/tensorflow/python/debug/wrappers/hooks.py @@ -19,8 +19,8 @@ from __future__ import print_function from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.debug import debug_utils -from tensorflow.python.debug import stepper +from tensorflow.python.debug.lib import debug_utils +from tensorflow.python.debug.lib import stepper from tensorflow.python.debug.wrappers import dumping_wrapper from tensorflow.python.debug.wrappers import framework from tensorflow.python.debug.wrappers import local_cli_wrapper diff --git a/tensorflow/python/debug/wrappers/local_cli_wrapper.py b/tensorflow/python/debug/wrappers/local_cli_wrapper.py index e1f5904c213d0f..55acb697f12b54 100644 --- a/tensorflow/python/debug/wrappers/local_cli_wrapper.py +++ b/tensorflow/python/debug/wrappers/local_cli_wrapper.py @@ -24,12 +24,12 @@ import tempfile # Google-internal import(s). -from tensorflow.python.debug import debug_data from tensorflow.python.debug.cli import analyzer_cli from tensorflow.python.debug.cli import cli_shared from tensorflow.python.debug.cli import debugger_cli_common from tensorflow.python.debug.cli import stepper_cli from tensorflow.python.debug.cli import ui_factory +from tensorflow.python.debug.lib import debug_data from tensorflow.python.debug.wrappers import framework From fc4e26d69eb74f46b30c48d59cf6000fd2045560 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Tue, 14 Feb 2017 11:44:31 -0800 Subject: [PATCH 16/50] Remove generator.py changes from cherrypicks. --- tensorflow/tools/docs/BUILD | 98 +------ tensorflow/tools/docs/generate.py | 408 ------------------------------ 2 files changed, 4 insertions(+), 502 deletions(-) delete mode 100644 tensorflow/tools/docs/generate.py diff --git a/tensorflow/tools/docs/BUILD b/tensorflow/tools/docs/BUILD index a3829482d29c7f..529c4fa6db623d 100644 --- a/tensorflow/tools/docs/BUILD +++ b/tensorflow/tools/docs/BUILD @@ -16,100 +16,6 @@ py_binary( deps = ["//tensorflow:tensorflow_py"], ) -py_library( - name = "doc_generator_visitor", - srcs = [ - "doc_generator_visitor.py", - ], - srcs_version = "PY2AND3", -) - -py_test( - name = "doc_generator_visitor_test", - size = "small", - srcs = [ - "doc_generator_visitor_test.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":doc_generator_visitor", - "//tensorflow/python:platform_test", - ], -) - -py_library( - name = "parser", - srcs = [ - "parser.py", - ], - srcs_version = "PY2AND3", -) - -py_test( - name = "parser_test", - size = "small", - srcs = [ - "parser_test.py", - ], - srcs_version = "PY2AND3", - tags = ["manual"], - deps = [ - ":parser", - "//tensorflow/python:platform_test", - ], -) - -py_binary( - name = "generate", - srcs = ["generate.py"], - srcs_version = "PY2AND3", - deps = [ - ":doc_generator_visitor", - ":parser", - ":py_guide_parser", - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/ffmpeg:ffmpeg_ops_py", - "//tensorflow/python/debug:debug_py", - "//tensorflow/tools/common:public_api", - "//tensorflow/tools/common:traverse", - ], -) - -py_test( - name = "generate_test", - size = "small", - srcs = [ - "generate_test.py", - ], - srcs_version = "PY2AND3", - tags = ["manual"], - deps = [ - ":generate", - "//tensorflow/python:platform_test", - ], -) - -py_library( - name = "py_guide_parser", - srcs = [ - "py_guide_parser.py", - ], - srcs_version = "PY2AND3", -) - -py_test( - name = "py_guide_parser_test", - size = "small", - srcs = [ - "py_guide_parser_test.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":py_guide_parser", - "//tensorflow/python:client_testlib", - ], -) - filegroup( name = "doxy_config", srcs = ["tf-doxy_for_md-config"], @@ -136,6 +42,10 @@ sh_test( "//tensorflow/core:all_files", "//tensorflow/python:all_files", ], + tags = [ + "manual", + "notap", + ], ) filegroup( diff --git a/tensorflow/tools/docs/generate.py b/tensorflow/tools/docs/generate.py deleted file mode 100644 index 79acf05caa42f3..00000000000000 --- a/tensorflow/tools/docs/generate.py +++ /dev/null @@ -1,408 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Generate docs for the TensorFlow Python API.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import inspect -import os -import sys - -import six -import tensorflow as tf - -from tensorflow.python import debug as tf_debug - -from tensorflow.tools.common import public_api -from tensorflow.tools.common import traverse -from tensorflow.tools.docs import doc_generator_visitor -from tensorflow.tools.docs import parser -from tensorflow.tools.docs import py_guide_parser - - -def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree, - reverse_index, doc_index, guide_index): - """Write previously extracted docs to disk. - - Write a docs page for each symbol in `index` to a tree of docs at - `output_dir`. - - Symbols with multiple aliases will have only one page written about them, - which is referenced for all aliases. `duplicate_of` and `duplicates` are used - to determine which docs pages to write. - - Args: - output_dir: Directory to write documentation markdown files to. Will be - created if it doesn't exist. - base_dir: Base directory of the code being documented. This prefix is - stripped from all file paths that are part of the documentation. - duplicate_of: A `dict` mapping fully qualified names to "master" names. This - is used to resolve "@{symbol}" references to the "master" name. - duplicates: A `dict` mapping fully qualified names to a set of all - aliases of this name. This is used to automatically generate a list of all - aliases for each name. - index: A `dict` mapping fully qualified names to the corresponding Python - objects. Used to produce docs for child objects, and to check the validity - of "@{symbol}" references. - tree: A `dict` mapping a fully qualified name to the names of all its - members. Used to populate the members section of a class or module page. - reverse_index: A `dict` mapping object ids to fully qualified names. - doc_index: A `dict` mapping a doc key to a DocInfo. - guide_index: A `dict` mapping symbol name strings to GuideRef. - """ - # Make output_dir. - try: - if not os.path.exists(output_dir): - os.makedirs(output_dir) - except OSError as e: - print('Creating output dir "%s" failed: %s' % (output_dir, e)) - raise - - # Parse and write Markdown pages, resolving cross-links (@{symbol}). - for full_name, py_object in six.iteritems(index): - - if full_name in duplicate_of: - continue - - # Methods and some routines are documented only as part of their class. - if not (inspect.ismodule(py_object) or - inspect.isclass(py_object) or - inspect.isfunction(py_object)): - continue - - print('Writing docs for %s (%r).' % (full_name, py_object)) - - # Generate docs for `py_object`, resolving references. - markdown = parser.generate_markdown(full_name, py_object, - duplicate_of=duplicate_of, - duplicates=duplicates, - index=index, - tree=tree, - reverse_index=reverse_index, - doc_index=doc_index, - guide_index=guide_index, - base_dir=base_dir) - - # TODO(deannarubin): use _tree to generate sidebar information. - - path = os.path.join(output_dir, parser.documentation_path(full_name)) - directory = os.path.dirname(path) - try: - if not os.path.exists(directory): - os.makedirs(directory) - with open(path, 'w') as f: - f.write(markdown) - except OSError as e: - print('Cannot write documentation for %s to %s: %s' % (full_name, - directory, e)) - raise - # TODO(deannarubin): write sidebar file? - - # Write a global index containing all full names with links. - with open(os.path.join(output_dir, 'index.md'), 'w') as f: - f.write(parser.generate_global_index('TensorFlow', index, duplicate_of)) - - -def extract(): - """Extract docs from tf namespace and write them to disk.""" - visitor = doc_generator_visitor.DocGeneratorVisitor('tf') - api_visitor = public_api.PublicAPIVisitor(visitor) - - # Access something in contrib so tf.contrib is properly loaded (it's hidden - # behind lazy loading) - _ = tf.contrib.__name__ - - # Exclude some libaries in contrib from the documentation altogether. - # TODO(wicke): Shrink this list. - api_visitor.do_not_descend_map.update({ - 'contrib': [ - 'compiler', - 'factorization', - 'grid_rnn', - 'labeled_tensor', - 'ndlstm', - 'quantization', - 'session_bundle', - 'slim', - 'solvers', - 'specs', - 'tensor_forest', - 'tensorboard', - 'testing', - 'tfprof', - 'training', - ], - 'contrib.bayesflow': [ - 'entropy', 'monte_carlo', - 'special_math', 'stochastic_gradient_estimators', - 'stochastic_graph', 'stochastic_tensor', - 'stochastic_variables', 'variational_inference' - ], - 'contrib.distributions': ['bijector'], - 'contrib.ffmpeg': ['ffmpeg_ops'], - 'contrib.graph_editor': [ - 'edit', - 'match', - 'reroute', - 'subgraph', - 'transform', - 'select', - 'util' - ], - 'contrib.layers': [ - 'feature_column', - 'summaries' - ], - 'contrib.learn': [ - 'datasets', - 'head', - 'graph_actions', - 'io', - 'models', - 'monitors', - 'ops', - 'preprocessing', - 'utils', - ], - 'contrib.util': ['loader'], - }) - - traverse.traverse(tf, api_visitor) - - # tf_debug is not imported with tf, it's a separate module altogether - visitor.set_root_name('tfdbg') - traverse.traverse(tf_debug, api_visitor) - - return visitor - - -class GetMarkdownTitle(py_guide_parser.PyGuideParser): - """Extract the title from a .md file.""" - - def __init__(self): - self.title = None - py_guide_parser.PyGuideParser.__init__(self) - - def process_title(self, _, title): - if self.title is None: # only use the first title - self.title = title - - -class DocInfo(object): - """A simple struct for holding a doc's url and title.""" - - def __init__(self, url, title): - self.url = url - self.title = title - - -def build_doc_index(src_dir): - """Build an index from a keyword designating a doc to DocInfo objects.""" - doc_index = {} - for dirpath, _, filenames in os.walk(src_dir): - suffix = os.path.relpath(path=dirpath, start=src_dir) - for base_name in filenames: - if not base_name.endswith('.md'): continue - title_parser = GetMarkdownTitle() - title_parser.process(os.path.join(dirpath, base_name)) - key_parts = os.path.join(suffix, base_name[:-3]).split('/') - if key_parts[-1] == 'index': - key_parts = key_parts[:-1] - doc_info = DocInfo(os.path.join(suffix, base_name), title_parser.title) - doc_index[key_parts[-1]] = doc_info - if len(key_parts) > 1: - doc_index['/'.join(key_parts[-2:])] = doc_info - - return doc_index - - -class GuideRef(object): - - def __init__(self, base_name, title, section_title, section_tag): - self.url = 'api_guides/python/' + ( - ('%s#%s' % (base_name, section_tag)) if section_tag else base_name) - self.link_text = (('%s > %s' % (title, section_title)) - if section_title else title) - - def make_md_link(self, url_prefix): - return '[%s](%s%s)' % (self.link_text, url_prefix, self.url) - - -class GenerateGuideIndex(py_guide_parser.PyGuideParser): - """Turn guide files into an index from symbol name to a list of GuideRefs.""" - - def __init__(self): - self.index = {} - py_guide_parser.PyGuideParser.__init__(self) - - def process(self, full_path, base_name): - """Index a file, reading from `full_path`, with `base_name` as the link.""" - self.full_path = full_path - self.base_name = base_name - self.title = None - self.section_title = None - self.section_tag = None - py_guide_parser.PyGuideParser.process(self, full_path) - - def process_title(self, _, title): - if self.title is None: # only use the first title - self.title = title - - def process_section(self, _, section_title, tag): - self.section_title = section_title - self.section_tag = tag - - def process_line(self, _, line): - """Index @{symbol} references as in the current file & section.""" - for match in parser.SYMBOL_REFERENCE_RE.finditer(line): - val = self.index.get(match.group(1), []) - val.append(GuideRef( - self.base_name, self.title, self.section_title, self.section_tag)) - self.index[match.group(1)] = val - - -def build_guide_index(guide_src_dir): - """Return dict: symbol name -> GuideRef from the files in `guide_src_dir`.""" - index_generator = GenerateGuideIndex() - for full_path, base_name in py_guide_parser.md_files_in_dir(guide_src_dir): - index_generator.process(full_path, base_name) - return index_generator.index - - -def write(output_dir, base_dir, doc_index, guide_index, visitor): - """Write documentation for an index in a `DocGeneratorVisitor` to disk. - - This function will create `output_dir` if it doesn't exist, and write - the documentation contained in `visitor`. - - Args: - output_dir: The directory to write documentation to. Must not exist. - base_dir: The base dir of the library `visitor` has traversed. This is used - to compute relative paths for file references. - doc_index: A `dict` mapping a doc key to a DocInfo. - guide_index: A `dict` mapping symbol name strings to GuideRef. - visitor: A `DocGeneratorVisitor` that has traversed a library located at - `base_dir`. - """ - write_docs(output_dir, os.path.abspath(base_dir), - visitor.duplicate_of, visitor.duplicates, - visitor.index, visitor.tree, visitor.reverse_index, - doc_index, guide_index) - - -class UpdateTags(py_guide_parser.PyGuideParser): - """Rewrites a Python guide so that each section has an explicit tag.""" - - def process_section(self, line_number, section_title, tag): - self.replace_line(line_number, '

%s

' % (tag, section_title)) - - -def other_docs(src_dir, output_dir, visitor, doc_index): - """Convert all the files in `src_dir` and write results to `output_dir`.""" - header = '\n' - - # Iterate through all the source files and process them. - tag_updater = UpdateTags() - for dirpath, _, filenames in os.walk(src_dir): - # How to get from `dirpath` to api_docs/python/ - relative_path_to_root = os.path.relpath( - path=os.path.join(src_dir, 'api_docs/python'), start=dirpath) - - # Make the directory under output_dir. - new_dir = os.path.join(output_dir, - os.path.relpath(path=dirpath, start=src_dir)) - try: - if not os.path.exists(new_dir): - os.makedirs(new_dir) - except OSError as e: - print('Creating output dir "%s" failed: %s' % (new_dir, e)) - raise - - for base_name in filenames: - full_in_path = os.path.join(dirpath, base_name) - suffix = os.path.relpath(path=full_in_path, start=src_dir) - full_out_path = os.path.join(output_dir, suffix) - if not base_name.endswith('.md'): - print('Copying non-md file %s...' % suffix) - open(full_out_path, 'w').write(open(full_in_path).read()) - continue - if dirpath.endswith('/api_guides/python'): - print('Processing Python guide %s...' % base_name) - md_string = tag_updater.process(full_in_path) - else: - print('Processing doc %s...' % suffix) - md_string = open(full_in_path).read() - - output = parser.replace_references( - md_string, relative_path_to_root, visitor.duplicate_of, - doc_index=doc_index, index=visitor.index) - with open(full_out_path, 'w') as f: - f.write(header + output) - - print('Done.') - - -def _main(src_dir, output_dir, base_dir): - doc_index = build_doc_index(src_dir) - visitor = extract() - write(os.path.join(output_dir, 'api_docs/python'), base_dir, - doc_index, - build_guide_index(os.path.join(src_dir, 'api_guides/python')), - visitor) - other_docs(src_dir, output_dir, visitor, doc_index) - - -if __name__ == '__main__': - argument_parser = argparse.ArgumentParser() - argument_parser.add_argument( - '--output_dir', - type=str, - default=None, - required=True, - help='Directory to write docs to.' - ) - - argument_parser.add_argument( - '--src_dir', - type=str, - default=None, - required=True, - help='Directory with the source docs.' - ) - - # This doc generator works on the TensorFlow codebase. Since this script lives - # at tensorflow/tools/docs, and all code is defined somewhere inside - # tensorflow/, we can compute the base directory (two levels up), which is - # valid unless we're trying to apply this to a different code base, or are - # moving the script around. - script_dir = os.path.dirname(inspect.getfile(inspect.currentframe())) - default_base_dir = os.path.join(script_dir, '..', '..') - - argument_parser.add_argument( - '--base_dir', - type=str, - default=default_base_dir, - help=('Base directory to to strip from file names referenced in docs. ' - 'Defaults to two directories up from the location of this file.') - ) - - flags, _ = argument_parser.parse_known_args() - _main(flags.src_dir, flags.output_dir, flags.base_dir) - if parser.all_errors: - print('Errors during processing:' + '\n '.join(parser.all_errors)) - sys.exit(1) From 5a60bb88a05fe6c615476c029aa2afcce4d6e9d6 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Tue, 14 Feb 2017 13:42:11 -0800 Subject: [PATCH 17/50] Remove unused debug_utils import in stepper.py. --- tensorflow/python/debug/lib/stepper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/python/debug/lib/stepper.py b/tensorflow/python/debug/lib/stepper.py index 887c2ddc10bad7..47e42efbbbf8fd 100644 --- a/tensorflow/python/debug/lib/stepper.py +++ b/tensorflow/python/debug/lib/stepper.py @@ -23,7 +23,6 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.python.debug.lib import debug_data -from tensorflow.python.debug.lib import debug_utils from tensorflow.python.framework import ops from tensorflow.python.ops import session_ops From 04af67226360e2aa233189c72907a3c58f01ff4b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 13 Feb 2017 16:52:47 -0800 Subject: [PATCH 18/50] Module docstring updates. Change: 147412093 --- tensorflow/python/debug/__init__.py | 35 +---------- tensorflow/python/platform/test.py | 37 +----------- tensorflow/python/training/training.py | 84 ++++---------------------- 3 files changed, 14 insertions(+), 142 deletions(-) diff --git a/tensorflow/python/debug/__init__.py b/tensorflow/python/debug/__init__.py index c9c702089b0018..3a3318a9deb6cb 100644 --- a/tensorflow/python/debug/__init__.py +++ b/tensorflow/python/debug/__init__.py @@ -14,52 +14,19 @@ # ============================================================================== """Public Python API of TensorFlow Debugger (tfdbg). -## Functions for adding debug watches - -These functions help you modify `RunOptions` to specify which `Tensor`s are to -be watched when the TensorFlow graph is executed at runtime. +See the @{$python/tfdbg} guide. @@add_debug_tensor_watch @@watch_graph @@watch_graph_with_blacklists - - -## Classes for debug-dump data and directories - -These classes allow you to load and inspect tensor values dumped from -TensorFlow graphs during runtime. - @@DebugTensorDatum @@DebugDumpDir - - -## Functions for loading debug-dump data - @@load_tensor_from_event_file - - -## Tensor-value predicates - -Built-in tensor-filter predicates to support conditional breakpoint between -runs. See `DebugDumpDir.find()` for more details. - @@has_inf_or_nan - - -## Session wrapper class and `SessionRunHook` implementations - -These classes allow you to - -* wrap aroundTensorFlow `Session` objects to debug plain TensorFlow models - (see `DumpingDebugWrapperSession` and `LocalCLIDebugWrapperSession`), or -* generate `SessionRunHook` objects to debug `tf.contrib.learn` models (see - `DumpingDebugHook` and `LocalCLIDebugHook`). - @@DumpingDebugHook @@DumpingDebugWrapperSession @@LocalCLIDebugHook @@LocalCLIDebugWrapperSession - """ from __future__ import absolute_import diff --git a/tensorflow/python/platform/test.py b/tensorflow/python/platform/test.py index 3f2c1d97b4876d..18cf8f7b99c3e7 100644 --- a/tensorflow/python/platform/test.py +++ b/tensorflow/python/platform/test.py @@ -13,53 +13,20 @@ # limitations under the License. # ============================================================================== -# pylint: disable=g-short-docstring-punctuation -"""## Unit tests - -TensorFlow provides a convenience class inheriting from `unittest.TestCase` -which adds methods relevant to TensorFlow tests. Here is an example: - -```python - import tensorflow as tf - - - class SquareTest(tf.test.TestCase): - - def testSquare(self): - with self.test_session(): - x = tf.square([2, 3]) - self.assertAllEqual(x.eval(), [4, 9]) - - - if __name__ == '__main__': - tf.test.main() -``` - -`tf.test.TestCase` inherits from `unittest.TestCase` but adds a few additional -methods. We will document these methods soon. +"""Testing. See the @{$python/test} guide. @@main @@TestCase @@test_src_dir_path - -## Utilities - @@assert_equal_graph_def @@get_temp_dir @@is_built_with_cuda @@is_gpu_available @@gpu_device_name - -## Gradient checking - -[`compute_gradient`](#compute_gradient) and -[`compute_gradient_error`](#compute_gradient_error) perform numerical -differentiation of graphs for comparison against registered analytic gradients. - @@compute_gradient @@compute_gradient_error - """ + from __future__ import absolute_import from __future__ import division from __future__ import print_function diff --git a/tensorflow/python/training/training.py b/tensorflow/python/training/training.py index 3a2415629a9cfa..c3405bfc28310f 100644 --- a/tensorflow/python/training/training.py +++ b/tensorflow/python/training/training.py @@ -13,20 +13,9 @@ # limitations under the License. # ============================================================================== -# pylint: disable=line-too-long -"""This library provides a set of classes and functions that helps train models. - -## Optimizers - -The Optimizer base class provides methods to compute gradients for a loss and -apply gradients to variables. A collection of subclasses implement classic -optimization algorithms such as GradientDescent and Adagrad. - -You never instantiate the Optimizer class itself, but instead instantiate one -of the subclasses. +"""Support for training models. See the @{$python/train} guide. @@Optimizer - @@GradientDescentOptimizer @@AdadeltaOptimizer @@AdagradOptimizer @@ -37,67 +26,25 @@ @@ProximalGradientDescentOptimizer @@ProximalAdagradOptimizer @@RMSPropOptimizer - -## Gradient Computation - -TensorFlow provides functions to compute the derivatives for a given -TensorFlow computation graph, adding operations to the graph. The -optimizer classes automatically compute derivatives on your graph, but -creators of new Optimizers or expert users can call the lower-level -functions below. - @@gradients @@AggregationMethod - @@stop_gradient - @@hessians - - -## Gradient Clipping - -TensorFlow provides several operations that you can use to add clipping -functions to your graph. You can use these functions to perform general data -clipping, but they're particularly useful for handling exploding or vanishing -gradients. - @@clip_by_value @@clip_by_norm @@clip_by_average_norm @@clip_by_global_norm @@global_norm - -## Decaying the learning rate @@exponential_decay @@inverse_time_decay @@natural_exp_decay @@piecewise_constant @@polynomial_decay - -## Moving Averages - -Some training algorithms, such as GradientDescent and Momentum often benefit -from maintaining a moving average of variables during optimization. Using the -moving averages for evaluations often improve results significantly. - @@ExponentialMovingAverage - -## Coordinator and QueueRunner - -See [Threading and Queues](../../how_tos/threading_and_queues/index.md) -for how to use threads and queues. For documentation on the Queue API, -see [Queues](../../api_docs/python/io_ops.md#queues). - @@Coordinator @@QueueRunner @@add_queue_runner @@start_queue_runners - -## Distributed execution - -See [Distributed TensorFlow](../../how_tos/distributed/index.md) for -more information about how to configure a distributed TensorFlow program. - @@Server @@Supervisor @@SessionManager @@ -110,23 +57,11 @@ @@SessionCreator @@ChiefSessionCreator @@WorkerSessionCreator - -## Reading Summaries from Event Files - -See [Summaries and -TensorBoard](../../how_tos/summaries_and_tensorboard/index.md) for an -overview of summaries, event files, and visualization in TensorBoard. - @@summary_iterator - -## Training Utilities - -@@global_step -@@basic_train_loop -@@get_global_step -@@assert_global_step -@@write_graph @@SessionRunHook +@@SessionRunArgs +@@SessionRunContext +@@SessionRunValues @@LoggingTensorHook @@StopAtStepHook @@CheckpointSaverHook @@ -136,12 +71,15 @@ @@NanTensorHook @@SummarySaverHook @@GlobalStepWaiterHook -@@SessionRunArgs -@@SessionRunContext -@@SessionRunValues @@LooperThread +@@FinalOpsHook +@@FeedFnHook +@@global_step +@@basic_train_loop +@@get_global_step +@@assert_global_step +@@write_graph """ -# pylint: enable=line-too-long # Optimizers. from __future__ import absolute_import From 40ec3db0bf1033ca435aa5b488868b29235fcf7c Mon Sep 17 00:00:00 2001 From: Patrick Nguyen Date: Tue, 14 Feb 2017 11:35:14 -0800 Subject: [PATCH 19/50] Fix links according to new convention. Change: 147499520 --- tensorflow/python/client/session.py | 34 +++++----- .../python/debug/wrappers/dumping_wrapper.py | 2 +- tensorflow/python/framework/errors_impl.py | 34 +++++----- .../python/framework/gen_docs_combined.py | 2 +- tensorflow/python/framework/importer.py | 7 ++- tensorflow/python/framework/ops.py | 62 +++++++++---------- tensorflow/python/framework/sparse_tensor.py | 2 +- tensorflow/python/framework/tensor_shape.py | 5 +- tensorflow/python/layers/core.py | 4 +- tensorflow/python/ops/control_flow_ops.py | 2 +- tensorflow/python/ops/data_flow_ops.py | 32 +++++----- tensorflow/python/ops/embedding_ops.py | 2 +- tensorflow/python/ops/image_ops_impl.py | 28 ++++----- tensorflow/python/ops/init_ops.py | 16 ++--- tensorflow/python/ops/nn_impl.py | 8 +-- tensorflow/python/ops/nn_ops.py | 16 ++--- tensorflow/python/ops/random_ops.py | 57 ++++++++++++++--- tensorflow/python/ops/template.py | 2 +- tensorflow/python/ops/variable_scope.py | 4 +- tensorflow/python/ops/variables.py | 14 ++--- tensorflow/python/summary/summary_iterator.py | 4 +- tensorflow/python/summary/writer/writer.py | 4 +- tensorflow/python/training/input.py | 2 +- tensorflow/python/training/moving_averages.py | 2 +- tensorflow/python/training/saver.py | 2 +- tensorflow/python/training/server_lib.py | 10 +-- tensorflow/python/training/supervisor.py | 2 +- 27 files changed, 198 insertions(+), 161 deletions(-) diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py index 3f70b6240f2067..808f15e7077c43 100644 --- a/tensorflow/python/client/session.py +++ b/tensorflow/python/client/session.py @@ -605,8 +605,8 @@ def as_default(self): """Returns a context manager that makes this object the default session. Use with the `with` keyword to specify that calls to - [`Operation.run()`](../../api_docs/python/framework.md#Operation.run) or - [`Tensor.eval()`](../../api_docs/python/framework.md#Tensor.eval) should be + @{tf.Operation.run} or + @{tf.Tensor.eval} should be executed in this session. ```python @@ -619,7 +619,7 @@ def as_default(self): ``` To get the current default session, use - [`tf.get_default_session()`](#get_default_session). + @{tf.get_default_session}. *N.B.* The `as_default` context manager *does not* close the @@ -665,14 +665,14 @@ def run(self, fetches, feed_dict=None, options=None, run_metadata=None): nested list, tuple, namedtuple, dict, or OrderedDict containing graph elements at its leaves. A graph element can be one of the following types: - * An [`Operation`](../../api_docs/python/framework.md#Operation). + * An @{tf.Operation}. The corresponding fetched value will be `None`. - * A [`Tensor`](../../api_docs/python/framework.md#Tensor). + * A @{tf.Tensor}. The corresponding fetched value will be a numpy ndarray containing the value of that tensor. - * A [`SparseTensor`](../../api_docs/python/sparse_ops.md#SparseTensor). + * A @{tf.SparseTensor}. The corresponding fetched value will be a - [`SparseTensorValue`](../../api_docs/python/sparse_ops.md#SparseTensorValue) + @{tf.SparseTensorValue} containing the value of that sparse tensor. * A `get_tensor_handle` op. The corresponding fetched value will be a numpy ndarray containing the handle of that tensor. @@ -708,16 +708,16 @@ def run(self, fetches, feed_dict=None, options=None, run_metadata=None): the value of tensors in the graph. Each key in `feed_dict` can be one of the following types: - * If the key is a [`Tensor`](../../api_docs/python/framework.md#Tensor), the + * If the key is a @{tf.Tensor}, the value may be a Python scalar, string, list, or numpy ndarray that can be converted to the same `dtype` as that tensor. Additionally, if the key is a - [placeholder](../../api_docs/python/io_ops.md#placeholder), the shape of + @{tf.placeholder}, the shape of the value will be checked for compatibility with the placeholder. * If the key is a - [`SparseTensor`](../../api_docs/python/sparse_ops.md#SparseTensor), + @{tf.SparseTensor}, the value should be a - [`SparseTensorValue`](../../api_docs/python/sparse_ops.md#SparseTensorValue). + @{tf.SparseTensorValue}. * If the key is a nested tuple of `Tensor`s or `SparseTensor`s, the value should be a nested tuple with the same structure that maps to their corresponding values as above. @@ -1120,10 +1120,10 @@ class Session(BaseSession): ``` A session may own resources, such as - [variables](../../api_docs/python/state_ops.md#Variable), [queues](../../api_docs/python/io_ops.md#QueueBase), - and [readers](../../api_docs/python/io_ops.md#ReaderBase). It is important to release + @{tf.Variable}, @{tf.QueueBase}, + and @{tf.ReaderBase}. It is important to release these resources when they are no longer required. To do this, either - invoke the [`close()`](#Session.close) method on the session, or use + invoke the @{tf.Session.close} method on the session, or use the session as a context manager. The following two examples are equivalent: @@ -1166,7 +1166,7 @@ def __init__(self, target='', graph=None, config=None): Args: target: (Optional.) The execution engine to connect to. Defaults to using an in-process engine. See - [Distributed Tensorflow](https://www.tensorflow.org/how_tos/distributed/index.html) + @{$distributed$Distributed TensorFlow} for more examples. graph: (Optional.) The `Graph` to be launched (described above). config: (Optional.) A [`ConfigProto`](https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto) @@ -1242,8 +1242,8 @@ class InteractiveSession(BaseSession): The only difference with a regular `Session` is that an `InteractiveSession` installs itself as the default session on construction. - The methods [`Tensor.eval()`](../../api_docs/python/framework.md#Tensor.eval) - and [`Operation.run()`](../../api_docs/python/framework.md#Operation.run) + The methods @{tf.Tensor.eval} + and @{tf.Operation.run} will use that session to run ops. This is convenient in interactive shells and [IPython diff --git a/tensorflow/python/debug/wrappers/dumping_wrapper.py b/tensorflow/python/debug/wrappers/dumping_wrapper.py index 9f1cde59c5737f..fdde52183f4442 100644 --- a/tensorflow/python/debug/wrappers/dumping_wrapper.py +++ b/tensorflow/python/debug/wrappers/dumping_wrapper.py @@ -39,7 +39,7 @@ def __init__(self, sess, session_root, watch_fn=None, log_usage=True): session_root: (`str`) Path to the session root directory. Must be a directory that does not exist or an empty directory. If the directory does not exist, it will be created by the debugger core during debug - [`Session.run()`](../../../g3doc/api_docs/python/client.md#session.run) + @{tf.Session.run} calls. As the `run()` calls occur, subdirectories will be added to `session_root`. The subdirectories' names has the following pattern: diff --git a/tensorflow/python/framework/errors_impl.py b/tensorflow/python/framework/errors_impl.py index e73bdefb21adf7..04a6e4d7fbfb1f 100644 --- a/tensorflow/python/framework/errors_impl.py +++ b/tensorflow/python/framework/errors_impl.py @@ -61,9 +61,9 @@ def op(self): *N.B.* If the failed op was synthesized at runtime, e.g. a `Send` or `Recv` op, there will be no corresponding - [`Operation`](../../api_docs/python/framework.md#Operation) + @{tf.Operation} object. In that case, this will return `None`, and you should - instead use the [`OpError.node_def`](#OpError.node_def) to + instead use the @{tf.OpError.node_def} to discover information about the op. Returns: @@ -156,10 +156,10 @@ class CancelledError(OpError): """Raised when an operation or step is cancelled. For example, a long-running operation (e.g. - [`queue.enqueue()`](../../api_docs/python/io_ops.md#QueueBase.enqueue) may be + @{tf.QueueBase.enqueue} may be cancelled by running another operation (e.g. - [`queue.close(cancel_pending_enqueues=True)`](../../api_docs/python/io_ops.md#QueueBase.close), - or by [closing the session](../../api_docs/python/client.md#Session.close). + @{tf.QueueBase.close}, + or by @{tf.Session.close}. A step that is running such a long-running operation will fail by raising `CancelledError`. @@ -194,9 +194,9 @@ class InvalidArgumentError(OpError): This may occur, for example, if an operation is receives an input tensor that has an invalid value or shape. For example, the - [`tf.matmul()`](../../api_docs/python/math_ops.md#matmul) op will raise this + @{tf.matmul} op will raise this error if it receives an input that is not a matrix, and the - [`tf.reshape()`](../../api_docs/python/array_ops.md#reshape) op will raise + @{tf.reshape} op will raise this error if the new shape does not match the number of elements in the input tensor. @@ -227,7 +227,7 @@ class NotFoundError(OpError): """Raised when a requested entity (e.g., a file or directory) was not found. For example, running the - [`tf.WholeFileReader.read()`](../../api_docs/python/io_ops.md#WholeFileReader) + @{tf.WholeFileReader.read} operation could raise `NotFoundError` if it receives the name of a file that does not exist. @@ -243,7 +243,7 @@ class AlreadyExistsError(OpError): """Raised when an entity that we attempted to create already exists. For example, running an operation that saves a file - (e.g. [`tf.train.Saver.save()`](../../api_docs/python/train.md#Saver.save)) + (e.g. @{tf.train.Saver.save}) could potentially raise this exception if an explicit filename for an existing file was passed. @@ -260,7 +260,7 @@ class PermissionDeniedError(OpError): """Raised when the caller does not have permission to run an operation. For example, running the - [`tf.WholeFileReader.read()`](../../api_docs/python/io_ops.md#WholeFileReader) + @{tf.WholeFileReader.read} operation could raise `PermissionDeniedError` if it receives the name of a file for which the user does not have the read file permission. @@ -306,7 +306,7 @@ class FailedPreconditionError(OpError): """Operation was rejected because the system is not in a state to execute it. This exception is most commonly raised when running an operation - that reads a [`tf.Variable`](../../api_docs/python/state_ops.md#Variable) + that reads a @{tf.Variable} before it has been initialized. @@__init__ @@ -322,9 +322,9 @@ class AbortedError(OpError): """The operation was aborted, typically due to a concurrent action. For example, running a - [`queue.enqueue()`](../../api_docs/python/io_ops.md#QueueBase.enqueue) + @{tf.QueueBase.enqueue} operation may raise `AbortedError` if a - [`queue.close()`](../../api_docs/python/io_ops.md#QueueBase.close) operation + @{tf.QueueBase.close} operation previously ran. @@__init__ @@ -339,9 +339,9 @@ class OutOfRangeError(OpError): """Raised when an operation iterates past the valid input range. This exception is raised in "end-of-file" conditions, such as when a - [`queue.dequeue()`](../../api_docs/python/io_ops.md#QueueBase.dequeue) + @{tf.QueueBase.dequeue} operation is blocked on an empty queue, and a - [`queue.close()`](../../api_docs/python/io_ops.md#QueueBase.close) + @{tf.QueueBase.close} operation executes. @@__init__ @@ -358,7 +358,7 @@ class UnimplementedError(OpError): Some operations may raise this error when passed otherwise-valid arguments that it does not currently support. For example, running - the [`tf.nn.max_pool()`](../../api_docs/python/nn.md#max_pool) operation + the @{tf.nn.max_pool} operation would raise this error if pooling was requested on the batch dimension, because this is not yet supported. @@ -403,7 +403,7 @@ class DataLossError(OpError): """Raised when unrecoverable data loss or corruption is encountered. For example, this may be raised by running a - [`tf.WholeFileReader.read()`](../../api_docs/python/io_ops.md#WholeFileReader) + @{tf.WholeFileReader.read} operation, if the file is truncated while it is being read. @@__init__ diff --git a/tensorflow/python/framework/gen_docs_combined.py b/tensorflow/python/framework/gen_docs_combined.py index fb0a4dda3d2eb8..65379dda209225 100644 --- a/tensorflow/python/framework/gen_docs_combined.py +++ b/tensorflow/python/framework/gen_docs_combined.py @@ -37,7 +37,7 @@ PREFIX_TEXT = """ Note: Functions taking `Tensor` arguments can also take anything accepted by -[`tf.convert_to_tensor`](framework.md#convert_to_tensor). +@{tf.convert_to_tensor}. """ diff --git a/tensorflow/python/framework/importer.py b/tensorflow/python/framework/importer.py index 77d0822431586f..cddcb8590a272e 100644 --- a/tensorflow/python/framework/importer.py +++ b/tensorflow/python/framework/importer.py @@ -156,9 +156,10 @@ def import_graph_def(graph_def, input_map=None, return_elements=None, This function provides a way to import a serialized TensorFlow [`GraphDef`](https://www.tensorflow.org/code/tensorflow/core/framework/graph.proto) protocol buffer, and extract individual objects in the `GraphDef` as - [`Tensor`](#Tensor) and [`Operation`](#Operation) objects. See - [`Graph.as_graph_def()`](#Graph.as_graph_def) for a way to create a - `GraphDef` proto. + @{tf.Tensor} and @{tf.Operation} objects. Once extracted, + these objects are placed into the current default `Graph`. See + @{tf.Graph.as_graph_def} for a way to create a `GraphDef` + proto. Args: graph_def: A `GraphDef` proto containing operations to be imported into diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 065b8977f4b6d4..4b734fd146c86d 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -199,7 +199,7 @@ class Tensor(_TensorLike): A `Tensor` is a symbolic handle to one of the outputs of an `Operation`. It does not hold the values of that operation's output, but instead provides a means of computing those values in a - TensorFlow [`Session`](../../api_docs/python/client.md#Session). + TensorFlow @{tf.Session}. This class has two primary purposes: @@ -210,7 +210,7 @@ class Tensor(_TensorLike): 2. After the graph has been launched in a session, the value of the `Tensor` can be computed by passing it to - [`Session.run()`](../../api_docs/python/client.md#Session.run). + @{tf.Session.run}. `t.eval()` is a shortcut for calling `tf.get_default_session().run(t)`. @@ -327,7 +327,7 @@ def shape(self): The shape is computed using shape inference functions that are registered in the Op for each `Operation`. See - [`TensorShape`](../../api_docs/python/framework.md#TensorShape) + @{tf.TensorShape} for more details of what a shape represents. The inferred shape of a tensor is used to provide shape @@ -555,7 +555,7 @@ def eval(self, feed_dict=None, session=None): Args: feed_dict: A dictionary that maps `Tensor` objects to feed values. - See [`Session.run()`](../../api_docs/python/client.md#Session.run) for a + See @{tf.Session.run} for a description of the valid feed values. session: (Optional.) The `Session` to be used to evaluate this tensor. If none, the default session will be used. @@ -1002,10 +1002,10 @@ class IndexedSlices(_TensorLike): The `IndexedSlices` class is used principally in the definition of gradients for operations that have sparse gradients - (e.g. [`tf.gather`](../../api_docs/python/array_ops.md#gather)). + (e.g. @{tf.gather}). Contrast this representation with - [`SparseTensor`](../../api_docs/python/sparse_ops.md#SparseTensor), + @{tf.SparseTensor}, which uses multi-dimensional indices and scalar values. """ @@ -1119,8 +1119,8 @@ class Operation(object): more `Tensor` objects as input, and produces zero or more `Tensor` objects as output. Objects of type `Operation` are created by calling a Python op constructor (such as - [`tf.matmul()`](../../api_docs/python/math_ops.md#matmul)) - or [`Graph.create_op()`](../../api_docs/python/framework.md#Graph.create_op). + @{tf.matmul}) + or @{tf.Graph.create_op}. For example `c = tf.matmul(a, b)` creates an `Operation` of type "MatMul" that takes tensors `a` and `b` as input, and produces `c` @@ -1128,7 +1128,7 @@ class Operation(object): After the graph has been launched in a session, an `Operation` can be executed by passing it to - [`Session.run()`](../../api_docs/python/client.md#Session.run). + @{tf.Session.run}. `op.run()` is a shortcut for calling `tf.get_default_session().run(op)`. """ @@ -1542,7 +1542,7 @@ def run(self, feed_dict=None, session=None): Args: feed_dict: A dictionary that maps `Tensor` objects to feed values. - See [`Session.run()`](../../api_docs/python/client.md#Session.run) + See @{tf.Session.run} for a description of the valid feed values. session: (Optional.) The `Session` to be used to run to this operation. If none, the default session will be used. @@ -1892,13 +1892,13 @@ class Graph(object): """A TensorFlow computation, represented as a dataflow graph. A `Graph` contains a set of - [`Operation`](../../api_docs/python/framework.md#Operation) objects, + @{tf.Operation} objects, which represent units of computation; and - [`Tensor`](../../api_docs/python/framework.md#Tensor) objects, which represent + @{tf.Tensor} objects, which represent the units of data that flow between operations. A default `Graph` is always registered, and accessible by calling - [`tf.get_default_graph()`](../../api_docs/python/framework.md#get_default_graph). + @{tf.get_default_graph}. To add an operation to the default graph, simply call one of the functions that defines a new `Operation`: @@ -1908,7 +1908,7 @@ class Graph(object): ``` Another typical usage involves the - [`Graph.as_default()`](../../api_docs/python/framework.md#Graph.as_default) + @{tf.Graph.as_default} context manager, which overrides the current default graph for the lifetime of the context: @@ -1939,7 +1939,7 @@ class Graph(object): that are identified by name. For convenience when building a large graph, collections can store groups of related objects: for example, the `tf.Variable` uses a collection (named - [`tf.GraphKeys.GLOBAL_VARIABLES`](../../api_docs/python/framework.md#GraphKeys)) for + @{tf.GraphKeys.GLOBAL_VARIABLES}) for all variables that are created during the construction of a graph. The caller may define additional collections by specifying a new name. @@ -2069,7 +2069,7 @@ def version(self): """Returns a version number that increases as ops are added to the graph. Note that this is unrelated to the - [GraphDef version](#Graph.graph_def_version). + @{tf.Graph.graph_def_versions}. """ if self._finalized: return self._version @@ -2109,7 +2109,7 @@ def finalize(self): After calling `g.finalize()`, no new operations can be added to `g`. This method is used to ensure that no operations are added to a graph when it is shared between multiple threads, for example - when using a [`QueueRunner`](../../api_docs/python/train.md#QueueRunner). + when using a @{tf.train.QueueRunner}. """ self._finalized = True @@ -2143,7 +2143,7 @@ def _as_graph_def(self, from_version=None, add_shapes=False): """Returns a serialized `GraphDef` representation of this graph. The serialized `GraphDef` can be imported into another `Graph` - (using [`import_graph_def()`](#import_graph_def)) or used with the + (using @{tf.import_graph_def}) or used with the [C++ Session API](../../api_docs/cc/index.md). This method is thread-safe. @@ -2197,7 +2197,7 @@ def as_graph_def(self, from_version=None, add_shapes=False): """Returns a serialized `GraphDef` representation of this graph. The serialized `GraphDef` can be imported into another `Graph` - (using [`import_graph_def()`](#import_graph_def)) or used with the + (using @{tf.import_graph_def}) or used with the [C++ Session API](../../api_docs/cc/index.md). This method is thread-safe. @@ -3563,7 +3563,7 @@ def device(device_name_or_function): """Wrapper for `Graph.device()` using the default graph. See - [`Graph.device()`](../../api_docs/python/framework.md#Graph.device) + @{tf.Graph.device} for more details. Args: @@ -3597,7 +3597,7 @@ def colocate_with(op, ignore_existing=False): def control_dependencies(control_inputs): """Wrapper for `Graph.control_dependencies()` using the default graph. - See [`Graph.control_dependencies()`](../../api_docs/python/framework.md#Graph.control_dependencies) + See @{tf.Graph.control_dependencies} for more details. Args: @@ -3950,7 +3950,7 @@ class GraphKeys(object): * `GLOBAL_VARIABLES`: the default collection of `Variable` objects, shared across distributed environment (model variables are subset of these). See - [`tf.global_variables()`](../../api_docs/python/state_ops.md#global_variables) + @{tf.global_variables} for more details. Commonly, all `TRAINABLE_VARIABLES` variables will be in `MODEL_VARIABLES`, and all `MODEL_VARIABLES` variables will be in `GLOBAL_VARIABLES`. @@ -3962,19 +3962,19 @@ class GraphKeys(object): `tf.contrib.framework.model_variable` to add to this collection. * `TRAINABLE_VARIABLES`: the subset of `Variable` objects that will be trained by an optimizer. See - [`tf.trainable_variables()`](../../api_docs/python/state_ops.md#trainable_variables) + @{tf.trainable_variables} for more details. * `SUMMARIES`: the summary `Tensor` objects that have been created in the graph. See - [`tf.summary.merge_all()`](../../api_docs/python/summary.md#merge_all) + @{tf.summary.merge_all} for more details. * `QUEUE_RUNNERS`: the `QueueRunner` objects that are used to produce input for a computation. See - [`tf.start_queue_runners()`](../../api_docs/python/train.md#start_queue_runners) + @{tf.train.start_queue_runners} for more details. * `MOVING_AVERAGE_VARIABLES`: the subset of `Variable` objects that will also keep moving averages. See - [`tf.moving_average_variables()`](../../api_docs/python/state_ops.md#moving_average_variables) + @{tf.moving_average_variables} for more details. * `REGULARIZATION_LOSSES`: regularization losses collected during graph construction. @@ -4060,7 +4060,7 @@ def VARIABLES(cls): # pylint: disable=no-self-argument def add_to_collection(name, value): """Wrapper for `Graph.add_to_collection()` using the default graph. - See [`Graph.add_to_collection()`](../../api_docs/python/framework.md#Graph.add_to_collection) + See @{tf.Graph.add_to_collection} for more details. Args: @@ -4074,7 +4074,7 @@ def add_to_collection(name, value): def add_to_collections(names, value): """Wrapper for `Graph.add_to_collections()` using the default graph. - See [`Graph.add_to_collections()`](../../api_docs/python/framework.md#Graph.add_to_collections) + See @{tf.Graph.add_to_collections} for more details. Args: @@ -4088,7 +4088,7 @@ def add_to_collections(names, value): def get_collection_ref(key): """Wrapper for `Graph.get_collection_ref()` using the default graph. - See [`Graph.get_collection_ref()`](../../api_docs/python/framework.md#Graph.get_collection_ref) + See @{tf.Graph.get_collection_ref} for more details. Args: @@ -4107,7 +4107,7 @@ def get_collection_ref(key): def get_collection(key, scope=None): """Wrapper for `Graph.get_collection()` using the default graph. - See [`Graph.get_collection()`](../../api_docs/python/framework.md#Graph.get_collection) + See @{tf.Graph.get_collection} for more details. Args: @@ -4141,7 +4141,7 @@ def name_scope(name, default_name=None, values=None): This context manager validates that the given `values` are from the same graph, makes that graph the default graph, and pushes a name scope in that graph (see - [`Graph.name_scope()`](../../api_docs/python/framework.md#Graph.name_scope) + @{tf.Graph.name_scope} for more details on that). For example, to define a new Python op called `my_op`: diff --git a/tensorflow/python/framework/sparse_tensor.py b/tensorflow/python/framework/sparse_tensor.py index 4f65e433a562ab..10f5579ae599bc 100644 --- a/tensorflow/python/framework/sparse_tensor.py +++ b/tensorflow/python/framework/sparse_tensor.py @@ -203,7 +203,7 @@ def eval(self, feed_dict=None, session=None): Args: feed_dict: A dictionary that maps `Tensor` objects to feed values. - See [`Session.run()`](../../api_docs/python/client.md#Session.run) for a + See @{tf.Session.run} for a description of the valid feed values. session: (Optional.) The `Session` to be used to evaluate this sparse tensor. If none, the default session will be used. diff --git a/tensorflow/python/framework/tensor_shape.py b/tensorflow/python/framework/tensor_shape.py index 5fa3310bb6195d..3664710caa331a 100644 --- a/tensorflow/python/framework/tensor_shape.py +++ b/tensorflow/python/framework/tensor_shape.py @@ -393,11 +393,10 @@ class TensorShape(object): If a tensor is produced by an operation of type `"Foo"`, its shape may be inferred if there is a registered shape function for - `"Foo"`. See [`Shape functions in - C++`](../../how_tos/adding_an_op/index.md#shape-functions-in-c) for + `"Foo"`. See @{$adding_an_op#shape-functions-in-c$`Shape functions in C++`} for details of shape functions and how to register them. Alternatively, the shape may be set explicitly using - [`Tensor.set_shape()`](../../api_docs/python/framework.md#Tensor.set_shape). + @{tf.Tensor.set_shape}. """ def __init__(self, dims): diff --git a/tensorflow/python/layers/core.py b/tensorflow/python/layers/core.py index 92894e14472c0e..254ebb73bb75f6 100644 --- a/tensorflow/python/layers/core.py +++ b/tensorflow/python/layers/core.py @@ -234,7 +234,7 @@ class Dropout(base._Layer): # pylint: disable=protected-access to be the same for all timesteps, you can use `noise_shape=[batch_size, 1, features]`. seed: A Python integer. Used to create random seeds. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. name: The name of the layer (string). """ @@ -283,7 +283,7 @@ def dropout(inputs, to be the same for all timesteps, you can use `noise_shape=[batch_size, 1, features]`. seed: A Python integer. Used to create random seeds. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. training: Either a Python boolean, or a TensorFlow boolean scalar tensor (e.g. a placeholder). Whether to return the output in training mode diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py index e7c34e482ce298..9a1b38dd28d785 100644 --- a/tensorflow/python/ops/control_flow_ops.py +++ b/tensorflow/python/ops/control_flow_ops.py @@ -2503,7 +2503,7 @@ def while_loop(cond, body, loop_vars, shape_invariants=None, `loop_vars` is the same in every iteration. The `shape_invariants` argument allows the caller to specify a less specific shape invariant for each loop variable, which is needed if the shape varies between iterations. The - [`Tensor.set_shape()`](../../api_docs/python/framework.md#Tensor.set_shape) + @{tf.Tensor.set_shape} function may also be used in the `body` function to indicate that the output loop variable has a particular shape. The shape invariant for SparseTensor and IndexedSlices are treated specially as follows: diff --git a/tensorflow/python/ops/data_flow_ops.py b/tensorflow/python/ops/data_flow_ops.py index a963a176f0d094..3730b79eaa245f 100644 --- a/tensorflow/python/ops/data_flow_ops.py +++ b/tensorflow/python/ops/data_flow_ops.py @@ -119,8 +119,8 @@ class QueueBase(object): handle single elements, versions that support enqueuing and dequeuing a batch of elements at once. - See [`tf.FIFOQueue`](#FIFOQueue) and - [`tf.RandomShuffleQueue`](#RandomShuffleQueue) for concrete + See @{tf.FIFOQueue} and + @{tf.RandomShuffleQueue} for concrete implementations of this class, and instructions on how to create them. """ @@ -292,12 +292,12 @@ def enqueue(self, vals, name=None): until the element has been enqueued. At runtime, this operation may raise an error if the queue is - [closed](#QueueBase.close) before or during its execution. If the + @{tf.QueueBase.close} before or during its execution. If the queue is closed before this operation runs, `tf.errors.CancelledError` will be raised. If this operation is blocked, and either (i) the queue is closed by a close operation with `cancel_pending_enqueues=True`, or (ii) the session is - [closed](../../api_docs/python/client.md#Session.close), + @{tf.Session.close}, `tf.errors.CancelledError` will be raised. Args: @@ -335,12 +335,12 @@ def enqueue_many(self, vals, name=None): until all of the elements have been enqueued. At runtime, this operation may raise an error if the queue is - [closed](#QueueBase.close) before or during its execution. If the + @{tf.QueueBase.close} before or during its execution. If the queue is closed before this operation runs, `tf.errors.CancelledError` will be raised. If this operation is blocked, and either (i) the queue is closed by a close operation with `cancel_pending_enqueues=True`, or (ii) the session is - [closed](../../api_docs/python/client.md#Session.close), + @{tf.Session.close}, `tf.errors.CancelledError` will be raised. Args: @@ -396,11 +396,11 @@ def dequeue(self, name=None): until there is an element to dequeue. At runtime, this operation may raise an error if the queue is - [closed](#QueueBase.close) before or during its execution. If the + @{tf.QueueBase.close} before or during its execution. If the queue is closed, the queue is empty, and there are no pending enqueue operations that can fulfill this request, `tf.errors.OutOfRangeError` will be raised. If the session is - [closed](../../api_docs/python/client.md#Session.close), + @{tf.Session.close}, `tf.errors.CancelledError` will be raised. Args: @@ -437,11 +437,11 @@ def dequeue_many(self, n, name=None): `OutOfRange` exception is raised. At runtime, this operation may raise an error if the queue is - [closed](#QueueBase.close) before or during its execution. If the + @{tf.QueueBase.close} before or during its execution. If the queue is closed, the queue contains fewer than `n` elements, and there are no pending enqueue operations that can fulfill this request, `tf.errors.OutOfRangeError` will be raised. If the - session is [closed](../../api_docs/python/client.md#Session.close), + session is @{tf.Session.close}, `tf.errors.CancelledError` will be raised. Args: @@ -479,7 +479,7 @@ def dequeue_up_to(self, n, name=None): If the queue is closed and there are more than `0` but fewer than `n` elements remaining, then instead of raising a - `tf.errors.OutOfRangeError` like [`dequeue_many`](#QueueBase.dequeue_many), + `tf.errors.OutOfRangeError` like @{tf.QueueBase.dequeue_many}, less than `n` elements are returned immediately. If the queue is closed and there are `0` elements left in the queue, then a `tf.errors.OutOfRangeError` is raised just like in `dequeue_many`. @@ -558,7 +558,7 @@ def size(self, name=None): class RandomShuffleQueue(QueueBase): """A queue implementation that dequeues elements in a random order. - See [`tf.QueueBase`](#QueueBase) for a description of the methods on + See @{tf.QueueBase} for a description of the methods on this class. """ @@ -601,7 +601,7 @@ def __init__(self, capacity, min_after_dequeue, dtypes, shapes=None, with the same length as `dtypes`, or `None`. If specified the dequeue methods return a dictionary with the names as keys. seed: A Python integer. Used to create a random seed. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. shared_name: (Optional.) If non-empty, this queue will be shared under the given name across multiple sessions. @@ -631,7 +631,7 @@ def __init__(self, capacity, min_after_dequeue, dtypes, shapes=None, class FIFOQueue(QueueBase): """A queue implementation that dequeues elements in first-in first-out order. - See [`tf.QueueBase`](#QueueBase) for a description of the methods on + See @{tf.QueueBase} for a description of the methods on this class. """ @@ -682,7 +682,7 @@ class PaddingFIFOQueue(QueueBase): A `PaddingFIFOQueue` may contain components with dynamic shape, while also supporting `dequeue_many`. See the constructor for more details. - See [`tf.QueueBase`](#QueueBase) for a description of the methods on + See @{tf.QueueBase} for a description of the methods on this class. """ @@ -744,7 +744,7 @@ def __init__(self, capacity, dtypes, shapes, names=None, shared_name=None, class PriorityQueue(QueueBase): """A queue implementation that dequeues elements in prioritized order. - See [`tf.QueueBase`](#QueueBase) for a description of the methods on + See @{tf.QueueBase} for a description of the methods on this class. """ diff --git a/tensorflow/python/ops/embedding_ops.py b/tensorflow/python/ops/embedding_ops.py index 80507024d256a0..14a658be1eb7c9 100644 --- a/tensorflow/python/ops/embedding_ops.py +++ b/tensorflow/python/ops/embedding_ops.py @@ -39,7 +39,7 @@ def embedding_lookup(params, ids, partition_strategy="mod", name=None, This function is used to perform parallel lookups on the list of tensors in `params`. It is a generalization of - [`tf.gather()`](../../api_docs/python/array_ops.md#gather), where `params` is + @{tf.gather}, where `params` is interpreted as a partitioning of a large embedding tensor. `params` may be a `PartitionedVariable` as returned by using `tf.get_variable()` with a partitioner. diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py index d2f373ed129e88..7d185a81376e49 100644 --- a/tensorflow/python/ops/image_ops_impl.py +++ b/tensorflow/python/ops/image_ops_impl.py @@ -188,7 +188,7 @@ def random_flip_up_down(image, seed=None): Args: image: A 3-D tensor of shape `[height, width, channels].` seed: A Python integer. Used to create a random seed. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. Returns: @@ -215,7 +215,7 @@ def random_flip_left_right(image, seed=None): Args: image: A 3-D tensor of shape `[height, width, channels].` seed: A Python integer. Used to create a random seed. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. Returns: @@ -608,7 +608,7 @@ def resize_images(images, Resized images will be distorted if their original aspect ratio is not the same as `size`. To avoid distortions see - [`resize_image_with_crop_or_pad`](#resize_image_with_crop_or_pad). + @{tf.image.resize_image_with_crop_or_pad}. `method` can be one of: @@ -750,7 +750,7 @@ def random_brightness(image, max_delta, seed=None): image: An image. max_delta: float, must be non-negative. seed: A Python integer. Used to create a random seed. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. Returns: @@ -777,7 +777,7 @@ def random_contrast(image, lower, upper, seed=None): lower: float. Lower bound for the random contrast factor. upper: float. Upper bound for the random contrast factor. seed: A Python integer. Used to create a random seed. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. Returns: @@ -1218,24 +1218,24 @@ def adjust_saturation(image, saturation_factor, name=None): def decode_image(contents, channels=None, name=None): """Convenience function for `decode_gif`, `decode_jpeg`, and `decode_png`. - Detects whether an image is a GIF, JPEG, or PNG, and performs the appropriate + Detects whether an image is a GIF, JPEG, or PNG, and performs the appropriate operation to convert the input bytes `string` into a `Tensor` of type `uint8`. - Note: `decode_gif` returns a 4-D array `[num_frames, height, width, 3]`, as - opposed to `decode_jpeg` and `decode_png`, which return 3-D arrays - `[height, width, num_channels]`. Make sure to take this into account when - constructing your graph if you are intermixing GIF files with JPEG and/or PNG + Note: `decode_gif` returns a 4-D array `[num_frames, height, width, 3]`, as + opposed to `decode_jpeg` and `decode_png`, which return 3-D arrays + `[height, width, num_channels]`. Make sure to take this into account when + constructing your graph if you are intermixing GIF files with JPEG and/or PNG files. Args: contents: 0-D `string`. The encoded image bytes. - channels: An optional `int`. Defaults to `0`. Number of color channels for + channels: An optional `int`. Defaults to `0`. Number of color channels for the decoded image. name: A name for the operation (optional) - + Returns: - `Tensor` with type `uint8` with shape `[height, width, num_channels]` for - JPEG and PNG images and shape `[num_frames, height, width, 3]` for GIF + `Tensor` with type `uint8` with shape `[height, width, num_channels]` for + JPEG and PNG images and shape `[num_frames, height, width, 3]` for GIF images. """ with ops.name_scope(name, 'decode_image') as scope: diff --git a/tensorflow/python/ops/init_ops.py b/tensorflow/python/ops/init_ops.py index dc49e3f0e02fa3..ce2f0ef4606962 100644 --- a/tensorflow/python/ops/init_ops.py +++ b/tensorflow/python/ops/init_ops.py @@ -162,7 +162,7 @@ class RandomUniform(Initializer): maxval: A python scalar or a scalar tensor. Upper bound of the range of random values to generate. Defaults to 1 for float types. seed: A Python integer. Used to create random seeds. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. dtype: The data type. """ @@ -189,7 +189,7 @@ class RandomNormal(Initializer): stddev: a python scalar or a scalar tensor. Standard deviation of the random values to generate. seed: A Python integer. Used to create random seeds. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. dtype: The data type. Only floating point types are supported. """ @@ -221,7 +221,7 @@ class TruncatedNormal(Initializer): stddev: a python scalar or a scalar tensor. Standard deviation of the random values to generate. seed: A Python integer. Used to create random seeds. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. dtype: The data type. Only floating point types are supported. """ @@ -261,7 +261,7 @@ class UniformUnitScaling(Initializer): Args: factor: Float. A multiplicative factor by which the values will be scaled. seed: A Python integer. Used to create random seeds. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. dtype: The data type. Only floating point types are supported. """ @@ -309,7 +309,7 @@ class VarianceScaling(Initializer): mode: One of "fan_in", "fan_out", "fan_avg". distribution: Random distribution to use. One of "normal", "uniform". seed: A Python integer. Used to create random seeds. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. dtype: The data type. Only floating point types are supported. @@ -376,7 +376,7 @@ class Orthogonal(Initializer): gain: multiplicative factor to apply to the orthogonal matrix dtype: The type of the output. seed: A Python integer. Used to create random seeds. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. """ @@ -441,7 +441,7 @@ def glorot_uniform_initializer(seed=None, dtype=dtypes.float32): Arguments: seed: A Python integer. Used to create random seeds. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. dtype: The data type. Only floating point types are supported. @@ -467,7 +467,7 @@ def glorot_normal_initializer(seed=None, dtype=dtypes.float32): Arguments: seed: A Python integer. Used to create random seeds. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. dtype: The data type. Only floating point types are supported. diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py index f621bc2d906074..c94713f3ef0555 100644 --- a/tensorflow/python/ops/nn_impl.py +++ b/tensorflow/python/ops/nn_impl.py @@ -349,8 +349,7 @@ def depthwise_conv2d(input, filter, strides, padding, rate=None, name=None): strides: 1-D of size 4. The stride of the sliding window for each dimension of `input`. padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. - See the [comment - here](https://www.tensorflow.org/api_docs/python/nn.html#convolution) + See the @{tf.nn.convolution$comment here} rate: 1-D of size 2. The dilation rate in which we sample input values across the `height` and `width` dimensions in atrous convolution. If it is greater than 1, then all values of strides must be 1. @@ -426,8 +425,7 @@ def separable_conv2d(input, strides: 1-D of size 4. The strides for the depthwise convolution for each dimension of `input`. padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. - See the [comment - here](https://www.tensorflow.org/api_docs/python/nn.html#convolution) + See the @{tf.nn.convolution$comment here} rate: 1-D of size 2. The dilation rate in which we sample input values across the `height` and `width` dimensions in atrous convolution. If it is greater than 1, then all values of strides must be 1. @@ -1058,7 +1056,7 @@ def nce_loss(weights, Note: By default this uses a log-uniform (Zipfian) distribution for sampling, so your labels must be sorted in order of decreasing frequency to achieve good results. For more details, see - [log_uniform_candidate_sampler](#log_uniform_candidate_sampler). + @{tf.nn.log_uniform_candidate_sampler}. Note: In the case where `num_true` > 1, we assign to each target class the target probability 1 / `num_true` so that the target probabilities diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index 344a5921065297..5b4419595412d2 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -516,7 +516,7 @@ def convolution(input, filter, # pylint: disable=redefined-builtin where `padded_input` is obtained by zero padding the input using an effective spatial filter shape of `(spatial_filter_shape-1) * dilation_rate + 1` and output striding `strides` as described in the - [comment here](https://www.tensorflow.org/api_docs/python/nn.html#convolution). + @{tf.nn.convolution$comment here}. In the case that `data_format` does start with `"NC"`, the `input` and output (but not the `filter`) are simply transposed as follows: @@ -665,7 +665,7 @@ def pool(input, # pylint: disable=redefined-builtin where the reduction function REDUCE depends on the value of `pooling_type`, and pad_before is defined based on the value of `padding` as described in the - [comment here](https://www.tensorflow.org/api_docs/python/nn.html#convolution). + @{tf.nn.convolution$comment here}. The reduction never includes out-of-bounds positions. In the case that `data_format` starts with `"NC"`, the `input` and output are @@ -685,7 +685,7 @@ def pool(input, # pylint: disable=redefined-builtin window_shape: Sequence of N ints >= 1. pooling_type: Specifies pooling operation, must be "AVG" or "MAX". padding: The padding algorithm, must be "SAME" or "VALID". - See the [comment here](https://www.tensorflow.org/api_docs/python/nn.html#convolution) + See the @{tf.nn.convolution$comment here} dilation_rate: Optional. Dilation rate. List of N ints >= 1. Defaults to [1]*N. If any value of dilation_rate is > 1, then all values of strides must be 1. @@ -1029,7 +1029,7 @@ def conv2d_transpose(value, strides: A list of ints. The stride of the sliding window for each dimension of the input tensor. padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. - See the [comment here](https://www.tensorflow.org/api_docs/python/nn.html#convolution) + See the @{tf.nn.convolution$comment here} data_format: A string. 'NHWC' and 'NCHW' are supported. name: Optional name for the returned tensor. @@ -1246,7 +1246,7 @@ def conv3d_transpose(value, strides: A list of ints. The stride of the sliding window for each dimension of the input tensor. padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. - See the [comment here](https://www.tensorflow.org/api_docs/python/nn.html#convolution) + See the @{tf.nn.convolution$comment here} name: Optional name for the returned tensor. Returns: @@ -1748,7 +1748,7 @@ def avg_pool(value, ksize, strides, padding, data_format="NHWC", name=None): The stride of the sliding window for each dimension of the input tensor. padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. - See the [comment here](https://www.tensorflow.org/api_docs/python/nn.html#convolution) + See the @{tf.nn.convolution$comment here} data_format: A string. 'NHWC' and 'NCHW' are supported. name: Optional name for the operation. @@ -1776,7 +1776,7 @@ def max_pool(value, ksize, strides, padding, data_format="NHWC", name=None): strides: A list of ints that has length >= 4. The stride of the sliding window for each dimension of the input tensor. padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. - See the [comment here](https://www.tensorflow.org/api_docs/python/nn.html#convolution) + See the @{tf.nn.convolution$comment here} data_format: A string. 'NHWC' and 'NCHW' are supported. name: Optional name for the operation. @@ -1904,7 +1904,7 @@ def dropout(x, keep_prob, noise_shape=None, seed=None, name=None): # pylint: di noise_shape: A 1-D `Tensor` of type `int32`, representing the shape for randomly generated keep/drop flags. seed: A Python integer. Used to create random seeds. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. name: A name for this operation (optional). diff --git a/tensorflow/python/ops/random_ops.py b/tensorflow/python/ops/random_ops.py index 34b4d361021ef6..5f14ea90a1e48b 100644 --- a/tensorflow/python/ops/random_ops.py +++ b/tensorflow/python/ops/random_ops.py @@ -59,7 +59,7 @@ def random_normal(shape, dtype: The type of the output. seed: A Python integer. Used to create a random seed for the distribution. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. name: A name for the operation (optional). @@ -110,7 +110,7 @@ def parameterized_truncated_normal(shape, dtype: The type of the output. seed: A Python integer. Used to create a random seed for the distribution. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. name: A name for the operation (optional). @@ -156,7 +156,7 @@ def truncated_normal(shape, dtype: The type of the output. seed: A Python integer. Used to create a random seed for the distribution. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. name: A name for the operation (optional). @@ -210,8 +210,7 @@ def random_uniform(shape, floating point. dtype: The type of the output: `float32`, `float64`, `int32`, or `int64`. seed: A Python integer. Used to create a random seed for the distribution. - See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + See @{tf.set_random_seed} for behavior. name: A name for the operation (optional). @@ -266,7 +265,7 @@ def random_shuffle(value, seed=None, name=None): value: A Tensor to be shuffled. seed: A Python integer. Used to create a random seed for the distribution. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. name: A name for the operation (optional). @@ -295,7 +294,7 @@ def random_crop(value, size, seed=None, name=None): value: Input tensor to crop. size: 1-D tensor with size the rank of `value`. seed: Python integer. Used to create a random seed. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. name: A name for this operation (optional). @@ -339,7 +338,7 @@ def multinomial(logits, num_samples, seed=None, name=None): num_samples: 0-D. Number of independent samples to draw for each row slice. seed: A Python integer. Used to create a random seed for the distribution. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. name: Optional name for the operation. @@ -415,7 +414,7 @@ def random_gamma(shape, `float64`. seed: A Python integer. Used to create a random seed for the distributions. See - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) + @{tf.set_random_seed} for behavior. name: Optional name for the operation. @@ -438,3 +437,43 @@ def random_gamma(shape, ops.NotDifferentiable("RandomGamma") + + +def random_poisson(lam, shape, dtype=dtypes.float32, seed=None, name=None): + """Draws `shape` samples from each of the given Poisson distribution(s). + + `lam` is the rate parameter describing the distribution(s). + + Example: + + samples = tf.random_poisson([0.5, 1.5], [10]) + # samples has shape [10, 2], where each slice [:, 0] and [:, 1] represents + # the samples drawn from each distribution + + samples = tf.random_poisson([12.2, 3.3], [7, 5]) + # samples has shape [7, 5, 2], where each slice [:, :, 0] and [:, :, 1] + # represents the 7x5 samples drawn from each of the two distributions + + Args: + lam: A Tensor or Python value or N-D array of type `dtype`. + `lam` provides the rate parameter(s) describing the poisson + distribution(s) to sample. + shape: A 1-D integer Tensor or Python array. The shape of the output samples + to be drawn per "rate"-parameterized distribution. + dtype: The type of `lam` and the output: `float16`, `float32`, or + `float64`. + seed: A Python integer. Used to create a random seed for the distributions. + See + @{tf.set_random_seed} + for behavior. + name: Optional name for the operation. + + Returns: + samples: a `Tensor` of shape `tf.concat(shape, tf.shape(lam))` with + values of type `dtype`. + """ + with ops.name_scope(name, "random_poisson", [lam, shape]): + lam = ops.convert_to_tensor(lam, name="lam", dtype=dtype) + shape = ops.convert_to_tensor(shape, name="shape", dtype=dtypes.int32) + seed1, seed2 = random_seed.get_seed(seed) + return gen_random_ops._random_poisson(shape, lam, seed=seed1, seed2=seed2) diff --git a/tensorflow/python/ops/template.py b/tensorflow/python/ops/template.py index c9fbb854f07a06..d4778977b34754 100644 --- a/tensorflow/python/ops/template.py +++ b/tensorflow/python/ops/template.py @@ -120,7 +120,7 @@ def my_op(x, scalar_name): template of the same scope/unique_name already exists and reuse is false, an error is raised. Defaults to None. custom_getter_: Optional custom getter for variables used in `func_`. See - the [`get_variable`](#get_variable) `custom_getter` documentation for + the @{tf.get_variable} `custom_getter` documentation for more information. **kwargs: Keyword arguments to apply to `func_`. diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py index ddba73f7e9a7d7..c419b8186d7347 100644 --- a/tensorflow/python/ops/variable_scope.py +++ b/tensorflow/python/ops/variable_scope.py @@ -991,7 +991,7 @@ def get_variable(name, %sThis function prefixes the name with the current variable scope and performs reuse checks. See the -[Variable Scope How To](../../how_tos/variable_scope/index.md) +@{$variable_scope$Variable Scope How To} for an extensive description of how reusing works. Here is a basic example: ```python @@ -1325,7 +1325,7 @@ def variable_scope(name_or_scope, Variable scope allows to create new variables and to share already created ones while providing checks to not create or share by accident. For details, - see the [Variable Scope How To](../../how_tos/variable_scope/index.md), + see the @{$variable_scope$Variable Scope How To}, here we present only a few basic examples. Simple example of how to create a new variable: diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index b9f41f80e26cf5..bbf305ec13f57c 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -31,7 +31,7 @@ class Variable(object): - """See the [Variables How To](../../how_tos/variables/index.md) for a high + """See the @{$variables$Variables How To} for a high level overview. A variable maintains state in the graph across calls to `run()`. You add a @@ -380,7 +380,7 @@ def _as_graph_element(self): def _AsTensor(self): # pylint: disable=invalid-name """Converts this variable to a Tensor. - See [`value()`](#Variable.value). + See @{tf.Variable.value}. Returns: A `Tensor` containing the value of the variable. @@ -437,7 +437,7 @@ def _ref(self): Returns is a `Tensor` which holds a reference to the variable. You can assign a new value to the variable by passing the tensor to an assign op. - See [`value()`](#Variable.value) if you want to get the value of the + See @{tf.Variable.value} if you want to get the value of the variable. Returns: @@ -461,7 +461,7 @@ def eval(self, session=None): This convenience method requires a session where the graph containing this variable has been launched. If no session is passed, the default session is - used. See the [Session class](../../api_docs/python/client.md#Session) for + used. See the @{tf.Session} for more information on launching a graph and on sessions. ```python @@ -634,7 +634,7 @@ def load(self, value, session=None): This convenience method requires a session where the graph containing this variable has been launched. If no session is passed, the default session is - used. See the [Session class](../../api_docs/python/client.md#Session) for + used. See the @{tf.Session} for more information on launching a graph and on sessions. ```python @@ -1079,7 +1079,7 @@ def global_variables(): This convenience function returns the contents of that collection. An alternative to global variables are local variables. See - [`tf.local_variables()`](../../api_docs/python/state_ops.md#local_variables) + @{tf.local_variables} Returns: A list of `Variable` objects. @@ -1116,7 +1116,7 @@ def local_variables(): This convenience function returns the contents of that collection. An alternative to local variables are global variables. See - [`tf.global_variables()`](../../api_docs/python/state_ops.md#global_variables) + @{tf.global_variables} Returns: A list of local `Variable` objects. diff --git a/tensorflow/python/summary/summary_iterator.py b/tensorflow/python/summary/summary_iterator.py index 4196fd815d98ad..301f560d41378b 100644 --- a/tensorflow/python/summary/summary_iterator.py +++ b/tensorflow/python/summary/summary_iterator.py @@ -124,8 +124,8 @@ def add_summary(self, summary, global_step=None): and adds it to the event file. You can pass the result of evaluating any summary op, using - [`Session.run()`](client.md#Session.run) or - [`Tensor.eval()`](framework.md#Tensor.eval), to this + @{tf.Session.run} or + @{tf.Tensor.eval}, to this function. Alternatively, you can pass a `tf.Summary` protocol buffer that you populate with your own data. The latter is commonly done to report evaluation results in event files. diff --git a/tensorflow/python/summary/writer/writer.py b/tensorflow/python/summary/writer/writer.py index 51565d72f34c53..f87afc53dd9882 100644 --- a/tensorflow/python/summary/writer/writer.py +++ b/tensorflow/python/summary/writer/writer.py @@ -89,8 +89,8 @@ def add_summary(self, summary, global_step=None): and adds it to the event file. You can pass the result of evaluating any summary op, using - [`Session.run()`](client.md#Session.run) or - [`Tensor.eval()`](framework.md#Tensor.eval), to this + @{tf.Session.run} or + @{tf.Tensor.eval}, to this function. Alternatively, you can pass a `tf.Summary` protocol buffer that you populate with your own data. The latter is commonly done to report evaluation results in event files. diff --git a/tensorflow/python/training/input.py b/tensorflow/python/training/input.py index 62b2b85a04f331..a1bb377bc9f0d2 100644 --- a/tensorflow/python/training/input.py +++ b/tensorflow/python/training/input.py @@ -15,7 +15,7 @@ """Input pipeline. -Please see the [reading data how-to](../../how_tos/reading_data/index.md) +Please see the @{$reading_data$reading data how-to} for context. """ diff --git a/tensorflow/python/training/moving_averages.py b/tensorflow/python/training/moving_averages.py index 4b1f668a949f3b..0ab8ea983d3251 100644 --- a/tensorflow/python/training/moving_averages.py +++ b/tensorflow/python/training/moving_averages.py @@ -269,7 +269,7 @@ class ExponentialMovingAverage(object): for a given variable. * Build a model normally but load the checkpoint files to evaluate by using the shadow variable names. For this use the `average_name()` method. See - the [Saver class](../../api_docs/python/train.md#Saver) for more + the @{tf.train.Saver} for more information on restoring saved variables. Example of restoring the shadow variable values: diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py index e563355080cd3a..282bc18161515c 100644 --- a/tensorflow/python/training/saver.py +++ b/tensorflow/python/training/saver.py @@ -856,7 +856,7 @@ def get_checkpoint_state(checkpoint_dir, latest_filename=None): class Saver(object): """Saves and restores variables. - See [Variables](../../how_tos/variables/index.md) + See @{$variables$Variables} for an overview of variables, saving and restoring. The `Saver` class adds ops to save and restore variables to and from diff --git a/tensorflow/python/training/server_lib.py b/tensorflow/python/training/server_lib.py index 97cf1ddbc22d20..d2ccf37d8856dc 100644 --- a/tensorflow/python/training/server_lib.py +++ b/tensorflow/python/training/server_lib.py @@ -95,9 +95,9 @@ class Server(object): """An in-process TensorFlow server, for use in distributed training. A `tf.train.Server` instance encapsulates a set of devices and a - [`tf.Session`](../../api_docs/python/client.md#Session) target that + @{tf.Session} target that can participate in distributed training. A server belongs to a - cluster (specified by a [`tf.train.ClusterSpec`](#ClusterSpec)), and + cluster (specified by a @{tf.train.ClusterSpec}), and corresponds to a particular task in a named job. The server can communicate with any other server in the same cluster. """ @@ -182,7 +182,7 @@ def target(self): """Returns the target for a `tf.Session` to connect to this server. To create a - [`tf.Session`](../../api_docs/python/client.md#Session) that + @{tf.Session} that connects to this server, use the following snippet: ```python @@ -225,7 +225,7 @@ class ClusterSpec(object): A `tf.train.ClusterSpec` represents the set of processes that participate in a distributed TensorFlow computation. Every - [`tf.train.Server`](#Server) is constructed in a particular cluster. + @{tf.train.Server} is constructed in a particular cluster. To create a cluster with two jobs and five tasks, you specify the mapping from job names to lists of network addresses (typically @@ -410,7 +410,7 @@ def job_tasks(self, job_name): NOTE: For backwards compatibility, this method returns a list. If the given job was defined with a sparse set of task indices, the length of this list may not reflect the number of tasks defined in - this job. Use the [`num_tasks()`](#ClusterSpec.num_tasks) method + this job. Use the @{tf.train.ClusterSpec.num_tasks} method to find the number of tasks defined in a particular job. Args: diff --git a/tensorflow/python/training/supervisor.py b/tensorflow/python/training/supervisor.py index bacddd9d826c0a..7b933919118bdb 100644 --- a/tensorflow/python/training/supervisor.py +++ b/tensorflow/python/training/supervisor.py @@ -127,7 +127,7 @@ class Supervisor(object): * Specifying `'local'` requests a session that uses the RPC-based "Master interface" to run TensorFlow programs. See - [`tf.train.Server.create_local_server()`](#Server.create_local_server) for + @{tf.train.Server.create_local_server} for details. * Specifying `'grpc://hostname:port'` requests a session that uses From da8758be2ef167461f9abd325306ba08f2041558 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 14 Feb 2017 13:39:05 -0800 Subject: [PATCH 20/50] Manual documentation fixes. Change: 147514603 --- tensorflow/python/ops/nn_impl.py | 4 ++-- tensorflow/python/ops/variables.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py index c94713f3ef0555..70ff97d6307e95 100644 --- a/tensorflow/python/ops/nn_impl.py +++ b/tensorflow/python/ops/nn_impl.py @@ -349,7 +349,7 @@ def depthwise_conv2d(input, filter, strides, padding, rate=None, name=None): strides: 1-D of size 4. The stride of the sliding window for each dimension of `input`. padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. - See the @{tf.nn.convolution$comment here} + See the @{tf.nn.convolution$comment here} rate: 1-D of size 2. The dilation rate in which we sample input values across the `height` and `width` dimensions in atrous convolution. If it is greater than 1, then all values of strides must be 1. @@ -425,7 +425,7 @@ def separable_conv2d(input, strides: 1-D of size 4. The strides for the depthwise convolution for each dimension of `input`. padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. - See the @{tf.nn.convolution$comment here} + See the @{tf.nn.convolution$comment here} rate: 1-D of size 2. The dilation rate in which we sample input values across the `height` and `width` dimensions in atrous convolution. If it is greater than 1, then all values of strides must be 1. diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index bbf305ec13f57c..6ad617bc97e994 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -459,10 +459,10 @@ def eval(self, session=None): This is not a graph construction method, it does not add ops to the graph. - This convenience method requires a session where the graph containing this - variable has been launched. If no session is passed, the default session is - used. See the @{tf.Session} for - more information on launching a graph and on sessions. + This convenience method requires a session where the graph + containing this variable has been launched. If no session is + passed, the default session is used. See @{tf.Session} for more + information on launching a graph and on sessions. ```python v = tf.Variable([1, 2]) @@ -632,10 +632,10 @@ def load(self, value, session=None): Writes new value to variable's memory. Doesn't add ops to the graph. - This convenience method requires a session where the graph containing this - variable has been launched. If no session is passed, the default session is - used. See the @{tf.Session} for - more information on launching a graph and on sessions. + This convenience method requires a session where the graph + containing this variable has been launched. If no session is + passed, the default session is used. See @{tf.Session} for more + information on launching a graph and on sessions. ```python v = tf.Variable([1, 2]) From 419665fdedae548de460c79cbd50b18497159131 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 14 Feb 2017 14:17:33 -0800 Subject: [PATCH 21/50] Update Python doc strings for the new doc generator pipeline. Change: 147519957 --- tensorflow/python/framework/ops.py | 30 ------------------- tensorflow/python/layers/layers.py | 12 -------- tensorflow/python/ops/sdca_ops.py | 5 +--- tensorflow/python/ops/sets.py | 2 +- tensorflow/python/ops/variables.py | 29 ------------------ tensorflow/python/platform/resource_loader.py | 2 +- tensorflow/python/platform/sysconfig.py | 2 +- 7 files changed, 4 insertions(+), 78 deletions(-) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 4b734fd146c86d..8ef99730f650ff 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -1576,8 +1576,6 @@ def _sub_grad(unused_op, grad): The decorator argument `op_type` is the string type of an operation. This corresponds to the `OpDef.name` field for the proto that defines the operation. - - @@__init__ """ def __init__(self, op_type): @@ -1925,16 +1923,6 @@ class Graph(object): synchronization must be provided. Unless otherwise specified, all methods are not thread-safe. - @@__init__ - @@as_default - @@as_graph_def - @@finalize - @@finalized - - @@control_dependencies - @@device - @@name_scope - A `Graph` instance supports an arbitrary number of "collections" that are identified by name. For convenience when building a large graph, collections can store groups of related objects: for @@ -1942,24 +1930,6 @@ class Graph(object): @{tf.GraphKeys.GLOBAL_VARIABLES}) for all variables that are created during the construction of a graph. The caller may define additional collections by specifying a new name. - - @@add_to_collection - @@add_to_collections - @@get_collection - @@get_collection_ref - - @@as_graph_element - @@get_operation_by_name - @@get_tensor_by_name - @@get_operations - - @@seed - @@unique_name - @@version - @@graph_def_versions - - @@create_op - @@gradient_override_map """ def __init__(self): diff --git a/tensorflow/python/layers/layers.py b/tensorflow/python/layers/layers.py index c25361d562a66b..9f02757d5bc754 100644 --- a/tensorflow/python/layers/layers.py +++ b/tensorflow/python/layers/layers.py @@ -16,32 +16,20 @@ # pylint: disable=line-too-long """This library provides a set of high-level neural networks layers. -## Core layers - @@dense @@dropout - -## Convolutional layers - @@conv1d @@conv2d @@conv3d @@separable_conv2d @@conv2d_transpose - -## Pooling layers - @@average_pooling1d @@max_pooling1d @@average_pooling2d @@max_pooling2d @@average_pooling3d @@max_pooling3d - -## Normalization layers - @@batch_normalization - """ from __future__ import absolute_import diff --git a/tensorflow/python/ops/sdca_ops.py b/tensorflow/python/ops/sdca_ops.py index 3876bc96421991..8b7e5abbc22735 100644 --- a/tensorflow/python/ops/sdca_ops.py +++ b/tensorflow/python/ops/sdca_ops.py @@ -12,10 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""## Script Language Operators. - -A Dual Cordinate Ascent optimizer for TensorFlow for training fast linear -models. +"""A Dual Coordinate Ascent optimizer library for training fast linear models. @@sdca_optimizer @@sdca_fprint diff --git a/tensorflow/python/ops/sets.py b/tensorflow/python/ops/sets.py index 1eff9033559b35..ea4677befe6fca 100644 --- a/tensorflow/python/ops/sets.py +++ b/tensorflow/python/ops/sets.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Python layer for sets. +"""Tensorflow set operations. @@set_size @@set_intersection diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index 6ad617bc97e994..8d34395fe3aeb1 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -116,37 +116,8 @@ class Variable(object): `trainable_variables()` returns the contents of this collection. The various `Optimizer` classes use this collection as the default list of variables to optimize. - - - Creating a variable. - - @@__init__ - @@initialized_value - - Changing a variable value. - - @@assign - @@assign_add - @@assign_sub - @@scatter_sub - @@count_up_to - - @@eval - - Properties. - - @@name - @@dtype - @@get_shape - @@device - @@initializer - @@graph - @@op """ - # TODO(touts): Add @@value and @@ref in the docstring above once they are - # ready for consumption. - def __init__(self, initial_value=None, trainable=True, diff --git a/tensorflow/python/platform/resource_loader.py b/tensorflow/python/platform/resource_loader.py index cee83c6185365e..880f14b8bbc76b 100644 --- a/tensorflow/python/platform/resource_loader.py +++ b/tensorflow/python/platform/resource_loader.py @@ -13,7 +13,7 @@ # limitations under the License. # ============================================================================== -"""## Resource management. +"""Resource management library. @@get_data_files_path @@get_path_to_datafile diff --git a/tensorflow/python/platform/sysconfig.py b/tensorflow/python/platform/sysconfig.py index 8b1e2c329d5047..70765168586da7 100644 --- a/tensorflow/python/platform/sysconfig.py +++ b/tensorflow/python/platform/sysconfig.py @@ -13,7 +13,7 @@ # limitations under the License. # ============================================================================== -"""## System configuration. +"""System configuration library. @@get_include @@get_lib From 929739370e90f21755ae5e9d7b054b33a6b94f8c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sat, 11 Feb 2017 14:28:39 -0800 Subject: [PATCH 22/50] Change to remove BUILD dependency on docs. Change: 147255697 --- tensorflow/BUILD | 2 - tensorflow/g3doc/how_tos/adding_an_op/BUILD | 157 -------------------- tensorflow/g3doc/tutorials/BUILD | 19 --- 3 files changed, 178 deletions(-) delete mode 100644 tensorflow/g3doc/how_tos/adding_an_op/BUILD delete mode 100644 tensorflow/g3doc/tutorials/BUILD diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 355e48d582f845..7379b1f63c301a 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -194,8 +194,6 @@ filegroup( "//tensorflow/examples/tutorials/estimators:all_files", "//tensorflow/examples/tutorials/mnist:all_files", "//tensorflow/examples/tutorials/word2vec:all_files", - "//tensorflow/g3doc/how_tos/adding_an_op:all_files", - "//tensorflow/g3doc/tutorials:all_files", "//tensorflow/go:all_files", "//tensorflow/java:all_files", "//tensorflow/java/src/main/java/org/tensorflow/examples:all_files", diff --git a/tensorflow/g3doc/how_tos/adding_an_op/BUILD b/tensorflow/g3doc/how_tos/adding_an_op/BUILD deleted file mode 100644 index ffaf9349d2a104..00000000000000 --- a/tensorflow/g3doc/how_tos/adding_an_op/BUILD +++ /dev/null @@ -1,157 +0,0 @@ -# Description: -# Code examples referenced by adding_an_op - -package( - default_visibility = ["//tensorflow:internal"], - features = [ - "-layering_check", - "-parse_headers", - ], -) - -licenses(["notice"]) # Apache 2.0 - -load("//tensorflow:tensorflow.bzl", "tf_custom_op_library") -load("//tensorflow:tensorflow.bzl", "tf_cuda_tests_tags") - -exports_files(["LICENSE"]) - -tf_custom_op_library( - name = "zero_out_op_kernel_1.so", - srcs = ["zero_out_op_kernel_1.cc"], -) - -py_library( - name = "zero_out_op_1", - srcs = ["zero_out_op_1.py"], - data = [":zero_out_op_kernel_1.so"], - srcs_version = "PY2AND3", -) - -tf_custom_op_library( - name = "zero_out_op_kernel_2.so", - srcs = ["zero_out_op_kernel_2.cc"], -) - -py_library( - name = "zero_out_op_2", - srcs = ["zero_out_op_2.py"], - data = [":zero_out_op_kernel_2.so"], - srcs_version = "PY2AND3", -) - -tf_custom_op_library( - name = "zero_out_op_kernel_3.so", - srcs = ["zero_out_op_kernel_3.cc"], -) - -py_library( - name = "zero_out_op_3", - srcs = ["zero_out_op_3.py"], - data = [":zero_out_op_kernel_3.so"], - srcs_version = "PY2AND3", -) - -py_library( - name = "zero_out_grad_2", - srcs = ["zero_out_grad_2.py"], - srcs_version = "PY2AND3", - deps = [ - ":zero_out_op_2", - "//tensorflow:tensorflow_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:sparse_ops", - ], -) - -py_test( - name = "zero_out_1_test", - size = "small", - srcs = ["zero_out_1_test.py"], - srcs_version = "PY2AND3", - deps = [ - ":zero_out_op_1", - "//tensorflow:tensorflow_py", - "//third_party/py/numpy", - ], -) - -py_test( - name = "zero_out_2_test", - size = "small", - srcs = ["zero_out_2_test.py"], - srcs_version = "PY2AND3", - deps = [ - ":zero_out_grad_2", - ":zero_out_op_2", - "//tensorflow:tensorflow_py", - ], -) - -py_test( - name = "zero_out_3_test", - size = "small", - srcs = ["zero_out_3_test.py"], - srcs_version = "PY2AND3", - deps = [ - ":zero_out_op_3", - "//tensorflow:tensorflow_py", - "//third_party/py/numpy", - ], -) - -tf_custom_op_library( - name = "cuda_op_kernel.so", - srcs = ["cuda_op_kernel.cc"], - gpu_srcs = ["cuda_op_kernel.cu.cc"], -) - -py_library( - name = "cuda_op", - srcs = ["cuda_op.py"], - data = [":cuda_op_kernel.so"], - srcs_version = "PY2AND3", -) - -py_test( - name = "cuda_op_test", - size = "small", - srcs = ["cuda_op_test.py"], - srcs_version = "PY2AND3", - tags = tf_cuda_tests_tags(), - deps = [ - ":cuda_op", - "//tensorflow:tensorflow_py", - ], -) - -py_test( - name = "fact_test", - size = "small", - srcs = ["fact_test.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - ], -) - -cc_binary( - name = "attr_examples", - srcs = ["attr_examples.cc"], - deps = [ - "//tensorflow/core", - ], -) - -filegroup( - name = "all_files", - srcs = glob( - ["**/*"], - exclude = [ - "**/METADATA", - "**/OWNERS", - ], - ), - visibility = ["//tensorflow:__subpackages__"], -) diff --git a/tensorflow/g3doc/tutorials/BUILD b/tensorflow/g3doc/tutorials/BUILD deleted file mode 100644 index 5642ade160e872..00000000000000 --- a/tensorflow/g3doc/tutorials/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -# Description: -# Top-level tutorials files - -package(default_visibility = ["//tensorflow:internal"]) - -licenses(["notice"]) # Apache 2.0 - -exports_files(["LICENSE"]) - -filegroup( - name = "all_files", - srcs = glob( - ["**/*"], - exclude = [ - "**/METADATA", - "**/OWNERS", - ], - ), -) From 8e51bfdb45687cbcd0faece394b5cb992c7aec5c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sat, 11 Feb 2017 16:00:39 -0800 Subject: [PATCH 23/50] Add linking to methods, and warnings for links to things that are not in the index. This CL creates an overabundance of tags on class pages, but that doesn't hurt as long as we know ours. Demo reference to class field (tf.GraphKeys.REGULARIZATION_LOSSES) is in get_variable.md. Change: 147257988 --- tensorflow/python/ops/variable_scope.py | 6 +++--- tensorflow/tools/common/public_api.py | 12 ++++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py index c419b8186d7347..6bcd32bbd05342 100644 --- a/tensorflow/python/ops/variable_scope.py +++ b/tensorflow/python/ops/variable_scope.py @@ -739,9 +739,9 @@ def no_regularizer(_): class VariableScope(object): - """Variable scope object to carry defaults to provide to get_variable. + """Variable scope object to carry defaults to provide to `get_variable`. - Many of the arguments we need for get_variable in a variable store are most + Many of the arguments we need for `get_variable` in a variable store are most easily handled with a context. This object is used for the defaults. Attributes: @@ -1025,7 +1025,7 @@ def get_variable(name, initializer: Initializer for the variable if one is created. regularizer: A (Tensor -> Tensor or None) function; the result of applying it on a newly created variable will be added to the collection - GraphKeys.REGULARIZATION_LOSSES and can be used for regularization. + @{tf.GraphKeys.REGULARIZATION_LOSSES} and can be used for regularization. %scollections: List of graph collections keys to add the Variable to. Defaults to `[%s]` (see `tf.Variable`). caching_device: Optional device string or function describing where the diff --git a/tensorflow/tools/common/public_api.py b/tensorflow/tools/common/public_api.py index 5d70cb7b76736c..f289db9e61c016 100644 --- a/tensorflow/tools/common/public_api.py +++ b/tensorflow/tools/common/public_api.py @@ -39,8 +39,16 @@ def __init__(self, visitor): # sytem modules exposed through platforms for compatibility reasons. # Each entry maps a module path to a name to ignore in traversal. _do_not_descend_map = { - # TODO(drpng): This can be removed once sealed off. - '': ['platform', 'pywrap_tensorflow'], + '': [ + 'core', + 'examples', + 'flags', # Don't add flags + 'platform', # TODO(drpng): This can be removed once sealed off. + 'pywrap_tensorflow', # TODO(drpng): This can be removed once sealed. + 'user_ops', # TODO(drpng): This can be removed once sealed. + 'python', + 'tools' + ], # Some implementations have this internal module that we shouldn't expose. 'flags': ['cpp_flags'], From e8da7b7bdd239869c64b719fc726759ce35c1eb6 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Thu, 16 Feb 2017 16:08:57 -0800 Subject: [PATCH 24/50] Add --pull to docker build to automatically pull the latest image in FROM. --- tensorflow/tools/docker/parameterized_docker_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/docker/parameterized_docker_build.sh b/tensorflow/tools/docker/parameterized_docker_build.sh index 35c12184700342..886266caaf8c0b 100755 --- a/tensorflow/tools/docker/parameterized_docker_build.sh +++ b/tensorflow/tools/docker/parameterized_docker_build.sh @@ -266,7 +266,7 @@ fi IMG="${USER}/tensorflow:${FINAL_TAG}" echo "Building docker image with image name and tag: ${IMG}" -"${DOCKER_BINARY}" build --no-cache -t "${IMG}" -f "${DOCKERFILE}" "${TMP_DIR}" +"${DOCKER_BINARY}" build --no-cache --pull -t "${IMG}" -f "${DOCKERFILE}" "${TMP_DIR}" if [[ $? == "0" ]]; then echo "${DOCKER_BINARY} build of ${IMG} succeeded" else From c56c873fbaf976d26d487ad57c8efbc87f05331c Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Thu, 16 Feb 2017 18:03:19 -0800 Subject: [PATCH 25/50] Add --pull to docker build to automatically pull the latest image in (#7597) FROM. --- tensorflow/tools/docker/parameterized_docker_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/docker/parameterized_docker_build.sh b/tensorflow/tools/docker/parameterized_docker_build.sh index 35c12184700342..886266caaf8c0b 100755 --- a/tensorflow/tools/docker/parameterized_docker_build.sh +++ b/tensorflow/tools/docker/parameterized_docker_build.sh @@ -266,7 +266,7 @@ fi IMG="${USER}/tensorflow:${FINAL_TAG}" echo "Building docker image with image name and tag: ${IMG}" -"${DOCKER_BINARY}" build --no-cache -t "${IMG}" -f "${DOCKERFILE}" "${TMP_DIR}" +"${DOCKER_BINARY}" build --no-cache --pull -t "${IMG}" -f "${DOCKERFILE}" "${TMP_DIR}" if [[ $? == "0" ]]; then echo "${DOCKER_BINARY} build of ${IMG} succeeded" else From 16198047300a31488f13dee6048579732f7db542 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Tue, 21 Feb 2017 10:44:15 -0800 Subject: [PATCH 26/50] Update release note for tf.nn.rnn_cell. --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index 2044598904a0ac..d732672f8d93ee 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -85,6 +85,7 @@ To help you upgrade your existing TensorFlow Python code to match the API change * In the C++ API (in tensorflow/cc), Input, Output, etc. have moved from the tensorflow::ops namespace to tensorflow. * Change arg order for `{softmax,sparse_softmax,sigmoid}_cross_entropy_with_logits` to be (labels, predictions), and force use of named args. +* tf.nn.rnn_cell.* and most functions in tf.nn.rnn.* (with the exception of dynamic_rnn and raw_rnn) are temporarily in tf.contrib.rnn. They will be moved back into core for TF 1.1. ## Bug Fixes and Other Changes * Numerous C++ API updates. From d21baff6a50f79a3dd4b8f7bbd3f024823558b28 Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Tue, 21 Feb 2017 18:12:14 -0800 Subject: [PATCH 27/50] Add MAVX2_FMA_DBG build option and rename MAVXDBG to MAVX_DBG. --- tensorflow/tools/ci_build/ci_parameterized_build.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh index f915678199198f..b7d5a8d144b5f4 100755 --- a/tensorflow/tools/ci_build/ci_parameterized_build.sh +++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh @@ -84,7 +84,8 @@ # support for Google Cloud Platform (GCP), which is # enabled by default. # TF_BUILD_OPTIONS: -# (FASTBUILD | OPT | OPTDBG | MAVX | MAVX2) +# (FASTBUILD | OPT | OPTDBG | MAVX | MAVX2 | MAVX_DBG | +# MAVX2_FMA_DBG) # Use the specified configurations when building. # When set, overrides TF_BUILD_IS_OPT and TF_BUILD_MAVX # options, as this will replace the two. @@ -305,12 +306,15 @@ else MAVX) OPT_FLAG="${OPT_FLAG} -c opt --copt=-mavx" ;; - MAVXDBG) + MAVX_DBG) OPT_FLAG="${OPT_FLAG} -c opt --copt=-g --copt=-mavx" ;; MAVX2) OPT_FLAG="${OPT_FLAG} -c opt --copt=-mavx2" ;; + MAVX2_FMA_DBG) + OPT_FLAG="${OPT_FLAG} -c opt --copt=-g --copt=-mavx2 --copt=-mfma" + ;; esac fi From 49b8223c307151a396830049ac07bc871d66334c Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Tue, 21 Feb 2017 21:28:37 -0800 Subject: [PATCH 28/50] Use Ubuntu 14.04 for the 1.0 GPU build. --- tensorflow/tools/ci_build/Dockerfile.gpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu b/tensorflow/tools/ci_build/Dockerfile.gpu index 4d7f6ef95da425..7657d40f12de5c 100644 --- a/tensorflow/tools/ci_build/Dockerfile.gpu +++ b/tensorflow/tools/ci_build/Dockerfile.gpu @@ -1,4 +1,4 @@ -FROM nvidia/cuda:8.0-cudnn5-devel +FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 MAINTAINER Jan Prach From d7950e02630459e8e4b51c5c16224ee29fb1cbf6 Mon Sep 17 00:00:00 2001 From: alanwang93 Date: Wed, 22 Feb 2017 15:17:02 +0100 Subject: [PATCH 29/50] fix some documentation style faults for dynamic_rnn, dynamic_rn_decoder and tensordot --- tensorflow/contrib/seq2seq/python/ops/seq2seq.py | 1 + tensorflow/python/ops/math_ops.py | 2 +- tensorflow/python/ops/rnn.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/seq2seq/python/ops/seq2seq.py b/tensorflow/contrib/seq2seq/python/ops/seq2seq.py index 67132a3677dd89..6495f3e54f052f 100644 --- a/tensorflow/contrib/seq2seq/python/ops/seq2seq.py +++ b/tensorflow/contrib/seq2seq/python/ops/seq2seq.py @@ -76,6 +76,7 @@ def dynamic_rnn_decoder(cell, decoder_fn, inputs=None, sequence_length=None, The input to `cell` at each time step will be a `Tensor` with dimensions `[batch_size, ...]`. + sequence_length: (optional) An int32/int64 vector sized `[batch_size]`. if `inputs` is not None and `sequence_length` is None it is inferred from the `inputs` as the maximal possible sequence length. diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py index e5c5265a8c58b6..ecb1af6982b285 100644 --- a/tensorflow/python/ops/math_ops.py +++ b/tensorflow/python/ops/math_ops.py @@ -2169,7 +2169,7 @@ def tensordot(a, b, axes, name=None): Example 2: When `a` and `b` are matrices (order 2), the case `axes = [[1], [0]]` is equivalent to matrix multiplication. - Example 3: Suppose that \\(a_ijk\\) and \\(b_lmn\\) represent two + Example 3: Suppose that \\(a_{ijk}\\) and \\(b_{lmn}\\) represent two tensors of order 3. Then, `contract(a, b, [0], [2])` is the order 4 tensor \\(c_{jklm}\\) whose entry corresponding to the indices \\((j,k,l,m)\\) is given by: diff --git a/tensorflow/python/ops/rnn.py b/tensorflow/python/ops/rnn.py index 06ae3589a27b56..790446b75682ad 100644 --- a/tensorflow/python/ops/rnn.py +++ b/tensorflow/python/ops/rnn.py @@ -419,6 +419,7 @@ def dynamic_rnn(cell, inputs, sequence_length=None, initial_state=None, The input to `cell` at each time step will be a `Tensor` or (possibly nested) tuple of Tensors each with dimensions `[batch_size, ...]`. + sequence_length: (optional) An int32/int64 vector sized `[batch_size]`. initial_state: (optional) An initial state for the RNN. If `cell.state_size` is an integer, this must be From b0c34b0a6e87733ec51a28ee6c4232f3a95ebf37 Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Wed, 22 Feb 2017 08:25:20 -0800 Subject: [PATCH 30/50] Workaround for cudnn file location issue --- tensorflow/tools/ci_build/Dockerfile.gpu | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu b/tensorflow/tools/ci_build/Dockerfile.gpu index 7657d40f12de5c..0c452aa6ccfd99 100644 --- a/tensorflow/tools/ci_build/Dockerfile.gpu +++ b/tensorflow/tools/ci_build/Dockerfile.gpu @@ -2,6 +2,11 @@ FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 MAINTAINER Jan Prach +# In the Ubuntu 14.04 images, cudnn is placed in system paths. Move them to +# /usr/local/cuda +RUN cp /usr/include/cudnn.h /usr/local/cuda/include +RUN cp /usr/lib/x86_64-linux-gnu/libcudnn* /usr/local/cuda/lib64 + # Copy and run the install scripts. COPY install/*.sh /install/ RUN /install/install_bootstrap_deb_packages.sh From 81d8255cb5064fe76238b749c06f84ec405cf1e6 Mon Sep 17 00:00:00 2001 From: Jan Prach Date: Wed, 22 Feb 2017 10:48:23 -0800 Subject: [PATCH 31/50] fix debian jessie ci_build --- tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu b/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu index fa74320b1e5f80..1639e2ecddc69d 100644 --- a/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu +++ b/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu @@ -5,7 +5,15 @@ MAINTAINER Jan Prach # Copy and run the install scripts. COPY install/*.sh /install/ RUN /install/install_bootstrap_deb_packages.sh -RUN echo "deb http://http.debian.net/debian jessie-backports main" | tee -a /etc/apt/sources.list +RUN echo "deb http://http.debian.net/debian jessie-backports main" | \ + tee -a /etc/apt/sources.list +# Workaround bug in Jessie backport repository deb packages +# http://serverfault.com/questions/830636/cannot-install-openjdk-8-jre-headless-on-debian-jessie +RUN apt-get update && \ + apt-get install -y --no-install-recommends -t jessie-backports \ + openjdk-8-jre-headless ca-certificates-java && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* RUN /install/install_deb_packages.sh RUN /install/install_pip_packages.sh RUN /install/install_bazel.sh From 01e57647635d547912fc5fcfc97ccfdc1467e825 Mon Sep 17 00:00:00 2001 From: Marek Kolodziej Date: Wed, 18 Jan 2017 16:36:59 -0800 Subject: [PATCH 32/50] Add pre-compilation of Pascal binaries (CC 6.0/6.1) to CUDA-enabled Docker build (#6941) (cherry picked from commit 83a5427ce8cf8bbfa665022fdd6aa125d8a8883b) --- tensorflow/tools/docker/Dockerfile.devel-gpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu index f693481fdfd285..b877086ffaa860 100644 --- a/tensorflow/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow/tools/docker/Dockerfile.devel-gpu @@ -89,7 +89,7 @@ WORKDIR /tensorflow ENV CI_BUILD_PYTHON python ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH ENV TF_NEED_CUDA 1 -ENV TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2 +ENV TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2,6.0,6.1 RUN tensorflow/tools/ci_build/builds/configured GPU \ bazel build -c opt --config=cuda tensorflow/tools/pip_package:build_pip_package && \ From a868af707dcf56cb92d32b9ef3802c1362e5c0f8 Mon Sep 17 00:00:00 2001 From: Alexey Surkov Date: Mon, 13 Feb 2017 18:00:46 -0800 Subject: [PATCH 33/50] Retry failures when requesting auth token from GCE metadata server. Also - the necessary refactoring of the retrying logic - increases the retry cap to 5 - more verbose HTTP error messages - an env flag to suppress the token retrieval logic in testing Change: 147419297 --- tensorflow/core/platform/cloud/BUILD | 16 ++ .../platform/cloud/google_auth_provider.cc | 67 +++-- .../platform/cloud/google_auth_provider.h | 12 +- .../cloud/google_auth_provider_test.cc | 56 +++-- .../core/platform/cloud/http_request.cc | 22 +- .../core/platform/cloud/http_request_test.cc | 238 ++++++++++-------- .../platform/cloud/retrying_file_system.cc | 128 ++++------ .../cloud/retrying_file_system_test.cc | 144 ++++------- .../core/platform/cloud/retrying_utils.cc | 67 +++++ .../core/platform/cloud/retrying_utils.h | 39 +++ .../ops/cloud/bigquery_reader_ops_test.py | 4 + 11 files changed, 461 insertions(+), 332 deletions(-) create mode 100644 tensorflow/core/platform/cloud/retrying_utils.cc create mode 100644 tensorflow/core/platform/cloud/retrying_utils.h diff --git a/tensorflow/core/platform/cloud/BUILD b/tensorflow/core/platform/cloud/BUILD index c3c995174d6b92..84b0bda896c0c0 100644 --- a/tensorflow/core/platform/cloud/BUILD +++ b/tensorflow/core/platform/cloud/BUILD @@ -87,6 +87,7 @@ cc_library( deps = [ ":http_request", ":oauth_client", + ":retrying_utils", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", "@jsoncpp_git//:jsoncpp", @@ -110,6 +111,20 @@ cc_library( ], ) +cc_library( + name = "retrying_utils", + srcs = [ + "retrying_utils.cc", + ], + hdrs = [ + "retrying_utils.h", + ], + deps = [ + "//tensorflow/core:framework_headers_lib", + "//tensorflow/core:lib_internal", + ], +) + cc_library( name = "retrying_file_system", srcs = [ @@ -119,6 +134,7 @@ cc_library( "retrying_file_system.h", ], deps = [ + ":retrying_utils", "//tensorflow/core:framework_headers_lib", "//tensorflow/core:lib_internal", ], diff --git a/tensorflow/core/platform/cloud/google_auth_provider.cc b/tensorflow/core/platform/cloud/google_auth_provider.cc index 73e9f4f7cf342b..6f29d4597f18dc 100644 --- a/tensorflow/core/platform/cloud/google_auth_provider.cc +++ b/tensorflow/core/platform/cloud/google_auth_provider.cc @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/lib/strings/base64.h" #include "tensorflow/core/platform/cloud/http_request.h" +#include "tensorflow/core/platform/cloud/retrying_utils.h" #include "tensorflow/core/platform/env.h" namespace tensorflow { @@ -34,6 +35,9 @@ namespace { constexpr char kGoogleApplicationCredentials[] = "GOOGLE_APPLICATION_CREDENTIALS"; +// The environment variable to override token generation for testing. +constexpr char kGoogleAuthTokenForTesting[] = "GOOGLE_AUTH_TOKEN_FOR_TESTING"; + // The environment variable which can override '~/.config/gcloud' if set. constexpr char kCloudSdkConfig[] = "CLOUDSDK_CONFIG"; @@ -63,6 +67,9 @@ constexpr char kGceTokenUrl[] = // The authentication token scope to request. constexpr char kOAuthScope[] = "https://www.googleapis.com/auth/cloud-platform"; +// The default intial delay between retries with exponential backoff. +constexpr int kInitialRetryDelayUsec = 500000; // 0.5 sec + /// Returns whether the given path points to a readable file. bool IsFile(const string& filename) { std::ifstream fstream(filename.c_str()); @@ -115,14 +122,16 @@ GoogleAuthProvider::GoogleAuthProvider() : GoogleAuthProvider( std::unique_ptr(new OAuthClient()), std::unique_ptr(new HttpRequest::Factory()), - Env::Default()) {} + Env::Default(), kInitialRetryDelayUsec) {} GoogleAuthProvider::GoogleAuthProvider( std::unique_ptr oauth_client, - std::unique_ptr http_request_factory, Env* env) + std::unique_ptr http_request_factory, Env* env, + int64 initial_retry_delay_usec) : oauth_client_(std::move(oauth_client)), http_request_factory_(std::move(http_request_factory)), - env_(env) {} + env_(env), + initial_retry_delay_usec_(initial_retry_delay_usec) {} Status GoogleAuthProvider::GetToken(string* t) { mutex_lock lock(mu_); @@ -134,12 +143,12 @@ Status GoogleAuthProvider::GetToken(string* t) { return Status::OK(); } - // First, try to get the token using credentials stored in a few special - // file locations. - auto token_from_files_status = GetTokenFromFiles(); + if (GetTokenForTesting().ok()) { + *t = current_token_; + return Status::OK(); + } - // If that didn't work, try to get the token assuming we're running on - // Google Compute Engine (GCE). + auto token_from_files_status = GetTokenFromFiles(); auto token_from_gce_status = token_from_files_status.ok() ? Status::OK() : GetTokenFromGce(); @@ -195,20 +204,34 @@ Status GoogleAuthProvider::GetTokenFromFiles() { } Status GoogleAuthProvider::GetTokenFromGce() { - std::unique_ptr request(http_request_factory_->Create()); - std::vector response_buffer; - const uint64 request_timestamp_sec = env_->NowSeconds(); - TF_RETURN_IF_ERROR(request->Init()); - TF_RETURN_IF_ERROR(request->SetUri(kGceTokenUrl)); - TF_RETURN_IF_ERROR(request->AddHeader("Metadata-Flavor", "Google")); - TF_RETURN_IF_ERROR(request->SetResultBuffer(&response_buffer)); - TF_RETURN_IF_ERROR(request->Send()); - StringPiece response = - StringPiece(&response_buffer[0], response_buffer.size()); - - TF_RETURN_IF_ERROR(oauth_client_->ParseOAuthResponse( - response, request_timestamp_sec, ¤t_token_, - &expiration_timestamp_sec_)); + const auto get_token_from_gce = [this]() { + std::unique_ptr request(http_request_factory_->Create()); + std::vector response_buffer; + const uint64 request_timestamp_sec = env_->NowSeconds(); + TF_RETURN_IF_ERROR(request->Init()); + TF_RETURN_IF_ERROR(request->SetUri(kGceTokenUrl)); + TF_RETURN_IF_ERROR(request->AddHeader("Metadata-Flavor", "Google")); + TF_RETURN_IF_ERROR(request->SetResultBuffer(&response_buffer)); + TF_RETURN_IF_ERROR(request->Send()); + StringPiece response = + StringPiece(&response_buffer[0], response_buffer.size()); + + TF_RETURN_IF_ERROR(oauth_client_->ParseOAuthResponse( + response, request_timestamp_sec, ¤t_token_, + &expiration_timestamp_sec_)); + return Status::OK(); + }; + return RetryingUtils::CallWithRetries(get_token_from_gce, + initial_retry_delay_usec_); +} + +Status GoogleAuthProvider::GetTokenForTesting() { + const char* token = std::getenv(kGoogleAuthTokenForTesting); + if (!token) { + return errors::NotFound("The env variable for testing was not set."); + } + expiration_timestamp_sec_ = UINT64_MAX; + current_token_ = token; return Status::OK(); } diff --git a/tensorflow/core/platform/cloud/google_auth_provider.h b/tensorflow/core/platform/cloud/google_auth_provider.h index 88b8445584043f..00da25a9593a40 100644 --- a/tensorflow/core/platform/cloud/google_auth_provider.h +++ b/tensorflow/core/platform/cloud/google_auth_provider.h @@ -30,7 +30,8 @@ class GoogleAuthProvider : public AuthProvider { GoogleAuthProvider(); explicit GoogleAuthProvider( std::unique_ptr oauth_client, - std::unique_ptr http_request_factory, Env* env); + std::unique_ptr http_request_factory, Env* env, + int64 initial_retry_delay_usec); virtual ~GoogleAuthProvider() {} /// \brief Returns the short-term authentication bearer token. @@ -43,10 +44,13 @@ class GoogleAuthProvider : public AuthProvider { /// /// Tries the file from $GOOGLE_APPLICATION_CREDENTIALS and the /// standard gcloud tool's location. - Status GetTokenFromFiles(); + Status GetTokenFromFiles() EXCLUSIVE_LOCKS_REQUIRED(mu_); /// Gets the bearer token from Google Compute Engine environment. - Status GetTokenFromGce(); + Status GetTokenFromGce() EXCLUSIVE_LOCKS_REQUIRED(mu_); + + /// Gets the bearer token from the systen env variable, for testing purposes. + Status GetTokenForTesting() EXCLUSIVE_LOCKS_REQUIRED(mu_); std::unique_ptr oauth_client_; std::unique_ptr http_request_factory_; @@ -54,6 +58,8 @@ class GoogleAuthProvider : public AuthProvider { mutex mu_; string current_token_ GUARDED_BY(mu_); uint64 expiration_timestamp_sec_ GUARDED_BY(mu_) = 0; + // The initial delay for exponential backoffs when retrying failed calls. + const int64 initial_retry_delay_usec_; TF_DISALLOW_COPY_AND_ASSIGN(GoogleAuthProvider); }; diff --git a/tensorflow/core/platform/cloud/google_auth_provider_test.cc b/tensorflow/core/platform/cloud/google_auth_provider_test.cc index 3626528d5c987f..4281c6c73738db 100644 --- a/tensorflow/core/platform/cloud/google_auth_provider_test.cc +++ b/tensorflow/core/platform/cloud/google_auth_provider_test.cc @@ -62,7 +62,20 @@ class FakeOAuthClient : public OAuthClient { } // namespace -TEST(GoogleAuthProvider, EnvironmentVariable_Caching) { +class GoogleAuthProviderTest : public ::testing::Test { + protected: + void SetUp() override { ClearEnvVars(); } + + void TearDown() override { ClearEnvVars(); } + + void ClearEnvVars() { + unsetenv("GOOGLE_APPLICATION_CREDENTIALS"); + unsetenv("CLOUDSDK_CONFIG"); + unsetenv("GOOGLE_AUTH_TOKEN_FOR_TESTING"); + } +}; + +TEST_F(GoogleAuthProviderTest, EnvironmentVariable_Caching) { setenv("GOOGLE_APPLICATION_CREDENTIALS", io::JoinPath( io::JoinPath(testing::TensorFlowSrcRoot(), kTestData).c_str(), @@ -80,7 +93,7 @@ TEST(GoogleAuthProvider, EnvironmentVariable_Caching) { GoogleAuthProvider provider(std::unique_ptr(oauth_client), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - &env); + &env, 0); oauth_client->return_token = "fake-token"; oauth_client->return_expiration_timestamp = env.NowSeconds() + 3600; @@ -103,8 +116,7 @@ TEST(GoogleAuthProvider, EnvironmentVariable_Caching) { EXPECT_EQ("new-fake-token", token); } -TEST(GoogleAuthProvider, GCloudRefreshToken) { - setenv("GOOGLE_APPLICATION_CREDENTIALS", "", 1); +TEST_F(GoogleAuthProviderTest, GCloudRefreshToken) { setenv("CLOUDSDK_CONFIG", io::JoinPath(testing::TensorFlowSrcRoot(), kTestData).c_str(), 1); @@ -115,7 +127,7 @@ TEST(GoogleAuthProvider, GCloudRefreshToken) { GoogleAuthProvider provider(std::unique_ptr(oauth_client), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - &env); + &env, 0); oauth_client->return_token = "fake-token"; oauth_client->return_expiration_timestamp = env.NowSeconds() + 3600; @@ -127,10 +139,7 @@ TEST(GoogleAuthProvider, GCloudRefreshToken) { .asString()); } -TEST(GoogleAuthProvider, RunningOnGCE) { - setenv("GOOGLE_APPLICATION_CREDENTIALS", "", 1); - setenv("CLOUDSDK_CONFIG", "", 1); - +TEST_F(GoogleAuthProviderTest, RunningOnGCE) { auto oauth_client = new FakeOAuthClient; std::vector requests( {new FakeHttpRequest( @@ -143,6 +152,12 @@ TEST(GoogleAuthProvider, RunningOnGCE) { "expires_in": 3920, "token_type":"Bearer" })"), + // The first token refresh request fails and will be retried. + new FakeHttpRequest( + "Uri: http://metadata/computeMetadata/v1/instance/service-accounts" + "/default/token\n" + "Header Metadata-Flavor: Google\n", + "", errors::Unavailable("503"), 503), new FakeHttpRequest( "Uri: http://metadata/computeMetadata/v1/instance/service-accounts" "/default/token\n" @@ -158,7 +173,7 @@ TEST(GoogleAuthProvider, RunningOnGCE) { GoogleAuthProvider provider(std::unique_ptr(oauth_client), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - &env); + &env, 0); string token; TF_EXPECT_OK(provider.GetToken(&token)); @@ -175,10 +190,23 @@ TEST(GoogleAuthProvider, RunningOnGCE) { EXPECT_EQ("new-fake-gce-token", token); } -TEST(GoogleAuthProvider, NothingAvailable) { - setenv("GOOGLE_APPLICATION_CREDENTIALS", "", 1); - setenv("CLOUDSDK_CONFIG", "", 1); +TEST_F(GoogleAuthProviderTest, OverrideForTesting) { + setenv("GOOGLE_AUTH_TOKEN_FOR_TESTING", "tokenForTesting", 1); + + auto oauth_client = new FakeOAuthClient; + std::vector empty_requests; + FakeEnv env; + GoogleAuthProvider provider(std::unique_ptr(oauth_client), + std::unique_ptr( + new FakeHttpRequestFactory(&empty_requests)), + &env, 0); + + string token; + TF_EXPECT_OK(provider.GetToken(&token)); + EXPECT_EQ("tokenForTesting", token); +} +TEST_F(GoogleAuthProviderTest, NothingAvailable) { auto oauth_client = new FakeOAuthClient; std::vector requests({new FakeHttpRequest( @@ -191,7 +219,7 @@ TEST(GoogleAuthProvider, NothingAvailable) { GoogleAuthProvider provider(std::unique_ptr(oauth_client), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - &env); + &env, 0); string token; TF_EXPECT_OK(provider.GetToken(&token)); diff --git a/tensorflow/core/platform/cloud/http_request.cc b/tensorflow/core/platform/cloud/http_request.cc index 62453b49ca7cf9..aaff6c11f04abe 100644 --- a/tensorflow/core/platform/cloud/http_request.cc +++ b/tensorflow/core/platform/cloud/http_request.cc @@ -357,7 +357,7 @@ Status HttpRequest::Send() { libcurl_->curl_easy_setopt(curl_, CURLOPT_HEADERFUNCTION, &HttpRequest::HeaderCallback); - char error_buffer[CURL_ERROR_SIZE]; + char error_buffer[CURL_ERROR_SIZE] = {0}; libcurl_->curl_easy_setopt(curl_, CURLOPT_ERRORBUFFER, error_buffer); const auto curl_result = libcurl_->curl_easy_perform(curl_); @@ -367,30 +367,32 @@ Status HttpRequest::Send() { libcurl_->curl_easy_getinfo(curl_, CURLINFO_RESPONSE_CODE, &response_code_); + const auto& error_message = strings::StrCat( + "Error executing an HTTP request (HTTP response code ", response_code_, + ", error code ", curl_result, ", error message '", error_buffer, "')"); switch (response_code_) { case 200: // OK case 201: // Created case 204: // No Content case 206: // Partial Content if (curl_result != CURLE_OK) { - // UNAVAILABLE can be retried by the caller, e.g by RetryingFileSystem. - return errors::Unavailable( - strings::StrCat("libcurl failed with error code ", curl_result, - ": ", error_buffer)); + // UNAVAILABLE can be retried by the caller, e.g by + // RetryingFileSystem. + return errors::Unavailable(error_message); } return Status::OK(); case 401: case 403: - return errors::PermissionDenied("Not authorized to access the resource."); + return errors::PermissionDenied(error_message); case 404: - return errors::NotFound("The requested resource was not found."); + return errors::NotFound(error_message); case 416: // Requested Range Not Satisfiable response_buffer_->clear(); return Status::OK(); default: - // UNAVAILABLE can be retried by the caller, e.g by RetryingFileSystem. - return errors::Unavailable( - strings::StrCat("Unexpected response code ", response_code_)); + // UNAVAILABLE can be retried by the caller, e.g by + // RetryingFileSystem. + return errors::Unavailable(error_message); } } diff --git a/tensorflow/core/platform/cloud/http_request_test.cc b/tensorflow/core/platform/cloud/http_request_test.cc index 31ba3e337f90b9..e9bd9dab1af36c 100644 --- a/tensorflow/core/platform/cloud/http_request_test.cc +++ b/tensorflow/core/platform/cloud/http_request_test.cc @@ -29,14 +29,14 @@ const string kTestContent = "random original scratch content"; class FakeLibCurl : public LibCurl { public: FakeLibCurl(const string& response_content, uint64 response_code) - : response_content(response_content), response_code(response_code) {} + : response_content_(response_content), response_code_(response_code) {} FakeLibCurl(const string& response_content, uint64 response_code, const std::vector& response_headers) - : response_content(response_content), - response_code(response_code), - response_headers(response_headers) {} + : response_content_(response_content), + response_code_(response_code), + response_headers_(response_headers) {} CURL* curl_easy_init() override { - is_initialized = true; + is_initialized_ = true; // The reuslt just needs to be non-null. return reinterpret_cast(this); } @@ -44,10 +44,10 @@ class FakeLibCurl : public LibCurl { uint64 param) override { switch (option) { case CURLOPT_POST: - is_post = param; + is_post_ = param; break; case CURLOPT_PUT: - is_put = param; + is_put_ = param; break; default: break; @@ -63,28 +63,28 @@ class FakeLibCurl : public LibCurl { void* param) override { switch (option) { case CURLOPT_URL: - url = reinterpret_cast(param); + url_ = reinterpret_cast(param); break; case CURLOPT_RANGE: - range = reinterpret_cast(param); + range_ = reinterpret_cast(param); break; case CURLOPT_CUSTOMREQUEST: - custom_request = reinterpret_cast(param); + custom_request_ = reinterpret_cast(param); break; case CURLOPT_HTTPHEADER: - headers = reinterpret_cast*>(param); + headers_ = reinterpret_cast*>(param); break; case CURLOPT_ERRORBUFFER: - error_buffer = reinterpret_cast(param); + error_buffer_ = reinterpret_cast(param); break; case CURLOPT_WRITEDATA: - write_data = reinterpret_cast(param); + write_data_ = reinterpret_cast(param); break; case CURLOPT_HEADERDATA: - header_data = reinterpret_cast(param); + header_data_ = reinterpret_cast(param); break; case CURLOPT_READDATA: - read_data = reinterpret_cast(param); + read_data_ = reinterpret_cast(param); break; default: break; @@ -94,7 +94,7 @@ class FakeLibCurl : public LibCurl { CURLcode curl_easy_setopt(CURL* curl, CURLoption option, size_t (*param)(void*, size_t, size_t, FILE*)) override { - read_callback = param; + read_callback_ = param; return CURLE_OK; } CURLcode curl_easy_setopt(CURL* curl, CURLoption option, @@ -102,10 +102,10 @@ class FakeLibCurl : public LibCurl { void*)) override { switch (option) { case CURLOPT_WRITEFUNCTION: - write_callback = param; + write_callback_ = param; break; case CURLOPT_HEADERFUNCTION: - header_callback = param; + header_callback_ = param; break; default: break; @@ -113,30 +113,34 @@ class FakeLibCurl : public LibCurl { return CURLE_OK; } CURLcode curl_easy_perform(CURL* curl) override { - if (read_data) { + if (read_data_) { char buffer[3]; int bytes_read; - posted_content = ""; + posted_content_ = ""; do { - bytes_read = read_callback(buffer, 1, sizeof(buffer), read_data); - posted_content = - strings::StrCat(posted_content, StringPiece(buffer, bytes_read)); + bytes_read = read_callback_(buffer, 1, sizeof(buffer), read_data_); + posted_content_ = + strings::StrCat(posted_content_, StringPiece(buffer, bytes_read)); } while (bytes_read > 0); } - if (write_data) { - write_callback(response_content.c_str(), 1, response_content.size(), - write_data); + if (write_data_) { + write_callback_(response_content_.c_str(), 1, response_content_.size(), + write_data_); } - for (const auto& header : response_headers) { - header_callback(header.c_str(), 1, header.size(), header_data); + for (const auto& header : response_headers_) { + header_callback_(header.c_str(), 1, header.size(), header_data_); } - return curl_easy_perform_result; + if (error_buffer_) { + strncpy(error_buffer_, curl_easy_perform_error_message_.c_str(), + curl_easy_perform_error_message_.size() + 1); + } + return curl_easy_perform_result_; } CURLcode curl_easy_getinfo(CURL* curl, CURLINFO info, uint64* value) override { switch (info) { case CURLINFO_RESPONSE_CODE: - *value = response_code; + *value = response_code_; break; default: break; @@ -147,14 +151,14 @@ class FakeLibCurl : public LibCurl { double* value) override { switch (info) { case CURLINFO_SIZE_DOWNLOAD: - *value = response_content.size(); + *value = response_content_.size(); break; default: break; } return CURLE_OK; } - void curl_easy_cleanup(CURL* curl) override { is_cleaned_up = true; } + void curl_easy_cleanup(CURL* curl) override { is_cleaned_up_ = true; } curl_slist* curl_slist_append(curl_slist* list, const char* str) override { std::vector* v = list ? reinterpret_cast*>(list) : new std::vector(); @@ -173,8 +177,8 @@ class FakeLibCurl : public LibCurl { temp_str.replace(n, victim.size(), encoded); n += encoded.size(); } - char* out_char_str = - (char*)port::Malloc(sizeof(char) * temp_str.size() + 1); + char* out_char_str = reinterpret_cast( + port::Malloc(sizeof(char) * temp_str.size() + 1)); std::copy(temp_str.begin(), temp_str.end(), out_char_str); out_char_str[temp_str.size()] = '\0'; return out_char_str; @@ -185,32 +189,33 @@ class FakeLibCurl : public LibCurl { void curl_free(void* p) override { port::Free(p); } // Variables defining the behavior of this fake. - string response_content; - uint64 response_code; - std::vector response_headers; + string response_content_; + uint64 response_code_; + std::vector response_headers_; // Internal variables to store the libcurl state. - string url; - string range; - string custom_request; - char* error_buffer = nullptr; - bool is_initialized = false; - bool is_cleaned_up = false; - std::vector* headers = nullptr; - bool is_post = false; - bool is_put = false; - void* write_data = nullptr; - size_t (*write_callback)(const void* ptr, size_t size, size_t nmemb, - void* userdata) = nullptr; - void* header_data = nullptr; - size_t (*header_callback)(const void* ptr, size_t size, size_t nmemb, + string url_; + string range_; + string custom_request_; + char* error_buffer_ = nullptr; + bool is_initialized_ = false; + bool is_cleaned_up_ = false; + std::vector* headers_ = nullptr; + bool is_post_ = false; + bool is_put_ = false; + void* write_data_ = nullptr; + size_t (*write_callback_)(const void* ptr, size_t size, size_t nmemb, void* userdata) = nullptr; - FILE* read_data = nullptr; - size_t (*read_callback)(void* ptr, size_t size, size_t nmemb, - FILE* userdata) = &fread; + void* header_data_ = nullptr; + size_t (*header_callback_)(const void* ptr, size_t size, size_t nmemb, + void* userdata) = nullptr; + FILE* read_data_ = nullptr; + size_t (*read_callback_)(void* ptr, size_t size, size_t nmemb, + FILE* userdata) = &fread; // Outcome of performing the request. - string posted_content; - CURLcode curl_easy_perform_result = CURLE_OK; + string posted_content_; + CURLcode curl_easy_perform_result_ = CURLE_OK; + string curl_easy_perform_error_message_; }; TEST(HttpRequestTest, GetRequest) { @@ -232,13 +237,13 @@ TEST(HttpRequestTest, GetRequest) { EXPECT_EQ("get response", string(scratch.begin(), scratch.end())); // Check interactions with libcurl. - EXPECT_TRUE(libcurl.is_initialized); - EXPECT_EQ("http://www.testuri.com", libcurl.url); - EXPECT_EQ("100-199", libcurl.range); - EXPECT_EQ("", libcurl.custom_request); - EXPECT_EQ(1, libcurl.headers->size()); - EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers)[0]); - EXPECT_FALSE(libcurl.is_post); + EXPECT_TRUE(libcurl.is_initialized_); + EXPECT_EQ("http://www.testuri.com", libcurl.url_); + EXPECT_EQ("100-199", libcurl.range_); + EXPECT_EQ("", libcurl.custom_request_); + EXPECT_EQ(1, libcurl.headers_->size()); + EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]); + EXPECT_FALSE(libcurl.is_post_); EXPECT_EQ(200, http_request.GetResponseCode()); } @@ -259,19 +264,19 @@ TEST(HttpRequestTest, GetRequest_Empty) { EXPECT_TRUE(scratch.empty()); // Check interactions with libcurl. - EXPECT_TRUE(libcurl.is_initialized); - EXPECT_EQ("http://www.testuri.com", libcurl.url); - EXPECT_EQ("100-199", libcurl.range); - EXPECT_EQ("", libcurl.custom_request); - EXPECT_EQ(1, libcurl.headers->size()); - EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers)[0]); - EXPECT_FALSE(libcurl.is_post); + EXPECT_TRUE(libcurl.is_initialized_); + EXPECT_EQ("http://www.testuri.com", libcurl.url_); + EXPECT_EQ("100-199", libcurl.range_); + EXPECT_EQ("", libcurl.custom_request_); + EXPECT_EQ(1, libcurl.headers_->size()); + EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]); + EXPECT_FALSE(libcurl.is_post_); EXPECT_EQ(200, http_request.GetResponseCode()); } TEST(HttpRequestTest, GetRequest_RangeOutOfBound) { FakeLibCurl libcurl("get response", 416); - libcurl.curl_easy_perform_result = CURLE_WRITE_ERROR; + libcurl.curl_easy_perform_result_ = CURLE_WRITE_ERROR; HttpRequest http_request(&libcurl); TF_EXPECT_OK(http_request.Init()); @@ -290,7 +295,7 @@ TEST(HttpRequestTest, GetRequest_RangeOutOfBound) { TEST(HttpRequestTest, GetRequest_503) { FakeLibCurl libcurl("get response", 503); - libcurl.curl_easy_perform_result = CURLE_WRITE_ERROR; + libcurl.curl_easy_perform_result_ = CURLE_WRITE_ERROR; HttpRequest http_request(&libcurl); TF_EXPECT_OK(http_request.Init()); @@ -301,10 +306,35 @@ TEST(HttpRequestTest, GetRequest_503) { TF_EXPECT_OK(http_request.AddAuthBearerHeader("fake-bearer")); TF_EXPECT_OK(http_request.SetRange(100, 199)); TF_EXPECT_OK(http_request.SetResultBuffer(&scratch)); - EXPECT_EQ(error::UNAVAILABLE, http_request.Send().code()); + const auto& status = http_request.Send(); + EXPECT_EQ(error::UNAVAILABLE, status.code()); + EXPECT_EQ( + "Error executing an HTTP request (HTTP response code 503, " + "error code 23, error message '')", + status.error_message()); EXPECT_EQ(503, http_request.GetResponseCode()); } +TEST(HttpRequestTest, GetRequest_HttpCode0) { + FakeLibCurl libcurl("get response", 0); + libcurl.curl_easy_perform_result_ = CURLE_OPERATION_TIMEDOUT; + libcurl.curl_easy_perform_error_message_ = "Operation timed out"; + HttpRequest http_request(&libcurl); + TF_EXPECT_OK(http_request.Init()); + + std::vector scratch; + scratch.insert(scratch.end(), kTestContent.begin(), kTestContent.end()); + + TF_EXPECT_OK(http_request.SetUri("http://www.testuri.com")); + const auto& status = http_request.Send(); + EXPECT_EQ(error::UNAVAILABLE, status.code()); + EXPECT_EQ( + "Error executing an HTTP request (HTTP response code 0, " + "error code 28, error message 'Operation timed out')", + status.error_message()); + EXPECT_EQ(0, http_request.GetResponseCode()); +} + TEST(HttpRequestTest, ResponseHeaders) { FakeLibCurl libcurl( "get response", 200, @@ -336,14 +366,14 @@ TEST(HttpRequestTest, PutRequest_WithBody_FromFile) { TF_EXPECT_OK(http_request.Send()); // Check interactions with libcurl. - EXPECT_TRUE(libcurl.is_initialized); - EXPECT_EQ("http://www.testuri.com", libcurl.url); - EXPECT_EQ("", libcurl.custom_request); - EXPECT_EQ(2, libcurl.headers->size()); - EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers)[0]); - EXPECT_EQ("Content-Length: 17", (*libcurl.headers)[1]); - EXPECT_TRUE(libcurl.is_put); - EXPECT_EQ("post body content", libcurl.posted_content); + EXPECT_TRUE(libcurl.is_initialized_); + EXPECT_EQ("http://www.testuri.com", libcurl.url_); + EXPECT_EQ("", libcurl.custom_request_); + EXPECT_EQ(2, libcurl.headers_->size()); + EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]); + EXPECT_EQ("Content-Length: 17", (*libcurl.headers_)[1]); + EXPECT_TRUE(libcurl.is_put_); + EXPECT_EQ("post body content", libcurl.posted_content_); std::remove(content_filename.c_str()); } @@ -364,7 +394,7 @@ TEST(HttpRequestTest, PutRequest_WithBody_FromFile_NonZeroOffset) { TF_EXPECT_OK(http_request.Send()); // Check interactions with libcurl. - EXPECT_EQ("dy content", libcurl.posted_content); + EXPECT_EQ("dy content", libcurl.posted_content_); std::remove(content_filename.c_str()); } @@ -382,14 +412,14 @@ TEST(HttpRequestTest, PostRequest_WithBody_FromMemory) { TF_EXPECT_OK(http_request.Send()); // Check interactions with libcurl. - EXPECT_TRUE(libcurl.is_initialized); - EXPECT_EQ("http://www.testuri.com", libcurl.url); - EXPECT_EQ("", libcurl.custom_request); - EXPECT_EQ(2, libcurl.headers->size()); - EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers)[0]); - EXPECT_EQ("Content-Length: 17", (*libcurl.headers)[1]); - EXPECT_TRUE(libcurl.is_post); - EXPECT_EQ("post body content", libcurl.posted_content); + EXPECT_TRUE(libcurl.is_initialized_); + EXPECT_EQ("http://www.testuri.com", libcurl.url_); + EXPECT_EQ("", libcurl.custom_request_); + EXPECT_EQ(2, libcurl.headers_->size()); + EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]); + EXPECT_EQ("Content-Length: 17", (*libcurl.headers_)[1]); + EXPECT_TRUE(libcurl.is_post_); + EXPECT_EQ("post body content", libcurl.posted_content_); } TEST(HttpRequestTest, PostRequest_WithoutBody) { @@ -403,14 +433,14 @@ TEST(HttpRequestTest, PostRequest_WithoutBody) { TF_EXPECT_OK(http_request.Send()); // Check interactions with libcurl. - EXPECT_TRUE(libcurl.is_initialized); - EXPECT_EQ("http://www.testuri.com", libcurl.url); - EXPECT_EQ("", libcurl.custom_request); - EXPECT_EQ(2, libcurl.headers->size()); - EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers)[0]); - EXPECT_EQ("Content-Length: 0", (*libcurl.headers)[1]); - EXPECT_TRUE(libcurl.is_post); - EXPECT_EQ("", libcurl.posted_content); + EXPECT_TRUE(libcurl.is_initialized_); + EXPECT_EQ("http://www.testuri.com", libcurl.url_); + EXPECT_EQ("", libcurl.custom_request_); + EXPECT_EQ(2, libcurl.headers_->size()); + EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]); + EXPECT_EQ("Content-Length: 0", (*libcurl.headers_)[1]); + EXPECT_TRUE(libcurl.is_post_); + EXPECT_EQ("", libcurl.posted_content_); } TEST(HttpRequestTest, DeleteRequest) { @@ -424,12 +454,12 @@ TEST(HttpRequestTest, DeleteRequest) { TF_EXPECT_OK(http_request.Send()); // Check interactions with libcurl. - EXPECT_TRUE(libcurl.is_initialized); - EXPECT_EQ("http://www.testuri.com", libcurl.url); - EXPECT_EQ("DELETE", libcurl.custom_request); - EXPECT_EQ(1, libcurl.headers->size()); - EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers)[0]); - EXPECT_FALSE(libcurl.is_post); + EXPECT_TRUE(libcurl.is_initialized_); + EXPECT_EQ("http://www.testuri.com", libcurl.url_); + EXPECT_EQ("DELETE", libcurl.custom_request_); + EXPECT_EQ(1, libcurl.headers_->size()); + EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]); + EXPECT_FALSE(libcurl.is_post_); } TEST(HttpRequestTest, WrongSequenceOfCalls_NoUri) { diff --git a/tensorflow/core/platform/cloud/retrying_file_system.cc b/tensorflow/core/platform/cloud/retrying_file_system.cc index 7e98cafa8b8cb6..2da3104a72d0a7 100644 --- a/tensorflow/core/platform/cloud/retrying_file_system.cc +++ b/tensorflow/core/platform/cloud/retrying_file_system.cc @@ -17,6 +17,7 @@ limitations under the License. #include #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/random/random.h" +#include "tensorflow/core/platform/cloud/retrying_utils.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/file_system.h" @@ -24,44 +25,6 @@ namespace tensorflow { namespace { -// In case of failure, every call will be retried kMaxRetries times. -constexpr int kMaxRetries = 3; -// Maximum backoff time in microseconds. -constexpr int64 kMaximumBackoffMicroseconds = 32000000; - -bool IsRetriable(Status status) { - switch (status.code()) { - case error::UNAVAILABLE: - case error::DEADLINE_EXCEEDED: - case error::UNKNOWN: - return true; - default: - // OK also falls here. - return false; - } -} - -void WaitBeforeRetry(const int64 delay_micros) { - const int64 random_micros = random::New64() % 1000000; - Env::Default()->SleepForMicroseconds(std::min(delay_micros + random_micros, - kMaximumBackoffMicroseconds)); -} - -Status CallWithRetries(const std::function& f, - const int64 initial_delay_microseconds) { - int retries = 0; - while (true) { - auto status = f(); - if (!IsRetriable(status) || retries >= kMaxRetries) { - return status; - } - const int64 delay_micros = initial_delay_microseconds << retries; - if (delay_micros > 0) { - WaitBeforeRetry(delay_micros); - } - retries++; - } -} class RetryingRandomAccessFile : public RandomAccessFile { public: @@ -72,9 +35,10 @@ class RetryingRandomAccessFile : public RandomAccessFile { Status Read(uint64 offset, size_t n, StringPiece* result, char* scratch) const override { - return CallWithRetries(std::bind(&RandomAccessFile::Read, base_file_.get(), - offset, n, result, scratch), - initial_delay_microseconds_); + return RetryingUtils::CallWithRetries( + std::bind(&RandomAccessFile::Read, base_file_.get(), offset, n, result, + scratch), + initial_delay_microseconds_); } private: @@ -95,21 +59,24 @@ class RetryingWritableFile : public WritableFile { } Status Append(const StringPiece& data) override { - return CallWithRetries( + return RetryingUtils::CallWithRetries( std::bind(&WritableFile::Append, base_file_.get(), data), initial_delay_microseconds_); } Status Close() override { - return CallWithRetries(std::bind(&WritableFile::Close, base_file_.get()), - initial_delay_microseconds_); + return RetryingUtils::CallWithRetries( + std::bind(&WritableFile::Close, base_file_.get()), + initial_delay_microseconds_); } Status Flush() override { - return CallWithRetries(std::bind(&WritableFile::Flush, base_file_.get()), - initial_delay_microseconds_); + return RetryingUtils::CallWithRetries( + std::bind(&WritableFile::Flush, base_file_.get()), + initial_delay_microseconds_); } Status Sync() override { - return CallWithRetries(std::bind(&WritableFile::Sync, base_file_.get()), - initial_delay_microseconds_); + return RetryingUtils::CallWithRetries( + std::bind(&WritableFile::Sync, base_file_.get()), + initial_delay_microseconds_); } private: @@ -122,10 +89,10 @@ class RetryingWritableFile : public WritableFile { Status RetryingFileSystem::NewRandomAccessFile( const string& filename, std::unique_ptr* result) { std::unique_ptr base_file; - TF_RETURN_IF_ERROR(CallWithRetries(std::bind(&FileSystem::NewRandomAccessFile, - base_file_system_.get(), - filename, &base_file), - initial_delay_microseconds_)); + TF_RETURN_IF_ERROR(RetryingUtils::CallWithRetries( + std::bind(&FileSystem::NewRandomAccessFile, base_file_system_.get(), + filename, &base_file), + initial_delay_microseconds_)); result->reset(new RetryingRandomAccessFile(std::move(base_file), initial_delay_microseconds_)); return Status::OK(); @@ -134,10 +101,10 @@ Status RetryingFileSystem::NewRandomAccessFile( Status RetryingFileSystem::NewWritableFile( const string& filename, std::unique_ptr* result) { std::unique_ptr base_file; - TF_RETURN_IF_ERROR(CallWithRetries(std::bind(&FileSystem::NewWritableFile, - base_file_system_.get(), - filename, &base_file), - initial_delay_microseconds_)); + TF_RETURN_IF_ERROR(RetryingUtils::CallWithRetries( + std::bind(&FileSystem::NewWritableFile, base_file_system_.get(), filename, + &base_file), + initial_delay_microseconds_)); result->reset(new RetryingWritableFile(std::move(base_file), initial_delay_microseconds_)); return Status::OK(); @@ -146,10 +113,10 @@ Status RetryingFileSystem::NewWritableFile( Status RetryingFileSystem::NewAppendableFile( const string& filename, std::unique_ptr* result) { std::unique_ptr base_file; - TF_RETURN_IF_ERROR(CallWithRetries(std::bind(&FileSystem::NewAppendableFile, - base_file_system_.get(), - filename, &base_file), - initial_delay_microseconds_)); + TF_RETURN_IF_ERROR(RetryingUtils::CallWithRetries( + std::bind(&FileSystem::NewAppendableFile, base_file_system_.get(), + filename, &base_file), + initial_delay_microseconds_)); result->reset(new RetryingWritableFile(std::move(base_file), initial_delay_microseconds_)); return Status::OK(); @@ -157,9 +124,10 @@ Status RetryingFileSystem::NewAppendableFile( Status RetryingFileSystem::NewReadOnlyMemoryRegionFromFile( const string& filename, std::unique_ptr* result) { - return CallWithRetries(std::bind(&FileSystem::NewReadOnlyMemoryRegionFromFile, - base_file_system_.get(), filename, result), - initial_delay_microseconds_); + return RetryingUtils::CallWithRetries( + std::bind(&FileSystem::NewReadOnlyMemoryRegionFromFile, + base_file_system_.get(), filename, result), + initial_delay_microseconds_); } Status RetryingFileSystem::FileExists(const string& fname) { @@ -168,57 +136,59 @@ Status RetryingFileSystem::FileExists(const string& fname) { } Status RetryingFileSystem::Stat(const string& fname, FileStatistics* stat) { - return CallWithRetries( + return RetryingUtils::CallWithRetries( std::bind(&FileSystem::Stat, base_file_system_.get(), fname, stat), initial_delay_microseconds_); } Status RetryingFileSystem::GetChildren(const string& dir, std::vector* result) { - return CallWithRetries(std::bind(&FileSystem::GetChildren, - base_file_system_.get(), dir, result), - initial_delay_microseconds_); + return RetryingUtils::CallWithRetries( + std::bind(&FileSystem::GetChildren, base_file_system_.get(), dir, result), + initial_delay_microseconds_); } Status RetryingFileSystem::GetMatchingPaths(const string& pattern, std::vector* result) { - return CallWithRetries(std::bind(&FileSystem::GetMatchingPaths, - base_file_system_.get(), pattern, result), - initial_delay_microseconds_); + return RetryingUtils::CallWithRetries( + std::bind(&FileSystem::GetMatchingPaths, base_file_system_.get(), pattern, + result), + initial_delay_microseconds_); } Status RetryingFileSystem::DeleteFile(const string& fname) { - return CallWithRetries( + return RetryingUtils::CallWithRetries( std::bind(&FileSystem::DeleteFile, base_file_system_.get(), fname), initial_delay_microseconds_); } Status RetryingFileSystem::CreateDir(const string& dirname) { - return CallWithRetries( + return RetryingUtils::CallWithRetries( std::bind(&FileSystem::CreateDir, base_file_system_.get(), dirname), initial_delay_microseconds_); } Status RetryingFileSystem::DeleteDir(const string& dirname) { - return CallWithRetries( + return RetryingUtils::CallWithRetries( std::bind(&FileSystem::DeleteDir, base_file_system_.get(), dirname), initial_delay_microseconds_); } Status RetryingFileSystem::GetFileSize(const string& fname, uint64* file_size) { - return CallWithRetries(std::bind(&FileSystem::GetFileSize, - base_file_system_.get(), fname, file_size), - initial_delay_microseconds_); + return RetryingUtils::CallWithRetries( + std::bind(&FileSystem::GetFileSize, base_file_system_.get(), fname, + file_size), + initial_delay_microseconds_); } Status RetryingFileSystem::RenameFile(const string& src, const string& target) { - return CallWithRetries( + return RetryingUtils::CallWithRetries( std::bind(&FileSystem::RenameFile, base_file_system_.get(), src, target), initial_delay_microseconds_); } Status RetryingFileSystem::IsDirectory(const string& dirname) { - return CallWithRetries( + return RetryingUtils::CallWithRetries( std::bind(&FileSystem::IsDirectory, base_file_system_.get(), dirname), initial_delay_microseconds_); } @@ -226,7 +196,7 @@ Status RetryingFileSystem::IsDirectory(const string& dirname) { Status RetryingFileSystem::DeleteRecursively(const string& dirname, int64* undeleted_files, int64* undeleted_dirs) { - return CallWithRetries( + return RetryingUtils::CallWithRetries( std::bind(&FileSystem::DeleteRecursively, base_file_system_.get(), dirname, undeleted_files, undeleted_dirs), initial_delay_microseconds_); diff --git a/tensorflow/core/platform/cloud/retrying_file_system_test.cc b/tensorflow/core/platform/cloud/retrying_file_system_test.cc index ccb9def3cb330e..fe0b0a765007bf 100644 --- a/tensorflow/core/platform/cloud/retrying_file_system_test.cc +++ b/tensorflow/core/platform/cloud/retrying_file_system_test.cc @@ -23,6 +23,15 @@ namespace { typedef std::vector> ExpectedCalls; +ExpectedCalls CreateRetriableErrors(const string& method, int n) { + ExpectedCalls expected_calls; + for (int i = 0; i < n; i++) { + expected_calls.emplace_back(std::make_tuple( + method, errors::Unavailable(strings::StrCat("Retriable error #", i)))); + } + return expected_calls; +} + // A class to manage call expectations on mock implementations. class MockCallSequence { public: @@ -204,11 +213,7 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_SuccessWith3rdTry) { TEST(RetryingFileSystemTest, NewRandomAccessFile_AllRetriesFailed) { // Configure the mock base random access file. - ExpectedCalls expected_file_calls( - {std::make_tuple("Read", errors::Unavailable("Something is wrong")), - std::make_tuple("Read", errors::Unavailable("Wrong again")), - std::make_tuple("Read", errors::Unavailable("And again")), - std::make_tuple("Read", errors::Unavailable("Last error"))}); + ExpectedCalls expected_file_calls = CreateRetriableErrors("Read", 6); std::unique_ptr base_file( new MockRandomAccessFile(expected_file_calls)); @@ -227,7 +232,7 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_AllRetriesFailed) { // Use it and check the results. StringPiece result; char scratch[10]; - EXPECT_EQ("Last error", + EXPECT_EQ("Retriable error #5", random_access_file->Read(0, 10, &result, scratch).error_message()); } @@ -361,12 +366,8 @@ TEST(RetryingFileSystemTest, NewAppendableFile_SuccessWith3rdTry) { TEST(RetryingFileSystemTest, NewWritableFile_AllRetriesFailed) { // Configure the mock base random access file. - ExpectedCalls expected_file_calls( - {std::make_tuple("Sync", errors::Unavailable("Something is wrong")), - std::make_tuple("Sync", errors::Unavailable("Something is wrong again")), - std::make_tuple("Sync", errors::Unavailable("...and again")), - std::make_tuple("Sync", errors::Unavailable("And again")), - std::make_tuple("Close", Status::OK())}); + ExpectedCalls expected_file_calls = CreateRetriableErrors("Sync", 6); + expected_file_calls.emplace_back(std::make_tuple("Close", Status::OK())); std::unique_ptr base_file( new MockWritableFile(expected_file_calls)); @@ -383,7 +384,7 @@ TEST(RetryingFileSystemTest, NewWritableFile_AllRetriesFailed) { TF_EXPECT_OK(fs.NewWritableFile("filename.txt", &writable_file)); // Use it and check the results. - EXPECT_EQ("And again", writable_file->Sync().error_message()); + EXPECT_EQ("Retriable error #5", writable_file->Sync().error_message()); } TEST(RetryingFileSystemTest, @@ -401,21 +402,14 @@ TEST(RetryingFileSystemTest, } TEST(RetryingFileSystemTest, NewReadOnlyMemoryRegionFromFile_AllRetriesFailed) { - ExpectedCalls expected_fs_calls( - {std::make_tuple("NewReadOnlyMemoryRegionFromFile", - errors::Unavailable("Something is wrong")), - std::make_tuple("NewReadOnlyMemoryRegionFromFile", - errors::Unavailable("Something is wrong again")), - std::make_tuple("NewReadOnlyMemoryRegionFromFile", - errors::Unavailable("and again")), - std::make_tuple("NewReadOnlyMemoryRegionFromFile", - errors::Unavailable("Last error"))}); + ExpectedCalls expected_fs_calls = + CreateRetriableErrors("NewReadOnlyMemoryRegionFromFile", 6); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); std::unique_ptr result; - EXPECT_EQ("Last error", + EXPECT_EQ("Retriable error #5", fs.NewReadOnlyMemoryRegionFromFile("filename.txt", &result) .error_message()); } @@ -434,19 +428,14 @@ TEST(RetryingFileSystemTest, GetChildren_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, GetChildren_AllRetriesFailed) { - ExpectedCalls expected_fs_calls( - {std::make_tuple("GetChildren", - errors::Unavailable("Something is wrong")), - std::make_tuple("GetChildren", - errors::Unavailable("Something is wrong again")), - std::make_tuple("GetChildren", errors::Unavailable("And again")), - std::make_tuple("GetChildren", errors::Unavailable("Last error"))}); + ExpectedCalls expected_fs_calls = CreateRetriableErrors("GetChildren", 6); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); std::vector result; - EXPECT_EQ("Last error", fs.GetChildren("gs://path", &result).error_message()); + EXPECT_EQ("Retriable error #5", + fs.GetChildren("gs://path", &result).error_message()); } TEST(RetryingFileSystemTest, GetMatchingPaths_SuccessWith2ndTry) { @@ -463,19 +452,14 @@ TEST(RetryingFileSystemTest, GetMatchingPaths_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, GetMatchingPaths_AllRetriesFailed) { - ExpectedCalls expected_fs_calls( - {std::make_tuple("GetMatchingPaths", - errors::Unavailable("Something is wrong")), - std::make_tuple("GetMatchingPaths", - errors::Unavailable("Something is wrong again")), - std::make_tuple("GetMatchingPaths", errors::Unavailable("And again")), - std::make_tuple("GetMatchingPaths", errors::Unavailable("Last error"))}); + ExpectedCalls expected_fs_calls = + CreateRetriableErrors("GetMatchingPaths", 6); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); std::vector result; - EXPECT_EQ("Last error", + EXPECT_EQ("Retriable error #5", fs.GetMatchingPaths("gs://path/dir", &result).error_message()); } @@ -492,18 +476,14 @@ TEST(RetryingFileSystemTest, DeleteFile_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, DeleteFile_AllRetriesFailed) { - ExpectedCalls expected_fs_calls( - {std::make_tuple("DeleteFile", errors::Unavailable("Something is wrong")), - std::make_tuple("DeleteFile", - errors::Unavailable("Something is wrong again")), - std::make_tuple("DeleteFile", errors::Unavailable("And again")), - std::make_tuple("DeleteFile", errors::Unavailable("Last error"))}); + ExpectedCalls expected_fs_calls = CreateRetriableErrors("DeleteFile", 6); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); std::vector result; - EXPECT_EQ("Last error", fs.DeleteFile("gs://path/file.txt").error_message()); + EXPECT_EQ("Retriable error #5", + fs.DeleteFile("gs://path/file.txt").error_message()); } TEST(RetryingFileSystemTest, CreateDir_SuccessWith2ndTry) { @@ -519,18 +499,14 @@ TEST(RetryingFileSystemTest, CreateDir_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, CreateDir_AllRetriesFailed) { - ExpectedCalls expected_fs_calls( - {std::make_tuple("CreateDir", errors::Unavailable("Something is wrong")), - std::make_tuple("CreateDir", - errors::Unavailable("Something is wrong again")), - std::make_tuple("CreateDir", errors::Unavailable("And again")), - std::make_tuple("CreateDir", errors::Unavailable("Last error"))}); + ExpectedCalls expected_fs_calls = CreateRetriableErrors("CreateDir", 6); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); std::vector result; - EXPECT_EQ("Last error", fs.CreateDir("gs://path/newdir").error_message()); + EXPECT_EQ("Retriable error #5", + fs.CreateDir("gs://path/newdir").error_message()); } TEST(RetryingFileSystemTest, DeleteDir_SuccessWith2ndTry) { @@ -546,18 +522,14 @@ TEST(RetryingFileSystemTest, DeleteDir_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, DeleteDir_AllRetriesFailed) { - ExpectedCalls expected_fs_calls( - {std::make_tuple("DeleteDir", errors::Unavailable("Something is wrong")), - std::make_tuple("DeleteDir", - errors::Unavailable("Something is wrong again")), - std::make_tuple("DeleteDir", errors::Unavailable("And again")), - std::make_tuple("DeleteDir", errors::Unavailable("Last error"))}); + ExpectedCalls expected_fs_calls = CreateRetriableErrors("DeleteDir", 6); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); std::vector result; - EXPECT_EQ("Last error", fs.DeleteDir("gs://path/dir").error_message()); + EXPECT_EQ("Retriable error #5", + fs.DeleteDir("gs://path/dir").error_message()); } TEST(RetryingFileSystemTest, GetFileSize_SuccessWith2ndTry) { @@ -574,19 +546,13 @@ TEST(RetryingFileSystemTest, GetFileSize_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, GetFileSize_AllRetriesFailed) { - ExpectedCalls expected_fs_calls( - {std::make_tuple("GetFileSize", - errors::Unavailable("Something is wrong")), - std::make_tuple("GetFileSize", - errors::Unavailable("Something is wrong again")), - std::make_tuple("GetFileSize", errors::Unavailable("And again")), - std::make_tuple("GetFileSize", errors::Unavailable("Last error"))}); + ExpectedCalls expected_fs_calls = CreateRetriableErrors("GetFileSize", 6); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); uint64 size; - EXPECT_EQ("Last error", + EXPECT_EQ("Retriable error #5", fs.GetFileSize("gs://path/file.txt", &size).error_message()); } @@ -602,18 +568,12 @@ TEST(RetryingFileSystemTest, RenameFile_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, RenameFile_AllRetriesFailed) { - ExpectedCalls expected_fs_calls({ - std::make_tuple("RenameFile", errors::Unavailable("Something is wrong")), - std::make_tuple("RenameFile", - errors::Unavailable("Something is wrong again")), - std::make_tuple("RenameFile", errors::Unavailable("And again")), - std::make_tuple("RenameFile", errors::Unavailable("Last error")), - }); + ExpectedCalls expected_fs_calls = CreateRetriableErrors("RenameFile", 6); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); - EXPECT_EQ("Last error", + EXPECT_EQ("Retriable error #5", fs.RenameFile("old_name", "new_name").error_message()); } @@ -630,18 +590,13 @@ TEST(RetryingFileSystemTest, Stat_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, Stat_AllRetriesFailed) { - ExpectedCalls expected_fs_calls({ - std::make_tuple("Stat", errors::Unavailable("Something is wrong")), - std::make_tuple("Stat", errors::Unavailable("Something is wrong again")), - std::make_tuple("Stat", errors::Unavailable("And again")), - std::make_tuple("Stat", errors::Unavailable("Last error")), - }); + ExpectedCalls expected_fs_calls = CreateRetriableErrors("Stat", 6); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); FileStatistics stat; - EXPECT_EQ("Last error", fs.Stat("file_name", &stat).error_message()); + EXPECT_EQ("Retriable error #5", fs.Stat("file_name", &stat).error_message()); } TEST(RetryingFileSystemTest, IsDirectory_SuccessWith2ndTry) { @@ -657,18 +612,13 @@ TEST(RetryingFileSystemTest, IsDirectory_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, IsDirectory_AllRetriesFailed) { - ExpectedCalls expected_fs_calls( - {std::make_tuple("IsDirectory", - errors::Unavailable("Something is wrong")), - std::make_tuple("IsDirectory", - errors::Unavailable("Something is wrong again")), - std::make_tuple("IsDirectory", errors::Unavailable("And again")), - std::make_tuple("IsDirectory", errors::Unavailable("Last error"))}); + ExpectedCalls expected_fs_calls = CreateRetriableErrors("IsDirectory", 6); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); - EXPECT_EQ("Last error", fs.IsDirectory("gs://path/dir").error_message()); + EXPECT_EQ("Retriable error #5", + fs.IsDirectory("gs://path/dir").error_message()); } TEST(RetryingFileSystemTest, DeleteRecursively_SuccessWith2ndTry) { @@ -686,21 +636,15 @@ TEST(RetryingFileSystemTest, DeleteRecursively_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, DeleteRecursively_AllRetriesFailed) { - ExpectedCalls expected_fs_calls( - {std::make_tuple("DeleteRecursively", - errors::Unavailable("Something is wrong")), - std::make_tuple("DeleteRecursively", - errors::Unavailable("Something is wrong again")), - std::make_tuple("DeleteRecursively", errors::Unavailable("And again")), - std::make_tuple("DeleteRecursively", - errors::Unavailable("Last error"))}); + ExpectedCalls expected_fs_calls = + CreateRetriableErrors("DeleteRecursively", 6); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); int64 undeleted_files, undeleted_dirs; EXPECT_EQ( - "Last error", + "Retriable error #5", fs.DeleteRecursively("gs://path/dir", &undeleted_files, &undeleted_dirs) .error_message()); } diff --git a/tensorflow/core/platform/cloud/retrying_utils.cc b/tensorflow/core/platform/cloud/retrying_utils.cc new file mode 100644 index 00000000000000..dc05f9b62bee66 --- /dev/null +++ b/tensorflow/core/platform/cloud/retrying_utils.cc @@ -0,0 +1,67 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/platform/cloud/retrying_utils.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/random/random.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/file_system.h" + +namespace tensorflow { + +namespace { + +// In case of failure, every call will be retried kMaxRetries times. +constexpr int kMaxRetries = 5; +// Maximum backoff time in microseconds. +constexpr int64 kMaximumBackoffMicroseconds = 32000000; // 32 seconds. + +bool IsRetriable(Status status) { + switch (status.code()) { + case error::UNAVAILABLE: + case error::DEADLINE_EXCEEDED: + case error::UNKNOWN: + return true; + default: + // OK also falls here. + return false; + } +} + +void WaitBeforeRetry(const int64 delay_micros) { + const int64 random_micros = random::New64() % 1000000; + Env::Default()->SleepForMicroseconds(std::min(delay_micros + random_micros, + kMaximumBackoffMicroseconds)); +} + +} // namespace + +Status RetryingUtils::CallWithRetries(const std::function& f, + const int64 initial_delay_microseconds) { + int retries = 0; + while (true) { + auto status = f(); + if (!IsRetriable(status) || retries >= kMaxRetries) { + return status; + } + const int64 delay_micros = initial_delay_microseconds << retries; + if (delay_micros > 0) { + WaitBeforeRetry(delay_micros); + } + retries++; + } +} + +} // namespace tensorflow diff --git a/tensorflow/core/platform/cloud/retrying_utils.h b/tensorflow/core/platform/cloud/retrying_utils.h new file mode 100644 index 00000000000000..19bbf51725eb0e --- /dev/null +++ b/tensorflow/core/platform/cloud/retrying_utils.h @@ -0,0 +1,39 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef THIRD_PARTY_TENSORFLOW_CORE_PLATFORM_CLOUD_RETRYING_UTILS_H_ +#define THIRD_PARTY_TENSORFLOW_CORE_PLATFORM_CLOUD_RETRYING_UTILS_H_ + +#include +#include "tensorflow/core/lib/core/status.h" + +namespace tensorflow { + +class RetryingUtils { + public: + /// \brief Retries the function in case of failure with exponential backoff. + /// + /// The provided callback is retried with an exponential backoff until it + /// returns OK or a non-retriable error status. + /// If initial_delay_microseconds is zero, no delays will be made between + /// retries. + /// If all retries failed, returns the last error status. + static Status CallWithRetries(const std::function& f, + const int64 initial_delay_microseconds); +}; + +} // namespace tensorflow + +#endif // THIRD_PARTY_TENSORFLOW_CORE_PLATFORM_CLOUD_RETRYING_UTILS_H_ diff --git a/tensorflow/python/ops/cloud/bigquery_reader_ops_test.py b/tensorflow/python/ops/cloud/bigquery_reader_ops_test.py index 196991f68a7f54..141b0af901813f 100644 --- a/tensorflow/python/ops/cloud/bigquery_reader_ops_test.py +++ b/tensorflow/python/ops/cloud/bigquery_reader_ops_test.py @@ -19,6 +19,7 @@ from __future__ import print_function import json +import os import re import threading @@ -175,6 +176,9 @@ def setUp(self): self.server.start() logging.info("server address is %s:%s", self.server.httpd.server_address[0], self.server.httpd.server_address[1]) + # An override to bypass the GCP auth token retrieval logic + # in google_auth_provider.cc. + os.environ["GOOGLE_AUTH_TOKEN_FOR_TESTING"] = "not-used" def tearDown(self): self.server.shutdown() From 96950778d529ec7f410ed8770ec02412139a3151 Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Wed, 22 Feb 2017 18:31:14 -0800 Subject: [PATCH 34/50] Change pip.sh to take Bazel flags directly so AVX packages are built properly. (#7797) * Fix flags for the pip package * Remove condition * Add shift * Small nit --- tensorflow/tools/ci_build/README.md | 4 +-- tensorflow/tools/ci_build/builds/pip.sh | 30 ++++++++++--------- tensorflow/tools/ci_build/ci_build.sh | 2 +- .../tools/ci_build/ci_parameterized_build.sh | 7 +---- 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/tensorflow/tools/ci_build/README.md b/tensorflow/tools/ci_build/README.md index 5c90fceaf704c0..eede1f5fac5835 100644 --- a/tensorflow/tools/ci_build/README.md +++ b/tensorflow/tools/ci_build/README.md @@ -53,10 +53,10 @@ tensorflow/tools/ci_build/ci_build.sh CPU bazel test //tensorflow/... tensorflow/tools/ci_build/ci_build.sh GPU bazel test -c opt --config=cuda //tensorflow/... # build pip with gpu support -tensorflow/tools/ci_build/ci_build.sh GPU tensorflow/tools/ci_build/builds/pip.sh GPU +tensorflow/tools/ci_build/ci_build.sh GPU tensorflow/tools/ci_build/builds/pip.sh GPU -c opt --config=cuda # build and run gpu tests using python 3 -CI_DOCKER_EXTRA_PARAMS="-e CI_BUILD_PYTHON=python3" tensorflow/tools/ci_build/ci_build.sh GPU tensorflow/tools/ci_build/builds/pip.sh GPU +CI_DOCKER_EXTRA_PARAMS="-e CI_BUILD_PYTHON=python3" tensorflow/tools/ci_build/ci_build.sh GPU tensorflow/tools/ci_build/builds/pip.sh GPU -c opt --config=cuda # build android example app tensorflow/tools/ci_build/ci_build.sh ANDROID tensorflow/tools/ci_build/builds/android.sh diff --git a/tensorflow/tools/ci_build/builds/pip.sh b/tensorflow/tools/ci_build/builds/pip.sh index 1e55ad01245e80..2b0bb1b6c03ac2 100755 --- a/tensorflow/tools/ci_build/builds/pip.sh +++ b/tensorflow/tools/ci_build/builds/pip.sh @@ -19,8 +19,7 @@ # The PIP installation is done using the --user flag. # # Usage: -# pip.sh CONTAINER_TYPE [--mavx] [--mavx2] -# [--test_tutorials] [--integration_tests] +# pip.sh CONTAINER_TYPE [--test_tutorials] [--integration_tests] [bazel flags] # # When executing the Python unit tests, the script obeys the shell # variables: TF_BUILD_BAZEL_CLEAN, TF_BUILD_INSTALL_EXTRA_PIP_PACKAGES, @@ -39,8 +38,7 @@ # If NO_TEST_USER_OPS has any non-empty and non-0 value, the testing of user- # defined ops against the installation will be skipped. # -# Use --mavx or --mavx2 to let bazel use --copt=-mavx or --copt=-mavx2 options -# while building the pip package, respectively. +# Any flags not listed in the usage above will be passed directly to Bazel. # # If the --test_tutorials flag is set, it will cause the script to run the # tutorial tests (see test_tutorials.sh) after the PIP @@ -49,6 +47,11 @@ # to run. # +# Helper function: Strip leading and trailing whitespaces +str_strip () { + echo -e "$1" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' +} + # Fixed naming patterns for wheel (.whl) files given different python versions if [[ $(uname) == "Linux" ]]; then declare -A WHL_TAGS @@ -66,6 +69,7 @@ source "${SCRIPT_DIR}/builds_common.sh" # Get the command line arguments CONTAINER_TYPE=$( echo "$1" | tr '[:upper:]' '[:lower:]' ) +shift if [[ ! -z "${TF_BUILD_BAZEL_CLEAN}" ]] && \ [[ "${TF_BUILD_BAZEL_CLEAN}" != "0" ]]; then @@ -82,16 +86,14 @@ fi DO_TEST_TUTORIALS=0 DO_INTEGRATION_TESTS=0 -MAVX_FLAG="" +BAZEL_FLAGS="" while true; do if [[ "${1}" == "--test_tutorials" ]]; then DO_TEST_TUTORIALS=1 elif [[ "${1}" == "--integration_tests" ]]; then DO_INTEGRATION_TESTS=1 - elif [[ "${1}" == "--mavx" ]]; then - MAVX_FLAG="--copt=-mavx" - elif [[ "${1}" == "--mavx2" ]]; then - MAVX_FLAG="--copt=-mavx2" + else + BAZEL_FLAGS="${BAZEL_FLAGS} ${1}" fi shift @@ -100,18 +102,18 @@ while true; do fi done -if [[ ! -z "${MAVX_FLAG}" ]]; then - echo "Using MAVX flag: ${MAVX_FLAG}" -fi +BAZEL_FLAGS=$(str_strip "${BAZEL_FLAGS}") + +echo "Using Bazel flags: ${BAZEL_FLAGS}" PIP_BUILD_TARGET="//tensorflow/tools/pip_package:build_pip_package" GPU_FLAG="" if [[ ${CONTAINER_TYPE} == "cpu" ]] || \ [[ ${CONTAINER_TYPE} == "debian.jessie.cpu" ]]; then - bazel build -c opt ${MAVX_FLAG} ${PIP_BUILD_TARGET} || \ + bazel build ${BAZEL_FLAGS} ${PIP_BUILD_TARGET} || \ die "Build failed." elif [[ ${CONTAINER_TYPE} == "gpu" ]]; then - bazel build -c opt --config=cuda ${MAVX_FLAG} ${PIP_BUILD_TARGET} || \ + bazel build ${BAZEL_FLAGS} ${PIP_BUILD_TARGET} || \ die "Build failed." GPU_FLAG="--gpu" else diff --git a/tensorflow/tools/ci_build/ci_build.sh b/tensorflow/tools/ci_build/ci_build.sh index 3697fd46a0ea08..f0fa8a9381060a 100755 --- a/tensorflow/tools/ci_build/ci_build.sh +++ b/tensorflow/tools/ci_build/ci_build.sh @@ -26,7 +26,7 @@ # directory as this script will be used. # # COMMAND: Command to be executed in the docker container, e.g., -# tensorflow/tools/ci_build/builds/pip.sh gpu +# tensorflow/tools/ci_build/builds/pip.sh gpu -c opt --config=cuda SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${SCRIPT_DIR}/builds/builds_common.sh" diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh index b7d5a8d144b5f4..ec243f8d0e8455 100755 --- a/tensorflow/tools/ci_build/ci_parameterized_build.sh +++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh @@ -376,12 +376,7 @@ if [[ ${TF_BUILD_IS_PIP} == "pip" ]] || exit 0 fi - PIP_MAIN_CMD="${MAIN_CMD} ${PIP_CMD} ${CTYPE} ${EXTRA_AGRS}" - - # Add flag for mavx/mavx2 - if [[ ! -z "${TF_BUILD_MAVX}" ]]; then - PIP_MAIN_CMD="${PIP_MAIN_CMD} --${TF_BUILD_MAVX}" - fi + PIP_MAIN_CMD="${MAIN_CMD} ${PIP_CMD} ${CTYPE} ${EXTRA_ARGS} ${OPT_FLAG}" # Add flag for integration tests if [[ ! -z "${TF_BUILD_INTEGRATION_TESTS}" ]] && From 1ded48061f8899c899a16edfe370bc09fe485c33 Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Thu, 23 Feb 2017 17:08:07 -0800 Subject: [PATCH 35/50] Add a non-debug avx2 fma build --- tensorflow/tools/ci_build/ci_parameterized_build.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh index ec243f8d0e8455..f1c8b51b376f21 100755 --- a/tensorflow/tools/ci_build/ci_parameterized_build.sh +++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh @@ -84,8 +84,8 @@ # support for Google Cloud Platform (GCP), which is # enabled by default. # TF_BUILD_OPTIONS: -# (FASTBUILD | OPT | OPTDBG | MAVX | MAVX2 | MAVX_DBG | -# MAVX2_FMA_DBG) +# (FASTBUILD | OPT | OPTDBG | MAVX | MAVX2 | MAVX2_FMA | +# MAVX_DBG | MAVX2_FMA_DBG) # Use the specified configurations when building. # When set, overrides TF_BUILD_IS_OPT and TF_BUILD_MAVX # options, as this will replace the two. @@ -312,6 +312,9 @@ else MAVX2) OPT_FLAG="${OPT_FLAG} -c opt --copt=-mavx2" ;; + MAVX2_FMA) + OPT_FLAG="${OPT_FLAG} -c opt --copt=-mavx2 --copt=-mfma" + ;; MAVX2_FMA_DBG) OPT_FLAG="${OPT_FLAG} -c opt --copt=-g --copt=-mavx2 --copt=-mfma" ;; From d4fd3b106c62787206ac2780d202abe2af3ebdb7 Mon Sep 17 00:00:00 2001 From: Alexey Surkov Date: Fri, 3 Mar 2017 11:08:42 -0800 Subject: [PATCH 36/50] Merge --- tensorflow/core/platform/cloud/BUILD | 14 ++ .../core/platform/cloud/gcs_file_system.cc | 88 ++++----- .../core/platform/cloud/gcs_file_system.h | 7 +- .../platform/cloud/gcs_file_system_test.cc | 170 +++++++++--------- .../cloud/retrying_file_system_test.cc | 72 +++++--- .../core/platform/cloud/retrying_utils.cc | 20 ++- .../core/platform/cloud/retrying_utils.h | 4 + .../platform/cloud/retrying_utils_test.cc | 73 ++++++++ 8 files changed, 287 insertions(+), 161 deletions(-) create mode 100644 tensorflow/core/platform/cloud/retrying_utils_test.cc diff --git a/tensorflow/core/platform/cloud/BUILD b/tensorflow/core/platform/cloud/BUILD index 84b0bda896c0c0..4df88b8d37439c 100644 --- a/tensorflow/core/platform/cloud/BUILD +++ b/tensorflow/core/platform/cloud/BUILD @@ -38,6 +38,7 @@ cc_library( ":google_auth_provider", ":http_request", ":retrying_file_system", + ":retrying_utils", ":time_util", "//tensorflow/core:framework_headers_lib", "//tensorflow/core:lib_internal", @@ -239,3 +240,16 @@ tf_cc_test( "//tensorflow/core:test_main", ], ) + +tf_cc_test( + name = "retrying_utils_test", + size = "small", + srcs = ["retrying_utils_test.cc"], + deps = [ + ":retrying_utils", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) diff --git a/tensorflow/core/platform/cloud/gcs_file_system.cc b/tensorflow/core/platform/cloud/gcs_file_system.cc index ab56ad09d11580..6b9e714b432c8a 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system.cc @@ -30,6 +30,7 @@ limitations under the License. #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/cloud/google_auth_provider.h" +#include "tensorflow/core/platform/cloud/retrying_utils.h" #include "tensorflow/core/platform/cloud/time_util.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/mutex.h" @@ -240,7 +241,11 @@ class GcsRandomAccessFile : public RandomAccessFile { buffer_.reserve(desired_buffer_size); } + // Shift the offset and clear the buffer so that the state stays + // consistent if loading from GCS fails. buffer_start_offset_ = offset; + buffer_.clear(); + TF_RETURN_IF_ERROR(LoadBufferFromGCS()); // Set the results. @@ -302,13 +307,13 @@ class GcsWritableFile : public WritableFile { GcsWritableFile(const string& bucket, const string& object, AuthProvider* auth_provider, HttpRequest::Factory* http_request_factory, - int32 max_upload_attempts) + int64 initial_retry_delay_usec) : bucket_(bucket), object_(object), auth_provider_(auth_provider), http_request_factory_(http_request_factory), sync_needed_(true), - max_upload_attempts_(max_upload_attempts) { + initial_retry_delay_usec_(initial_retry_delay_usec) { if (GetTmpFilename(&tmp_content_filename_).ok()) { outfile_.open(tmp_content_filename_, std::ofstream::binary | std::ofstream::app); @@ -324,13 +329,13 @@ class GcsWritableFile : public WritableFile { AuthProvider* auth_provider, const string& tmp_content_filename, HttpRequest::Factory* http_request_factory, - int32 max_upload_attempts) + int64 initial_retry_delay_usec) : bucket_(bucket), object_(object), auth_provider_(auth_provider), http_request_factory_(http_request_factory), sync_needed_(true), - max_upload_attempts_(max_upload_attempts) { + initial_retry_delay_usec_(initial_retry_delay_usec) { tmp_content_filename_ = tmp_content_filename; outfile_.open(tmp_content_filename_, std::ofstream::binary | std::ofstream::app); @@ -388,41 +393,40 @@ class GcsWritableFile : public WritableFile { string session_uri; TF_RETURN_IF_ERROR(CreateNewUploadSession(&session_uri)); uint64 already_uploaded = 0; - for (int attempt = 0; attempt < max_upload_attempts_; attempt++) { - if (attempt > 0) { - bool completed; - TF_RETURN_IF_ERROR(RequestUploadSessionStatus(session_uri, &completed, - &already_uploaded)); - if (completed) { - // It's unclear why UploadToSession didn't return OK in the previous - // attempt, but GCS reports that the file is fully uploaded, - // so succeed. - return Status::OK(); - } - } - const Status upload_status = - UploadToSession(session_uri, already_uploaded); - if (upload_status.ok()) { + bool first_attempt = true; + const Status upload_status = RetryingUtils::CallWithRetries( + [&first_attempt, &already_uploaded, &session_uri, this]() { + if (!first_attempt) { + bool completed; + TF_RETURN_IF_ERROR(RequestUploadSessionStatus( + session_uri, &completed, &already_uploaded)); + if (completed) { + // It's unclear why UploadToSession didn't return OK in the + // previous attempt, but GCS reports that the file is fully + // uploaded, so succeed. + return Status::OK(); + } + } + first_attempt = false; + return UploadToSession(session_uri, already_uploaded); + }, + initial_retry_delay_usec_); + switch (upload_status.code()) { + case errors::Code::OK: return Status::OK(); - } - switch (upload_status.code()) { - case errors::Code::NOT_FOUND: - // GCS docs recommend retrying the whole upload. We're relying on the - // RetryingFileSystem to retry the Sync() call. - return errors::Unavailable("Could not upload gs://", bucket_, "/", - object_); - case errors::Code::UNAVAILABLE: - // The upload can be resumed, but GCS docs recommend an exponential - // back-off. - Env::Default()->SleepForMicroseconds(kUploadRetryDelayMicros - << attempt); - break; - default: - // Something unexpected happen, fail. - return upload_status; - } + case errors::Code::NOT_FOUND: + // GCS docs recommend retrying the whole upload. We're relying on the + // RetryingFileSystem to retry the Sync() call. + return errors::Unavailable("Could not upload gs://", bucket_, "/", + object_); + case errors::Code::UNAVAILABLE: + // Return ABORTED so that RetryingFileSystem doesn't retry again. + return errors::Aborted("Upload gs://", bucket_, "/", object_, + " failed."); + default: + // Something unexpected happen, fail. + return upload_status; } - return errors::Aborted("Upload gs://", bucket_, "/", object_, " failed."); } Status CheckWritable() const { @@ -571,7 +575,7 @@ class GcsWritableFile : public WritableFile { std::ofstream outfile_; HttpRequest::Factory* http_request_factory_; bool sync_needed_; // whether there is buffered data that needs to be synced - int32 max_upload_attempts_; + int64 initial_retry_delay_usec_; }; class GcsReadOnlyMemoryRegion : public ReadOnlyMemoryRegion { @@ -594,11 +598,11 @@ GcsFileSystem::GcsFileSystem() GcsFileSystem::GcsFileSystem( std::unique_ptr auth_provider, std::unique_ptr http_request_factory, - size_t read_ahead_bytes, int32 max_upload_attempts) + size_t read_ahead_bytes, int64 initial_retry_delay_usec) : auth_provider_(std::move(auth_provider)), http_request_factory_(std::move(http_request_factory)), read_ahead_bytes_(read_ahead_bytes), - max_upload_attempts_(max_upload_attempts) {} + initial_retry_delay_usec_(initial_retry_delay_usec) {} Status GcsFileSystem::NewRandomAccessFile( const string& fname, std::unique_ptr* result) { @@ -616,7 +620,7 @@ Status GcsFileSystem::NewWritableFile(const string& fname, TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object)); result->reset(new GcsWritableFile(bucket, object, auth_provider_.get(), http_request_factory_.get(), - max_upload_attempts_)); + initial_retry_delay_usec_)); return Status::OK(); } @@ -656,7 +660,7 @@ Status GcsFileSystem::NewAppendableFile(const string& fname, TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object)); result->reset(new GcsWritableFile( bucket, object, auth_provider_.get(), old_content_filename, - http_request_factory_.get(), max_upload_attempts_)); + http_request_factory_.get(), initial_retry_delay_usec_)); return Status::OK(); } diff --git a/tensorflow/core/platform/cloud/gcs_file_system.h b/tensorflow/core/platform/cloud/gcs_file_system.h index 28e6a85a089425..6a6437f070e348 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.h +++ b/tensorflow/core/platform/cloud/gcs_file_system.h @@ -35,7 +35,7 @@ class GcsFileSystem : public FileSystem { GcsFileSystem(); GcsFileSystem(std::unique_ptr auth_provider, std::unique_ptr http_request_factory, - size_t read_ahead_bytes, int32 max_upload_attempts); + size_t read_ahead_bytes, int64 initial_retry_delay_usec); Status NewRandomAccessFile( const string& filename, @@ -114,9 +114,8 @@ class GcsFileSystem : public FileSystem { // RandomAccessFile implementation. Defaults to 256Mb. const size_t read_ahead_bytes_ = 256 * 1024 * 1024; - // The max number of attempts to upload a file to GCS using the resumable - // upload API. - const int32 max_upload_attempts_ = 5; + // The initial delay for exponential backoffs when retrying failed calls. + const int64 initial_retry_delay_usec_ = 1000000L; TF_DISALLOW_COPY_AND_ASSIGN(GcsFileSystem); }; diff --git a/tensorflow/core/platform/cloud/gcs_file_system_test.cc b/tensorflow/core/platform/cloud/gcs_file_system_test.cc index 84f219616a96ba..47675976a8eeb6 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system_test.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system_test.cc @@ -45,7 +45,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoReadAhead) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -79,7 +79,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoReadAhead_differentN) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -136,7 +136,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithReadAhead) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 5 /* read ahead bytes */, 5 /* max upload attempts */); + 5 /* read ahead bytes */, 0 /* initial retry delay */); char scratch[100]; StringPiece result; @@ -197,7 +197,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoObjectName) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 5 /* read ahead bytes */, 5 /* max upload attempts */); + 5 /* read ahead bytes */, 0 /* initial retry delay */); std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -221,7 +221,7 @@ TEST(GcsFileSystemTest, NewWritableFile) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::unique_ptr file; TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); @@ -275,7 +275,7 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::unique_ptr file; TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); @@ -307,7 +307,7 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceedsOnGetStatus) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::unique_ptr file; TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); @@ -330,32 +330,36 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) { "Auth Token: fake_token\n" "Header Content-Range: bytes 0-16/17\n" "Put body: content1,content2\n", - "", errors::Unavailable("503"), 503), - new FakeHttpRequest("Uri: https://custom/upload/location\n" - "Auth Token: fake_token\n" - "Header Content-Range: bytes */17\n" - "Put: yes\n", - "", errors::Unavailable("308"), nullptr, - {{"Range", "0-10"}}, 308), - new FakeHttpRequest("Uri: https://custom/upload/location\n" - "Auth Token: fake_token\n" - "Header Content-Range: bytes 11-16/17\n" - "Put body: ntent2\n", - "", errors::Unavailable("503"), 503), - // These calls will be made in the Close() attempt from the destructor. - // Letting the destructor succeed. - new FakeHttpRequest( - "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" - "uploadType=resumable&name=path%2Fwriteable.txt\n" - "Auth Token: fake_token\n" - "Header X-Upload-Content-Length: 17\n" - "Post: yes\n", - "", {{"Location", "https://custom/upload/location"}}), - new FakeHttpRequest("Uri: https://custom/upload/location\n" - "Auth Token: fake_token\n" - "Header Content-Range: bytes 0-16/17\n" - "Put body: content1,content2\n", - "")}); + "", errors::Unavailable("503"), 503)}); + for (int i = 0; i < 10; i++) { + requests.emplace_back(new FakeHttpRequest( + "Uri: https://custom/upload/location\n" + "Auth Token: fake_token\n" + "Header Content-Range: bytes */17\n" + "Put: yes\n", + "", errors::Unavailable("308"), nullptr, {{"Range", "0-10"}}, 308)); + requests.emplace_back( + new FakeHttpRequest("Uri: https://custom/upload/location\n" + "Auth Token: fake_token\n" + "Header Content-Range: bytes 11-16/17\n" + "Put body: ntent2\n", + "", errors::Unavailable("503"), 503)); + } + // These calls will be made in the Close() attempt from the destructor. + // Letting the destructor succeed. + requests.emplace_back(new FakeHttpRequest( + "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" + "uploadType=resumable&name=path%2Fwriteable.txt\n" + "Auth Token: fake_token\n" + "Header X-Upload-Content-Length: 17\n" + "Post: yes\n", + "", {{"Location", "https://custom/upload/location"}})); + requests.emplace_back( + new FakeHttpRequest("Uri: https://custom/upload/location\n" + "Auth Token: fake_token\n" + "Header Content-Range: bytes 0-16/17\n" + "Put body: content1,content2\n", + "")); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), @@ -400,7 +404,7 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns404) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::unique_ptr file; TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); @@ -415,7 +419,7 @@ TEST(GcsFileSystemTest, NewWritableFile_NoObjectName) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 5 /* read ahead bytes */, 5 /* max upload attempts */); + 5 /* read ahead bytes */, 0 /* initial retry delay */); std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -444,7 +448,7 @@ TEST(GcsFileSystemTest, NewAppendableFile) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::unique_ptr file; TF_EXPECT_OK(fs.NewAppendableFile("gs://bucket/path/appendable.txt", &file)); @@ -458,7 +462,7 @@ TEST(GcsFileSystemTest, NewAppendableFile_NoObjectName) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 5 /* read ahead bytes */, 5 /* max upload attempts */); + 5 /* read ahead bytes */, 0 /* initial retry delay */); std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -484,7 +488,7 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::unique_ptr region; TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile( @@ -499,7 +503,7 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile_NoObjectName) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 5 /* read ahead bytes */, 5 /* max upload attempts */); + 5 /* read ahead bytes */, 0 /* initial retry delay */); std::unique_ptr region; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -516,7 +520,7 @@ TEST(GcsFileSystemTest, FileExists_YesAsObject) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt")); } @@ -538,7 +542,7 @@ TEST(GcsFileSystemTest, FileExists_YesAsFolder) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder")); } @@ -556,7 +560,7 @@ TEST(GcsFileSystemTest, FileExists_YesAsBucket) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); TF_EXPECT_OK(fs.FileExists("gs://bucket1")); TF_EXPECT_OK(fs.FileExists("gs://bucket1/")); @@ -578,7 +582,7 @@ TEST(GcsFileSystemTest, FileExists_NotAsObjectOrFolder) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); EXPECT_EQ(errors::Code::NOT_FOUND, fs.FileExists("gs://bucket/path/file1.txt").code()); @@ -597,7 +601,7 @@ TEST(GcsFileSystemTest, FileExists_NotAsBucket) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); EXPECT_EQ(errors::Code::INVALID_ARGUMENT, fs.FileExists("gs://bucket2/").code()); EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -614,7 +618,7 @@ TEST(GcsFileSystemTest, GetChildren_NoItems) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -635,7 +639,7 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -657,7 +661,7 @@ TEST(GcsFileSystemTest, GetChildren_SelfDirectoryMarker) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -678,7 +682,7 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles_NoSlash) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children)); @@ -696,7 +700,7 @@ TEST(GcsFileSystemTest, GetChildren_Root) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket-a-b-c", &children)); @@ -714,7 +718,7 @@ TEST(GcsFileSystemTest, GetChildren_Empty) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -747,7 +751,7 @@ TEST(GcsFileSystemTest, GetChildren_Pagination) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children)); @@ -767,7 +771,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_NoWildcard) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::vector result; TF_EXPECT_OK( @@ -788,7 +792,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_BucketAndWildcard) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", &result)); @@ -810,7 +814,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_Matches) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file2.txt", &result)); @@ -829,7 +833,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_SelfDirectoryMarker) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", &result)); @@ -848,7 +852,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file3.txt", &result)); @@ -860,7 +864,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_OnlyWildcard) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); std::vector result; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -877,7 +881,7 @@ TEST(GcsFileSystemTest, DeleteFile) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); TF_EXPECT_OK(fs.DeleteFile("gs://bucket/path/file1.txt")); } @@ -887,7 +891,7 @@ TEST(GcsFileSystemTest, DeleteFile_NoObjectName) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 5 /* read ahead bytes */, 5 /* max upload attempts */); + 5 /* read ahead bytes */, 0 /* initial retry delay */); EXPECT_EQ(errors::Code::INVALID_ARGUMENT, fs.DeleteFile("gs://bucket/").code()); @@ -902,7 +906,7 @@ TEST(GcsFileSystemTest, DeleteDir_Empty) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/")); } @@ -923,7 +927,7 @@ TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/")); } @@ -936,7 +940,7 @@ TEST(GcsFileSystemTest, DeleteDir_BucketOnly) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); TF_EXPECT_OK(fs.DeleteDir("gs://bucket")); } @@ -951,7 +955,7 @@ TEST(GcsFileSystemTest, DeleteDir_NonEmpty) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); EXPECT_EQ(error::Code::FAILED_PRECONDITION, fs.DeleteDir("gs://bucket/path/").code()); @@ -967,7 +971,7 @@ TEST(GcsFileSystemTest, GetFileSize) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); uint64 size; TF_EXPECT_OK(fs.GetFileSize("gs://bucket/file.txt", &size)); @@ -979,7 +983,7 @@ TEST(GcsFileSystemTest, GetFileSize_NoObjectName) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 5 /* read ahead bytes */, 5 /* max upload attempts */); + 5 /* read ahead bytes */, 0 /* initial retry delay */); uint64 size; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -1051,7 +1055,7 @@ TEST(GcsFileSystemTest, RenameFile_Folder) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); TF_EXPECT_OK(fs.RenameFile("gs://bucket/path1", "gs://bucket/path2/")); } @@ -1089,7 +1093,7 @@ TEST(GcsFileSystemTest, RenameFile_Object) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); TF_EXPECT_OK( fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt")); @@ -1122,7 +1126,7 @@ TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); EXPECT_EQ( errors::Code::UNIMPLEMENTED, @@ -1140,7 +1144,7 @@ TEST(GcsFileSystemTest, Stat_Object) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat)); @@ -1166,7 +1170,7 @@ TEST(GcsFileSystemTest, Stat_Folder) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder", &stat)); @@ -1191,7 +1195,7 @@ TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); FileStatistics stat; EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/path", &stat).code()); @@ -1205,7 +1209,7 @@ TEST(GcsFileSystemTest, Stat_Bucket) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/", &stat)); @@ -1222,7 +1226,7 @@ TEST(GcsFileSystemTest, Stat_BucketNotFound) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); FileStatistics stat; EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/", &stat).code()); @@ -1244,7 +1248,7 @@ TEST(GcsFileSystemTest, IsDirectory_NotFound) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); EXPECT_EQ(error::Code::NOT_FOUND, fs.IsDirectory("gs://bucket/file.txt").code()); @@ -1267,7 +1271,7 @@ TEST(GcsFileSystemTest, IsDirectory_NotDirectoryButObject) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); EXPECT_EQ(error::Code::FAILED_PRECONDITION, fs.IsDirectory("gs://bucket/file.txt").code()); @@ -1290,7 +1294,7 @@ TEST(GcsFileSystemTest, IsDirectory_Yes) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder")); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder/")); @@ -1309,7 +1313,7 @@ TEST(GcsFileSystemTest, IsDirectory_Bucket) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); TF_EXPECT_OK(fs.IsDirectory("gs://bucket")); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/")); @@ -1323,7 +1327,7 @@ TEST(GcsFileSystemTest, IsDirectory_BucketNotFound) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); EXPECT_EQ(error::Code::NOT_FOUND, fs.IsDirectory("gs://bucket/").code()); } @@ -1355,7 +1359,7 @@ TEST(GcsFileSystemTest, CreateDir_Folder) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath")); TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath/")); @@ -1374,7 +1378,7 @@ TEST(GcsFileSystemTest, CreateDir_Bucket) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); TF_EXPECT_OK(fs.CreateDir("gs://bucket/")); TF_EXPECT_OK(fs.CreateDir("gs://bucket")); @@ -1427,7 +1431,7 @@ TEST(GcsFileSystemTest, DeleteRecursively_Ok) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); int64 undeleted_files, undeleted_dirs; TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files, @@ -1505,7 +1509,7 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); int64 undeleted_files, undeleted_dirs; TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files, @@ -1532,7 +1536,7 @@ TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) { GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 5 /* max upload attempts */); + 0 /* read ahead bytes */, 0 /* initial retry delay */); int64 undeleted_files, undeleted_dirs; EXPECT_EQ(error::Code::NOT_FOUND, diff --git a/tensorflow/core/platform/cloud/retrying_file_system_test.cc b/tensorflow/core/platform/cloud/retrying_file_system_test.cc index fe0b0a765007bf..c5922c2c6ff749 100644 --- a/tensorflow/core/platform/cloud/retrying_file_system_test.cc +++ b/tensorflow/core/platform/cloud/retrying_file_system_test.cc @@ -213,7 +213,7 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_SuccessWith3rdTry) { TEST(RetryingFileSystemTest, NewRandomAccessFile_AllRetriesFailed) { // Configure the mock base random access file. - ExpectedCalls expected_file_calls = CreateRetriableErrors("Read", 6); + ExpectedCalls expected_file_calls = CreateRetriableErrors("Read", 11); std::unique_ptr base_file( new MockRandomAccessFile(expected_file_calls)); @@ -232,7 +232,7 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_AllRetriesFailed) { // Use it and check the results. StringPiece result; char scratch[10]; - EXPECT_EQ("Retriable error #5", + EXPECT_EQ("Retriable error #10", random_access_file->Read(0, 10, &result, scratch).error_message()); } @@ -366,7 +366,7 @@ TEST(RetryingFileSystemTest, NewAppendableFile_SuccessWith3rdTry) { TEST(RetryingFileSystemTest, NewWritableFile_AllRetriesFailed) { // Configure the mock base random access file. - ExpectedCalls expected_file_calls = CreateRetriableErrors("Sync", 6); + ExpectedCalls expected_file_calls = CreateRetriableErrors("Sync", 11); expected_file_calls.emplace_back(std::make_tuple("Close", Status::OK())); std::unique_ptr base_file( new MockWritableFile(expected_file_calls)); @@ -384,7 +384,7 @@ TEST(RetryingFileSystemTest, NewWritableFile_AllRetriesFailed) { TF_EXPECT_OK(fs.NewWritableFile("filename.txt", &writable_file)); // Use it and check the results. - EXPECT_EQ("Retriable error #5", writable_file->Sync().error_message()); + EXPECT_EQ("Retriable error #10", writable_file->Sync().error_message()); } TEST(RetryingFileSystemTest, @@ -403,13 +403,13 @@ TEST(RetryingFileSystemTest, TEST(RetryingFileSystemTest, NewReadOnlyMemoryRegionFromFile_AllRetriesFailed) { ExpectedCalls expected_fs_calls = - CreateRetriableErrors("NewReadOnlyMemoryRegionFromFile", 6); + CreateRetriableErrors("NewReadOnlyMemoryRegionFromFile", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); std::unique_ptr result; - EXPECT_EQ("Retriable error #5", + EXPECT_EQ("Retriable error #10", fs.NewReadOnlyMemoryRegionFromFile("filename.txt", &result) .error_message()); } @@ -428,13 +428,13 @@ TEST(RetryingFileSystemTest, GetChildren_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, GetChildren_AllRetriesFailed) { - ExpectedCalls expected_fs_calls = CreateRetriableErrors("GetChildren", 6); + ExpectedCalls expected_fs_calls = CreateRetriableErrors("GetChildren", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); std::vector result; - EXPECT_EQ("Retriable error #5", + EXPECT_EQ("Retriable error #10", fs.GetChildren("gs://path", &result).error_message()); } @@ -453,13 +453,13 @@ TEST(RetryingFileSystemTest, GetMatchingPaths_SuccessWith2ndTry) { TEST(RetryingFileSystemTest, GetMatchingPaths_AllRetriesFailed) { ExpectedCalls expected_fs_calls = - CreateRetriableErrors("GetMatchingPaths", 6); + CreateRetriableErrors("GetMatchingPaths", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); std::vector result; - EXPECT_EQ("Retriable error #5", + EXPECT_EQ("Retriable error #10", fs.GetMatchingPaths("gs://path/dir", &result).error_message()); } @@ -476,13 +476,13 @@ TEST(RetryingFileSystemTest, DeleteFile_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, DeleteFile_AllRetriesFailed) { - ExpectedCalls expected_fs_calls = CreateRetriableErrors("DeleteFile", 6); + ExpectedCalls expected_fs_calls = CreateRetriableErrors("DeleteFile", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); std::vector result; - EXPECT_EQ("Retriable error #5", + EXPECT_EQ("Retriable error #10", fs.DeleteFile("gs://path/file.txt").error_message()); } @@ -499,13 +499,13 @@ TEST(RetryingFileSystemTest, CreateDir_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, CreateDir_AllRetriesFailed) { - ExpectedCalls expected_fs_calls = CreateRetriableErrors("CreateDir", 6); + ExpectedCalls expected_fs_calls = CreateRetriableErrors("CreateDir", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); std::vector result; - EXPECT_EQ("Retriable error #5", + EXPECT_EQ("Retriable error #10", fs.CreateDir("gs://path/newdir").error_message()); } @@ -522,13 +522,13 @@ TEST(RetryingFileSystemTest, DeleteDir_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, DeleteDir_AllRetriesFailed) { - ExpectedCalls expected_fs_calls = CreateRetriableErrors("DeleteDir", 6); + ExpectedCalls expected_fs_calls = CreateRetriableErrors("DeleteDir", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); std::vector result; - EXPECT_EQ("Retriable error #5", + EXPECT_EQ("Retriable error #10", fs.DeleteDir("gs://path/dir").error_message()); } @@ -546,13 +546,13 @@ TEST(RetryingFileSystemTest, GetFileSize_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, GetFileSize_AllRetriesFailed) { - ExpectedCalls expected_fs_calls = CreateRetriableErrors("GetFileSize", 6); + ExpectedCalls expected_fs_calls = CreateRetriableErrors("GetFileSize", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); uint64 size; - EXPECT_EQ("Retriable error #5", + EXPECT_EQ("Retriable error #10", fs.GetFileSize("gs://path/file.txt", &size).error_message()); } @@ -568,12 +568,12 @@ TEST(RetryingFileSystemTest, RenameFile_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, RenameFile_AllRetriesFailed) { - ExpectedCalls expected_fs_calls = CreateRetriableErrors("RenameFile", 6); + ExpectedCalls expected_fs_calls = CreateRetriableErrors("RenameFile", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); - EXPECT_EQ("Retriable error #5", + EXPECT_EQ("Retriable error #10", fs.RenameFile("old_name", "new_name").error_message()); } @@ -590,13 +590,33 @@ TEST(RetryingFileSystemTest, Stat_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, Stat_AllRetriesFailed) { - ExpectedCalls expected_fs_calls = CreateRetriableErrors("Stat", 6); + ExpectedCalls expected_fs_calls = CreateRetriableErrors("Stat", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); FileStatistics stat; - EXPECT_EQ("Retriable error #5", fs.Stat("file_name", &stat).error_message()); + EXPECT_EQ("Retriable error #10", fs.Stat("file_name", &stat).error_message()); +} + +TEST(RetryingFileSystemTest, FileExists_AllRetriesFailed) { + ExpectedCalls expected_fs_calls = CreateRetriableErrors("FileExists", 11); + std::unique_ptr base_fs( + new MockFileSystem(expected_fs_calls)); + RetryingFileSystem fs(std::move(base_fs), 0); + + EXPECT_EQ("Retriable error #10", fs.FileExists("file_name").error_message()); +} + +TEST(RetryingFileSystemTest, FileExists_SuccessWith2ndTry) { + ExpectedCalls expected_fs_calls( + {std::make_tuple("FileExists", errors::Unavailable("Something is wrong")), + std::make_tuple("FileExists", Status::OK())}); + std::unique_ptr base_fs( + new MockFileSystem(expected_fs_calls)); + RetryingFileSystem fs(std::move(base_fs), 0); + + TF_EXPECT_OK(fs.FileExists("gs://path/dir")); } TEST(RetryingFileSystemTest, IsDirectory_SuccessWith2ndTry) { @@ -612,12 +632,12 @@ TEST(RetryingFileSystemTest, IsDirectory_SuccessWith2ndTry) { } TEST(RetryingFileSystemTest, IsDirectory_AllRetriesFailed) { - ExpectedCalls expected_fs_calls = CreateRetriableErrors("IsDirectory", 6); + ExpectedCalls expected_fs_calls = CreateRetriableErrors("IsDirectory", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); - EXPECT_EQ("Retriable error #5", + EXPECT_EQ("Retriable error #10", fs.IsDirectory("gs://path/dir").error_message()); } @@ -637,14 +657,14 @@ TEST(RetryingFileSystemTest, DeleteRecursively_SuccessWith2ndTry) { TEST(RetryingFileSystemTest, DeleteRecursively_AllRetriesFailed) { ExpectedCalls expected_fs_calls = - CreateRetriableErrors("DeleteRecursively", 6); + CreateRetriableErrors("DeleteRecursively", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); RetryingFileSystem fs(std::move(base_fs), 0); int64 undeleted_files, undeleted_dirs; EXPECT_EQ( - "Retriable error #5", + "Retriable error #10", fs.DeleteRecursively("gs://path/dir", &undeleted_files, &undeleted_dirs) .error_message()); } diff --git a/tensorflow/core/platform/cloud/retrying_utils.cc b/tensorflow/core/platform/cloud/retrying_utils.cc index dc05f9b62bee66..afc66abe44f4b0 100644 --- a/tensorflow/core/platform/cloud/retrying_utils.cc +++ b/tensorflow/core/platform/cloud/retrying_utils.cc @@ -24,7 +24,7 @@ namespace tensorflow { namespace { // In case of failure, every call will be retried kMaxRetries times. -constexpr int kMaxRetries = 5; +constexpr int kMaxRetries = 10; // Maximum backoff time in microseconds. constexpr int64 kMaximumBackoffMicroseconds = 32000000; // 32 seconds. @@ -40,16 +40,25 @@ bool IsRetriable(Status status) { } } -void WaitBeforeRetry(const int64 delay_micros) { +void WaitBeforeRetry(const int64 delay_micros, + const std::function& sleep_usec) { const int64 random_micros = random::New64() % 1000000; - Env::Default()->SleepForMicroseconds(std::min(delay_micros + random_micros, - kMaximumBackoffMicroseconds)); + sleep_usec(std::min(delay_micros, kMaximumBackoffMicroseconds) + + random_micros); } } // namespace Status RetryingUtils::CallWithRetries(const std::function& f, const int64 initial_delay_microseconds) { + return CallWithRetries(f, initial_delay_microseconds, + std::bind(&Env::SleepForMicroseconds, Env::Default(), + std::placeholders::_1)); +} + +Status RetryingUtils::CallWithRetries( + const std::function& f, const int64 initial_delay_microseconds, + const std::function& sleep_usec) { int retries = 0; while (true) { auto status = f(); @@ -58,10 +67,9 @@ Status RetryingUtils::CallWithRetries(const std::function& f, } const int64 delay_micros = initial_delay_microseconds << retries; if (delay_micros > 0) { - WaitBeforeRetry(delay_micros); + WaitBeforeRetry(delay_micros, sleep_usec); } retries++; } } - } // namespace tensorflow diff --git a/tensorflow/core/platform/cloud/retrying_utils.h b/tensorflow/core/platform/cloud/retrying_utils.h index 19bbf51725eb0e..ca5c40dc9368d1 100644 --- a/tensorflow/core/platform/cloud/retrying_utils.h +++ b/tensorflow/core/platform/cloud/retrying_utils.h @@ -32,6 +32,10 @@ class RetryingUtils { /// If all retries failed, returns the last error status. static Status CallWithRetries(const std::function& f, const int64 initial_delay_microseconds); + /// sleep_usec is a function that sleeps for the given number of microseconds. + static Status CallWithRetries(const std::function& f, + const int64 initial_delay_microseconds, + const std::function& sleep_usec); }; } // namespace tensorflow diff --git a/tensorflow/core/platform/cloud/retrying_utils_test.cc b/tensorflow/core/platform/cloud/retrying_utils_test.cc new file mode 100644 index 00000000000000..59af77a7a24e5e --- /dev/null +++ b/tensorflow/core/platform/cloud/retrying_utils_test.cc @@ -0,0 +1,73 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/platform/cloud/retrying_utils.h" +#include +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace { + +TEST(RetryingUtilsTest, RetryDelays) { + std::vector requested_delays; // requested delays in seconds + std::function sleep = [&requested_delays](int64 delay) { + requested_delays.emplace_back(delay / 1000000.0); + }; + std::function f = []() { return errors::Unavailable(""); }; + + EXPECT_EQ(errors::Code::UNAVAILABLE, + RetryingUtils::CallWithRetries(f, 500000L, sleep).code()); + + EXPECT_EQ(10, requested_delays.size()); + EXPECT_NEAR(0.5, requested_delays[0], 1.0); + EXPECT_NEAR(1.0, requested_delays[1], 1.0); + EXPECT_NEAR(2.0, requested_delays[2], 1.0); + EXPECT_NEAR(4.0, requested_delays[3], 1.0); + EXPECT_NEAR(8.0, requested_delays[4], 1.0); + EXPECT_NEAR(16.0, requested_delays[5], 1.0); + + // All subsequent delays are capped at 32 seconds (plus jitter). + EXPECT_NEAR(32.0, requested_delays[6], 1.0); + EXPECT_NEAR(32.0, requested_delays[7], 1.0); + EXPECT_NEAR(32.0, requested_delays[8], 1.0); + EXPECT_NEAR(32.0, requested_delays[9], 1.0); +} + +TEST(RetryingUtilsTest, NotFoundIsNotRetried) { + std::vector results({errors::Unavailable(""), errors::NotFound("")}); + std::function f = [&results]() { + auto result = results[0]; + results.erase(results.begin()); + return result; + }; + EXPECT_EQ(errors::Code::NOT_FOUND, + RetryingUtils::CallWithRetries(f, 0).code()); +} + +TEST(RetryingUtilsTest, EventualSuccess) { + std::vector results( + {errors::Unavailable(""), errors::Unavailable(""), Status::OK()}); + std::function f = [&results]() { + auto result = results[0]; + results.erase(results.begin()); + return result; + }; + TF_EXPECT_OK(RetryingUtils::CallWithRetries(f, 0)); +} + +} // namespace +} // namespace tensorflow From aee4914fadca79289c87eccf1c24dfd6030bf2b6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 2 Mar 2017 13:17:15 -0800 Subject: [PATCH 37/50] Retrying FileExists errors in case of errors to overcome intermittent GCS errors. Change: 149038920 --- .../platform/cloud/retrying_file_system.cc | 6 +++-- .../cloud/retrying_file_system_test.cc | 24 ++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/platform/cloud/retrying_file_system.cc b/tensorflow/core/platform/cloud/retrying_file_system.cc index 2da3104a72d0a7..6f9c7e1a128283 100644 --- a/tensorflow/core/platform/cloud/retrying_file_system.cc +++ b/tensorflow/core/platform/cloud/retrying_file_system.cc @@ -131,8 +131,10 @@ Status RetryingFileSystem::NewReadOnlyMemoryRegionFromFile( } Status RetryingFileSystem::FileExists(const string& fname) { - // No status -- no retries. - return base_file_system_->FileExists(fname); + return RetryingUtils::CallWithRetries( + std::bind(&FileSystem::FileExists, base_file_system_.get(), fname), + initial_delay_microseconds_); + ; } Status RetryingFileSystem::Stat(const string& fname, FileStatistics* stat) { diff --git a/tensorflow/core/platform/cloud/retrying_file_system_test.cc b/tensorflow/core/platform/cloud/retrying_file_system_test.cc index c5922c2c6ff749..65625d991b37b3 100644 --- a/tensorflow/core/platform/cloud/retrying_file_system_test.cc +++ b/tensorflow/core/platform/cloud/retrying_file_system_test.cc @@ -109,7 +109,9 @@ class MockFileSystem : public FileSystem { return calls_.ConsumeNextCall("NewReadOnlyMemoryRegionFromFile"); } - Status FileExists(const string& fname) override { return Status::OK(); } + Status FileExists(const string& fname) override { + return calls_.ConsumeNextCall("FileExists"); + } Status GetChildren(const string& dir, std::vector* result) override { return calls_.ConsumeNextCall("GetChildren"); @@ -619,6 +621,26 @@ TEST(RetryingFileSystemTest, FileExists_SuccessWith2ndTry) { TF_EXPECT_OK(fs.FileExists("gs://path/dir")); } +TEST(RetryingFileSystemTest, FileExists_AllRetriesFailed) { + ExpectedCalls expected_fs_calls = CreateRetriableErrors("FileExists", 6); + std::unique_ptr base_fs( + new MockFileSystem(expected_fs_calls)); + RetryingFileSystem fs(std::move(base_fs), 0); + + EXPECT_EQ("Retriable error #5", fs.FileExists("file_name").error_message()); +} + +TEST(RetryingFileSystemTest, FileExists_SuccessWith2ndTry) { + ExpectedCalls expected_fs_calls( + {std::make_tuple("FileExists", errors::Unavailable("Something is wrong")), + std::make_tuple("FileExists", Status::OK())}); + std::unique_ptr base_fs( + new MockFileSystem(expected_fs_calls)); + RetryingFileSystem fs(std::move(base_fs), 0); + + TF_EXPECT_OK(fs.FileExists("gs://path/dir")); +} + TEST(RetryingFileSystemTest, IsDirectory_SuccessWith2ndTry) { ExpectedCalls expected_fs_calls( {std::make_tuple("IsDirectory", From cbb612e502995fbb8e6e980ab3385d1ba105b65c Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Thu, 2 Mar 2017 16:34:59 -0800 Subject: [PATCH 38/50] Upon any HTTP error, clear the response buffer. Change: 149062390 --- .../core/platform/cloud/http_request.cc | 3 +++ .../core/platform/cloud/http_request_test.cc | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/tensorflow/core/platform/cloud/http_request.cc b/tensorflow/core/platform/cloud/http_request.cc index aaff6c11f04abe..8a8d1e448a6b25 100644 --- a/tensorflow/core/platform/cloud/http_request.cc +++ b/tensorflow/core/platform/cloud/http_request.cc @@ -383,8 +383,10 @@ Status HttpRequest::Send() { return Status::OK(); case 401: case 403: + response_buffer_->clear(); return errors::PermissionDenied(error_message); case 404: + response_buffer_->clear(); return errors::NotFound(error_message); case 416: // Requested Range Not Satisfiable response_buffer_->clear(); @@ -392,6 +394,7 @@ Status HttpRequest::Send() { default: // UNAVAILABLE can be retried by the caller, e.g by // RetryingFileSystem. + response_buffer_->clear(); return errors::Unavailable(error_message); } } diff --git a/tensorflow/core/platform/cloud/http_request_test.cc b/tensorflow/core/platform/cloud/http_request_test.cc index e9bd9dab1af36c..6ac637a81e603b 100644 --- a/tensorflow/core/platform/cloud/http_request_test.cc +++ b/tensorflow/core/platform/cloud/http_request_test.cc @@ -528,5 +528,24 @@ TEST(HttpRequestTest, EscapeString) { EXPECT_EQ("a%2Fb%2Fc", http_request.EscapeString(test_string)); } +TEST(HttpRequestTest, ErrorReturnsNoResponse) { + FakeLibCurl libcurl("get response", 500); + HttpRequest http_request(&libcurl); + TF_EXPECT_OK(http_request.Init()); + + std::vector scratch; + scratch.insert(scratch.begin(), kTestContent.begin(), kTestContent.end()); + StringPiece result; + scratch.reserve(100); + + TF_EXPECT_OK(http_request.SetUri("http://www.testuri.com")); + TF_EXPECT_OK(http_request.AddAuthBearerHeader("fake-bearer")); + TF_EXPECT_OK(http_request.SetRange(100, 199)); + TF_EXPECT_OK(http_request.SetResultBuffer(&scratch)); + EXPECT_EQ(error::UNAVAILABLE, http_request.Send().code()); + + EXPECT_EQ("", string(scratch.begin(), scratch.end())); +} + } // namespace } // namespace tensorflow From 2924e923a0ca3705e551511a9f69b6da65e43e5a Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Fri, 3 Mar 2017 13:32:01 -0800 Subject: [PATCH 39/50] Fix merge --- .../cloud/retrying_file_system_test.cc | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/tensorflow/core/platform/cloud/retrying_file_system_test.cc b/tensorflow/core/platform/cloud/retrying_file_system_test.cc index 65625d991b37b3..7ace6e5b065bf8 100644 --- a/tensorflow/core/platform/cloud/retrying_file_system_test.cc +++ b/tensorflow/core/platform/cloud/retrying_file_system_test.cc @@ -621,26 +621,6 @@ TEST(RetryingFileSystemTest, FileExists_SuccessWith2ndTry) { TF_EXPECT_OK(fs.FileExists("gs://path/dir")); } -TEST(RetryingFileSystemTest, FileExists_AllRetriesFailed) { - ExpectedCalls expected_fs_calls = CreateRetriableErrors("FileExists", 6); - std::unique_ptr base_fs( - new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); - - EXPECT_EQ("Retriable error #5", fs.FileExists("file_name").error_message()); -} - -TEST(RetryingFileSystemTest, FileExists_SuccessWith2ndTry) { - ExpectedCalls expected_fs_calls( - {std::make_tuple("FileExists", errors::Unavailable("Something is wrong")), - std::make_tuple("FileExists", Status::OK())}); - std::unique_ptr base_fs( - new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); - - TF_EXPECT_OK(fs.FileExists("gs://path/dir")); -} - TEST(RetryingFileSystemTest, IsDirectory_SuccessWith2ndTry) { ExpectedCalls expected_fs_calls( {std::make_tuple("IsDirectory", From a0802b5bd3fab97bd0ddf4805955b943ed8702f1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 3 Mar 2017 14:14:16 -0800 Subject: [PATCH 40/50] Add the graphdef version to InferenceContext and to ShapeRefiner::AddNode. Use this to allow loading reductions saved with older graphdefs. Change GraphConstructor to not increase the version when importing, but instead take the min of all versions. Change: 149152437 --- tensorflow/c/c_api.cc | 2 +- tensorflow/cc/framework/scope.cc | 3 +- .../core/common_runtime/shape_refiner.cc | 12 +- .../core/common_runtime/shape_refiner.h | 5 +- .../core/common_runtime/shape_refiner_test.cc | 49 +++---- tensorflow/core/framework/common_shape_fns.cc | 8 +- .../core/framework/common_shape_fns_test.cc | 135 +++++++++++------- tensorflow/core/framework/shape_inference.cc | 10 +- tensorflow/core/framework/shape_inference.h | 9 +- .../core/framework/shape_inference_test.cc | 107 +++++++------- .../framework/shape_inference_testutil.cc | 5 +- .../core/framework/shape_inference_testutil.h | 2 + tensorflow/core/graph/graph_constructor.cc | 8 +- .../core/graph/graph_constructor_test.cc | 78 ++++++++-- .../core/kernels/hexagon/graph_transferer.cc | 2 +- tensorflow/core/ops/array_ops_test.cc | 7 +- tensorflow/core/ops/math_ops_test.cc | 4 +- tensorflow/core/public/version.h | 3 + tensorflow/python/BUILD | 1 + tensorflow/python/framework/common_shapes.py | 5 +- .../python/framework/cpp_shape_inference.cc | 11 +- .../python/framework/cpp_shape_inference.h | 2 +- tensorflow/python/framework/importer_test.py | 19 +++ tensorflow/python/framework/test_ops.cc | 11 ++ .../python/kernel_tests/reduction_ops_test.py | 7 + 25 files changed, 337 insertions(+), 168 deletions(-) diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc index 83ce3e25d46046..bdab3f7134dfd2 100644 --- a/tensorflow/c/c_api.cc +++ b/tensorflow/c/c_api.cc @@ -729,7 +729,7 @@ extern "C" { struct TF_Graph { TF_Graph() : graph(OpRegistry::Global()), - refiner(graph.op_registry()), + refiner(graph.versions().producer(), graph.op_registry()), num_sessions(0), delete_requested(false) {} mutex mu; diff --git a/tensorflow/cc/framework/scope.cc b/tensorflow/cc/framework/scope.cc index e1af5b36e8c226..2193427a0673d9 100644 --- a/tensorflow/cc/framework/scope.cc +++ b/tensorflow/cc/framework/scope.cc @@ -34,7 +34,8 @@ Scope::Scope(Graph* graph, Status* status, Scope::NameMap* name_map, Scope Scope::NewRootScope() { Graph* graph = new Graph(OpRegistry::Global()); - ShapeRefiner* refiner = new ShapeRefiner(graph->op_registry()); + ShapeRefiner* refiner = new ShapeRefiner( + graph->versions().producer(), graph->op_registry()); return Scope(graph, new Status, new Scope::NameMap, refiner); } diff --git a/tensorflow/core/common_runtime/shape_refiner.cc b/tensorflow/core/common_runtime/shape_refiner.cc index dc1272c5d68b10..c0e7f5e19f6fe4 100644 --- a/tensorflow/core/common_runtime/shape_refiner.cc +++ b/tensorflow/core/common_runtime/shape_refiner.cc @@ -31,8 +31,9 @@ using shape_inference::DimensionHandle; using shape_inference::InferenceContext; using shape_inference::ShapeHandle; -ShapeRefiner::ShapeRefiner(const OpRegistryInterface* ops) - : ops_registry_(ops) {} +ShapeRefiner::ShapeRefiner(int graph_def_version, + const OpRegistryInterface* ops) + : graph_def_version_(graph_def_version), ops_registry_(ops) {} ShapeRefiner::~ShapeRefiner() { gtl::STLDeleteValues(&node_to_context_); } @@ -87,9 +88,10 @@ Status ShapeRefiner::AddNode(const Node* node) { std::vector input_tensors_as_shapes; // Create the inference context for this node with the existing input shapes. - std::unique_ptr c(new InferenceContext( - &node->def(), node->op_def(), input_shapes, input_tensors, - input_tensors_as_shapes, input_handle_shapes, input_handle_dtypes)); + std::unique_ptr c( + new InferenceContext(graph_def_version_, &node->def(), node->op_def(), + input_shapes, input_tensors, input_tensors_as_shapes, + input_handle_shapes, input_handle_dtypes)); if (!c->construction_status().ok()) { return c->construction_status(); } diff --git a/tensorflow/core/common_runtime/shape_refiner.h b/tensorflow/core/common_runtime/shape_refiner.h index 6ce5ddb3661b24..bce63d65dbe8ad 100644 --- a/tensorflow/core/common_runtime/shape_refiner.h +++ b/tensorflow/core/common_runtime/shape_refiner.h @@ -31,7 +31,7 @@ namespace tensorflow { // construction time. class ShapeRefiner { public: - explicit ShapeRefiner(const OpRegistryInterface* ops); + ShapeRefiner(int graph_def_version, const OpRegistryInterface* ops); ~ShapeRefiner(); // Performs validation of 'node' and runs 'node's shape function, @@ -99,7 +99,8 @@ class ShapeRefiner { const Node* node, int dst_idx, shape_inference::ShapeHandle* result); - const OpRegistryInterface* ops_registry_ = nullptr; + const int graph_def_version_; + const OpRegistryInterface* const ops_registry_; // Stores a map from a node to its InferenceContext. // diff --git a/tensorflow/core/common_runtime/shape_refiner_test.cc b/tensorflow/core/common_runtime/shape_refiner_test.cc index 54a00ac9ffa957..05274ff311233b 100644 --- a/tensorflow/core/common_runtime/shape_refiner_test.cc +++ b/tensorflow/core/common_runtime/shape_refiner_test.cc @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/platform/test.h" +#include "tensorflow/core/public/version.h" namespace tensorflow { namespace { @@ -38,14 +39,14 @@ TEST(ShapeRefinerTest, Constant) { // and that its shape is correct. Scope root = Scope::NewRootScope(); auto c = ops::Const(root, 42.0f); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); TF_ASSERT_OK(m.AddNode(c.node())); EXPECT_SHAPE("[]", m, c, 0); } TEST(ShapeRefinerTest, MatMul) { - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); Scope root = Scope::NewRootScope(); auto a = ops::Const(root, {{1.0f}, {2.0f}}); @@ -62,7 +63,7 @@ TEST(ShapeRefinerTest, MatMul) { } TEST(ShapeRefinerTest, InvalidOrder) { - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); Scope root = Scope::NewRootScope(); auto a = ops::Const(root, {{1.0f}, {2.0f}}); auto b = ops::Const(root, {{1.0f, 2.0f}}); @@ -77,7 +78,7 @@ TEST(ShapeRefinerTest, InvalidOrder) { } TEST(ShapeRefinerTest, BadShapes) { - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); Scope root = Scope::NewRootScope(); auto a = ops::Const(root, {{1.0f}, {2.0f}}); auto b = ops::Const(root, {{1.0f}, {2.0f}}); @@ -94,7 +95,7 @@ TEST(ShapeRefinerTest, BadShapes) { } TEST(ShapeRefinerTest, SetShape) { - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); Scope root = Scope::NewRootScope(); auto a = ops::Placeholder(root, DT_FLOAT); @@ -136,7 +137,7 @@ TEST(ShapeRefinerTest, PropagateConstants) { auto dim = ops::Variable(root, {}, DT_INT32); auto am = ops::ArgMax(root, input, dim); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); TF_ASSERT_OK(m.AddNode(input.node())); TF_ASSERT_OK(m.AddNode(dim.node())); TF_ASSERT_OK(m.AddNode(am.node())); @@ -153,7 +154,7 @@ TEST(ShapeRefinerTest, PropagateConstants) { auto dim = ops::Const(root, 1); auto am = ops::ArgMax(root, input, dim); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); TF_ASSERT_OK(m.AddNode(input.node())); TF_ASSERT_OK(m.AddNode(dim.node())); TF_ASSERT_OK(m.AddNode(am.node())); @@ -169,7 +170,7 @@ TEST(ShapeRefinerTest, PropagateConstants) { auto dim = ops::Const(root, 0); auto am = ops::ArgMax(root, input, dim); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); TF_ASSERT_OK(m.AddNode(input.node())); TF_ASSERT_OK(m.AddNode(dim.node())); TF_ASSERT_OK(m.AddNode(am.node())); @@ -199,7 +200,7 @@ REGISTER_OP("TestOp") } // namespace TEST(ShapeRefinerTest, InputTensorDependencies) { - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); Graph graph(OpRegistry::Global()); Node* node; @@ -260,7 +261,7 @@ TEST(ShapeRefinerTest, PropagateShape) { .Input(shape.node()) .Finalize(root.graph(), &shape_data)); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); TF_ASSERT_OK(m.AddNode(input.node())); TF_ASSERT_OK(m.AddNode(shape.node())); TF_ASSERT_OK(m.AddNode(shape_data)); @@ -281,7 +282,7 @@ TEST(ShapeRefinerTest, PropagateSize) { .Input(size.node()) .Finalize(root.graph(), &shape_data)); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); TF_ASSERT_OK(m.AddNode(input.node())); TF_ASSERT_OK(m.AddNode(size.node())); TF_ASSERT_OK(m.AddNode(shape_data)); @@ -302,7 +303,7 @@ TEST(ShapeRefinerTest, PropagateRank) { .Input(rank.node()) .Finalize(root.graph(), &shape_data)); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); TF_ASSERT_OK(m.AddNode(input.node())); TF_ASSERT_OK(m.AddNode(rank.node())); TF_ASSERT_OK(m.AddNode(shape_data)); @@ -323,7 +324,7 @@ TEST(ShapeRefinerTest, PropagateRange) { .Input(range.node()) .Finalize(root.graph(), &shape_data)); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); TF_ASSERT_OK(m.AddNode(begin.node())); TF_ASSERT_OK(m.AddNode(limit.node())); TF_ASSERT_OK(m.AddNode(delta.node())); @@ -346,7 +347,7 @@ TEST(ShapeRefinerTest, ConstantValueTwoInputsToSameNode) { .Input(range.node()) .Finalize(root.graph(), &shape_data)); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); TF_ASSERT_OK(m.AddNode(begin_and_delta.node())); TF_ASSERT_OK(m.AddNode(limit.node())); TF_ASSERT_OK(m.AddNode(range.node())); @@ -381,7 +382,7 @@ TEST(ShapeRefinerTest, ConstantValueVisitNodeTwice) { .Input(range.node()) .Finalize(root.graph(), &shape_data)); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); TF_ASSERT_OK(m.AddNode(begin.node())); TF_ASSERT_OK(m.AddNode(limit.node())); TF_ASSERT_OK(m.AddNode(delta.node())); @@ -477,7 +478,7 @@ TEST(ShapeRefinerTest, ConstantValueAsShape_EmptyVector) { .Input(input) .Finalize(root.graph(), &result)); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); TF_ASSERT_OK(m.AddNode(input)); TF_ASSERT_OK(m.AddNode(result)); @@ -498,7 +499,7 @@ TEST(ShapeRefinerTest, ConstantValueAsShape_Shape) { .Input(shape.node()) .Finalize(root.graph(), &result)); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); TF_ASSERT_OK(m.AddNode(input)); TF_ASSERT_OK(m.AddNode(shape.node())); TF_ASSERT_OK(m.AddNode(result)); @@ -533,7 +534,7 @@ TEST(ShapeRefinerTest, ConstantValueAsShape_PackInt32) { .Input(pack.node()) .Finalize(root.graph(), &result)); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); for (auto input : inputs) { TF_ASSERT_OK(m.AddNode(input.node())); } @@ -565,7 +566,7 @@ TEST(ShapeRefinerTest, ConstantValueAsShape_PackInt64) { .Input(pack.node()) .Finalize(root.graph(), &result)); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); for (const auto& input : inputs) { TF_ASSERT_OK(m.AddNode(input.node())); } @@ -591,7 +592,7 @@ TEST(ShapeRefinerTest, ConstantValueAsShape_PackUnknownDim) { .Input(pack.node()) .Finalize(root.graph(), &result)); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); for (const auto& input : inputs) { TF_ASSERT_OK(m.AddNode(input.node())); } @@ -618,7 +619,7 @@ TEST(ShapeRefinerTest, ConstantValueAsShape_PackInvalidInput) { .Input(pack.node()) .Finalize(root.graph(), &result)); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); for (const auto& input : inputs) { TF_ASSERT_OK(m.AddNode(input.node())); } @@ -650,7 +651,7 @@ TEST(ShapeRefinerTest, ConstantValueAsShape_Concat) { .Input(concat.node()) .Finalize(g, &result)); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); TF_ASSERT_OK(m.AddNode(partial_1)); TF_ASSERT_OK(m.AddNode(partial_2)); for (const auto& o : concat_inputs) { @@ -692,7 +693,7 @@ TEST(ShapeRefinerTest, ConstantValueAsShape_ConcatWithUnknown) { .Input(concat.node()) .Finalize(g, &result)); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); TF_ASSERT_OK(m.AddNode(partial_1)); TF_ASSERT_OK(m.AddNode(partial_2)); TF_ASSERT_OK(m.AddNode(unknown)); @@ -734,7 +735,7 @@ TEST(ShapeRefinerTest, ConstantValueAsShape_ConcatInvalidDimValue) { .Input(concat.node()) .Finalize(g, &result)); - ShapeRefiner m(OpRegistry::Global()); + ShapeRefiner m(TF_GRAPH_DEF_VERSION, OpRegistry::Global()); TF_ASSERT_OK(m.AddNode(partial_1)); TF_ASSERT_OK(m.AddNode(partial_2)); for (const auto& o : concat_inputs) { diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc index 9d5d212ddddf2b..ede0452f14dbc9 100644 --- a/tensorflow/core/framework/common_shape_fns.cc +++ b/tensorflow/core/framework/common_shape_fns.cc @@ -590,7 +590,13 @@ Status ReductionShape(InferenceContext* c) { ShapeHandle input = c->input(0); ShapeHandle indices; - TF_RETURN_IF_ERROR(c->WithRankAtMost(c->input(1), 1, &indices)); + // Older versions of TensorFlow accidentally allowed higher rank tensors like + // [[1,2]] or [[1],[2]] to represent axis=[1,2]. + if (c->graph_def_version() < 21) { + indices = c->input(1); + } else { + TF_RETURN_IF_ERROR(c->WithRankAtMost(c->input(1), 1, &indices)); + } bool keep_dims; TF_RETURN_IF_ERROR(c->GetAttr("keep_dims", &keep_dims)); diff --git a/tensorflow/core/framework/common_shape_fns_test.cc b/tensorflow/core/framework/common_shape_fns_test.cc index 89acf1202cac82..2d9e96e6bc6d49 100644 --- a/tensorflow/core/framework/common_shape_fns_test.cc +++ b/tensorflow/core/framework/common_shape_fns_test.cc @@ -69,7 +69,8 @@ TEST(CommonShapeFnsTest, NoOutputShapeTest) { .Input({{"data", 0, DT_FLOAT}}) .Finalize(&def)); - InferenceContext c(&def, op_def, {S({}), S({10})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, {S({}), S({10})}, {}, + {}, {}, {}); TF_EXPECT_OK(NoOutputs(&c)); EXPECT_EQ(0, c.num_outputs()); } @@ -87,14 +88,16 @@ TEST(CommonShapeFnsTest, ScalarShapeTest) { NodeDefBuilder("test", "L2Loss").Input("t", 0, DT_FLOAT).Finalize(&def)); { - InferenceContext c(&def, op_def, {S({})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, {S({})}, {}, {}, {}, + {}); TF_EXPECT_OK(ScalarShape(&c)); ShapeHandle output = c.output(0); EXPECT_EQ(0, c.Rank(output)); } { - InferenceContext c(&def, op_def, {S({1, 23, 4, 4, 2})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, + {S({1, 23, 4, 4, 2})}, {}, {}, {}, {}); TF_EXPECT_OK(ScalarShape(&c)); ShapeHandle output = c.output(0); EXPECT_EQ(0, c.Rank(output)); @@ -121,7 +124,8 @@ TEST(CommonShapeFnsTest, MatMulShapeTest) { .Finalize(&def)); { - InferenceContext c(&def, op_def, {S({2, 3}), S({3, 4})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, + {S({2, 3}), S({3, 4})}, {}, {}, {}, {}); TF_EXPECT_OK(MatMulShape(&c)); ShapeHandle output = c.output(0); EXPECT_EQ(2, c.Value(c.Dim(output, 0))); @@ -130,7 +134,8 @@ TEST(CommonShapeFnsTest, MatMulShapeTest) { { // Unknown inner dimension for one - InferenceContext c(&def, op_def, {S({2, -1}), S({3, 4})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, + {S({2, -1}), S({3, 4})}, {}, {}, {}, {}); TF_EXPECT_OK(MatMulShape(&c)); ShapeHandle output = c.output(0); EXPECT_EQ(2, c.Value(c.Dim(output, 0))); @@ -139,7 +144,8 @@ TEST(CommonShapeFnsTest, MatMulShapeTest) { { // Invalid rank. - InferenceContext c(&def, op_def, {S({2}), S({3, 4})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, {S({2}), S({3, 4})}, + {}, {}, {}, {}); auto s = MatMulShape(&c); EXPECT_FALSE(s.ok()); EXPECT_TRUE( @@ -149,7 +155,8 @@ TEST(CommonShapeFnsTest, MatMulShapeTest) { { // Unknown outer dimension - InferenceContext c(&def, op_def, {S({2, 3}), S({3, -1})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, + {S({2, 3}), S({3, -1})}, {}, {}, {}, {}); TF_EXPECT_OK(MatMulShape(&c)); ShapeHandle output = c.output(0); EXPECT_EQ(2, c.Value(c.Dim(output, 0))); @@ -158,7 +165,8 @@ TEST(CommonShapeFnsTest, MatMulShapeTest) { { // Inner shapes not compatible - InferenceContext c(&def, op_def, {S({2, 5}), S({3, 4})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, + {S({2, 5}), S({3, 4})}, {}, {}, {}, {}); auto s = MatMulShape(&c); EXPECT_FALSE(s.ok()); EXPECT_TRUE( @@ -169,8 +177,8 @@ TEST(CommonShapeFnsTest, MatMulShapeTest) { { // Inner shapes not compatible - InferenceContext c(&def, op_def, {S({2, 5, 3}), S({3, 5, 4})}, {}, {}, {}, - {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, + {S({2, 5, 3}), S({3, 5, 4})}, {}, {}, {}, {}); auto s = MatMulShape(&c); EXPECT_FALSE(s.ok()); EXPECT_TRUE( @@ -188,7 +196,8 @@ TEST(CommonShapeFnsTest, MatMulShapeTest) { .Attr("type", DT_FLOAT) .Finalize(&def)); - InferenceContext c(&def, op_def, {S({3, 2}), S({3, 4})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, + {S({3, 2}), S({3, 4})}, {}, {}, {}, {}); auto s = MatMulShape(&c); ShapeHandle output = c.output(0); EXPECT_EQ(2, c.Value(c.Dim(output, 0))); @@ -205,7 +214,8 @@ TEST(CommonShapeFnsTest, MatMulShapeTest) { .Attr("type", DT_FLOAT) .Finalize(&def)); - InferenceContext c(&def, op_def, {S({2, 3}), S({4, 3})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, + {S({2, 3}), S({4, 3})}, {}, {}, {}, {}); auto s = MatMulShape(&c); ShapeHandle output = c.output(0); EXPECT_EQ(2, c.Value(c.Dim(output, 0))); @@ -229,7 +239,8 @@ TEST(CommonShapeFnsTest, BiasAddShapeTest) { .Finalize(&def)); { - InferenceContext c(&def, op_def, {S({2, 10}), S({10})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, + {S({2, 10}), S({10})}, {}, {}, {}, {}); TF_EXPECT_OK(BiasAddShape(&c)); ShapeHandle output = c.output(0); EXPECT_EQ(2, c.Value(c.Dim(output, 0))); @@ -238,7 +249,8 @@ TEST(CommonShapeFnsTest, BiasAddShapeTest) { { // Unknown ranks. - InferenceContext c(&def, op_def, {Unknown(), Unknown()}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, + {Unknown(), Unknown()}, {}, {}, {}, {}); TF_EXPECT_OK(BiasAddShape(&c)); ShapeHandle output = c.output(0); EXPECT_FALSE(c.RankKnown(output)); @@ -246,8 +258,8 @@ TEST(CommonShapeFnsTest, BiasAddShapeTest) { { // Rank > 2 - InferenceContext c(&def, op_def, {S({4, 3, 4, 2, 15}), S({15})}, {}, {}, {}, - {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, + {S({4, 3, 4, 2, 15}), S({15})}, {}, {}, {}, {}); TF_EXPECT_OK(BiasAddShape(&c)); ShapeHandle output = c.output(0); EXPECT_EQ("[4,3,4,2,15]", c.DebugString(output)); @@ -260,7 +272,8 @@ TEST(CommonShapeFnsTest, BiasAddShapeTest) { .Input("b", 0, DT_FLOAT) .Attr("data_format", "NCHW") .Finalize(&def)); - InferenceContext c(&def, op_def, {S({2, 3, 4, 5}), S({3})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, + {S({2, 3, 4, 5}), S({3})}, {}, {}, {}, {}); TF_EXPECT_OK(BiasAddShape(&c)); ShapeHandle output = c.output(0); EXPECT_EQ("[2,3,4,5]", c.DebugString(output)); @@ -273,8 +286,8 @@ TEST(CommonShapeFnsTest, BiasAddShapeTest) { .Input("b", 0, DT_FLOAT) .Attr("data_format", "NCHW") .Finalize(&def)); - InferenceContext c(&def, op_def, {S({8, 6, 4, 2, 3, 4, 5}), S({3})}, {}, {}, - {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, + {S({8, 6, 4, 2, 3, 4, 5}), S({3})}, {}, {}, {}, {}); TF_EXPECT_OK(BiasAddShape(&c)); ShapeHandle output = c.output(0); EXPECT_EQ("[8,6,4,2,3,4,5]", c.DebugString(output)); @@ -287,8 +300,8 @@ TEST(CommonShapeFnsTest, BiasAddShapeTest) { .Input("b", 0, DT_FLOAT) .Attr("data_format", "NCHW") .Finalize(&def)); - InferenceContext c(&def, op_def, {S({10, 11, 12}), S({10})}, {}, {}, {}, - {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, + {S({10, 11, 12}), S({10})}, {}, {}, {}, {}); TF_EXPECT_OK(BiasAddShape(&c)); ShapeHandle output = c.output(0); EXPECT_EQ("[10,11,12]", c.DebugString(output)); @@ -296,7 +309,8 @@ TEST(CommonShapeFnsTest, BiasAddShapeTest) { { // Input rank not high enough - InferenceContext c(&def, op_def, {S({3}), S({3})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, {S({3}), S({3})}, {}, + {}, {}, {}); EXPECT_FALSE(BiasAddShape(&c).ok()); } @@ -308,7 +322,8 @@ TEST(CommonShapeFnsTest, BiasAddShapeTest) { .Attr("data_format", "NCHW") .Finalize(&def)); // NCHW format - InferenceContext c(&def, op_def, {S({2, 3}), S({3})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, {S({2, 3}), S({3})}, + {}, {}, {}, {}); EXPECT_FALSE(BiasAddShape(&c).ok()); } } @@ -327,7 +342,8 @@ TEST(CommonShapeFnsTest, BiasAddGradShapeTest) { .Finalize(&def)); { - InferenceContext c(&def, op_def, {S({2, 10})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, {S({2, 10})}, {}, {}, + {}, {}); TF_EXPECT_OK(BiasAddGradShape(&c)); ShapeHandle output = c.output(0); EXPECT_EQ(10, c.Value(c.Dim(output, 0))); @@ -335,7 +351,8 @@ TEST(CommonShapeFnsTest, BiasAddGradShapeTest) { { // Rank > 2 - InferenceContext c(&def, op_def, {S({5, 7, 2, 10})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, {S({5, 7, 2, 10})}, + {}, {}, {}, {}); TF_EXPECT_OK(BiasAddGradShape(&c)); ShapeHandle output = c.output(0); EXPECT_EQ(10, c.Value(c.Dim(output, 0))); @@ -347,7 +364,8 @@ TEST(CommonShapeFnsTest, BiasAddGradShapeTest) { .Input("a", 0, DT_FLOAT) .Attr("data_format", "NCHW") .Finalize(&def)); - InferenceContext c(&def, op_def, {S({2, 3, 4, 5})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, {S({2, 3, 4, 5})}, + {}, {}, {}, {}); TF_EXPECT_OK(BiasAddGradShape(&c)); ShapeHandle output = c.output(0); EXPECT_EQ(3, c.Value(c.Dim(output, 0))); @@ -359,8 +377,8 @@ TEST(CommonShapeFnsTest, BiasAddGradShapeTest) { .Input("a", 0, DT_FLOAT) .Attr("data_format", "NCHW") .Finalize(&def)); - InferenceContext c(&def, op_def, {S({8, 6, 4, 2, 3, 4, 5})}, {}, {}, {}, - {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, + {S({8, 6, 4, 2, 3, 4, 5})}, {}, {}, {}, {}); TF_EXPECT_OK(BiasAddGradShape(&c)); ShapeHandle output = c.output(0); EXPECT_EQ(3, c.Value(c.Dim(output, 0))); @@ -372,7 +390,8 @@ TEST(CommonShapeFnsTest, BiasAddGradShapeTest) { .Input("a", 0, DT_FLOAT) .Attr("data_format", "NCHW") .Finalize(&def)); - InferenceContext c(&def, op_def, {S({10, 11, 12})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, {S({10, 11, 12})}, + {}, {}, {}, {}); TF_EXPECT_OK(BiasAddGradShape(&c)); ShapeHandle output = c.output(0); EXPECT_EQ(10, c.Value(c.Dim(output, 0))); @@ -380,7 +399,8 @@ TEST(CommonShapeFnsTest, BiasAddGradShapeTest) { { // Input rank not high enough - InferenceContext c(&def, op_def, {S({3})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, {S({3})}, {}, {}, {}, + {}); EXPECT_FALSE(BiasAddGradShape(&c).ok()); } @@ -391,7 +411,8 @@ TEST(CommonShapeFnsTest, BiasAddGradShapeTest) { .Attr("data_format", "NCHW") .Finalize(&def)); // NCHW format - InferenceContext c(&def, op_def, {S({2, 3})}, {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, op_def, {S({2, 3})}, {}, {}, + {}, {}); EXPECT_FALSE(BiasAddGradShape(&c).ok()); } } @@ -781,12 +802,24 @@ TEST(CommonShapeFnsTest, Reduce_ShapeFn) { op.input_tensors[1] = nullptr; INFER_OK(op, "[?,?,?];?", "[?,?,?]"); INFER_OK(op, "[?,?,?];[2]", "[?,?,?]"); + + // Reduction indices with too many dimensions. + INFER_ERROR("must be at most rank 1 but is rank 2", op, "[?,?,?];[?,?]"); + // With older graph-def version, this is allowed. + op.graph_def_version = 20; + INFER_OK(op, "[?,?,?];[?,?]", "[?,?,?]"); + // And when the tensor is specified, it's still allowed. + op.input_tensors[1] = &indices; + indices = test::AsTensor({-1, -2}, TensorShape({2, 1})); + INFER_OK(op, "[2,4,5];[2,1]", "[d0_0, 1, 1]"); + indices = test::AsTensor({-1, -2}, TensorShape({1, 2})); + INFER_OK(op, "[2,4,5];[1,2]", "[d0_0, 1, 1]"); } TEST(CommonShapeFnsTest, ValidateSparseTensor_UnknownShapes) { NodeDef def; - InferenceContext c(&def, MakeOpDef(3, 1), {Unknown(), Unknown(), Unknown()}, - {}, {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, MakeOpDef(3, 1), + {Unknown(), Unknown(), Unknown()}, {}, {}, {}, {}); EXPECT_EQ(3, c.num_inputs()); EXPECT_EQ(1, c.num_outputs()); @@ -798,8 +831,8 @@ TEST(CommonShapeFnsTest, ValidateSparseTensor_UnknownShapes) { TEST(CommonShapeFnsTest, ValidateSparseTensor_UnknownDims) { NodeDef def; - InferenceContext c(&def, MakeOpDef(3, 1), {S({-1, -1}), S({-1}), S({-1})}, {}, - {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, MakeOpDef(3, 1), + {S({-1, -1}), S({-1}), S({-1})}, {}, {}, {}, {}); EXPECT_EQ(3, c.num_inputs()); EXPECT_EQ(1, c.num_outputs()); @@ -811,8 +844,8 @@ TEST(CommonShapeFnsTest, ValidateSparseTensor_UnknownDims) { TEST(CommonShapeFnsTest, ValidateSparseTensor_InvalidIndicesRank) { NodeDef def; - InferenceContext c(&def, MakeOpDef(3, 1), {S({-1}), S({-1}), S({-1})}, {}, {}, - {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, MakeOpDef(3, 1), + {S({-1}), S({-1}), S({-1})}, {}, {}, {}, {}); EXPECT_EQ(3, c.num_inputs()); EXPECT_EQ(1, c.num_outputs()); @@ -825,8 +858,8 @@ TEST(CommonShapeFnsTest, ValidateSparseTensor_InvalidIndicesRank) { TEST(CommonShapeFnsTest, ValidateSparseTensor_InvalidNumElements) { NodeDef def; - InferenceContext c(&def, MakeOpDef(3, 1), {S({5, 3}), S({4}), S({3})}, {}, {}, - {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, MakeOpDef(3, 1), + {S({5, 3}), S({4}), S({3})}, {}, {}, {}, {}); EXPECT_EQ(3, c.num_inputs()); EXPECT_EQ(1, c.num_outputs()); @@ -839,8 +872,8 @@ TEST(CommonShapeFnsTest, ValidateSparseTensor_InvalidNumElements) { TEST(CommonShapeFnsTest, ValidateSparseTensor_InvalidRank) { NodeDef def; - InferenceContext c(&def, MakeOpDef(3, 1), {S({5, 3}), S({5}), S({4})}, {}, {}, - {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, MakeOpDef(3, 1), + {S({5, 3}), S({5}), S({4})}, {}, {}, {}, {}); EXPECT_EQ(3, c.num_inputs()); EXPECT_EQ(1, c.num_outputs()); @@ -853,8 +886,8 @@ TEST(CommonShapeFnsTest, ValidateSparseTensor_InvalidRank) { TEST(CommonShapeFnsTest, ValidateSparseTensor_UnknownNumIndexElements) { NodeDef def; - InferenceContext c(&def, MakeOpDef(3, 1), {S({-1, 3}), S({5}), S({3})}, {}, - {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, MakeOpDef(3, 1), + {S({-1, 3}), S({5}), S({3})}, {}, {}, {}, {}); EXPECT_EQ(3, c.num_inputs()); EXPECT_EQ(1, c.num_outputs()); @@ -866,8 +899,8 @@ TEST(CommonShapeFnsTest, ValidateSparseTensor_UnknownNumIndexElements) { TEST(CommonShapeFnsTest, ValidateSparseTensor_UnknownNumValueElements) { NodeDef def; - InferenceContext c(&def, MakeOpDef(3, 1), {S({5, 3}), S({-1}), S({3})}, {}, - {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, MakeOpDef(3, 1), + {S({5, 3}), S({-1}), S({3})}, {}, {}, {}, {}); EXPECT_EQ(3, c.num_inputs()); EXPECT_EQ(1, c.num_outputs()); @@ -879,8 +912,8 @@ TEST(CommonShapeFnsTest, ValidateSparseTensor_UnknownNumValueElements) { TEST(CommonShapeFnsTest, ValidateSparseTensor_UnknownIndexRank) { NodeDef def; - InferenceContext c(&def, MakeOpDef(3, 1), {S({5, -1}), S({5}), S({3})}, {}, - {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, MakeOpDef(3, 1), + {S({5, -1}), S({5}), S({3})}, {}, {}, {}, {}); EXPECT_EQ(3, c.num_inputs()); EXPECT_EQ(1, c.num_outputs()); @@ -892,8 +925,8 @@ TEST(CommonShapeFnsTest, ValidateSparseTensor_UnknownIndexRank) { TEST(CommonShapeFnsTest, ValidateSparseTensor_UnknownShapeRank) { NodeDef def; - InferenceContext c(&def, MakeOpDef(3, 1), {S({5, 3}), S({5}), S({-1})}, {}, - {}, {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, MakeOpDef(3, 1), + {S({5, 3}), S({5}), S({-1})}, {}, {}, {}, {}); EXPECT_EQ(3, c.num_inputs()); EXPECT_EQ(1, c.num_outputs()); @@ -905,8 +938,8 @@ TEST(CommonShapeFnsTest, ValidateSparseTensor_UnknownShapeRank) { TEST(CommonShapeFnsTest, ValidateSparseTensor) { NodeDef def; - InferenceContext c(&def, MakeOpDef(3, 1), {S({5, 3}), S({5}), S({3})}, {}, {}, - {}, {}); + InferenceContext c(TF_GRAPH_DEF_VERSION, &def, MakeOpDef(3, 1), + {S({5, 3}), S({5}), S({3})}, {}, {}, {}, {}); EXPECT_EQ(3, c.num_inputs()); EXPECT_EQ(1, c.num_outputs()); diff --git a/tensorflow/core/framework/shape_inference.cc b/tensorflow/core/framework/shape_inference.cc index 02fdc16e882212..3411306c25d63c 100644 --- a/tensorflow/core/framework/shape_inference.cc +++ b/tensorflow/core/framework/shape_inference.cc @@ -29,13 +29,14 @@ constexpr int32 InferenceContext::kUnknownRank; constexpr int64 InferenceContext::kUnknownDim; InferenceContext::InferenceContext( - const NodeDef* node_def, const OpDef& op_def, + int graph_def_version, const NodeDef* node_def, const OpDef& op_def, const std::vector& input_shapes, const std::vector& input_tensors, const std::vector& input_tensors_as_shapes, const std::vector& input_handle_shapes, const std::vector& input_handle_dtypes) - : node_def_(*CHECK_NOTNULL(node_def)) { + : graph_def_version_(graph_def_version), + node_def_(*CHECK_NOTNULL(node_def)) { std::vector input_tensors_as_shape_handles; for (const TensorShapeProto& p : input_tensors_as_shapes) { ShapeHandle shape; @@ -68,13 +69,14 @@ InferenceContext::InferenceContext( } InferenceContext::InferenceContext( - const NodeDef* node_def, const OpDef& op_def, + int graph_def_version, const NodeDef* node_def, const OpDef& op_def, const std::vector& input_shapes, const std::vector& input_tensors, const std::vector& input_tensors_as_shapes, const std::vector& input_handle_shapes, const std::vector& input_handle_dtypes) - : node_def_(*CHECK_NOTNULL(node_def)) { + : graph_def_version_(graph_def_version), + node_def_(*CHECK_NOTNULL(node_def)) { PreInputInit(op_def, input_tensors, input_tensors_as_shapes); if (!construction_status_.ok()) return; inputs_ = input_shapes; diff --git a/tensorflow/core/framework/shape_inference.h b/tensorflow/core/framework/shape_inference.h index fd4e25c7283cb7..dba8d3030271ec 100644 --- a/tensorflow/core/framework/shape_inference.h +++ b/tensorflow/core/framework/shape_inference.h @@ -144,7 +144,8 @@ class InferenceContext { // Values of do not need to outlive the context. // // REQUIRES: is not NULL, and must outlive the InferenceContext. - InferenceContext(const NodeDef* node_def, const OpDef& op_def, + InferenceContext(int graph_def_version, const NodeDef* node_def, + const OpDef& op_def, const std::vector& input_shapes, const std::vector& input_tensors, const std::vector& input_tensors_as_shapes, @@ -161,7 +162,8 @@ class InferenceContext { // Values of do not need to outlive the context. // // REQUIRES: is not NULL, and must outlive the InferenceContext. - InferenceContext(const NodeDef* node_def, const OpDef& op_def, + InferenceContext(int graph_def_version, const NodeDef* node_def, + const OpDef& op_def, const std::vector& input_shapes, const std::vector& input_tensors, const std::vector& input_tensors_as_shapes, @@ -436,6 +438,8 @@ class InferenceContext { Status MakeShapeFromTensor(const Tensor* t, ShapeHandle tensor_shape, ShapeHandle* out); + int graph_def_version() const { return graph_def_version_; } + private: // Creates and stores shapes for use in InferenceContext. class ShapeManager { @@ -508,6 +512,7 @@ class InferenceContext { std::vector output_handle_shape_; std::vector output_handle_dtype_; + const int graph_def_version_; const NodeDef& node_def_; NameRangeMap input_name_map_; NameRangeMap output_name_map_; diff --git a/tensorflow/core/framework/shape_inference_test.cc b/tensorflow/core/framework/shape_inference_test.cc index 80a8639c021944..e9756afd4b31fe 100644 --- a/tensorflow/core/framework/shape_inference_test.cc +++ b/tensorflow/core/framework/shape_inference_test.cc @@ -61,6 +61,8 @@ class ShapeInferenceTest : public ::testing::Test { bool SameHandle(ShapeHandle a, ShapeHandle b) { return a.SameHandle(b); } bool IsSet(DimensionHandle d) { return d.IsSet(); } bool IsSet(ShapeHandle s) { return s.IsSet(); } + + static const int kVersion = 0; // used for graph-def version. }; TEST_F(ShapeInferenceTest, InputOutputByName) { @@ -71,8 +73,8 @@ TEST_F(ShapeInferenceTest, InputOutputByName) { .Attr("N", 3) .Input(FakeInput(DT_FLOAT)) .Finalize(&def); - InferenceContext c(&def, op_def, {S({1, 5}), S({2, 5}), S({1, 3})}, {}, {}, - {}, {}); + InferenceContext c(kVersion, &def, op_def, {S({1, 5}), S({2, 5}), S({1, 3})}, + {}, {}, {}, {}); EXPECT_EQ("5", c.DebugString(c.NumElements(c.input(0)))); EXPECT_EQ("10", c.DebugString(c.NumElements(c.input(1)))); @@ -108,7 +110,8 @@ static OpDef MakeOpDef(int num_inputs, int num_outputs) { TEST_F(ShapeInferenceTest, DimensionOrConstant) { NodeDef def; - InferenceContext c(&def, MakeOpDef(1, 1), {Unknown()}, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(1, 1), {Unknown()}, {}, {}, {}, + {}); EXPECT_EQ(InferenceContext::kUnknownDim, c.Value(InferenceContext::kUnknownDim)); EXPECT_EQ(1, c.Value(1)); @@ -123,7 +126,8 @@ TEST_F(ShapeInferenceTest, Run) { NodeDef def; def.set_name("foo"); def.set_op("foo_op"); - InferenceContext c(&def, MakeOpDef(3, 2), {S({1})}, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(1, 2), {S({1})}, {}, {}, {}, {}); + TF_ASSERT_OK(c.construction_status()); { auto fn = [](InferenceContext* c) { @@ -154,8 +158,8 @@ TEST_F(ShapeInferenceTest, Run) { TEST_F(ShapeInferenceTest, RankAndDimInspection) { NodeDef def; - InferenceContext c(&def, MakeOpDef(3, 2), {Unknown(), S({1, -1, 3}), S({})}, - {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(3, 2), + {Unknown(), S({1, -1, 3}), S({})}, {}, {}, {}, {}); EXPECT_EQ(3, c.num_inputs()); EXPECT_EQ(2, c.num_outputs()); @@ -195,7 +199,7 @@ TEST_F(ShapeInferenceTest, RankAndDimInspection) { TEST_F(ShapeInferenceTest, NumElements) { NodeDef def; - InferenceContext c(&def, MakeOpDef(3, 2), + InferenceContext c(kVersion, &def, MakeOpDef(3, 2), {Unknown(), S({1, -1, 3}), S({5, 4, 3, 2})}, {}, {}, {}, {}); @@ -210,8 +214,8 @@ TEST_F(ShapeInferenceTest, NumElements) { TEST_F(ShapeInferenceTest, WithRank) { NodeDef def; - InferenceContext c(&def, MakeOpDef(2, 2), {Unknown(), S({1, -1, 3})}, {}, {}, - {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(2, 2), + {Unknown(), S({1, -1, 3})}, {}, {}, {}, {}); auto in0 = c.input(0); auto in1 = c.input(1); @@ -249,8 +253,8 @@ TEST_F(ShapeInferenceTest, WithRank) { TEST_F(ShapeInferenceTest, WithRankAtMost) { NodeDef def; - InferenceContext c(&def, MakeOpDef(2, 2), {Unknown(), S({1, -1, 3})}, {}, {}, - {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(2, 2), + {Unknown(), S({1, -1, 3})}, {}, {}, {}, {}); auto in0 = c.input(0); auto in1 = c.input(1); @@ -288,8 +292,8 @@ TEST_F(ShapeInferenceTest, WithRankAtMost) { TEST_F(ShapeInferenceTest, WithRankAtLeast) { NodeDef def; - InferenceContext c(&def, MakeOpDef(2, 2), {Unknown(), S({1, -1, 3})}, {}, {}, - {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(2, 2), + {Unknown(), S({1, -1, 3})}, {}, {}, {}, {}); auto in0 = c.input(0); auto in1 = c.input(1); @@ -327,7 +331,8 @@ TEST_F(ShapeInferenceTest, WithRankAtLeast) { TEST_F(ShapeInferenceTest, WithValue) { NodeDef def; - InferenceContext c(&def, MakeOpDef(1, 2), {S({1, -1})}, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(1, 2), {S({1, -1})}, {}, {}, {}, + {}); auto d0 = c.Dim(c.input(0), 0); auto d1 = c.Dim(c.input(0), 1); @@ -368,8 +373,8 @@ TEST_F(ShapeInferenceTest, WithValue) { TEST_F(ShapeInferenceTest, MergeDim) { NodeDef def; - InferenceContext c(&def, MakeOpDef(1, 2), {S({2, -1, 2, 1, -1})}, {}, {}, {}, - {}); + InferenceContext c(kVersion, &def, MakeOpDef(1, 2), {S({2, -1, 2, 1, -1})}, + {}, {}, {}, {}); auto d2 = c.Dim(c.input(0), 0); auto d_unknown = c.Dim(c.input(0), 1); @@ -415,7 +420,7 @@ TEST_F(ShapeInferenceTest, MergeDim) { TEST_F(ShapeInferenceTest, MergeShape) { NodeDef def; - InferenceContext c(&def, MakeOpDef(7, 2), + InferenceContext c(kVersion, &def, MakeOpDef(7, 2), {Unknown(), S({1, 2}), S({-1, 2}), S({1, -1}), S({1, 3}), Unknown(), S({1})}, {}, {}, {}, {}); @@ -485,7 +490,7 @@ TEST_F(ShapeInferenceTest, MergeShape) { TEST_F(ShapeInferenceTest, MergePrefix) { NodeDef def; - InferenceContext c(&def, MakeOpDef(4, 2), + InferenceContext c(kVersion, &def, MakeOpDef(4, 2), { Unknown(), S({-1, 2}), S({1, -1, 3}), S({2, 4}), }, @@ -541,8 +546,8 @@ TEST_F(ShapeInferenceTest, MergePrefix) { TEST_F(ShapeInferenceTest, Subshape) { NodeDef def; - InferenceContext c(&def, MakeOpDef(2, 2), {S({1, 2, 3, -1, 5}), Unknown()}, - {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(2, 2), + {S({1, 2, 3, -1, 5}), Unknown()}, {}, {}, {}, {}); ShapeHandle unknown = c.input(1); ShapeHandle out; @@ -616,7 +621,7 @@ TEST_F(ShapeInferenceTest, Subshape) { TEST_F(ShapeInferenceTest, Concatenate) { NodeDef def; - InferenceContext c(&def, MakeOpDef(3, 2), + InferenceContext c(kVersion, &def, MakeOpDef(3, 2), {S({1, -1, 3}), S({4, 5}), Unknown()}, {}, {}, {}, {}); auto in0 = c.input(0); @@ -643,8 +648,8 @@ TEST_F(ShapeInferenceTest, Concatenate) { TEST_F(ShapeInferenceTest, ReplaceDim) { NodeDef def; - InferenceContext c(&def, MakeOpDef(2, 0), {S({1, 2, 3}), Unknown()}, {}, {}, - {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(2, 0), {S({1, 2, 3}), Unknown()}, + {}, {}, {}, {}); auto in = c.input(0); auto unknown = c.input(1); @@ -675,8 +680,8 @@ TEST_F(ShapeInferenceTest, ReplaceDim) { TEST_F(ShapeInferenceTest, MakeShape) { NodeDef def; - InferenceContext c(&def, MakeOpDef(1, 2), {S({1, 2, 3, -1, 5})}, {}, {}, {}, - {}); + InferenceContext c(kVersion, &def, MakeOpDef(1, 2), {S({1, 2, 3, -1, 5})}, {}, + {}, {}, {}); std::vector dims; auto in0 = c.input(0); @@ -701,7 +706,7 @@ TEST_F(ShapeInferenceTest, MakeShape) { TEST_F(ShapeInferenceTest, UnknownShape) { NodeDef def; std::vector empty; - InferenceContext c(&def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); auto u0 = c.UnknownShape(); auto u1 = c.UnknownShape(); @@ -713,7 +718,7 @@ TEST_F(ShapeInferenceTest, UnknownShape) { TEST_F(ShapeInferenceTest, Scalar) { NodeDef def; std::vector empty; - InferenceContext c(&def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); auto s0 = c.Scalar(); EXPECT_EQ("[]", c.DebugString(s0)); @@ -724,7 +729,7 @@ TEST_F(ShapeInferenceTest, Scalar) { TEST_F(ShapeInferenceTest, Vector) { NodeDef def; std::vector empty; - InferenceContext c(&def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); auto s0 = c.Vector(1); EXPECT_EQ("[1]", c.DebugString(s0)); @@ -740,7 +745,7 @@ TEST_F(ShapeInferenceTest, Vector) { TEST_F(ShapeInferenceTest, Matrix) { NodeDef def; std::vector empty; - InferenceContext c(&def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); auto s0 = c.Matrix(1, 2); EXPECT_EQ("[1,2]", c.DebugString(s0)); @@ -762,7 +767,8 @@ TEST_F(ShapeInferenceTest, Matrix) { TEST_F(ShapeInferenceTest, MakeShapeFromShapeTensor) { auto create = [&](Tensor* t) { NodeDef def; - InferenceContext c(&def, MakeOpDef(1, 0), {Unknown()}, {t}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(1, 0), {Unknown()}, {t}, {}, + {}, {}); ShapeHandle out; Status s = c.MakeShapeFromShapeTensor(0, &out); if (s.ok()) { @@ -814,8 +820,8 @@ TEST_F(ShapeInferenceTest, MakeShapeFromShapeTensor) { // Test when the input shape is wrong. { NodeDef def; - InferenceContext c(&def, MakeOpDef(1, 0), {S({1, -1})}, {nullptr}, {}, {}, - {}); + InferenceContext c(kVersion, &def, MakeOpDef(1, 0), {S({1, -1})}, {nullptr}, + {}, {}, {}); ShapeHandle out; EXPECT_EQ("Shape must be rank 1 but is rank 2", c.MakeShapeFromShapeTensor(0, &out).error_message()); @@ -825,7 +831,7 @@ TEST_F(ShapeInferenceTest, MakeShapeFromShapeTensor) { TEST_F(ShapeInferenceTest, MakeShapeFromShapeProto) { NodeDef def; std::vector empty; - InferenceContext c(&def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); TensorShapeProto proto; // With a set unknown rank. @@ -861,7 +867,7 @@ TEST_F(ShapeInferenceTest, MakeShapeFromShapeProto) { TEST_F(ShapeInferenceTest, MakeDim) { NodeDef def; std::vector empty; - InferenceContext c(&def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); auto d0 = c.MakeDim(1); auto d1 = c.MakeDim(1); @@ -875,7 +881,7 @@ TEST_F(ShapeInferenceTest, MakeDim) { TEST_F(ShapeInferenceTest, UnknownDim) { NodeDef def; std::vector empty; - InferenceContext c(&def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); auto d0 = c.UnknownDim(); auto d1 = c.UnknownDim(); @@ -887,7 +893,7 @@ TEST_F(ShapeInferenceTest, UnknownDim) { TEST_F(ShapeInferenceTest, UnknownShapeOfRank) { NodeDef def; std::vector empty; - InferenceContext c(&def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); auto unknown_shape_of_rank_3 = c.UnknownShapeOfRank(3); EXPECT_EQ("[?,?,?]", c.DebugString(unknown_shape_of_rank_3)); @@ -900,7 +906,7 @@ TEST_F(ShapeInferenceTest, InputTensors) { const Tensor t1 = tensorflow::test::AsTensor({10}); const Tensor t2 = tensorflow::test::AsTensor({20, 30}); NodeDef def; - InferenceContext c(&def, MakeOpDef(3, 2), {S({1}), S({2}), S({3})}, + InferenceContext c(kVersion, &def, MakeOpDef(3, 2), {S({1}), S({2}), S({3})}, {&t1, &t2}, {}, {}, {}); EXPECT_TRUE(c.input_tensor(0) == &t1); @@ -912,8 +918,8 @@ TEST_F(ShapeInferenceTest, MakeDimForScalarInput) { Tensor t1 = tensorflow::test::AsScalar(20); Tensor t2 = tensorflow::test::AsScalar(-1); NodeDef def; - InferenceContext c(&def, MakeOpDef(2, 2), {S({}), S({})}, {&t1, &t2}, {}, {}, - {}); + InferenceContext c(kVersion, &def, MakeOpDef(2, 2), {S({}), S({})}, + {&t1, &t2}, {}, {}, {}); DimensionHandle d; EXPECT_TRUE(c.MakeDimForScalarInput(0, &d).ok()); @@ -944,7 +950,7 @@ TEST_F(ShapeInferenceTest, GetAttr) { .ok()); std::vector empty; - InferenceContext c(&def, op_reg_data.op_def, empty, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, op_reg_data.op_def, empty, {}, {}, {}, {}); string value; EXPECT_TRUE(c.GetAttr("foo", &value).ok()); EXPECT_EQ("bar", value); @@ -952,8 +958,8 @@ TEST_F(ShapeInferenceTest, GetAttr) { TEST_F(ShapeInferenceTest, Divide) { NodeDef def; - InferenceContext c(&def, MakeOpDef(1, 2), {S({6, -1, 1, 2, 0})}, {}, {}, {}, - {}); + InferenceContext c(kVersion, &def, MakeOpDef(1, 2), {S({6, -1, 1, 2, 0})}, {}, + {}, {}, {}); auto s = c.input(0); auto d_6 = c.Dim(s, 0); @@ -1015,7 +1021,8 @@ TEST_F(ShapeInferenceTest, Divide) { TEST_F(ShapeInferenceTest, Add) { NodeDef def; - InferenceContext c(&def, MakeOpDef(1, 2), {S({6, -1, 0})}, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(1, 2), {S({6, -1, 0})}, {}, {}, + {}, {}); auto s = c.input(0); auto d_6 = c.Dim(s, 0); @@ -1066,7 +1073,8 @@ TEST_F(ShapeInferenceTest, Add) { TEST_F(ShapeInferenceTest, Subtract) { NodeDef def; - InferenceContext c(&def, MakeOpDef(1, 2), {S({6, -1, 0, 5})}, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(1, 2), {S({6, -1, 0, 5})}, {}, + {}, {}, {}); auto s = c.input(0); auto d_6 = c.Dim(s, 0); @@ -1115,7 +1123,8 @@ TEST_F(ShapeInferenceTest, Subtract) { TEST_F(ShapeInferenceTest, Multiply) { NodeDef def; - InferenceContext c(&def, MakeOpDef(1, 2), {S({6, -1, 0, 1})}, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(1, 2), {S({6, -1, 0, 1})}, {}, + {}, {}, {}); auto s = c.input(0); auto d_6 = c.Dim(s, 0); @@ -1168,7 +1177,7 @@ TEST_F(ShapeInferenceTest, Multiply) { TEST_F(ShapeInferenceTest, FullyDefined) { NodeDef def; std::vector empty; - InferenceContext c(&def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(0, 2), empty, {}, {}, {}, {}); // No rank or missing dimension information should return false. EXPECT_FALSE(c.FullyDefined(c.UnknownShape())); @@ -1181,7 +1190,8 @@ TEST_F(ShapeInferenceTest, FullyDefined) { TEST_F(ShapeInferenceTest, Min) { NodeDef def; - InferenceContext c(&def, MakeOpDef(1, 2), {S({1, 2, -1, 0})}, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(1, 2), {S({1, 2, -1, 0})}, {}, + {}, {}, {}); auto s = c.input(0); auto d_1 = c.Dim(s, 0); @@ -1229,7 +1239,8 @@ TEST_F(ShapeInferenceTest, Min) { TEST_F(ShapeInferenceTest, Max) { NodeDef def; - InferenceContext c(&def, MakeOpDef(1, 2), {S({1, 2, -1})}, {}, {}, {}, {}); + InferenceContext c(kVersion, &def, MakeOpDef(1, 2), {S({1, 2, -1})}, {}, {}, + {}, {}); auto s = c.input(0); auto d_1 = c.Dim(s, 0); diff --git a/tensorflow/core/framework/shape_inference_testutil.cc b/tensorflow/core/framework/shape_inference_testutil.cc index a225824f8260f7..85e085af99ae37 100644 --- a/tensorflow/core/framework/shape_inference_testutil.cc +++ b/tensorflow/core/framework/shape_inference_testutil.cc @@ -43,8 +43,9 @@ Status ShapeInferenceTestutil::InferShapes(ShapeInferenceTestOp op, in_shapes.push_back(shape); } - shape_inference::InferenceContext c(&op.node_def, op_reg_data->op_def, - in_shapes, op.input_tensors, {}, {}, {}); + shape_inference::InferenceContext c(op.graph_def_version, &op.node_def, + op_reg_data->op_def, in_shapes, + op.input_tensors, {}, {}, {}); TF_RETURN_IF_ERROR(c.construction_status()); if (op_reg_data->shape_inference_fn == nullptr) { return errors::InvalidArgument( diff --git a/tensorflow/core/framework/shape_inference_testutil.h b/tensorflow/core/framework/shape_inference_testutil.h index 64067464fb93aa..996281e70e6aa6 100644 --- a/tensorflow/core/framework/shape_inference_testutil.h +++ b/tensorflow/core/framework/shape_inference_testutil.h @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/public/version.h" // Contains utilities for writing tests for shape inference functions. @@ -34,6 +35,7 @@ struct ShapeInferenceTestOp { string name; NodeDef node_def; std::vector input_tensors; + int graph_def_version = TF_GRAPH_DEF_VERSION; }; namespace shape_inference { diff --git a/tensorflow/core/graph/graph_constructor.cc b/tensorflow/core/graph/graph_constructor.cc index d1550512737803..4a4ac044eace73 100644 --- a/tensorflow/core/graph/graph_constructor.cc +++ b/tensorflow/core/graph/graph_constructor.cc @@ -734,8 +734,8 @@ Status GraphConstructor::UpdateVersionDef() { return Status::OK(); } VersionDef versions = g_->versions(); - // This new graph is being "produced" by the binary invoking ImportGraphDef. - versions.set_producer(TF_GRAPH_DEF_VERSION); + versions.set_producer( + std::min(versions.producer(), gdef_->versions().producer())); versions.set_min_consumer( std::max(versions.min_consumer(), gdef_->versions().min_consumer())); if (gdef_->versions().bad_consumers_size() > 0) { @@ -779,13 +779,13 @@ Status GraphConstructor::MakeEdge(Node* src, int output_index, Node* dst, Status ConvertGraphDefToGraph(const GraphConstructorOptions& opts, const GraphDef& gdef, Graph* g) { - ShapeRefiner refiner(g->op_registry()); + ShapeRefiner refiner(gdef.versions().producer(), g->op_registry()); return GraphConstructor::Construct(opts, &gdef, g, &refiner); } Status ImportGraphDef(const ImportGraphDefOptions& opts, const GraphDef& gdef, Graph* g, ShapeRefiner* refiner) { - ShapeRefiner default_refiner(g->op_registry()); + ShapeRefiner default_refiner(gdef.versions().producer(), g->op_registry()); if (refiner == nullptr) { refiner = &default_refiner; } diff --git a/tensorflow/core/graph/graph_constructor_test.cc b/tensorflow/core/graph/graph_constructor_test.cc index a173d3a627571b..dd7bc8fb09c144 100644 --- a/tensorflow/core/graph/graph_constructor_test.cc +++ b/tensorflow/core/graph/graph_constructor_test.cc @@ -198,6 +198,15 @@ REGISTER_OP("TestOneInputOneOutput") REGISTER_OP("TestDefaultAttr") .Attr("default_int: int=31415") .SetShapeFn(shape_inference::NoOutputs); +REGISTER_OP("RequiresCurrentGraphVersion") + .Output("version: int32") + .SetIsStateful() + .SetShapeFn([](shape_inference::InferenceContext* c) { + if (c->graph_def_version() != TF_GRAPH_DEF_VERSION) { + return errors::InvalidArgument("Wrong graph version for shape"); + } + return shape_inference::ScalarShape(c); + }); TEST_F(GraphConstructorTest, InvalidNodeName) { auto expect_invalid_name = [this](const char* name) { @@ -716,7 +725,7 @@ TEST_F(GraphConstructorTest, ImportGraphDef_ShapeWhitelist) { } TEST_F(GraphConstructorTest, ImportGraphDef_InputMap) { - ShapeRefiner refiner(graph_.op_registry()); + ShapeRefiner refiner(TF_GRAPH_DEF_VERSION, graph_.op_registry()); // Populate graph with node we'll use in input map ExpectOK("node { name: 'input' op: 'TestInput' }", ImportGraphDefOptions(), @@ -756,7 +765,7 @@ TEST_F(GraphConstructorTest, ImportGraphDef_InputMap) { } TEST_F(GraphConstructorTest, ImportGraphDef_InputMapWithPrefix) { - ShapeRefiner refiner(graph_.op_registry()); + ShapeRefiner refiner(TF_GRAPH_DEF_VERSION, graph_.op_registry()); // Populate graph with node we'll use in input map ExpectOK( @@ -819,7 +828,7 @@ TEST_F(GraphConstructorTest, ImportGraphDef_InputMapWithPrefix) { } TEST_F(GraphConstructorTest, ImportGraphDef_InputMapWithControlEdges) { - ShapeRefiner refiner(graph_.op_registry()); + ShapeRefiner refiner(TF_GRAPH_DEF_VERSION, graph_.op_registry()); // Populate graph with node we'll use in input map ExpectOK("node { name: 'W1' op: 'TestParams' }", ImportGraphDefOptions(), @@ -883,7 +892,7 @@ TEST_F(GraphConstructorTest, ImportGraphDef_InputMapWithControlEdges) { } TEST_F(GraphConstructorTest, ImportGraphDef_InputMapWithBadControlEdge) { - ShapeRefiner refiner(graph_.op_registry()); + ShapeRefiner refiner(TF_GRAPH_DEF_VERSION, graph_.op_registry()); // Populate graph with node we'll use in input map ExpectOK("node { name: 'W1' op: 'TestParams' }", ImportGraphDefOptions(), @@ -915,7 +924,7 @@ TEST_F(GraphConstructorTest, ImportGraphDef_InputMapWithBadControlEdge) { } TEST_F(GraphConstructorTest, ImportGraphDef_InputMapWithInvalidNodeIndex) { - ShapeRefiner refiner(graph_.op_registry()); + ShapeRefiner refiner(TF_GRAPH_DEF_VERSION, graph_.op_registry()); // Populate graph with node we'll use in input map ExpectOK("node { name: 'input1' op: 'TestInput' }", ImportGraphDefOptions(), @@ -936,7 +945,7 @@ TEST_F(GraphConstructorTest, ImportGraphDef_InputMapWithInvalidNodeIndex) { } TEST_F(GraphConstructorTest, ImportGraphDef_InputMapWithMissingEntries) { - ShapeRefiner refiner(graph_.op_registry()); + ShapeRefiner refiner(TF_GRAPH_DEF_VERSION, graph_.op_registry()); // Populate graph with node we'll use in input map ExpectOK("node { name: 'W1' op: 'TestParams' }", ImportGraphDefOptions(), @@ -957,7 +966,7 @@ TEST_F(GraphConstructorTest, ImportGraphDef_InputMapWithMissingEntries) { } TEST_F(GraphConstructorTest, ImportGraphDef_InputMapDuplicateNodeNames) { - ShapeRefiner refiner(graph_.op_registry()); + ShapeRefiner refiner(TF_GRAPH_DEF_VERSION, graph_.op_registry()); // Add two nodes with the same name to graph Node* node; @@ -1200,7 +1209,7 @@ versions { } TEST_F(GraphConstructorTest, ImportGraphDef_ControlDeps) { - ShapeRefiner refiner(graph_.op_registry()); + ShapeRefiner refiner(TF_GRAPH_DEF_VERSION, graph_.op_registry()); // Populate graph with nodes we'll use in control deps and input map ExpectOK( @@ -1267,7 +1276,7 @@ TEST_F(GraphConstructorTest, ImportGraphDef_ControlDeps) { } TEST_F(GraphConstructorTest, ImportGraphDef_ControlDepsWithCycle) { - ShapeRefiner refiner(graph_.op_registry()); + ShapeRefiner refiner(TF_GRAPH_DEF_VERSION, graph_.op_registry()); // Populate graph with nodes we'll use in control deps and input map ExpectOK( @@ -1478,5 +1487,56 @@ TEST_F(GraphConstructorTest, CopyGraph) { EXPECT_EQ(dst.versions().bad_consumers(0), bad); } +// Confirms that graph def version in the graph reaches the shape inference +// function. +TEST_F(GraphConstructorTest, GraphDefVersionUsedForShapeInference) { + string gdef_ascii = strings::StrCat(R"EOF( + node{ name:"A" op:"RequiresCurrentGraphVersion" } + versions { producer: )EOF", + TF_GRAPH_DEF_VERSION - 1, "}"); + ImportGraphDefOptions opts; + ExpectError(gdef_ascii, opts, {"Wrong graph version for shape"}); + gdef_ascii = strings::StrCat(R"EOF( + node{ name:"A" op:"RequiresCurrentGraphVersion" } + versions { producer: )EOF", + TF_GRAPH_DEF_VERSION, "}"); + ExpectOK(gdef_ascii, opts); +} + +TEST_F(GraphConstructorTest, GraphDefVersionMergingDuringImport) { + ImportGraphDefOptions opts; + ExpectOK( + "versions { producer: 15 min_consumer: 5 bad_consumers: 2 bad_consumers: " + "3 " + "}", + opts); + EXPECT_EQ(15, graph_.versions().producer()); + EXPECT_EQ(5, graph_.versions().min_consumer()); + ASSERT_EQ(2, graph_.versions().bad_consumers_size()); + EXPECT_EQ(2, graph_.versions().bad_consumers(0)); + EXPECT_EQ(3, graph_.versions().bad_consumers(1)); + + ExpectOK( + "versions { producer: 10 min_consumer: 8 bad_consumers: 1 bad_consumers: " + "3 " + "}", + opts); + EXPECT_EQ(10, graph_.versions().producer()); + EXPECT_EQ(8, graph_.versions().min_consumer()); + ASSERT_EQ(3, graph_.versions().bad_consumers_size()); + EXPECT_EQ(1, graph_.versions().bad_consumers(0)); + EXPECT_EQ(2, graph_.versions().bad_consumers(1)); + EXPECT_EQ(3, graph_.versions().bad_consumers(2)); + + // This one is a no-op. + ExpectOK("versions { producer: 20 min_consumer: 7 }", opts); + EXPECT_EQ(10, graph_.versions().producer()); + EXPECT_EQ(8, graph_.versions().min_consumer()); + ASSERT_EQ(3, graph_.versions().bad_consumers_size()); + EXPECT_EQ(1, graph_.versions().bad_consumers(0)); + EXPECT_EQ(2, graph_.versions().bad_consumers(1)); + EXPECT_EQ(3, graph_.versions().bad_consumers(2)); +} + } // namespace } // namespace tensorflow diff --git a/tensorflow/core/kernels/hexagon/graph_transferer.cc b/tensorflow/core/kernels/hexagon/graph_transferer.cc index 5b2a95a3712522..3dfaaf08bebbb0 100644 --- a/tensorflow/core/kernels/hexagon/graph_transferer.cc +++ b/tensorflow/core/kernels/hexagon/graph_transferer.cc @@ -71,7 +71,7 @@ Status GraphTransferer::LoadGraphFromProto( const OutputTensorMap& output_tensor_map) { ImportGraphDefOptions opts; Graph graph(OpRegistry::Global()); - ShapeRefiner shape_refiner(graph.op_registry()); + ShapeRefiner shape_refiner(graph.versions().producer(), graph.op_registry()); VLOG(1) << "Start import graph"; Status status = ImportGraphDef(opts, graph_def, &graph, &shape_refiner); if (!status.ok()) { diff --git a/tensorflow/core/ops/array_ops_test.cc b/tensorflow/core/ops/array_ops_test.cc index 0f5d71fcd72b4a..98b94838942529 100644 --- a/tensorflow/core/ops/array_ops_test.cc +++ b/tensorflow/core/ops/array_ops_test.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/platform/test.h" +#include "tensorflow/core/public/version.h" namespace tensorflow { @@ -161,9 +162,9 @@ TEST(ArrayOpsTest, Identity_ShapeFnHandles) { // Check that handle dtypes are preserved. const OpRegistrationData* op_reg_data; TF_ASSERT_OK(OpRegistry::Global()->LookUp(op.name, &op_reg_data)); - shape_inference::InferenceContext c(&op.node_def, op_reg_data->op_def, - {TensorShapeProto()}, {}, {}, {}, - {DT_BOOL}); + shape_inference::InferenceContext c(TF_GRAPH_DEF_VERSION, &op.node_def, + op_reg_data->op_def, {TensorShapeProto()}, + {}, {}, {}, {DT_BOOL}); TF_ASSERT_OK(c.construction_status()); ASSERT_TRUE(op_reg_data->shape_inference_fn != nullptr); TF_ASSERT_OK(c.Run(op_reg_data->shape_inference_fn)); diff --git a/tensorflow/core/ops/math_ops_test.cc b/tensorflow/core/ops/math_ops_test.cc index 84264f13dc6698..8881857b29b69e 100644 --- a/tensorflow/core/ops/math_ops_test.cc +++ b/tensorflow/core/ops/math_ops_test.cc @@ -229,7 +229,7 @@ TEST(MathOpsTest, Select_ShapeFn) { ASSERT_TRUE(op_reg_data->shape_inference_fn != nullptr); shape_inference::InferenceContext c( - &op.node_def, op_reg_data->op_def, + TF_GRAPH_DEF_VERSION, &op.node_def, op_reg_data->op_def, {TensorShapeProto(), TensorShapeProto(), TensorShapeProto()}, {}, {}, {TensorShapeProto(), i0, i1}, {}); TF_ASSERT_OK(c.construction_status()); @@ -242,7 +242,7 @@ TEST(MathOpsTest, Select_ShapeFn) { i1.add_dim()->set_size(2); i1.add_dim()->set_size(2); shape_inference::InferenceContext c2( - &op.node_def, op_reg_data->op_def, + TF_GRAPH_DEF_VERSION, &op.node_def, op_reg_data->op_def, {TensorShapeProto(), TensorShapeProto(), TensorShapeProto()}, {}, {}, {TensorShapeProto(), i0, i2}, {}); TF_ASSERT_OK(c.construction_status()); diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 9c7c4b38a6ef76..0d8fd18b696fa9 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -79,6 +79,9 @@ limitations under the License. // used for tf.split, ReverseV2 is now used by tf.reverse, ConcatV2 is // now used by tf.concat_v2 (and soon tf.concat). Graphs use flooring // division and mod semantics. TensorArrayV3. (12dec2016) +// Also considered the version for when it is required for reduction +// ops' indices to be scalar or vector, and not higher rank. +// Some earlier graph def versions allowed this. // 21. Dropped FunctionDef.Node support, switched to node_def introduced // in version 12. (11jan2017) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index e36e32f59f1503..9fbbb30d7ac972 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -579,6 +579,7 @@ py_test( ":nn_grad", ":nn_ops", ":random_ops", + ":test_ops", ":variables", "//tensorflow/core:protos_all_py", "//third_party/py/numpy", diff --git a/tensorflow/python/framework/common_shapes.py b/tensorflow/python/framework/common_shapes.py index 487387cd836aed..5ec73afa9974b9 100644 --- a/tensorflow/python/framework/common_shapes.py +++ b/tensorflow/python/framework/common_shapes.py @@ -635,6 +635,7 @@ def _call_cpp_shape_fn_impl( input_tensors_as_shapes_needed, debug_python_shape_fn, require_shape_fn): """Core implementaton of call_cpp_shape_fn.""" + graph_def_version = op.graph.graph_def_versions.producer node_def_str = op.node_def.SerializeToString() def tensor_to_inference_result(t): @@ -666,8 +667,8 @@ def tensor_to_inference_result(t): try: with errors.raise_exception_on_not_ok_status() as status: output = pywrap_tensorflow.RunCppShapeInference( - node_def_str, input_shapes, input_tensors, input_tensors_as_shapes, - status) + graph_def_version, node_def_str, input_shapes, input_tensors, + input_tensors_as_shapes, status) except errors.InvalidArgumentError as err: if err.message.startswith("No shape inference function exists for op"): missing_shape_fn = True diff --git a/tensorflow/python/framework/cpp_shape_inference.cc b/tensorflow/python/framework/cpp_shape_inference.cc index cc08e3b70506ae..e1fab4fc2d1a22 100644 --- a/tensorflow/python/framework/cpp_shape_inference.cc +++ b/tensorflow/python/framework/cpp_shape_inference.cc @@ -47,7 +47,7 @@ void ProtoFromShapeHandle(tensorflow::shape_inference::ShapeHandle s, } Status RunCppShapeInferenceImpl( - const string& serialized_node_def, + int graph_def_version, const string& serialized_node_def, const std::vector& input_serialized_shapes, const std::vector& input_constant_tensor_values, const std::vector& input_constant_tensor_as_shape_values, @@ -115,8 +115,9 @@ Status RunCppShapeInferenceImpl( // Run shape inference. tensorflow::shape_inference::InferenceContext c( - &node, op_reg_data->op_def, input_shapes, input_tensors, - input_tensor_as_shapes_protos, input_handle_shapes, input_handle_dtypes); + graph_def_version, &node, op_reg_data->op_def, input_shapes, + input_tensors, input_tensor_as_shapes_protos, input_handle_shapes, + input_handle_dtypes); TF_RETURN_IF_ERROR(c.construction_status()); TF_RETURN_IF_ERROR(c.Run(op_reg_data->shape_inference_fn)); @@ -151,7 +152,7 @@ Status RunCppShapeInferenceImpl( } // namespace std::vector RunCppShapeInference( - const string& serialized_node_def, + int graph_def_version, const string& serialized_node_def, const std::vector& input_serialized_shapes, PyObject* input_constant_tensor_values, const std::vector& input_constant_tensor_as_shape_values, @@ -171,7 +172,7 @@ std::vector RunCppShapeInference( std::vector output; string input_tensors_needed_out; tensorflow::Status status = RunCppShapeInferenceImpl( - serialized_node_def, input_serialized_shapes, + graph_def_version, serialized_node_def, input_serialized_shapes, input_constant_tensor_values_v, input_constant_tensor_as_shape_values, &output, &input_tensors_needed_out); diff --git a/tensorflow/python/framework/cpp_shape_inference.h b/tensorflow/python/framework/cpp_shape_inference.h index 79b37aa6b42ea3..afca7277c77506 100644 --- a/tensorflow/python/framework/cpp_shape_inference.h +++ b/tensorflow/python/framework/cpp_shape_inference.h @@ -42,7 +42,7 @@ namespace swig { // This is temporary code to be used during the migration // from python shape inference functions to C++ shape inference functions. std::vector RunCppShapeInference( - const string& serialized_node_def, + int graph_def_version, const string& serialized_node_def, const std::vector& input_serialized_shapes, PyObject* input_constant_tensor_values, const std::vector& input_constant_tensor_as_shape_values, diff --git a/tensorflow/python/framework/importer_test.py b/tensorflow/python/framework/importer_test.py index 813f27bd17805d..76618f5234cb89 100644 --- a/tensorflow/python/framework/importer_test.py +++ b/tensorflow/python/framework/importer_test.py @@ -31,6 +31,7 @@ from tensorflow.python.framework import op_def_registry from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_ops # pylint: disable=unused-import from tensorflow.python.framework import versions from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradients_impl @@ -829,6 +830,24 @@ def testVersionHigh(self): with self.assertRaisesRegexp(Exception, pat): sess.run(x) + def testVersionAppliesToOpConstruction(self): + """These tests rely on shape fns in test_ops.cc.""" + with ops.Graph().as_default(): + importer.import_graph_def( + self._MakeGraphDef( + "node { name: 'A' op: 'RequiresOlderGraphVersion' }", + producer=versions.GRAPH_DEF_VERSION - 1), + return_elements=["A"]) + + with ops.Graph().as_default(): + with self.assertRaisesWithPredicateMatch(ValueError, + "Wrong graph version.*"): + importer.import_graph_def( + self._MakeGraphDef( + "node { name: 'A' op: 'RequiresOlderGraphVersion' }", + producer=versions.GRAPH_DEF_VERSION), + return_elements=["A"]) + def testDefaultAttrsAdded(self): with ops.Graph().as_default(): a = importer.import_graph_def( diff --git a/tensorflow/python/framework/test_ops.cc b/tensorflow/python/framework/test_ops.cc index c19094847d88c0..19f07fb754bc4d 100644 --- a/tensorflow/python/framework/test_ops.cc +++ b/tensorflow/python/framework/test_ops.cc @@ -19,6 +19,7 @@ limitations under the License. #include "tensorflow/core/framework/resource_handle.pb.h" #include "tensorflow/core/framework/resource_mgr.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/public/version.h" namespace tensorflow { @@ -31,6 +32,16 @@ REGISTER_OP("GraphDefVersion") .SetIsStateful() .SetShapeFn(shape_inference::ScalarShape); +REGISTER_OP("RequiresOlderGraphVersion") + .Output("version: int32") + .SetIsStateful() + .SetShapeFn([](shape_inference::InferenceContext* c) { + if (c->graph_def_version() != TF_GRAPH_DEF_VERSION - 1) { + return errors::InvalidArgument("Wrong graph version for shape"); + } + return shape_inference::ScalarShape(c); + }); + REGISTER_OP("Old") .SetShapeFn(shape_inference::UnknownShape) .Deprecated(8, "For reasons"); diff --git a/tensorflow/python/kernel_tests/reduction_ops_test.py b/tensorflow/python/kernel_tests/reduction_ops_test.py index 0da5a2ecc563e2..316c23609cd667 100644 --- a/tensorflow/python/kernel_tests/reduction_ops_test.py +++ b/tensorflow/python/kernel_tests/reduction_ops_test.py @@ -241,6 +241,13 @@ def testPartialShapes(self): c_unknown_indices, unknown_indices, keep_dims=True) self.assertEqual(2, s_unknown_indices_keep.get_shape().ndims) + def testWrongShapeForReductionIndices(self): + reduction_axes = [[1], [2]] + c_unknown = array_ops.placeholder(dtypes.float32) + with self.assertRaisesWithPredicateMatch(ValueError, + ".*must be at most rank 1.*"): + math_ops.reduce_sum(c_unknown, reduction_axes) + # Int64?? def _compareGradient(self, shape, sum_shape, reduction_axes): From 15c5bb2137865896565aee3f4c16af5ddab196e5 Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Fri, 3 Mar 2017 16:48:54 -0800 Subject: [PATCH 41/50] Running the update_version.sh script to update all version #s to 1.0.1. --- README.md | 8 +-- tensorflow/contrib/cmake/setup.py | 2 +- tensorflow/core/public/version.h | 2 +- tensorflow/g3doc/get_started/os_setup.md | 92 ++++++++++++------------ tensorflow/tools/pip_package/setup.py | 2 +- 5 files changed, 53 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index c77b2924c6b82a..d85e537760aaba 100644 --- a/README.md +++ b/README.md @@ -33,10 +33,10 @@ and discussion.** People who are a little more adventurous can also try our nightly binaries: -* Linux CPU-only: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.0.0-cp27-none-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave)) / [Python 3.4](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.0.0-cp34-cp34m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/)) / [Python 3.5](https://ci.tensorflow.org/view/Nightly/job/nightly-python35-linux-cpu/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.0.0-cp35-cp35m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-python35-linux-cpu/)) -* Linux GPU: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow_gpu-1.0.0-cp27-none-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/)) / [Python 3.4](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow_gpu-1.0.0-cp34-cp34m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/)) / [Python 3.5](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow_gpu-1.0.0-cp35-cp35m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/)) -* Mac CPU-only: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.0.0-py2-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac-slave/)) / [Python 3](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.0.0-py3-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac-slave/)) -* Mac GPU: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-mac-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-mac/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow_gpu-1.0.0-py2-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-mac-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-mac/)) / [Python 3](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-mac-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-mac/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow_gpu-1.0.0-py3-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-mac-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-mac/)) +* Linux CPU-only: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.0.1-cp27-none-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave)) / [Python 3.4](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.0.1-cp34-cp34m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/)) / [Python 3.5](https://ci.tensorflow.org/view/Nightly/job/nightly-python35-linux-cpu/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.0.1-cp35-cp35m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-python35-linux-cpu/)) +* Linux GPU: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow_gpu-1.0.1-cp27-none-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/)) / [Python 3.4](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow_gpu-1.0.1-cp34-cp34m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/)) / [Python 3.5](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow_gpu-1.0.1-cp35-cp35m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/)) +* Mac CPU-only: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.0.1-py2-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac-slave/)) / [Python 3](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.0.1-py3-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac-slave/)) +* Mac GPU: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-mac-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-mac/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow_gpu-1.0.1-py2-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-mac-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-mac/)) / [Python 3](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-mac-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-mac/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow_gpu-1.0.1-py3-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-mac-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-mac/)) * [Android](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-android/TF_BUILD_CONTAINER_TYPE=ANDROID,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=NO_PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=android-slave/lastSuccessfulBuild/artifact/bazel-out/local_linux/bin/tensorflow/examples/android/tensorflow_demo.apk) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-android/TF_BUILD_CONTAINER_TYPE=ANDROID,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=NO_PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=android-slave/)) * Android: [demo APK](https://ci.tensorflow.org/view/Nightly/job/nightly-android/lastSuccessfulBuild/artifact/out/tensorflow_demo.apk), [native libs](http://ci.tensorflow.org/view/Nightly/job/nightly-android/lastSuccessfulBuild/artifact/out/native/) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-android/)) diff --git a/tensorflow/contrib/cmake/setup.py b/tensorflow/contrib/cmake/setup.py index ce5e12b9ff6241..b35d73862553b1 100644 --- a/tensorflow/contrib/cmake/setup.py +++ b/tensorflow/contrib/cmake/setup.py @@ -26,7 +26,7 @@ from setuptools.command.install import install as InstallCommandBase from setuptools.dist import Distribution -_VERSION = '1.0.0-cmake-experimental' +_VERSION = '1.0.1-cmake-experimental' REQUIRED_PACKAGES = [ 'numpy >= 1.11.0', diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 9c7c4b38a6ef76..3f052aeb0f056c 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -20,7 +20,7 @@ limitations under the License. #define TF_MAJOR_VERSION 1 #define TF_MINOR_VERSION 0 -#define TF_PATCH_VERSION 0 +#define TF_PATCH_VERSION 1 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", // "-beta", "-rc", "-rc.1") diff --git a/tensorflow/g3doc/get_started/os_setup.md b/tensorflow/g3doc/get_started/os_setup.md index d6e822a2796cef..af626ac1389580 100644 --- a/tensorflow/g3doc/get_started/os_setup.md +++ b/tensorflow/g3doc/get_started/os_setup.md @@ -78,51 +78,51 @@ If the above commands do not work on your system, you can follow these instructi ```bash # Ubuntu/Linux 64-bit, CPU only, Python 2.7 -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.0-cp27-none-linux_x86_64.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp27-none-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 2.7 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Installing from sources" below. -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.0-cp27-none-linux_x86_64.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.1-cp27-none-linux_x86_64.whl # Mac OS X, CPU only, Python 2.7: -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.0.0-py2-none-any.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.0.1-py2-none-any.whl # Mac OS X, GPU enabled, Python 2.7: -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow_gpu-1.0.0-py2-none-any.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow_gpu-1.0.1-py2-none-any.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.3 -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.0-cp33-cp33m-linux_x86_64.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp33-cp33m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.3 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Installing from sources" below. -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.0-cp33-cp33m-linux_x86_64.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.1-cp33-cp33m-linux_x86_64.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.4 -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.0-cp34-cp34m-linux_x86_64.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.4 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Installing from sources" below. -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.0-cp34-cp34m-linux_x86_64.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.1-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.5 -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.0-cp35-cp35m-linux_x86_64.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp35-cp35m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.5 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Installing from sources" below. -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.0-cp35-cp35m-linux_x86_64.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.1-cp35-cp35m-linux_x86_64.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.6 -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.0-cp36-cp36m-linux_x86_64.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp36-cp36m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.6 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Installing from sources" below. -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.0-cp36-cp36m-linux_x86_64.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.1-cp36-cp36m-linux_x86_64.whl # Mac OS X, CPU only, Python 3.4 or 3.5: -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.0.0-py3-none-any.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.0.1-py3-none-any.whl # Mac OS X, GPU enabled, Python 3.4 or 3.5: -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow_gpu-1.0.0-py3-none-any.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow_gpu-1.0.1-py3-none-any.whl ``` Install TensorFlow: @@ -164,14 +164,14 @@ Both distributions include pip. To install the CPU-only version of TensorFlow, enter the following command at a command prompt: ```bat -C:\> pip install --upgrade https://storage.googleapis.com/tensorflow/windows/cpu/tensorflow-1.0.0-cp35-cp35m-win_amd64.whl +C:\> pip install --upgrade https://storage.googleapis.com/tensorflow/windows/cpu/tensorflow-1.0.1-cp35-cp35m-win_amd64.whl ``` To install the GPU version of TensorFlow, enter the following command at a command prompt: ```bat -C:\> pip install --upgrade https://storage.googleapis.com/tensorflow/windows/gpu/tensorflow_gpu-1.0.0-cp35-cp35m-win_amd64.whl +C:\> pip install --upgrade https://storage.googleapis.com/tensorflow/windows/gpu/tensorflow_gpu-1.0.1-cp35-cp35m-win_amd64.whl ``` You can now [test your installation](#test-the-tensorflow-installation). @@ -226,51 +226,51 @@ Now, install TensorFlow just as you would for a regular Pip installation. First ```bash # Ubuntu/Linux 64-bit, CPU only, Python 2.7 -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.0-cp27-none-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp27-none-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 2.7 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Installing from sources" below. -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.0-cp27-none-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.1-cp27-none-linux_x86_64.whl # Mac OS X, CPU only, Python 2.7: -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.0.0-py2-none-any.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.0.1-py2-none-any.whl # Mac OS X, GPU enabled, Python 2.7: -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow_gpu-1.0.0-py2-none-any.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow_gpu-1.0.1-py2-none-any.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.3 -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.0-cp33-cp33m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp33-cp33m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.3 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Installing from sources" below. -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.0-cp33-cp33m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.1-cp33-cp33m-linux_x86_64.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.4 -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.0-cp34-cp34m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.4 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Installing from sources" below. -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.0-cp34-cp34m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.1-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.5 -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.0-cp35-cp35m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp35-cp35m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.5 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Installing from sources" below. -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.0-cp35-cp35m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.1-cp35-cp35m-linux_x86_64.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.6 -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.0-cp36-cp36m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp36-cp36m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.6 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Installing from sources" below. -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.0-cp36-cp36m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.1-cp36-cp36m-linux_x86_64.whl # Mac OS X, CPU only, Python 3.4 or 3.5: -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.0.0-py3-none-any.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.0.1-py3-none-any.whl # Mac OS X, GPU enabled, Python 3.4 or 3.5: -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow_gpu-1.0.0-py3-none-any.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow_gpu-1.0.1-py3-none-any.whl ``` Finally install TensorFlow: @@ -392,52 +392,52 @@ select the correct binary to install: ```bash # Ubuntu/Linux 64-bit, CPU only, Python 2.7 -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.0-cp27-none-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp27-none-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 2.7 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Installing from sources" below. -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.0-cp27-none-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.1-cp27-none-linux_x86_64.whl # Mac OS X, CPU only, Python 2.7: -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.0.0-py2-none-any.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.0.1-py2-none-any.whl # Mac OS X, GPU enabled, Python 2.7: -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow_gpu-1.0.0-py2-none-any.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow_gpu-1.0.1-py2-none-any.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.3 -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.0-cp33-cp33m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp33-cp33m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.3 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Installing from sources" below. -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.0-cp33-cp33m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.1-cp33-cp33m-linux_x86_64.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.4 -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.0-cp34-cp34m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.4 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Installing from sources" below. -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.0-cp34-cp34m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.1-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.5 -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.0-cp35-cp35m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp35-cp35m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.5 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Installing from sources" below. -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.0-cp35-cp35m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.1-cp35-cp35m-linux_x86_64.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.6 -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.0-cp36-cp36m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp36-cp36m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.6 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Installing from sources" below. -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.0-cp36-cp36m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.0.1-cp36-cp36m-linux_x86_64.whl # Mac OS X, CPU only, Python 3.4 or 3.5: -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.0.0-py3-none-any.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.0.1-py3-none-any.whl # Mac OS X, GPU enabled, Python 3.4 or 3.5: -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow_gpu-1.0.0-py3-none-any.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow_gpu-1.0.1-py3-none-any.whl ``` Finally install TensorFlow: @@ -505,7 +505,7 @@ code. code. We also have tags with `latest` replaced by a released version (e.g., -`1.0.0-gpu`). +`1.0.1-gpu`). With Docker the installation is as follows: @@ -910,7 +910,7 @@ $ bazel build -c opt --config=sycl //tensorflow/tools/pip_package:build_pip_pack $ bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg # The name of the .whl file will depend on your platform. -$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.0.0-py2-none-any.whl +$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.0.1-py2-none-any.whl ``` ## Optimizing CPU performance diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index fba2ead700ba4a..070ff5536c3037 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -29,7 +29,7 @@ # This version string is semver compatible, but incompatible with pip. # For pip, we will remove all '-' characters from this string, and use the # result for pip. -_VERSION = '1.0.0' +_VERSION = '1.0.1' REQUIRED_PACKAGES = [ 'numpy >= 1.11.0', From 96d555e2ddb6bcb233a38f23f1d752657e6cf9b9 Mon Sep 17 00:00:00 2001 From: Patrick Nguyen Date: Sat, 4 Mar 2017 15:04:11 -0800 Subject: [PATCH 42/50] Remove extra exported symbols. Change: 149211889 --- tensorflow/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tensorflow/__init__.py b/tensorflow/__init__.py index 92d390a9764d42..0bca6f8fb80519 100644 --- a/tensorflow/__init__.py +++ b/tensorflow/__init__.py @@ -37,3 +37,16 @@ def __getattr__(self, item): contrib = _LazyContribLoader() + +del absolute_import +del division +del print_function + +# These symbols appear because we import the python package which +# in turn imports from tensorflow.core and tensorflow.python. They +# must come from this module. So python adds these symbols for the +# resolution to succeed. +# pylint: disable=undefined-variable +del python +del core +# pylint: enable=undefined-variable From e8b9bd928d20edd565e2898766beb7eb21def7e0 Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Mon, 13 Feb 2017 14:45:27 -0800 Subject: [PATCH 43/50] Install numpy from source for Python 3.5 to fix the nightly build. Change: 147395987 --- .../ci_build/install/install_python3.5_pip_packages.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh b/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh index f1c3f38812d8c5..a0cd540e91d1f5 100755 --- a/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh +++ b/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh @@ -99,7 +99,11 @@ pip3.5 install --upgrade protobuf==3.0.0 rm -rf /usr/lib/python3/dist-packages/six* # Install numpy, scipy and scikit-learn required by the builds -pip3.5 install --upgrade numpy + +# numpy needs to be installed from source to fix segfaults. See: +# https://github.com/tensorflow/tensorflow/issues/6968 +# This workaround isn't needed for Ubuntu 16.04 or later. +pip3.5 install --no-binary=:all: --upgrade numpy set +e SCIPY_VERSION="0.17.1" From 74ab5f163d9b337a498652be7ae6325a4217dc6c Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Sat, 4 Mar 2017 23:30:22 -0800 Subject: [PATCH 44/50] Bump the installed protobuf for the python 3.5 build to protobuf 3.1.0 --- .../tools/ci_build/install/install_python3.5_pip_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh b/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh index a0cd540e91d1f5..9dddcb83ac2546 100755 --- a/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh +++ b/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh @@ -93,7 +93,7 @@ set -e pip3.5 install --upgrade six==1.10.0 # Install protobuf. -pip3.5 install --upgrade protobuf==3.0.0 +pip3.5 install --upgrade protobuf==3.1.0 # Remove obsolete version of six, which can sometimes confuse virtualenv. rm -rf /usr/lib/python3/dist-packages/six* From 5d2a7d9e3b865bdb1432cfca200a9a0cc33f57ed Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Mon, 6 Mar 2017 09:52:08 -0800 Subject: [PATCH 45/50] Remove the extra linkopt in ci_parameterized_build script. --- tensorflow/tools/ci_build/ci_parameterized_build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh index f1c8b51b376f21..5ac4f0f0a98dd1 100755 --- a/tensorflow/tools/ci_build/ci_parameterized_build.sh +++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh @@ -118,8 +118,7 @@ DOCKER_MAIN_CMD="${CI_BUILD_DIR}/ci_build.sh" NO_DOCKER_MAIN_CMD="${CI_BUILD_DIR}/builds/configured" # Additional option flags to apply when Docker is unavailable (e.g., on Mac) -NO_DOCKER_OPT_FLAG="--linkopt=-headerpad_max_install_names "\ -"--genrule_strategy=standalone" +NO_DOCKER_OPT_FLAG="--genrule_strategy=standalone" DO_DOCKER=1 From 30fa482af97b3ebf5ab0b72ed7e2711bdb66374a Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Mon, 6 Mar 2017 17:09:52 -0800 Subject: [PATCH 46/50] Updating the RELEASE.md for patch version 1.0.1 --- RELEASE.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index d732672f8d93ee..72367af1c42ab6 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,9 @@ +# Release 1.0.1 + +## Bug Fixes and Other Changes +* Change GraphConstructor to not increase the version when importing, but instead take the min of all versions. +* Google Cloud Storage fixes + # Release 1.0.0 ## Major Features and Improvements From 60d35bbfc623740955c4fbbe988b3d115d687e3b Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Tue, 7 Mar 2017 10:39:02 -0800 Subject: [PATCH 47/50] Implementing Gunan's suggestions to the RELEASE.md file. Including missed changes in 1.0 docs. --- RELEASE.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index 72367af1c42ab6..8f413f38bbf9e5 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -3,6 +3,7 @@ ## Bug Fixes and Other Changes * Change GraphConstructor to not increase the version when importing, but instead take the min of all versions. * Google Cloud Storage fixes +* Removed `tf.core` and `tf.python` modules from the API. These were never intended to be exposed. Please use the same objects through top-level `tf` module instead. # Release 1.0.0 @@ -92,6 +93,8 @@ To help you upgrade your existing TensorFlow Python code to match the API change from the tensorflow::ops namespace to tensorflow. * Change arg order for `{softmax,sparse_softmax,sigmoid}_cross_entropy_with_logits` to be (labels, predictions), and force use of named args. * tf.nn.rnn_cell.* and most functions in tf.nn.rnn.* (with the exception of dynamic_rnn and raw_rnn) are temporarily in tf.contrib.rnn. They will be moved back into core for TF 1.1. +* `tf.nn.sampled_softmax_loss` and `tf.nn.nce_loss` have both changed their API such that you need to switch the `inputs, labels` to `labels, inputs` parameters. +* The shape keyword argument of the SparseTensor constructor changes its name to dense_shape between Tensorflow 0.12 and Tensorflow 1.0. ## Bug Fixes and Other Changes * Numerous C++ API updates. From e895d5ca395c2362df4f5c8f08b68501b41f8a98 Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Tue, 7 Mar 2017 12:49:01 -0800 Subject: [PATCH 48/50] Fixing 2 minor issues as suggested by Gunan. --- RELEASE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 8f413f38bbf9e5..c569491504d6f3 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -2,7 +2,7 @@ ## Bug Fixes and Other Changes * Change GraphConstructor to not increase the version when importing, but instead take the min of all versions. -* Google Cloud Storage fixes +* Google Cloud Storage fixes. * Removed `tf.core` and `tf.python` modules from the API. These were never intended to be exposed. Please use the same objects through top-level `tf` module instead. # Release 1.0.0 @@ -94,7 +94,7 @@ To help you upgrade your existing TensorFlow Python code to match the API change * Change arg order for `{softmax,sparse_softmax,sigmoid}_cross_entropy_with_logits` to be (labels, predictions), and force use of named args. * tf.nn.rnn_cell.* and most functions in tf.nn.rnn.* (with the exception of dynamic_rnn and raw_rnn) are temporarily in tf.contrib.rnn. They will be moved back into core for TF 1.1. * `tf.nn.sampled_softmax_loss` and `tf.nn.nce_loss` have both changed their API such that you need to switch the `inputs, labels` to `labels, inputs` parameters. -* The shape keyword argument of the SparseTensor constructor changes its name to dense_shape between Tensorflow 0.12 and Tensorflow 1.0. +* The shape keyword argument of the `SparseTensor` constructor changes its name to `dense_shape` between Tensorflow 0.12 and Tensorflow 1.0. ## Bug Fixes and Other Changes * Numerous C++ API updates. From 00cff0694a36f815c7a5d34d1b1255ba1e025527 Mon Sep 17 00:00:00 2001 From: Neal Wu Date: Fri, 28 Apr 2017 15:01:23 -0700 Subject: [PATCH 49/50] Indicate that tf.contrib.rnn functions will be moved back to core for TF 1.2, not 1.1 --- RELEASE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index c569491504d6f3..72aa4ba7753054 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -85,14 +85,14 @@ To help you upgrade your existing TensorFlow Python code to match the API change * `SparseTensor.shape` has been renamed to `SparseTensor.dense_shape`. Same for `SparseTensorValue.shape`. * Replace tf.scalar_summary, tf.histogram_summary, tf.audio_summary, tf.image_summary with tf.summary.scalar, tf.summary.histogram, tf.summary.audio, tf.summary.image, respectively. The new summary ops take name rather than tag as their first argument, meaning summary ops now respect TensorFlow name scopes. -* Replace tf.train.SummaryWriter and tf.train.SummaryWriterCache with tf.summary.FileWriter and tf.summary.FileWriterCache. +* Replace tf.train.SummaryWriter and tf.train.SummaryWriterCache with tf.summary.FileWriter and tf.summary.FileWriterCache. * Removes RegisterShape from public API. Use C++ shape function registration instead. * Deprecated `_ref` dtypes from the python API. * In the C++ API (in tensorflow/cc), Input, Output, etc. have moved from the tensorflow::ops namespace to tensorflow. * Change arg order for `{softmax,sparse_softmax,sigmoid}_cross_entropy_with_logits` to be (labels, predictions), and force use of named args. -* tf.nn.rnn_cell.* and most functions in tf.nn.rnn.* (with the exception of dynamic_rnn and raw_rnn) are temporarily in tf.contrib.rnn. They will be moved back into core for TF 1.1. +* tf.nn.rnn_cell.* and most functions in tf.nn.rnn.* (with the exception of dynamic_rnn and raw_rnn) are temporarily in tf.contrib.rnn. They will be moved back into core for TF 1.2. * `tf.nn.sampled_softmax_loss` and `tf.nn.nce_loss` have both changed their API such that you need to switch the `inputs, labels` to `labels, inputs` parameters. * The shape keyword argument of the `SparseTensor` constructor changes its name to `dense_shape` between Tensorflow 0.12 and Tensorflow 1.0. From 905662a1c047f314ad295e27d2e2a00b4aa51e90 Mon Sep 17 00:00:00 2001 From: Jingtian Peng Date: Wed, 3 May 2017 05:30:48 +0800 Subject: [PATCH 50/50] [issue#9328]fix invalid pointer when free() (#9595) --- tensorflow/tools/docker/Dockerfile | 1 + tensorflow/tools/docker/Dockerfile.devel | 1 + tensorflow/tools/docker/Dockerfile.devel-gpu | 1 + tensorflow/tools/docker/Dockerfile.gpu | 1 + 4 files changed, 4 insertions(+) diff --git a/tensorflow/tools/docker/Dockerfile b/tensorflow/tools/docker/Dockerfile index a9852586e933c8..bd9b3665a2b0a6 100644 --- a/tensorflow/tools/docker/Dockerfile +++ b/tensorflow/tools/docker/Dockerfile @@ -9,6 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libfreetype6-dev \ libpng12-dev \ libzmq3-dev \ + libtcmalloc-minimal4 \ pkg-config \ python \ python-dev \ diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel index c910b83cffcc7e..74daa16b6323a7 100644 --- a/tensorflow/tools/docker/Dockerfile.devel +++ b/tensorflow/tools/docker/Dockerfile.devel @@ -10,6 +10,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libfreetype6-dev \ libpng12-dev \ libzmq3-dev \ + libtcmalloc-minimal4 \ pkg-config \ python-dev \ rsync \ diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu index b877086ffaa860..6d987926c23212 100644 --- a/tensorflow/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow/tools/docker/Dockerfile.devel-gpu @@ -10,6 +10,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libfreetype6-dev \ libpng12-dev \ libzmq3-dev \ + libtcmalloc-minimal4 \ pkg-config \ python-dev \ rsync \ diff --git a/tensorflow/tools/docker/Dockerfile.gpu b/tensorflow/tools/docker/Dockerfile.gpu index ca3252e1d934a7..b4a964bc5485da 100644 --- a/tensorflow/tools/docker/Dockerfile.gpu +++ b/tensorflow/tools/docker/Dockerfile.gpu @@ -9,6 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libfreetype6-dev \ libpng12-dev \ libzmq3-dev \ + libtcmalloc-minimal4 \ pkg-config \ python \ python-dev \