diff --git a/.github/workflows/pylint-presubmit.yml b/.github/workflows/pylint-presubmit.yml deleted file mode 100644 index 3d76dd960ded2b..00000000000000 --- a/.github/workflows/pylint-presubmit.yml +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -name: PyLint -on: - pull_request: - paths: - - '**.py' - -jobs: - build: - name: PyLint - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Get file changes - id: get_file_changes - uses: trilom/file-changes-action@v1.2.4 - with: - output: ' ' - - name: Report list of changed files - run: | - echo Changed files: ${{ steps.get_file_changes.outputs.files }} - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install pylint numpy wheel - pip install keras_preprocessing --no-deps - - name: Run PyLint on changed files - run: | - echo "${{ steps.get_file_changes.outputs.files}}" | tr " " "\n" | grep ".py$" | xargs pylint --rcfile=tensorflow/tools/ci_build/pylintrc diff --git a/.github/workflows/update-nightly.yml b/.github/workflows/update-nightly.yml deleted file mode 100644 index 0265ffbebe2ec0..00000000000000 --- a/.github/workflows/update-nightly.yml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -on: - workflow_dispatch: # Allow manual triggers - schedule: - - cron: 0 4 * * * # 4am UTC is 9pm PDT and 8pm PST -name: Set nightly branch to master HEAD -jobs: - master-to-nightly: - if: github.repository == 'tensorflow/tensorflow' # Don't do this in forks - runs-on: ubuntu-latest - steps: - - uses: zofrex/mirror-branch@v1 - name: Set nightly branch to master HEAD - with: - target-branch: 'nightly' diff --git a/RELEASE.md b/RELEASE.md index 21147f6e07b5e4..c7840136158481 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,166 @@ +# Release 2.7.4 + +This releases introduces several vulnerability fixes: + +* Fixes a `CHECK` failure in tf.reshape caused by overflows ([CVE-2022-35934](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35934)) +* Fixes a `CHECK` failure in `SobolSample` caused by missing validation ([CVE-2022-35935](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35935)) +* Fixes an OOB read in `Gather_nd` op in TF Lite ([CVE-2022-35937](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35937)) +* Fixes a `CHECK` failure in `TensorListReserve` caused by missing validation ([CVE-2022-35960](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35960)) +* Fixes an OOB write in `Scatter_nd` op in TF Lite ([CVE-2022-35939](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35939)) +* Fixes an integer overflow in `RaggedRangeOp` ([CVE-2022-35940](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35940)) +* Fixes a `CHECK` failure in `AvgPoolOp` ([CVE-2022-35941](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35941)) +* Fixes a `CHECK` failures in `UnbatchGradOp` ([CVE-2022-35952](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35952)) +* Fixes a segfault TFLite converter on per-channel quantized transposed convolutions ([CVE-2022-36027](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36027)) +* Fixes a `CHECK` failures in `AvgPool3DGrad` ([CVE-2022-35959](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35959)) +* Fixes a `CHECK` failures in `FractionalAvgPoolGrad` ([CVE-2022-35963](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35963)) +* Fixes a segfault in `BlockLSTMGradV2` ([CVE-2022-35964](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35964)) +* Fixes a segfault in `LowerBound` and `UpperBound` ([CVE-2022-35965](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35965)) +* Fixes a segfault in `QuantizedAvgPool` ([CVE-2022-35966](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35966)) +* Fixes a segfault in `QuantizedAdd` ([CVE-2022-35967](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35967)) +* Fixes a `CHECK` fail in `AvgPoolGrad` ([CVE-2022-35968](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35968)) +* Fixes a `CHECK` fail in `Conv2DBackpropInput` ([CVE-2022-35969](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35969)) +* Fixes a segfault in `QuantizedInstanceNorm` ([CVE-2022-35970](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35970)) +* Fixes a `CHECK` fail in `FakeQuantWithMinMaxVars` ([CVE-2022-35971](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35971)) +* Fixes a segfault in `Requantize` ([CVE-2022-36017](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36017)) +* Fixes a segfault in `QuantizedBiasAdd` ([CVE-2022-35972](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35972)) +* Fixes a `CHECK` fail in `FakeQuantWithMinMaxVarsPerChannel` ([CVE-2022-36019](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36019)) +* Fixes a segfault in `QuantizedMatMul` ([CVE-2022-35973](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35973)) +* Fixes a segfault in `QuantizeDownAndShrinkRange` ([CVE-2022-35974](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35974)) +* Fixes segfaults in `QuantizedRelu` and `QuantizedRelu6` ([CVE-2022-35979](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35979)) +* Fixes a `CHECK` fail in `FractionalMaxPoolGrad` ([CVE-2022-35981](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35981)) +* Fixes a `CHECK` fail in `RaggedTensorToVariant` ([CVE-2022-36018](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36018)) +* Fixes a `CHECK` fail in `QuantizeAndDequantizeV3` ([CVE-2022-36026](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36026)) +* Fixes a segfault in `SparseBincount` ([CVE-2022-35982](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35982)) +* Fixes a `CHECK` fail in `Save` and `SaveSlices` ([CVE-2022-35983](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35983)) +* Fixes a `CHECK` fail in `ParameterizedTruncatedNormal` ([CVE-2022-35984](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35984)) +* Fixes a `CHECK` fail in `LRNGrad` ([CVE-2022-35985](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35985)) +* Fixes a segfault in `RaggedBincount` ([CVE-2022-35986](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35986)) +* Fixes a `CHECK` fail in `DenseBincount` ([CVE-2022-35987](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35987)) +* Fixes a `CHECK` fail in `tf.linalg.matrix_rank` ([CVE-2022-35988](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35988)) +* Fixes a `CHECK` fail in `MaxPool` ([CVE-2022-35989](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35989)) +* Fixes a `CHECK` fail in `Conv2DBackpropInput` ([CVE-2022-35999](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35999)) +* Fixes a `CHECK` fail in `EmptyTensorList` ([CVE-2022-35998](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35998)) +* Fixes a `CHECK` fail in `tf.sparse.cross` ([CVE-2022-35997](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35997)) +* Fixes a floating point exception in `Conv2D` ([CVE-2022-35996](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35996)) +* Fixes a `CHECK` fail in `AudioSummaryV2` ([CVE-2022-35995](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35995)) +* Fixes a `CHECK` fail in `CollectiveGather` ([CVE-2022-35994](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35994)) +* Fixes a `CHECK` fail in `SetSize` ([CVE-2022-35993](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35993)) +* Fixes a `CHECK` fail in `TensorListFromTensor` ([CVE-2022-35992](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35992)) +* Fixes a `CHECK` fail in `TensorListScatter` and `TensorListScatterV2` ([CVE-2022-35991](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35991)) +* Fixes a `CHECK` fail in `FakeQuantWithMinMaxVarsPerChannelGradient` ([CVE-2022-35990](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35990)) +* Fixes a `CHECK` fail in `FakeQuantWithMinMaxVarsGradient` ([CVE-2022-36005](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36005)) +* Fixes a `CHECK` fail in `tf.random.gamma` ([CVE-2022-36004](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36004)) +* Fixes a `CHECK` fail in `RandomPoissonV2` ([CVE-2022-36003](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36003)) +* Fixes a `CHECK` fail in `Unbatch` ([CVE-2022-36002](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36002)) +* Fixes a `CHECK` fail in `DrawBoundingBoxes` ([CVE-2022-36001](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36001)) +* Fixes a `CHECK` fail in `Eig` ([CVE-2022-36000](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36000)) +* Fixes a null dereference on MLIR on empty function attributes ([CVE-2022-36011](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36011)) +* Fixes an assertion failure on MLIR empty edge names ([CVE-2022-36012](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36012)) +* Fixes a null-dereference in `mlir::tfg::GraphDefImporter::ConvertNodeDef` ([CVE-2022-36013](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36013)) +* Fixes a null-dereference in `mlir::tfg::TFOp::nameAttr` ([CVE-2022-36014](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36014)) +* Fixes an integer overflow in math ops ([CVE-2022-36015](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36015)) +* Fixes a `CHECK`-fail in `tensorflow::full_type::SubstituteFromAttrs` ([CVE-2022-36016](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36016)) +* Fixes an OOB read in `Gather_nd` op in TF Lite Micro ([CVE-2022-35938](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-35938)) + +# Release 2.7.3 + +Add an upper bound for `protobuf` in `setup.py` since `protobuf` after version 3.20 is currently incompatible with TensorFlow. See https://github.com/tensorflow/tensorflow/issues/53234, https://github.com/protocolbuffers/protobuf/issues/9954 and https://github.com/tensorflow/tensorflow/issues/56077. + +# Release 2.7.2 + +This releases introduces several vulnerability fixes: + +* Fixes a code injection in `saved_model_cli` ([CVE-2022-29216](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29216)) +* Fixes a missing validation which causes `TensorSummaryV2` to crash ([CVE-2022-29193](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29193)) +* Fixes a missing validation which crashes `QuantizeAndDequantizeV4Grad` ([CVE-2022-29192](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29192)) +* Fixes a missing validation which causes denial of service via `DeleteSessionTensor` ([CVE-2022-29194](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29194)) +* Fixes a missing validation which causes denial of service via `GetSessionTensor` ([CVE-2022-29191](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29191)) +* Fixes a missing validation which causes denial of service via `StagePeek` ([CVE-2022-29195](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29195)) +* Fixes a missing validation which causes denial of service via `UnsortedSegmentJoin` ([CVE-2022-29197](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29197)) +* Fixes a missing validation which causes denial of service via `LoadAndRemapMatrix` ([CVE-2022-29199](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29199)) +* Fixes a missing validation which causes denial of service via `SparseTensorToCSRSparseMatrix` ([CVE-2022-29198](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29198)) +* Fixes a missing validation which causes denial of service via `LSTMBlockCell` ([CVE-2022-29200](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29200)) +* Fixes a missing validation which causes denial of service via `Conv3DBackpropFilterV2` ([CVE-2022-29196](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29196)) +* Fixes a `CHECK` failure in depthwise ops via overflows ([CVE-2021-41197](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41197)) +* Fixes issues arising from undefined behavior stemming from users supplying invalid resource handles ([CVE-2022-29207](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29207)) +* Fixes a segfault due to missing support for quantized types ([CVE-2022-29205](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29205)) +* Fixes a missing validation which results in undefined behavior in `SparseTensorDenseAdd` ([CVE-2022-29206](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29206)) +* Fixes a missing validation which results in undefined behavior in `QuantizedConv2D` ([CVE-2022-29201](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29201)) +* Fixes an integer overflow in `SpaceToBatchND` ([CVE-2022-29203](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29203)) +* Fixes a segfault and OOB write due to incomplete validation in `EditDistance` ([CVE-2022-29208](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29208)) +* Fixes a missing validation which causes denial of service via `Conv3DBackpropFilterV2` ([CVE-2022-29204](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29204)) +* Fixes a denial of service in `tf.ragged.constant` due to lack of validation ([CVE-2022-29202](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29202)) +* Fixes a segfault when `tf.histogram_fixed_width` is called with NaN values ([CVE-2022-29211](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29211)) +* Fixes a core dump when loading TFLite models with quantization ([CVE-2022-29212](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29212)) +* Fixes crashes stemming from incomplete validation in signal ops ([CVE-2022-29213](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29213)) +* Fixes a type confusion leading to `CHECK`-failure based denial of service ([CVE-2022-29209](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29209)) +* Updates `curl` to `7.83.1` to handle ([CVE-2022-22576](https://cve.mitre.org/cgi-bin/cvename.cgi?name=VE-2022-22576), ([CVE-2022-27774](https://cve.mitre.org/cgi-bin/cvename.cgi?name=VE-2022-27774), ([CVE-2022-27775](https://cve.mitre.org/cgi-bin/cvename.cgi?name=VE-2022-27775), ([CVE-2022-27776](https://cve.mitre.org/cgi-bin/cvename.cgi?name=VE-2022-27776), ([CVE-2022-27778](https://cve.mitre.org/cgi-bin/cvename.cgi?name=VE-2022-27778), ([CVE-2022-27779](https://cve.mitre.org/cgi-bin/cvename.cgi?name=VE-2022-27779), ([CVE-2022-27780](https://cve.mitre.org/cgi-bin/cvename.cgi?name=VE-2022-27780), ([CVE-2022-27781](https://cve.mitre.org/cgi-bin/cvename.cgi?name=VE-2022-27781), ([CVE-2022-27782](https://cve.mitre.org/cgi-bin/cvename.cgi?name=VE-2022-27782) and ([CVE-2022-30115](https://cve.mitre.org/cgi-bin/cvename.cgi?name=VE-2022-30115) +* Updates `zlib` to `1.2.12` after `1.2.11` was pulled due to [security issue](https://www.openwall.com/lists/oss-security/2022/03/28/1) + + +# Release 2.7.1 + +This releases introduces several vulnerability fixes: + +* Fixes a floating point division by 0 when executing convolution operators ([CVE-2022-21725](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21725)) +* Fixes a heap OOB read in shape inference for `ReverseSequence` ([CVE-2022-21728](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21728)) +* Fixes a heap OOB access in `Dequantize` ([CVE-2022-21726](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21726)) +* Fixes an integer overflow in shape inference for `Dequantize` ([CVE-2022-21727](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21727)) +* Fixes a heap OOB access in `FractionalAvgPoolGrad` ([CVE-2022-21730](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21730)) +* Fixes an overflow and divide by zero in `UnravelIndex` ([CVE-2022-21729](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21729)) +* Fixes a type confusion in shape inference for `ConcatV2` ([CVE-2022-21731](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21731)) +* Fixes an OOM in `ThreadPoolHandle` ([CVE-2022-21732](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21732)) +* Fixes an OOM due to integer overflow in `StringNGrams` ([CVE-2022-21733](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21733)) +* Fixes more issues caused by incomplete validation in boosted trees code ([CVE-2021-41208](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41208)) +* Fixes an integer overflows in most sparse component-wise ops ([CVE-2022-23567](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23567)) +* Fixes an integer overflows in `AddManySparseToTensorsMap` ([CVE-2022-23568](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23568)) +* Fixes a number of `CHECK`-failures in `MapStage` ([CVE-2022-21734](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21734)) +* Fixes a division by zero in `FractionalMaxPool` ([CVE-2022-21735](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21735)) +* Fixes a number of `CHECK`-fails when building invalid/overflowing tensor shapes ([CVE-2022-23569](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23569)) +* Fixes an undefined behavior in `SparseTensorSliceDataset` ([CVE-2022-21736](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21736)) +* Fixes an assertion failure based denial of service via faulty bin count operations ([CVE-2022-21737](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21737)) +* Fixes a reference binding to null pointer in `QuantizedMaxPool` ([CVE-2022-21739](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21739)) +* Fixes an integer overflow leading to crash in `SparseCountSparseOutput` ([CVE-2022-21738](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21738)) +* Fixes a heap overflow in `SparseCountSparseOutput` ([CVE-2022-21740](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21740)) +* Fixes an FPE in `BiasAndClamp` in TFLite ([CVE-2022-23557](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23557)) +* Fixes an FPE in depthwise convolutions in TFLite ([CVE-2022-21741](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21741)) +* Fixes an integer overflow in TFLite array creation ([CVE-2022-23558](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23558)) +* Fixes an integer overflow in TFLite ([CVE-2022-23559](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23559)) +* Fixes a dangerous OOB write in TFLite ([CVE-2022-23561](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23561)) +* Fixes a vulnerability leading to read and write outside of bounds in TFLite ([CVE-2022-23560](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23560)) +* Fixes a set of vulnerabilities caused by using insecure temporary files ([CVE-2022-23563](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23563)) +* Fixes an integer overflow in Range resulting in undefined behavior and OOM ([CVE-2022-23562](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23562)) +* Fixes a vulnerability where missing validation causes `tf.sparse.split` to crash when `axis` is a tuple ([CVE-2021-41206](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41206)) +* Fixes a `CHECK`-fail when decoding resource handles from proto ([CVE-2022-23564](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23564)) +* Fixes a `CHECK`-fail with repeated `AttrDef` ([CVE-2022-23565](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23565)) +* Fixes a heap OOB write in Grappler ([CVE-2022-23566](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23566)) +* Fixes a `CHECK`-fail when decoding invalid tensors from proto ([CVE-2022-23571](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23571)) +* Fixes a null-dereference when specializing tensor type ([CVE-2022-23570](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23570)) +* Fixes a crash when type cannot be specialized ([CVE-2022-23572](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23572)) +* Fixes a heap OOB read/write in `SpecializeType` ([CVE-2022-23574](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23574)) +* Fixes an unitialized variable access in `AssignOp` ([CVE-2022-23573](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23573)) +* Fixes an integer overflow in `OpLevelCostEstimator::CalculateTensorSize` ([CVE-2022-23575](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23575)) +* Fixes an integer overflow in `OpLevelCostEstimator::CalculateOutputSize` ([CVE-2022-23576](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23576)) +* Fixes a null dereference in `GetInitOp` ([CVE-2022-23577](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23577)) +* Fixes a memory leak when a graph node is invalid ([CVE-2022-23578](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23578)) +* Fixes an abort caused by allocating a vector that is too large ([CVE-2022-23580](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23580)) +* Fixes multiple `CHECK`-failures during Grappler's `IsSimplifiableReshape` ([CVE-2022-23581](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23581)) +* Fixes multiple `CHECK`-failures during Grappler's `SafeToRemoveIdentity` ([CVE-2022-23579](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23579)) +* Fixes multiple `CHECK`-failures in `TensorByteSize` ([CVE-2022-23582](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23582)) +* Fixes multiple `CHECK`-failures in binary ops due to type confusion ([CVE-2022-23583](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23583)) +* Fixes a use after free in `DecodePng` kernel ([CVE-2022-23584](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23584)) +* Fixes a memory leak in decoding PNG images ([CVE-2022-23585](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23585)) +* Fixes multiple `CHECK`-fails in `function.cc` ([CVE-2022-23586](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23586)) +* Fixes multiple `CHECK`-fails due to attempting to build a reference tensor ([CVE-2022-23588](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23588)) +* Fixes an integer overflow in Grappler cost estimation of crop and resize operation ([CVE-2022-23587](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23587)) +* Fixes a null pointer dereference in Grappler's `IsConstant` ([CVE-2022-23589](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23589)) +* Fixes a `CHECK` failure in constant folding ([CVE-2021-41197](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41197)) +* Fixes a stack overflow due to self-recursive function in `GraphDef` ([CVE-2022-23591](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23591)) +* Fixes a crash due to erroneous `StatusOr` ([CVE-2022-23590](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23590)) +* Fixes multiple crashes and heap OOB accesses in TFG dialect (MLIR) ([CVE-2022-23594](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23594)) +* Fixes a null pointer dereference in `BuildXlaCompilationCache` (XLA) ([CVE-2022-23595](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-23595)) +* Updates `icu` to `69.1` to handle [CVE-2020-10531](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10531) + # Release 2.7.0 ## Breaking Changes @@ -120,6 +283,7 @@ * Move autotuning options from`tf.data.Options.experimental_optimization.autotune*` to a newly created `tf.data.Options.autotune.*` and remove support for `tf.data.Options.experimental_optimization.autotune_buffers`. * Add support for user-defined names of tf.data core Python API, which can be used to disambiguate tf.data events in TF Profiler Trace Viewer. * Promote `tf.data.experimental.sample_from_datasets` API to `tf.data.Dataset.sample_from_datasets` and deprecate the experimental endpoint. + * Added `TF_GPU_ALLOCATOR=cuda_malloc_async` that use cudaMallocAsync from CUDA 11.2. This could become the default in the future. * TF SavedModel: * Custom gradients are now saved by default. See `tf.saved_model.SaveOptions` to disable this. * The saved_model_cli's `--input_examples` inputs are now restricted to @@ -130,6 +294,36 @@ * XLA:GPU works with Horovod (OSS contribution by Trent Lo from NVidia) * `tf.saved_model.save`: * When saving a model, not specifying a namespace whitelist for custom ops with a namespace will now default to allowing rather than rejecting them all. +* Deterministic Op Functionality (enabled by setting the environment variable `TF_DETERMINISTIC_OPS` to `"true"` or `"1"`): + * Add determinsitic GPU implementations of: + * `tf.math.segment_sum` + * `tf.math.segment_prod` + * `tf.math.segment_mean` + * `tf.math.unsorted_segment_sum` + * `tf.math.unsorted_segment_prod` + * `tf.math.unsorted_segment_sqrt` + * `tf.math.unsorted_segment_mean` + * `tf.gather` backprop + * `tf.convert_to_tensor` when fed with (sparse) `tf.IndexedSlices` + * `tf.nn.sparse_softmax_crossentropy_with_logits` + * `tf.nn.ctc_loss` (resolved, possibly in prior release, and confirmed with tests) + * stateful ops used in `tf.data.Dataset` + * Run the following ops on CPU (with significant performance penalty): + * `tf.scatter_nd` and other related scatter functions, such as `tf.tensor_scatter_nd_update` + * Add determinism-unimplemented exception-throwing to the following ops. When op-determinism is expected (i.e. when the environment variable `TF_DETERMINISTIC_OPS` is set to `"true"` or `"1"`), an attempt to use the specified paths through the following ops on a GPU will cause `tf.errors.UnimplementedError` (with an understandable message), unless otherwise specified, to be thrown. + * `tf.compat.v1.nn.fused_batch_norm` backprop to `offset` when `is_training=False` + * `tf.image.adjust_contrast` forward + * `tf.nn.depthwise_conv2d` backprop to `filter` when not using cuDNN convolution + * `tf.image.resize` with `method=ResizeMethod.NEAREST` backprop + * `tf.math.bincount` - TODO: confirm exception added + * `tf.raw_ops.DebugNumericSummary` and `tf.raw_ops.DebugNumericSummaryV2` + * `tf.Variable.scatter_add` (and other scatter methods, both on ref and resource variables) + * `tf.linalg.svd` + * `tf.nn.dilation2d` gradient + * `tf.nn.max_pool_with_argmax` gradient + * `tf.timestamp`. Throws `FailedPrecondition` + * The random-number-generating ops in the `tf.random` module when the global random seed has not yet been set (via `tf.random.set_seed`). Throws `RuntimeError` from Python or `InvalidArgument` from C++ + * `tf.compat.v1.get_seed` if the global random seed has not yet been set (via `tf.random.set_seed`). Throws `RuntimeError` from Python or `InvalidArgument` from C++ ## Security @@ -179,7 +373,7 @@ This release contains contributions from many people at Google, as well as: -8bitmp3, Abhilash Majumder, abhilash1910, AdeshChoudhar, Adrian Garcia Badaracco, Adrian Ratiu, ag.ramesh, Aleksandr Nikolaev, Alexander Bosch, Alexander Grund, Annie Tallund, Anush Elangovan, Artem Sokolovskii, azazhu, Balint Cristian, Bas Aarts, Ben Barsdell, bhack, cfRod, Cheney-Wang, Cheng Ren, Christopher Bate, collin, Danila Bespalov, David Datascientist, Deven Desai, Ehsan Kia, Ellie, Fan Du, fo40225, Frederic Bastien, fsx950223, Gauri1 Deshpande, geetachavan1, Guillaume Klein, guozhong.zhuang, helen, Håkon Sandsmark, japm48, jgehw, Jinzhe Zeng, Jonathan Dekhtiar, Kai Zhu, Kaixi Hou, Kanvi Khanna, Koan-Sin Tan, Koki Ibukuro, Kulin Seth, KumaTea, Kun-Lu, Lemo, lipracer, liuyuanqiang, Mahmoud Abuzaina, Marius Brehler, Maxiwell S. Garcia, mdfaijul, metarutaiga, Michal Szutenberg, nammbash, Neil Girdhar, Nishidha Panpaliya, Nyadla-Sys, Patrice Vignola, Peter Kasting, Philipp Hack, PINTO0309, Prateek Gupta, puneeshkhanna, Rahul Butani, Rajeshwar Reddy T, Reza Rahimi, RinozaJiffry, rmothukuru, Rohit Santhanam, Saduf2019, Samuel Marks, sclarkson, Sergii Khomenko, Sheng, Yang, Sidong-Wei, slowy07, Srinivasan Narayanamoorthy, Srishti Srivastava, stanley, Stella Alice Schlotter, Steven I Reeves, stevenireeves, svobora, Takayoshi Koizumi, Tamas Bela Feher, Thibaut Goetghebuer-Planchon, Trent Lo, Twice, Varghese, Jojimon, Vishnuvardhan Janapati, Wang Yanzhang, Wang,Quintin, William Muir, William Raveane, Yasir Modak, Yasuhiro Matsumoto, Yi Li, Yong Tang, zhaozheng09, Zhoulong Jiang, zzpmiracle +8bitmp3, Abhilash Majumder, abhilash1910, AdeshChoudhar, Adrian Garcia Badaracco, Adrian Ratiu, ag.ramesh, Aleksandr Nikolaev, Alexander Bosch, Alexander Grund, Annie Tallund, Anush Elangovan, Artem Sokolovskii, azazhu, Balint Cristian, Bas Aarts, Ben Barsdell, bhack, cfRod, Cheney-Wang, Cheng Ren, Christopher Bate, collin, Danila Bespalov, David Datascientist, Deven Desai, Duncan Riach, Ehsan Kia, Ellie, Fan Du, fo40225, Frederic Bastien, fsx950223, Gauri1 Deshpande, geetachavan1, Guillaume Klein, guozhong.zhuang, helen, Håkon Sandsmark, japm48, jgehw, Jinzhe Zeng, Jonathan Dekhtiar, Kai Zhu, Kaixi Hou, Kanvi Khanna, Koan-Sin Tan, Koki Ibukuro, Kulin Seth, KumaTea, Kun-Lu, Lemo, lipracer, liuyuanqiang, Mahmoud Abuzaina, Marius Brehler, Maxiwell S. Garcia, mdfaijul, metarutaiga, Michal Szutenberg, nammbash, Neil Girdhar, Nishidha Panpaliya, Nyadla-Sys, Patrice Vignola, Peter Kasting, Philipp Hack, PINTO0309, Prateek Gupta, puneeshkhanna, Rahul Butani, Rajeshwar Reddy T, Reza Rahimi, RinozaJiffry, rmothukuru, Rohit Santhanam, Saduf2019, Samuel Marks, sclarkson, Sergii Khomenko, Sheng, Yang, Sidong-Wei, slowy07, Srinivasan Narayanamoorthy, Srishti Srivastava, stanley, Stella Alice Schlotter, Steven I Reeves, stevenireeves, svobora, Takayoshi Koizumi, Tamas Bela Feher, Thibaut Goetghebuer-Planchon, Trent Lo, Twice, Varghese, Jojimon, Vishnuvardhan Janapati, Wang Yanzhang, Wang,Quintin, William Muir, William Raveane, Yasir Modak, Yasuhiro Matsumoto, Yi Li, Yong Tang, zhaozheng09, Zhoulong Jiang, zzpmiracle # Release 2.6.0 @@ -308,8 +502,6 @@ This release contains contributions from many people at Google, as well as: * Added `tf.config.experimental.reset_memory_stats` to reset the tracked peak memory returned by `tf.config.experimental.get_memory_info`. - * Added `TF_GPU_ALLOCATOR=cuda_malloc_async` that use cudaMallocAsync - from CUDA 11.2. This could become the default in the future. * `tf.data`: diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc index b06f2281d6a595..1022793fda08ef 100644 --- a/tensorflow/cc/saved_model/loader.cc +++ b/tensorflow/cc/saved_model/loader.cc @@ -25,6 +25,7 @@ limitations under the License. #include "tensorflow/core/framework/attr_value.pb.h" #include "tensorflow/core/framework/function.pb.h" #include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/op_def.pb.h" #include "tensorflow/core/framework/tensor.pb.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/lib/monitoring/counter.h" @@ -99,6 +100,19 @@ static Status ValidateNode(const NodeDef& node) { return Status::OK(); } +static Status ValidateFunctionNotRecursive(const FunctionDef& function) { + const auto& function_name = function.signature().name(); + for (const auto& node : function.node_def()) { + if (node.op() == function_name) { + return errors::FailedPrecondition( + "Function ", function_name, + " is self recursive and TensorFlow does not support this scenario."); + } + } + + return Status::OK(); +} + static Status ValidateSavedTensors(const GraphDef& graph_def) { for (const auto& node : graph_def.node()) { TF_RETURN_IF_ERROR(ValidateNode(node)); @@ -110,6 +124,10 @@ static Status ValidateSavedTensors(const GraphDef& graph_def) { for (const auto& node : function.node_def()) { TF_RETURN_IF_ERROR(ValidateNode(node)); } + + // Also check that there is no recursivity in the library + // TODO(mihaimaruseac): Do more than self-recursivity + TF_RETURN_IF_ERROR(ValidateFunctionNotRecursive(function)); } } diff --git a/tensorflow/cc/saved_model/loader_util.cc b/tensorflow/cc/saved_model/loader_util.cc index 100cae2291333f..411dc41fd44837 100644 --- a/tensorflow/cc/saved_model/loader_util.cc +++ b/tensorflow/cc/saved_model/loader_util.cc @@ -34,9 +34,14 @@ Status GetInitOp(const string& export_dir, const MetaGraphDef& meta_graph_def, const auto& init_op_sig_it = meta_graph_def.signature_def().find(kSavedModelInitOpSignatureKey); if (init_op_sig_it != sig_def_map.end()) { - *init_op_name = init_op_sig_it->second.outputs() - .find(kSavedModelInitOpSignatureKey) - ->second.name(); + const auto& sig_def_outputs = init_op_sig_it->second.outputs(); + const auto& sig_def_outputs_it = + sig_def_outputs.find(kSavedModelInitOpSignatureKey); + if (sig_def_outputs_it == sig_def_outputs.end()) { + return errors::FailedPrecondition("Could not find output ", + kSavedModelInitOpSignatureKey); + } + *init_op_name = sig_def_outputs_it->second.name(); return Status::OK(); } diff --git a/tensorflow/compiler/jit/xla_platform_info.cc b/tensorflow/compiler/jit/xla_platform_info.cc index 8a1382befdec3e..c9334972346867 100644 --- a/tensorflow/compiler/jit/xla_platform_info.cc +++ b/tensorflow/compiler/jit/xla_platform_info.cc @@ -82,11 +82,13 @@ Status BuildXlaCompilationCache(DeviceBase* device, FunctionLibraryRuntime* flr, client_options.set_intra_op_parallelism_threads( device->tensorflow_cpu_worker_threads()->num_threads); - string allowed_gpus = - flr->config_proto()->gpu_options().visible_device_list(); - TF_ASSIGN_OR_RETURN(absl::optional> gpu_ids, - ParseVisibleDeviceList(allowed_gpus)); - client_options.set_allowed_devices(gpu_ids); + if (flr->config_proto()) { + string allowed_gpus = + flr->config_proto()->gpu_options().visible_device_list(); + TF_ASSIGN_OR_RETURN(absl::optional> gpu_ids, + ParseVisibleDeviceList(allowed_gpus)); + client_options.set_allowed_devices(gpu_ids); + } auto client = xla::ClientLibrary::GetOrCreateLocalClient(client_options); if (!client.ok()) { diff --git a/tensorflow/compiler/tests/spacetobatch_op_test.py b/tensorflow/compiler/tests/spacetobatch_op_test.py index 74f5f7bb48f52a..0873281200cd46 100644 --- a/tensorflow/compiler/tests/spacetobatch_op_test.py +++ b/tensorflow/compiler/tests/spacetobatch_op_test.py @@ -21,6 +21,7 @@ import numpy as np from tensorflow.compiler.tests import xla_test +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_array_ops @@ -149,6 +150,29 @@ def testLargerInputBatch2x2(self): self._testOne(x_np, block_size, x_out) +class SpaceToBatchNDErrorHandlingTest(xla_test.XLATestCase): + + def testInvalidBlockShape(self): + with self.assertRaisesRegex(ValueError, "block_shape must be positive"): + with self.session() as sess, self.test_scope(): + tf_in = constant_op.constant( + -3.5e+35, shape=[10, 20, 20], dtype=dtypes.float32) + block_shape = constant_op.constant(-10, shape=[2], dtype=dtypes.int64) + paddings = constant_op.constant(0, shape=[2, 2], dtype=dtypes.int32) + sess.run(array_ops.space_to_batch_nd(tf_in, block_shape, paddings)) + + def testOutputSizeOutOfBounds(self): + with self.assertRaisesRegex(ValueError, + "Negative.* dimension size caused by overflow"): + with self.session() as sess, self.test_scope(): + tf_in = constant_op.constant( + -3.5e+35, shape=[10, 19, 22], dtype=dtypes.float32) + block_shape = constant_op.constant( + 1879048192, shape=[2], dtype=dtypes.int64) + paddings = constant_op.constant(0, shape=[2, 2], dtype=dtypes.int32) + sess.run(array_ops.space_to_batch_nd(tf_in, block_shape, paddings)) + + class SpaceToBatchNDTest(xla_test.XLATestCase): """Tests input-output pairs for the SpaceToBatchND and BatchToSpaceND ops.""" diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD index 4cc82379d2eacc..6e7678353bf8d2 100644 --- a/tensorflow/compiler/tf2xla/BUILD +++ b/tensorflow/compiler/tf2xla/BUILD @@ -378,6 +378,7 @@ cc_library( "//tensorflow/compiler/xla/client:xla_builder", "//tensorflow/compiler/xla/client:xla_computation", "//tensorflow/compiler/xla/service:hlo", + "//tensorflow/core/util:overflow", "//tensorflow/core:core_cpu", "//tensorflow/core:core_cpu_internal", "//tensorflow/core:framework", diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD index 6938e132ab6be8..0240342c768a1b 100644 --- a/tensorflow/compiler/tf2xla/kernels/BUILD +++ b/tensorflow/compiler/tf2xla/kernels/BUILD @@ -205,6 +205,7 @@ tf_kernel_library( "//tensorflow/core/kernels:stateful_random_ops_header", "//tensorflow/core/kernels:stateless_random_ops_v2_header", "//tensorflow/core/tpu:tpu_defs", + "//tensorflow/core/util:overflow", "//tensorflow/stream_executor/lib", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/container:flat_hash_map", diff --git a/tensorflow/compiler/tf2xla/kernels/spacetobatch_op.cc b/tensorflow/compiler/tf2xla/kernels/spacetobatch_op.cc index a4e9aec1c97058..d6e38f1309f91c 100644 --- a/tensorflow/compiler/tf2xla/kernels/spacetobatch_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/spacetobatch_op.cc @@ -17,6 +17,7 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/core/util/overflow.h" namespace tensorflow { namespace { @@ -60,10 +61,14 @@ void SpaceToBatch(XlaOpKernelContext* ctx, const xla::XlaOp& input, int64_t pad_end = paddings.Get({i, 1}); OP_REQUIRES(ctx, pad_start >= 0 && pad_end >= 0, errors::InvalidArgument("Paddings must be non-negative")); + OP_REQUIRES(ctx, block_shape[i] >= 1, + errors::InvalidArgument( + "All values in block_shape must be positive, got value, ", + block_shape[i], " at index ", i, ".")); dim->set_edge_padding_low(pad_start); dim->set_edge_padding_high(pad_end); padded_shape[1 + i] += pad_start + pad_end; - block_num_elems *= block_shape[i]; + block_num_elems = MultiplyWithoutOverflow(block_num_elems, block_shape[i]); } // Don't pad the remainder dimensions. for (int i = 0; i < remainder_shape.size(); ++i) { @@ -72,6 +77,16 @@ void SpaceToBatch(XlaOpKernelContext* ctx, const xla::XlaOp& input, OP_REQUIRES(ctx, block_num_elems > 0, errors::InvalidArgument( "The product of the block dimensions must be positive")); + const int64_t batch_size = input_shape[0]; + const int64_t output_dim = + MultiplyWithoutOverflow(batch_size, block_num_elems); + if (output_dim < 0) { + OP_REQUIRES( + ctx, output_dim >= 0, + errors::InvalidArgument("Negative output dimension size caused by " + "overflow when multiplying ", + batch_size, " and ", block_num_elems)); + } xla::XlaOp padded = xla::Pad(input, XlaHelpers::Zero(b, input_dtype), padding_config); @@ -85,7 +100,6 @@ void SpaceToBatch(XlaOpKernelContext* ctx, const xla::XlaOp& input, // padded_shape[M] / block_shape[M-1], // block_shape[M-1]] + // remaining_shape - const int64_t batch_size = input_shape[0]; std::vector reshaped_padded_shape(input_rank + block_rank); reshaped_padded_shape[0] = batch_size; for (int i = 0; i < block_rank; ++i) { @@ -134,7 +148,7 @@ void SpaceToBatch(XlaOpKernelContext* ctx, const xla::XlaOp& input, // Determine the length of the prefix of block dims that can be combined // into the batch dimension due to having no padding and block_shape=1. std::vector output_shape(input_rank); - output_shape[0] = batch_size * block_num_elems; + output_shape[0] = output_dim; for (int i = 0; i < block_rank; ++i) { output_shape[1 + i] = padded_shape[1 + i] / block_shape[i]; } diff --git a/tensorflow/compiler/tf2xla/xla_op_kernel.cc b/tensorflow/compiler/tf2xla/xla_op_kernel.cc index 4c2c0664d5c00e..0697f5550481e5 100644 --- a/tensorflow/compiler/tf2xla/xla_op_kernel.cc +++ b/tensorflow/compiler/tf2xla/xla_op_kernel.cc @@ -29,6 +29,7 @@ limitations under the License. #include "tensorflow/compiler/xla/status_macros.h" #include "tensorflow/core/common_runtime/dma_helper.h" #include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/util/overflow.h" namespace tensorflow { @@ -466,6 +467,16 @@ Status XlaOpKernelContext::ConstantInputAsShape(int index, TensorShape* shape, TF_RETURN_IF_ERROR(ConstantInput(index, &literal, mode)); std::vector dims; TF_RETURN_IF_ERROR(LiteralToInt64Vector(literal, &dims)); + + int64_t num_elements = 1; + for (auto i = dims.begin(); i != dims.end(); ++i) { + num_elements = MultiplyWithoutOverflow(num_elements, *i); + if (num_elements < 0) + return errors::InvalidArgument( + "The total elements specified by orig_input_shape is too large.", + "Encountered overflow after multiplying", *i, + ", result: ", num_elements); + } *shape = TensorShape(dims); return Status::OK(); } diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc index f59480bbea0ea2..91661d56124ace 100644 --- a/tensorflow/core/common_runtime/eager/execute.cc +++ b/tensorflow/core/common_runtime/eager/execute.cc @@ -297,6 +297,9 @@ Status GetDeviceForInput(const EagerContext& ctx, TensorHandle* tensor_handle, const Tensor* tensor; // TODO(fishx): Avoid blocking here. TF_RETURN_IF_ERROR(tensor_handle->Tensor(&tensor)); + if (tensor->NumElements() == 0) { + return errors::InvalidArgument("Empty resource handle"); + } const ResourceHandle& handle = tensor->flat()(0); device_name = handle.device(); diff --git a/tensorflow/core/common_runtime/immutable_executor_state.cc b/tensorflow/core/common_runtime/immutable_executor_state.cc index 1f728334e2b7ee..25822540f024ae 100644 --- a/tensorflow/core/common_runtime/immutable_executor_state.cc +++ b/tensorflow/core/common_runtime/immutable_executor_state.cc @@ -131,6 +131,7 @@ Status ImmutableExecutorState::Initialize(const Graph& graph) { Status s = params_.create_kernel(n->properties(), &item->kernel); if (!s.ok()) { + params_.delete_kernel(item->kernel); item->kernel = nullptr; s = AttachDef(s, *n); return s; diff --git a/tensorflow/core/framework/BUILD b/tensorflow/core/framework/BUILD index c42e5ff3da0712..5f43655ffa313f 100644 --- a/tensorflow/core/framework/BUILD +++ b/tensorflow/core/framework/BUILD @@ -730,7 +730,9 @@ cc_library( "//tensorflow/core/lib/core:errors", "//tensorflow/core/lib/strings:strcat", "//tensorflow/core/platform:casts", + "//tensorflow/core/platform:errors", "//tensorflow/core/platform:intrusive_ptr", + "//tensorflow/core/platform:macros", "//tensorflow/core/platform:statusor", "//tensorflow/core/platform:tensor_coding", "//tensorflow/core/platform:types", @@ -876,6 +878,7 @@ cc_library( "//tensorflow/core/lib/strings:scanner", "//tensorflow/core/lib/strings:str_util", "//tensorflow/core/platform:macros", + "//tensorflow/core/util:overflow", "@com_google_absl//absl/memory", ], ) diff --git a/tensorflow/core/framework/attr_value_util.cc b/tensorflow/core/framework/attr_value_util.cc index 20f34cc46c8294..019063e22d3c62 100644 --- a/tensorflow/core/framework/attr_value_util.cc +++ b/tensorflow/core/framework/attr_value_util.cc @@ -45,7 +45,7 @@ constexpr int kMaxTensorNestDepth = 100; // not fully defined return -1. int64_t TensorByteSize(const TensorProto& t) { // num_elements returns -1 if shape is not fully defined. - int64_t num_elems = TensorShape(t.tensor_shape()).num_elements(); + int64_t num_elems = PartialTensorShape(t.tensor_shape()).num_elements(); return num_elems < 0 ? -1 : num_elems * DataTypeSize(t.dtype()); } diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc index 7d8fd93b1b0e6d..ffbbc629a1afe8 100644 --- a/tensorflow/core/framework/common_shape_fns.cc +++ b/tensorflow/core/framework/common_shape_fns.cc @@ -2005,7 +2005,7 @@ Status ConcatShapeHelper(InferenceContext* c, int start_value_index, } // Minimum required number of dimensions. - const int min_rank = concat_dim < 0 ? -concat_dim : concat_dim + 1; + const int64 min_rank = concat_dim < 0 ? -concat_dim : concat_dim + 1; ShapeHandle output_before; ShapeHandle output_after; diff --git a/tensorflow/core/framework/full_type_util.cc b/tensorflow/core/framework/full_type_util.cc index 5d2b33c3099341..89617dc97f2496 100644 --- a/tensorflow/core/framework/full_type_util.cc +++ b/tensorflow/core/framework/full_type_util.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/core/framework/op_def.pb.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/platform/statusor.h" +#include "tensorflow/core/protobuf/error_codes.pb.h" namespace tensorflow { @@ -99,10 +100,14 @@ StatusOr SpecializeType(const AttrSlice& attrs, // verifications are needed, they should be done by separately, and in a // way that can be reused for type inference. for (int j = 0; j < t->args_size(); j++) { - auto* arg = t->mutable_args(i); + auto* arg = t->mutable_args(j); if (arg->type_id() == TFT_VAR) { const auto* attr = attrs.Find(arg->s()); - DCHECK(attr != nullptr); + if (attr == nullptr) { + return Status( + error::INVALID_ARGUMENT, + absl::StrCat("Could not find an attribute for key ", arg->s())); + } if (attr->value_case() == AttrValue::kList) { const auto& attr_list = attr->list(); arg->set_type_id(TFT_PRODUCT); diff --git a/tensorflow/core/framework/function.cc b/tensorflow/core/framework/function.cc index 41b8d446149694..b8318ef346ae08 100644 --- a/tensorflow/core/framework/function.cc +++ b/tensorflow/core/framework/function.cc @@ -181,7 +181,9 @@ class FunctionInstantiationHelper { DataTypeVector dtypes; TF_RETURN_IF_ERROR( ArgNumType(attr_values, arg_def, &is_type_list, &dtypes)); - CHECK_GE(dtypes.size(), size_t{1}); + if (dtypes.size() < size_t{1}) { + return errors::Internal("Expected a list of at least one dtype"); + } int arg_index = result_.nodes.size(); TF_RETURN_IF_ERROR( AddItem(arg_def.name(), {true, arg_index, 0, is_type_list, dtypes})); @@ -189,7 +191,11 @@ class FunctionInstantiationHelper { for (size_t i = 0; i < dtypes.size(); ++i) { TF_RETURN_IF_ERROR(AddItem(strings::StrCat(arg_def.name(), ":", i), {true, arg_index, 0, false, {dtypes[i]}})); - DCHECK_EQ(arg_index, result_.nodes.size()); + if (arg_index != result_.nodes.size()) { + return errors::Internal( + "Expected arg_index to be equal to the number of nodes in result.", + " Got ", arg_index, " and ", result_.nodes.size()); + } string name = arg_def.name(); if (dtypes.size() > 1) { strings::StrAppend(&name, "_", i); diff --git a/tensorflow/core/framework/op_def_util.cc b/tensorflow/core/framework/op_def_util.cc index dcbe5f38ce88ea..6127913d9ba1f0 100644 --- a/tensorflow/core/framework/op_def_util.cc +++ b/tensorflow/core/framework/op_def_util.cc @@ -821,9 +821,10 @@ bool RepeatedAttrDefEqual( const protobuf::RepeatedPtrField& a2) { std::unordered_map a1_set; for (const OpDef::AttrDef& def : a1) { - DCHECK(a1_set.find(def.name()) == a1_set.end()) - << "AttrDef names must be unique, but '" << def.name() - << "' appears more than once"; + if (a1_set.find(def.name()) != a1_set.end()) { + LOG(ERROR) << "AttrDef names must be unique, but '" << def.name() + << "' appears more than once"; + } a1_set[def.name()] = &def; } for (const OpDef::AttrDef& def : a2) { diff --git a/tensorflow/core/framework/resource_handle.cc b/tensorflow/core/framework/resource_handle.cc index 0ca9fc8dfcb8ab..c0eefe58e47bab 100644 --- a/tensorflow/core/framework/resource_handle.cc +++ b/tensorflow/core/framework/resource_handle.cc @@ -16,8 +16,11 @@ limitations under the License. #include "tensorflow/core/framework/resource_handle.h" #include "tensorflow/core/framework/resource_handle.pb.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/macros.h" namespace tensorflow { @@ -27,7 +30,15 @@ namespace tensorflow { ResourceHandle::ResourceHandle() {} ResourceHandle::ResourceHandle(const ResourceHandleProto& proto) { - FromProto(proto); + TF_CHECK_OK(FromProto(proto)); +} + +Status ResourceHandle::BuildResourceHandle(const ResourceHandleProto& proto, + ResourceHandle* out) { + if (out == nullptr) + return errors::Internal( + "BuildResourceHandle() was called with nullptr for the output"); + return out->FromProto(proto); } ResourceHandle::~ResourceHandle() {} @@ -52,7 +63,7 @@ void ResourceHandle::AsProto(ResourceHandleProto* proto) const { } } -void ResourceHandle::FromProto(const ResourceHandleProto& proto) { +Status ResourceHandle::FromProto(const ResourceHandleProto& proto) { set_device(proto.device()); set_container(proto.container()); set_name(proto.name()); @@ -61,10 +72,16 @@ void ResourceHandle::FromProto(const ResourceHandleProto& proto) { std::vector dtypes_and_shapes; for (const auto& dtype_and_shape : proto.dtypes_and_shapes()) { DataType dtype = dtype_and_shape.dtype(); - PartialTensorShape shape(dtype_and_shape.shape()); + PartialTensorShape shape; + Status s = PartialTensorShape::BuildPartialTensorShape( + dtype_and_shape.shape(), &shape); + if (!s.ok()) { + return s; + } dtypes_and_shapes.push_back(DtypeAndPartialTensorShape{dtype, shape}); } dtypes_and_shapes_ = std::move(dtypes_and_shapes); + return Status::OK(); } string ResourceHandle::SerializeAsString() const { @@ -75,9 +92,7 @@ string ResourceHandle::SerializeAsString() const { bool ResourceHandle::ParseFromString(const string& s) { ResourceHandleProto proto; - const bool status = proto.ParseFromString(s); - if (status) FromProto(proto); - return status; + return proto.ParseFromString(s) && FromProto(proto).ok(); } string ResourceHandle::DebugString() const { @@ -144,7 +159,9 @@ bool DecodeResourceHandleList(std::unique_ptr d, if (!proto.ParseFromArray(d->Data(sizes[i]), sizes[i])) { return false; } - ps[i].FromProto(proto); + if (!ps[i].FromProto(proto).ok()) { + return false; + } } return true; } diff --git a/tensorflow/core/framework/resource_handle.h b/tensorflow/core/framework/resource_handle.h index 0db8e2fcf4d21e..46926720a00577 100644 --- a/tensorflow/core/framework/resource_handle.h +++ b/tensorflow/core/framework/resource_handle.h @@ -46,6 +46,11 @@ class ResourceHandle { ResourceHandle(const ResourceHandleProto& proto); ~ResourceHandle(); + // Use this factory method if the `proto` comes from user controlled input, to + // prevent a denial of service. + static Status BuildResourceHandle(const ResourceHandleProto& proto, + ResourceHandle* out); + // Unique name for the device containing the resource. const std::string& device() const { return device_; } @@ -91,7 +96,7 @@ class ResourceHandle { // Conversion to and from ResourceHandleProto void AsProto(ResourceHandleProto* proto) const; - void FromProto(const ResourceHandleProto& proto); + Status FromProto(const ResourceHandleProto& proto); // Serialization via ResourceHandleProto std::string SerializeAsString() const; diff --git a/tensorflow/core/framework/shape_inference.cc b/tensorflow/core/framework/shape_inference.cc index 04185dfba2e90c..c36337704e2599 100644 --- a/tensorflow/core/framework/shape_inference.cc +++ b/tensorflow/core/framework/shape_inference.cc @@ -14,6 +14,8 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/framework/shape_inference.h" +#include + #include "tensorflow/core/framework/bounds_check.h" #include "tensorflow/core/framework/full_type_util.h" #include "tensorflow/core/framework/node_def.pb.h" @@ -24,6 +26,7 @@ limitations under the License. #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/scanner.h" #include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/util/overflow.h" namespace tensorflow { namespace shape_inference { @@ -170,7 +173,10 @@ void InferenceContext::PreInputInit( const std::vector& input_tensors_as_shapes) { // TODO(mdan): This is also done at graph construction. Run only here instead? const auto ret = full_type::SpecializeType(attrs_, op_def); - DCHECK(ret.status().ok()) << "while instantiating types: " << ret.status(); + if (!ret.status().ok()) { + construction_status_ = ret.status(); + return; + } ret_types_ = ret.ValueOrDie(); input_tensors_ = input_tensors; @@ -786,6 +792,19 @@ Status InferenceContext::InternalMakeShapeFromTensor( return ReturnUnknownShape(out); } const auto num_dims = Value(shape_dim); + // TODO(mihaimaruseac): Should be `TensorShape::MaxDimensions()` as we are + // not able to materialize shapes with more than this number of dimensions + // but then shape inference would fail for operations such as + // `tf.range`/`tf.ones`, etc. where the shape is not really materialized, + // only used during the inference. Hence, just prevent doing a `reserve` + // with a very large argument. + const int64_t max_dimensions = 1 << 25; + if (num_dims >= max_dimensions) { + return errors::Internal( + "Cannot create a tensor with ", num_dims, + " dimensions, as these would be more than maximum of ", + max_dimensions); + } std::vector dims; dims.reserve(num_dims); for (int i = 0; i < num_dims; i++) dims.push_back(UnknownDim()); @@ -1081,7 +1100,7 @@ Status InferenceContext::Multiply(DimensionHandle first, *out = UnknownDim(); } else { // Invariant: Both values are known and greater than 1. - const int64_t product = first_value * second_value; + const int64_t product = MultiplyWithoutOverflow(first_value, second_value); if (product < 0) { return errors::InvalidArgument( "Negative dimension size caused by overflow when multiplying ", diff --git a/tensorflow/core/framework/tensor.cc b/tensorflow/core/framework/tensor.cc index 5e26bf05c03ca9..c7a08ee0808043 100644 --- a/tensorflow/core/framework/tensor.cc +++ b/tensorflow/core/framework/tensor.cc @@ -537,6 +537,46 @@ TensorBuffer* FromProtoField(Allocator* a, const TensorProto& in, int64_t n) { return buf; } +// Separate implementation for `ResourceHandle` to handle the case when the +// proto for the resource is invalid. See `resource_handle.h` constructor and +// static factory builder. +template <> +TensorBuffer* FromProtoField(Allocator* a, + const TensorProto& in, int64_t n) { + CHECK_GT(n, 0); + Buffer* buf = new Buffer(a, n); + ResourceHandle* data = buf->template base(); + if (data == nullptr) { + buf->Unref(); + return nullptr; + } + const int64_t in_n = ProtoHelper::NumElements(in); + if (in_n <= 0) { + std::fill_n(data, n, ResourceHandle()); + } else { + // If tensor shape says we have n < in_n elements in the output tensor + // then make sure to only decode the first n out of the in_n elements in the + // in tensors. In all other cases, we decode all in_n elements of in and set + // the remaining elements up to n to be the default ResourceHandle() value. + const int64_t real_n = n < in_n ? n : in_n; + for (int64_t i = 0; i < real_n; ++i) { + Status s = ResourceHandle::BuildResourceHandle(in.resource_handle_val(i), + &data[i]); + if (!s.ok()) { + LOG(ERROR) << "Could not decode resource handle from proto \"" + << in.resource_handle_val(i).ShortDebugString() + << "\", returned status: " << s.ToString(); + buf->Unref(); + return nullptr; + } + } + for (int64_t i = in_n; i < n; ++i) { + data[i] = ResourceHandle(); + } + } + return buf; +} + template <> TensorBuffer* FromProtoField(Allocator* a, const TensorProto& in, int64_t n) { @@ -943,6 +983,15 @@ bool Tensor::FromProto(Allocator* a, const TensorProto& proto) { dtype_error = true, dtype_error = true); } if (dtype_error || p == nullptr) return false; + } else { + // Handle the case of empty tensors (N = 0) or tensors with incomplete shape + // (N = -1). All other values of `shape.num_elements()` should be invalid by + // construction. + // Here, we just need to validate that the `proto.dtype()` value is valid. + bool dtype_error = false; + CASES_WITH_DEFAULT(proto.dtype(), break, dtype_error = true, + dtype_error = true); + if (dtype_error) return false; } shape_ = shape; set_dtype(proto.dtype()); diff --git a/tensorflow/core/framework/tensor_shape.cc b/tensorflow/core/framework/tensor_shape.cc index d3b2f68a8fb890..163471f3931e27 100644 --- a/tensorflow/core/framework/tensor_shape.cc +++ b/tensorflow/core/framework/tensor_shape.cc @@ -229,7 +229,7 @@ Status TensorShapeBase::InitDims(gtl::ArraySlice dim_sizes) { if (!kIsPartial && !large_size) { for (auto s : dim_sizes) { if (TF_PREDICT_FALSE(s < 0)) { - return errors::Internal( + return errors::InvalidArgument( "Expected shape dimensions to be non-negative, got ", s); } } @@ -411,7 +411,8 @@ template Status TensorShapeBase::AddDimWithStatus(int64_t size) { if (!kIsPartial) { if (TF_PREDICT_FALSE(size < 0)) { - return errors::Internal("Expected a non-negative size, got ", size); + return errors::InvalidArgument("Expected a non-negative size, got ", + size); } } @@ -420,7 +421,7 @@ Status TensorShapeBase::AddDimWithStatus(int64_t size) { } if (TF_PREDICT_FALSE(ndims_byte() >= MaxDimensions())) { - return errors::Internal("Too many dimensions in tensor"); + return errors::InvalidArgument("Too many dimensions in tensor"); } int64_t new_num_elements; @@ -429,9 +430,9 @@ Status TensorShapeBase::AddDimWithStatus(int64_t size) { } else { new_num_elements = MultiplyWithoutOverflow(num_elements(), size); if (TF_PREDICT_FALSE(new_num_elements < 0)) { - return errors::Internal("Encountered overflow when multiplying ", - num_elements(), " with ", size, - ", result: ", new_num_elements); + return errors::InvalidArgument("Encountered overflow when multiplying ", + num_elements(), " with ", size, + ", result: ", new_num_elements); } } @@ -522,7 +523,8 @@ template Status TensorShapeBase::InsertDimWithStatus(int d, int64_t size) { if (!kIsPartial) { if (TF_PREDICT_FALSE(size < 0)) { - return errors::Internal("Expected a non-negative size, got ", size); + return errors::InvalidArgument("Expected a non-negative size, got ", + size); } } @@ -594,13 +596,14 @@ void TensorShapeBase::set_dim(int d, int64_t size) { template Status TensorShapeBase::SetDimWithStatus(int d, int64_t size) { if (TF_PREDICT_FALSE(d < 0)) { - return errors::Internal("Index must be non-negative, got ", d); + return errors::InvalidArgument("Index must be non-negative, got ", d); } if (TF_PREDICT_FALSE(d >= dims())) { - return errors::Internal("Index must be less than ", dims(), ", got ", d); + return errors::InvalidArgument("Index must be less than ", dims(), ", got ", + d); } if (TF_PREDICT_FALSE(!kIsPartial && size < 0)) { - return errors::Internal("Expected a non-negative size, got ", size); + return errors::InvalidArgument("Expected a non-negative size, got ", size); } if (tag() == REP16 && size < kMaxRep16) { diff --git a/tensorflow/core/framework/tensor_shape.h b/tensorflow/core/framework/tensor_shape.h index c18bad8d52e142..8cadd50081df69 100644 --- a/tensorflow/core/framework/tensor_shape.h +++ b/tensorflow/core/framework/tensor_shape.h @@ -359,6 +359,23 @@ class TensorShape : public TensorShapeBase { public: using TensorShapeBase::TensorShapeBase; + // These factory methods should be used instead of the constructors that take + // an array of sizes if calling code cannot validate that the sizes specify a + // valid `TensorShape`. + // The value in `*out` is valid iff the returned value is `Status::OK`. + static Status BuildTensorShape(gtl::ArraySlice dim_sizes, + TensorShape* out) { + return BuildTensorShapeBase(dim_sizes, out); + } + static Status BuildTensorShape(std::initializer_list dim_sizes, + TensorShape* out) { + return BuildTensorShape(gtl::ArraySlice(dim_sizes), out); + } + static Status BuildTensorShape(const TensorShapeProto& proto, + TensorShape* out) { + return BuildTensorShapeBase(proto, out); + } + /// Allow a TensorShape to be used as a PartialTensorShape without copying operator const PartialTensorShape&() const; // NOLINT(runtime/explicit) @@ -511,6 +528,23 @@ class PartialTensorShape : public TensorShapeBase { PartialTensorShape() {} using TensorShapeBase::TensorShapeBase; + // These factory methods should be used instead of the constructors that take + // an array of sizes if calling code cannot validate that the sizes specify a + // valid `PartialTensorShape`. + // The value in `*out` is valid iff the returned value is `Status::OK`. + static Status BuildPartialTensorShape(gtl::ArraySlice dim_sizes, + PartialTensorShape* out) { + return BuildTensorShapeBase(dim_sizes, out); + } + static Status BuildPartialTensorShape( + std::initializer_list dim_sizes, PartialTensorShape* out) { + return BuildPartialTensorShape(gtl::ArraySlice(dim_sizes), out); + } + static Status BuildPartialTensorShape(const TensorShapeProto& proto, + PartialTensorShape* out) { + return BuildTensorShapeBase(proto, out); + } + /// Add a dimension to the end ("inner-most"), returns a new /// PartialTensorShape. /// REQUIRES: `size >= -1`, where -1 means unknown diff --git a/tensorflow/core/framework/tensor_shape_test.cc b/tensorflow/core/framework/tensor_shape_test.cc index 3ca52eebf8a492..518cdeca87092d 100644 --- a/tensorflow/core/framework/tensor_shape_test.cc +++ b/tensorflow/core/framework/tensor_shape_test.cc @@ -214,7 +214,7 @@ TEST(TensorShapeTest, AddDimWithStatus) { ASSERT_EQ(4, s.dims()); status = s.AddDimWithStatus(-1); - EXPECT_EQ(tensorflow::error::INTERNAL, status.code()); + EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, status.code()); } TEST(TensorShapeTest, Factory) { @@ -225,7 +225,7 @@ TEST(TensorShapeTest, Factory) { ASSERT_EQ(3, s.dims()); status = TensorShape::BuildTensorShapeBase({-10, 5, 20}, &s); - EXPECT_EQ(tensorflow::error::INTERNAL, status.code()); + EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, status.code()); } // ----------------------------------------------------------------------- diff --git a/tensorflow/core/graph/graph.cc b/tensorflow/core/graph/graph.cc index ea5c789072c6fb..f6e444d00c9aa5 100644 --- a/tensorflow/core/graph/graph.cc +++ b/tensorflow/core/graph/graph.cc @@ -485,6 +485,23 @@ Node* Graph::AddNode(NodeDef node_def, Status* status) { ? Node::NC_FUNCTION_OP : Node::GetNodeClassForOp(node_def.op()); + if (op_reg_data->type_ctor != nullptr) { + VLOG(3) << "AddNode: found type constructor for " << node_def.name(); + const auto ctor_type = + full_type::SpecializeType(AttrSlice(node_def), op_reg_data->op_def); + if (!ctor_type.ok()) { + *status = errors::InvalidArgument("type error: ", + ctor_type.status().ToString()); + return nullptr; + } + const FullTypeDef ctor_typedef = ctor_type.ValueOrDie(); + if (ctor_typedef.type_id() != TFT_UNSET) { + *(node_def.mutable_experimental_type()) = ctor_typedef; + } + } else { + VLOG(3) << "AddNode: no type constructor for " << node_def.name(); + } + Node* node = AllocateNode( std::make_shared(&op_reg_data->op_def, std::move(node_def), inputs, outputs), diff --git a/tensorflow/core/grappler/costs/BUILD b/tensorflow/core/grappler/costs/BUILD index 66e0c6cd7c5f15..34bd9c7fc2120c 100644 --- a/tensorflow/core/grappler/costs/BUILD +++ b/tensorflow/core/grappler/costs/BUILD @@ -182,6 +182,7 @@ tf_cuda_library( "//tensorflow/core:lib_proto_parsing", "//tensorflow/core:protos_all_cc", "//tensorflow/core/grappler:utils", + "//tensorflow/core/util:overflow", "//tensorflow/core/grappler/clusters:utils", ] + tf_protos_grappler(), ) @@ -340,6 +341,7 @@ cc_library( "//tensorflow/core:lib", "//tensorflow/core:protos_all_cc", "//tensorflow/core/grappler/clusters:utils", + "//tensorflow/core/util:overflow", ] + tf_protos_grappler(), ) @@ -353,6 +355,7 @@ tf_cc_test( "//tensorflow/core:protos_all_cc", "//tensorflow/core:test", "//tensorflow/core:test_main", + "//tensorflow/core/platform:status_matchers", ], ) diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc index 740374b9b621db..1744a4401b57f8 100644 --- a/tensorflow/core/grappler/costs/graph_properties.cc +++ b/tensorflow/core/grappler/costs/graph_properties.cc @@ -1134,7 +1134,12 @@ class SymbolicShapeRefiner { GetUnknownOutputShape(node, output_port); InferenceContext* ctx = GetContext(node); if (ctx == nullptr) { - return errors::InvalidArgument("Missing context"); + return errors::InvalidArgument("SetUnknownShape: Missing context"); + } + if (output_port < 0 || output_port >= ctx->num_outputs()) { + return errors::InvalidArgument( + "SetUnknownShape: output_port must be in [0, ", ctx->num_outputs(), + ") but was ", output_port); } ctx->set_output(output_port, shape); return Status::OK(); diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc index 10acbf54595da4..3dab8bccc543ee 100644 --- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc +++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc @@ -27,6 +27,7 @@ limitations under the License. #include "tensorflow/core/grappler/costs/op_context.h" #include "tensorflow/core/grappler/costs/utils.h" #include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/util/overflow.h" namespace tensorflow { namespace grappler { @@ -1544,7 +1545,14 @@ int64_t OpLevelCostEstimator::CalculateTensorElementCount( auto tensor_shape = MaybeGetMinimumShape(tensor.shape(), num_dims, found_unknown_shapes); for (const auto& dim : tensor_shape.dim()) { - tensor_size *= dim.size(); + int64_t new_tensor_size = MultiplyWithoutOverflow(tensor_size, dim.size()); + if (new_tensor_size < 0) { + VLOG(1) << "Overflow encountered when computing element count of a " + "tensor, multiplying " + << tensor_size << " with " << dim.size(); + return -1; + } + tensor_size = new_tensor_size; } return tensor_size; } @@ -1554,7 +1562,13 @@ int64_t OpLevelCostEstimator::CalculateTensorSize( int64_t count = CalculateTensorElementCount(tensor, found_unknown_shapes); int size = DataTypeSize(BaseType(tensor.dtype())); VLOG(2) << "Count: " << count << " DataTypeSize: " << size; - return count * size; + int64_t tensor_size = MultiplyWithoutOverflow(count, size); + if (tensor_size < 0) { + VLOG(1) << "Overflow encountered when computing tensor size, multiplying " + << count << " with " << size; + return -1; + } + return tensor_size; } int64_t OpLevelCostEstimator::CalculateInputSize(const OpInfo& op_info, @@ -1607,7 +1621,14 @@ int64_t OpLevelCostEstimator::CalculateOutputSize(const OpInfo& op_info, auto output_shape = MaybeGetMinimumShape(original_output_shape, num_dims, found_unknown_shapes); for (const auto& dim : output_shape.dim()) { - output_size *= dim.size(); + int64_t new_output_size = + MultiplyWithoutOverflow(output_size, dim.size()); + if (new_output_size < 0) { + VLOG(1) << "Overflow encountered when estimating cost, multiplying " + << output_size << " with " << dim.size(); + return -1; + } + output_size = new_output_size; } total_output_size += output_size; VLOG(1) << "Output Size: " << output_size @@ -2132,7 +2153,7 @@ OpInfo::TensorProperties OpLevelCostEstimator::DescribeTensor( } /* static */ -OpLevelCostEstimator::ConvolutionDimensions +StatusOr OpLevelCostEstimator::OpDimensionsFromInputs( const TensorShapeProto& original_image_shape, const OpInfo& op_info, bool* found_unknown_shapes) { @@ -2169,6 +2190,11 @@ OpLevelCostEstimator::OpDimensionsFromInputs( std::vector strides = GetStrides(op_info); int64_t sx = strides[x_index]; int64_t sy = strides[y_index]; + if (sx == 0 || sy == 0) { + return errors::InvalidArgument( + "Stride must be > 0 for Height and Width, but got (", sy, ", ", sx, + ")"); + } const auto padding = GetPadding(op_info); int64_t ox = GetOutputSize(ix, kx, sx, padding); @@ -2185,8 +2211,9 @@ Status OpLevelCostEstimator::PredictMaxPool(const OpContext& op_context, bool found_unknown_shapes = false; const auto& op_info = op_context.op_info; // x: op_info.inputs(0) - ConvolutionDimensions dims = OpDimensionsFromInputs( - op_info.inputs(0).shape(), op_info, &found_unknown_shapes); + TF_ASSIGN_OR_RETURN(ConvolutionDimensions dims, + OpDimensionsFromInputs(op_info.inputs(0).shape(), op_info, + &found_unknown_shapes)); // kx * ky - 1 comparisons per output (kx * xy > 1) // or 1 copy per output (kx * k1 = 1). int per_output_ops = dims.kx * dims.ky == 1 ? 1 : dims.kx * dims.ky - 1; @@ -2227,8 +2254,9 @@ Status OpLevelCostEstimator::PredictMaxPoolGrad(const OpContext& op_context, op_info.ShortDebugString()); } - ConvolutionDimensions dims = OpDimensionsFromInputs( - op_info.inputs(0).shape(), op_info, &found_unknown_shapes); + TF_ASSIGN_OR_RETURN(ConvolutionDimensions dims, + OpDimensionsFromInputs(op_info.inputs(0).shape(), op_info, + &found_unknown_shapes)); int64_t ops = 0; if (dims.kx == 1 && dims.ky == 1) { @@ -2303,8 +2331,9 @@ Status OpLevelCostEstimator::PredictAvgPool(const OpContext& op_context, bool found_unknown_shapes = false; const auto& op_info = op_context.op_info; // x: op_info.inputs(0) - ConvolutionDimensions dims = OpDimensionsFromInputs( - op_info.inputs(0).shape(), op_info, &found_unknown_shapes); + TF_ASSIGN_OR_RETURN(ConvolutionDimensions dims, + OpDimensionsFromInputs(op_info.inputs(0).shape(), op_info, + &found_unknown_shapes)); // kx * ky - 1 additions and 1 multiplication per output. int64_t ops = dims.batch * dims.ox * dims.oy * dims.oz * dims.kx * dims.ky; @@ -2361,8 +2390,9 @@ Status OpLevelCostEstimator::PredictAvgPoolGrad(const OpContext& op_context, found_unknown_shapes = true; } - ConvolutionDimensions dims = - OpDimensionsFromInputs(x_shape, op_info, &found_unknown_shapes); + TF_ASSIGN_OR_RETURN( + ConvolutionDimensions dims, + OpDimensionsFromInputs(x_shape, op_info, &found_unknown_shapes)); int64_t ops = 0; if (dims.kx <= dims.sx && dims.ky <= dims.sy) { @@ -2388,8 +2418,9 @@ Status OpLevelCostEstimator::PredictFusedBatchNorm( // offset: op_info.inputs(2) // mean: op_info.inputs(3) --> only for inference // variance: op_info.inputs(4) --> only for inference - ConvolutionDimensions dims = OpDimensionsFromInputs( - op_info.inputs(0).shape(), op_info, &found_unknown_shapes); + TF_ASSIGN_OR_RETURN(ConvolutionDimensions dims, + OpDimensionsFromInputs(op_info.inputs(0).shape(), op_info, + &found_unknown_shapes)); const bool is_training = IsTraining(op_info); int64_t ops = 0; @@ -2438,8 +2469,9 @@ Status OpLevelCostEstimator::PredictFusedBatchNormGrad( // scale: op_info.inputs(2) // mean: op_info.inputs(3) // variance or inverse of variance: op_info.inputs(4) - ConvolutionDimensions dims = OpDimensionsFromInputs( - op_info.inputs(1).shape(), op_info, &found_unknown_shapes); + TF_ASSIGN_OR_RETURN(ConvolutionDimensions dims, + OpDimensionsFromInputs(op_info.inputs(1).shape(), op_info, + &found_unknown_shapes)); int64_t ops = 0; const auto rsqrt_cost = Eigen::internal::functor_traits< @@ -2660,27 +2692,42 @@ Status OpLevelCostEstimator::PredictCropAndResize(const OpContext& op_context, // calculation differs from rough estimate in implementation, as it separates // out cost per box from cost per pixel and cost per element. + // Since crop arguments are user controlled, check for overflow. + int64_t crop_area = MultiplyWithoutOverflow(crop_height, crop_width); + if (crop_area < 0) + return errors::InvalidArgument("Cannot estimate cost, multiplying ", + crop_height, " with ", crop_width, + " would overflow"); + int64_t crop_volume = MultiplyWithoutOverflow(crop_area, num_boxes); + if (crop_volume < 0) + return errors::InvalidArgument("Cannot estimate cost, multiplying ", + crop_area, " with ", num_boxes, + " would overflow"); + int64_t crop_depth = MultiplyWithoutOverflow(crop_height, num_boxes); + if (crop_depth < 0) + return errors::InvalidArgument("Cannot estimate cost, multiplying ", + crop_height, " with ", num_boxes, + " would overflow"); + // Ops for variables height_scale and width_scale. int64_t ops = (sub_cost * 6 + mul_cost * 2 + div_cost * 2) * num_boxes; // Ops for variable in_y. - ops += (mul_cost * 2 + sub_cost + add_cost) * crop_height * num_boxes; + ops += (mul_cost * 2 + sub_cost + add_cost) * crop_depth; // Ops for variable in_x (same computation across both branches). - ops += (mul_cost * 2 + sub_cost + add_cost) * crop_height * crop_width * - num_boxes; + ops += (mul_cost * 2 + sub_cost + add_cost) * crop_volume; // Specify op_cost based on the method. if (use_bilinear_interp) { // Ops for variables top_y_index, bottom_y_index, y_lerp. - ops += (floor_cost + ceil_cost + sub_cost) * crop_height * num_boxes; + ops += (floor_cost + ceil_cost + sub_cost) * crop_depth; // Ops for variables left_x, right_x, x_lerp; - ops += (floor_cost + ceil_cost + sub_cost) * crop_height * crop_width * - num_boxes; + ops += (floor_cost + ceil_cost + sub_cost) * crop_volume; // Ops for innermost loop across depth. ops += (cast_to_float_cost * 4 + add_cost * 3 + sub_cost * 3 + mul_cost * 3) * output_elements; } else /* method == "nearest" */ { // Ops for variables closest_x_index and closest_y_index. - ops += round_cost * 2 * crop_height * crop_width * num_boxes; + ops += round_cost * 2 * crop_volume; // Ops for innermost loop across depth. ops += cast_to_float_cost * output_elements; } diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.h b/tensorflow/core/grappler/costs/op_level_cost_estimator.h index 6c5764f3fe3a89..f7de566548a784 100644 --- a/tensorflow/core/grappler/costs/op_level_cost_estimator.h +++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.h @@ -290,7 +290,7 @@ class OpLevelCostEstimator { bool* found_unknown_shapes); // For Pooling, FusedBatchNorm, and their grad ops. - static ConvolutionDimensions OpDimensionsFromInputs( + static StatusOr OpDimensionsFromInputs( const TensorShapeProto& original_image_shape, const OpInfo& op_info, bool* found_unknown_shapes); diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc index 560845ee980eba..0cddce42df2f04 100644 --- a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc +++ b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_shape.pb.h" #include "tensorflow/core/framework/types.h" +#include "tensorflow/core/platform/status_matchers.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/protobuf/device_properties.pb.h" @@ -558,9 +559,10 @@ class OpLevelCostEstimatorTest : public ::testing::Test { } bool found_unknown_shapes; - auto dims = OpLevelCostEstimator::OpDimensionsFromInputs( - op_context.op_info.inputs(0).shape(), op_context.op_info, - &found_unknown_shapes); + TF_ASSERT_OK_AND_ASSIGN( + auto dims, OpLevelCostEstimator::OpDimensionsFromInputs( + op_context.op_info.inputs(0).shape(), op_context.op_info, + &found_unknown_shapes)); Padding padding_enum; if (padding == "VALID") { padding_enum = Padding::VALID; @@ -581,6 +583,38 @@ class OpLevelCostEstimatorTest : public ::testing::Test { EXPECT_EQ(padding_enum, dims.padding); } + StatusOr + CallOpDimensionsFromInputs(const int n, const int h, const int w, const int c, + const int kx, const int ky, const int sx, + const int sy, const string& data_format, + const string& padding) { + OpContext op_context; + + const std::vector x = {n, h, w, c}; + const std::vector ksize = {1, kx, ky, 1}; + std::vector strides; + if (data_format == "NHWC") { + strides = {1, sy, sx, 1}; + } else { + strides = {1, 1, sy, sx}; + } + + auto& op_info = op_context.op_info; + SetCpuDevice(&op_info); + op_info.set_op("MaxPool"); + + DescribeTensor4D(x[0], x[1], x[2], x[3], op_info.add_inputs()); + auto* attr = op_info.mutable_attr(); + SetAttrValue(data_format, &(*attr)["data_format"]); + SetAttrValue(padding, &(*attr)["padding"]); + SetAttrValue(strides, &(*attr)["strides"]); + SetAttrValue(ksize, &(*attr)["ksize"]); + bool found_unknown_shapes; + return OpLevelCostEstimator::OpDimensionsFromInputs( + op_context.op_info.inputs(0).shape(), op_context.op_info, + &found_unknown_shapes); + } + OpLevelCostEstimator estimator_; }; @@ -1383,6 +1417,26 @@ TEST_F(OpLevelCostEstimatorTest, OpDimensionsFromInputs) { } } +TEST_F(OpLevelCostEstimatorTest, OpDimensionsFromInputsError) { + std::vector paddings = {"VALID", "SAME"}; + std::vector formats = {"NHWC", "NCHW"}; + for (const auto& p : paddings) { + for (const auto& f : formats) { + // n, h, w, c, kx, ky, sx, sy, data_format, padding. + ASSERT_THAT( + CallOpDimensionsFromInputs(10, 14, 14, 3840, 3, 3, 0, 2, f, p), + testing::StatusIs( + error::INVALID_ARGUMENT, + "Stride must be > 0 for Height and Width, but got (2, 0)")); + ASSERT_THAT( + CallOpDimensionsFromInputs(10, 14, 14, 3840, 3, 3, 2, 0, f, p), + testing::StatusIs( + error::INVALID_ARGUMENT, + "Stride must be > 0 for Height and Width, but got (0, 2)")); + } + } +} + TEST_F(OpLevelCostEstimatorTest, PredictMaxPool) { auto predict_max_pool = [this](const int n, const int in, const int c, const int k, const int s, diff --git a/tensorflow/core/grappler/costs/utils.cc b/tensorflow/core/grappler/costs/utils.cc index b6dddf00d61c81..abfc12ec4e8fd5 100644 --- a/tensorflow/core/grappler/costs/utils.cc +++ b/tensorflow/core/grappler/costs/utils.cc @@ -45,6 +45,7 @@ limitations under the License. #include "tensorflow/core/platform/protobuf.h" #include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow/core/util/device_name_utils.h" +#include "tensorflow/core/util/overflow.h" namespace tensorflow { namespace grappler { @@ -217,7 +218,13 @@ int64_t CalculateTensorSize(const OpInfo::TensorProperties& prop) { } int64_t num_elems = TensorShape(shape).num_elements(); - return num_elems * size; + int64_t tensor_size = MultiplyWithoutOverflow(num_elems, size); + if (tensor_size < 0) { + VLOG(1) << "Overflow encountered when computing tensor size, multiplying " + << num_elems << " with " << size; + return -1; + } + return tensor_size; } int64_t CalculateOutputSize( diff --git a/tensorflow/core/grappler/costs/utils_test.cc b/tensorflow/core/grappler/costs/utils_test.cc index 944f5356e84190..5db6991a1b27ea 100644 --- a/tensorflow/core/grappler/costs/utils_test.cc +++ b/tensorflow/core/grappler/costs/utils_test.cc @@ -202,6 +202,10 @@ TEST(UtilsTest, CalculateTensorSize) { EXPECT_EQ( DataTypeSize(DT_FLOAT) * 1 * 7 * 1 * 99, CalculateTensorSize(ShapeToTensorProperty({-1, 7, -1, 99}, DT_FLOAT))); + + // Test overflow + EXPECT_EQ(-1, CalculateTensorSize(ShapeToTensorProperty( + {4096, 4096, 4096, 33554432}, DT_FLOAT))); } TEST(UtilsTest, CalculateOutputSize) { diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc index e1eeb42e9c4f44..d70dc2e01c1484 100644 --- a/tensorflow/core/grappler/mutable_graph_view.cc +++ b/tensorflow/core/grappler/mutable_graph_view.cc @@ -68,6 +68,9 @@ bool IsIdentityConsumingSwitch(const MutableGraphView& graph, } NodeDef* input_node = graph.GetNode(tensor_id.node()); + if (input_node == nullptr) { + return false; + } return IsSwitch(*input_node); } return false; diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc index 3706d4096ecbb3..701689c45c9b72 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding.cc @@ -1017,7 +1017,12 @@ bool ConstantFolding::IsFoldableUncached( } } for (const auto& output_prop : output_props) { - const PartialTensorShape output_shape(output_prop.shape()); + PartialTensorShape output_shape; + if (!PartialTensorShape::BuildPartialTensorShape(output_prop.shape(), + &output_shape) + .ok()) { + return false; + } if (output_shape.IsFullyDefined()) { const int64_t num_bytes = output_shape.num_elements() * DataTypeSize(output_prop.dtype()); @@ -1363,6 +1368,11 @@ Status ConstantFolding::EvaluateOneFoldable(const NodeDef& node, input_tensor.ToString(), " has a dtype of DT_INVALID.")); } + if (IsRefType(raw_val.dtype())) { + return errors::InvalidArgument( + "Not allowed to construct a tensor with reference dtype, got ", + DataTypeString(raw_val.dtype())); + } Tensor* value = new Tensor(raw_val.dtype(), raw_val.tensor_shape()); if (!value->FromProto(raw_val)) { delete (value); @@ -1684,15 +1694,21 @@ Status ConstantFolding::FoldGraph( return Status::OK(); } -bool ConstantFolding::IsSimplifiableReshape( +Status ConstantFolding::IsSimplifiableReshape( const NodeDef& node, const GraphProperties& properties) const { if (!IsReshape(node)) { - return false; + return errors::Internal("Node ", node.name(), " is not a Reshape node"); + } + if (2 > node.input_size()) { + return errors::Internal("Node ", node.name(), + " must have at most 2 inputs but has ", + node.input_size()); } - CHECK_LE(2, node.input_size()); const NodeDef* new_shape = node_map_->GetNode(node.input(1)); if (!IsReallyConstant(*new_shape)) { - return false; + return errors::Internal("Node ", node.name(), " has shape ", + new_shape->DebugString(), + " which is not a constant"); } TensorVector outputs; auto outputs_cleanup = gtl::MakeCleanup([&outputs] { @@ -1703,22 +1719,29 @@ bool ConstantFolding::IsSimplifiableReshape( Status s = EvaluateNode(*new_shape, TensorVector(), &outputs); if (!s.ok()) { - return false; + return errors::Internal("Could not evaluate node ", node.name()); + } + if (outputs.size() != 1) { + return errors::Internal("Node ", node.name(), + " must have exactly 1 output but has ", + outputs.size()); } - CHECK_EQ(1, outputs.size()); const std::vector& props = properties.GetInputProperties(node.name()); if (props.empty()) { - return false; + return errors::Internal("Node ", node.name(), " has no properties"); } const OpInfo::TensorProperties& prop = props[0]; if (prop.dtype() == DT_INVALID) { - return false; + return errors::Internal("Node ", node.name(), " has property ", + prop.DebugString(), " with invalid dtype"); } const PartialTensorShape shape(prop.shape()); if (!shape.IsFullyDefined()) { - return false; + return errors::Internal("Node ", node.name(), " has property ", + prop.DebugString(), " with shape ", + shape.DebugString(), " which is not fully defined"); } PartialTensorShape new_dims; @@ -1728,17 +1751,23 @@ bool ConstantFolding::IsSimplifiableReshape( int32_t dim = outputs[0]->flat()(i); shp.push_back(dim); } - TF_CHECK_OK(TensorShapeUtils::MakeShape(shp, &new_dims)); + s = TensorShapeUtils::MakeShape(shp, &new_dims); + if (!s.ok()) return s; } else { std::vector shp; for (int i = 0; i < outputs[0]->NumElements(); ++i) { int64_t dim = outputs[0]->flat()(i); shp.push_back(dim); } - TF_CHECK_OK(TensorShapeUtils::MakeShape(shp, &new_dims)); + if (!s.ok()) return s; } - return shape.IsCompatibleWith(new_dims); + if (!shape.IsCompatibleWith(new_dims)) { + return errors::Internal("Expected shape ", shape.DebugString(), + "to be compatible with ", new_dims.DebugString()); + } + + return Status::OK(); } #define IS_VALUE_CASE(DTYPE, VALUE) \ @@ -2925,7 +2954,7 @@ bool ConstantFolding::SimplifyReduction(GraphDef* optimized_graph, bool ConstantFolding::SimplifyReshape(const GraphProperties& properties, bool use_shape_info, NodeDef* node) { if (!use_shape_info || node->attr().count("T") == 0 || - !IsSimplifiableReshape(*node, properties)) { + !IsSimplifiableReshape(*node, properties).ok()) { return false; } DataType output_type = node->attr().at("T").type(); @@ -3480,6 +3509,9 @@ bool ConstantFolding::MulConvPushDown(GraphDef* optimized_graph, NodeDef* node, NodeDef* mul_left_child = node_map_->GetNode(node->input(0)); NodeDef* mul_right_child = node_map_->GetNode(node->input(1)); + if (mul_left_child == nullptr || mul_right_child == nullptr) { + return false; + } // One child must be constant, and the second must be Conv op. const bool left_child_is_constant = IsReallyConstant(*mul_left_child); const bool right_child_is_constant = IsReallyConstant(*mul_right_child); diff --git a/tensorflow/core/grappler/optimizers/constant_folding.h b/tensorflow/core/grappler/optimizers/constant_folding.h index 1de88fed0f99cf..9305aa09764f3d 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.h +++ b/tensorflow/core/grappler/optimizers/constant_folding.h @@ -129,8 +129,8 @@ class ConstantFolding : public GraphOptimizer { Status FoldGraph(const GraphProperties& properties, GraphDef* output, absl::flat_hash_set* nodes_to_not_simplify); - bool IsSimplifiableReshape(const NodeDef& node, - const GraphProperties& properties) const; + Status IsSimplifiableReshape(const NodeDef& node, + const GraphProperties& properties) const; Status SimplifyGraph(GraphDef* optimized_graph, GraphProperties* properties, absl::flat_hash_set* nodes_to_not_simplify); Status SimplifyNode(NodeDef* node, GraphDef* optimized_graph, diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc index aadea833a4fc48..bfd98a58a77718 100644 --- a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc @@ -75,8 +75,10 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) const { } const NodeDef* input = node_map_->GetNode(NodeName(node.input(0))); - CHECK(input != nullptr) << "node = " << node.name() - << " input = " << node.input(0); + if (input == nullptr) { + VLOG(1) << "node = " << node.name() << " input = " << node.input(0); + return false; + } // Don't remove Identity nodes corresponding to Variable reads or following // Recv. if (IsVariable(*input) || IsRecv(*input)) { diff --git a/tensorflow/core/ir/importexport/convert_attributes.cc b/tensorflow/core/ir/importexport/convert_attributes.cc index dd91ad0b0b3beb..21425f4ab7e241 100644 --- a/tensorflow/core/ir/importexport/convert_attributes.cc +++ b/tensorflow/core/ir/importexport/convert_attributes.cc @@ -365,6 +365,8 @@ tensorflow::StatusOr ConvertNonFuncAttributeValue( TF_ASSIGN_OR_RETURN( auto attr, ConvertAttributeValue(subattr.second, builder, tfgDialect)); + if (subattr.first.empty()) + return InvalidArgument("empty func_attr name"); subattrs.push_back(builder.getNamedAttr(subattr.first, attr)); } attrs.push_back(FuncAttr::get(builder.getContext(), func_attr.name(), @@ -389,6 +391,7 @@ tensorflow::StatusOr ConvertAttributeValue( case AttrValue::kFunc: { NamedAttrList attrs; for (const auto& func_attr : value.func().attr()) { + if (func_attr.first.empty()) return InvalidArgument("empty attr name"); TF_ASSIGN_OR_RETURN( auto attr, ConvertAttributeValue(func_attr.second, builder, tfgDialect)); diff --git a/tensorflow/core/ir/importexport/functiondef_import.cc b/tensorflow/core/ir/importexport/functiondef_import.cc index 55f520e510de08..3400f16c0d2317 100644 --- a/tensorflow/core/ir/importexport/functiondef_import.cc +++ b/tensorflow/core/ir/importexport/functiondef_import.cc @@ -36,6 +36,7 @@ limitations under the License. #include "tensorflow/core/platform/protobuf.h" #include "tensorflow/core/platform/status.h" #include "tensorflow/core/protobuf/graph_debug_info.pb.h" +#include "tensorflow/core/platform/statusor.h" using tensorflow::AttrValue; using tensorflow::FunctionDef; @@ -43,6 +44,7 @@ using tensorflow::NodeDef; using tensorflow::OpDef; using tensorflow::OpDef_AttrDef; using tensorflow::Status; +using tensorflow::StatusOr; using tensorflow::errors::InvalidArgument; using tensorflow::protobuf::RepeatedPtrField; @@ -65,12 +67,18 @@ class ValueMapManager { placeholder_ty_(placeholder_ty), control_ty_(control_ty) {} - void DefineOperation(Operation* op, StringRef node_name) { + Status DefineOperation(Operation* op, StringRef node_name) { llvm::StringMap>& op_info = values_map_[node_name]; SmallVector& base_operation = op_info["^"]; // Replace placeholders. if (!base_operation.empty()) { Operation* placeholder = base_operation[0].getDefiningOp(); + if (!placeholder || + placeholder->getName().getStringRef() != "tfg.__mlir_placeholder") + return InvalidArgument(absl::StrCat( + "Duplicated node (or function argument) with the same name: `", + node_name.str(), "`")); + op->moveBefore(placeholder); placeholder->replaceAllUsesWith(op); placeholder->erase(); @@ -78,6 +86,7 @@ class ValueMapManager { } base_operation.push_back(op->getResult(1)); base_operation.push_back(op->getResult(0)); + return Status::OK(); } Value GetValueOrCreatePlaceholder(StringRef full_name) { @@ -158,11 +167,15 @@ Status ImportNodes(ValueMapManager value_manager, // Process every node and create a matching MLIR operation for (const NodeDef& node : nodes) { DVLOG(0) << "Processing node " << node.name() << "\n"; + if (node.op().empty()) return InvalidArgument("empty op type"); OperationState state(unknown_loc, absl::StrCat("tfg.", node.op())); // Fetch the inputs, creating placeholder if an input hasn't been visited. - for (const std::string& input : node.input()) + for (const std::string& input : node.input()) { + if (input.empty()) + return InvalidArgument("Node '", node.name(), "' has an empty input"); state.operands.push_back( value_manager.GetValueOrCreatePlaceholder(input)); + } // Retrieve the entry in the nodes_map for this node and infer the result // count from what was inferred during the first traversal above. state.types.push_back(placeholder_ty); @@ -186,7 +199,7 @@ Status ImportNodes(ValueMapManager value_manager, if (colon_sep != StringRef::npos) node_name = node_name.take_front(colon_sep); } - value_manager.DefineOperation(op, node_name); + TF_RETURN_IF_ERROR(value_manager.DefineOperation(op, node_name)); } // We don't expect any placeholder left at this point, fail if any. for (Operation& op : *builder.getInsertionBlock()) { @@ -248,6 +261,8 @@ Status ImportGenericFunction( TFGraphDialect* tfgDialect = cast(func_op->getDialect()); NamedAttrList attrs; DictionaryAttr func_attrs = builder.getDictionaryAttr({}); + if (signature.name().empty()) + return InvalidArgument("generic function without a name"); attrs.append("sym_name", builder.getStringAttr(signature.name())); attrs.append("generic", builder.getUnitAttr()); if (!signature.description().empty()) @@ -314,6 +329,8 @@ Status ImportGenericFunction( // Import the function attributes with a `tf.` prefix to match the current // infrastructure expectations. for (const auto& namedAttr : func.attr()) { + if (namedAttr.first.empty()) + return InvalidArgument("Invalid function attribute name"); const std::string& name = "tf." + namedAttr.first; const AttrValue& tf_attr = namedAttr.second; TF_ASSIGN_OR_RETURN(Attribute attr, @@ -403,7 +420,9 @@ Status ImportGenericFunction( // Import the function body here, after this we have a function with all // the nodes, and the nodes_map contains the mapping from node_name to actual // MLIR Operations. - TF_RETURN_IF_ERROR(ImportNodes(value_manager, func.node_def(), body_builder)); + TF_RETURN_WITH_CONTEXT_IF_ERROR( + ImportNodes(value_manager, func.node_def(), body_builder), + " when importing function ", func.signature().name()); // After the body, the final part is to setup the return. It comes in two // parts: the `ret` field from the FunctionDef for the regular output and the @@ -436,21 +455,31 @@ Status ImportGenericFunction( ret_vals.resize(func.ret_size() + func.control_ret_size(), Value()); for (const auto& ret_val : func.ret()) { auto position = output_name_to_position.find(ret_val.first); - if (position == output_name_to_position.end()) + if (position == output_name_to_position.end()) { return InvalidArgument( "Can't import function, returned value references unknown output " "argument ", ret_val.first); + } + if (ret_val.second.empty()) { + return InvalidArgument("Function '", func.signature().name(), + "' has empty result name"); + } ret_vals[position->second] = value_manager.GetValueOrCreatePlaceholder(ret_val.second); } for (const auto& ret_val : func.control_ret()) { auto position = control_output_to_position.find(ret_val.first); - if (position == control_output_to_position.end()) + if (position == control_output_to_position.end()) { return InvalidArgument( "Can't import function, returned value references unknown output " "argument ", ret_val.first); + } + if (ret_val.second.empty()) { + return InvalidArgument("Function '", func.signature().name(), + "' has empty control result name"); + } Value result = value_manager.GetValueOrCreatePlaceholder( (Twine("^") + ret_val.second).str()); if (!result.getType().isa()) diff --git a/tensorflow/core/ir/importexport/import.cc b/tensorflow/core/ir/importexport/import.cc index c346cc3eca287a..3f25668d22cad7 100644 --- a/tensorflow/core/ir/importexport/import.cc +++ b/tensorflow/core/ir/importexport/import.cc @@ -588,6 +588,7 @@ Status GraphImporter::ConvertNode(const Node& node) { StringAttr::get(context_, node.name())); for (const auto& namedAttr : node.attrs()) { const std::string& name = namedAttr.first; + if (name.empty()) return InvalidArgument("empty attr name"); const AttrValue& tf_attr = namedAttr.second; TF_ASSIGN_OR_RETURN(Attribute attr, ConvertAttributeValue(tf_attr, builder_, dialect_)); @@ -729,6 +730,8 @@ tensorflow::StatusOr ImportFunctionDef( ConvertAttributeValue(tf_attr, builder, tfgDialect)); attrs.append(name, attr); } + if (signature.name().empty()) + return InvalidArgument("function without a name"); attrs.append("sym_name", builder.getStringAttr(name)); if (!signature.description().empty()) @@ -921,9 +924,10 @@ tensorflow::StatusOr ConvertHandleData( // Two entries: a type and a shape. SmallVector dtype_and_shape; for (const auto& handle : handle_data) { + if (handle.dtype() == tensorflow::DT_INVALID) + return InvalidArgument("Invalid dtype for handle_data"); Type dtype; - if (handle.dtype() != tensorflow::DT_INVALID) - TF_RETURN_IF_ERROR(ConvertDataType(handle.dtype(), builder, &dtype)); + TF_RETURN_IF_ERROR(ConvertDataType(handle.dtype(), builder, &dtype)); TF_ASSIGN_OR_RETURN( Attribute shape, ConvertTensorShapeProto(handle.shape(), builder.getContext())); diff --git a/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_arg_name.pbtxt b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_arg_name.pbtxt new file mode 100644 index 00000000000000..f6fbb73ac15ebc --- /dev/null +++ b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_arg_name.pbtxt @@ -0,0 +1,50 @@ +# RUN: not tfg-translate -graphdef-to-mlir %s 2>&1 | FileCheck %s + +# CHECK: INVALID_ARGUMENT: Duplicated node (or function argument) with the same name: `x` +# CHECK-NEXT: when importing function DuplicatedName + +library { + function { + signature { + name: "DuplicatedName" + input_arg { + name: "x" + type_attr: "T" + } + output_arg { + name: "y" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_INT64 + } + } + } + } + node_def { + name: "x" + op: "Identity" + input: "x" + attr { + key: "T" + value { + placeholder: "T" + } + } + } + ret { + key: "y" + value: "x" + } + } +} +versions { + producer: 762 +} diff --git a/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_duplicated_node_name.pbtxt b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_duplicated_node_name.pbtxt new file mode 100644 index 00000000000000..fda3c904ff86ff --- /dev/null +++ b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_duplicated_node_name.pbtxt @@ -0,0 +1,61 @@ +# RUN: not tfg-translate -graphdef-to-mlir %s 2>&1 | FileCheck %s + +# CHECK: INVALID_ARGUMENT: Duplicated node (or function argument) with the same name: `y` +# CHECK-NEXT: when importing function DuplicatedName + +library { + function { + signature { + name: "DuplicatedName" + input_arg { + name: "x" + type_attr: "T" + } + output_arg { + name: "y" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_INT64 + } + } + } + } + node_def { + name: "y" + op: "Identity" + input: "x" + attr { + key: "T" + value { + placeholder: "T" + } + } + } + node_def { + name: "y" + op: "Identity" + input: "x" + attr { + key: "T" + value { + placeholder: "T" + } + } + } + ret { + key: "y" + value: "y" + } + } +} +versions { + producer: 762 +} diff --git a/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_empty_attr_key.pbtxt b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_empty_attr_key.pbtxt new file mode 100644 index 00000000000000..5e9c003b010451 --- /dev/null +++ b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_empty_attr_key.pbtxt @@ -0,0 +1,55 @@ +# RUN: not tfg-translate -graphdef-to-mlir %s 2>&1 | FileCheck %s + +# CHECK: empty attr name + +node { + name: "SaveV/" + op: "PartitionedCall" + device: "/job:localhost/replica:0/task:0/device:CPU:0" + attr { + key: "" + value { + } + } + attr { + key: "Tin" + value { + } + } + attr { + key: "Tout" + value { + } + } + attr { + key: "config" + value { + s: "" + } + } + attr { + key: "config_proto" + value { + s: "" + } + } + attr { + key: "executor_type" + value { + s: "" + } + } + attr { + key: "f" + value { + func { + } + } + } +} +library { + function { + } +} +versions { +} diff --git a/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_empty_func_attr_key.pbtxt b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_empty_func_attr_key.pbtxt new file mode 100644 index 00000000000000..cc4742efcc05ab --- /dev/null +++ b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_empty_func_attr_key.pbtxt @@ -0,0 +1,165 @@ +# RUN: not tfg-translate -graphdef-to-mlir %s 2>&1 | FileCheck %s + +# CHECK: empty attr name + +node { + name: "SaveV/" + op: "PartitionedCall" + device: "/job:localhost/replica:0/task:0/device:CPU:0" + attr { + key: "T" + value { + func { + attr { + key: "" + value { + } + } + attr { + key: "\036" + value { + } + } + attr { + key: " " + value { + } + } + attr { + key: "1" + value { + } + } + attr { + key: "2" + value { + } + } + attr { + key: "loc:@c" + value { + } + } + attr { + key: "\177" + value { + func { + attr { + key: "" + value { + } + } + attr { + key: "\036" + value { + } + } + attr { + key: "1" + value { + } + } + attr { + key: "2" + value { + } + } + attr { + key: "_class" + value { + } + } + attr { + key: "\177" + value { + } + } + attr { + key: "\177\177" + value { + } + } + } + } + } + } + } + } + attr { + key: "Tin" + value { + } + } + attr { + key: "Tout" + value { + } + } + attr { + key: "config" + value { + s: "" + } + } + attr { + key: "config_proto" + value { + s: "" + } + } + attr { + key: "executor_type" + value { + s: "" + } + } + attr { + key: "f" + value { + func { + attr { + key: "E" + value { + } + } + } + } + } + attr { + key: "shape" + value { + } + } +} +node { + name: "serving_default_x" + op: "Placeholder" + device: "/job:localhost/replica:0/task:0/device:CPU:0" + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "shape" + value { + shape { + unknown_rank: true + } + } + } +} +library { + function { + signature { + is_distributed_communication: true + } + } + gradient { + gradient_func: "\001\000\000\000\000\000\000\000" + } +} +versions { +} + diff --git a/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_empty_func_attr_name.pbtxt b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_empty_func_attr_name.pbtxt new file mode 100644 index 00000000000000..40be7f8b0ced67 --- /dev/null +++ b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_empty_func_attr_name.pbtxt @@ -0,0 +1,38 @@ +# RUN: not tfg-translate -graphdef-to-mlir %s 2>&1 | FileCheck %s + +# CHECK: empty func_attr name + +node { + name: "NoOp" + op: "NoOp" + attr { + key: "dense_shapes" + value { + list { + shape { + dim { + size: 1 + } + } + shape { + dim { + size: 1 + } + } + func { + attr { + key: "" + value { + type: DT_QINT16 + } + } + } + } + } + } +} +library { +} +versions { + producer: 27 +} diff --git a/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_empty_op_type.pbtxt b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_empty_op_type.pbtxt new file mode 100644 index 00000000000000..0ed0763eef75c2 --- /dev/null +++ b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_empty_op_type.pbtxt @@ -0,0 +1,22 @@ +# RUN: not tfg-translate -graphdef-to-mlir %s 2>&1 | FileCheck %s + +# CHECK: empty op type + +library { + function { + signature { + name: "XTimesTwo" + } + node_def { + name: "value" + attr { + key: "Tin" + value { + placeholder: "\t" + } + } + } + } +} +versions { +} diff --git a/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_func_with_empty_name.pbtxt b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_func_with_empty_name.pbtxt new file mode 100644 index 00000000000000..327da3f2153240 --- /dev/null +++ b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_func_with_empty_name.pbtxt @@ -0,0 +1,10 @@ +# RUN: not tfg-translate -graphdef-to-mlir %s 2>&1 | FileCheck %s + +# CHECK: function without a name + +library { + function { + signature { + } + } +} diff --git a/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_generic_func_with_empty_control_result.pbtxt b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_generic_func_with_empty_control_result.pbtxt new file mode 100644 index 00000000000000..b7d82f87842dc7 --- /dev/null +++ b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_generic_func_with_empty_control_result.pbtxt @@ -0,0 +1,26 @@ +# RUN: not tfg-translate -graphdef-to-mlir %s 2>&1 | FileCheck %s + +# CHECK: Function 'foo' has empty control result name + +library { + function { + signature { + name: "foo" + control_output: "output" + } + node_def { + name: "y" + op: "NoOp" + attr { + key: "T" + value { + placeholder: "T" + } + } + } + control_ret { + key: "output" + value: "" + } + } +} diff --git a/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_generic_func_with_empty_input.pbtxt b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_generic_func_with_empty_input.pbtxt new file mode 100644 index 00000000000000..5b1c3cff4f85ab --- /dev/null +++ b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_generic_func_with_empty_input.pbtxt @@ -0,0 +1,22 @@ +# RUN: not tfg-translate -graphdef-to-mlir %s 2>&1 | FileCheck %s + +# CHECK: Node 'y' has an empty input + +library { + function { + signature { + name: "foo" + } + node_def { + name: "y" + input: "" + op: "Identity" + attr { + key: "T" + value { + placeholder: "T" + } + } + } + } +} diff --git a/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_generic_func_with_empty_name.pbtxt b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_generic_func_with_empty_name.pbtxt new file mode 100644 index 00000000000000..e396699069d6fa --- /dev/null +++ b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_generic_func_with_empty_name.pbtxt @@ -0,0 +1,20 @@ +# RUN: not tfg-translate -graphdef-to-mlir %s 2>&1 | FileCheck %s + +# CHECK: generic function without a name + +library { + function { + signature { + } + node_def { + name: "y" + op: "NoOp" + attr { + key: "T" + value { + placeholder: "T" + } + } + } + } +} diff --git a/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_generic_func_with_empty_result.pbtxt b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_generic_func_with_empty_result.pbtxt new file mode 100644 index 00000000000000..f4fc5263ebf790 --- /dev/null +++ b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_generic_func_with_empty_result.pbtxt @@ -0,0 +1,29 @@ +# RUN: not tfg-translate -graphdef-to-mlir %s 2>&1 | FileCheck %s + +# CHECK: Function 'foo' has empty result name + +library { + function { + signature { + name: "foo" + output_arg { + name: "output" + type: DT_INT32 + } + } + node_def { + name: "y" + op: "NoOp" + attr { + key: "T" + value { + placeholder: "T" + } + } + } + ret { + key: "output" + value: "" + } + } +} diff --git a/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_generic_function_attr_name.pbtxt b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_generic_function_attr_name.pbtxt new file mode 100644 index 00000000000000..7a0f18f6732027 --- /dev/null +++ b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_generic_function_attr_name.pbtxt @@ -0,0 +1,52 @@ +# RUN: not tfg-translate -graphdef-to-mlir %s 2>&1 | FileCheck %s + +# CHECK: Invalid function attribute name + +library { + function { + signature { + name: "foo" + input_arg { + name: "a" + } + output_arg { + name: "d" + } + } + node_def { + op: "Const" + attr { + key: "_b" + value { + placeholder: "T" + } + } + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + } + } + } + } + ret { + key: "d" + value: "a" + } + attr { + key: "" + value { + s: "a" + } + } + } +} diff --git a/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_handle_data.pbtxt b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_handle_data.pbtxt new file mode 100644 index 00000000000000..2a6fb92da04397 --- /dev/null +++ b/tensorflow/core/ir/importexport/tests/graphdef-to-mlir/invalid_handle_data.pbtxt @@ -0,0 +1,92 @@ +# RUN: not tfg-translate -graphdef-to-mlir %s 2>&1 | FileCheck %s + +# CHECK: INVALID_ARGUMENT: Invalid dtype for handle_data + +library { + function { + signature { + name: "XTimesTwo" + input_arg { + name: "x" + type_attr: "T" + } + output_arg { + name: "y" + type_attr: "T" + # This empty handle_data is invalid. + handle_data { + } + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_INT64 + } + } + } + } + node_def { + name: "two" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT64 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT64 + tensor_shape { + } + int64_val: 2 + } + } + } + } + node_def { + name: "scale" + op: "Cast" + input: "two:output:0" + attr { + key: "DstT" + value { + placeholder: "T" + } + } + attr { + key: "SrcT" + value { + type: DT_INT64 + } + } + } + node_def { + name: "y" + op: "Mul" + input: "x" + input: "scale:y:0" + attr { + key: "T" + value { + placeholder: "T" + } + } + } + ret { + key: "y" + value: "y:z:0" + } + } +} +versions { + producer: 762 +} + diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 9757fc9fdddc66..bee81e28e0de43 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -26,6 +26,10 @@ load( "tf_fingerprint_deps", "tf_kernel_tests_linkstatic", ) +load( + "//third_party/mkl:build_defs.bzl", + "mkl_deps", +) # buildifier: disable=same-origin-load load("//tensorflow:tensorflow.bzl", "cc_header_only_library") @@ -57,10 +61,6 @@ load( "//tensorflow/core/platform:build_config_root.bzl", "tf_cuda_tests_tags", ) -load( - "//third_party/mkl:build_defs.bzl", - "mkl_deps", -) load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda") load( "@local_config_rocm//rocm:build_defs.bzl", @@ -435,6 +435,7 @@ tf_cc_test( "//tensorflow/core:protos_all_cc", "//tensorflow/core:test", "//tensorflow/core:test_main", + "//tensorflow/core/platform:status_matchers", "@com_google_absl//absl/base:core_headers", ], ) @@ -4289,6 +4290,7 @@ tf_kernel_library( deps = [ ":fill_functor", ":gpu_prim_hdrs", + ":sparse_utils", "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", @@ -4494,6 +4496,7 @@ tf_kernel_library( "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core/framework:bounds_check", + "//tensorflow/core/util:overflow", "//third_party/eigen3", ], ) @@ -4802,6 +4805,7 @@ cc_library( SPARSE_DEPS = [ "//tensorflow/core:framework", "//tensorflow/core:lib", + ":sparse_utils", ] tf_kernel_library( @@ -6239,6 +6243,7 @@ filegroup( "sparse_reorder_op.h", "sparse_slice_op.h", "sparse_tensor_dense_matmul_op.h", + "sparse_utils.h", "string_util.h", "string_to_hash_bucket_op.h", "string_to_hash_bucket_fast_op.h", @@ -6496,6 +6501,7 @@ filegroup( "random_op_cpu.h", "random_ops_util.h", "random_poisson_op.cc", + "sparse_utils.cc", "random_shuffle_op.cc", "reduce_join_op.cc", "reduction_ops_all.cc", diff --git a/tensorflow/core/kernels/assign_op.h b/tensorflow/core/kernels/assign_op.h index ca822a58cdacc4..1e84436b27e0d8 100644 --- a/tensorflow/core/kernels/assign_op.h +++ b/tensorflow/core/kernels/assign_op.h @@ -50,6 +50,12 @@ class AssignOp : public OpKernel { // We always return the input ref. context->forward_ref_input_to_ref_output(0, 0); + // Prevent copying uninitialized data, to solve harder to debug undefined + // behaviors that cannot be traced back to the original tensor. + OP_REQUIRES( + context, rhs.IsInitialized(), + errors::Internal("Right hand side of AssignOp is not initialized")); + // We can't always know how this value will be used downstream, so make // conservative assumptions in specifying constraints on the memory // allocation attributes, unless the Grappler graph analysis determined that diff --git a/tensorflow/core/kernels/avgpooling_op.cc b/tensorflow/core/kernels/avgpooling_op.cc index 0429d50cdec23a..bc536d187512af 100644 --- a/tensorflow/core/kernels/avgpooling_op.cc +++ b/tensorflow/core/kernels/avgpooling_op.cc @@ -35,6 +35,7 @@ limitations under the License. #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/gtl/array_slice.h" #include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/util/overflow.h" #include "tensorflow/core/util/padding.h" #include "tensorflow/core/util/tensor_format.h" @@ -77,10 +78,10 @@ class AvgPoolingOp : public UnaryOp { OP_REQUIRES(context, ksize_[0] == 1 && stride_[0] == 1, errors::Unimplemented( "Pooling is not yet supported on the batch dimension.")); - for (int i = 0; i < ksize_.size(); ++i) { - OP_REQUIRES(context, ksize_[i] != 0, - errors::InvalidArgument("ksize cannot be zero")); + OP_REQUIRES(context, ksize_[i] > 0, + errors::InvalidArgument( + "ksize must be a postive int32 value, got:", ksize_[i])); } } @@ -142,6 +143,11 @@ class AvgPoolingOp : public UnaryOp { OP_REQUIRES(context, ksize_.size() == 4, errors::InvalidArgument("Sliding window ksize field must " "specify 4 dimensions")); + for (int i = 0; i < ksize_.size(); ++i) { + OP_REQUIRES(context, ksize_[i] > 0, + errors::InvalidArgument( + "ksize must be a postive int32 value, got:", ksize_[i])); + } OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_)); OP_REQUIRES(context, stride_.size() == 4, errors::InvalidArgument("Sliding window stride field must " @@ -298,7 +304,7 @@ class AvgPoolingGradOp : public OpKernel { TensorShape output_shape; auto shape_vec = tensor_in_shape.vec(); for (int64_t i = 0; i < tensor_in_shape.NumElements(); ++i) { - output_shape.AddDim(shape_vec(i)); + OP_REQUIRES_OK(context, output_shape.AddDimWithStatus(shape_vec(i))); } const int64_t in_rows = output_shape.dim_size(1); const int64_t in_cols = output_shape.dim_size(2); @@ -457,7 +463,7 @@ class AvgPoolingGradOp : public OpKernel { TensorShape output_shape; auto shape_vec = tensor_in_shape.vec(); for (int64_t i = 0; i < tensor_in_shape.NumElements(); ++i) { - output_shape.AddDim(shape_vec(i)); + OP_REQUIRES_OK(context, output_shape.AddDimWithStatus(shape_vec(i))); } if (output_shape.num_elements() == 0) { @@ -543,7 +549,7 @@ class AvgPoolingGradOpCustomGPUKernel : public OpKernel { TensorShape output_shape; auto shape_vec = tensor_in_shape.vec(); for (int64_t i = 0; i < tensor_in_shape.NumElements(); ++i) { - output_shape.AddDim(shape_vec(i)); + OP_REQUIRES_OK(context, output_shape.AddDimWithStatus(shape_vec(i))); } if (output_shape.num_elements() == 0) { Tensor* output = nullptr; diff --git a/tensorflow/core/kernels/batch_kernels.cc b/tensorflow/core/kernels/batch_kernels.cc index 8f0be9d0a3506d..f2168fbd2ef2d5 100644 --- a/tensorflow/core/kernels/batch_kernels.cc +++ b/tensorflow/core/kernels/batch_kernels.cc @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow/core/framework/op_requires.h" #include "tensorflow/core/framework/resource_mgr.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_util.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/kernels/batching_util/adaptive_shared_batch_scheduler.h" @@ -648,6 +649,12 @@ class UnbatchResource : public ResourceBase { batch_index_t.shape().dim_size(1), "."); } + if (!TensorShapeUtils::IsScalar(context->input(2).shape())) { + return errors::InvalidArgument( + "Input id should be scalar; " + "Got: ", + context->input(2).DebugString(), "."); + } const int64_t batch_key = context->input(2).scalar()(); const bool nonempty_input = batch_index_t.dim_size(0) > 0; @@ -879,8 +886,13 @@ class UnbatchGradResource : public ResourceBase { const Tensor& data_t = context->input(0); const Tensor& batch_index_t = context->input(1); const Tensor& grad_t = context->input(2); + const Tensor& batch_key_t = context->input(3); mutex_lock ml(mu_); + if (batch_key_t.NumElements() != 1) { + return errors::InvalidArgument("Expected `id` to be scalar. Received ", + batch_key_t.DebugString()); + } const int64_t batch_key = context->input(3).scalar()(); // Mark our tensor as available. @@ -896,6 +908,11 @@ class UnbatchGradResource : public ResourceBase { "batch_index is empty while the tensor isn't."); } std::unordered_set missing_tensors; + if (batch_index_t.NumElements() != batch_index_t.dim_size(0) * 3) { + return errors::InvalidArgument( + "batch_index should contain ", batch_index_t.dim_size(0) * 3, + " elements. Received ", batch_index_t.NumElements()); + } const auto batch_index = batch_index_t.shaped({batch_index_t.dim_size(0), 3}); for (int i = 0; i < batch_index_t.dim_size(0); ++i) { diff --git a/tensorflow/core/kernels/bincount_op.cc b/tensorflow/core/kernels/bincount_op.cc index 0f661dd9f201a6..c8fb81f9591546 100644 --- a/tensorflow/core/kernels/bincount_op.cc +++ b/tensorflow/core/kernels/bincount_op.cc @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow/core/framework/types.h" #include "tensorflow/core/kernels/bincount_op.h" #include "tensorflow/core/kernels/fill_functor.h" +#include "tensorflow/core/kernels/sparse_utils.h" #include "tensorflow/core/lib/core/threadpool.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/determinism.h" @@ -276,6 +277,17 @@ class DenseBincountOp : public OpKernel { const Tensor& size_t = ctx->input(1); const Tensor& weights = ctx->input(2); + OP_REQUIRES(ctx, size_t.dims() == 0, + errors::InvalidArgument("Shape must be rank 0 but is rank ", + size_t.dims())); + OP_REQUIRES(ctx, + weights.shape() == data.shape() || weights.NumElements() == 0, + errors::InvalidArgument( + "`weights` must be the same shape as `arr` or a length-0 " + "`Tensor`, in which case it acts as all weights equal to " + "1. Received ", + weights.shape().DebugString())); + Tidx size = size_t.scalar()(); OP_REQUIRES( ctx, size >= 0, @@ -366,16 +378,23 @@ class SparseBincountOp : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& indices = ctx->input(0); - const auto values = ctx->input(1).flat(); + const Tensor& values = ctx->input(1); + const auto values_flat = values.flat(); const Tensor& dense_shape = ctx->input(2); const Tensor& size_t = ctx->input(3); const auto weights = ctx->input(4).flat(); const int64_t weights_size = weights.size(); + OP_REQUIRES(ctx, size_t.dims() == 0, + errors::InvalidArgument("Shape must be rank 0 but is rank ", + size_t.dims())); Tidx size = size_t.scalar()(); OP_REQUIRES( ctx, size >= 0, errors::InvalidArgument("size (", size, ") must be non-negative")); + OP_REQUIRES_OK(ctx, sparse_utils::ValidateSparseTensor( + indices, values, dense_shape, + sparse_utils::IndexValidation::kUnordered)); bool is_1d = dense_shape.NumElements() == 1; @@ -388,11 +407,11 @@ class SparseBincountOp : public OpKernel { if (binary_output_) { OP_REQUIRES_OK(ctx, functor::BincountFunctor::Compute( - ctx, values, weights, out, size)); + ctx, values_flat, weights, out, size)); } else { OP_REQUIRES_OK( ctx, functor::BincountFunctor::Compute( - ctx, values, weights, out, size)); + ctx, values_flat, weights, out, size)); } } else { const auto shape = dense_shape.flat(); @@ -404,7 +423,7 @@ class SparseBincountOp : public OpKernel { const auto indices_mat = indices.matrix(); for (int64_t i = 0; i < indices_mat.dimension(0); ++i) { const int64_t batch = indices_mat(i, 0); - const Tidx bin = values(i); + const Tidx bin = values_flat(i); OP_REQUIRES( ctx, batch < out.dimension(0), errors::InvalidArgument("Index out of bound. `batch` (", batch, @@ -462,6 +481,9 @@ class RaggedBincountOp : public OpKernel { const auto weights = ctx->input(3).flat(); const int64_t weights_size = weights.size(); + OP_REQUIRES(ctx, size_t.dims() == 0, + errors::InvalidArgument("Shape must be rank 0 but is rank ", + size_t.dims())); Tidx size = size_t.scalar()(); OP_REQUIRES( ctx, size >= 0, @@ -471,6 +493,9 @@ class RaggedBincountOp : public OpKernel { int num_values = values.size(); int batch_idx = 0; + OP_REQUIRES(ctx, splits.size() > 0, + errors::InvalidArgument("Splits must be non-empty")); + OP_REQUIRES(ctx, splits(0) == 0, errors::InvalidArgument("Splits must start with 0, not with ", splits(0))); diff --git a/tensorflow/core/kernels/boosted_trees/prediction_ops.cc b/tensorflow/core/kernels/boosted_trees/prediction_ops.cc index 831f6f2f7802dd..2f25ff81554a39 100644 --- a/tensorflow/core/kernels/boosted_trees/prediction_ops.cc +++ b/tensorflow/core/kernels/boosted_trees/prediction_ops.cc @@ -37,7 +37,7 @@ limitations under the License. namespace tensorflow { static void ConvertVectorsToMatrices( - const OpInputList bucketized_features_list, + OpKernelContext* const context, const OpInputList bucketized_features_list, std::vector::ConstMatrix>& bucketized_features) { for (const Tensor& tensor : bucketized_features_list) { if (tensor.dims() == 1) { @@ -45,6 +45,10 @@ static void ConvertVectorsToMatrices( bucketized_features.emplace_back( TTypes::ConstMatrix(v.data(), v.size(), 1)); } else { + OP_REQUIRES(context, TensorShapeUtils::IsMatrix(tensor.shape()), + errors::Internal("Cannot use tensor as matrix, expected " + "vector or matrix, received shape ", + tensor.shape().DebugString())); bucketized_features.emplace_back(tensor.matrix()); } } @@ -58,6 +62,9 @@ class BoostedTreesTrainingPredictOp : public OpKernel { public: explicit BoostedTreesTrainingPredictOp(OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr("num_bucketized_features", &num_bucketized_features_)); OP_REQUIRES_OK(context, @@ -76,17 +83,26 @@ class BoostedTreesTrainingPredictOp : public OpKernel { &bucketized_features_list)); std::vector::ConstMatrix> bucketized_features; bucketized_features.reserve(bucketized_features_list.size()); - ConvertVectorsToMatrices(bucketized_features_list, bucketized_features); + ConvertVectorsToMatrices(context, bucketized_features_list, + bucketized_features); const int batch_size = bucketized_features[0].dimension(0); const Tensor* cached_tree_ids_t; OP_REQUIRES_OK(context, context->input("cached_tree_ids", &cached_tree_ids_t)); + OP_REQUIRES(context, TensorShapeUtils::IsVector(cached_tree_ids_t->shape()), + errors::InvalidArgument( + "cached_tree_ids must be a vector, received shape ", + cached_tree_ids_t->shape().DebugString())); const auto cached_tree_ids = cached_tree_ids_t->vec(); const Tensor* cached_node_ids_t; OP_REQUIRES_OK(context, context->input("cached_node_ids", &cached_node_ids_t)); + OP_REQUIRES(context, TensorShapeUtils::IsVector(cached_node_ids_t->shape()), + errors::InvalidArgument( + "cached_node_ids must be a vector, received shape ", + cached_node_ids_t->shape().DebugString())); const auto cached_node_ids = cached_node_ids_t->vec(); // Allocate outputs. @@ -118,9 +134,9 @@ class BoostedTreesTrainingPredictOp : public OpKernel { output_partial_logits.setZero(); } else { output_tree_ids.setConstant(latest_tree); - auto do_work = [&resource, &bucketized_features, &cached_tree_ids, - &cached_node_ids, &output_partial_logits, - &output_node_ids, latest_tree, + auto do_work = [&context, &resource, &bucketized_features, + &cached_tree_ids, &cached_node_ids, + &output_partial_logits, &output_node_ids, latest_tree, this](int64_t start, int64_t end) { for (int32_t i = start; i < end; ++i) { int32_t tree_id = cached_tree_ids(i); @@ -138,7 +154,11 @@ class BoostedTreesTrainingPredictOp : public OpKernel { // node's value. The following logic handles both of these cases. const auto& node_logits = resource->node_value(tree_id, node_id); if (!node_logits.empty()) { - DCHECK_EQ(node_logits.size(), logits_dimension_); + OP_REQUIRES( + context, node_logits.size() == logits_dimension_, + errors::Internal( + "Expected node_logits.size() == logits_dimension_, got ", + node_logits.size(), " vs ", logits_dimension_)); for (int32_t j = 0; j < logits_dimension_; ++j) { partial_tree_logits[j] -= node_logits[j]; } @@ -151,7 +171,11 @@ class BoostedTreesTrainingPredictOp : public OpKernel { while (true) { if (resource->is_leaf(tree_id, node_id)) { const auto& leaf_logits = resource->node_value(tree_id, node_id); - DCHECK_EQ(leaf_logits.size(), logits_dimension_); + OP_REQUIRES( + context, leaf_logits.size() == logits_dimension_, + errors::Internal( + "Expected leaf_logits.size() == logits_dimension_, got ", + leaf_logits.size(), " vs ", logits_dimension_)); // Tree is done const float tree_weight = resource->GetTreeWeight(tree_id); for (int32_t j = 0; j < logits_dimension_; ++j) { @@ -201,6 +225,9 @@ class BoostedTreesPredictOp : public OpKernel { public: explicit BoostedTreesPredictOp(OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr("num_bucketized_features", &num_bucketized_features_)); OP_REQUIRES_OK(context, @@ -219,7 +246,8 @@ class BoostedTreesPredictOp : public OpKernel { &bucketized_features_list)); std::vector::ConstMatrix> bucketized_features; bucketized_features.reserve(bucketized_features_list.size()); - ConvertVectorsToMatrices(bucketized_features_list, bucketized_features); + ConvertVectorsToMatrices(context, bucketized_features_list, + bucketized_features); const int batch_size = bucketized_features[0].dimension(0); // Allocate outputs. @@ -236,8 +264,8 @@ class BoostedTreesPredictOp : public OpKernel { } const int32_t last_tree = resource->num_trees() - 1; - auto do_work = [&resource, &bucketized_features, &output_logits, last_tree, - this](int64_t start, int64_t end) { + auto do_work = [&context, &resource, &bucketized_features, &output_logits, + last_tree, this](int64_t start, int64_t end) { for (int32_t i = start; i < end; ++i) { std::vector tree_logits(logits_dimension_, 0.0); int32_t tree_id = 0; @@ -246,7 +274,11 @@ class BoostedTreesPredictOp : public OpKernel { if (resource->is_leaf(tree_id, node_id)) { const float tree_weight = resource->GetTreeWeight(tree_id); const auto& leaf_logits = resource->node_value(tree_id, node_id); - DCHECK_EQ(leaf_logits.size(), logits_dimension_); + OP_REQUIRES( + context, leaf_logits.size() == logits_dimension_, + errors::Internal( + "Expected leaf_logits.size() == logits_dimension_, got ", + leaf_logits.size(), " vs ", logits_dimension_)); for (int32_t j = 0; j < logits_dimension_; ++j) { tree_logits[j] += tree_weight * leaf_logits[j]; } @@ -298,6 +330,9 @@ class BoostedTreesExampleDebugOutputsOp : public OpKernel { explicit BoostedTreesExampleDebugOutputsOp( OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr("num_bucketized_features", &num_bucketized_features_)); OP_REQUIRES_OK(context, @@ -319,7 +354,8 @@ class BoostedTreesExampleDebugOutputsOp : public OpKernel { &bucketized_features_list)); std::vector::ConstMatrix> bucketized_features; bucketized_features.reserve(bucketized_features_list.size()); - ConvertVectorsToMatrices(bucketized_features_list, bucketized_features); + ConvertVectorsToMatrices(context, bucketized_features_list, + bucketized_features); const int batch_size = bucketized_features[0].dimension(0); // We need to get the feature ids used for splitting and the logits after @@ -339,14 +375,16 @@ class BoostedTreesExampleDebugOutputsOp : public OpKernel { // features used to split and the associated logits at each point along the // path. Note: feature_ids has one less value than logits_path because the // first value of each logit path will be the bias. - auto do_work = [&resource, &bucketized_features, &output_debug_info, - last_tree](int64_t start, int64_t end) { + auto do_work = [&context, &resource, &bucketized_features, + &output_debug_info, last_tree](int64_t start, int64_t end) { for (int32_t i = start; i < end; ++i) { // Proto to store debug outputs, per example. boosted_trees::DebugOutput example_debug_info; // Initial bias prediction. E.g., prediction based off training mean. const auto& tree_logits = resource->node_value(0, 0); - DCHECK_EQ(tree_logits.size(), 1); + OP_REQUIRES(context, tree_logits.size() == 1, + errors::Internal("Expected tree_logits.size() == 1, got ", + tree_logits.size())); float tree_logit = resource->GetTreeWeight(0) * tree_logits[0]; example_debug_info.add_logits_path(tree_logit); int32_t node_id = 0; @@ -372,7 +410,10 @@ class BoostedTreesExampleDebugOutputsOp : public OpKernel { node_id = resource->next_node(tree_id, node_id, i, bucketized_features); const auto& tree_logits = resource->node_value(tree_id, node_id); - DCHECK_EQ(tree_logits.size(), 1); + OP_REQUIRES( + context, tree_logits.size() == 1, + errors::Internal("Expected tree_logits.size() == 1, got ", + tree_logits.size())); tree_logit = resource->GetTreeWeight(tree_id) * tree_logits[0]; // Output logit incorporates sum of leaf logits from prior trees. example_debug_info.add_logits_path(tree_logit + past_trees_logit); diff --git a/tensorflow/core/kernels/boosted_trees/quantile_ops.cc b/tensorflow/core/kernels/boosted_trees/quantile_ops.cc index 97bbdb95537a83..f14ec5ee2c552c 100644 --- a/tensorflow/core/kernels/boosted_trees/quantile_ops.cc +++ b/tensorflow/core/kernels/boosted_trees/quantile_ops.cc @@ -98,6 +98,9 @@ class BoostedTreesCreateQuantileStreamResourceOp : public OpKernel { explicit BoostedTreesCreateQuantileStreamResourceOp( OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr(kMaxElementsName, &max_elements_)); } @@ -108,6 +111,10 @@ class BoostedTreesCreateQuantileStreamResourceOp : public OpKernel { // disallowed. const Tensor* epsilon_t; OP_REQUIRES_OK(context, context->input(kEpsilonName, &epsilon_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(epsilon_t->shape()), + errors::InvalidArgument( + "epsilon must be a scalar, got a tensor of shape ", + epsilon_t->shape().DebugString())); float epsilon = epsilon_t->scalar()(); OP_REQUIRES( context, epsilon > 0, @@ -115,6 +122,10 @@ class BoostedTreesCreateQuantileStreamResourceOp : public OpKernel { const Tensor* num_streams_t; OP_REQUIRES_OK(context, context->input(kNumStreamsName, &num_streams_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(num_streams_t->shape()), + errors::InvalidArgument( + "num_streams must be a scalar, got a tensor of shape ", + num_streams_t->shape().DebugString())); int64_t num_streams = num_streams_t->scalar()(); OP_REQUIRES(context, num_streams >= 0, errors::InvalidArgument( @@ -143,6 +154,9 @@ class BoostedTreesMakeQuantileSummariesOp : public OpKernel { explicit BoostedTreesMakeQuantileSummariesOp( OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr(kNumFeaturesName, &num_features_)); } @@ -156,7 +170,8 @@ class BoostedTreesMakeQuantileSummariesOp : public OpKernel { const Tensor* example_weights_t; OP_REQUIRES_OK(context, context->input(kExampleWeightsName, &example_weights_t)); - DCHECK(float_features_list.size() > 0) << "Got empty feature list"; + OP_REQUIRES(context, float_features_list.size() > 0, + errors::Internal("Got empty feature list")); auto example_weights = example_weights_t->flat(); const int64_t weight_size = example_weights.size(); const int64_t batch_size = float_features_list[0].flat().size(); @@ -166,6 +181,10 @@ class BoostedTreesMakeQuantileSummariesOp : public OpKernel { "Weights should be a single value or same size as features."))); const Tensor* epsilon_t; OP_REQUIRES_OK(context, context->input(kEpsilonName, &epsilon_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(epsilon_t->shape()), + errors::InvalidArgument( + "epsilon must be a scalar, got a tensor of shape ", + epsilon_t->shape().DebugString())); float epsilon = epsilon_t->scalar()(); OpOutputList summaries_output_list; @@ -190,7 +209,8 @@ class BoostedTreesMakeQuantileSummariesOp : public OpKernel { context, summaries_output_list.allocate( index, - TensorShape({static_cast(summary_entry_list.size()), 4}), + TensorShape( + {static_cast(summary_entry_list.size()), 4}), &output_t)); auto output = output_t->matrix(); for (auto row = 0; row < summary_entry_list.size(); row++) { @@ -223,6 +243,9 @@ class BoostedTreesFlushQuantileSummariesOp : public OpKernel { explicit BoostedTreesFlushQuantileSummariesOp( OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr(kNumFeaturesName, &num_features_)); } @@ -248,7 +271,7 @@ class BoostedTreesFlushQuantileSummariesOp : public OpKernel { const auto summary_list = stream->GetFinalSummary().GetEntryList(); Tensor* output_t; const int64_t summary_list_size = - static_cast(summary_list.size()); + static_cast(summary_list.size()); OP_REQUIRES_OK(context, summaries_output_list.allocate( index, TensorShape({summary_list_size, 4}), &output_t)); @@ -283,7 +306,11 @@ class BoostedTreesQuantileStreamResourceAddSummariesOp : public OpKernel { public: explicit BoostedTreesQuantileStreamResourceAddSummariesOp( OpKernelConstruction* const context) - : OpKernel(context) {} + : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; + } void Compute(OpKernelContext* context) override { ResourceHandle handle; @@ -298,8 +325,11 @@ class BoostedTreesQuantileStreamResourceAddSummariesOp : public OpKernel { OpInputList summaries_list; OP_REQUIRES_OK(context, context->input_list(kSummariesName, &summaries_list)); - int32_t num_streams = stream_resource->num_streams(); - CHECK_EQ(static_cast(num_streams), summaries_list.size()); + auto num_streams = stream_resource->num_streams(); + OP_REQUIRES( + context, num_streams == summaries_list.size(), + errors::Internal("Expected num_streams == summaries_list.size(), got ", + num_streams, " vs ", summaries_list.size())); auto do_quantile_add_summary = [&](const int64_t begin, const int64_t end) { // Iterating all features. @@ -314,7 +344,10 @@ class BoostedTreesQuantileStreamResourceAddSummariesOp : public OpKernel { const auto summary_values = summaries.matrix(); const auto& tensor_shape = summaries.shape(); const int64_t entries_size = tensor_shape.dim_size(0); - CHECK_EQ(tensor_shape.dim_size(1), 4); + OP_REQUIRES( + context, tensor_shape.dim_size(1) == 4, + errors::Internal("Expected tensor_shape.dim_size(1) == 4, got ", + tensor_shape.dim_size(1))); std::vector summary_entries; summary_entries.reserve(entries_size); for (int64_t i = 0; i < entries_size; i++) { @@ -347,6 +380,9 @@ class BoostedTreesQuantileStreamResourceDeserializeOp : public OpKernel { explicit BoostedTreesQuantileStreamResourceDeserializeOp( OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr(kNumStreamsName, &num_features_)); } @@ -366,6 +402,12 @@ class BoostedTreesQuantileStreamResourceDeserializeOp : public OpKernel { // Iterating over all streams. for (int64_t stream_idx = begin; stream_idx < end; stream_idx++) { const Tensor& bucket_boundaries_t = bucket_boundaries_list[stream_idx]; + OP_REQUIRES( + context, TensorShapeUtils::IsVector(bucket_boundaries_t.shape()), + errors::InvalidArgument("bucket boundaries for each stream must be " + "a vector, received shape ", + bucket_boundaries_t.shape().DebugString(), + " for stream ", stream_idx)); const auto& bucket_boundaries = bucket_boundaries_t.vec(); std::vector result; result.reserve(bucket_boundaries.size()); @@ -397,6 +439,9 @@ class BoostedTreesQuantileStreamResourceFlushOp : public OpKernel { explicit BoostedTreesQuantileStreamResourceFlushOp( OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr(kGenerateQuantiles, &generate_quantiles_)); } @@ -413,6 +458,10 @@ class BoostedTreesQuantileStreamResourceFlushOp : public OpKernel { const Tensor* num_buckets_t; OP_REQUIRES_OK(context, context->input(kNumBucketsName, &num_buckets_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(num_buckets_t->shape()), + errors::InvalidArgument( + "num_buckets must be a scalar, got a tensor of shape ", + num_buckets_t->shape().DebugString())); const int64_t num_buckets = num_buckets_t->scalar()(); const int64_t num_streams = stream_resource->num_streams(); @@ -453,6 +502,9 @@ class BoostedTreesQuantileStreamResourceGetBucketBoundariesOp explicit BoostedTreesQuantileStreamResourceGetBucketBoundariesOp( OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr(kNumFeaturesName, &num_features_)); } @@ -467,7 +519,9 @@ class BoostedTreesQuantileStreamResourceGetBucketBoundariesOp mutex_lock l(*stream_resource->mutex()); const int64_t num_streams = stream_resource->num_streams(); - CHECK_EQ(num_features_, num_streams); + OP_REQUIRES(context, num_streams == num_features_, + errors::Internal("Expected num_streams == num_features_, got ", + num_streams, " vs ", num_features_)); OpOutputList bucket_boundaries_list; OP_REQUIRES_OK(context, context->output_list(kBucketBoundariesName, &bucket_boundaries_list)); @@ -477,10 +531,10 @@ class BoostedTreesQuantileStreamResourceGetBucketBoundariesOp for (int64_t stream_idx = begin; stream_idx < end; stream_idx++) { const auto& boundaries = stream_resource->boundaries(stream_idx); Tensor* bucket_boundaries_t = nullptr; - OP_REQUIRES_OK(context, - bucket_boundaries_list.allocate( - stream_idx, {static_cast(boundaries.size())}, - &bucket_boundaries_t)); + OP_REQUIRES_OK( + context, bucket_boundaries_list.allocate( + stream_idx, {static_cast(boundaries.size())}, + &bucket_boundaries_t)); auto* quantiles_flat = bucket_boundaries_t->flat().data(); memcpy(quantiles_flat, boundaries.data(), sizeof(float) * boundaries.size()); @@ -511,6 +565,9 @@ class BoostedTreesBucketizeOp : public OpKernel { public: explicit BoostedTreesBucketizeOp(OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr(kNumFeaturesName, &num_features_)); } diff --git a/tensorflow/core/kernels/boosted_trees/resource_ops.cc b/tensorflow/core/kernels/boosted_trees/resource_ops.cc index 435c7d2880e2f5..f9e1a9a01ddcd3 100644 --- a/tensorflow/core/kernels/boosted_trees/resource_ops.cc +++ b/tensorflow/core/kernels/boosted_trees/resource_ops.cc @@ -36,18 +36,32 @@ REGISTER_KERNEL_BUILDER( class BoostedTreesCreateEnsembleOp : public OpKernel { public: explicit BoostedTreesCreateEnsembleOp(OpKernelConstruction* context) - : OpKernel(context) {} + : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; + } void Compute(OpKernelContext* context) override { // Get the stamp token. const Tensor* stamp_token_t; OP_REQUIRES_OK(context, context->input("stamp_token", &stamp_token_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(stamp_token_t->shape()), + errors::InvalidArgument( + "stamp_token must be a scalar, got a tensor of shape ", + stamp_token_t->shape().DebugString())); int64_t stamp_token = stamp_token_t->scalar()(); // Get the tree ensemble proto. const Tensor* tree_ensemble_serialized_t; OP_REQUIRES_OK(context, context->input("tree_ensemble_serialized", &tree_ensemble_serialized_t)); + OP_REQUIRES( + context, + TensorShapeUtils::IsScalar(tree_ensemble_serialized_t->shape()), + errors::InvalidArgument( + "tree_ensemble_serialized must be a scalar, got a tensor of shape ", + tree_ensemble_serialized_t->shape().DebugString())); std::unique_ptr result( new BoostedTreesEnsembleResource()); if (!result->InitFromSerialized( @@ -76,7 +90,11 @@ REGISTER_KERNEL_BUILDER(Name("BoostedTreesCreateEnsemble").Device(DEVICE_CPU), class BoostedTreesGetEnsembleStatesOp : public OpKernel { public: explicit BoostedTreesGetEnsembleStatesOp(OpKernelConstruction* context) - : OpKernel(context) {} + : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; + } void Compute(OpKernelContext* context) override { // Looks up the resource. @@ -139,7 +157,11 @@ REGISTER_KERNEL_BUILDER( class BoostedTreesSerializeEnsembleOp : public OpKernel { public: explicit BoostedTreesSerializeEnsembleOp(OpKernelConstruction* context) - : OpKernel(context) {} + : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; + } void Compute(OpKernelContext* context) override { core::RefCountPtr tree_ensemble_resource; @@ -166,7 +188,11 @@ REGISTER_KERNEL_BUILDER( class BoostedTreesDeserializeEnsembleOp : public OpKernel { public: explicit BoostedTreesDeserializeEnsembleOp(OpKernelConstruction* context) - : OpKernel(context) {} + : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; + } void Compute(OpKernelContext* context) override { core::RefCountPtr tree_ensemble_resource; @@ -177,12 +203,22 @@ class BoostedTreesDeserializeEnsembleOp : public OpKernel { // Get the stamp token. const Tensor* stamp_token_t; OP_REQUIRES_OK(context, context->input("stamp_token", &stamp_token_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(stamp_token_t->shape()), + errors::InvalidArgument( + "stamp_token must be a scalar, got a tensor of shape ", + stamp_token_t->shape().DebugString())); int64_t stamp_token = stamp_token_t->scalar()(); // Get the tree ensemble proto. const Tensor* tree_ensemble_serialized_t; OP_REQUIRES_OK(context, context->input("tree_ensemble_serialized", &tree_ensemble_serialized_t)); + OP_REQUIRES( + context, + TensorShapeUtils::IsScalar(tree_ensemble_serialized_t->shape()), + errors::InvalidArgument( + "tree_ensemble_serialized must be a scalar, got a tensor of shape ", + tree_ensemble_serialized_t->shape().DebugString())); // Deallocate all the previous objects on the resource. tree_ensemble_resource->Reset(); OP_REQUIRES( diff --git a/tensorflow/core/kernels/boosted_trees/stats_ops.cc b/tensorflow/core/kernels/boosted_trees/stats_ops.cc index fe48695358b5d8..103e4b201587d8 100644 --- a/tensorflow/core/kernels/boosted_trees/stats_ops.cc +++ b/tensorflow/core/kernels/boosted_trees/stats_ops.cc @@ -45,6 +45,9 @@ class BoostedTreesCalculateBestGainsPerFeatureOp : public OpKernel { explicit BoostedTreesCalculateBestGainsPerFeatureOp( OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr("max_splits", &max_splits_)); OP_REQUIRES_OK(context, context->GetAttr("num_features", &num_features_)); } @@ -83,17 +86,33 @@ class BoostedTreesCalculateBestGainsPerFeatureOp : public OpKernel { } const Tensor* l1_t; OP_REQUIRES_OK(context, context->input("l1", &l1_t)); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(l1_t->shape()), + errors::InvalidArgument("l1 must be a scalar, got a tensor of shape ", + l1_t->shape().DebugString())); const auto l1 = l1_t->scalar()(); const Tensor* l2_t; OP_REQUIRES_OK(context, context->input("l2", &l2_t)); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(l2_t->shape()), + errors::InvalidArgument("l2 must be a scalar, got a tensor of shape ", + l2_t->shape().DebugString())); const auto l2 = l2_t->scalar()(); const Tensor* tree_complexity_t; OP_REQUIRES_OK(context, context->input("tree_complexity", &tree_complexity_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(tree_complexity_t->shape()), + errors::InvalidArgument( + "tree_complexity must be a scalar, got a tensor of shape ", + tree_complexity_t->shape().DebugString())); const auto tree_complexity = tree_complexity_t->scalar()(); const Tensor* min_node_weight_t; OP_REQUIRES_OK(context, context->input("min_node_weight", &min_node_weight_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(min_node_weight_t->shape()), + errors::InvalidArgument( + "min_node_weight must be a scalar, got a tensor of shape ", + min_node_weight_t->shape().DebugString())); const auto min_node_weight = min_node_weight_t->scalar()(); // Allocate output lists of tensors: @@ -251,6 +270,9 @@ class BoostedTreesCalculateBestFeatureSplitOp : public OpKernel { explicit BoostedTreesCalculateBestFeatureSplitOp( OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr("logits_dimension", &logits_dim_)); OP_REQUIRES_OK(context, context->GetAttr("split_type", &split_type_)); } @@ -262,6 +284,10 @@ class BoostedTreesCalculateBestFeatureSplitOp : public OpKernel { OP_REQUIRES( context, node_id_range_t->NumElements() == 2, errors::InvalidArgument("node_id_range argument must have shape [2]")); + OP_REQUIRES(context, TensorShapeUtils::IsVector(node_id_range_t->shape()), + errors::InvalidArgument( + "node_id_range must be a vector, received shape ", + node_id_range_t->shape().DebugString())); const auto node_id_range = node_id_range_t->vec(); const int32_t node_id_first = node_id_range(0); // inclusive const int32_t node_id_last = node_id_range(1); // exclusive @@ -291,10 +317,14 @@ class BoostedTreesCalculateBestFeatureSplitOp : public OpKernel { OP_REQUIRES(context, l1_t->NumElements() == 1, errors::InvalidArgument("l1 argument must be a scalar")); const auto l1 = l1_t->scalar()(); - DCHECK_GE(l1, 0); + OP_REQUIRES(context, l1 >= 0, + errors::InvalidArgument("l1 = ", l1, " but it should be >= 0")); if (logits_dim_ > 1) { // Multi-class L1 regularization not supported yet. - DCHECK_EQ(l1, 0); + OP_REQUIRES( + context, l1 == 0, + errors::InvalidArgument( + "l1 != 0 is not yet supported for multi-class regularization")); } const Tensor* l2_t; @@ -302,7 +332,8 @@ class BoostedTreesCalculateBestFeatureSplitOp : public OpKernel { OP_REQUIRES(context, l2_t->NumElements() == 1, errors::InvalidArgument("l2 argument must be a scalar")); const auto l2 = l2_t->scalar()(); - DCHECK_GE(l2, 0); + OP_REQUIRES(context, l2 >= 0, + errors::InvalidArgument("l2 = ", l2, " but it should be >= 0")); const Tensor* tree_complexity_t; OP_REQUIRES_OK(context, @@ -597,6 +628,9 @@ class BoostedTreesCalculateBestFeatureSplitV2 : public OpKernel { explicit BoostedTreesCalculateBestFeatureSplitV2( OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr("logits_dimension", &logits_dim_)); OP_REQUIRES_OK(context, context->GetAttr("num_features", &num_features_)); } @@ -605,6 +639,10 @@ class BoostedTreesCalculateBestFeatureSplitV2 : public OpKernel { // node_id_range const Tensor* node_id_range_t; OP_REQUIRES_OK(context, context->input("node_id_range", &node_id_range_t)); + OP_REQUIRES(context, TensorShapeUtils::IsVector(node_id_range_t->shape()), + errors::InvalidArgument( + "node_id_range must be a vector, received shape ", + node_id_range_t->shape().DebugString())); const auto node_id_range = node_id_range_t->vec(); OP_REQUIRES( context, node_id_range_t->dims() == 1, @@ -625,7 +663,9 @@ class BoostedTreesCalculateBestFeatureSplitV2 : public OpKernel { &stats_summaries_list)); // Infer dimensions of a stats_summary. - DCHECK_GT(stats_summaries_list.size(), 0); + OP_REQUIRES( + context, stats_summaries_list.size() >= 0, + errors::InvalidArgument("Got an empty list for stats_summaries_list")); const int32_t feature_dims = stats_summaries_list[0].dim_size(1); // The last bucket is for default/missing value. const int32_t num_buckets = stats_summaries_list[0].dim_size(2) - 1; @@ -643,7 +683,11 @@ class BoostedTreesCalculateBestFeatureSplitV2 : public OpKernel { // Vector of stats_summaries; each element is stats for feature of shape // [max_splits, feature_dim, num_buckets, logits_dim + hessian_dim]. std::vector::ConstTensor> stats_summaries; - DCHECK_EQ(stats_summaries_list.size(), num_features_); + OP_REQUIRES(context, stats_summaries_list.size() == num_features_, + errors::InvalidArgument( + "Invalid stats_summaries_list size, got ", + stats_summaries_list.size(), + " but expected to match num_features ", num_features_)); stats_summaries.reserve(num_features_); for (const auto& tensor : stats_summaries_list) { stats_summaries.emplace_back(tensor.tensor()); @@ -652,8 +696,15 @@ class BoostedTreesCalculateBestFeatureSplitV2 : public OpKernel { // Split types. const Tensor* split_types_t; OP_REQUIRES_OK(context, context->input("split_types", &split_types_t)); + OP_REQUIRES( + context, TensorShapeUtils::IsVector(split_types_t->shape()), + errors::InvalidArgument("split_types must be a vector, received shape ", + split_types_t->shape().DebugString())); const auto split_types = split_types_t->vec(); - DCHECK_EQ(split_types.size(), num_features_); + OP_REQUIRES(context, split_types.size() == num_features_, + errors::InvalidArgument( + "Invalid split_types size, got ", split_types.size(), + " but expected to match num_features ", num_features_)); // Validate. for (int i = 0; i < num_features_; ++i) { if (!(split_types(i) == kInequalitySplit || @@ -668,29 +719,59 @@ class BoostedTreesCalculateBestFeatureSplitV2 : public OpKernel { const Tensor* candidate_feature_ids_t; OP_REQUIRES_OK(context, context->input("candidate_feature_ids", &candidate_feature_ids_t)); + OP_REQUIRES(context, + TensorShapeUtils::IsVector(candidate_feature_ids_t->shape()), + errors::InvalidArgument( + "candidate_feature_ids must be a vector, received shape ", + candidate_feature_ids_t->shape().DebugString())); const auto candidate_feature_ids = candidate_feature_ids_t->vec(); - DCHECK_EQ(candidate_feature_ids.size(), num_features_); + OP_REQUIRES(context, candidate_feature_ids.size() == num_features_, + errors::InvalidArgument( + "Invalid candidate_feature_ids size, got ", + candidate_feature_ids.size(), + " but expected to match num_features ", num_features_)); // L1, L2, tree_complexity, min_node_weight. const Tensor* l1_t; OP_REQUIRES_OK(context, context->input("l1", &l1_t)); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(l1_t->shape()), + errors::InvalidArgument("l1 must be a scalar, got a tensor of shape ", + l1_t->shape().DebugString())); const auto l1 = l1_t->scalar()(); - DCHECK_GE(l1, 0); + OP_REQUIRES(context, l1 >= 0, + errors::InvalidArgument("l1 = ", l1, " but it should be >= 0")); if (logits_dim_ > 1) { // Multi-class L1 regularization not supported yet. - DCHECK_EQ(l1, 0); + OP_REQUIRES( + context, l1 == 0, + errors::InvalidArgument( + "l1 != 0 is not yet supported for multi-class regularization")); } const Tensor* l2_t; OP_REQUIRES_OK(context, context->input("l2", &l2_t)); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(l2_t->shape()), + errors::InvalidArgument("l2 must be a scalar, got a tensor of shape ", + l2_t->shape().DebugString())); const auto l2 = l2_t->scalar()(); - DCHECK_GE(l2, 0); + OP_REQUIRES(context, l2 >= 0, + errors::InvalidArgument("l2 = ", l2, " but it should be >= 0")); const Tensor* tree_complexity_t; OP_REQUIRES_OK(context, context->input("tree_complexity", &tree_complexity_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(tree_complexity_t->shape()), + errors::InvalidArgument( + "tree_complexity must be a scalar, got a tensor of shape ", + tree_complexity_t->shape().DebugString())); const auto tree_complexity = tree_complexity_t->scalar()(); const Tensor* min_node_weight_t; OP_REQUIRES_OK(context, context->input("min_node_weight", &min_node_weight_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(min_node_weight_t->shape()), + errors::InvalidArgument( + "min_node_weight must be a scalar, got a tensor of shape ", + min_node_weight_t->shape().DebugString())); const auto min_node_weight = min_node_weight_t->scalar()(); std::vector output_node_ids; @@ -1004,6 +1085,9 @@ class BoostedTreesSparseCalculateBestFeatureSplitOp : public OpKernel { explicit BoostedTreesSparseCalculateBestFeatureSplitOp( OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; // TODO(crawles): Using logits_dim_ for multi-class split. OP_REQUIRES_OK(context, context->GetAttr("logits_dimension", &logits_dim_)); // TODO(tanzheny): Using this for equality split. @@ -1014,6 +1098,10 @@ class BoostedTreesSparseCalculateBestFeatureSplitOp : public OpKernel { // node_id_range const Tensor* node_id_range_t; OP_REQUIRES_OK(context, context->input("node_id_range", &node_id_range_t)); + OP_REQUIRES(context, TensorShapeUtils::IsVector(node_id_range_t->shape()), + errors::InvalidArgument( + "node_id_range must be a scalar, got a tensor of shape ", + node_id_range_t->shape().DebugString())); const auto node_id_range = node_id_range_t->vec(); OP_REQUIRES( context, node_id_range.size() == 2, @@ -1025,37 +1113,68 @@ class BoostedTreesSparseCalculateBestFeatureSplitOp : public OpKernel { const Tensor* stats_summary_indices_t; OP_REQUIRES_OK(context, context->input("stats_summary_indices", &stats_summary_indices_t)); + OP_REQUIRES( + context, TensorShapeUtils::IsMatrix(stats_summary_indices_t->shape()), + errors::InvalidArgument( + "stats_summary_indices must be a matrix, got a tensor of shape ", + stats_summary_indices_t->shape().DebugString())); const auto stats_summary_indices = stats_summary_indices_t->matrix(); const int32_t num_sparse_entries = stats_summary_indices_t->dim_size(0); const Tensor* stats_summary_values_t; OP_REQUIRES_OK(context, context->input("stats_summary_values", &stats_summary_values_t)); + OP_REQUIRES( + context, TensorShapeUtils::IsVector(stats_summary_values_t->shape()), + errors::InvalidArgument( + "stats_summary_values must be a vector, got a tensor of shape ", + stats_summary_values_t->shape().DebugString())); const auto stats_summary_values = stats_summary_values_t->vec(); const Tensor* stats_summary_shape_t; OP_REQUIRES_OK( context, context->input("stats_summary_shape", &stats_summary_shape_t)); + OP_REQUIRES( + context, TensorShapeUtils::IsVector(stats_summary_shape_t->shape()), + errors::InvalidArgument( + "stats_summary_shape must be a vector, got a tensor of shape ", + stats_summary_shape_t->shape().DebugString())); const auto stats_summary_shape = stats_summary_shape_t->vec(); const int32_t num_buckets = stats_summary_shape(2) - 1; const int32_t stats_dims = stats_summary_shape(3); const Tensor* l1_t; OP_REQUIRES_OK(context, context->input("l1", &l1_t)); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(l1_t->shape()), + errors::InvalidArgument("l1 must be a scalar, got a tensor of shape ", + l1_t->shape().DebugString())); const auto l1 = l1_t->scalar()(); const Tensor* l2_t; OP_REQUIRES_OK(context, context->input("l2", &l2_t)); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(l2_t->shape()), + errors::InvalidArgument("l2 must be a scalar, got a tensor of shape ", + l2_t->shape().DebugString())); const auto l2 = l2_t->scalar()(); const Tensor* tree_complexity_t; OP_REQUIRES_OK(context, context->input("tree_complexity", &tree_complexity_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(tree_complexity_t->shape()), + errors::InvalidArgument( + "tree_complexity must be a scalar, got a tensor of shape ", + tree_complexity_t->shape().DebugString())); const auto tree_complexity = tree_complexity_t->scalar()(); const Tensor* min_node_weight_t; OP_REQUIRES_OK(context, context->input("min_node_weight", &min_node_weight_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(min_node_weight_t->shape()), + errors::InvalidArgument( + "min_node_weight must be a scalar, got a tensor of shape ", + min_node_weight_t->shape().DebugString())); const auto min_node_weight = min_node_weight_t->scalar()(); std::vector output_node_ids; @@ -1080,8 +1199,10 @@ class BoostedTreesSparseCalculateBestFeatureSplitOp : public OpKernel { f_map.clear(); } previous_node_id = node_id; - DCHECK_LE(node_id_first, node_id); - DCHECK_LT(node_id, node_id_last); + OP_REQUIRES( + context, node_id_first <= node_id && node_id < node_id_last, + errors::InvalidArgument("node_id = ", node_id, " which is not in [", + node_id_first, ", ", node_id_last, ")")); const int32_t feature_dim = stats_summary_indices(idx, 1); const int32_t bucket_id = stats_summary_indices(idx, 2); const int32_t stat_dim = stats_summary_indices(idx, 3); @@ -1315,6 +1436,9 @@ class BoostedTreesMakeStatsSummaryOp : public OpKernel { public: explicit BoostedTreesMakeStatsSummaryOp(OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr("max_splits", &max_splits_)); OP_REQUIRES_OK(context, context->GetAttr("num_buckets", &num_buckets_)); OP_REQUIRES_OK(context, context->GetAttr("num_features", &num_features_)); @@ -1324,10 +1448,18 @@ class BoostedTreesMakeStatsSummaryOp : public OpKernel { // node_ids const Tensor* node_ids_t; OP_REQUIRES_OK(context, context->input("node_ids", &node_ids_t)); + OP_REQUIRES(context, TensorShapeUtils::IsVector(node_ids_t->shape()), + errors::InvalidArgument( + "node_ids must be a vector, got a tensor of shape ", + node_ids_t->shape().DebugString())); const auto node_ids = node_ids_t->vec(); // gradients const Tensor* gradients_t; OP_REQUIRES_OK(context, context->input("gradients", &gradients_t)); + OP_REQUIRES(context, TensorShapeUtils::IsMatrix(gradients_t->shape()), + errors::InvalidArgument( + "gradients must be a matrix, got a tensor of shape ", + gradients_t->shape().DebugString())); const auto gradients = gradients_t->matrix(); OP_REQUIRES( context, node_ids.size() == gradients.dimension(0), @@ -1338,7 +1470,17 @@ class BoostedTreesMakeStatsSummaryOp : public OpKernel { // hessians const Tensor* hessians_t; OP_REQUIRES_OK(context, context->input("hessians", &hessians_t)); + OP_REQUIRES(context, TensorShapeUtils::IsMatrix(hessians_t->shape()), + errors::InvalidArgument( + "hessians must be a matrix, got a tensor of shape ", + hessians_t->shape().DebugString())); const auto hessians = hessians_t->matrix(); + OP_REQUIRES( + context, node_ids.size() == hessians.dimension(0), + errors::InvalidArgument( + "node_ids size should match 0th dim of hessians. node ids " + "size: ", + node_ids.size(), ", hessians dim0: ", hessians.dimension(0))); // bucketized_features OpInputList bucketized_features_list; OP_REQUIRES_OK(context, context->input_list("bucketized_features_list", @@ -1358,6 +1500,11 @@ class BoostedTreesMakeStatsSummaryOp : public OpKernel { // Partition by node, and then bucketize. for (int feature_idx = 0; feature_idx < num_features_; ++feature_idx) { const auto& features = bucketized_features_list[feature_idx].vec(); + OP_REQUIRES( + context, features.size() == node_ids.size(), + errors::InvalidArgument("feature ", feature_idx, + " should have same size as node_ids, got ", + features.size(), " and ", node_ids.size())); for (int i = 0; i < batch_size; ++i) { const int32_t node = node_ids(i); const int32_t bucket = features(i); @@ -1389,6 +1536,9 @@ class BoostedTreesAggregateStatsOp : public OpKernel { public: explicit BoostedTreesAggregateStatsOp(OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr("max_splits", &max_splits_)); OP_REQUIRES_OK(context, context->GetAttr("num_buckets", &num_buckets_)); } @@ -1397,11 +1547,19 @@ class BoostedTreesAggregateStatsOp : public OpKernel { // node_ids. const Tensor* node_ids_t; OP_REQUIRES_OK(context, context->input("node_ids", &node_ids_t)); + OP_REQUIRES(context, TensorShapeUtils::IsVector(node_ids_t->shape()), + errors::InvalidArgument( + "node_ids must be a vector, got a tensor of shape ", + node_ids_t->shape().DebugString())); const auto node_ids = node_ids_t->vec(); // gradients. const Tensor* gradients_t; OP_REQUIRES_OK(context, context->input("gradients", &gradients_t)); + OP_REQUIRES(context, TensorShapeUtils::IsMatrix(gradients_t->shape()), + errors::InvalidArgument( + "gradients must be a matrix, got a tensor of shape ", + gradients_t->shape().DebugString())); const auto gradients = gradients_t->matrix(); OP_REQUIRES( @@ -1414,11 +1572,19 @@ class BoostedTreesAggregateStatsOp : public OpKernel { // hessians. const Tensor* hessians_t; OP_REQUIRES_OK(context, context->input("hessians", &hessians_t)); + OP_REQUIRES(context, TensorShapeUtils::IsMatrix(hessians_t->shape()), + errors::InvalidArgument( + "hessians must be a matrix, got a tensor of shape ", + hessians_t->shape().DebugString())); const auto hessians = hessians_t->matrix(); // feature. const Tensor* feature_t; OP_REQUIRES_OK(context, context->input("feature", &feature_t)); + OP_REQUIRES(context, TensorShapeUtils::IsMatrix(feature_t->shape()), + errors::InvalidArgument( + "feature must be a matrix, got a tensor of shape ", + feature_t->shape().DebugString())); const auto feature = feature_t->matrix(); // Infer batch size, feature dimension and stats dimension. @@ -1572,7 +1738,8 @@ static void AddInstanceStatsToMap( // Add statistics to StatsPartitionMap for bucket_id ranging from // (start_instance, start_feature_dim) to (end_instance, end_feature_dim), // inclusive on start and end instances, exclusive on end feature dim. -static void AddRangeStats(const int start_instance, const int end_instance, +static void AddRangeStats(OpKernelContext* const context, + const int start_instance, const int end_instance, const int start_feature_dim, const int end_feature_dim, StatsPartitionMap* stats_map, @@ -1581,9 +1748,15 @@ static void AddRangeStats(const int start_instance, const int end_instance, const TTypes::ConstVec& node_ids, const int32_t feature_dims, const int32_t bucket_id, const int32_t logits_dims, const int32_t stats_dims) { - DCHECK_LE(start_instance, end_instance); + OP_REQUIRES(context, start_instance <= end_instance, + errors::InvalidArgument( + "start_instance = ", start_instance, + " which is not at most end_instance=", end_instance)); if (start_instance == end_instance) { - DCHECK_LT(start_feature_dim, end_feature_dim); + OP_REQUIRES(context, start_feature_dim < end_feature_dim, + errors::InvalidArgument( + "start_feature_dim = ", start_feature_dim, + " which is not at most end_feature_dim=", end_feature_dim)); } for (int32_t instance = start_instance; instance <= end_instance; ++instance) { @@ -1603,6 +1776,9 @@ class BoostedTreesSparseAggregateStatsOp : public OpKernel { explicit BoostedTreesSparseAggregateStatsOp( OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr("max_splits", &max_splits_)); OP_REQUIRES_OK(context, context->GetAttr("num_buckets", &num_buckets_)); } @@ -1611,29 +1787,71 @@ class BoostedTreesSparseAggregateStatsOp : public OpKernel { // node_ids. const Tensor* node_ids_t; OP_REQUIRES_OK(context, context->input("node_ids", &node_ids_t)); + OP_REQUIRES( + context, TensorShapeUtils::IsVector(node_ids_t->shape()), + errors::InvalidArgument("node_ids must be a vector, received shape ", + node_ids_t->shape().DebugString())); const auto node_ids = node_ids_t->vec(); + const auto num_nodes = node_ids_t->NumElements(); + for (int i = 0; i < num_nodes; ++i) { + OP_REQUIRES( + context, node_ids(i) <= max_splits_, + errors::InvalidArgument( + "Nodes in node_ids must be at most max_splits. Node ", i, " is ", + node_ids(i), " which is greater than ", max_splits_)); + } // gradients. const Tensor* gradients_t; OP_REQUIRES_OK(context, context->input("gradients", &gradients_t)); + OP_REQUIRES( + context, TensorShapeUtils::IsMatrix(gradients_t->shape()), + errors::InvalidArgument("gradients must be a matrix, received shape ", + gradients_t->shape().DebugString())); const auto gradients = gradients_t->matrix(); // hessians. const Tensor* hessians_t; OP_REQUIRES_OK(context, context->input("hessians", &hessians_t)); + OP_REQUIRES( + context, TensorShapeUtils::IsMatrix(hessians_t->shape()), + errors::InvalidArgument("hessians must be a matrix, received shape ", + hessians_t->shape().DebugString())); const auto hessians = hessians_t->matrix(); // feature indices. const Tensor* feature_indices_t; OP_REQUIRES_OK(context, context->input("feature_indices", &feature_indices_t)); + OP_REQUIRES(context, TensorShapeUtils::IsMatrix(feature_indices_t->shape()), + errors::InvalidArgument( + "feature_indices must be a matrix, received shape ", + feature_indices_t->shape().DebugString())); + OP_REQUIRES( + context, feature_indices_t->shape().dim_size(1) == 2, + errors::InvalidArgument( + "feature_indices must be a matrix of shape [?, 2], received shape ", + feature_indices_t->shape().DebugString())); const auto feature_indices = feature_indices_t->matrix(); // feature values. const Tensor* feature_values_t; OP_REQUIRES_OK(context, context->input("feature_values", &feature_values_t)); + OP_REQUIRES(context, TensorShapeUtils::IsVector(feature_values_t->shape()), + errors::InvalidArgument( + "feature_values must be a vector, received shape ", + feature_values_t->shape().DebugString())); const auto feature_values = feature_values_t->vec(); + const auto num_features = feature_values_t->NumElements(); + for (int i = 0; i < num_features; ++i) { + OP_REQUIRES( + context, feature_values(i) <= num_buckets_, + errors::InvalidArgument( + "Features in feature_values must be at most num_buckets. Node ", + i, " is ", feature_values(i), " which is greater than ", + num_buckets_)); + } // feature shape. const Tensor* feature_shape_t; @@ -1650,6 +1868,20 @@ class BoostedTreesSparseAggregateStatsOp : public OpKernel { const int64_t stats_dims = logits_dims + hessians_dims; const int64_t num_sparse_entries = feature_indices_t->dim_size(0); const int32_t feature_dims = feature_shape(1); + OP_REQUIRES(context, num_features == num_sparse_entries, + errors::InvalidArgument( + "Number of elements in feature_values must match number of " + "sparse entries in feature_indices. Got ", + num_features, " and ", num_sparse_entries)); + for (int i = 0; i < num_sparse_entries; ++i) { + const int32_t f_dim = feature_indices(i, 1); + OP_REQUIRES( + context, f_dim <= feature_dims, + errors::InvalidArgument( + "Got invalid feature index feature_indices(", i, "1) = ", f_dim, + " which is above ", feature_dims, + " (from feature_shape: ", feature_shape_t->DebugString(), ")")); + } OP_REQUIRES(context, num_sparse_entries <= batch_size * feature_dims, errors::InvalidArgument( "feature_indices dim0 should be <= gradients dim0 * " @@ -1663,26 +1895,44 @@ class BoostedTreesSparseAggregateStatsOp : public OpKernel { int prev_instance = 0; int prev_f_dim = -1; + if (num_sparse_entries > 0) { + OP_REQUIRES( + context, feature_indices(0, 0) >= 0, + errors::InvalidArgument("feature_indices should be non-negative but " + "got feature_indices(0, 0)=", + feature_indices(0, 0))); + } + for (int i = 0; i < num_sparse_entries; ++i) { // the instance number within a batch const int32_t instance = feature_indices(i, 0); - DCHECK_LE(instance, batch_size); - DCHECK_GE(instance, prev_instance); + OP_REQUIRES(context, instance <= batch_size, + errors::InvalidArgument("feature_indices(", i, + "0) should be at most batch size (", + batch_size, " but got ", instance)); + OP_REQUIRES( + context, instance >= prev_instance, + errors::InvalidArgument( + "feature_indices should be increasing but got feature_indices(", + i, ", 0) < ", prev_instance, " (feature_indices(", i - 1, "0))")); // the node id within a tree. - const int32_t node_id = node_ids(instance); - DCHECK_LE(node_id, max_splits_); + // We don't need the node id here, we just validate that the `instance` + // is a valid index as this is needed later in the code. + OP_REQUIRES(context, instance < num_nodes, + errors::InvalidArgument("feature_indices(", i, + "0) is not a valid index in the " + "node_ids vector (must be less than ", + num_nodes, ", got ", instance, ")")); // the feature dimension. const int32_t f_dim = feature_indices(i, 1); - DCHECK_LE(f_dim, feature_dims); // the bucket id of the value. const int32_t bucket_id = feature_values(i); - DCHECK_LE(bucket_id, num_buckets_); // Add statistics for the missing entries into default bucket. // The last bucket is default bucket. const int missing_entry_bucket = num_buckets_; - AddRangeStats(prev_instance, instance, prev_f_dim, f_dim, &stats_map, - gradients, hessians, node_ids, feature_dims, + AddRangeStats(context, prev_instance, instance, prev_f_dim, f_dim, + &stats_map, gradients, hessians, node_ids, feature_dims, missing_entry_bucket, logits_dims, stats_dims); prev_instance = instance; prev_f_dim = f_dim; @@ -1691,9 +1941,9 @@ class BoostedTreesSparseAggregateStatsOp : public OpKernel { AddInstanceStatsToMap(instance, f_dim, bucket_id, logits_dims, stats_dims, &stats_map, gradients, hessians, node_ids); } - AddRangeStats(prev_instance, batch_size - 1, prev_f_dim, feature_dims, - &stats_map, gradients, hessians, node_ids, feature_dims, - num_buckets_, logits_dims, stats_dims); + AddRangeStats(context, prev_instance, batch_size - 1, prev_f_dim, + feature_dims, &stats_map, gradients, hessians, node_ids, + feature_dims, num_buckets_, logits_dims, stats_dims); // Serialize statistics info map to tensor output. const int64_t num_slots = stats_map.size() * stats_dims; diff --git a/tensorflow/core/kernels/boosted_trees/training_ops.cc b/tensorflow/core/kernels/boosted_trees/training_ops.cc index 3115d1056c3965..8a10741ae5f707 100644 --- a/tensorflow/core/kernels/boosted_trees/training_ops.cc +++ b/tensorflow/core/kernels/boosted_trees/training_ops.cc @@ -35,6 +35,9 @@ class BoostedTreesUpdateEnsembleOp : public OpKernel { public: explicit BoostedTreesUpdateEnsembleOp(OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr("num_features", &num_features_)); int32_t pruning_index; @@ -68,14 +71,26 @@ class BoostedTreesUpdateEnsembleOp : public OpKernel { const Tensor* feature_ids_t; OP_REQUIRES_OK(context, context->input("feature_ids", &feature_ids_t)); + OP_REQUIRES( + context, TensorShapeUtils::IsVector(feature_ids_t->shape()), + errors::InvalidArgument("feature_ids must be a vector, received shape ", + feature_ids_t->shape().DebugString())); const auto feature_ids = feature_ids_t->vec(); const Tensor* max_depth_t; OP_REQUIRES_OK(context, context->input("max_depth", &max_depth_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(max_depth_t->shape()), + errors::InvalidArgument( + "max_depth must be a scalar, got a tensor of shape ", + max_depth_t->shape().DebugString())); const auto max_depth = max_depth_t->scalar()(); const Tensor* learning_rate_t; OP_REQUIRES_OK(context, context->input("learning_rate", &learning_rate_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(learning_rate_t->shape()), + errors::InvalidArgument( + "learning_rate must be a scalar, got a tensor of shape ", + learning_rate_t->shape().DebugString())); const auto learning_rate = learning_rate_t->scalar()(); // Op does not support multi-class, the V2 op below does however. int32_t logits_dimension = 1; @@ -176,11 +191,50 @@ class BoostedTreesUpdateEnsembleOp : public OpKernel { std::map* best_split_per_node) { // Find best split per node going through every feature candidate. for (int64_t feature_idx = 0; feature_idx < num_features_; ++feature_idx) { + OP_REQUIRES( + context, + TensorShapeUtils::IsVector(node_ids_list[feature_idx].shape()), + errors::InvalidArgument( + "Each node_id in node_ids_list must be a vector, received shape ", + node_ids_list[feature_idx].shape().DebugString(), " at index ", + feature_idx)); const auto& node_ids = node_ids_list[feature_idx].vec(); + OP_REQUIRES( + context, TensorShapeUtils::IsVector(gains_list[feature_idx].shape()), + errors::InvalidArgument( + "Each gain in gains_list must be a vector, received shape ", + gains_list[feature_idx].shape().DebugString(), " at index ", + feature_idx)); const auto& gains = gains_list[feature_idx].vec(); + OP_REQUIRES( + context, + TensorShapeUtils::IsVector(thresholds_list[feature_idx].shape()), + errors::InvalidArgument( + "Each threshold in thresholds_list must be a vector, received " + "shape ", + thresholds_list[feature_idx].shape().DebugString(), " at index ", + feature_idx)); const auto& thresholds = thresholds_list[feature_idx].vec(); + OP_REQUIRES( + context, + TensorShapeUtils::IsMatrix( + left_node_contribs_list[feature_idx].shape()), + errors::InvalidArgument( + "Each left_node_contribs in left_node_contribs_list must be a " + "matrix, received shape ", + left_node_contribs_list[feature_idx].shape().DebugString(), + " at index ", feature_idx)); const auto& left_node_contribs = left_node_contribs_list[feature_idx].matrix(); + OP_REQUIRES( + context, + TensorShapeUtils::IsMatrix( + right_node_contribs_list[feature_idx].shape()), + errors::InvalidArgument( + "Each right_node_contribs in right_node_contribs_list must be a " + "matrix, received shape ", + right_node_contribs_list[feature_idx].shape().DebugString(), + " at index ", feature_idx)); const auto& right_node_contribs = right_node_contribs_list[feature_idx].matrix(); @@ -234,6 +288,9 @@ class BoostedTreesUpdateEnsembleV2Op : public OpKernel { public: explicit BoostedTreesUpdateEnsembleV2Op(OpKernelConstruction* const context) : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; OP_REQUIRES_OK(context, context->GetAttr("logits_dimension", &logits_dim_)); OP_REQUIRES_OK(context, context->GetAttr("num_groups", &num_groups_)); } @@ -274,14 +331,26 @@ class BoostedTreesUpdateEnsembleV2Op : public OpKernel { const Tensor* max_depth_t; OP_REQUIRES_OK(context, context->input("max_depth", &max_depth_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(max_depth_t->shape()), + errors::InvalidArgument( + "max_depth must be a scalar, got a tensor of shape ", + max_depth_t->shape().DebugString())); const auto max_depth = max_depth_t->scalar()(); const Tensor* learning_rate_t; OP_REQUIRES_OK(context, context->input("learning_rate", &learning_rate_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(learning_rate_t->shape()), + errors::InvalidArgument( + "learning_rate must be a scalar, got a tensor of shape ", + learning_rate_t->shape().DebugString())); const auto learning_rate = learning_rate_t->scalar()(); const Tensor* pruning_mode_t; OP_REQUIRES_OK(context, context->input("pruning_mode", &pruning_mode_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(pruning_mode_t->shape()), + errors::InvalidArgument( + "pruning_mode must be a scalar, got a tensor of shape ", + pruning_mode_t->shape().DebugString())); const auto pruning_mode = static_cast(pruning_mode_t->scalar()()); // Find best splits for each active node. @@ -327,7 +396,7 @@ class BoostedTreesUpdateEnsembleV2Op : public OpKernel { boosted_trees::SplitTypeWithDefault split_type_with_default; bool parsed = boosted_trees::SplitTypeWithDefault_Parse( split_type, &split_type_with_default); - DCHECK(parsed); + OP_REQUIRES(context, parsed, errors::Internal("Parse failed")); if (split_type_with_default == boosted_trees::EQUALITY_DEFAULT_RIGHT) { // Add equality split to the node. ensemble_resource->AddCategoricalSplitNode(current_tree, split_entry, @@ -396,15 +465,75 @@ class BoostedTreesUpdateEnsembleV2Op : public OpKernel { std::map* best_split_per_node) { // Find best split per node going through every feature candidate. for (int64_t group_idx = 0; group_idx < num_groups_; ++group_idx) { + OP_REQUIRES( + context, TensorShapeUtils::IsVector(node_ids_list[group_idx].shape()), + errors::InvalidArgument( + "Each node_id in node_ids_list must be a vector, received shape ", + node_ids_list[group_idx].shape().DebugString(), " at index ", + group_idx)); const auto& node_ids = node_ids_list[group_idx].vec(); + OP_REQUIRES( + context, TensorShapeUtils::IsVector(gains_list[group_idx].shape()), + errors::InvalidArgument( + "Each gain in gains_list must be a vector, received shape ", + gains_list[group_idx].shape().DebugString(), " at index ", + group_idx)); const auto& gains = gains_list[group_idx].vec(); + OP_REQUIRES( + context, + TensorShapeUtils::IsVector(feature_ids_list[group_idx].shape()), + errors::InvalidArgument( + "Each feature_id in feature_ids_lists must be a vector, received " + "shape ", + feature_ids_list[group_idx].shape().DebugString(), " at index ", + group_idx)); const auto& feature_ids = feature_ids_list[group_idx].vec(); + OP_REQUIRES( + context, + TensorShapeUtils::IsVector(thresholds_list[group_idx].shape()), + errors::InvalidArgument( + "Each threshold in thresholds_list must be a vector, received " + "shape ", + thresholds_list[group_idx].shape().DebugString(), " at index ", + group_idx)); const auto& thresholds = thresholds_list[group_idx].vec(); + OP_REQUIRES( + context, + TensorShapeUtils::IsVector(dimension_ids_list[group_idx].shape()), + errors::InvalidArgument( + "Each dimension_id in dimension_ids_list must be a vector, " + "received shape ", + dimension_ids_list[group_idx].shape().DebugString(), " at index ", + group_idx)); const auto& dimension_ids = dimension_ids_list[group_idx].vec(); + OP_REQUIRES(context, + TensorShapeUtils::IsMatrix( + left_node_contribs_list[group_idx].shape()), + errors::InvalidArgument( + "Each left_node_contribs in right_node_contribs_list " + "must be a matrix, received shape ", + left_node_contribs_list[group_idx].shape().DebugString(), + " at index ", group_idx)); const auto& left_node_contribs = left_node_contribs_list[group_idx].matrix(); + OP_REQUIRES(context, + TensorShapeUtils::IsMatrix( + right_node_contribs_list[group_idx].shape()), + errors::InvalidArgument( + "Each right_node_contribs in right_node_contribs_list " + "must be a matrix, received shape ", + right_node_contribs_list[group_idx].shape().DebugString(), + " at index ", group_idx)); const auto& right_node_contribs = right_node_contribs_list[group_idx].matrix(); + OP_REQUIRES( + context, + TensorShapeUtils::IsVector(split_types_list[group_idx].shape()), + errors::InvalidArgument( + "Each split_type in split_types_list must be a vector, received " + "shape ", + split_types_list[group_idx].shape().DebugString(), " at index ", + group_idx)); const auto& split_types = split_types_list[group_idx].vec(); for (size_t candidate_idx = 0; candidate_idx < node_ids.size(); @@ -457,7 +586,11 @@ REGISTER_KERNEL_BUILDER(Name("BoostedTreesUpdateEnsembleV2").Device(DEVICE_CPU), class BoostedTreesCenterBiasOp : public OpKernel { public: explicit BoostedTreesCenterBiasOp(OpKernelConstruction* const context) - : OpKernel(context) {} + : OpKernel(context) { + VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use " + << "TensorFlow Decision Forests instead " + << "(https://github.com/tensorflow/decision-forests).\n"; + } void Compute(OpKernelContext* const context) override { // Get decision tree ensemble. @@ -479,9 +612,17 @@ class BoostedTreesCenterBiasOp : public OpKernel { // Get the regularization options. const Tensor* l1_t; OP_REQUIRES_OK(context, context->input("l1", &l1_t)); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(l1_t->shape()), + errors::InvalidArgument("l1 must be a scalar, got a tensor of shape ", + l1_t->shape().DebugString())); const auto l1 = l1_t->scalar()(); const Tensor* l2_t; OP_REQUIRES_OK(context, context->input("l2", &l2_t)); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(l2_t->shape()), + errors::InvalidArgument("l2 must be a scalar, got a tensor of shape ", + l2_t->shape().DebugString())); const auto l2 = l2_t->scalar()(); // For now, assume 1-dimensional weight on leaves. @@ -489,7 +630,8 @@ class BoostedTreesCenterBiasOp : public OpKernel { float unused_gain; // TODO(crawles): Support multiclass. - DCHECK_EQ(logits_dim, 1); + OP_REQUIRES(context, logits_dim == 1, + errors::Internal("Expected logits_dim == 1, got ", logits_dim)); Eigen::VectorXf gradients_mean(1); Eigen::VectorXf hessians_mean(1); gradients_mean[0] = mean_gradients_t->flat()(0); @@ -506,7 +648,9 @@ class BoostedTreesCenterBiasOp : public OpKernel { current_bias = logits; } else { const auto& current_biases = ensemble_resource->node_value(0, 0); - DCHECK_EQ(current_biases.size(), 1); + OP_REQUIRES(context, current_biases.size() == 1, + errors::Internal("Expected current_biases.size() == 1, got ", + current_biases.size())); current_bias = current_biases[0]; continue_centering = std::abs(logits / current_bias) > kMinDeltaForCenterBias; diff --git a/tensorflow/core/kernels/collective_ops.cc b/tensorflow/core/kernels/collective_ops.cc index 1824148a401a3c..733e2b7566a6c0 100644 --- a/tensorflow/core/kernels/collective_ops.cc +++ b/tensorflow/core/kernels/collective_ops.cc @@ -176,6 +176,10 @@ class CollectiveGatherOpKernel : public CollectiveOpV1Kernel { void ComputeAsyncImpl(OpKernelContext* c, CollectiveExecutor* col_exec, DoneCallback done) override { auto output_shape = c->input(0).shape(); + OP_REQUIRES_ASYNC(c, output_shape.dims() > 0, + errors::InvalidArgument("input should have rank > 0, ", + "recieved ", output_shape.dims()), + done); output_shape.set_dim( 0, output_shape.dim_size(0) * col_params_->group.group_size); col_params_->instance.shape = output_shape; diff --git a/tensorflow/core/kernels/conv_grad_input_ops.h b/tensorflow/core/kernels/conv_grad_input_ops.h index 0d81bf1ae449ef..5f98cdac8daebf 100644 --- a/tensorflow/core/kernels/conv_grad_input_ops.h +++ b/tensorflow/core/kernels/conv_grad_input_ops.h @@ -37,6 +37,7 @@ limitations under the License. #include "tensorflow/core/kernels/conv_2d.h" #include "tensorflow/core/kernels/conv_grad_ops.h" #include "tensorflow/core/kernels/conv_grad_shape_utils.h" +#include "tensorflow/core/kernels/fill_functor.h" #ifdef TENSORFLOW_USE_LIBXSMM_CONVOLUTIONS #include "tensorflow/core/kernels/xsmm_conv2d.h" #endif @@ -421,6 +422,11 @@ class Conv2DBackpropInputOp : public OpKernel { const Tensor& filter = context->input(1); const Tensor& out_backprop = context->input(2); + OP_REQUIRES( + context, out_backprop.dims() == 4, + errors::InvalidArgument("input_sizes must be 4-dimensional, got: ", + out_backprop.dims())); + TensorShape input_shape; OP_REQUIRES_OK(context, Conv2DBackpropComputeInputShape(input_sizes, filter.shape(), @@ -436,6 +442,15 @@ class Conv2DBackpropInputOp : public OpKernel { return; } + // If shapes are valid but `out_backprop` is empty, in_backprop should be + // set to all zeros. Otherwise, cudnn/dnnl fail with an empty input. + if (out_backprop.NumElements() == 0) { + functor::SetZeroFunctor set_zero; + set_zero(context->eigen_device(), + in_backprop->template flat()); + return; + } + // For now we take the stride from the second and third dimensions only (we // do not support striding on the batch or depth dimension). const int stride_rows = GetTensorDim(strides_, data_format_, 'H'); @@ -517,6 +532,10 @@ class Conv2DCustomBackpropInputOp : public OpKernel { const Tensor& input_sizes = context->input(0); const Tensor& filter = context->input(1); const Tensor& out_backprop = context->input(2); + OP_REQUIRES( + context, out_backprop.dims() == 4, + errors::InvalidArgument("input_sizes must be 4-dimensional, got: ", + out_backprop.dims())); TensorShape input_shape; OP_REQUIRES_OK(context, @@ -551,6 +570,15 @@ class Conv2DCustomBackpropInputOp : public OpKernel { return; } + // If shapes are valid but `out_backprop` is empty, in_backprop should be + // set to all zeros. Otherwise, cudnn/dnnl fail with an empty input. + if (out_backprop.NumElements() == 0) { + functor::SetZeroFunctor set_zero; + set_zero(context->eigen_device(), + in_backprop->template flat()); + return; + } + // TODO(ezhulenev): Remove custom kernel and move XSMM support to // LaunchConv2DBackpropInputOp functor. #if defined TENSORFLOW_USE_LIBXSMM_CONVOLUTIONS && \ diff --git a/tensorflow/core/kernels/conv_grad_ops_3d.cc b/tensorflow/core/kernels/conv_grad_ops_3d.cc index 129abd0daea66d..dd40485041d5bd 100644 --- a/tensorflow/core/kernels/conv_grad_ops_3d.cc +++ b/tensorflow/core/kernels/conv_grad_ops_3d.cc @@ -741,6 +741,10 @@ class Conv3DBackpropFilterOp : public OpKernel { TensorShape filter_shape; if (takes_shape_) { const Tensor& filter_sizes = context->input(1); + OP_REQUIRES(context, TensorShapeUtils::IsVector(filter_sizes.shape()), + errors::InvalidArgument( + "filter_sizes shape must be rank 1 but is rank ", + filter_sizes.shape().dims())); OP_REQUIRES_OK(context, TensorShapeUtils::MakeShape( filter_sizes.vec(), &filter_shape)); } else { @@ -875,6 +879,10 @@ class Conv3DCustomBackpropFilterOp : public OpKernel { TensorShape filter_shape; if (takes_shape_) { const Tensor& filter_sizes = context->input(1); + OP_REQUIRES(context, TensorShapeUtils::IsVector(filter_sizes.shape()), + errors::InvalidArgument( + "filter_sizes shape must be rank 1 but is rank ", + filter_sizes.shape().dims())); OP_REQUIRES_OK(context, TensorShapeUtils::MakeShape( filter_sizes.vec(), &filter_shape)); } else { @@ -1651,6 +1659,10 @@ class Conv3DBackpropFilterOp : public OpKernel { TensorShape filter_shape; if (takes_shape_) { const Tensor& filter_sizes = context->input(1); + OP_REQUIRES(context, TensorShapeUtils::IsVector(filter_sizes.shape()), + errors::InvalidArgument( + "filter_sizes shape must be rank 1 but is rank ", + filter_sizes.shape().dims())); OP_REQUIRES_OK(context, tensor::MakeShape(filter_sizes, &filter_shape)); } else { filter_shape = context->input(1).shape(); diff --git a/tensorflow/core/kernels/conv_ops.cc b/tensorflow/core/kernels/conv_ops.cc index b3c44f4592148f..28c02efe6aef63 100644 --- a/tensorflow/core/kernels/conv_ops.cc +++ b/tensorflow/core/kernels/conv_ops.cc @@ -43,6 +43,7 @@ limitations under the License. #include "tensorflow/core/framework/types.h" #include "tensorflow/core/kernels/conv_2d.h" #include "tensorflow/core/kernels/deep_conv2d.h" +#include "tensorflow/core/kernels/fill_functor.h" #include "tensorflow/core/kernels/ops_util.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/gtl/array_slice.h" @@ -700,6 +701,15 @@ class Conv2DOp : public BinaryOp { return; } + // If the input is empty, result can only be due to padding. + if (input.NumElements() == 0) { + // Zero-out output and return. + functor::SetZeroFunctor()(context->eigen_device(), + output->template flat()); + + return; + } + #ifdef TENSORFLOW_USE_LIBXSMM_CONVOLUTIONS if (params_.padding != EXPLICIT && LaunchXsmmConvOp::Run( diff --git a/tensorflow/core/kernels/count_ops.cc b/tensorflow/core/kernels/count_ops.cc index 5330c36361e5e6..3f9df8f0d69d69 100644 --- a/tensorflow/core/kernels/count_ops.cc +++ b/tensorflow/core/kernels/count_ops.cc @@ -13,6 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include + #include "absl/container/flat_hash_map.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/op_requires.h" @@ -23,6 +25,9 @@ limitations under the License. namespace tensorflow { +// Don't allocate too large `BatchedMap` objects +static int kMaxBatches = std::numeric_limits::max(); + template using BatchedMap = std::vector>; @@ -185,6 +190,44 @@ class SparseCount : public OpKernel { errors::InvalidArgument( "Input indices must be a 2-dimensional tensor. Got: ", indices.shape().DebugString())); + OP_REQUIRES(context, TensorShapeUtils::IsVector(values.shape()), + errors::InvalidArgument("Input values must be a vector. Got: ", + values.shape().DebugString())); + OP_REQUIRES(context, TensorShapeUtils::IsVector(shape.shape()), + errors::InvalidArgument("Input shape must be a vector. Got: ", + shape.shape().DebugString())); + OP_REQUIRES(context, + values.shape().dim_size(0) == indices.shape().dim_size(0), + errors::InvalidArgument( + "Number of values must match first dimension of indices.", + "Got ", values.shape().dim_size(0), + " values, indices shape: ", indices.shape().DebugString())); + OP_REQUIRES( + context, shape.shape().dim_size(0) == indices.shape().dim_size(1), + errors::InvalidArgument( + "Number of dimensions must match second dimension of indices.", + "Got ", shape.shape().dim_size(0), + " dimensions, indices shape: ", indices.shape().DebugString())); + OP_REQUIRES(context, shape.NumElements() > 0, + errors::InvalidArgument( + "The shape argument requires at least one element.")); + // Validate indices: each index must be valid for the corresponding + // dimension. This could be possibly done better. + const auto indices_values = indices.matrix(); + const auto shape_vector = shape.vec(); + int num_values = values.NumElements(); // same as first dim of indices + int rank = indices.shape().dim_size(1); + for (int i = 0; i < num_values; ++i) { + for (int j = 0; j < rank; ++j) { + OP_REQUIRES( + context, + indices_values(i, j) >= 0 && indices_values(i, j) < shape_vector(j), + errors::InvalidArgument( + "Invalid index value at ", i, ": dimension ", j, " has value ", + indices_values(i, j), " which is not in [0, ", shape_vector(j), + ") (as given by dense shape ", shape.DebugString())); + } + } if (use_weights) { OP_REQUIRES( @@ -195,29 +238,13 @@ class SparseCount : public OpKernel { "; values shape: ", values.shape().DebugString())); } - OP_REQUIRES(context, shape.NumElements() != 0, - errors::InvalidArgument( - "The shape argument requires at least one element.")); - bool is_1d = shape.NumElements() == 1; - auto shape_vector = shape.flat(); int num_batches = is_1d ? 1 : shape_vector(0); - int num_values = values.NumElements(); - - for (int b = 0; b < shape_vector.size(); b++) { - OP_REQUIRES(context, shape_vector(b) >= 0, - errors::InvalidArgument( - "Elements in dense_shape must be >= 0. Instead got:", - shape.DebugString())); - } - - OP_REQUIRES(context, num_values == indices.shape().dim_size(0), - errors::InvalidArgument( - "Number of values must match first dimension of indices.", - "Got ", num_values, - " values, indices shape: ", indices.shape().DebugString())); + OP_REQUIRES( + context, 0 < num_batches && num_batches < kMaxBatches, + errors::InvalidArgument("Cannot allocate ", num_batches, + " batches, is the dense shape too wide?")); - const auto indices_values = indices.matrix(); const auto values_values = values.flat(); const auto weight_values = weights.flat(); @@ -225,16 +252,6 @@ class SparseCount : public OpKernel { T max_value = 0; - OP_REQUIRES(context, num_values <= indices.shape().dim_size(0), - errors::InvalidArgument( - "The first dimension of indices must be equal to or " - "greather than number of values. ( ", - indices.shape().dim_size(0), " vs. ", num_values, " )")); - OP_REQUIRES(context, indices.shape().dim_size(1) > 0, - errors::InvalidArgument("The second dimension of indices must " - "be greater than 0. Received: ", - indices.shape().dim_size(1))); - for (int idx = 0; idx < num_values; ++idx) { int batch = is_1d ? 0 : indices_values(idx, 0); if (batch >= num_batches) { diff --git a/tensorflow/core/kernels/cwise_ops_common.h b/tensorflow/core/kernels/cwise_ops_common.h index ed4125f45579f3..fa48717ab1c285 100644 --- a/tensorflow/core/kernels/cwise_ops_common.h +++ b/tensorflow/core/kernels/cwise_ops_common.h @@ -87,7 +87,17 @@ class BinaryOp : public BinaryOpShared { void Compute(OpKernelContext* ctx) override { const Tensor& input_0 = ctx->input(0); + OP_REQUIRES(ctx, input_0.dtype() == DataTypeToEnum::v(), + errors::InvalidArgument( + "Expected tensor of type ", + DataTypeString(DataTypeToEnum::v()), " but got type ", + DataTypeString(input_0.dtype()))); const Tensor& input_1 = ctx->input(1); + OP_REQUIRES(ctx, input_1.dtype() == DataTypeToEnum::v(), + errors::InvalidArgument( + "Expected tensor of type ", + DataTypeString(DataTypeToEnum::v()), " but got type ", + DataTypeString(input_1.dtype()))); const Device& eigen_device = ctx->eigen_device(); bool error = false; bool* const error_ptr = Functor::has_errors ? &error : nullptr; diff --git a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc index fae97f270f455e..365376b6c2eef0 100644 --- a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc +++ b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc @@ -39,6 +39,22 @@ namespace experimental { PrivateThreadPoolDatasetOp::kDatasetType; /* static */ constexpr const char* const PrivateThreadPoolDatasetOp::kDatasetOp; +namespace { +// To prevent integer overflow issues when allocating threadpool memory for an +// unreasonable number of threads. +constexpr int kThreadLimit = 65536; + +Status ValidateNumThreads(int32_t num_threads) { + if (num_threads < 0) { + return errors::InvalidArgument("`num_threads` must be >= 0"); + } + if (num_threads >= kThreadLimit) { + return errors::InvalidArgument("`num_threads` must be < ", kThreadLimit); + } + return Status::OK(); +} +} // namespace + class ThreadPoolResource : public ResourceBase { public: ThreadPoolResource(Env* env, const ThreadOptions& thread_options, @@ -83,9 +99,7 @@ class ThreadPoolHandleOp : public OpKernel { OP_REQUIRES_OK(ctx, ctx->GetAttr("num_threads", &num_threads_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("max_intra_op_parallelism", &max_intra_op_parallelism_)); - OP_REQUIRES( - ctx, num_threads_ > 0, - errors::InvalidArgument("`num_threads` must be greater than zero.")); + OP_REQUIRES_OK(ctx, ValidateNumThreads(num_threads_)); } // The resource is deleted from the resource manager only when it is private @@ -529,8 +543,7 @@ void PrivateThreadPoolDatasetOp::MakeDatasetFromOptions(OpKernelContext* ctx, DatasetBase* input, int32_t num_threads, DatasetBase** output) { - OP_REQUIRES(ctx, num_threads >= 0, - errors::InvalidArgument("`num_threads` must be >= 0")); + OP_REQUIRES_OK(ctx, ValidateNumThreads(num_threads)); *output = new Dataset(ctx, DatasetContext(DatasetContext::Params( {PrivateThreadPoolDatasetOp::kDatasetType, @@ -544,8 +557,7 @@ void PrivateThreadPoolDatasetOp::MakeDataset(OpKernelContext* ctx, int64_t num_threads = 0; OP_REQUIRES_OK( ctx, ParseScalarArgument(ctx, "num_threads", &num_threads)); - OP_REQUIRES(ctx, num_threads >= 0, - errors::InvalidArgument("`num_threads` must be >= 0")); + OP_REQUIRES_OK(ctx, ValidateNumThreads(num_threads)); *output = new Dataset(ctx, input, num_threads); } diff --git a/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc b/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc index 634e667046afb0..c07b237a70a4c1 100644 --- a/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc +++ b/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc @@ -238,28 +238,29 @@ class SparseTensorSliceDatasetOp : public DatasetOpKernel { OP_REQUIRES_OK(ctx, ctx->input("dense_shape", &dense_shape)); OP_REQUIRES(ctx, TensorShapeUtils::IsMatrix(indices->shape()), - errors::InvalidArgument( - "Input indices should be a matrix but received shape ", - indices->shape().DebugString())); - - const auto num_indices = indices->NumElements(); - const auto num_values = values->NumElements(); - if (num_indices == 0 || num_values == 0) { - OP_REQUIRES(ctx, num_indices == num_values, - errors::InvalidArgument( - "If indices or values are empty, the other one must also " - "be. Got indices of shape ", - indices->shape().DebugString(), " and values of shape ", - values->shape().DebugString())); - } + errors::InvalidArgument("Input indices must be a matrix. Got: ", + indices->shape().DebugString())); OP_REQUIRES(ctx, TensorShapeUtils::IsVector(values->shape()), - errors::InvalidArgument( - "Input values should be a vector but received shape ", - indices->shape().DebugString())); + errors::InvalidArgument("Input values must be a vector. Got: ", + values->shape().DebugString())); OP_REQUIRES(ctx, TensorShapeUtils::IsVector(dense_shape->shape()), + errors::InvalidArgument("Input shape must be a vector. Got: ", + dense_shape->shape().DebugString())); + OP_REQUIRES( + ctx, values->shape().dim_size(0) == indices->shape().dim_size(0), + errors::InvalidArgument( + "Number of values must match first dimension of indices. ", "Got ", + values->shape().dim_size(0), + " values, indices shape: ", indices->shape().DebugString())); + OP_REQUIRES( + ctx, dense_shape->shape().dim_size(0) == indices->shape().dim_size(1), + errors::InvalidArgument( + "Number of dimensions must match second dimension of indices. ", + "Got ", dense_shape->shape().dim_size(0), + " dimensions, indices shape: ", indices->shape().DebugString())); + OP_REQUIRES(ctx, dense_shape->NumElements() > 0, errors::InvalidArgument( - "Input shape should be a vector but received shape ", - dense_shape->shape().DebugString())); + "The shape argument requires at least one element.")); // We currently ensure that `sparse_tensor` is ordered in the // batch dimension. @@ -278,11 +279,12 @@ class SparseTensorSliceDatasetOp : public DatasetOpKernel { previous_batch_index = next_batch_index; } gtl::InlinedVector std_order(dense_shape->NumElements(), 0); + TensorShape shape; + OP_REQUIRES_OK(ctx, TensorShape::BuildTensorShape( + dense_shape->vec(), &shape)); sparse::SparseTensor tensor; - OP_REQUIRES_OK( - ctx, sparse::SparseTensor::Create( - *indices, *values, TensorShape(dense_shape->vec()), - std_order, &tensor)); + OP_REQUIRES_OK(ctx, sparse::SparseTensor::Create(*indices, *values, shape, + std_order, &tensor)); *output = new Dataset(ctx, std::move(tensor)); } diff --git a/tensorflow/core/kernels/depthwise_conv_grad_op.cc b/tensorflow/core/kernels/depthwise_conv_grad_op.cc index 22d338b778a98e..a0efc1a6e041de 100644 --- a/tensorflow/core/kernels/depthwise_conv_grad_op.cc +++ b/tensorflow/core/kernels/depthwise_conv_grad_op.cc @@ -623,7 +623,7 @@ class DepthwiseConv2dNativeBackpropInputOp : public OpKernel { OP_REQUIRES(context, in_sizes_data[i] >= 0, errors::InvalidArgument("Dimension ", i, " of input_sizes must be >= 0")); - input_shape.AddDim(in_sizes_data[i]); + OP_REQUIRES_OK(context, input_shape.AddDimWithStatus(in_sizes_data[i])); } const TensorShape& filter_shape = filter.shape(); EXTRACT_AND_VERIFY_DIMENSIONS("DepthwiseConv2DBackpropInput"); @@ -1120,7 +1120,8 @@ class DepthwiseConv2dNativeBackpropFilterOp : public OpKernel { OP_REQUIRES(context, filter_sizes_data[i] >= 0, errors::InvalidArgument("Dimension ", i, " of filter_sizes must be >= 0")); - filter_shape.AddDim(filter_sizes_data[i]); + OP_REQUIRES_OK(context, + filter_shape.AddDimWithStatus(filter_sizes_data[i])); } const TensorShape& input_shape = input.shape(); diff --git a/tensorflow/core/kernels/dequantize_op.cc b/tensorflow/core/kernels/dequantize_op.cc index 31a7a7498cf0c8..0a466f1cf118e2 100644 --- a/tensorflow/core/kernels/dequantize_op.cc +++ b/tensorflow/core/kernels/dequantize_op.cc @@ -94,6 +94,11 @@ class DequantizeOp : public OpKernel { const Tensor& input_min_tensor = ctx->input(1); const Tensor& input_max_tensor = ctx->input(2); + OP_REQUIRES( + ctx, axis_ < input.dims(), + errors::InvalidArgument("Axis must be less than input dimension(", + input.dims(), "), got ", axis_)); + int num_slices = 1; if (axis_ > -1) { num_slices = input.dim_size(axis_); diff --git a/tensorflow/core/kernels/edit_distance_op.cc b/tensorflow/core/kernels/edit_distance_op.cc index 3ff290e92b6103..3ed0f012b83ceb 100644 --- a/tensorflow/core/kernels/edit_distance_op.cc +++ b/tensorflow/core/kernels/edit_distance_op.cc @@ -203,9 +203,9 @@ class EditDistanceOp : public OpKernel { auto loc = std::inner_product(g_truth.begin(), g_truth.end(), output_strides.begin(), int64_t{0}); OP_REQUIRES( - ctx, loc < output_elements, + ctx, 0 <= loc && loc < output_elements, errors::Internal("Got an inner product ", loc, - " which would require in writing to outside of " + " which would require writing to outside of " "the buffer for the output tensor (max elements ", output_elements, ")")); output_t(loc) = @@ -218,9 +218,9 @@ class EditDistanceOp : public OpKernel { auto loc = std::inner_product(g_hypothesis.begin(), g_hypothesis.end(), output_strides.begin(), int64_t{0}); OP_REQUIRES( - ctx, loc < output_elements, + ctx, 0 <= loc && loc < output_elements, errors::Internal("Got an inner product ", loc, - " which would require in writing to outside of " + " which would require writing to outside of " "the buffer for the output tensor (max elements ", output_elements, ")")); output_t(loc) = hypothesis_seq.size(); @@ -232,9 +232,9 @@ class EditDistanceOp : public OpKernel { auto loc = std::inner_product(g_truth.begin(), g_truth.end(), output_strides.begin(), int64_t{0}); OP_REQUIRES( - ctx, loc < output_elements, + ctx, 0 <= loc && loc < output_elements, errors::Internal("Got an inner product ", loc, - " which would require in writing to outside of " + " which would require writing to outside of " "the buffer for the output tensor (max elements ", output_elements, ")")); output_t(loc) = (normalize_) ? 1.0 : truth_seq.size(); @@ -248,9 +248,9 @@ class EditDistanceOp : public OpKernel { auto loc = std::inner_product(g_hypothesis.begin(), g_hypothesis.end(), output_strides.begin(), int64_t{0}); OP_REQUIRES( - ctx, loc < output_elements, + ctx, 0 <= loc && loc < output_elements, errors::Internal("Got an inner product ", loc, - " which would require in writing to outside of the " + " which would require writing to outside of the " "buffer for the output tensor (max elements ", output_elements, ")")); output_t(loc) = hypothesis_seq.size(); @@ -266,9 +266,9 @@ class EditDistanceOp : public OpKernel { auto loc = std::inner_product(g_truth.begin(), g_truth.end(), output_strides.begin(), int64_t{0}); OP_REQUIRES( - ctx, loc < output_elements, + ctx, 0 <= loc && loc < output_elements, errors::Internal("Got an inner product ", loc, - " which would require in writing to outside of the " + " which would require writing to outside of the " "buffer for the output tensor (max elements ", output_elements, ")")); output_t(loc) = (normalize_) ? 1.0 : truth_seq.size(); diff --git a/tensorflow/core/kernels/fake_quant_ops.cc b/tensorflow/core/kernels/fake_quant_ops.cc index f08575c0e99b2c..7ada39ebec5b5f 100644 --- a/tensorflow/core/kernels/fake_quant_ops.cc +++ b/tensorflow/core/kernels/fake_quant_ops.cc @@ -24,6 +24,7 @@ limitations under the License. // Above is the related header but clang tidy doesn't recognize it. #include "tensorflow/core/framework/numeric_op.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/monitoring/gauge.h" #include "tensorflow/core/platform/protobuf.h" @@ -204,6 +205,13 @@ class FakeQuantWithMinMaxVarsOp : public OpKernel { const Tensor& min = context->input(1); const Tensor& max = context->input(2); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(min.shape()), + InvalidArgument("`min` must be rank 0 but is rank ", min.dims())); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(max.shape()), + InvalidArgument("`max` must be rank 0 but is rank ", max.dims())); + Tensor* output; OP_REQUIRES_OK(context, context->allocate_output(0, input.shape(), &output)); @@ -245,6 +253,12 @@ class FakeQuantWithMinMaxVarsGradientOp : public OpKernel { InvalidArgument("gradient and input must be the same size")); const Tensor& min = context->input(2); const Tensor& max = context->input(3); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(min.shape()), + InvalidArgument("`min` must be rank 0 but is rank ", min.dims())); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(max.shape()), + InvalidArgument("`max` must be rank 0 but is rank ", max.dims())); Tensor* grad_wrt_input; OP_REQUIRES_OK(context, @@ -334,10 +348,17 @@ class FakeQuantWithMinMaxVarsPerChannelOp : public OpKernel { const Tensor& input = context->input(0); const int depth = input.dim_size(input.dims() - 1); // last dimension size. const Tensor& min = context->input(1); + const Tensor& max = context->input(2); + + OP_REQUIRES( + context, TensorShapeUtils::IsVector(min.shape()), + InvalidArgument("`min` must be rank 1 but is rank ", min.dims())); OP_REQUIRES(context, min.dim_size(0) == depth, InvalidArgument("min has incorrect size, expected ", depth, " was ", min.dim_size(0))); - const Tensor& max = context->input(2); + OP_REQUIRES( + context, TensorShapeUtils::IsVector(max.shape()), + InvalidArgument("`max` must be rank 1 but is rank ", max.dims())); OP_REQUIRES(context, max.dim_size(0) == depth, InvalidArgument("max has incorrect size, expected ", depth, " was ", max.dim_size(0))); @@ -384,10 +405,16 @@ class FakeQuantWithMinMaxVarsPerChannelGradientOp : public OpKernel { InvalidArgument("gradient and input must be the same size")); const int depth = input.dim_size(input.dims() - 1); // last dimension size. const Tensor& min = context->input(2); + OP_REQUIRES( + context, TensorShapeUtils::IsVector(min.shape()), + InvalidArgument("`min` must be rank 1 but is rank ", min.dims())); OP_REQUIRES(context, min.dim_size(0) == depth, InvalidArgument("min has incorrect size, expected ", depth, " was ", min.dim_size(0))); const Tensor& max = context->input(3); + OP_REQUIRES( + context, TensorShapeUtils::IsVector(max.shape()), + InvalidArgument("`max` must be rank 1 but is rank ", max.dims())); OP_REQUIRES(context, max.dim_size(0) == depth, InvalidArgument("max has incorrect size, expected ", depth, " was ", max.dim_size(0))); diff --git a/tensorflow/core/kernels/fft_ops.cc b/tensorflow/core/kernels/fft_ops.cc index 14d0ebd983142f..3186863884b1ce 100644 --- a/tensorflow/core/kernels/fft_ops.cc +++ b/tensorflow/core/kernels/fft_ops.cc @@ -66,6 +66,10 @@ class FFTBase : public OpKernel { auto fft_length_as_vec = fft_length.vec(); for (int i = 0; i < fft_rank; ++i) { + OP_REQUIRES(ctx, fft_length_as_vec(i) >= 0, + errors::InvalidArgument( + "fft_length[", i, + "] must >= 0, but got: ", fft_length_as_vec(i))); fft_shape[i] = fft_length_as_vec(i); // Each input dimension must have length of at least fft_shape[i]. For // IRFFTs, the inner-most input dimension must have length of at least diff --git a/tensorflow/core/kernels/fractional_avg_pool_op.cc b/tensorflow/core/kernels/fractional_avg_pool_op.cc index afc72287edadc1..e1c536af7065de 100644 --- a/tensorflow/core/kernels/fractional_avg_pool_op.cc +++ b/tensorflow/core/kernels/fractional_avg_pool_op.cc @@ -12,6 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ + #define EIGEN_USE_THREADS #include @@ -19,15 +20,15 @@ limitations under the License. #include #include -#include "tensorflow/core/kernels/fractional_pool_common.h" - #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/numeric_op.h" #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/kernels/fractional_pool_common.h" #include "tensorflow/core/lib/random/random.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/util/guarded_philox_random.h" +#include "tensorflow/core/util/overflow.h" namespace tensorflow { typedef Eigen::ThreadPoolDevice CPUDevice; @@ -241,7 +242,32 @@ class FractionalAvgPoolGradOp : public OpKernel { orig_input_tensor_shape.NumElements() == 4, errors::InvalidArgument("original input tensor shape must be" "1-dimensional and 4 elements")); + int64_t num_elements = 1; + for (int i = 0; i < orig_input_tensor_shape.dims(); i++) { + OP_REQUIRES(context, orig_input_tensor_shape.dim_size(i) > 0, + errors::InvalidArgument( + "orig_input_tensor_shape must be positive, got: ", + orig_input_tensor_shape.dim_size(i))); + num_elements = MultiplyWithoutOverflow( + num_elements, orig_input_tensor_shape.dim_size(i)); + OP_REQUIRES( + context, num_elements > 0, + errors::InvalidArgument( + "The total elements specified by orig_input_tensor_shape", + " is too large. Encountered overflow after multiplying ", + orig_input_tensor_shape.dim_size(i), ", result: ", num_elements)); + } + const Tensor& out_backprop = context->input(1); + OP_REQUIRES(context, out_backprop.dims() == 4, + errors::InvalidArgument("out_backprop must be 4-dimensional")); + for (int i = 0; i < out_backprop.dims(); i++) { + OP_REQUIRES(context, out_backprop.dim_size(i) > 0, + errors::InvalidArgument( + "out_backprop must be positive for all dimension, got:", + out_backprop.dim_size(i))); + } + const Tensor& row_seq_tensor = context->input(2); const Tensor& col_seq_tensor = context->input(3); @@ -311,15 +337,26 @@ class FractionalAvgPoolGradOp : public OpKernel { for (int64_t b = 0; b < out_batch; ++b) { for (int64_t r = 0; r < out_rows; ++r) { const int64_t in_row_start = row_seq_tensor_flat(r); + int64_t in_row_end = overlapping_ ? row_seq_tensor_flat(r + 1) : row_seq_tensor_flat(r + 1) - 1; in_row_end = std::min(in_row_end, in_max_row_index); + OP_REQUIRES(context, in_row_start >= 0 && in_row_end >= 0, + errors::InvalidArgument( + "Row sequence tensor values must not be negative, got ", + row_seq_tensor_flat)); + for (int64_t c = 0; c < out_cols; ++c) { const int64_t in_col_start = col_seq_tensor_flat(c); int64_t in_col_end = overlapping_ ? col_seq_tensor_flat(c + 1) : col_seq_tensor_flat(c + 1) - 1; in_col_end = std::min(in_col_end, in_max_col_index); + OP_REQUIRES( + context, in_col_start >= 0 && in_col_end >= 0, + errors::InvalidArgument( + "Column sequence tensor values must not be negative, got ", + col_seq_tensor_flat)); const int64_t num_elements_in_pooling_cell = (in_row_end - in_row_start + 1) * (in_col_end - in_col_start + 1); const int64_t out_index = (b * out_rows + r) * out_cols + c; diff --git a/tensorflow/core/kernels/fractional_max_pool_op.cc b/tensorflow/core/kernels/fractional_max_pool_op.cc index 7519c84667409f..375786615ebe69 100644 --- a/tensorflow/core/kernels/fractional_max_pool_op.cc +++ b/tensorflow/core/kernels/fractional_max_pool_op.cc @@ -19,12 +19,13 @@ limitations under the License. #include #include -#include "tensorflow/core/kernels/fractional_pool_common.h" - #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/numeric_op.h" #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/op_requires.h" +#include "tensorflow/core/kernels/fractional_pool_common.h" #include "tensorflow/core/lib/random/random.h" +#include "tensorflow/core/platform/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/util/guarded_philox_random.h" @@ -83,6 +84,13 @@ class FractionalMaxPoolOp : public OpKernel { std::vector output_size(tensor_in_and_out_dims); for (int i = 0; i < tensor_in_and_out_dims; ++i) { input_size[i] = tensor_in.dim_size(i); + + OP_REQUIRES( + context, input_size[i] >= pooling_ratio_[i], + errors::InvalidArgument("Pooling ratio is higher than input " + "dimension size for dimension ", + i, ". Input dim size: ", input_size[i], + " pooling ratio: ", pooling_ratio_[i])); } // Output size. for (int i = 0; i < tensor_in_and_out_dims; ++i) { @@ -345,7 +353,9 @@ class FractionalMaxPoolGradOp : public OpKernel { output_size[2] * output_size[1] * output_size[0]; for (int64_t i = 0; i < num_reshaped_cols; ++i) { for (int64_t j = 0; j < output_size[3]; ++j) { - DCHECK_EQ(tensor_out_dup_mat(j, i), tensor_out_mat(j, i)); + OP_REQUIRES(context, tensor_out_dup_mat(j, i) == tensor_out_mat(j, i), + errors::InvalidArgument( + "tensor_out_dup is not the same as tensor_out")); } } @@ -362,11 +372,12 @@ class FractionalMaxPoolGradOp : public OpKernel { for (int index = 0; index < num_total_outputs; ++index) { int input_backprop_index = out_arg_max_flat(index); - // According to maxpooling_op.cc, the performance impact below is small. - CHECK(input_backprop_index >= 0 && - input_backprop_index < num_total_inputs) - << "Invalid input backprop index: " << input_backprop_index << ", " - << num_total_inputs; + OP_REQUIRES( + context, + input_backprop_index >= 0 && input_backprop_index < num_total_inputs, + errors::InvalidArgument( + "Invalid input backprop index: ", input_backprop_index, ", ", + num_total_inputs)); input_backprop_flat(input_backprop_index) += out_backprop_flat(index); } } diff --git a/tensorflow/core/kernels/histogram_op.cc b/tensorflow/core/kernels/histogram_op.cc index 8e1b53de6ee409..8ba9bccc9fbecd 100644 --- a/tensorflow/core/kernels/histogram_op.cc +++ b/tensorflow/core/kernels/histogram_op.cc @@ -50,6 +50,15 @@ struct HistogramFixedWidthFunctor { static_cast(nbins); const double nbins_minus_1 = static_cast(nbins - 1); + // We cannot handle NANs in the algorithm below (due to the case to int32) + const Eigen::Tensor nans_tensor = + values.isnan().template cast(); + const Eigen::Tensor reduced_tensor = nans_tensor.sum(); + const int num_nans = reduced_tensor(0); + if (num_nans > 0) { + return errors::InvalidArgument("Histogram values must not contain NaN"); + } + // The calculation is done by finding the slot of each value in `values`. // With [a, b]: // step = (b - a) / nbins @@ -98,12 +107,12 @@ class HistogramFixedWidthOp : public OpKernel { const auto nbins = nbins_tensor.scalar()(); OP_REQUIRES( - ctx, (value_range(0) < value_range(1)), + ctx, value_range(0) < value_range(1), errors::InvalidArgument("value_range should satisfy value_range[0] < " "value_range[1], but got '[", value_range(0), ", ", value_range(1), "]'")); OP_REQUIRES( - ctx, (nbins > 0), + ctx, nbins > 0, errors::InvalidArgument("nbins should be a positive number, but got '", nbins, "'")); diff --git a/tensorflow/core/kernels/image/decode_image_op.cc b/tensorflow/core/kernels/image/decode_image_op.cc index 8006f52171feda..d797a49725ac49 100644 --- a/tensorflow/core/kernels/image/decode_image_op.cc +++ b/tensorflow/core/kernels/image/decode_image_op.cc @@ -18,6 +18,8 @@ limitations under the License. #include #include +#include "tensorflow/core/lib/gtl/cleanup.h" + #define EIGEN_USE_THREADS #include "absl/strings/escaping.h" @@ -326,6 +328,16 @@ class DecodeImageV2Op : public OpKernel { context, png::CommonInitDecode(input, channels_, channel_bits, &decode), errors::InvalidArgument("Invalid PNG. Failed to initialize decoder.")); + // If we reach this point, then there is data in `decode` which must be + // freed by the time we end execution in this function. We cannot call + // `png::CommonFreeDecode()` before an `OP_REQUIRES` because if + // `OP_REQUIRES` constraint is satisfied then the data would be freed + // prematurely. Instead, let's use a `Cleanup` object. + auto cleanup = gtl::MakeCleanup([&decode]() { + std::cerr << "Cleanup called...\n"; + png::CommonFreeDecode(&decode); + }); + // Verify that width and height are not too large: // - verify width and height don't overflow int. // - width can later be multiplied by channels_ and sizeof(uint16), so @@ -339,22 +351,24 @@ class DecodeImageV2Op : public OpKernel { if (width != static_cast(decode.width) || width <= 0 || width >= (1LL << 27) || height != static_cast(decode.height) || height <= 0 || height >= (1LL << 27) || total_size >= (1LL << 29)) { - png::CommonFreeDecode(&decode); OP_REQUIRES(context, false, errors::InvalidArgument("PNG size too large for int: ", decode.width, " by ", decode.height)); } Tensor* output = nullptr; - Status status; // By the existing API, we support decoding PNG with `DecodeGif` op. // We need to make sure to return 4-D shapes when using `DecodeGif`. if (op_type_ == "DecodeGif") { - status = context->allocate_output( - 0, TensorShape({1, height, width, decode.channels}), &output); + OP_REQUIRES_OK( + context, + context->allocate_output( + 0, TensorShape({1, height, width, decode.channels}), &output)); } else { - status = context->allocate_output( - 0, TensorShape({height, width, decode.channels}), &output); + OP_REQUIRES_OK( + context, + context->allocate_output( + 0, TensorShape({height, width, decode.channels}), &output)); } if (op_type_ == "DecodeBmp") { @@ -374,9 +388,6 @@ class DecodeImageV2Op : public OpKernel { "detected PNG.")); } - if (!status.ok()) png::CommonFreeDecode(&decode); - OP_REQUIRES_OK(context, status); - if (data_type_ == DataType::DT_UINT8) { OP_REQUIRES( context, diff --git a/tensorflow/core/kernels/image/draw_bounding_box_op.cc b/tensorflow/core/kernels/image/draw_bounding_box_op.cc index bc8452d6164cbb..eff76f0811df81 100644 --- a/tensorflow/core/kernels/image/draw_bounding_box_op.cc +++ b/tensorflow/core/kernels/image/draw_bounding_box_op.cc @@ -119,7 +119,7 @@ class DrawBoundingBoxesOp : public OpKernel { for (int64_t b = 0; b < batch_size; ++b) { const int64_t num_boxes = boxes.dim_size(1); - const auto tboxes = boxes.tensor(); + const auto tboxes = boxes.tensor(); for (int64_t bb = 0; bb < num_boxes; ++bb) { int64_t color_index = bb % color_table.size(); const int64_t min_box_row = diff --git a/tensorflow/core/kernels/linalg/linalg_ops_common.cc b/tensorflow/core/kernels/linalg/linalg_ops_common.cc index bb55f7de0011a9..676111f4bf14d6 100644 --- a/tensorflow/core/kernels/linalg/linalg_ops_common.cc +++ b/tensorflow/core/kernels/linalg/linalg_ops_common.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/core/kernels/linalg/linalg_ops_common.h" +#include #include #include "third_party/eigen3/Eigen/Core" @@ -22,7 +23,9 @@ limitations under the License. #include "tensorflow/core/framework/kernel_def_builder.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/types.h" @@ -152,6 +155,10 @@ void LinearAlgebraOp::AnalyzeInputs( input_matrix_shapes->emplace_back( std::initializer_list({num_rows, num_cols})); inputs->emplace_back(&in); + OP_REQUIRES( + context, in.dtype() == DataTypeToEnum::v(), + errors::InvalidArgument("Invalid input dtype ", in.dtype(), " vs ", + DataTypeToEnum::v())); } // Have the derived class validate that the inputs are as expected. ValidateInputMatrixShapes(context, *input_matrix_shapes); @@ -212,6 +219,11 @@ void LinearAlgebraOp::PrepareOutputs( OP_REQUIRES_OK(context, context->allocate_output( output_idx, output_tensor_shape, &out)); } + OP_REQUIRES( + context, out->dtype() == DataTypeToEnum::v(), + errors::InvalidArgument("Invalid output dtype ", out->dtype(), " vs ", + DataTypeToEnum::v())); + outputs->emplace_back(out); } } diff --git a/tensorflow/core/kernels/linalg/svd_op_gpu.cu.cc b/tensorflow/core/kernels/linalg/svd_op_gpu.cu.cc index a3532f765a414b..6168baac069d68 100644 --- a/tensorflow/core/kernels/linalg/svd_op_gpu.cu.cc +++ b/tensorflow/core/kernels/linalg/svd_op_gpu.cu.cc @@ -395,6 +395,12 @@ class SvdOpGpu : public AsyncOpKernel { OP_REQUIRES_OK_ASYNC(context, context->allocate_output(2, shapeV, &outputV), done); + // If there are zero batches, we are done. + if (shapeRaw.num_elements() == 0) { + done(); + return; + } + if (n == 0 || m == 0) { if (n == m || !compute_uv_ || !full_matrices_) { // S, U, and V are all empty. Nothing to do. diff --git a/tensorflow/core/kernels/list_kernels.cc b/tensorflow/core/kernels/list_kernels.cc index 5f7943a9bad044..ab3b77230c16c4 100644 --- a/tensorflow/core/kernels/list_kernels.cc +++ b/tensorflow/core/kernels/list_kernels.cc @@ -21,19 +21,21 @@ limitations under the License. #include "tensorflow/core/kernels/list_kernels.h" +#include +#include #include +#include +#include #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/allocator.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/framework/variant.h" #include "tensorflow/core/framework/variant_op_registry.h" -#include "tensorflow/core/kernels/concat_lib.h" -#include "tensorflow/core/lib/core/coding.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/util/util.h" +#include "tensorflow/core/platform/errors.h" namespace tensorflow { @@ -49,6 +51,9 @@ Status TensorShapeFromTensor(const Tensor& t, PartialTensorShape* out) { return errors::InvalidArgument( "The only valid scalar shape tensor is the fully unknown shape " "specified as -1."); + } else if (t.shape().dims() != 1) { + return errors::InvalidArgument("Shape must be at most rank 1 but is rank ", + t.shape().dims()); } if (t.dtype() == DT_INT32) { return PartialTensorShape::MakePartialShape(t.vec().data(), @@ -319,6 +324,11 @@ class TensorListReserve : public OpKernel { void Compute(OpKernelContext* c) override { PartialTensorShape element_shape; OP_REQUIRES_OK(c, TensorShapeFromTensor(c->input(0), &element_shape)); + OP_REQUIRES( + c, TensorShapeUtils::IsScalar(c->input(1).shape()), + errors::InvalidArgument( + "The num_elements to reserve must be a tensor size 1, but got ", + c->input(1).shape())); int32_t num_elements = c->input(1).scalar()(); OP_REQUIRES(c, num_elements >= 0, errors::InvalidArgument("The num_elements to reserve must be a " diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h index 539f7e5200ec96..b8889125c33a6f 100644 --- a/tensorflow/core/kernels/list_kernels.h +++ b/tensorflow/core/kernels/list_kernels.h @@ -768,6 +768,11 @@ class TensorListFromTensor : public OpKernel { attr.set_on_host(true); OP_REQUIRES_OK(c, c->allocate_output(0, {}, &output_tensor, attr)); PartialTensorShape element_shape; + OP_REQUIRES( + c, !TensorShapeUtils::IsMatrixOrHigher(c->input(1).shape()), + errors::InvalidArgument( + "TensorListFromTensor: element_shape must be at most rank 1 but ", + "has the shape of ", c->input(1).shape().DebugString())); OP_REQUIRES_OK(c, TensorShapeFromTensor(c->input(1), &element_shape)); TensorList output_list; const Tensor& t = c->input(0); @@ -894,6 +899,11 @@ class TensorListScatter : public OpKernel { OP_REQUIRES_OK(c, c->allocate_output(0, {}, &output_tensor, attr)); Tensor indices = c->input(1); PartialTensorShape element_shape; + OP_REQUIRES( + c, !TensorShapeUtils::IsMatrixOrHigher(c->input(2).shape()), + errors::InvalidArgument( + "TensorListScatter: element_shape must be at most rank 1 but has ", + "the shape of ", c->input(2).shape().DebugString())); OP_REQUIRES_OK(c, TensorShapeFromTensor(c->input(2), &element_shape)); // TensorListScatterV2 passes the num_elements input, TensorListScatter does // not. diff --git a/tensorflow/core/kernels/load_and_remap_matrix_op.cc b/tensorflow/core/kernels/load_and_remap_matrix_op.cc index 3fa753251c0f4f..4276e16059a9c6 100644 --- a/tensorflow/core/kernels/load_and_remap_matrix_op.cc +++ b/tensorflow/core/kernels/load_and_remap_matrix_op.cc @@ -74,6 +74,11 @@ class LoadAndRemapMatrixOp : public OpKernel { std::vector row_id_present; const Tensor* row_remapping_t; OP_REQUIRES_OK(context, context->input("row_remapping", &row_remapping_t)); + OP_REQUIRES( + context, row_remapping_t->dims() == 1, + errors::InvalidArgument("The `row_remapping` tensor must be 1-D, got " + "a tensor of shape ", + row_remapping_t->shape().DebugString())); const auto row_remapping = row_remapping_t->vec(); OP_REQUIRES(context, row_remapping.size() == num_rows_, errors::InvalidArgument(strings::StrCat( diff --git a/tensorflow/core/kernels/lrn_op.cc b/tensorflow/core/kernels/lrn_op.cc index fad12d7c985ce1..0caa0d6d971318 100644 --- a/tensorflow/core/kernels/lrn_op.cc +++ b/tensorflow/core/kernels/lrn_op.cc @@ -668,7 +668,8 @@ class LRNGradOp : public OpKernel { in_image.dim_size(0) == batch && in_image.dim_size(1) == rows && in_image.dim_size(2) == cols && in_image.dim_size(3) == depth && out_image.dim_size(0) == batch && out_image.dim_size(1) == rows && - out_image.dim_size(2) == cols && out_image.dim_size(3) == depth, + out_image.dim_size(2) == cols && out_image.dim_size(3) == depth && + out_image.dims() == 4, errors::InvalidArgument( "input_grads, input_image, and out_image should have the same " "shape")); diff --git a/tensorflow/core/kernels/map_stage_op.cc b/tensorflow/core/kernels/map_stage_op.cc index aee53ae78d3af8..463e2a8ced5c2d 100644 --- a/tensorflow/core/kernels/map_stage_op.cc +++ b/tensorflow/core/kernels/map_stage_op.cc @@ -536,6 +536,11 @@ class MapStageOp : public OpKernel { OP_REQUIRES(ctx, key_tensor->NumElements() > 0, errors::InvalidArgument("key must not be empty")); + OP_REQUIRES(ctx, key_tensor->NumElements() == 1, + errors::InvalidArgument( + "key must be an int64 scalar, got tensor with shape: ", + key_tensor->shape())); + // Create copy for insertion into Staging Area Tensor key(*key_tensor); diff --git a/tensorflow/core/kernels/maxpooling_op.cc b/tensorflow/core/kernels/maxpooling_op.cc index 10625e4bf75523..60f2b952c651b3 100644 --- a/tensorflow/core/kernels/maxpooling_op.cc +++ b/tensorflow/core/kernels/maxpooling_op.cc @@ -1268,6 +1268,13 @@ class MaxPoolingNoMaskOp : public OpKernel { ShapeFromFormat(data_format_, params.tensor_in_batch, params.out_height, params.out_width, params.depth); + // Degenerate pooling output should return an empty tensor. + if (out_shape.num_elements() == 0) { + Tensor* output = nullptr; + OP_REQUIRES_OK(context, context->allocate_output(0, out_shape, &output)); + return; + } + // Assuming qint8 <--> NCHW_VECT_C (int8x4) here. constexpr bool is_int8x4 = std::is_same::value; OP_REQUIRES(context, (is_int8x4 == (data_format_ == FORMAT_NCHW_VECT_C)), diff --git a/tensorflow/core/kernels/parameterized_truncated_normal_op.cc b/tensorflow/core/kernels/parameterized_truncated_normal_op.cc index 24b7e3f4ebdd5a..a007d37c4e290a 100644 --- a/tensorflow/core/kernels/parameterized_truncated_normal_op.cc +++ b/tensorflow/core/kernels/parameterized_truncated_normal_op.cc @@ -32,6 +32,7 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/tensor_util.h" #include "tensorflow/core/kernels/stateless_random_ops.h" #include "tensorflow/core/lib/random/random_distributions.h" #include "tensorflow/core/platform/logging.h" @@ -630,20 +631,18 @@ class ParameterizedTruncatedNormalOp : public OpKernel { OP_REQUIRES(ctx, shape_tensor.NumElements() > 0, errors::InvalidArgument("Shape tensor must not be empty, got ", shape_tensor.DebugString())); - int32_t num_batches = shape_tensor.flat()(0); + TensorShape tensor_shape; + OP_REQUIRES_OK(ctx, tensor::MakeShape(shape_tensor, &tensor_shape)); + int32_t num_batches = tensor_shape.dim_size(0); int32_t samples_per_batch = 1; - const int32_t num_dims = shape_tensor.dim_size(0); + const int32_t num_dims = tensor_shape.dims(); for (int32_t i = 1; i < num_dims; i++) { - samples_per_batch *= shape_tensor.flat()(i); + samples_per_batch *= tensor_shape.dim_size(i); } const int32_t num_elements = num_batches * samples_per_batch; // Allocate the output before fudging num_batches and samples_per_batch. - auto shape_vec = shape_tensor.flat(); - TensorShape tensor_shape; - OP_REQUIRES_OK(ctx, TensorShapeUtils::MakeShape( - shape_vec.data(), shape_vec.size(), &tensor_shape)); Tensor* samples_tensor; OP_REQUIRES_OK(ctx, ctx->allocate_output(0, tensor_shape, &samples_tensor)); diff --git a/tensorflow/core/kernels/pooling_ops_3d.cc b/tensorflow/core/kernels/pooling_ops_3d.cc index 3eee1f753449c0..2d80630ae1fd89 100644 --- a/tensorflow/core/kernels/pooling_ops_3d.cc +++ b/tensorflow/core/kernels/pooling_ops_3d.cc @@ -526,7 +526,7 @@ class AvgPooling3dGradOp : public OpKernel { TensorShape output_shape; auto shape_vec = tensor_in_shape.vec(); for (int64_t i = 0; i < tensor_in_shape.NumElements(); ++i) { - output_shape.AddDim(shape_vec(i)); + OP_REQUIRES_OK(context, output_shape.AddDimWithStatus(shape_vec(i))); } Tensor* output; diff --git a/tensorflow/core/kernels/quantize_and_dequantize_op.cc b/tensorflow/core/kernels/quantize_and_dequantize_op.cc index d63a49a04be621..ae02b57861ac02 100644 --- a/tensorflow/core/kernels/quantize_and_dequantize_op.cc +++ b/tensorflow/core/kernels/quantize_and_dequantize_op.cc @@ -21,19 +21,23 @@ limitations under the License. #define EIGEN_USE_GPU #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/quantize_and_dequantize_op.h" - #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/type_traits.h" #include "tensorflow/core/framework/types.h" +#include "tensorflow/core/kernels/quantize_and_dequantize_op.h" #include "tensorflow/core/lib/core/errors.h" namespace tensorflow { +namespace { -typedef Eigen::ThreadPoolDevice CPUDevice; -typedef Eigen::GpuDevice GPUDevice; +using CpuDevice = ::Eigen::ThreadPoolDevice; +using GpuDevice = ::Eigen::GpuDevice; +using ::tensorflow::errors::InvalidArgument; + +} // namespace // Simulate quantization precision loss in a float tensor by: // 1. Quantize the tensor to fixed point numbers, which should match the target @@ -49,8 +53,8 @@ class QuantizeAndDequantizeV2Op : public OpKernel { OP_REQUIRES_OK(ctx, ctx->GetAttr("axis", &axis_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("num_bits", &num_bits_)); OP_REQUIRES(ctx, num_bits_ > 0 && num_bits_ < (signed_input_ ? 62 : 63), - errors::InvalidArgument("num_bits is out of range: ", num_bits_, - " with signed_input_ ", signed_input_)); + InvalidArgument("num_bits is out of range: ", num_bits_, + " with signed_input_ ", signed_input_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("range_given", &range_given_)); string round_mode_string; @@ -58,10 +62,10 @@ class QuantizeAndDequantizeV2Op : public OpKernel { OP_REQUIRES( ctx, (round_mode_string == "HALF_UP" || round_mode_string == "HALF_TO_EVEN"), - errors::InvalidArgument("Round mode string must be " - "'HALF_UP' or " - "'HALF_TO_EVEN', is '" + - round_mode_string + "'")); + InvalidArgument("Round mode string must be " + "'HALF_UP' or " + "'HALF_TO_EVEN', is '" + + round_mode_string + "'")); if (round_mode_string == "HALF_UP") { round_mode_ = ROUND_HALF_UP; } else if (round_mode_string == "HALF_TO_EVEN") { @@ -72,12 +76,10 @@ class QuantizeAndDequantizeV2Op : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& input = ctx->input(0); - OP_REQUIRES( - ctx, axis_ >= -1, - errors::InvalidArgument("Axis must be at least -1. Found ", axis_)); - OP_REQUIRES( - ctx, (axis_ == -1 || axis_ < input.shape().dims()), - errors::InvalidArgument("Shape must be at least rank ", axis_ + 1, + OP_REQUIRES(ctx, axis_ >= -1, + InvalidArgument("Axis must be at least -1. Found ", axis_)); + OP_REQUIRES(ctx, (axis_ == -1 || axis_ < input.shape().dims()), + InvalidArgument("Shape must be at least rank ", axis_ + 1, " but is rank ", input.shape().dims())); const int depth = (axis_ == -1) ? 1 : input.dim_size(axis_); Tensor input_min_tensor; @@ -91,21 +93,21 @@ class QuantizeAndDequantizeV2Op : public OpKernel { auto min_val = input_min_tensor.scalar()(); auto max_val = input_max_tensor.scalar()(); OP_REQUIRES(ctx, min_val <= max_val, - errors::InvalidArgument("Invalid range: input_min ", - min_val, " > input_max ", max_val)); + InvalidArgument("Invalid range: input_min ", min_val, + " > input_max ", max_val)); } else { - OP_REQUIRES(ctx, input_min_tensor.dim_size(0) == depth, - errors::InvalidArgument( - "input_min_tensor has incorrect size, was ", - input_min_tensor.dim_size(0), " expected ", depth, - " to match dim ", axis_, " of the input ", - input_min_tensor.shape())); - OP_REQUIRES(ctx, input_max_tensor.dim_size(0) == depth, - errors::InvalidArgument( - "input_max_tensor has incorrect size, was ", - input_max_tensor.dim_size(0), " expected ", depth, - " to match dim ", axis_, " of the input ", - input_max_tensor.shape())); + OP_REQUIRES( + ctx, input_min_tensor.dim_size(0) == depth, + InvalidArgument("input_min_tensor has incorrect size, was ", + input_min_tensor.dim_size(0), " expected ", depth, + " to match dim ", axis_, " of the input ", + input_min_tensor.shape())); + OP_REQUIRES( + ctx, input_max_tensor.dim_size(0) == depth, + InvalidArgument("input_max_tensor has incorrect size, was ", + input_max_tensor.dim_size(0), " expected ", depth, + " to match dim ", axis_, " of the input ", + input_max_tensor.shape())); } } else { auto range_shape = (axis_ == -1) ? TensorShape({}) : TensorShape({depth}); @@ -158,38 +160,34 @@ class QuantizeAndDequantizeV4GradientOp : public OpKernel { Tensor* input_backprop = nullptr; OP_REQUIRES_OK(ctx, ctx->allocate_output(0, input.shape(), &input_backprop)); - OP_REQUIRES( - ctx, axis_ >= -1, - errors::InvalidArgument("Axis must be at least -1. Found ", axis_)); + OP_REQUIRES(ctx, axis_ >= -1, + InvalidArgument("Axis must be at least -1. Found ", axis_)); OP_REQUIRES(ctx, (axis_ == -1 || axis_ < input.shape().dims()), - errors::InvalidArgument( + InvalidArgument( "Axis should be -1 or 0 or a positive value less than ", input.shape().dims(), "but given axis value was ", axis_)); - OP_REQUIRES( - ctx, input.IsSameSize(gradient), - errors::InvalidArgument("gradient and input must be the same size")); + OP_REQUIRES(ctx, input.IsSameSize(gradient), + InvalidArgument("gradient and input must be the same size")); const int depth = (axis_ == -1) ? 1 : input.dim_size(axis_); const Tensor& input_min_tensor = ctx->input(2); OP_REQUIRES(ctx, input_min_tensor.dims() == 0 || input_min_tensor.dims() == 1, - errors::InvalidArgument( - "Input min tensor must have dimension 1. Recieved ", + InvalidArgument( + "Input min tensor must have dimension 0 or 1. Received ", input_min_tensor.dims(), ".")); const Tensor& input_max_tensor = ctx->input(3); OP_REQUIRES(ctx, input_max_tensor.dims() == 0 || input_max_tensor.dims() == 1, - errors::InvalidArgument( - "Input max tensor must have dimension 1. Recieved ", + InvalidArgument( + "Input max tensor must have dimension 0 or 1. Received ", input_max_tensor.dims(), ".")); if (axis_ != -1) { - OP_REQUIRES( - ctx, input_min_tensor.dim_size(0) == depth, - errors::InvalidArgument("min has incorrect size, expected ", depth, + OP_REQUIRES(ctx, input_min_tensor.dim_size(0) == depth, + InvalidArgument("min has incorrect size, expected ", depth, " was ", input_min_tensor.dim_size(0))); - OP_REQUIRES( - ctx, input_max_tensor.dim_size(0) == depth, - errors::InvalidArgument("max has incorrect size, expected ", depth, + OP_REQUIRES(ctx, input_max_tensor.dim_size(0) == depth, + InvalidArgument("max has incorrect size, expected ", depth, " was ", input_max_tensor.dim_size(0))); } @@ -203,6 +201,12 @@ class QuantizeAndDequantizeV4GradientOp : public OpKernel { ctx->allocate_output(2, min_max_shape, &input_max_backprop)); if (axis_ == -1) { + OP_REQUIRES( + ctx, TensorShapeUtils::IsScalar(input_min_tensor.shape()), + InvalidArgument("input_min must be a scalar if axis is unspecified")); + OP_REQUIRES( + ctx, TensorShapeUtils::IsScalar(input_max_tensor.shape()), + InvalidArgument("input_max must be a scalar if axis is unspecified")); functor::QuantizeAndDequantizeOneScaleGradientFunctor f; f(ctx->eigen_device(), gradient.template flat(), input.template flat(), input_min_tensor.scalar(), @@ -246,21 +250,25 @@ class QuantizeAndDequantizeV3Op : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& input = ctx->input(0); OP_REQUIRES(ctx, axis_ < input.dims(), - errors::InvalidArgument( + InvalidArgument( "Axis requested is larger than input dimensions. Axis: ", axis_, " Input Dimensions: ", input.dims())); const int depth = (axis_ == -1) ? 1 : input.dim_size(axis_); Tensor* output = nullptr; OP_REQUIRES_OK(ctx, ctx->allocate_output(0, input.shape(), &output)); - Tensor num_bits_tensor; - num_bits_tensor = ctx->input(3); - int num_bits_val = num_bits_tensor.scalar()(); + // Get num_bits and validate. + const Tensor num_bits_tensor = ctx->input(3); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(num_bits_tensor.shape()), + InvalidArgument("Invalid shape. The `num_bits` tensor should " + "be a scalar. Got dimensions: ", + num_bits_tensor.dims())); - OP_REQUIRES( - ctx, num_bits_val > 0 && num_bits_val < (signed_input_ ? 62 : 63), - errors::InvalidArgument("num_bits is out of range: ", num_bits_val, - " with signed_input_ ", signed_input_)); + const int num_bits_val = num_bits_tensor.scalar()(); + OP_REQUIRES(ctx, + num_bits_val > 0 && num_bits_val < (signed_input_ ? 62 : 63), + InvalidArgument("num_bits is out of range: ", num_bits_val, + " with `signed_input_` ", signed_input_)); Tensor input_min_tensor; Tensor input_max_tensor; @@ -268,24 +276,24 @@ class QuantizeAndDequantizeV3Op : public OpKernel { input_min_tensor = ctx->input(1); input_max_tensor = ctx->input(2); if (axis_ == -1) { - auto min_val = input_min_tensor.scalar()(); - auto max_val = input_max_tensor.scalar()(); + const auto min_val = input_min_tensor.scalar()(); + const auto max_val = input_max_tensor.scalar()(); OP_REQUIRES(ctx, min_val <= max_val, - errors::InvalidArgument("Invalid range: input_min ", - min_val, " > input_max ", max_val)); + InvalidArgument("Invalid range: input_min ", min_val, + " > input_max ", max_val)); } else { - OP_REQUIRES(ctx, input_min_tensor.dim_size(0) == depth, - errors::InvalidArgument( - "input_min_tensor has incorrect size, was ", - input_min_tensor.dim_size(0), " expected ", depth, - " to match dim ", axis_, " of the input ", - input_min_tensor.shape())); - OP_REQUIRES(ctx, input_max_tensor.dim_size(0) == depth, - errors::InvalidArgument( - "input_max_tensor has incorrect size, was ", - input_max_tensor.dim_size(0), " expected ", depth, - " to match dim ", axis_, " of the input ", - input_max_tensor.shape())); + OP_REQUIRES( + ctx, input_min_tensor.dim_size(0) == depth, + InvalidArgument("input_min_tensor has incorrect size, was ", + input_min_tensor.dim_size(0), " expected ", depth, + " to match dim ", axis_, " of the input ", + input_min_tensor.shape())); + OP_REQUIRES( + ctx, input_max_tensor.dim_size(0) == depth, + InvalidArgument("input_max_tensor has incorrect size, was ", + input_max_tensor.dim_size(0), " expected ", depth, + " to match dim ", axis_, " of the input ", + input_max_tensor.shape())); } } else { auto range_shape = (axis_ == -1) ? TensorShape({}) : TensorShape({depth}); @@ -325,15 +333,14 @@ class QuantizeAndDequantizeOp : public OpKernel { OP_REQUIRES_OK(ctx, ctx->GetAttr("signed_input", &signed_input_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("num_bits", &num_bits_)); OP_REQUIRES(ctx, num_bits_ > 0 && num_bits_ < (signed_input_ ? 62 : 63), - errors::InvalidArgument("num_bits is out of range: ", num_bits_, - " with signed_input_ ", signed_input_)); + InvalidArgument("num_bits is out of range: ", num_bits_, + " with signed_input_ ", signed_input_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("range_given", &range_given_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("input_min", &input_min_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("input_max", &input_max_)); if (range_given_) { - OP_REQUIRES( - ctx, input_min_ <= input_max_, - errors::InvalidArgument("Invalid range: input_min ", input_min_, + OP_REQUIRES(ctx, input_min_ <= input_max_, + InvalidArgument("Invalid range: input_min ", input_min_, " > input_max ", input_max_)); } } @@ -365,53 +372,53 @@ class QuantizeAndDequantizeOp : public OpKernel { float input_max_; }; -// Specializations for CPUDevice. +// Specializations for CpuDevice. namespace functor { template -struct QuantizeAndDequantizeOneScaleFunctor { - void operator()(const CPUDevice& d, typename TTypes::ConstVec input, +struct QuantizeAndDequantizeOneScaleFunctor { + void operator()(const CpuDevice& d, typename TTypes::ConstVec input, const bool signed_input, const int num_bits, const bool range_given, Tensor* input_min_tensor, Tensor* input_max_tensor, QuantizerRoundMode round_mode, bool narrow_range, typename TTypes::Vec out) { - QuantizeAndDequantizeOneScaleImpl::Compute( + QuantizeAndDequantizeOneScaleImpl::Compute( d, input, signed_input, num_bits, range_given, input_min_tensor, input_max_tensor, round_mode, narrow_range, out); } }; template -struct QuantizeAndDequantizePerChannelFunctor { - void operator()(const CPUDevice& d, typename TTypes::ConstTensor input, +struct QuantizeAndDequantizePerChannelFunctor { + void operator()(const CpuDevice& d, typename TTypes::ConstTensor input, bool signed_input, int num_bits, bool range_given, Tensor* input_min_tensor, Tensor* input_max_tensor, QuantizerRoundMode round_mode, bool narrow_range, typename TTypes::Tensor out) { - QuantizeAndDequantizePerChannelImpl::Compute( + QuantizeAndDequantizePerChannelImpl::Compute( d, input, signed_input, num_bits, range_given, input_min_tensor, input_max_tensor, round_mode, narrow_range, out); } }; template -struct QuantizeAndDequantizeOneScaleGradientFunctor { - void operator()(const CPUDevice& d, typename TTypes::ConstFlat gradient, +struct QuantizeAndDequantizeOneScaleGradientFunctor { + void operator()(const CpuDevice& d, typename TTypes::ConstFlat gradient, typename TTypes::ConstFlat input, typename TTypes::ConstScalar input_min_tensor, typename TTypes::ConstScalar input_max_tensor, typename TTypes::Flat input_backprop, typename TTypes::Scalar input_min_backprop, typename TTypes::Scalar input_max_backprop) { - QuantizeAndDequantizeOneScaleGradientImpl::Compute( + QuantizeAndDequantizeOneScaleGradientImpl::Compute( d, gradient, input, input_min_tensor, input_max_tensor, input_backprop, input_min_backprop, input_max_backprop); } }; template -struct QuantizeAndDequantizePerChannelGradientFunctor { - void operator()(const CPUDevice& d, +struct QuantizeAndDequantizePerChannelGradientFunctor { + void operator()(const CpuDevice& d, typename TTypes::ConstTensor gradient, typename TTypes::ConstTensor input, const Tensor* input_min_tensor, @@ -419,16 +426,16 @@ struct QuantizeAndDequantizePerChannelGradientFunctor { typename TTypes::Tensor input_backprop, typename TTypes::Flat input_min_backprop, typename TTypes::Flat input_max_backprop) { - QuantizeAndDequantizePerChannelGradientImpl::Compute( + QuantizeAndDequantizePerChannelGradientImpl::Compute( d, gradient, input, input_min_tensor, input_max_tensor, input_backprop, input_min_backprop, input_max_backprop); } }; -template struct functor::QuantizeAndDequantizeOneScaleGradientFunctor; template struct functor::QuantizeAndDequantizePerChannelGradientFunctor< - CPUDevice, double>; + CpuDevice, double>; } // namespace functor @@ -436,22 +443,22 @@ template struct functor::QuantizeAndDequantizePerChannelGradientFunctor< REGISTER_KERNEL_BUILDER(Name("QuantizeAndDequantizeV2") \ .Device(DEVICE_CPU) \ .TypeConstraint("T"), \ - QuantizeAndDequantizeV2Op); \ + QuantizeAndDequantizeV2Op); \ REGISTER_KERNEL_BUILDER(Name("QuantizeAndDequantizeV3") \ .Device(DEVICE_CPU) \ .TypeConstraint("T"), \ - QuantizeAndDequantizeV3Op); \ + QuantizeAndDequantizeV3Op); \ REGISTER_KERNEL_BUILDER(Name("QuantizeAndDequantizeV4") \ .Device(DEVICE_CPU) \ .TypeConstraint("T"), \ - QuantizeAndDequantizeV2Op); \ + QuantizeAndDequantizeV2Op); \ REGISTER_KERNEL_BUILDER(Name("QuantizeAndDequantizeV4Grad") \ .Device(DEVICE_CPU) \ .TypeConstraint("T"), \ - QuantizeAndDequantizeV4GradientOp); \ + QuantizeAndDequantizeV4GradientOp); \ REGISTER_KERNEL_BUILDER( \ Name("QuantizeAndDequantize").Device(DEVICE_CPU).TypeConstraint("T"), \ - QuantizeAndDequantizeOp); + QuantizeAndDequantizeOp); TF_CALL_float(REGISTER_CPU_KERNEL); TF_CALL_double(REGISTER_CPU_KERNEL); #undef REGISTER_CPU_KERNEL @@ -464,29 +471,29 @@ TF_CALL_double(REGISTER_CPU_KERNEL); .HostMemory("input_min") \ .HostMemory("input_max") \ .TypeConstraint("T"), \ - QuantizeAndDequantizeV2Op); \ + QuantizeAndDequantizeV2Op); \ REGISTER_KERNEL_BUILDER(Name("QuantizeAndDequantizeV3") \ .Device(DEVICE_GPU) \ .HostMemory("input_min") \ .HostMemory("input_max") \ .HostMemory("num_bits") \ .TypeConstraint("T"), \ - QuantizeAndDequantizeV3Op); \ + QuantizeAndDequantizeV3Op); \ REGISTER_KERNEL_BUILDER(Name("QuantizeAndDequantizeV4") \ .Device(DEVICE_GPU) \ .HostMemory("input_min") \ .HostMemory("input_max") \ .TypeConstraint("T"), \ - QuantizeAndDequantizeV2Op); \ + QuantizeAndDequantizeV2Op); \ REGISTER_KERNEL_BUILDER(Name("QuantizeAndDequantizeV4Grad") \ .Device(DEVICE_GPU) \ .HostMemory("input_min") \ .HostMemory("input_max") \ .TypeConstraint("T"), \ - QuantizeAndDequantizeV4GradientOp); \ + QuantizeAndDequantizeV4GradientOp); \ REGISTER_KERNEL_BUILDER( \ Name("QuantizeAndDequantize").Device(DEVICE_GPU).TypeConstraint("T"), \ - QuantizeAndDequantizeOp); + QuantizeAndDequantizeOp); TF_CALL_float(REGISTER_GPU_KERNEL); TF_CALL_double(REGISTER_GPU_KERNEL); #undef REGISTER_GPU_KERNEL diff --git a/tensorflow/core/kernels/quantize_down_and_shrink_range.cc b/tensorflow/core/kernels/quantize_down_and_shrink_range.cc index 1b948c8108de87..83f8996b4cc746 100644 --- a/tensorflow/core/kernels/quantize_down_and_shrink_range.cc +++ b/tensorflow/core/kernels/quantize_down_and_shrink_range.cc @@ -40,8 +40,20 @@ class QuantizeDownAndShrinkRangeOp : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& input = ctx->input(0); - const float input_min_float = ctx->input(1).flat()(0); - const float input_max_float = ctx->input(2).flat()(0); + const Tensor& input_min = ctx->input(1); + const Tensor& input_max = ctx->input(2); + + OP_REQUIRES( + ctx, TensorShapeUtils::IsScalar(input_min.shape()), + errors::InvalidArgument("`input_min` must be rank 0 but is rank ", + input_min.dims())); + OP_REQUIRES( + ctx, TensorShapeUtils::IsScalar(input_max.shape()), + errors::InvalidArgument("`input_max` must be rank 0 but is rank ", + input_max.dims())); + + const float input_min_float = input_min.scalar()(); + const float input_max_float = input_max.scalar()(); Tensor* output = nullptr; OP_REQUIRES_OK(ctx, ctx->allocate_output(0, input.shape(), &output)); Tensor* output_min = nullptr; diff --git a/tensorflow/core/kernels/quantize_down_and_shrink_range_op_test.cc b/tensorflow/core/kernels/quantize_down_and_shrink_range_op_test.cc index 48b56ae0ac20c4..0c34a9da7218a8 100644 --- a/tensorflow/core/kernels/quantize_down_and_shrink_range_op_test.cc +++ b/tensorflow/core/kernels/quantize_down_and_shrink_range_op_test.cc @@ -53,8 +53,8 @@ TEST_F(QuantizeDownAndShrinkRangeTest, HandCrafted) { const int value_count = 3; AddInputFromArray(TensorShape({value_count}), {-(1 << 23), 0, (1 << 23)}); - AddInputFromArray(TensorShape({1}), {-256.0f}); - AddInputFromArray(TensorShape({1}), {256.0f}); + AddInputFromArray(TensorShape({}), {-256.0f}); + AddInputFromArray(TensorShape({}), {256.0f}); TF_ASSERT_OK(RunOpKernel()); Tensor expected(allocator(), DT_QUINT8, TensorShape({value_count})); test::FillValues(&expected, {0, 127, 255}); diff --git a/tensorflow/core/kernels/quantized_activation_ops.cc b/tensorflow/core/kernels/quantized_activation_ops.cc index 2896c3d45a7023..36d321a8e17138 100644 --- a/tensorflow/core/kernels/quantized_activation_ops.cc +++ b/tensorflow/core/kernels/quantized_activation_ops.cc @@ -32,8 +32,21 @@ class QuantizedReluOp : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& input = context->input(0); - const float min_input = context->input(1).flat()(0); - const float max_input = context->input(2).flat()(0); + const Tensor& min_input_tensor = context->input(1); + const Tensor& max_input_tensor = context->input(2); + + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(min_input_tensor.shape()), + errors::InvalidArgument("`min_input` must be rank 0 but is rank ", + min_input_tensor.dims())); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(max_input_tensor.shape()), + errors::InvalidArgument("`max_input` must be rank 0 but is rank ", + max_input_tensor.dims())); + + const float min_input = min_input_tensor.scalar()(); + const float max_input = max_input_tensor.scalar()(); + Tensor* output = nullptr; OP_REQUIRES_OK(context, context->allocate_output(0, input.shape(), &output)); @@ -65,8 +78,21 @@ class QuantizedRelu6Op : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& input = context->input(0); - const float min_input = context->input(1).flat()(0); - const float max_input = context->input(2).flat()(0); + const Tensor& min_input_tensor = context->input(1); + const Tensor& max_input_tensor = context->input(2); + + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(min_input_tensor.shape()), + errors::InvalidArgument("`min_input` must be rank 0 but is rank ", + min_input_tensor.dims())); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(max_input_tensor.shape()), + errors::InvalidArgument("`max_input` must be rank 0 but is rank ", + max_input_tensor.dims())); + + const float min_input = min_input_tensor.scalar()(); + const float max_input = max_input_tensor.scalar()(); + Tensor* output = nullptr; OP_REQUIRES_OK(context, context->allocate_output(0, input.shape(), &output)); diff --git a/tensorflow/core/kernels/quantized_activation_ops_test.cc b/tensorflow/core/kernels/quantized_activation_ops_test.cc index b3b7cb58b9a455..34c5130f4759b5 100644 --- a/tensorflow/core/kernels/quantized_activation_ops_test.cc +++ b/tensorflow/core/kernels/quantized_activation_ops_test.cc @@ -55,8 +55,8 @@ TEST_F(QuantizedActivationsTest, TestRelu) { AddInputFromArray(input_quantized.shape(), input_quantized.flat()); - AddInputFromArray(TensorShape({1}), {input_min}); - AddInputFromArray(TensorShape({1}), {input_max}); + AddInputFromArray(TensorShape({}), {input_min}); + AddInputFromArray(TensorShape({}), {input_max}); TF_ASSERT_OK(RunOpKernel()); const Tensor& output_quantized = *GetOutput(0); const float output_min = GetOutput(1)->flat()(0); @@ -86,8 +86,8 @@ TEST_F(QuantizedActivationsTest, TestRelu6) { AddInputFromArray(input_quantized.shape(), input_quantized.flat()); - AddInputFromArray(TensorShape({1}), {input_min}); - AddInputFromArray(TensorShape({1}), {input_max}); + AddInputFromArray(TensorShape({}), {input_min}); + AddInputFromArray(TensorShape({}), {input_max}); TF_ASSERT_OK(RunOpKernel()); const Tensor& output_quantized = *GetOutput(0); const float output_min = GetOutput(1)->flat()(0); diff --git a/tensorflow/core/kernels/quantized_add_op.cc b/tensorflow/core/kernels/quantized_add_op.cc index 21e07671ed8fd8..4ddb6e04223009 100644 --- a/tensorflow/core/kernels/quantized_add_op.cc +++ b/tensorflow/core/kernels/quantized_add_op.cc @@ -25,6 +25,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/kernels/meta_support.h" #include "tensorflow/core/kernels/quantization_utils.h" #include "tensorflow/core/lib/core/errors.h" @@ -457,10 +458,28 @@ class QuantizedAddOp : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& x = context->input(0); const Tensor& y = context->input(1); - const float min_x = context->input(2).flat()(0); - const float max_x = context->input(3).flat()(0); - const float min_y = context->input(4).flat()(0); - const float max_y = context->input(5).flat()(0); + const Tensor& min_x_tensor = context->input(2); + const Tensor& max_x_tensor = context->input(3); + const Tensor& min_y_tensor = context->input(4); + const Tensor& max_y_tensor = context->input(5); + + OP_REQUIRES(context, TensorShapeUtils::IsScalar(min_x_tensor.shape()), + errors::InvalidArgument("`min_x` must be rank 0 but is rank ", + min_x_tensor.dims())); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(max_x_tensor.shape()), + errors::InvalidArgument("`max_x` must be rank 0 but is rank ", + max_x_tensor.dims())); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(min_y_tensor.shape()), + errors::InvalidArgument("`min_y` must be rank 0 but is rank ", + min_y_tensor.dims())); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(max_y_tensor.shape()), + errors::InvalidArgument("`max_y` must be rank 0 but is rank ", + max_y_tensor.dims())); + + const float min_x = min_x_tensor.scalar()(); + const float max_x = max_x_tensor.scalar()(); + const float min_y = min_y_tensor.scalar()(); + const float max_y = max_y_tensor.scalar()(); BCast bcast(BCast::FromShape(x.shape()), BCast::FromShape(y.shape())); if (!bcast.IsValid()) { diff --git a/tensorflow/core/kernels/quantized_bias_add_op.cc b/tensorflow/core/kernels/quantized_bias_add_op.cc index db0e21a498011d..c064f9b1b21e25 100644 --- a/tensorflow/core/kernels/quantized_bias_add_op.cc +++ b/tensorflow/core/kernels/quantized_bias_add_op.cc @@ -20,6 +20,7 @@ limitations under the License. #include "tensorflow/core/framework/numeric_op.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/kernels/meta_support.h" #include "tensorflow/core/kernels/ops_util.h" #include "tensorflow/core/kernels/quantization_utils.h" @@ -38,10 +39,30 @@ class QuantizedBiasAddOp : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& input = context->input(0); const Tensor& bias = context->input(1); - const float input_min = context->input(2).flat()(0); - const float input_max = context->input(3).flat()(0); - const float bias_min = context->input(4).flat()(0); - const float bias_max = context->input(5).flat()(0); + + const Tensor& min_input = context->input(2); + const Tensor& max_input = context->input(3); + const Tensor& min_bias = context->input(4); + const Tensor& max_bias = context->input(5); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(min_input.shape()), + errors::InvalidArgument("`min_input` must be rank 0 but is rank ", + min_input.dims())); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(max_input.shape()), + errors::InvalidArgument("`max_input` must be rank 0 but is rank ", + max_input.dims())); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(min_bias.shape()), + errors::InvalidArgument( + "`min_bias` must be rank 0 but is rank ", min_bias.dims())); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(max_bias.shape()), + errors::InvalidArgument( + "`max_bias` must be rank 0 but is rank ", max_bias.dims())); + + const float input_min = min_input.flat()(0); + const float input_max = max_input.flat()(0); + const float bias_min = min_bias.flat()(0); + const float bias_max = max_bias.flat()(0); OP_REQUIRES(context, TensorShapeUtils::IsMatrixOrHigher(input.shape()), errors::InvalidArgument("Input tensor must be at least 2D: ", diff --git a/tensorflow/core/kernels/quantized_bias_add_op_test.cc b/tensorflow/core/kernels/quantized_bias_add_op_test.cc index 7b99ceafe261b7..edfae98efa953b 100644 --- a/tensorflow/core/kernels/quantized_bias_add_op_test.cc +++ b/tensorflow/core/kernels/quantized_bias_add_op_test.cc @@ -74,10 +74,10 @@ TEST_F(QuantizedBiasAddTest, Small) { input_quantized.flat()); AddInputFromArray(bias_quantized.shape(), bias_quantized.flat()); - AddInputFromArray(TensorShape({1}), {input_min}); - AddInputFromArray(TensorShape({1}), {input_max}); - AddInputFromArray(TensorShape({1}), {bias_min}); - AddInputFromArray(TensorShape({1}), {bias_max}); + AddInputFromArray(TensorShape({}), {input_min}); + AddInputFromArray(TensorShape({}), {input_max}); + AddInputFromArray(TensorShape({}), {bias_min}); + AddInputFromArray(TensorShape({}), {bias_max}); TF_ASSERT_OK(RunOpKernel()); const Tensor& output_quantized = *GetOutput(0); const float output_min = GetOutput(1)->flat()(0); @@ -156,10 +156,10 @@ TEST_F(QuantizedBiasAddTest, RealData) { input_quantized.flat()); AddInputFromArray(bias_quantized.shape(), bias_quantized.flat()); - AddInputFromArray(TensorShape({1}), {input_min}); - AddInputFromArray(TensorShape({1}), {input_max}); - AddInputFromArray(TensorShape({1}), {bias_min}); - AddInputFromArray(TensorShape({1}), {bias_max}); + AddInputFromArray(TensorShape({}), {input_min}); + AddInputFromArray(TensorShape({}), {input_max}); + AddInputFromArray(TensorShape({}), {bias_min}); + AddInputFromArray(TensorShape({}), {bias_max}); TF_ASSERT_OK(RunOpKernel()); const Tensor& output_quantized = *GetOutput(0); const float output_min = GetOutput(1)->flat()(0); diff --git a/tensorflow/core/kernels/quantized_conv_ops.cc b/tensorflow/core/kernels/quantized_conv_ops.cc index 39824ab2546d33..0519256b613be6 100644 --- a/tensorflow/core/kernels/quantized_conv_ops.cc +++ b/tensorflow/core/kernels/quantized_conv_ops.cc @@ -18,8 +18,6 @@ limitations under the License. #include #include -#include "tensorflow/core/platform/errors.h" - #define EIGEN_USE_THREADS #define GEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK @@ -32,6 +30,7 @@ limitations under the License. #include "tensorflow/core/kernels/quantization_utils.h" #include "tensorflow/core/kernels/reference_gemm.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/errors.h" #include "tensorflow/core/util/padding.h" namespace tensorflow { @@ -499,11 +498,26 @@ class QuantizedConv2DOp : public OpKernel { // For 2D convolution, there should be 4 dimensions. OP_REQUIRES(context, input.dims() == 4, - errors::InvalidArgument("input must be 4-dimensional", - input.shape().DebugString())); + errors::InvalidArgument("input must be rank 4 but is rank ", + input.shape().dims())); OP_REQUIRES(context, filter.dims() == 4, - errors::InvalidArgument("filter must be 4-dimensional: ", - filter.shape().DebugString())); + errors::InvalidArgument("filter must be rank 4 but is rank ", + filter.shape().dims())); + + OP_REQUIRES(context, TensorShapeUtils::IsScalar(context->input(2).shape()), + errors::InvalidArgument("min_input must be rank 0 but is rank ", + context->input(2).shape().dims())); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(context->input(3).shape()), + errors::InvalidArgument("max_input must be rank 0 but is rank ", + context->input(3).shape().dims())); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(context->input(4).shape()), + errors::InvalidArgument("min_filter must be rank 0 but is rank ", + context->input(4).shape().dims())); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(context->input(5).shape()), + errors::InvalidArgument("max_filter must be rank 0 but is rank ", + context->input(5).shape().dims())); const float min_input = context->input(2).flat()(0); const float max_input = context->input(3).flat()(0); diff --git a/tensorflow/core/kernels/quantized_conv_ops_test.cc b/tensorflow/core/kernels/quantized_conv_ops_test.cc index 4226378bb64683..dd0878a36df7b0 100644 --- a/tensorflow/core/kernels/quantized_conv_ops_test.cc +++ b/tensorflow/core/kernels/quantized_conv_ops_test.cc @@ -91,10 +91,10 @@ TEST_F(QuantizedConv2DTest, Small) { image_quantized.flat()); AddInputFromArray(filter_quantized.shape(), filter_quantized.flat()); - AddInputFromArray(TensorShape({1}), {image_min}); - AddInputFromArray(TensorShape({1}), {image_max}); - AddInputFromArray(TensorShape({1}), {filter_min}); - AddInputFromArray(TensorShape({1}), {filter_max}); + AddInputFromArray(TensorShape({}), {image_min}); + AddInputFromArray(TensorShape({}), {image_max}); + AddInputFromArray(TensorShape({}), {filter_min}); + AddInputFromArray(TensorShape({}), {filter_max}); TF_ASSERT_OK(RunOpKernel()); // We're sliding the 3x3 filter across the 3x4 image, with accesses outside @@ -158,10 +158,10 @@ TEST_F(QuantizedConv2DTest, Small32Bit) { AddInputFromArray( TensorShape({filter_size, filter_size, depth, filter_count}), {10, 40, 70, 20, 50, 80, 30, 60, 90}); - AddInputFromArray(TensorShape({1}), {0}); - AddInputFromArray(TensorShape({1}), {255.0f}); - AddInputFromArray(TensorShape({1}), {0}); - AddInputFromArray(TensorShape({1}), {255.0f}); + AddInputFromArray(TensorShape({}), {0}); + AddInputFromArray(TensorShape({}), {255.0f}); + AddInputFromArray(TensorShape({}), {0}); + AddInputFromArray(TensorShape({}), {255.0f}); TF_ASSERT_OK(RunOpKernel()); const int expected_width = image_width; @@ -201,10 +201,10 @@ TEST_F(QuantizedConv2DTest, OddPadding) { AddInputFromArray( TensorShape({filter_size, filter_size, depth, filter_count}), {1, 2, 3, 4, 5, 6, 7, 8, 9}); - AddInputFromArray(TensorShape({1}), {0}); - AddInputFromArray(TensorShape({1}), {255.0f}); - AddInputFromArray(TensorShape({1}), {0}); - AddInputFromArray(TensorShape({1}), {255.0f}); + AddInputFromArray(TensorShape({}), {0}); + AddInputFromArray(TensorShape({}), {255.0f}); + AddInputFromArray(TensorShape({}), {0}); + AddInputFromArray(TensorShape({}), {255.0f}); TF_ASSERT_OK(RunOpKernel()); const int expected_width = image_width / stride; @@ -244,10 +244,10 @@ TEST_F(QuantizedConv2DTest, OddPaddingBatch) { AddInputFromArray( TensorShape({filter_size, filter_size, depth, filter_count}), {1, 2, 3, 4, 5, 6, 7, 8, 9}); - AddInputFromArray(TensorShape({1}), {0}); - AddInputFromArray(TensorShape({1}), {255.0f}); - AddInputFromArray(TensorShape({1}), {0}); - AddInputFromArray(TensorShape({1}), {255.0f}); + AddInputFromArray(TensorShape({}), {0}); + AddInputFromArray(TensorShape({}), {255.0f}); + AddInputFromArray(TensorShape({}), {0}); + AddInputFromArray(TensorShape({}), {255.0f}); TF_ASSERT_OK(RunOpKernel()); const int expected_width = image_width / stride; @@ -302,10 +302,10 @@ TEST_F(QuantizedConv2DTest, SmallWithNoZero) { image_quantized.flat()); AddInputFromArray(filter_quantized.shape(), filter_quantized.flat()); - AddInputFromArray(TensorShape({1}), {image_min}); - AddInputFromArray(TensorShape({1}), {image_max}); - AddInputFromArray(TensorShape({1}), {filter_min}); - AddInputFromArray(TensorShape({1}), {filter_max}); + AddInputFromArray(TensorShape({}), {image_min}); + AddInputFromArray(TensorShape({}), {image_max}); + AddInputFromArray(TensorShape({}), {filter_min}); + AddInputFromArray(TensorShape({}), {filter_max}); TF_ASSERT_OK(RunOpKernel()); const int expected_width = image_width; const int expected_height = image_height * filter_count; diff --git a/tensorflow/core/kernels/quantized_instance_norm.cc b/tensorflow/core/kernels/quantized_instance_norm.cc index d62094cc9fad85..6cc2ad55e430cc 100644 --- a/tensorflow/core/kernels/quantized_instance_norm.cc +++ b/tensorflow/core/kernels/quantized_instance_norm.cc @@ -25,7 +25,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" - +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/kernels/quantization_utils.h" #ifdef USE_NEON @@ -274,8 +274,16 @@ class QuantizedInstanceNorm : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& input = context->input(0); - float input_min = context->input(1).flat()(0); - float input_max = context->input(2).flat()(0); + const Tensor& x_min = context->input(1); + const Tensor& x_max = context->input(2); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(x_min.shape()), + errors::InvalidArgument("`x_min` must be rank 0 but is rank ", + x_min.dims())); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(x_max.shape()), + errors::InvalidArgument("`x_max` must be rank 0 but is rank ", + x_max.dims())); + float input_min = x_min.scalar()(); + float input_max = x_max.scalar()(); float input_scale = (input_max - input_min) / 255.0f; OP_REQUIRES(context, input_min < input_max, diff --git a/tensorflow/core/kernels/quantized_matmul_op.cc b/tensorflow/core/kernels/quantized_matmul_op.cc index 9d3b5279e4bb82..ae65dc3b5e38ce 100644 --- a/tensorflow/core/kernels/quantized_matmul_op.cc +++ b/tensorflow/core/kernels/quantized_matmul_op.cc @@ -20,11 +20,14 @@ limitations under the License. #define GEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK #include "public/gemmlowp.h" #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/op_requires.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/kernels/meta_support.h" #include "tensorflow/core/kernels/quantization_utils.h" #include "tensorflow/core/kernels/reference_gemm.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/errors.h" namespace tensorflow { @@ -75,9 +78,21 @@ class QuantizedMatMulOp : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& a = context->input(0); const Tensor& b = context->input(1); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(context->input(2).shape()), + errors::InvalidArgument("min_a must be a scalar, but got shape", + context->input(2).shape())); const float min_a = context->input(2).flat()(0); + OP_REQUIRES(context, context->input(3).NumElements() == 1, + errors::InvalidArgument("max_a must be a scalar, but got shape", + context->input(3).shape())); const float max_a = context->input(3).flat()(0); + OP_REQUIRES(context, context->input(4).NumElements() == 1, + errors::InvalidArgument("min_b must be a scalar, but got shape", + context->input(4).shape())); const float min_b = context->input(4).flat()(0); + OP_REQUIRES(context, context->input(5).NumElements() == 1, + errors::InvalidArgument("max_b must be a scalar, but got shape", + context->input(5).shape())); const float max_b = context->input(5).flat()(0); // Make sure that we have valid quantization ranges for the input buffers. diff --git a/tensorflow/core/kernels/quantized_matmul_op_test.cc b/tensorflow/core/kernels/quantized_matmul_op_test.cc index c9f05dbc10bb8b..f562a2ebcb744f 100644 --- a/tensorflow/core/kernels/quantized_matmul_op_test.cc +++ b/tensorflow/core/kernels/quantized_matmul_op_test.cc @@ -62,10 +62,10 @@ TEST_F(QuantizedMatMulTest, Small_NoParams) { // | 15 | 16 | 17 | 18 | AddInputFromArray(TensorShape({3, 4}), {7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18}); - AddInputFromArray(TensorShape({1}), {0}); - AddInputFromArray(TensorShape({1}), {255.0f}); - AddInputFromArray(TensorShape({1}), {0}); - AddInputFromArray(TensorShape({1}), {255.0f}); + AddInputFromArray(TensorShape({}), {0}); + AddInputFromArray(TensorShape({}), {255.0f}); + AddInputFromArray(TensorShape({}), {0}); + AddInputFromArray(TensorShape({}), {255.0f}); TF_ASSERT_OK(RunOpKernel()); // Here are the results we expect, from hand calculations: @@ -118,10 +118,10 @@ TEST_F(QuantizedMatMulTest, VerySmall_WithParams) { // The B matrix is: // | 1 | AddInputFromArray(TensorShape({b_rows, b_cols}), {0}); - AddInputFromArray(TensorShape({1}), {-12.0f}); - AddInputFromArray(TensorShape({1}), {243.0f}); - AddInputFromArray(TensorShape({1}), {1.0f}); - AddInputFromArray(TensorShape({1}), {256.0f}); + AddInputFromArray(TensorShape({}), {-12.0f}); + AddInputFromArray(TensorShape({}), {243.0f}); + AddInputFromArray(TensorShape({}), {1.0f}); + AddInputFromArray(TensorShape({}), {256.0f}); TF_ASSERT_OK(RunOpKernel()); // We're requesting C = A.transposed() * B, // so we expect to get these results: @@ -162,12 +162,50 @@ TEST_F(QuantizedMatMulTest, VerySmall_BadRange) { // The B matrix is: // | 1 | AddInputFromArray(TensorShape({b_rows, b_cols}), {0}); - AddInputFromArray(TensorShape({1}), {-12.0f}); - AddInputFromArray(TensorShape({1}), {243.0f}); + AddInputFromArray(TensorShape({}), {-12.0f}); + AddInputFromArray(TensorShape({}), {243.0f}); // Here we set the range so that the min and max are equal, so we expect to // see an error when we run. - AddInputFromArray(TensorShape({1}), {1.0f}); - AddInputFromArray(TensorShape({1}), {1.0f}); + AddInputFromArray(TensorShape({}), {1.0f}); + AddInputFromArray(TensorShape({}), {1.0f}); + EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, RunOpKernel().code()); +} + +// This test multiplies two 1x1 8bit matrices, but sets invalid quantized min +// and max values, so we expect to get an error +TEST_F(QuantizedMatMulTest, VerySmall_BadMinMax) { + // These parameters reflect a typical production usage of eight-bit matmuls + // in an Inception-style network. + const bool transpose_a = true; + const int a_rows = 1; + const int a_cols = 1; + const int b_rows = 1; + const int b_cols = 1; + const bool transpose_b = false; + TF_ASSERT_OK(NodeDefBuilder("quantized_mat_mul_op", "QuantizedMatMul") + .Input(FakeInput(DT_QUINT8)) + .Input(FakeInput(DT_QUINT8)) + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Attr("Toutput", DataTypeToEnum::v()) + .Attr("transpose_a", transpose_a) + .Attr("transpose_b", transpose_b) + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + // The A matrix is: + // | -1 | + AddInputFromArray(TensorShape({a_rows, a_cols}), {11}); + // The B matrix is: + // | 1 | + AddInputFromArray(TensorShape({b_rows, b_cols}), {0}); + // Here we set the error of a non scalar min_a value, so we expect to see an + // error when we run. + AddInputFromArray(TensorShape({1}), {2}); + AddInputFromArray(TensorShape({}), {243.0f}); + AddInputFromArray(TensorShape({}), {1.0f}); + AddInputFromArray(TensorShape({}), {256.0f}); EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, RunOpKernel().code()); } @@ -233,10 +271,10 @@ TEST_F(QuantizedMatMulTest, Small_WithParams) { 3, 6, }); - AddInputFromArray(TensorShape({1}), {-12.0f}); - AddInputFromArray(TensorShape({1}), {243.0f}); - AddInputFromArray(TensorShape({1}), {0}); - AddInputFromArray(TensorShape({1}), {255.0f}); + AddInputFromArray(TensorShape({}), {-12.0f}); + AddInputFromArray(TensorShape({}), {243.0f}); + AddInputFromArray(TensorShape({}), {0}); + AddInputFromArray(TensorShape({}), {255.0f}); TF_ASSERT_OK(RunOpKernel()); // We're requesting C = A.transposed() * B, // so we expect to get these results: @@ -326,10 +364,10 @@ TEST_F(QuantizedMatMulTest, Medium_WithParams) { AddInputFromArray(a_quantized.shape(), a_quantized.flat()); AddInputFromArray(b_quantized.shape(), b_quantized.flat()); - AddInputFromArray(TensorShape({1}), {a_min}); - AddInputFromArray(TensorShape({1}), {a_max}); - AddInputFromArray(TensorShape({1}), {b_min}); - AddInputFromArray(TensorShape({1}), {b_max}); + AddInputFromArray(TensorShape({}), {a_min}); + AddInputFromArray(TensorShape({}), {a_max}); + AddInputFromArray(TensorShape({}), {b_min}); + AddInputFromArray(TensorShape({}), {b_max}); TF_ASSERT_OK(RunOpKernel()); Tensor expected_float(DT_FLOAT, {a_cols, b_cols}); diff --git a/tensorflow/core/kernels/quantized_pooling_ops.cc b/tensorflow/core/kernels/quantized_pooling_ops.cc index 17df3c223676aa..5673fb6ee00a5b 100644 --- a/tensorflow/core/kernels/quantized_pooling_ops.cc +++ b/tensorflow/core/kernels/quantized_pooling_ops.cc @@ -20,11 +20,13 @@ limitations under the License. #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/numeric_op.h" #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/op_requires.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/kernels/ops_util.h" #include "tensorflow/core/kernels/pooling_ops_common.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/util/padding.h" #include "tensorflow/core/util/tensor_format.h" @@ -65,8 +67,20 @@ class QuantizedAvgPoolingOp : public OpKernel { return; } - const float min_input = context->input(1).flat()(0); - const float max_input = context->input(2).flat()(0); + const Tensor& min_input_tensor = context->input(1); + const Tensor& max_input_tensor = context->input(2); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(min_input_tensor.shape()), + errors::InvalidArgument( + "min_input shape must be rank 0 but is rank ", + min_input_tensor.dims(), + ", received shape: ", min_input_tensor.shape())); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(max_input_tensor.shape()), + errors::InvalidArgument( + "max_input shape must be rank 0 but is rank ", + max_input_tensor.dims(), + ", received shape: ", max_input_tensor.shape())); + const float min_input = context->input(1).scalar()(); + const float max_input = context->input(2).scalar()(); OP_REQUIRES(context, params.depth_window == 1, errors::Unimplemented("Non-spatial pooling is not " @@ -117,8 +131,20 @@ class QuantizedMaxPoolingOp : public MaxPoolingOp { : MaxPoolingOp(context) {} void Compute(OpKernelContext* context) override { - const float min_input = context->input(1).flat()(0); - const float max_input = context->input(2).flat()(0); + const Tensor& min_input_tensor = context->input(1); + const Tensor& max_input_tensor = context->input(2); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(min_input_tensor.shape()), + errors::InvalidArgument( + "min_input shape must be rank 0 but is rank ", + min_input_tensor.dims(), + ", received shape: ", min_input_tensor.shape())); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(max_input_tensor.shape()), + errors::InvalidArgument( + "max_input shape must be rank 0 but is rank ", + max_input_tensor.dims(), + ", received shape: ", max_input_tensor.shape())); + const float min_input = context->input(1).scalar()(); + const float max_input = context->input(2).scalar()(); MaxPoolingOp::Compute(context); Tensor* output_min = nullptr; OP_REQUIRES_OK(context, context->allocate_output(1, {}, &output_min)); diff --git a/tensorflow/core/kernels/quantized_pooling_ops_test.cc b/tensorflow/core/kernels/quantized_pooling_ops_test.cc index fc0417e5431b27..9e56890478be24 100644 --- a/tensorflow/core/kernels/quantized_pooling_ops_test.cc +++ b/tensorflow/core/kernels/quantized_pooling_ops_test.cc @@ -69,8 +69,8 @@ TEST_F(QuantizedPoolingTest, SmallAveragePooling) { AddInputFromArray(input_quantized.shape(), input_quantized.flat()); - AddInputFromArray(TensorShape({1}), {input_min}); - AddInputFromArray(TensorShape({1}), {input_max}); + AddInputFromArray(TensorShape({}), {input_min}); + AddInputFromArray(TensorShape({}), {input_max}); TF_ASSERT_OK(RunOpKernel()); const Tensor& output_quantized = *GetOutput(0); const float output_min = GetOutput(1)->flat()(0); @@ -114,8 +114,8 @@ TEST_F(QuantizedPoolingTest, SmallMaxPooling) { AddInputFromArray(input_quantized.shape(), input_quantized.flat()); - AddInputFromArray(TensorShape({1}), {input_min}); - AddInputFromArray(TensorShape({1}), {input_max}); + AddInputFromArray(TensorShape({}), {input_min}); + AddInputFromArray(TensorShape({}), {input_max}); TF_ASSERT_OK(RunOpKernel()); const Tensor& output_quantized = *GetOutput(0); const float output_min = GetOutput(1)->flat()(0); diff --git a/tensorflow/core/kernels/ragged_range_op.cc b/tensorflow/core/kernels/ragged_range_op.cc index 066e5d638bbc43..469ef06b4b3bb6 100644 --- a/tensorflow/core/kernels/ragged_range_op.cc +++ b/tensorflow/core/kernels/ragged_range_op.cc @@ -12,6 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include #include #include #include @@ -78,8 +79,25 @@ class RaggedRangeOp : public OpKernel { T limit = broadcast_limits ? limits(0) : limits(row); T delta = broadcast_deltas ? deltas(0) : deltas(row); OP_REQUIRES(context, delta != 0, InvalidArgument("Requires delta != 0")); - rt_nested_splits(row + 1) = - rt_nested_splits(row) + RangeSize(start, limit, delta); + int64_t size; // The number of elements in the specified range. + if (((delta > 0) && (limit < start)) || + ((delta < 0) && (limit > start))) { + size = 0; + } else if (std::is_integral::value) { + // The following is copied from tensorflow::RangeOp::Compute(). + size = Eigen::divup(Eigen::numext::abs(limit - start), + Eigen::numext::abs(delta)); + } else { + // The following is copied from tensorflow::RangeOp::Compute(). + auto size_auto = + Eigen::numext::ceil(Eigen::numext::abs((limit - start) / delta)); + OP_REQUIRES( + context, size_auto <= std::numeric_limits::max(), + errors::InvalidArgument("Requires ((limit - start) / delta) <= ", + std::numeric_limits::max())); + size = static_cast(size_auto); + } + rt_nested_splits(row + 1) = rt_nested_splits(row) + size; } SPLITS_TYPE nvals = rt_nested_splits(nrows); @@ -99,19 +117,6 @@ class RaggedRangeOp : public OpKernel { } } } - - private: - // Returns the number of elements in the specified range. - SPLITS_TYPE RangeSize(T start, T limit, T delta) { - if (((delta > 0) && (limit < start)) || ((delta < 0) && (limit > start))) { - return 0; - } - // The following is copied from tensorflow::RangeOp::Compute(). - return (std::is_integral::value - ? ((std::abs(limit - start) + std::abs(delta) - 1) / - std::abs(delta)) - : std::ceil(std::abs((limit - start) / delta))); - } }; #define REGISTER_CPU_KERNEL(TYPE) \ diff --git a/tensorflow/core/kernels/ragged_range_op_test.cc b/tensorflow/core/kernels/ragged_range_op_test.cc index 94aaedde3420e0..fc3b302eeb7e30 100644 --- a/tensorflow/core/kernels/ragged_range_op_test.cc +++ b/tensorflow/core/kernels/ragged_range_op_test.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include #include "tensorflow/core/framework/fake_input.h" #include "tensorflow/core/framework/node_def_builder.h" #include "tensorflow/core/framework/shape_inference.h" @@ -77,6 +78,17 @@ TEST_F(RaggedRangeOpTest, FloatValues) { test::AsTensor({0, 2, 4, 6, 5, 6, 5, 4, 3, 2}), 0.1); } +TEST_F(RaggedRangeOpTest, RangeSizeOverflow) { + BuildRaggedRangeGraph(); + AddInputFromArray(TensorShape({2}), {1.1, 0.1}); // starts + AddInputFromArray(TensorShape({2}), {10.0, 1e10}); // limits + AddInputFromArray(TensorShape({2}), {1, 1e-10}); // deltas + + EXPECT_EQ(absl::StrCat("Requires ((limit - start) / delta) <= ", + std::numeric_limits::max()), + RunOpKernel().error_message()); +} + TEST_F(RaggedRangeOpTest, BroadcastDeltas) { BuildRaggedRangeGraph(); AddInputFromArray(TensorShape({3}), {0, 5, 8}); // starts diff --git a/tensorflow/core/kernels/ragged_tensor_to_variant_op.cc b/tensorflow/core/kernels/ragged_tensor_to_variant_op.cc index 6975baa5bcff59..0883bc28893488 100644 --- a/tensorflow/core/kernels/ragged_tensor_to_variant_op.cc +++ b/tensorflow/core/kernels/ragged_tensor_to_variant_op.cc @@ -145,6 +145,10 @@ class RaggedTensorToVariantOp : public OpKernel { batched_ragged_input.mutable_nested_splits()->reserve( ragged_nested_splits_len); for (int i = 0; i < ragged_nested_splits_len; i++) { + OP_REQUIRES(context, ragged_nested_splits_in[i].dims() == 1, + errors::InvalidArgument("Requires nested_row_splits[", i, "]", + " to be rank 1 but is rank ", + ragged_nested_splits_in[i].dims())); batched_ragged_input.append_splits(ragged_nested_splits_in[i]); } diff --git a/tensorflow/core/kernels/random_op.cc b/tensorflow/core/kernels/random_op.cc index 8ec9ed7d24b081..b85c120eba9aa8 100644 --- a/tensorflow/core/kernels/random_op.cc +++ b/tensorflow/core/kernels/random_op.cc @@ -166,7 +166,7 @@ class RandomGammaOp : public OpKernel { } const int64_t samples_per_alpha = samples_shape.num_elements(); - samples_shape.AppendShape(alpha_t.shape()); + OP_REQUIRES_OK(ctx, samples_shape.AppendShapeWithStatus(alpha_t.shape())); // Allocate output samples. Tensor* samples_t = nullptr; OP_REQUIRES_OK(ctx, ctx->allocate_output(0, samples_shape, &samples_t)); diff --git a/tensorflow/core/kernels/random_poisson_op.cc b/tensorflow/core/kernels/random_poisson_op.cc index b4c4d5d95c1881..a14bee790753ca 100644 --- a/tensorflow/core/kernels/random_poisson_op.cc +++ b/tensorflow/core/kernels/random_poisson_op.cc @@ -296,8 +296,8 @@ class RandomPoissonOp : public OpKernel { TensorShape samples_shape; OP_REQUIRES_OK(ctx, tensor::MakeShape(shape_t, &samples_shape)); const int64_t num_samples = samples_shape.num_elements(); + OP_REQUIRES_OK(ctx, samples_shape.AppendShapeWithStatus(rate_t.shape())); - samples_shape.AppendShape(rate_t.shape()); // Allocate output samples. Tensor* samples_t = nullptr; OP_REQUIRES_OK(ctx, ctx->allocate_output(0, samples_shape, &samples_t)); diff --git a/tensorflow/core/kernels/requantize.cc b/tensorflow/core/kernels/requantize.cc index 3259e5ddd096aa..bc5de171639267 100644 --- a/tensorflow/core/kernels/requantize.cc +++ b/tensorflow/core/kernels/requantize.cc @@ -18,9 +18,11 @@ limitations under the License. #define EIGEN_USE_THREADS #include -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" + #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/type_traits.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/kernels/meta_support.h" @@ -38,10 +40,34 @@ class RequantizeOp : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& input = ctx->input(0); - const float input_min_float = ctx->input(1).flat()(0); - const float input_max_float = ctx->input(2).flat()(0); - const float requested_output_min_float = ctx->input(3).flat()(0); - const float requested_output_max_float = ctx->input(4).flat()(0); + + const Tensor& input_min = ctx->input(1); + const Tensor& input_max = ctx->input(2); + const Tensor& requested_output_min = ctx->input(3); + const Tensor& requested_output_max = ctx->input(4); + OP_REQUIRES( + ctx, TensorShapeUtils::IsScalar(input_min.shape()), + errors::InvalidArgument("`input_min` must be rank 0 but is rank ", + input_min.dims())); + OP_REQUIRES( + ctx, TensorShapeUtils::IsScalar(input_max.shape()), + errors::InvalidArgument("`input_max` must be rank 0 but is rank ", + input_max.dims())); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(requested_output_min.shape()), + errors::InvalidArgument( + "`requested_output_min` must be rank 0 but is rank ", + requested_output_min.dims())); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(requested_output_max.shape()), + errors::InvalidArgument( + "`requested_output_max` must be rank 0 but is rank ", + requested_output_max.dims())); + + const float input_min_float = input_min.flat()(0); + const float input_max_float = input_max.flat()(0); + const float requested_output_min_float = + requested_output_min.flat()(0); + const float requested_output_max_float = + requested_output_max.flat()(0); Tensor* output = nullptr; OP_REQUIRES_OK(ctx, ctx->allocate_output(0, input.shape(), &output)); diff --git a/tensorflow/core/kernels/requantize_op_test.cc b/tensorflow/core/kernels/requantize_op_test.cc index 133f92b029d8da..5663520fdbfda9 100644 --- a/tensorflow/core/kernels/requantize_op_test.cc +++ b/tensorflow/core/kernels/requantize_op_test.cc @@ -53,10 +53,10 @@ TEST_F(RequantizeTest, HandCraftedRequantize) { // Requantize to -1 to 1. AddInputFromArray(TensorShape({value_count}), {-(1 << 23), 0, (1 << 23)}); - AddInputFromArray(TensorShape({1}), {-256.0f}); - AddInputFromArray(TensorShape({1}), {256.0f}); - AddInputFromArray(TensorShape({1}), {-1.0f}); - AddInputFromArray(TensorShape({1}), {1.0f}); + AddInputFromArray(TensorShape({}), {-256.0f}); + AddInputFromArray(TensorShape({}), {256.0f}); + AddInputFromArray(TensorShape({}), {-1.0f}); + AddInputFromArray(TensorShape({}), {1.0f}); TF_ASSERT_OK(RunOpKernel()); Tensor expected(allocator(), DT_QUINT8, TensorShape({value_count})); test::FillValues(&expected, {0, 128, 255}); @@ -71,10 +71,10 @@ TEST_F(RequantizeTest, InvalidOutputMin) { AddInputFromArray(TensorShape({value_count}), {-(1 << 23), 0, (1 << 23)}); - AddInputFromArray(TensorShape({1}), {-256.0f}); - AddInputFromArray(TensorShape({1}), {256.0f}); - AddInputFromArray(TensorShape({1}), {0.01f}); - AddInputFromArray(TensorShape({1}), {1.0f}); + AddInputFromArray(TensorShape({}), {-256.0f}); + AddInputFromArray(TensorShape({}), {256.0f}); + AddInputFromArray(TensorShape({}), {0.01f}); + AddInputFromArray(TensorShape({}), {1.0f}); EXPECT_EQ("requested_output_min must be <= 0, but got 0.01", RunOpKernel().error_message()); } @@ -85,10 +85,10 @@ TEST_F(RequantizeTest, InvalidOutputMax) { AddInputFromArray(TensorShape({value_count}), {-(1 << 23), 0, (1 << 23)}); - AddInputFromArray(TensorShape({1}), {-256.0f}); - AddInputFromArray(TensorShape({1}), {256.0f}); - AddInputFromArray(TensorShape({1}), {-10.0f}); - AddInputFromArray(TensorShape({1}), {-11.0f}); + AddInputFromArray(TensorShape({}), {-256.0f}); + AddInputFromArray(TensorShape({}), {256.0f}); + AddInputFromArray(TensorShape({}), {-10.0f}); + AddInputFromArray(TensorShape({}), {-11.0f}); EXPECT_EQ( "requested_output_max must be >= requested_output_min, but got -11 and " "-10", diff --git a/tensorflow/core/kernels/reshape_op.h b/tensorflow/core/kernels/reshape_op.h index cd8ffefdff2274..9f6dd2c156cea2 100644 --- a/tensorflow/core/kernels/reshape_op.h +++ b/tensorflow/core/kernels/reshape_op.h @@ -45,6 +45,11 @@ class ReshapeOp : public OpKernel { TensorShapeUtils::IsScalar(sizes.shape())), errors::InvalidArgument("sizes input must be 1-D, not ", sizes.shape().DebugString())); + OP_REQUIRES( + context, sizes.NumElements() < TensorShape::MaxDimensions(), + errors::InvalidArgument("too many dimensions: must be < ", + TensorShape::MaxDimensions(), ", but received ", + sizes.NumElements())); // Compute the output shape. Determine product of specified // dimensions, and find the index of the unspecified one. diff --git a/tensorflow/core/kernels/reshape_util.cc b/tensorflow/core/kernels/reshape_util.cc index 71d93302a59a45..442faa49187fcf 100644 --- a/tensorflow/core/kernels/reshape_util.cc +++ b/tensorflow/core/kernels/reshape_util.cc @@ -23,8 +23,10 @@ limitations under the License. #include #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/op_requires.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_util.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/lib/gtl/inlined_vector.h" @@ -99,7 +101,9 @@ void ReshapeSparseTensor(OpKernelContext *context, target_shape_in.shape().DebugString())); const int64_t output_rank = target_shape_in.NumElements(); - const TensorShape input_shape(input_shape_in.vec()); + TensorShape input_shape; + OP_REQUIRES_OK(context, TensorShape::BuildTensorShape( + input_shape_in.vec(), &input_shape)); const int64_t dense_size = input_shape.num_elements(); const int64_t nnz = input_indices_in.shape().dim_size(0); diff --git a/tensorflow/core/kernels/rnn/lstm_ops.cc b/tensorflow/core/kernels/rnn/lstm_ops.cc index 711fc8f08275d8..b0a27c1914af74 100644 --- a/tensorflow/core/kernels/rnn/lstm_ops.cc +++ b/tensorflow/core/kernels/rnn/lstm_ops.cc @@ -416,6 +416,65 @@ class LSTMBlockCellOp : public OpKernel { const Device& device = ctx->eigen_device(); + // Sanity check that each of the tensors have the required NDIMS. + OP_REQUIRES(ctx, x_tensor->dims() == 2, + errors::InvalidArgument("x_tensor must be rank 2 but is rank ", + x_tensor->dims(), ".")); + OP_REQUIRES( + ctx, cs_prev_tensor->dims() == 2, + errors::InvalidArgument("cs_prev_tensor must be rank 2 but is rank ", + cs_prev_tensor->dims(), ".")); + OP_REQUIRES( + ctx, h_prev_tensor->dims() == 2, + errors::InvalidArgument("h_prev_tensor must be rank 2 but is rank ", + h_prev_tensor->dims(), ".")); + OP_REQUIRES(ctx, w_tensor->dims() == 2, + errors::InvalidArgument("w_tensor must be rank 2 but is rank ", + w_tensor->dims(), ".")); + OP_REQUIRES( + ctx, wci_tensor->dims() == 1, + errors::InvalidArgument("wci_tensor must be rank 1 but is rank ", + wci_tensor->dims(), ".")); + OP_REQUIRES( + ctx, wcf_tensor->dims() == 1, + errors::InvalidArgument("wcf_tensor must be rank 1 but is rank ", + wci_tensor->dims(), ".")); + OP_REQUIRES( + ctx, wco_tensor->dims() == 1, + errors::InvalidArgument("wco_tensor must be rank 1 but is rank ", + wco_tensor->dims(), ".")); + OP_REQUIRES(ctx, b_tensor->dims() == 1, + errors::InvalidArgument("b_tensor must be rank 1 but is rank ", + b_tensor->dims(), ".")); + OP_REQUIRES(ctx, xh_tensor.dims() == 2, + errors::InvalidArgument("xh_tensor must be rank 2 but is rank ", + xh_tensor.dims(), ".")); + OP_REQUIRES(ctx, i_tensor->dims() == 2, + errors::InvalidArgument("i_tensor must be rank 2 but is rank ", + i_tensor->dims(), ".")); + OP_REQUIRES(ctx, cs_tensor->dims() == 2, + errors::InvalidArgument("cs_tensor must be rank 2 but is rank ", + cs_tensor->dims(), ".")); + OP_REQUIRES(ctx, f_tensor->dims() == 2, + errors::InvalidArgument("f_tensor must be rank 2 but is rank ", + f_tensor->dims(), ".")); + OP_REQUIRES(ctx, o_tensor->dims() == 2, + errors::InvalidArgument("o_tensor must be rank 2 but is rank ", + o_tensor->dims(), ".")); + OP_REQUIRES(ctx, ci_tensor->dims() == 2, + errors::InvalidArgument("ci_tensor must be rank 2 but is rank ", + ci_tensor->dims(), ".")); + OP_REQUIRES(ctx, co_tensor->dims() == 2, + errors::InvalidArgument("co_tensor must be rank 2 but is rank ", + co_tensor->dims(), ".")); + OP_REQUIRES( + ctx, gates_tensor.dims() == 2, + errors::InvalidArgument("gates_tensor must be rank 2 but is rank ", + gates_tensor.dims(), ".")); + OP_REQUIRES(ctx, h_tensor->dims() == 2, + errors::InvalidArgument("h_tensor must be rank 2 but is rank ", + h_tensor->dims(), ".")); + functor::LSTMBlockCellFprop( batch_size, input_size, cell_size)( ctx, device, forget_bias_, cell_clip_, use_peephole_, @@ -1079,19 +1138,30 @@ class BlockLSTMGradOp : public OpKernel { const Tensor* x; OP_REQUIRES_OK(ctx, ctx->input("x", &x)); - OP_REQUIRES(ctx, x->dims() == 3, errors::InvalidArgument("x must be 3D")); + OP_REQUIRES( + ctx, x->dims() == 3, + errors::InvalidArgument("x must be rank 3 but is rank ", x->dims())); const int64_t timelen = x->dim_size(0); const int64_t batch_size = x->dim_size(1); const int64_t input_size = x->dim_size(2); const Tensor* cs_prev_tensor = nullptr; OP_REQUIRES_OK(ctx, ctx->input("cs_prev", &cs_prev_tensor)); + OP_REQUIRES(ctx, cs_prev_tensor->dims() == 2, + errors::InvalidArgument("cs_prev must be rank 2 but is rank ", + cs_prev_tensor->dims())); const Tensor* h_prev_tensor = nullptr; OP_REQUIRES_OK(ctx, ctx->input("h_prev", &h_prev_tensor)); + OP_REQUIRES(ctx, h_prev_tensor->dims() == 2, + errors::InvalidArgument("h_prev must be rank 2 but is rank ", + h_prev_tensor->dims())); const Tensor* w_tensor = nullptr; OP_REQUIRES_OK(ctx, ctx->input("w", &w_tensor)); + OP_REQUIRES(ctx, w_tensor->dims() == 2, + errors::InvalidArgument("w must be rank 2 but is rank ", + w_tensor->dims())); const int64_t cell_size = w_tensor->dim_size(1) / 4; OP_REQUIRES(ctx, input_size + cell_size == w_tensor->dim_size(0), errors::InvalidArgument( @@ -1100,15 +1170,27 @@ class BlockLSTMGradOp : public OpKernel { const Tensor* wci_tensor = nullptr; OP_REQUIRES_OK(ctx, ctx->input("wci", &wci_tensor)); + OP_REQUIRES(ctx, wci_tensor->dims() == 1, + errors::InvalidArgument("wci must be rank 1 but is rank ", + wci_tensor->dims())); const Tensor* wcf_tensor = nullptr; OP_REQUIRES_OK(ctx, ctx->input("wcf", &wcf_tensor)); + OP_REQUIRES(ctx, wcf_tensor->dims() == 1, + errors::InvalidArgument("wcf must be rank 1 but is rank ", + wcf_tensor->dims())); const Tensor* wco_tensor = nullptr; OP_REQUIRES_OK(ctx, ctx->input("wco", &wco_tensor)); + OP_REQUIRES(ctx, wco_tensor->dims() == 1, + errors::InvalidArgument("wco must be rank 1 but is rank ", + wco_tensor->dims())); const Tensor* b_tensor = nullptr; OP_REQUIRES_OK(ctx, ctx->input("b", &b_tensor)); + OP_REQUIRES(ctx, b_tensor->dims() == 1, + errors::InvalidArgument("b must be rank 1 but is rank ", + b_tensor->dims())); OP_REQUIRES( ctx, cell_size == b_tensor->dim_size(0) / 4, errors::InvalidArgument("w and b cell_size don't match: ", cell_size, diff --git a/tensorflow/core/kernels/searchsorted_op.cc b/tensorflow/core/kernels/searchsorted_op.cc index 019e704738f7dd..5c06ce1e4bfd43 100644 --- a/tensorflow/core/kernels/searchsorted_op.cc +++ b/tensorflow/core/kernels/searchsorted_op.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/kernels/fill_functor.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/types.h" @@ -115,6 +116,14 @@ class UpperBoundOp : public OpKernel { auto output = output_t->template flat(); const auto sorted_inputs = sorted_inputs_t.template flat(); const auto values = values_t.template flat(); + + // For empty inputs, all values will be placed at the zeroth position. + if (sorted_inputs.size() == 0) { + functor::SetZeroFunctor set_zero; + set_zero(ctx->eigen_device(), output); + return; + } + OP_REQUIRES_OK( ctx, functor::UpperBoundFunctor::Compute( ctx, sorted_inputs, values, sorted_inputs_t.dim_size(0), @@ -160,6 +169,14 @@ class LowerBoundOp : public OpKernel { auto output = output_t->template flat(); const auto sorted_inputs = sorted_inputs_t.template flat(); const auto values = values_t.template flat(); + + // For empty inputs, all values will be placed at the zeroth position. + if (sorted_inputs.size() == 0) { + functor::SetZeroFunctor set_zero; + set_zero(ctx->eigen_device(), output); + return; + } + OP_REQUIRES_OK( ctx, functor::LowerBoundFunctor::Compute( ctx, sorted_inputs, values, sorted_inputs_t.dim_size(0), diff --git a/tensorflow/core/kernels/segment_reduction_ops_impl.h b/tensorflow/core/kernels/segment_reduction_ops_impl.h index a44ede6dbe5319..da58e3cb252fc0 100644 --- a/tensorflow/core/kernels/segment_reduction_ops_impl.h +++ b/tensorflow/core/kernels/segment_reduction_ops_impl.h @@ -18,6 +18,10 @@ limitations under the License. #ifndef TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_IMPL_H_ #define TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_IMPL_H_ +#include + +#include "tensorflow/core/framework/op_requires.h" +#include "tensorflow/core/platform/types.h" #define EIGEN_USE_THREADS #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define EIGEN_USE_GPU @@ -474,6 +478,7 @@ class SparseSegmentReductionOpBase : public OpKernel { bool is_mean, bool is_sqrtn, bool has_num_segments, T default_value) : OpKernel(context), + dtidx_(DataTypeToEnum::v()), is_mean_(is_mean), is_sqrtn_(is_sqrtn), has_num_segments_(has_num_segments), @@ -503,10 +508,20 @@ class SparseSegmentReductionOpBase : public OpKernel { const auto segment_vec = segment_ids.vec(); // Note that the current implementation assumes that segment_vec values are // sorted. + const SegmentId last_segment_id = + num_indices > 0 ? segment_vec(num_indices - 1) : 0; + int64_t limit = dtidx_ == DataType::DT_INT32 ? kint32max : kint64max; + + OP_REQUIRES( + context, last_segment_id < limit, + errors::InvalidArgument("Last segment id must be < kintmax, got ", + last_segment_id, " limit ", limit)); + const SegmentId last_segment_id_plus_one = num_indices > 0 ? internal::SubtleMustCopy(segment_vec(num_indices - 1)) + 1 : 0; + if (has_num_segments_) { OP_REQUIRES( context, output_rows >= last_segment_id_plus_one, @@ -518,7 +533,7 @@ class SparseSegmentReductionOpBase : public OpKernel { errors::InvalidArgument("segment ids must be >= 0")); TensorShape output_shape = input.shape(); - output_shape.set_dim(0, output_rows); + OP_REQUIRES_OK(context, output_shape.SetDimWithStatus(0, output_rows)); // Note that we do not initialize the output buffer with a default value, so // we need to explicitly set missing indices to the default value. @@ -605,6 +620,7 @@ class SparseSegmentReductionOpBase : public OpKernel { } private: + const DataType dtidx_; template using EnableIfBfloat16OrHalf = typename std::enable_if::value || @@ -1098,7 +1114,7 @@ class SparseSegmentGradOpBase : public OpKernel { const auto segment_vec = segment_ids.vec(); TensorShape output_shape = input.shape(); - output_shape.set_dim(0, M); + OP_REQUIRES_OK(context, output_shape.SetDimWithStatus(0, M)); Tensor* output = nullptr; OP_REQUIRES_OK(context, context->allocate_output(0, output_shape, &output)); if (M == 0 || N == 0) return; diff --git a/tensorflow/core/kernels/sequence_ops.cc b/tensorflow/core/kernels/sequence_ops.cc index bdcf83eb3761f2..12a8785d8d17ee 100644 --- a/tensorflow/core/kernels/sequence_ops.cc +++ b/tensorflow/core/kernels/sequence_ops.cc @@ -71,13 +71,19 @@ class RangeOp : public OpKernel { errors::InvalidArgument( "Requires start >= limit when delta < 0: ", start, "/", limit)); } - int64_t size = 0; - if (std::is_integral::value) { - size = static_cast( - (std::abs(limit - start) + std::abs(delta) - 1) / std::abs(delta)); - } else { - size = static_cast(std::ceil(std::abs((limit - start) / delta))); - } + auto size_auto = (std::is_integral::value + ? (Eigen::numext::abs(limit - start) + + Eigen::numext::abs(delta) - T(1)) / + Eigen::numext::abs(delta) + : Eigen::numext::ceil( + Eigen::numext::abs((limit - start) / delta))); + OP_REQUIRES( + context, size_auto <= std::numeric_limits::max(), + errors::InvalidArgument("Requires ((limit - start) / delta) <= ", + std::numeric_limits::max())); + + int64 size = static_cast(size_auto); + TensorShape shape; OP_REQUIRES_OK(context, shape.AddDimWithStatus(size)); Tensor* out = nullptr; diff --git a/tensorflow/core/kernels/serialize_sparse_op.cc b/tensorflow/core/kernels/serialize_sparse_op.cc index 3e386746a0b31c..a7d64105afe11b 100644 --- a/tensorflow/core/kernels/serialize_sparse_op.cc +++ b/tensorflow/core/kernels/serialize_sparse_op.cc @@ -23,9 +23,11 @@ limitations under the License. #include "tensorflow/core/common_runtime/dma_helper.h" #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/op_requires.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor.pb.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_util.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/variant.h" @@ -366,7 +368,10 @@ class SerializeManySparseOp : public OpKernel { errors::InvalidArgument( "Rank of input SparseTensor should be > 1, but saw rank: ", rank)); - TensorShape tensor_input_shape(input_shape->vec()); + TensorShape tensor_input_shape; + OP_REQUIRES_OK(context, + TensorShape::BuildTensorShape(input_shape->vec(), + &tensor_input_shape)); gtl::InlinedVector std_order(rank); std::iota(std_order.begin(), std_order.end(), 0); SparseTensor input_st; diff --git a/tensorflow/core/kernels/session_ops.cc b/tensorflow/core/kernels/session_ops.cc index 11f991e1ba34fe..2feb79a24e329e 100644 --- a/tensorflow/core/kernels/session_ops.cc +++ b/tensorflow/core/kernels/session_ops.cc @@ -98,6 +98,8 @@ class GetSessionTensorOp : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& handle = ctx->input(0); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(handle.shape()), + errors::InvalidArgument("handle must be scalar")); const string& name = handle.scalar()(); Tensor val; auto session_state = ctx->session_state(); @@ -132,6 +134,8 @@ class DeleteSessionTensorOp : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& handle = ctx->input(0); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(handle.shape()), + errors::InvalidArgument("`handle` must be scalar")); const string& name = handle.scalar()(); auto session_state = ctx->session_state(); OP_REQUIRES(ctx, session_state != nullptr, diff --git a/tensorflow/core/kernels/set_kernels.cc b/tensorflow/core/kernels/set_kernels.cc index d18fbcc0536c8e..f9cac9dc3722bf 100644 --- a/tensorflow/core/kernels/set_kernels.cc +++ b/tensorflow/core/kernels/set_kernels.cc @@ -37,6 +37,7 @@ limitations under the License. #include "tensorflow/core/framework/types.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/errors.h" #include "tensorflow/core/util/sparse/sparse_tensor.h" namespace tensorflow { @@ -69,8 +70,13 @@ Status SparseTensorFromContext(OpKernelContext* ctx, const int32_t base_index, bool validate_indices, sparse::SparseTensor* tensor) { // Assume row-major order. - const TensorShape shape = - TensorShape(ctx->input(base_index + 2).vec()); + TensorShape shape; + const Tensor& shape_tensor = ctx->input(base_index + 2); + if (shape_tensor.dims() != 1) { + return errors::InvalidArgument("Shape must be a 1D tensor."); + } + TF_RETURN_IF_ERROR( + TensorShape::BuildTensorShape(shape_tensor.vec(), &shape)); CheckRankAtLeast2(ctx, shape); std::vector order(shape.dims()); std::iota(order.begin(), order.end(), 0); diff --git a/tensorflow/core/kernels/sobol_op.cc b/tensorflow/core/kernels/sobol_op.cc index 94fff6baea2688..ea3de5056f9e72 100644 --- a/tensorflow/core/kernels/sobol_op.cc +++ b/tensorflow/core/kernels/sobol_op.cc @@ -24,6 +24,7 @@ limitations under the License. #include "sobol_data.h" // from @sobol_data #include "tensorflow/core/framework/device_base.h" #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/lib/core/threadpool.h" #include "tensorflow/core/platform/platform_strings.h" @@ -134,8 +135,14 @@ class SobolSampleOp : public OpKernel { : OpKernel(context) {} void Compute(OpKernelContext* context) override { + OP_REQUIRES(context, TensorShapeUtils::IsScalar(context->input(0).shape()), + errors::InvalidArgument("dim must be a scalar")); int32_t dim = context->input(0).scalar()(); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(context->input(0).shape()), + errors::InvalidArgument("num_results must be a scalar")); int32_t num_results = context->input(1).scalar()(); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(context->input(0).shape()), + errors::InvalidArgument("skip must be a scalar")); int32_t skip = context->input(2).scalar()(); OP_REQUIRES(context, dim >= 1, diff --git a/tensorflow/core/kernels/spacetobatch_op.cc b/tensorflow/core/kernels/spacetobatch_op.cc index 009500f79268fc..e391529d852b04 100644 --- a/tensorflow/core/kernels/spacetobatch_op.cc +++ b/tensorflow/core/kernels/spacetobatch_op.cc @@ -21,8 +21,6 @@ limitations under the License. #include #include -#include "tensorflow/core/kernels/spacetobatch_functor.h" - #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/op_kernel.h" @@ -31,8 +29,10 @@ limitations under the License. #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/framework/types.h" +#include "tensorflow/core/kernels/spacetobatch_functor.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/overflow.h" namespace tensorflow { @@ -99,7 +99,13 @@ Status SpaceToBatchOpCompute(OpKernelContext* context, // Compute the product of the block_shape values. int64_t block_shape_product = 1; for (int block_dim = 0; block_dim < block_dims; ++block_dim) { - block_shape_product *= block_shape[block_dim]; + if (block_shape[block_dim] < 1) { + return errors::InvalidArgument( + "All values in block_shape must be positive, got value, ", + block_shape[block_dim], " at index ", block_dim, "."); + } + block_shape_product = + MultiplyWithoutOverflow(block_shape_product, block_shape[block_dim]); } if (block_shape_product <= 0) { return errors::InvalidArgument( @@ -131,8 +137,14 @@ Status SpaceToBatchOpCompute(OpKernelContext* context, // The actual output shape exposed to callers. TensorShape external_output_shape; - external_output_shape.AddDim(orig_input_tensor.dim_size(0) * - block_shape_product); + const int64_t output_shape = MultiplyWithoutOverflow( + orig_input_tensor.dim_size(0), block_shape_product); + if (output_shape < 0) { + return errors::InvalidArgument( + "Negative output dimension size caused by overflow when multiplying ", + orig_input_tensor.dim_size(0), " and ", block_shape_product); + } + external_output_shape.AddDim(output_shape); int64_t input_batch_size = orig_input_tensor.dim_size(0); for (int block_dim = 0; block_dim < removed_prefix_block_dims; ++block_dim) { diff --git a/tensorflow/core/kernels/sparse/sparse_tensor_to_csr_sparse_matrix_op.cc b/tensorflow/core/kernels/sparse/sparse_tensor_to_csr_sparse_matrix_op.cc index 942a538db0004d..3fa79fe2019b15 100644 --- a/tensorflow/core/kernels/sparse/sparse_tensor_to_csr_sparse_matrix_op.cc +++ b/tensorflow/core/kernels/sparse/sparse_tensor_to_csr_sparse_matrix_op.cc @@ -67,6 +67,13 @@ class SparseTensorToCSRSparseMatrixCPUOp : public OpKernel { const Tensor& values = ctx->input(1); const Tensor& dense_shape = ctx->input(2); const int rank = dense_shape.NumElements(); + OP_REQUIRES( + ctx, TensorShapeUtils::IsVector(dense_shape.shape()), + errors::InvalidArgument("dense_shape must be rank 1 but got rank", + dense_shape.shape().dims())); + OP_REQUIRES(ctx, TensorShapeUtils::IsMatrix(indices.shape()), + errors::InvalidArgument("indices must be rank 2 but got rank", + indices.shape().dims())); OP_REQUIRES(ctx, rank == 2 || rank == 3, errors::InvalidArgument("SparseTensor must have rank 2 or 3; ", "but indices has rank: ", rank)); @@ -76,10 +83,18 @@ class SparseTensorToCSRSparseMatrixCPUOp : public OpKernel { const int64_t total_nnz = values.NumElements(); // Allocate output Tensors. - Tensor batch_ptr(cpu_allocator(), DT_INT32, TensorShape({batch_size + 1})); - Tensor csr_col_ind(cpu_allocator(), DT_INT32, TensorShape({total_nnz})); - Tensor csr_row_ptr(cpu_allocator(), DT_INT32, - TensorShape({(num_rows + 1) * batch_size})); + TensorShape batch_ptr_shape; + OP_REQUIRES_OK( + ctx, TensorShape::BuildTensorShape({batch_size + 1}, &batch_ptr_shape)); + Tensor batch_ptr(cpu_allocator(), DT_INT32, batch_ptr_shape); + TensorShape csr_col_ind_shape; + OP_REQUIRES_OK( + ctx, TensorShape::BuildTensorShape({total_nnz}, &csr_col_ind_shape)); + Tensor csr_col_ind(cpu_allocator(), DT_INT32, csr_col_ind_shape); + TensorShape csr_row_ind_shape; + OP_REQUIRES_OK(ctx, TensorShape::BuildTensorShape( + {(num_rows + 1) * batch_size}, &csr_row_ind_shape)); + Tensor csr_row_ptr(cpu_allocator(), DT_INT32, csr_row_ind_shape); // Fill the row pointers with zeros. functor::SetZeroFunctor set_zero; diff --git a/tensorflow/core/kernels/sparse_cross_op.cc b/tensorflow/core/kernels/sparse_cross_op.cc index da9fe3bf0c2bc8..9833db11024ba8 100644 --- a/tensorflow/core/kernels/sparse_cross_op.cc +++ b/tensorflow/core/kernels/sparse_cross_op.cc @@ -24,12 +24,14 @@ limitations under the License. #include "tensorflow/core/framework/kernel_def_builder.h" #include "tensorflow/core/framework/op_def_builder.h" #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/op_requires.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/platform/errors.h" #include "tensorflow/core/platform/fingerprint.h" #include "tensorflow/core/platform/strong_hash.h" #include "tensorflow/core/util/work_sharder.h" @@ -832,6 +834,10 @@ class SparseCrossV2Op : public OpKernel { const Tensor* sep_t; OP_REQUIRES_OK(context, context->input("sep", &sep_t)); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(sep_t->shape()), + errors::InvalidArgument("Input separator should be a scalar. " + "Received: ", + sep_t->DebugString())); const tstring separator = sep_t->scalar()(); std::vector>> columns = diff --git a/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc b/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc index 4dbe6d8133bcc2..979a947f8879de 100644 --- a/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc +++ b/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc @@ -78,15 +78,30 @@ class SparseDenseBinaryOpShared : public OpKernel { "but received shapes: ", values_t->shape().DebugString(), " and ", shape_t->shape().DebugString())); + OP_REQUIRES( + ctx, TensorShapeUtils::IsVector(shape_t->shape()), + errors::InvalidArgument("Input sp_shape must be a vector. Got: ", + shape_t->shape().DebugString())); OP_REQUIRES( ctx, values_t->dim_size(0) == indices_t->dim_size(0), errors::InvalidArgument( "The first dimension of values and indices should match. (", values_t->dim_size(0), " vs. ", indices_t->dim_size(0), ")")); + OP_REQUIRES( + ctx, shape_t->shape().dim_size(0) == indices_t->shape().dim_size(1), + errors::InvalidArgument( + "Number of dimensions must match second dimension of indices. ", + "Got ", shape_t->shape().dim_size(0), + " dimensions, indices shape: ", indices_t->shape().DebugString())); + OP_REQUIRES(ctx, shape_t->NumElements() > 0, + errors::InvalidArgument( + "The shape argument requires at least one element.")); const auto indices_mat = indices_t->matrix(); const auto shape_vec = shape_t->vec(); - const auto lhs_dims = BCast::FromShape(TensorShape(shape_vec)); + TensorShape lhs_shape; + OP_REQUIRES_OK(ctx, TensorShape::BuildTensorShape(shape_vec, &lhs_shape)); + const auto lhs_dims = BCast::FromShape(lhs_shape); const auto rhs_dims = BCast::FromShape(dense_t->shape()); BCast b(lhs_dims, rhs_dims, false); // false for keeping the same num dims. diff --git a/tensorflow/core/kernels/sparse_reduce_op.cc b/tensorflow/core/kernels/sparse_reduce_op.cc index a4a3cb9ededde7..dbf421a0db6f48 100644 --- a/tensorflow/core/kernels/sparse_reduce_op.cc +++ b/tensorflow/core/kernels/sparse_reduce_op.cc @@ -18,8 +18,10 @@ limitations under the License. #define EIGEN_USE_THREADS #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/op_requires.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_util.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/util/sparse/sparse_tensor.h" @@ -172,10 +174,13 @@ class SparseReduceOp : public OpKernel { // making deep copies here. Remove this if/when we change Reorder()'s // semantics. const auto shape_vec = shape_t->vec(); + TensorShape shape; + OP_REQUIRES_OK(ctx, TensorShape::BuildTensorShape(shape_vec, &shape)); + SparseTensor sp; OP_REQUIRES_OK(ctx, SparseTensor::Create( tensor::DeepCopy(*indices_t), tensor::DeepCopy(*values_t), - TensorShape(shape_vec), &sp)); + shape, &sp)); ReduceDetails reduction = SparseTensorReduceHelper( sp, reduction_axes_t->flat(), keep_dims_); @@ -275,10 +280,13 @@ class SparseReduceSparseOp : public OpKernel { OP_REQUIRES_OK(ctx, ValidateInputs(shape_t, reduction_axes_t)); + TensorShape shape; + OP_REQUIRES_OK(ctx, TensorShape::BuildTensorShape(shape_t->vec(), + &shape)); SparseTensor sp; OP_REQUIRES_OK(ctx, SparseTensor::Create(tensor::DeepCopy(*indices_t), tensor::DeepCopy(*values_t), - TensorShape(shape_t->vec()), &sp)); + shape, &sp)); ReduceDetails reduction = SparseTensorReduceHelper( sp, reduction_axes_t->flat(), keep_dims_); diff --git a/tensorflow/core/kernels/sparse_slice_op.cc b/tensorflow/core/kernels/sparse_slice_op.cc index b0aa0efe833c8b..205429afe77ffa 100644 --- a/tensorflow/core/kernels/sparse_slice_op.cc +++ b/tensorflow/core/kernels/sparse_slice_op.cc @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/util/sparse/sparse_tensor.h" namespace tensorflow { @@ -39,27 +40,38 @@ struct SparseSliceFunctor { const int input_dims = input_shape.NumElements(); sparse::SparseTensor sparse_tensor; - OP_REQUIRES_OK( - context, sparse::SparseTensor::Create( - input_indices, input_values, - TensorShape(input_shape.vec()), &sparse_tensor)); + TensorShape sparse_tensor_shape; + OP_REQUIRES_OK(context, + TensorShapeBase::BuildTensorShapeBase( + input_shape.vec(), &sparse_tensor_shape)); + OP_REQUIRES_OK(context, sparse::SparseTensor::Create( + input_indices, input_values, + sparse_tensor_shape, &sparse_tensor)); const gtl::ArraySlice start(input_start.flat().data(), input_dims); const gtl::ArraySlice size(input_size.flat().data(), input_dims); - const sparse::SparseTensor output = + const StatusOr output_or = sparse::SparseTensor::Slice(sparse_tensor, start, size); + OP_REQUIRES_OK(context, output_or.status()); + auto output = output_or.ValueOrDie(); context->set_output(0, output.indices()); context->set_output(1, output.values()); - const TensorShape output_shape(output.shape()); + TensorShape output_shape; + OP_REQUIRES_OK(context, TensorShapeBase::BuildTensorShapeBase( + output.shape(), &output_shape)); + + TensorShape allocated_shape; + OP_REQUIRES_OK(context, TensorShapeBase::BuildTensorShapeBase( + {output_shape.dims()}, &allocated_shape)); Tensor* shape = nullptr; OP_REQUIRES_OK(context, - context->allocate_output(2, {output_shape.dims()}, &shape)); + context->allocate_output(2, allocated_shape, &shape)); for (int dim = 0; dim < output_shape.dims(); ++dim) { shape->vec()(dim) = output_shape.dim_size(dim); } diff --git a/tensorflow/core/kernels/sparse_softmax_op.cc b/tensorflow/core/kernels/sparse_softmax_op.cc index 062ff77dadd4bf..8a381345baa506 100644 --- a/tensorflow/core/kernels/sparse_softmax_op.cc +++ b/tensorflow/core/kernels/sparse_softmax_op.cc @@ -21,6 +21,7 @@ limitations under the License. #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/op_requires.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_util.h" @@ -62,14 +63,16 @@ class SparseSoftmaxOp : public OpKernel { errors::InvalidArgument( "Input should have rank >= 2, but received shape: ", shape_t->SummarizeValue(3))); + TensorShape shape; + OP_REQUIRES_OK(context, TensorShape::BuildTensorShape( + shape_t->flat(), &shape)); const int64_t nnz = indices_t->dim_size(0); const int rank = static_cast(indices_t->dim_size(1)); SparseTensor st; OP_REQUIRES_OK( - context, SparseTensor::Create( - tensor::DeepCopy(*indices_t), tensor::DeepCopy(*values_t), - TensorShape(shape_t->flat()), &st)); + context, SparseTensor::Create(tensor::DeepCopy(*indices_t), + tensor::DeepCopy(*values_t), shape, &st)); Tensor *output_values = nullptr; OP_REQUIRES_OK(context, context->allocate_output(0, TensorShape({nnz}), diff --git a/tensorflow/core/kernels/sparse_split_op.cc b/tensorflow/core/kernels/sparse_split_op.cc index ebbf7e2a04c474..884498041e9bf3 100644 --- a/tensorflow/core/kernels/sparse_split_op.cc +++ b/tensorflow/core/kernels/sparse_split_op.cc @@ -30,11 +30,15 @@ class SparseSplitOp : public OpKernel { } void Compute(OpKernelContext* context) override { - const int64_t axis_input = context->input(0).scalar()(); + const Tensor& input_axis = context->input(0); const Tensor& input_indices = context->input(1); const Tensor& input_values = context->input(2); const Tensor& input_shape = context->input(3); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(input_axis.shape()), + errors::InvalidArgument( + "Input axis should be a scalar but received shape ", + input_axis.shape().DebugString())); OP_REQUIRES(context, TensorShapeUtils::IsMatrix(input_indices.shape()), errors::InvalidArgument( "Input indices should be a matrix but received shape ", @@ -48,6 +52,7 @@ class SparseSplitOp : public OpKernel { "Input shape should be a vector but received shape ", input_shape.shape().DebugString())); + const int64_t axis_input = input_axis.scalar()(); const int64_t input_rank = input_shape.vec().size(); const int64_t axis = (axis_input < 0) ? input_rank + axis_input : axis_input; diff --git a/tensorflow/core/kernels/sparse_tensor_dense_add_op.cc b/tensorflow/core/kernels/sparse_tensor_dense_add_op.cc index 48803e4b939800..6d6b05bf70f30a 100644 --- a/tensorflow/core/kernels/sparse_tensor_dense_add_op.cc +++ b/tensorflow/core/kernels/sparse_tensor_dense_add_op.cc @@ -18,6 +18,7 @@ limitations under the License. #include "tensorflow/core/kernels/sparse_tensor_dense_add_op.h" #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/op_requires.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_util.h" @@ -47,6 +48,17 @@ Status ValidateInputs(const Tensor *a_indices, const Tensor *a_values, a_values->shape().DebugString(), " and ", a_shape->shape().DebugString()); } + int64_t nnz = a_indices->dim_size(0); + int64_t ndims = a_indices->dim_size(1); + if (a_values->dim_size(0) != nnz) { + return errors::InvalidArgument("Dimensions ", nnz, " and ", + a_values->dim_size(0), + " are not compatible"); + } + if (a_shape->dim_size(0) != ndims) { + return errors::InvalidArgument("Dimensions ", ndims, " and ", + a_shape->dim_size(0), " are not compatible"); + } if (a_shape->NumElements() != b->dims()) { return errors::InvalidArgument( "Two operands have different ranks; received: ", a_shape->NumElements(), @@ -61,6 +73,24 @@ Status ValidateInputs(const Tensor *a_indices, const Tensor *a_values, a_shape_flat(i), " vs dense side ", b->dim_size(i)); } } + + // Check for invalid indices. + const auto a_indices_mat = a_indices->flat_inner_dims(); + + for (int64_t zidx = 0; zidx < nnz; ++zidx) { + for (int64_t didx = 0; didx < ndims; ++didx) { + const Index idx = a_indices_mat(zidx, didx); + if (idx < 0 || idx >= a_shape_flat(didx)) { + return errors::InvalidArgument( + "Sparse tensor has an invalid index on dimension ", didx, + ": " + "a_indices(", + zidx, ",", didx, ") = ", idx, + ", dense tensor shape: ", a_shape_flat); + } + } + } + return Status::OK(); } diff --git a/tensorflow/core/kernels/sparse_tensors_map_ops.cc b/tensorflow/core/kernels/sparse_tensors_map_ops.cc index 04efed5fd90c75..b486a2b4dc3c92 100644 --- a/tensorflow/core/kernels/sparse_tensors_map_ops.cc +++ b/tensorflow/core/kernels/sparse_tensors_map_ops.cc @@ -231,16 +231,29 @@ class AddManySparseToTensorsMapOp : public SparseTensorAccessingOp { errors::InvalidArgument( "Input indices should be a matrix but received shape ", input_indices->shape().DebugString())); - OP_REQUIRES(context, TensorShapeUtils::IsVector(input_values->shape()), errors::InvalidArgument( "Input values should be a vector but received shape ", input_values->shape().DebugString())); - OP_REQUIRES(context, TensorShapeUtils::IsVector(input_shape->shape()), errors::InvalidArgument( "Input shape should be a vector but received shape ", input_shape->shape().DebugString())); + OP_REQUIRES( + context, + input_values->shape().dim_size(0) == input_indices->shape().dim_size(0), + errors::InvalidArgument( + "Number of values must match first dimension of indices. ", "Got ", + input_values->shape().dim_size(0), + " values, indices shape: ", input_indices->shape().DebugString())); + OP_REQUIRES( + context, + input_shape->shape().dim_size(0) == input_indices->shape().dim_size(1), + errors::InvalidArgument( + "Number of dimensions must match second dimension of indices. ", + "Got ", input_shape->shape().dim_size(0), + " dimensions, indices shape: ", + input_indices->shape().DebugString())); int rank = input_shape->NumElements(); @@ -250,22 +263,10 @@ class AddManySparseToTensorsMapOp : public SparseTensorAccessingOp { "Rank of input SparseTensor should be > 1, but saw rank: ", rank)); auto input_shape_vec = input_shape->vec(); - int new_num_elements = 1; - bool overflow_ocurred = false; - for (int i = 0; i < input_shape_vec.size(); i++) { - new_num_elements = - MultiplyWithoutOverflow(new_num_elements, input_shape_vec(i)); - if (new_num_elements < 0) { - overflow_ocurred = true; - break; - } - } - - OP_REQUIRES( - context, !overflow_ocurred, - errors::Internal("Encountered overflow from large input shape.")); - TensorShape tensor_input_shape(input_shape_vec); + TensorShape tensor_input_shape; + OP_REQUIRES_OK(context, TensorShape::BuildTensorShape(input_shape_vec, + &tensor_input_shape)); gtl::InlinedVector std_order(rank); std::iota(std_order.begin(), std_order.end(), 0); SparseTensor input_st; diff --git a/tensorflow/core/kernels/sparse_utils.cc b/tensorflow/core/kernels/sparse_utils.cc index 75e42eebe25bef..2e144f453ad59f 100644 --- a/tensorflow/core/kernels/sparse_utils.cc +++ b/tensorflow/core/kernels/sparse_utils.cc @@ -16,8 +16,12 @@ limitations under the License. #include "tensorflow/core/kernels/sparse_utils.h" #include +#include #include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/status.h" namespace tensorflow { namespace sparse_utils { @@ -140,6 +144,169 @@ bool ContainsEmptyRows(const std::vector& row_start_indices) { return false; } +namespace { + +// Ensures indices, values, shape are all of the proper ranks and are +// compatible. +Status ValidateSparseTensorShape(const Tensor& indices, const Tensor& values, + const Tensor& shape) { + + // Indices must be a matrix, and values/shape must be a vector. + if (!TensorShapeUtils::IsMatrix(indices.shape())) { + return errors::InvalidArgument("Sparse indices must be rank 2 but is rank ", + indices.shape().dim_sizes().size()); + } + if (!TensorShapeUtils::IsVector(values.shape())) { + return errors::InvalidArgument("Sparse values must be rank 1 but is rank ", + values.shape().dims()); + } + if (!TensorShapeUtils::IsVector(shape.shape())) { + return errors::InvalidArgument("Sparse shape must be rank 1 but is rank ", + shape.shape().dims()); + } + // Indices shape must be compatible with the values vector and dense shape. + int64_t nnz = indices.dim_size(0); + int64_t ndims = indices.dim_size(1); + if (values.dim_size(0) != nnz) { + return errors::InvalidArgument("Number of elements in indices (", nnz, + ") and values (", values.dim_size(0), + ") do not match"); + } + if (shape.NumElements() != ndims) { + return errors::InvalidArgument("Index rank (", ndims, ") and shape rank (", + shape.NumElements(), ") do not match"); + } + + return Status::OK(); +} + +// Creates a debug string for the index tuple in indices(row, :). +template +string CreateIndexString(const IndexTensor& indices, int64_t row) { + const int64_t ndims = indices.dimension(1); + string index_str = strings::StrCat("indices[", row, ", :] = ["); + for (int64_t dim = 0; dim < ndims; ++dim) { + strings::StrAppend(&index_str, indices(row, dim), + dim < ndims - 1 ? ", " : "]"); + } + if (ndims == 0) { + strings::StrAppend(&index_str, "]"); + } + return index_str; +} + +// Ensures all sparse indices are within correct bounds. +template +Status ValidateSparseTensorIndicesUnordered(const Tensor& indices, + const Tensor& shape) { + // Ensure no index is out-of-bounds. + const auto indices_mat = indices.flat_inner_dims(); + const auto shape_vec = shape.flat(); + int64_t nnz = indices.dim_size(0); + int64_t ndims = indices.dim_size(1); + + for (int64_t i = 0; i < nnz; ++i) { + for (int64_t dim = 0; dim < ndims; ++dim) { + const Tindices idx = indices_mat(i, dim); + if (TF_PREDICT_FALSE(idx < 0 || idx >= shape_vec(dim))) { + string index_str = CreateIndexString(indices_mat, i); + + return errors::InvalidArgument("Sparse index tuple ", index_str, + " is out of bounds"); + } + } + } + + return Status::OK(); +} + +// Ensures all sparse indices are within correct bounds and are +// lexicographically ordered. +template +Status ValidateSparseTensorIndicesOrdered(const Tensor& indices, + const Tensor& shape) { + const auto indices_mat = indices.flat_inner_dims(); + const auto shape_vec = shape.flat(); + int64_t nnz = indices.dim_size(0); + int64_t ndims = indices.dim_size(1); + + if (nnz == 0) { + return Status::OK(); + } + + // First set of indices must be within range. + for (int64_t dim = 0; dim < ndims; ++dim) { + const Tindices idx = indices_mat(0, dim); + if (TF_PREDICT_FALSE(idx < 0 || idx >= shape_vec(dim))) { + string index_str = CreateIndexString(indices_mat, 0); + return errors::InvalidArgument("Sparse index tuple ", index_str, + " is out of bounds"); + } + } + + // Remaining set of indices must be within range and lexicographically + // larger than the previous. + for (int64_t i = 1; i < nnz; ++i) { + bool different = false; + for (int64_t dim = 0; dim < ndims; ++dim) { + const Tindices idx = indices_mat(i, dim); + const Tindices prev_idx = indices_mat(i - 1, dim); + // If indices are already different from previous i, the new index can + // be anything within the valid range. + if (TF_PREDICT_TRUE(different)) { + if (TF_PREDICT_FALSE(idx < 0 || idx >= shape_vec(dim))) { + string index_str = CreateIndexString(indices_mat, i); + return errors::InvalidArgument("Sparse index tuple ", index_str, + " is out of bounds"); + } + } else { + // Otherwise, the new index must be >= previous and <= shape(dim). + if (TF_PREDICT_FALSE(idx < prev_idx || idx >= shape_vec(dim))) { + string index_str = CreateIndexString(indices_mat, i); + // Check if index is actually out of bounds. + if (TF_PREDICT_FALSE(idx < 0 || idx >= shape_vec(dim))) { + return errors::InvalidArgument("Sparse index tuple ", index_str, + " is out of bounds"); + } else { + return errors::InvalidArgument("Sparse index tuple ", index_str, + " is out of order"); + } + } else if (TF_PREDICT_TRUE(idx > prev_idx)) { + different = true; + } + } // if (different) + } // for dim in [0, ndims) + + if (TF_PREDICT_FALSE(!different)) { + string index_str = CreateIndexString(indices_mat, i); + return errors::InvalidArgument("Sparse index tuple ", index_str, + " is repeated"); + } + } // for i in [1, nnz) + + return Status::OK(); +} + + +} // namespace + +template +Status ValidateSparseTensor(const Tensor& indices, const Tensor& values, + const Tensor& shape, + IndexValidation index_validation) { + TF_RETURN_IF_ERROR(ValidateSparseTensorShape(indices, values, shape)); + switch (index_validation) { + case IndexValidation::kOrdered: + return ValidateSparseTensorIndicesOrdered(indices, shape); + case IndexValidation::kUnordered: + return ValidateSparseTensorIndicesUnordered(indices, shape); + case IndexValidation::kNone: { + } + + } + return Status::OK(); +} + #define REGISTER_SPARSE_UTIL_FUNCTIONS(TypeIndex) \ template TypeIndex FindNextDenseRowStartIndex( \ const TypeIndex sparse_index_begin, \ @@ -151,7 +318,11 @@ bool ContainsEmptyRows(const std::vector& row_start_indices) { const std::vector& row_start_indices); \ template std::vector ParseRowStartIndices( \ const tensorflow::Tensor& tensor, \ - const TypeIndex num_nonzero_entries_in_sparse_mat); + const TypeIndex num_nonzero_entries_in_sparse_mat); \ + template Status ValidateSparseTensor( \ + const Tensor& indices, const Tensor& values, const Tensor& shape, \ + IndexValidation index_validation) + REGISTER_SPARSE_UTIL_FUNCTIONS(int32); REGISTER_SPARSE_UTIL_FUNCTIONS(int64); diff --git a/tensorflow/core/kernels/sparse_utils.h b/tensorflow/core/kernels/sparse_utils.h index d43b2e34470a5e..fdc5c9f29e43f0 100644 --- a/tensorflow/core/kernels/sparse_utils.h +++ b/tensorflow/core/kernels/sparse_utils.h @@ -65,6 +65,24 @@ std::vector ParseRowStartIndices( template bool ContainsEmptyRows(const std::vector& row_start_indices); +// Methods for validating sparse indices. +enum class IndexValidation { + kNone, // Indices are not used by the op, or are not directly accessible + // (e.g. on GPU). + kOrdered, // Indices must be unique, in lexicographical order, and within + // safe bounds. + kUnordered // Indices must be within safe bounds, but may repeat or appear + // out-of-order. +}; + +// Validates the three component tensors of a sparse tensor have the proper +// shapes. Also validates index values according to the method supplied. +template +Status ValidateSparseTensor(const Tensor& indices, const Tensor& values, + const Tensor& shape, + IndexValidation index_validation); + + } // namespace sparse_utils } // namespace tensorflow diff --git a/tensorflow/core/kernels/sparse_utils_test.cc b/tensorflow/core/kernels/sparse_utils_test.cc index c480829237119c..ebed17bb5d358c 100644 --- a/tensorflow/core/kernels/sparse_utils_test.cc +++ b/tensorflow/core/kernels/sparse_utils_test.cc @@ -15,27 +15,29 @@ limitations under the License. #include "tensorflow/core/kernels/sparse_utils.h" +#include +#include +#include +#include #include +#include "absl/container/flat_hash_set.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/random/philox_random.h" +#include "tensorflow/core/lib/random/simple_philox.h" +#include "tensorflow/core/platform/status_matchers.h" #include "tensorflow/core/platform/test.h" +namespace tensorflow { +namespace sparse_utils { namespace { -using ::int64_t; -using tensorflow::DataType; -using tensorflow::int32; -using tensorflow::Tensor; -using tensorflow::TTypes; -using tensorflow::uint16; -using tensorflow::uint32; -using tensorflow::uint64; -using tensorflow::sparse_utils::ContainsEmptyRows; -using tensorflow::sparse_utils::FindNextDenseRowStartIndex; -using tensorflow::sparse_utils::GetStartIndicesOfEachDenseRow; -using tensorflow::sparse_utils::ParseRowStartIndices; +using ::tensorflow::testing::StatusIs; +using ::testing::MatchesRegex; TEST(SparseUtilsTest, GetStartIndicesOfEachDenseRow) { { @@ -260,4 +262,299 @@ TEST(SparseUtilsTest, FindNextDenseRowStartIndex) { } } +// Returns a shared random number generator. +::tensorflow::random::SimplePhilox& RandomPhilox() { + // Safe initialization of static random generator. + static auto* philox = + new ::tensorflow::random::PhiloxRandom(tensorflow::testing::RandomSeed()); + static auto* rnd = new ::tensorflow::random::SimplePhilox(philox); + return *rnd; +} + +// Fills a tensor of indices with a unique set of random index tuples. +// The `SetType` must be a std::set-like type (e.g. flat_hash_set, btree_set) +// that is used to ensure uniqueness and governs the final index tuple order. +// For example, use a hash set for unordered indices, and sorted set for +// lexicographically ordered indices. The `shape` is used to ensure proper index +// bounds. +template +void FillIndicesWithRandomTuples(const TensorShape& shape, Tensor& indices) { + const int64_t nnz = indices.dim_size(0); + const int64_t ndims = indices.dim_size(1); + + SetType indices_set; + int64_t count = 0; + // Generate nnz unique random tuples. + while (count < nnz) { + std::vector candidate(ndims); + for (int64_t d = 0; d < ndims; ++d) { + candidate[d] = RandomPhilox().Uniform64(shape.dim_size(d)); + } + auto it = indices_set.insert(std::move(candidate)); + if (it.second) { + ++count; + } + } + + // Copy index tuples from set into index tensor. + auto indices_mat = indices.matrix(); + int64_t row = 0; + for (const std::vector& idxs : indices_set) { + for (int64_t col = 0; col < ndims; ++col) { + indices_mat(row, col) = idxs[col]; + } + ++row; + } +} + +// Populates components of a sparse random tensor with provided number of +// non-zeros `max_nnz` and tensor shape `shape`. If `ordered`, output indices +// are ordered lexicographically. +void GenerateRandomSparseTensor(int64_t max_nnz, const TensorShape& shape, + bool ordered, Tensor& output_indices, + Tensor& output_values, Tensor& output_shape) { + const int64_t ndims = shape.dims(); + // We cannot generate more elements than the total in the tensor, so + // potentially reduce nnz. + const int64_t nnz = std::min(shape.num_elements(), max_nnz); + output_indices = Tensor(DT_INT64, TensorShape({nnz, ndims})); + output_values = Tensor(DT_FLOAT, TensorShape({nnz})); + output_shape = Tensor(DT_INT64, TensorShape({ndims})); + + // Generate random unique sparse indices. + if (ordered) { + // NOTE: absl::btree_set does not seem to be available in TF OSS. + FillIndicesWithRandomTuples>>(shape, + output_indices); + } else { + FillIndicesWithRandomTuples>>( + shape, output_indices); + } + + auto values_vec = output_values.vec(); + values_vec.setRandom(); + + auto shape_vec = output_shape.vec(); + for (int i = 0; i < shape.dims(); ++i) { + shape_vec(i) = shape.dim_size(i); + } +} + +using ValidateSparseTensorTest = ::testing::TestWithParam; + +TEST_P(ValidateSparseTensorTest, ValidSparseTensorPasses) { + constexpr int kNumNonZeros = 1000; + const TensorShape kTensorShapes[] = { + {}, {3}, {4, 5}, {6, 7, 8}, {9, 10, 11, 12}}; + const IndexValidation index_validation = GetParam(); + const bool ordered = (index_validation == IndexValidation::kOrdered); + for (const TensorShape& test_shape : kTensorShapes) { + Tensor indices, values, shape; + GenerateRandomSparseTensor(kNumNonZeros, test_shape, ordered, indices, + values, shape); + TF_EXPECT_OK((ValidateSparseTensor(indices, values, shape, + index_validation))); + } +} + +TEST_P(ValidateSparseTensorTest, InvalidIndicesRankFails) { + constexpr int kNumNonZeros = 1000; + constexpr int kNumDims = 3; + // Indices tensor must be rank 2, so try rank 0, 1, 3. + const TensorShape kInvalidIndicesShapes[] = { + {}, {kNumNonZeros}, {kNumNonZeros, kNumDims, 4}}; + const IndexValidation index_validation = GetParam(); + + for (const TensorShape& invalid_shape : kInvalidIndicesShapes) { + const Tensor indices = Tensor(DT_INT64, invalid_shape); + const Tensor values = Tensor(DT_FLOAT, TensorShape({kNumNonZeros})); + const Tensor shape = Tensor(DT_INT64, TensorShape({kNumDims})); + EXPECT_THAT((ValidateSparseTensor(indices, values, shape, + index_validation)), + StatusIs(error::INVALID_ARGUMENT, + MatchesRegex("Sparse indices must be rank 2 .*"))); + } +} + +TEST_P(ValidateSparseTensorTest, InvalidValuesRankFails) { + constexpr int kNumNonZeros = 1000; + constexpr int kNumDims = 3; + // Values tensor must be rank 1, so try rank 0, 2. + const TensorShape kInvalidValuesShapes[] = {{}, {kNumNonZeros, 2}}; + const IndexValidation index_validation = GetParam(); + + for (const TensorShape& invalid_shape : kInvalidValuesShapes) { + const Tensor indices = + Tensor(DT_INT64, TensorShape({kNumNonZeros, kNumDims})); + const Tensor values = Tensor(DT_FLOAT, invalid_shape); + const Tensor shape = Tensor(DT_INT64, TensorShape({kNumDims})); + EXPECT_THAT((ValidateSparseTensor(indices, values, shape, + index_validation)), + StatusIs(error::INVALID_ARGUMENT, + MatchesRegex("Sparse values must be rank 1 .*"))); + } +} + +TEST_P(ValidateSparseTensorTest, InvalidShapeRankFails) { + constexpr int kNumNonZeros = 1000; + constexpr int kNumDims = 3; + const IndexValidation index_validation = GetParam(); + + // Shape tensor must be rank 1, so try rank 0, 2. + const TensorShape kInvalidShapeShapes[] = {{}, {kNumDims, 2}}; + for (const TensorShape& invalid_shape : kInvalidShapeShapes) { + const Tensor indices = + Tensor(DT_INT64, TensorShape({kNumNonZeros, kNumDims})); + const Tensor values = Tensor(DT_FLOAT, TensorShape({kNumNonZeros})); + const Tensor shape = Tensor(DT_INT64, invalid_shape); + EXPECT_THAT((ValidateSparseTensor(indices, values, shape, + index_validation)), + + StatusIs(error::INVALID_ARGUMENT, + MatchesRegex("Sparse shape must be rank 1 .*"))); + } +} + +TEST_P(ValidateSparseTensorTest, IncompatibleShapesFails) { + constexpr int kNumNonZeros = 1000; + constexpr int kNumDims = 3; + const IndexValidation index_validation = GetParam(); + + + const Tensor values = Tensor(DT_FLOAT, TensorShape({kNumNonZeros})); + const Tensor shape = Tensor(DT_INT64, TensorShape({kNumDims})); + + // Indices and values must have the same size in dimension 0 (nnz). + { + const Tensor indices = + Tensor(DT_INT64, TensorShape({kNumNonZeros + 1, kNumDims})); + EXPECT_THAT((ValidateSparseTensor(indices, values, shape, + index_validation)), + + StatusIs(error::INVALID_ARGUMENT, + MatchesRegex("Number of elements in indices .* and " + "values .* do not match"))); + } + + // Each index tuple must have the same size in dimension 1 as the dense + // tensor shape (ndims). + { + const Tensor indices = + Tensor(DT_INT64, TensorShape({kNumNonZeros, kNumDims + 1})); + EXPECT_THAT( + (ValidateSparseTensor(indices, values, shape, + index_validation)), + + StatusIs(error::INVALID_ARGUMENT, + MatchesRegex("Index rank .* and shape rank .* do not match"))); + } +} + +TEST_P(ValidateSparseTensorTest, IndexOutOfBoundsFails) { + constexpr int kNumNonZeros = 1000; + constexpr int kNumTests = 100; + const IndexValidation index_validation = GetParam(); + const bool ordered = (index_validation == IndexValidation::kOrdered); + + const TensorShape kTensorShapes[] = {{3}, {4, 5}, {6, 7, 8}, {9, 10, 11, 12}}; + + for (const TensorShape& test_shape : kTensorShapes) { + Tensor indices, values, shape; + GenerateRandomSparseTensor(kNumNonZeros, test_shape, ordered, indices, + values, shape); + + // Access tensor values. + auto indices_mat = indices.matrix(); + for (int test = 0; test < kNumTests; ++test) { + // Pick a random entry and dimension, and make the index out of bounds. + int64_t row = RandomPhilox().Uniform64(indices.dim_size(0)); + int64_t dim = RandomPhilox().Uniform64(indices.dim_size(1)); + int64_t old_val = indices_mat(row, dim); + + for (int64_t val : {static_cast(-1), test_shape.dim_size(dim)}) { + indices_mat(row, dim) = val; + Status indices_valid = ValidateSparseTensor( + indices, values, shape, index_validation); + if (index_validation == IndexValidation::kNone) { + TF_EXPECT_OK(indices_valid); + } else { + EXPECT_THAT( + indices_valid, + StatusIs(error::INVALID_ARGUMENT, + MatchesRegex("Sparse index tuple .* is out of bounds"))) + << indices_mat; + } + } + + // Restore index for next test. + indices_mat(row, dim) = old_val; + } + } +} + +TEST_P(ValidateSparseTensorTest, IndexOutOfOrderFailsForOrderedValidation) { + constexpr int kNumNonZeros = 1000; + constexpr int kNumTests = 100; + const TensorShape kTensorShapes[] = {{3}, {4, 5}, {6, 7, 8}, {9, 10, 11, 12}}; + const IndexValidation index_validation = GetParam(); + const bool ordered = (index_validation == IndexValidation::kOrdered); + + for (const TensorShape& test_shape : kTensorShapes) { + Tensor indices, values, shape; + GenerateRandomSparseTensor(kNumNonZeros, test_shape, ordered, indices, + values, shape); + // Access tensor values. + auto indices_mat = indices.matrix(); + const int64_t nnz = indices.dim_size(0); + const int64_t ndims = indices.dim_size(1); + for (int test = 0; test < kNumTests; ++test) { + // Pick two random index entries to swap. + int64_t row1 = RandomPhilox().Uniform64(nnz); + int64_t row2; + do { + row2 = RandomPhilox().Uniform64(nnz); + } while (row1 == row2); + for (int dim = 0; dim < ndims; ++dim) { + std::swap(indices_mat(row1, dim), indices_mat(row2, dim)); + } + + Status indices_valid = ValidateSparseTensor( + indices, values, shape, index_validation); + if (ordered) { + EXPECT_THAT( + indices_valid, + StatusIs(error::INVALID_ARGUMENT, + MatchesRegex("Sparse index tuple .* is out of order"))); + } else { + TF_EXPECT_OK(indices_valid); + } + + // Restore index for next test. + for (int dim = 0; dim < ndims; ++dim) { + std::swap(indices_mat(row1, dim), indices_mat(row2, dim)); + } + + } + } +} + +INSTANTIATE_TEST_SUITE_P( + ValidateSparseTensorTestSuite, ValidateSparseTensorTest, + ::testing::Values(IndexValidation::kNone, IndexValidation::kOrdered, + IndexValidation::kUnordered), + [](const ::testing::TestParamInfo& + info) { + switch (info.param) { + case IndexValidation::kNone: + return "None"; + case IndexValidation::kUnordered: + return "Unordered"; + case IndexValidation::kOrdered: + return "Ordered"; + } + }); + + } // namespace +} // namespace sparse_utils +} // namespace tensorflow diff --git a/tensorflow/core/kernels/stage_op.cc b/tensorflow/core/kernels/stage_op.cc index 55c9db22ddf527..f7bb42f9c52b7d 100644 --- a/tensorflow/core/kernels/stage_op.cc +++ b/tensorflow/core/kernels/stage_op.cc @@ -258,6 +258,8 @@ class StagePeekOp : public OpKernel { core::ScopedUnref scope(buf); Buffer::Tuple tuple; + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(ctx->input(0).shape()), + errors::InvalidArgument("index must be scalar")); std::size_t index = ctx->input(0).scalar()(); OP_REQUIRES_OK(ctx, buf->Peek(index, &tuple)); diff --git a/tensorflow/core/kernels/strided_slice_op.cc b/tensorflow/core/kernels/strided_slice_op.cc index 30f083dae1a41d..3ec2d6bbcc3c17 100644 --- a/tensorflow/core/kernels/strided_slice_op.cc +++ b/tensorflow/core/kernels/strided_slice_op.cc @@ -431,6 +431,7 @@ class StridedSliceAssignOp : public OpKernel { StridedSliceAssignOp) TF_CALL_ALL_TYPES(REGISTER_STRIDED_SLICE); +TF_CALL_QUANTIZED_TYPES(REGISTER_STRIDED_SLICE); #undef REGISTER_STRIDED_SLICE diff --git a/tensorflow/core/kernels/strided_slice_op_impl.h b/tensorflow/core/kernels/strided_slice_op_impl.h index 11af7092f30cc2..65c0ecc055a28f 100644 --- a/tensorflow/core/kernels/strided_slice_op_impl.h +++ b/tensorflow/core/kernels/strided_slice_op_impl.h @@ -292,7 +292,7 @@ TF_CALL_GPU_ALL_TYPES(DECLARE_FOR_N_GPU); #endif // END GOOGLE_CUDA || TENSORFLOW_USE_ROCM TF_CALL_ALL_TYPES(DECLARE_FOR_N_CPU); - +TF_CALL_QUANTIZED_TYPES(DECLARE_FOR_N_CPU); #undef INSTANTIATE #undef DECLARE_FOR_N_CPU diff --git a/tensorflow/core/kernels/string_ngrams_op.cc b/tensorflow/core/kernels/string_ngrams_op.cc index c3fbf21bbbf546..005cd6cdf06d51 100644 --- a/tensorflow/core/kernels/string_ngrams_op.cc +++ b/tensorflow/core/kernels/string_ngrams_op.cc @@ -13,13 +13,16 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include #include #include #include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/op_requires.h" #include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/types.h" namespace tensorflow { namespace text { @@ -47,8 +50,14 @@ class StringNGramsOp : public tensorflow::OpKernel { ngram_width - 1); } - int get_num_ngrams(const int length, const int ngram_width) const { + StatusOr get_num_ngrams(const int length, const int ngram_width) const { + int64 limit = kint32max; int pad_width = get_pad_width(ngram_width); + if (pad_width > limit / 2 - length) { + return errors::InvalidArgument( + "Pad width could lead to integer overflow, got pad_width = ", + pad_width); + } return std::max(0, ((length + 2 * pad_width) - ngram_width) + 1); } @@ -112,8 +121,11 @@ class StringNGramsOp : public tensorflow::OpKernel { for (int i = 1; i <= num_batch_items; ++i) { int length = splits_vec(i) - splits_vec(i - 1); int num_ngrams = 0; - for (int ngram_width : ngram_widths_) - num_ngrams += get_num_ngrams(length, ngram_width); + for (int ngram_width : ngram_widths_) { + auto ngrams_or = get_num_ngrams(length, ngram_width); + OP_REQUIRES_OK(context, ngrams_or.status()); + num_ngrams += ngrams_or.ValueOrDie(); + } if (preserve_short_ && length > 0 && num_ngrams == 0) { num_ngrams = 1; } @@ -133,7 +145,9 @@ class StringNGramsOp : public tensorflow::OpKernel { for (int ngram_width : ngram_widths_) { auto output_start = &ngrams_data[output_start_idx]; int length = splits_vec(i + 1) - splits_vec(i); - int num_ngrams = get_num_ngrams(length, ngram_width); + auto ngrams_or = get_num_ngrams(length, ngram_width); + OP_REQUIRES_OK(context, ngrams_or.status()); + int num_ngrams = ngrams_or.ValueOrDie(); CreateNgrams(data_start, output_start, num_ngrams, ngram_width); output_start_idx += num_ngrams; } @@ -152,6 +166,16 @@ class StringNGramsOp : public tensorflow::OpKernel { // We don't have to worry about dynamic padding sizes here: if padding // was dynamic, every sequence would have had sufficient padding to // generate at least one ngram. + + // If reached here, pad_width should be > 0, pad_width_ = -1, + // which indicates max(ngram_widths) - 1 cannot be used here since + // ngram_width is not known. + OP_REQUIRES( + context, pad_width_ >= 0, + errors::InvalidArgument("Pad width should be >= 0 when " + "preserve_short_sequences is True and " + "ngram_widths are not provided, got ", + pad_width_)); int ngram_width = data_length + 2 * pad_width_; auto output_start = &ngrams_data[output_start_idx]; int num_ngrams = 1; diff --git a/tensorflow/core/kernels/summary_audio_op.cc b/tensorflow/core/kernels/summary_audio_op.cc index 09aab681f560aa..6015fe2ab5ebe4 100644 --- a/tensorflow/core/kernels/summary_audio_op.cc +++ b/tensorflow/core/kernels/summary_audio_op.cc @@ -49,6 +49,11 @@ class SummaryAudioOp : public OpKernel { float sample_rate = sample_rate_attr_; if (!has_sample_rate_attr_) { const Tensor& sample_rate_tensor = c->input(2); + OP_REQUIRES(c, + sample_rate_tensor.IsAligned() && + sample_rate_tensor.NumElements() == 1, + errors::InvalidArgument( + "sample_rate must be rank-0 or contain a single value")); sample_rate = sample_rate_tensor.scalar()(); } OP_REQUIRES(c, sample_rate > 0.0f, diff --git a/tensorflow/core/kernels/summary_tensor_op.cc b/tensorflow/core/kernels/summary_tensor_op.cc index e367045b02ab48..730ef6f38e5d62 100644 --- a/tensorflow/core/kernels/summary_tensor_op.cc +++ b/tensorflow/core/kernels/summary_tensor_op.cc @@ -36,6 +36,10 @@ class SummaryTensorOpV2 : public OpKernel { errors::InvalidArgument("tag must be scalar")); const Tensor& tensor = c->input(1); const Tensor& serialized_summary_metadata_tensor = c->input(2); + OP_REQUIRES( + c, + TensorShapeUtils::IsScalar(serialized_summary_metadata_tensor.shape()), + errors::InvalidArgument("serialized_summary_metadata must be scalar")); Summary s; Summary::Value* v = s.add_value(); diff --git a/tensorflow/core/kernels/unravel_index_op.cc b/tensorflow/core/kernels/unravel_index_op.cc index cff04387d60247..b8524e87598ceb 100644 --- a/tensorflow/core/kernels/unravel_index_op.cc +++ b/tensorflow/core/kernels/unravel_index_op.cc @@ -13,6 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include + +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/platform/types.h" #define EIGEN_USE_THREADS #include "tensorflow/core/framework/op_kernel.h" @@ -35,7 +39,8 @@ typedef Eigen::ThreadPoolDevice CPUDevice; template class UnravelIndexOp : public OpKernel { public: - explicit UnravelIndexOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + explicit UnravelIndexOp(OpKernelConstruction* ctx) + : OpKernel(ctx), dtidx_(DataTypeToEnum::v()) {} void Compute(OpKernelContext* ctx) override { const Tensor& indices_tensor = ctx->input(0); @@ -54,12 +59,31 @@ class UnravelIndexOp : public OpKernel { auto dims = dims_tensor.vec(); // Make sure dims does not contain a zero + double prod = 1; + uint64_t limit; + if (dtidx_ == DataType::DT_INT64) { + limit = kint64max; + } else { + limit = kint32max; + } + for (int i = 0; i < dims.size(); i++) { OP_REQUIRES( ctx, dims(i) != 0, errors::InvalidArgument("Input dims cannot contain a dim of zero, " "but dims contains zero at index ", i)); + OP_REQUIRES(ctx, dims(i) > 0, + errors::InvalidArgument( + "Input dims cannot be negative. Got dim = ", dims(i), + " at index ", i)); + // Check interger overflow + OP_REQUIRES( + ctx, prod <= limit / dims(i), + errors::InvalidArgument("Input dims product is causing integer " + "overflow: (", + dims, ")")); + prod = (prod * dims(i)); } // Check to make sure indices is not out of boundary @@ -132,6 +156,7 @@ class UnravelIndexOp : public OpKernel { strides_shifted.reshape(reshape).broadcast(bcast); } } + const DataType dtidx_; }; #define REGISTER_KERNEL(type) \ diff --git a/tensorflow/core/kernels/unsorted_segment_join_op.cc b/tensorflow/core/kernels/unsorted_segment_join_op.cc index 2406f9b0d7c5bd..1d13bd95697e89 100644 --- a/tensorflow/core/kernels/unsorted_segment_join_op.cc +++ b/tensorflow/core/kernels/unsorted_segment_join_op.cc @@ -92,7 +92,15 @@ class UnsortedSegmentJoinOp : public OpKernel { const Tensor& num_segments_tensor = context->input(2); OP_REQUIRES(context, num_segments_tensor.NumElements() != 0, errors::InvalidArgument("Number of segments cannot be empty.")); + OP_REQUIRES(context, + TensorShapeUtils::IsScalar(num_segments_tensor.shape()), + errors::InvalidArgument("Number of segments must be a scalar")); + auto num_segments = num_segments_tensor.scalar()(); + OP_REQUIRES( + context, num_segments >= 0, + errors::InvalidArgument( + "Number of segments must be non-negative but got ", num_segments)); OP_REQUIRES(context, segment_dims != 0, errors::InvalidArgument("Segment_id cannot have rank 0")); diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc index db09252ae0bab1..dad45b632679af 100644 --- a/tensorflow/core/ops/array_ops.cc +++ b/tensorflow/core/ops/array_ops.cc @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/mirror_pad_mode.h" #include "tensorflow/core/util/padding.h" #include "tensorflow/core/util/strided_slice_op.h" @@ -1650,11 +1651,21 @@ REGISTER_OP("ReverseSequence") return errors::InvalidArgument( "batch_dim must be < input rank: ", batch_dim, " vs. ", input_rank); } + if (seq_dim >= input_rank) { return errors::InvalidArgument( "seq_dim must be < input rank: ", seq_dim, " vs. ", input_rank); } + // To prevent out of bound access when calling c->Dim(input, batch_dim), + // batch_dim range [-1 * input rank, input rank) is allowed. However, + // the op implementation has a stricter bound for batch_dim requiring >= 0 + // value. Thus, perform strict check here. + if (batch_dim < 0) { + return errors::InvalidArgument("batch_dim must be >=0, got ", + batch_dim); + } + DimensionHandle batch_dim_dim = c->Dim(input, batch_dim); TF_RETURN_IF_ERROR( c->Merge(batch_dim_dim, c->Dim(seq_lens_shape, 0), &batch_dim_dim)); @@ -3015,6 +3026,12 @@ REGISTER_OP("Dequantize") return errors::InvalidArgument("axis should be at least -1, got ", axis); } + auto input_dims = c->Rank(c->input(0)); + if (axis > input_dims) { + return errors::InvalidArgument( + "Axis must be less than input dimension(", input_dims, "), got ", + axis); + } const int minmax_rank = (axis == -1) ? 0 : 1; TF_RETURN_IF_ERROR(shape_inference::UnchangedShape(c)); ShapeHandle minmax; @@ -3022,6 +3039,13 @@ REGISTER_OP("Dequantize") TF_RETURN_IF_ERROR(c->WithRank(c->input(2), minmax_rank, &minmax)); if (axis != -1) { ShapeHandle input; + if (axis >= kint32max) { + // Check int32 max bound for a corner case to prevent integer flow + // when input actually has kint32max rank and above bound check is not + // triggered. + return errors::InvalidArgument( + "Axis cannot be >= kint32max value, got ", axis); + } TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), axis + 1, &input)); DimensionHandle depth; TF_RETURN_IF_ERROR( diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index cad02305b91308..1e5c9cb5b21e07 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -1475,12 +1475,20 @@ Status RangeSize(const Tensor* start_t, const Tensor* limit_t, return errors::InvalidArgument("Requires delta != 0"); } - auto size = (std::is_integral::value - ? ((Eigen::numext::abs(limit - start) + - Eigen::numext::abs(delta) - T(1)) / - Eigen::numext::abs(delta)) - : (Eigen::numext::ceil( - Eigen::numext::abs((limit - start) / delta)))); + int64_t size; + if (std::is_integral::value) { + size = Eigen::divup(static_cast(Eigen::numext::abs(limit - start)), + static_cast(Eigen::numext::abs(delta))); + } else { + auto size_auto = + Eigen::numext::ceil(Eigen::numext::abs((limit - start) / delta)); + if (size_auto > std::numeric_limits::max()) { + return errors::InvalidArgument("Requires ((limit - start) / delta) <= ", + std::numeric_limits::max()); + } + size = static_cast(size_auto); + } + c->set_output(0, c->Vector(static_cast(size))); return Status::OK(); } @@ -1691,6 +1699,11 @@ REGISTER_OP("Bincount") return Status::OK(); } + if (size_tensor->dims() != 0) { + return errors::InvalidArgument("Shape must be rank 0 but is rank ", + size_tensor->dims()); + } + // Return `[size]` shape if size is known. int32_t size_val = size_tensor->scalar()(); if (size_val < 0) { @@ -1722,6 +1735,10 @@ REGISTER_OP("DenseBincount") c->set_output(0, c->UnknownShape()); return Status::OK(); } + if (size_tensor->dims() != 0) { + return errors::InvalidArgument("Shape must be rank 0 but is rank ", + size_tensor->dims()); + } int64_t size_val; DataType dtype; @@ -1763,6 +1780,10 @@ REGISTER_OP("SparseBincount") c->set_output(0, c->UnknownShape()); return Status::OK(); } + if (size_tensor->dims() != 0) { + return errors::InvalidArgument("Shape must be rank 0 but is rank ", + size_tensor->dims()); + } int64_t size_val; DataType dtype; diff --git a/tensorflow/core/platform/default/logging.h b/tensorflow/core/platform/default/logging.h index aa7700edbc14b2..621f9ffc9044da 100644 --- a/tensorflow/core/platform/default/logging.h +++ b/tensorflow/core/platform/default/logging.h @@ -85,7 +85,7 @@ class LogMessage : public std::basic_ostringstream { // that the ternary VLOG() implementation is balanced, type wise. struct Voidifier { template - void operator&(const T&)const {} + void operator&(const T&) const {} }; // LogMessageFatal ensures the process will exit in failure after @@ -348,11 +348,13 @@ string* MakeCheckOpString(const T1& v1, const T2& v2, const char* exprtext) { } // Helper functions for CHECK_OP macro. -// The (int, int) specialization works around the issue that the compiler +// We use the full name Check_EQ, Check_NE, etc. in case the file including +// base/logging.h provides its own #defines for the simpler names EQ, NE, etc. +// This happens if, for example, those are used as token names in a +// yacc grammar. +// The (int, int) overload works around the issue that the compiler // will not instantiate the template version of the function on values of // unnamed enum type - see comment below. -// The (size_t, int) and (int, size_t) specialization are to handle unsigned -// comparison errors while still being thorough with the comparison. #define TF_DEFINE_CHECK_OP_IMPL(name, op) \ template \ inline string* name##Impl(const T1& v1, const T2& v2, \ @@ -364,34 +366,77 @@ string* MakeCheckOpString(const T1& v1, const T2& v2, const char* exprtext) { } \ inline string* name##Impl(int v1, int v2, const char* exprtext) { \ return name##Impl(v1, v2, exprtext); \ - } \ - inline string* name##Impl(const size_t v1, const int v2, \ - const char* exprtext) { \ - if (TF_PREDICT_FALSE(v2 < 0)) { \ - return ::tensorflow::internal::MakeCheckOpString(v1, v2, exprtext); \ - } \ - return name##Impl(v1, v2, exprtext); \ - } \ - inline string* name##Impl(const int v1, const size_t v2, \ - const char* exprtext) { \ - if (TF_PREDICT_FALSE(v2 >= std::numeric_limits::max())) { \ - return ::tensorflow::internal::MakeCheckOpString(v1, v2, exprtext); \ - } \ - const size_t uval = (size_t)((unsigned)v2); \ - return name##Impl(v1, uval, exprtext); \ } -// We use the full name Check_EQ, Check_NE, etc. in case the file including -// base/logging.h provides its own #defines for the simpler names EQ, NE, etc. -// This happens if, for example, those are used as token names in a -// yacc grammar. -TF_DEFINE_CHECK_OP_IMPL(Check_EQ, - ==) // Compilation error with CHECK_EQ(NULL, x)? -TF_DEFINE_CHECK_OP_IMPL(Check_NE, !=) // Use CHECK(x == NULL) instead. +// The (size_t, int) and (int, size_t) specialization are to handle unsigned +// comparison errors while still being thorough with the comparison. + +TF_DEFINE_CHECK_OP_IMPL(Check_EQ, ==) +// Compilation error with CHECK_EQ(NULL, x)? +// Use CHECK(x == NULL) instead. + +inline string* Check_EQImpl(int v1, size_t v2, const char* exprtext) { + if (TF_PREDICT_FALSE(v1 < 0)) + ::tensorflow::internal::MakeCheckOpString(v1, v2, exprtext); + + return Check_EQImpl(size_t(v1), v2, exprtext); +} + +inline string* Check_EQImpl(size_t v1, int v2, const char* exprtext) { + return Check_EQImpl(v2, v1, exprtext); +} + +TF_DEFINE_CHECK_OP_IMPL(Check_NE, !=) + +inline string* Check_NEImpl(int v1, size_t v2, const char* exprtext) { + if (v1 < 0) return NULL; + + return Check_NEImpl(size_t(v1), v2, exprtext); +} + +inline string* Check_NEImpl(size_t v1, int v2, const char* exprtext) { + return Check_NEImpl(v2, v1, exprtext); +} + TF_DEFINE_CHECK_OP_IMPL(Check_LE, <=) + +inline string* Check_LEImpl(int v1, size_t v2, const char* exprtext) { + if (v1 <= 0) return NULL; + + return Check_LEImpl(size_t(v1), v2, exprtext); +} + +inline string* Check_LEImpl(size_t v1, int v2, const char* exprtext) { + if (TF_PREDICT_FALSE(v2 < 0)) + return ::tensorflow::internal::MakeCheckOpString(v1, v2, exprtext); + return Check_LEImpl(v1, size_t(v2), exprtext); +} + TF_DEFINE_CHECK_OP_IMPL(Check_LT, <) -TF_DEFINE_CHECK_OP_IMPL(Check_GE, >=) -TF_DEFINE_CHECK_OP_IMPL(Check_GT, >) + +inline string* Check_LTImpl(int v1, size_t v2, const char* exprtext) { + if (v1 < 0) return NULL; + + return Check_LTImpl(size_t(v1), v2, exprtext); +} + +inline string* Check_LTImpl(size_t v1, int v2, const char* exprtext) { + if (v2 < 0) + return ::tensorflow::internal::MakeCheckOpString(v1, v2, exprtext); + return Check_LTImpl(v1, size_t(v2), exprtext); +} + +// Implement GE,GT in terms of LE,LT +template +inline string* Check_GEImpl(const T1& v1, const T2& v2, const char* exprtext) { + return Check_LEImpl(v2, v1, exprtext); +} + +template +inline string* Check_GTImpl(const T1& v1, const T2& v2, const char* exprtext) { + return Check_LTImpl(v2, v1, exprtext); +} + #undef TF_DEFINE_CHECK_OP_IMPL // In optimized mode, use CheckOpString to hint to compiler that diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index c90b8dcd6b4daa..2d63f01b7f8e3c 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -22,7 +22,7 @@ limitations under the License. // tensorflow/tools/pip_package/setup.py #define TF_MAJOR_VERSION 2 #define TF_MINOR_VERSION 7 -#define TF_PATCH_VERSION 0 +#define TF_PATCH_VERSION 4 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", // "-beta", "-rc", "-rc.1") diff --git a/tensorflow/core/util/BUILD b/tensorflow/core/util/BUILD index 4d4a9fdd96d29a..16a205d0dd91cc 100644 --- a/tensorflow/core/util/BUILD +++ b/tensorflow/core/util/BUILD @@ -511,6 +511,9 @@ tf_cuda_library( cc_library( name = "overflow", hdrs = ["overflow.h"], + visibility = [ + "//tensorflow:internal", + ], deps = [ "//tensorflow/core/platform:logging", "//tensorflow/core/platform:macros", diff --git a/tensorflow/core/util/sparse/sparse_tensor.h b/tensorflow/core/util/sparse/sparse_tensor.h index 23bd8fa8679793..240b96ab47c49e 100644 --- a/tensorflow/core/util/sparse/sparse_tensor.h +++ b/tensorflow/core/util/sparse/sparse_tensor.h @@ -30,6 +30,7 @@ limitations under the License. #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/platform/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/sparse/dim_comparator.h" @@ -177,9 +178,9 @@ class SparseTensor { // element of the array representing one dimension. The start is the start // index at each dimension and the size is the size at each dimension. template - static SparseTensor Slice(const SparseTensor& tensor, - const gtl::ArraySlice& start, - const gtl::ArraySlice& size); + static StatusOr Slice(const SparseTensor& tensor, + const gtl::ArraySlice start, + const gtl::ArraySlice size); // Picks out the dimensions according to `dim_indices`. std::vector PickDims(gtl::ArraySlice dim_indices) const { @@ -577,9 +578,9 @@ inline Status SparseTensor::Split(const SparseTensor& input_tensor, } template -inline SparseTensor SparseTensor::Slice(const SparseTensor& input_tensor, - const gtl::ArraySlice& start, - const gtl::ArraySlice& size) { +inline StatusOr SparseTensor::Slice( + const SparseTensor& input_tensor, const gtl::ArraySlice start, + const gtl::ArraySlice size) { TensorShape output_shape(input_tensor.shape()); const int dims = input_tensor.dims(); @@ -590,15 +591,17 @@ inline SparseTensor SparseTensor::Slice(const SparseTensor& input_tensor, const int64_t input_size = output_shape.dim_size(dim); const int64_t start_index = start[dim]; const int64_t slice_size = size[dim]; - if (start_index + slice_size < input_size) { + + if (start_index < input_size - slice_size) { // The entire selection is within input boundaries. - output_shape.set_dim(dim, slice_size); + TF_RETURN_IF_ERROR(output_shape.SetDimWithStatus(dim, slice_size)); } else if (start_index < input_size) { // The selection starts within input boundaries, but goes beyond them. - output_shape.set_dim(dim, input_size - start_index); + TF_RETURN_IF_ERROR( + output_shape.SetDimWithStatus(dim, input_size - start_index)); } else { // The selection is entirely out of input boundaries. - output_shape.set_dim(dim, 0); + TF_RETURN_IF_ERROR(output_shape.SetDimWithStatus(dim, 0)); } } diff --git a/tensorflow/core/util/sparse/sparse_tensor_test.cc b/tensorflow/core/util/sparse/sparse_tensor_test.cc index 67c7111fcffdcb..84e4d2feb859d9 100644 --- a/tensorflow/core/util/sparse/sparse_tensor_test.cc +++ b/tensorflow/core/util/sparse/sparse_tensor_test.cc @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/random/simple_philox.h" #include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/platform/statusor.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/test_benchmark.h" @@ -690,7 +691,8 @@ TEST(SparseTensorTest, Slice) { size[0] = 2; size[1] = 3; - SparseTensor slice = SparseTensor::Slice(st, start, size); + TF_ASSERT_OK_AND_ASSIGN(SparseTensor slice, + SparseTensor::Slice(st, start, size)); EXPECT_EQ(TensorShape(slice.shape()), TensorShape({2, 3})); EXPECT_EQ(slice.values().NumElements(), 3); @@ -724,8 +726,9 @@ TEST(SparseTensorTest, SliceReducesOutputDimension) { TF_ASSERT_OK(SparseTensor::Create(ids, vals, TensorShape({num_rows, num_columns}), &st)); - SparseTensor slice = - SparseTensor::Slice(st, {num_rows + 1, 1}, {1, num_columns}); + TF_ASSERT_OK_AND_ASSIGN( + SparseTensor slice, + SparseTensor::Slice(st, {num_rows + 1, 1}, {1, num_columns})); EXPECT_EQ(TensorShape(slice.shape()), TensorShape({0, 1})); } diff --git a/tensorflow/core/util/tensor_slice_writer.cc b/tensorflow/core/util/tensor_slice_writer.cc index a74e2a04cedbe5..731bce308fc9fb 100644 --- a/tensorflow/core/util/tensor_slice_writer.cc +++ b/tensorflow/core/util/tensor_slice_writer.cc @@ -131,6 +131,16 @@ Status TensorSliceWriter::Finish() { /* static */ size_t TensorSliceWriter::MaxBytesPerElement(DataType dt) { + size_t max_bytes_per_element = + TensorSliceWriter::MaxBytesPerElementOrZero(dt); + if (max_bytes_per_element == 0) { + LOG(FATAL) << "MaxBytesPerElement not implemented for dtype: " << dt; + } + return max_bytes_per_element; +} + +/* static */ +size_t TensorSliceWriter::MaxBytesPerElementOrZero(DataType dt) { switch (dt) { case DT_FLOAT: return 4; @@ -170,9 +180,8 @@ size_t TensorSliceWriter::MaxBytesPerElement(DataType dt) { case DT_STRING: case DT_BFLOAT16: default: - LOG(FATAL) << "MaxBytesPerElement not implemented for dtype: " << dt; + return 0; } - return 0; } template <> diff --git a/tensorflow/core/util/tensor_slice_writer.h b/tensorflow/core/util/tensor_slice_writer.h index 01f2e62dfbd2bc..9aa51c29cb323d 100644 --- a/tensorflow/core/util/tensor_slice_writer.h +++ b/tensorflow/core/util/tensor_slice_writer.h @@ -68,6 +68,8 @@ class TensorSliceWriter { static size_t MaxBytesPerElement(DataType dt); private: + static size_t MaxBytesPerElementOrZero(DataType dt); + static constexpr size_t kMaxMessageBytes = 1LL << 31; // Filling in the TensorProto in a SavedSlice will add the following // header bytes, in addition to the data: @@ -162,9 +164,15 @@ Status TensorSliceWriter::Add(const string& name, const TensorShape& shape, template Status TensorSliceWriter::SaveData(const T* data, int64_t num_elements, SavedSlice* ss) { - size_t size_bound = - ss->ByteSize() + kTensorProtoHeaderBytes + - (MaxBytesPerElement(DataTypeToEnum::value) * num_elements); + size_t max_bytes_per_element = + MaxBytesPerElementOrZero(DataTypeToEnum::value); + if (max_bytes_per_element == 0) { + return errors::InvalidArgument( + "Tensor slice serialization not implemented for dtype ", + DataTypeToEnum::value); + } + size_t size_bound = ss->ByteSize() + kTensorProtoHeaderBytes + + (max_bytes_per_element * num_elements); if (size_bound > kMaxMessageBytes) { return errors::InvalidArgument( "Tensor slice is too large to serialize (conservative estimate: ", diff --git a/tensorflow/core/util/tensor_slice_writer_test.cc b/tensorflow/core/util/tensor_slice_writer_test.cc index 51aa781af11137..d09f92eed97553 100644 --- a/tensorflow/core/util/tensor_slice_writer_test.cc +++ b/tensorflow/core/util/tensor_slice_writer_test.cc @@ -15,17 +15,19 @@ limitations under the License. #include "tensorflow/core/util/tensor_slice_writer.h" +#include #include +#include +#include #include "tensorflow/core/framework/tensor_shape.pb.h" #include "tensorflow/core/framework/versions.pb.h" #include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/lib/core/stringpiece.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/path.h" #include "tensorflow/core/platform/protobuf.h" #include "tensorflow/core/platform/test.h" +#include "tensorflow/core/protobuf/error_codes.pb.h" #include "tensorflow/core/public/version.h" #include "tensorflow/core/util/saved_tensor_slice_util.h" #include "tensorflow/core/util/tensor_slice_reader.h" @@ -350,6 +352,17 @@ TEST(TensorSliceWriteTest, SizeErrors) { } } +TEST(TensorSliceWriterTest, InvalidInput) { + SavedSlice ss; + std::array data; + std::fill(data.begin(), data.end(), 1234); + Status s = TensorSliceWriter::SaveData(data.data(), data.size(), &ss); + EXPECT_EQ(s.code(), error::INVALID_ARGUMENT); + EXPECT_TRUE(absl::StrContains( + s.error_message(), + "Tensor slice serialization not implemented for dtype")); +} + } // namespace checkpoint } // namespace tensorflow diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD index d1e15ea1b7255d..690acb2e5e6748 100644 --- a/tensorflow/lite/BUILD +++ b/tensorflow/lite/BUILD @@ -1015,6 +1015,7 @@ cc_library( copts = tflite_copts_warnings() + tflite_copts(), deps = [ ":kernel_api", + ":macros", "//tensorflow/lite/c:common", "//tensorflow/lite/schema:schema_fbs", ], @@ -1043,6 +1044,7 @@ cc_test( features = ["-dynamic_link_test_srcs"], # see go/dynamic_link_test_srcs deps = [ ":util", + "//tensorflow/lite/c:c_api_types", "//tensorflow/lite/c:common", "//tensorflow/lite/schema:schema_fbs", "@com_google_googletest//:gtest_main", diff --git a/tensorflow/lite/c/common.c b/tensorflow/lite/c/common.c index 5456a889b75671..8bfb123b17eb28 100644 --- a/tensorflow/lite/c/common.c +++ b/tensorflow/lite/c/common.c @@ -21,7 +21,7 @@ limitations under the License. #include #endif // TF_LITE_STATIC_MEMORY -int TfLiteIntArrayGetSizeInBytes(int size) { +size_t TfLiteIntArrayGetSizeInBytes(int size) { static TfLiteIntArray dummy; return sizeof(dummy) + sizeof(dummy.data[0]) * size; } @@ -45,7 +45,7 @@ int TfLiteIntArrayEqualsArray(const TfLiteIntArray* a, int b_size, #ifndef TF_LITE_STATIC_MEMORY TfLiteIntArray* TfLiteIntArrayCreate(int size) { - int alloc_size = TfLiteIntArrayGetSizeInBytes(size); + size_t alloc_size = TfLiteIntArrayGetSizeInBytes(size); if (alloc_size <= 0) return NULL; TfLiteIntArray* ret = (TfLiteIntArray*)malloc(alloc_size); if (!ret) return ret; diff --git a/tensorflow/lite/c/common.h b/tensorflow/lite/c/common.h index b369e3807964c7..0c8e866a02b42c 100644 --- a/tensorflow/lite/c/common.h +++ b/tensorflow/lite/c/common.h @@ -94,7 +94,7 @@ typedef struct TfLiteIntArray { // Given the size (number of elements) in a TfLiteIntArray, calculate its size // in bytes. -int TfLiteIntArrayGetSizeInBytes(int size); +size_t TfLiteIntArrayGetSizeInBytes(int size); #ifndef TF_LITE_STATIC_MEMORY // Create a array of a given `size` (uninitialized entries). diff --git a/tensorflow/lite/core/subgraph.cc b/tensorflow/lite/core/subgraph.cc index d16540980c77fd..67c87eb059cb28 100644 --- a/tensorflow/lite/core/subgraph.cc +++ b/tensorflow/lite/core/subgraph.cc @@ -690,27 +690,6 @@ TfLiteStatus Subgraph::CheckInputAndOutputForOverlap(const int* input_indices, return kTfLiteOk; } -namespace { -// Multiply two sizes and return true if overflow occurred; -// This is based off tensorflow/overflow.h but is simpler as we already -// have unsigned numbers. It is also generalized to work where sizeof(size_t) -// is not 8. -TfLiteStatus MultiplyAndCheckOverflow(size_t a, size_t b, size_t* product) { - // Multiplying a * b where a and b are size_t cannot result in overflow in a - // size_t accumulator if both numbers have no non-zero bits in their upper - // half. - constexpr size_t size_t_bits = 8 * sizeof(size_t); - constexpr size_t overflow_upper_half_bit_position = size_t_bits / 2; - *product = a * b; - // If neither integers have non-zero bits past 32 bits can't overflow. - // Otherwise check using slow devision. - if (TFLITE_EXPECT_FALSE((a | b) >> overflow_upper_half_bit_position != 0)) { - if (a != 0 && *product / a != b) return kTfLiteError; - } - return kTfLiteOk; -} -} // namespace - TfLiteStatus Subgraph::BytesRequired(TfLiteType type, const int* dims, size_t dims_size, size_t* bytes) { TF_LITE_ENSURE(&context_, bytes != nullptr); diff --git a/tensorflow/lite/delegates/flex/BUILD b/tensorflow/lite/delegates/flex/BUILD index 2c34acdb436e4c..7ab90f45e737b1 100644 --- a/tensorflow/lite/delegates/flex/BUILD +++ b/tensorflow/lite/delegates/flex/BUILD @@ -122,7 +122,7 @@ tflite_flex_cc_library( # ops and kernels. The output library name is platform dependent: # - Linux/Android: `libtensorflowlite_flex.so` # - Mac: `libtensorflowlite_flex.dylib` -# - Windows: `libtensorflowlite_flex.dll` +# - Windows: `tensorflowlite_flex.dll` tflite_flex_shared_library( name = "tensorflowlite_flex", ) diff --git a/tensorflow/lite/delegates/flex/build_def.bzl b/tensorflow/lite/delegates/flex/build_def.bzl index 5021a3b10bdd8f..7616e78c9f8fc7 100644 --- a/tensorflow/lite/delegates/flex/build_def.bzl +++ b/tensorflow/lite/delegates/flex/build_def.bzl @@ -216,9 +216,6 @@ def tflite_flex_shared_library( tflite_cc_shared_object( name = name, - # Until we have more granular symbol export for the C++ API on Windows, - # export all symbols. - features = ["windows_export_all_symbols"], linkopts = select({ "//tensorflow:macos": [ "-Wl,-exported_symbols_list,$(location //tensorflow/lite/delegates/flex:exported_symbols.lds)", diff --git a/tensorflow/lite/delegates/hexagon/BUILD b/tensorflow/lite/delegates/hexagon/BUILD index a70a95b5062a3d..d2aa9e0d3fedf1 100644 --- a/tensorflow/lite/delegates/hexagon/BUILD +++ b/tensorflow/lite/delegates/hexagon/BUILD @@ -66,6 +66,14 @@ cc_library( ], ) +cc_library( + name = "enable_hexagon_delegate", + defines = select({ + "//tensorflow:arm_any": ["TFLITE_ENABLE_HEXAGON"], + "//conditions:default": [], + }), +) + cc_library( name = "hexagon_delegate", srcs = ["hexagon_delegate.cc"], @@ -82,7 +90,12 @@ cc_library( "//tensorflow/lite:minimal_logging", "//tensorflow/lite/c:common", "//tensorflow/lite/delegates/utils:simple_delegate", - ], + ] + select({ + "//tensorflow:ios": [], + "//tensorflow:ios_x86_64": [], + "//tensorflow:macos": [], + "//conditions:default": [":enable_hexagon_delegate"], + }), ) cc_library( diff --git a/tensorflow/lite/kernels/comparisons.cc b/tensorflow/lite/kernels/comparisons.cc index d0a1876c5c654f..c3824c1db01706 100644 --- a/tensorflow/lite/kernels/comparisons.cc +++ b/tensorflow/lite/kernels/comparisons.cc @@ -81,6 +81,17 @@ TfLiteStatus ComparisonPrepareStringAllowed(TfLiteContext* context, return ComparisonPrepareCommon(context, node, true); } +void QuantizeMultiplier(double double_multiplier, int32_t* quantized_multiplier, + int* left_shift) { + if (double_multiplier < 1.0) { + QuantizeMultiplierSmallerThanOneExp(double_multiplier, quantized_multiplier, + left_shift); + } else { + QuantizeMultiplierGreaterThanOne(double_multiplier, quantized_multiplier, + left_shift); + } +} + template opname> void ComparisonQuantized(const TfLiteTensor* input1, const TfLiteTensor* input2, TfLiteTensor* output, bool requires_broadcast) { @@ -90,13 +101,11 @@ void ComparisonQuantized(const TfLiteTensor* input1, const TfLiteTensor* input2, const int left_shift = 8; int32 input1_multiplier; - int input1_shift; - QuantizeMultiplierSmallerThanOneExp(input1->params.scale, - &input1_multiplier, &input1_shift); int32 input2_multiplier; + int input1_shift; int input2_shift; - QuantizeMultiplierSmallerThanOneExp(input2->params.scale, - &input2_multiplier, &input2_shift); + QuantizeMultiplier(input1->params.scale, &input1_multiplier, &input1_shift); + QuantizeMultiplier(input2->params.scale, &input2_multiplier, &input2_shift); ComparisonParams op_params; op_params.left_shift = left_shift; diff --git a/tensorflow/lite/kernels/comparisons_test.cc b/tensorflow/lite/kernels/comparisons_test.cc index f8cf6dee74c4bf..074d0f1f61513a 100644 --- a/tensorflow/lite/kernels/comparisons_test.cc +++ b/tensorflow/lite/kernels/comparisons_test.cc @@ -653,6 +653,26 @@ TEST(ComparisonsTest, QuantizedInt8GreaterWithBroadcast) { } } +TEST(ComparisonsTest, + QuantizedInt8GreaterWithBroadcastMultiplierGreaterThanOne) { + const float kMin = -127.f; + const float kMax = 127.f; + std::vector> test_shapes = { + {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}}; + for (int i = 0; i < test_shapes.size(); ++i) { + ComparisonOpModel model({TensorType_INT8, test_shapes[i], kMin, kMax}, + {TensorType_INT8, {}, kMin, kMax}, TensorType_INT8, + BuiltinOperator_GREATER); + model.QuantizeAndPopulate(model.input1(), + {572, -2, -71, 8, 11, 20}); + model.QuantizeAndPopulate(model.input2(), {8}); + model.Invoke(); + EXPECT_THAT(model.GetOutput(), + ElementsAre(true, false, false, false, true, true)) + << "With shape number " << i; + } +} + TEST(ComparisonsTest, QuantizedUInt8GreaterEqualWithBroadcast) { const float kMin = -1.f; const float kMax = 128.f; diff --git a/tensorflow/lite/kernels/depthwise_conv.cc b/tensorflow/lite/kernels/depthwise_conv.cc index 060b0827dafa74..e1a54adcc7fd05 100644 --- a/tensorflow/lite/kernels/depthwise_conv.cc +++ b/tensorflow/lite/kernels/depthwise_conv.cc @@ -115,6 +115,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, NumDimensions(input), 4); TF_LITE_ENSURE_EQ(context, NumDimensions(filter), 4); + TF_LITE_ENSURE(context, params->dilation_height_factor > 0); + TF_LITE_ENSURE(context, params->dilation_width_factor > 0); const TfLiteType data_type = input->type; diff --git a/tensorflow/lite/kernels/embedding_lookup_sparse.cc b/tensorflow/lite/kernels/embedding_lookup_sparse.cc index 4ad1054340c9c3..270ccc929d9bd8 100644 --- a/tensorflow/lite/kernels/embedding_lookup_sparse.cc +++ b/tensorflow/lite/kernels/embedding_lookup_sparse.cc @@ -72,6 +72,7 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/internal/tensor_utils.h" #include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/util.h" namespace tflite { namespace ops { @@ -158,6 +159,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 3, &weights)); const TfLiteTensor* value; TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 4, &value)); + const size_t values_size = NumElements(value); const int lookup_rank = SizeOfDimension(indices, 1); const int embedding_rank = NumDimensions(value); @@ -175,25 +177,33 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { TfLiteIntArray* output_shape = TfLiteIntArrayCreate(output_rank); TF_LITE_ENSURE(context, output_shape != nullptr); int k = 0; - int embedding_size = 1; - int lookup_size = 1; + size_t embedding_size = 1; + size_t lookup_size = 1; for (int i = 0; i < lookup_rank - 1; i++, k++) { - const int dim = dense_shape->data.i32[i]; - lookup_size *= dim; + const size_t dim = dense_shape->data.i32[i]; + TF_LITE_ENSURE_MSG( + context, + MultiplyAndCheckOverflow(lookup_size, dim, &lookup_size) == kTfLiteOk, + "Lookup size overflowed."); output_shape->data[k] = dim; } for (int i = 1; i < embedding_rank; i++, k++) { - const int dim = SizeOfDimension(value, i); - embedding_size *= dim; + const size_t dim = SizeOfDimension(value, i); + TF_LITE_ENSURE_MSG(context, + MultiplyAndCheckOverflow(embedding_size, dim, + &embedding_size) == kTfLiteOk, + "Embedding size overflowed."); output_shape->data[k] = dim; } TF_LITE_ENSURE_STATUS(context->ResizeTensor(context, output, output_shape)); - const int output_size = lookup_size * embedding_size; + const size_t output_size = lookup_size * embedding_size; TfLiteTensorRealloc(output_size * sizeof(float), output); float* output_ptr = GetTensorData(output); const float* weights_ptr = GetTensorData(weights); const float* value_ptr = GetTensorData(value); + // Makes sure reallocation was successful. + TF_LITE_ENSURE(context, output_ptr != nullptr); std::fill_n(output_ptr, output_size, 0.0f); @@ -244,6 +254,11 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { current_squares_weight += w * w; current_total_weight += w; for (int k = 0; k < embedding_size; k++) { + // only index if indices are valid + if (current_output_offset + k < 0) continue; + if (current_output_offset + k >= output_size) continue; + if (example_embedding_offset + k < 0) continue; + if (example_embedding_offset + k >= values_size) continue; output_ptr[current_output_offset + k] += value_ptr[example_embedding_offset + k] * w; } diff --git a/tensorflow/lite/kernels/fully_connected.cc b/tensorflow/lite/kernels/fully_connected.cc index d0010e302618dd..a328398494b358 100644 --- a/tensorflow/lite/kernels/fully_connected.cc +++ b/tensorflow/lite/kernels/fully_connected.cc @@ -879,6 +879,36 @@ TfLiteStatus EvalShuffledQuantized(TfLiteContext* context, TfLiteNode* node, return kTfLiteOk; } +// Verifies that sparsity values are valid given input/weight/output. +bool VerifySparsity(const RuntimeShape& weights_shape, + const RuntimeShape& input_shape, + const RuntimeShape& output_shape, + const TfLiteSparsity* sparsity) { + const int weights_dims_count = weights_shape.DimensionsCount(); + const int output_dims_count = output_shape.DimensionsCount(); + const int w0_size = sparsity->dim_metadata[0].dense_size; + const int accum_depth = weights_shape.Dims(weights_dims_count - 1); + const int output_elements = output_shape.FlatSize(); + const int input_elements = input_shape.FlatSize(); + const int batches = FlatSizeSkipDim(output_shape, output_dims_count - 1); + const int output_depth = MatchingDim(weights_shape, weights_dims_count - 2, + output_shape, output_dims_count - 1); + const int max_batch_index = batches - 1; + const int max_output = max_batch_index * output_depth + w0_size; + const int max_batch_depth = accum_depth * max_batch_index; + + // Verify output size is enough. + if (output_elements < max_output) return false; + + // Verify index from sparse in input is valid. + for (int i = 0; i < sparsity->dim_metadata[1].array_indices->size; ++i) { + if (input_elements <= + max_batch_depth + sparsity->dim_metadata[1].array_indices->data[i]) + return false; + } + return true; +} + template TfLiteStatus EvalFloat(TfLiteContext* context, TfLiteNode* node, TfLiteFullyConnectedParams* params, OpData* data, @@ -919,24 +949,32 @@ TfLiteStatus EvalFloat(TfLiteContext* context, TfLiteNode* node, "Unsupported sparse fully-connected weight format."); return kTfLiteError; } + const auto& input_shape = GetTensorShape(input); + const auto& filter_shape = GetTensorShape(filter); + const auto& output_shape = GetTensorShape(output); + const auto& bias_shape = GetTensorShape(bias); + if (!VerifySparsity(filter_shape, input_shape, output_shape, &sparsity)) { + TF_LITE_KERNEL_LOG(context, "Invalid sparse fully-connected format."); + return kTfLiteError; + } if (sparsity.dim_metadata_size == kDimMetadataSizeRandomSparse) { // Random sparse. optimized_ops::FullyConnectedSparseWeight( - sparsity, op_params, GetTensorShape(input), - GetTensorData(input), GetTensorShape(filter), - GetTensorData(filter), GetTensorShape(bias), - GetTensorData(bias), GetTensorShape(output), - GetTensorData(output)); + sparsity, op_params, // Disable formatting + input_shape, GetTensorData(input), // Disable formatting + filter_shape, GetTensorData(filter), // Disable formatting + bias_shape, GetTensorData(bias), // Disable formatting + output_shape, GetTensorData(output)); } else if (sparsity.dim_metadata_size == kDimMetadataSizeBlockSparse && sparsity.dim_metadata[2].dense_size == 4) { // Block sparse with block size of 1x4. optimized_ops::FullyConnectedSparseWeight1x4( - sparsity, op_params, GetTensorShape(input), - GetTensorData(input), GetTensorShape(filter), - GetTensorData(filter), GetTensorShape(bias), - GetTensorData(bias), GetTensorShape(output), - GetTensorData(output), + sparsity, op_params, // Disable formatting + input_shape, GetTensorData(input), // Disable formatting + filter_shape, GetTensorData(filter), // Disable formatting + bias_shape, GetTensorData(bias), // Disable formatting + output_shape, GetTensorData(output), CpuBackendContext::GetFromContext(context)); } else { TF_LITE_KERNEL_LOG(context, diff --git a/tensorflow/lite/kernels/gather_nd.cc b/tensorflow/lite/kernels/gather_nd.cc index c39917b478505f..47e4119b5e57ac 100644 --- a/tensorflow/lite/kernels/gather_nd.cc +++ b/tensorflow/lite/kernels/gather_nd.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include +#include "tensorflow/lite/c/c_api_types.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h" #include "tensorflow/lite/kernels/internal/reference/reference_ops.h" @@ -102,13 +103,16 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { } template -TfLiteStatus GatherNd(const TfLiteTensor* params, const TfLiteTensor* indices, - TfLiteTensor* output) { - reference_ops::GatherNd( +TfLiteStatus GatherNd(TfLiteContext* context, const TfLiteTensor* params, + const TfLiteTensor* indices, TfLiteTensor* output) { + const TfLiteStatus status = reference_ops::GatherNd( GetTensorShape(params), GetTensorData(params), GetTensorShape(indices), GetTensorData(indices), GetTensorShape(output), GetTensorData(output)); - return kTfLiteOk; + if (status != kTfLiteOk) { + TF_LITE_KERNEL_LOG(context, "gather_nd index out of bounds"); + } + return status; } template @@ -136,17 +140,17 @@ TfLiteStatus EvalGatherNd(TfLiteContext* context, const TfLiteTensor* params, switch (params->type) { case kTfLiteFloat32: - return GatherNd(params, indices, output); + return GatherNd(context, params, indices, output); case kTfLiteUInt8: - return GatherNd(params, indices, output); + return GatherNd(context, params, indices, output); case kTfLiteInt8: - return GatherNd(params, indices, output); + return GatherNd(context, params, indices, output); case kTfLiteInt16: - return GatherNd(params, indices, output); + return GatherNd(context, params, indices, output); case kTfLiteInt32: - return GatherNd(params, indices, output); + return GatherNd(context, params, indices, output); case kTfLiteInt64: - return GatherNd(params, indices, output); + return GatherNd(context, params, indices, output); case kTfLiteString: return GatherNdString(params, indices, output); default: diff --git a/tensorflow/lite/kernels/gather_nd_test.cc b/tensorflow/lite/kernels/gather_nd_test.cc index 1e9a6fce252255..99768de0ba976f 100644 --- a/tensorflow/lite/kernels/gather_nd_test.cc +++ b/tensorflow/lite/kernels/gather_nd_test.cc @@ -73,6 +73,22 @@ TEST(GatherNdOpTest, ElementIndexingIntoMatrix) { EXPECT_THAT(m.GetOutput(), ElementsAreArray({1.1, 2.2})); } +TEST(GatherNdOpTest, ErrorOnOutOfBoundsTooLarge) { + GatherNdOpModel m({TensorType_FLOAT32, {2, 2}}, {TensorType_INT32, {2, 2}}); + m.SetInput({1.1, 1.2, 2.1, 2.2}); + m.SetPositions({0, 0, 2, 0}); + EXPECT_EQ(m.Invoke(), kTfLiteError); + m.SetPositions({0, 0, 1, 2}); + EXPECT_EQ(m.Invoke(), kTfLiteError); +} + +TEST(GatherNdOpTest, ErrorOnOutOfBoundsNegative) { + GatherNdOpModel m({TensorType_FLOAT32, {2, 2}}, {TensorType_INT32, {2, 2}}); + m.SetInput({1.1, 1.2, 2.1, 2.2}); + m.SetPositions({1, -1, 1, 1}); + EXPECT_EQ(m.Invoke(), kTfLiteError); +} + TEST(GatherNdOpTest, SliceIndexingIntoMatrix) { GatherNdOpModel m({TensorType_FLOAT32, {2, 2}}, {TensorType_INT32, {2, 1}}); m.SetInput({1.1, 1.2, 2.1, 2.2}); diff --git a/tensorflow/lite/kernels/internal/common.h b/tensorflow/lite/kernels/internal/common.h index 098b6bebb8d9a3..74ce7e37168eb5 100644 --- a/tensorflow/lite/kernels/internal/common.h +++ b/tensorflow/lite/kernels/internal/common.h @@ -75,6 +75,7 @@ float ActivationFunction(float x) { inline void BiasAndClamp(float clamp_min, float clamp_max, int bias_size, const float* bias_data, int array_size, float* array_data) { + if (bias_size == 0) return; // Note: see b/132215220: in May 2019 we thought it would be OK to replace // this with the Eigen one-liner: // return (array.colwise() + bias).cwiseMin(clamp_max).cwiseMin(clamp_max). diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h index 969db74107b841..3b11e39df17658 100644 --- a/tensorflow/lite/kernels/internal/reference/reference_ops.h +++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h @@ -29,6 +29,7 @@ limitations under the License. #include "third_party/eigen3/Eigen/Core" #include "fixedpoint/fixedpoint.h" #include "ruy/profiler/instrumentation.h" // from @ruy +#include "tensorflow/lite/c/c_api_types.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/common.h" #include "tensorflow/lite/kernels/internal/quantization_util.h" @@ -971,23 +972,31 @@ inline GatherNdHelperResult GatherNdHelper(const RuntimeShape& params_shape, return ret; } +// Implements GatherNd. +// Returns an error if any of the indices_data would cause an out of bounds +// memory read. template -inline void GatherNd(const RuntimeShape& params_shape, - const ParamsT* params_data, - const RuntimeShape& indices_shape, - const IndicesT* indices_data, - const RuntimeShape& output_shape, ParamsT* output_data) { +inline TfLiteStatus GatherNd(const RuntimeShape& params_shape, + const ParamsT* params_data, + const RuntimeShape& indices_shape, + const IndicesT* indices_data, + const RuntimeShape& output_shape, + ParamsT* output_data) { ruy::profiler::ScopeLabel label("GatherNd"); const GatherNdHelperResult res = GatherNdHelper(params_shape, indices_shape); for (int i = 0; i < res.n_slices; ++i) { - int from_pos = 0; + int64_t from_pos = 0; for (int j = 0; j < res.indices_nd; ++j) { from_pos += indices_data[i * res.indices_nd + j] * res.dims_to_count[j]; } + if (from_pos < 0 || from_pos + res.slice_size > params_shape.FlatSize()) { + return kTfLiteError; + } std::memcpy(output_data + i * res.slice_size, params_data + from_pos, sizeof(ParamsT) * res.slice_size); } + return kTfLiteOk; } #ifndef TF_LITE_STATIC_MEMORY @@ -1016,11 +1025,12 @@ inline void GatherNdString(const RuntimeShape& params_shape, #endif template -inline void ScatterNd(const RuntimeShape& indices_shape, - const IndicesT* indices_data, - const RuntimeShape& updates_shape, - const UpdatesT* updates_data, - const RuntimeShape& output_shape, UpdatesT* output_data) { +inline TfLiteStatus ScatterNd(const RuntimeShape& indices_shape, + const IndicesT* indices_data, + const RuntimeShape& updates_shape, + const UpdatesT* updates_data, + const RuntimeShape& output_shape, + UpdatesT* output_data) { ruy::profiler::ScopeLabel label("ScatterNd"); int n_slices = 1; @@ -1043,18 +1053,24 @@ inline void ScatterNd(const RuntimeShape& indices_shape, remain_flat_size = dims_to_count[i]; } + if (n_slices * slice_size > updates_shape.FlatSize()) { + return kTfLiteError; + } memset(output_data, 0, sizeof(UpdatesT) * output_flat_size); for (int i = 0; i < n_slices; ++i) { int to_pos = 0; for (int j = 0; j < indices_nd; ++j) { IndicesT idx = indices_data[i * indices_nd + j]; - TFLITE_DCHECK(0 <= idx && idx < output_shape.Dims(j)); to_pos += idx * dims_to_count[j]; } + if (to_pos < 0 || to_pos + slice_size > output_flat_size) { + return kTfLiteError; + } for (int j = 0; j < slice_size; j++) { output_data[to_pos + j] += updates_data[i * slice_size + j]; } } + return kTfLiteOk; } template diff --git a/tensorflow/lite/kernels/internal/utils/sparsity_format_converter.cc b/tensorflow/lite/kernels/internal/utils/sparsity_format_converter.cc index 22aa5d019e7395..0595d49365c387 100644 --- a/tensorflow/lite/kernels/internal/utils/sparsity_format_converter.cc +++ b/tensorflow/lite/kernels/internal/utils/sparsity_format_converter.cc @@ -282,10 +282,12 @@ void FormatConverter::InitSparseToDenseConverter( block_size_.resize(block_map_.size()); for (int i = 0; i < original_rank; i++) { if (block_dim < block_map_.size() && block_map_[block_dim] == i) { - int orig_dim = traversal_order_[original_rank + block_dim]; - block_size_[block_dim] = dense_size[orig_dim]; - blocked_shape_[i] = dense_shape_[i] / dense_size[orig_dim]; - block_dim++; + if (original_rank + block_dim < traversal_order_.size()) { + int orig_dim = traversal_order_[original_rank + block_dim]; + block_size_[block_dim] = dense_size[orig_dim]; + blocked_shape_[i] = dense_shape_[i] / dense_size[orig_dim]; + block_dim++; + } } else { blocked_shape_[i] = dense_shape_[i]; } @@ -328,13 +330,15 @@ void FormatConverter::Populate(const T* src_data, std::vector indices, Populate(src_data, indices, level + 1, prev_idx * shape_of_level + i, src_data_ptr, dest_data); } - } else { + } else if (prev_idx + 1 < dim_metadata_[metadata_idx].size()) { const auto& array_segments = dim_metadata_[metadata_idx]; const auto& array_indices = dim_metadata_[metadata_idx + 1]; for (int i = array_segments[prev_idx]; i < array_segments[prev_idx + 1]; i++) { - indices[level] = array_indices[i]; - Populate(src_data, indices, level + 1, i, src_data_ptr, dest_data); + if (i < array_indices.size() && level < indices.size()) { + indices[level] = array_indices[i]; + Populate(src_data, indices, level + 1, i, src_data_ptr, dest_data); + } } } } diff --git a/tensorflow/lite/kernels/scatter_nd.cc b/tensorflow/lite/kernels/scatter_nd.cc index 93e2fe36c3fb26..144c07a2da7b4f 100644 --- a/tensorflow/lite/kernels/scatter_nd.cc +++ b/tensorflow/lite/kernels/scatter_nd.cc @@ -128,11 +128,10 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { template TfLiteStatus ScatterNd(const TfLiteTensor* indices, const TfLiteTensor* updates, TfLiteTensor* output) { - reference_ops::ScatterNd( + return reference_ops::ScatterNd( GetTensorShape(indices), GetTensorData(indices), GetTensorShape(updates), GetTensorData(updates), GetTensorShape(output), GetTensorData(output)); - return kTfLiteOk; } template @@ -148,23 +147,36 @@ TfLiteStatus EvalScatterNd(TfLiteContext* context, const TfLiteTensor* indices, ResizeOutputTensor(context, shape, output)); } + TfLiteStatus status = kTfLiteError; switch (updates->type) { case kTfLiteFloat32: - return ScatterNd(indices, updates, output); + status = ScatterNd(indices, updates, output); + break; case kTfLiteUInt8: - return ScatterNd(indices, updates, output); + status = ScatterNd(indices, updates, output); + break; + case kTfLiteBool: + status = ScatterNd(indices, updates, output); + break; case kTfLiteInt8: - return ScatterNd(indices, updates, output); + status = ScatterNd(indices, updates, output); + break; case kTfLiteInt32: - return ScatterNd(indices, updates, output); + status = ScatterNd(indices, updates, output); + break; case kTfLiteInt64: - return ScatterNd(indices, updates, output); + status = ScatterNd(indices, updates, output); + break; default: context->ReportError( context, "Updates of type '%s' are not supported by scatter_nd.", TfLiteTypeGetName(updates->type)); return kTfLiteError; } + if (status != kTfLiteOk) { + context->ReportError(context, "scatter_nd index out of bounds"); + } + return status; } TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { diff --git a/tensorflow/lite/kernels/scatter_nd_test.cc b/tensorflow/lite/kernels/scatter_nd_test.cc index 9fdf176fe1f8cd..154596cd4dd90c 100644 --- a/tensorflow/lite/kernels/scatter_nd_test.cc +++ b/tensorflow/lite/kernels/scatter_nd_test.cc @@ -347,5 +347,34 @@ TEST(ScatterNdOpTest, DynamicShape) { /*2, 3*/ 1, 2, 3, 4, 5})); } +TEST(ScatterNdOpTest, ReadAndWriteArrayLimits) { + ScatterNdOpModel m({TensorType_INT32, {5, 1}}, {TensorType_INT32, {5}}, + {TensorType_INT32, {1}}); + m.SetIndices({4, 3, 1, 0, 2}); + m.SetUpdates({1, 2, 3, 7, 9}); + m.SetShape({5}); + ASSERT_EQ(m.Invoke(), kTfLiteOk); + EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({5})); + EXPECT_THAT(m.GetOutput(), ElementsAreArray({7, 3, 9, 2, 1})); +} + +TEST(ScatterNdOpTest, OOBRead) { + ScatterNdOpModel m({TensorType_INT32, {1, 1}}, {TensorType_INT32, {1}}, + {TensorType_INT32, {1}}); + m.SetIndices({4}); + m.SetUpdates({1}); + m.SetShape({1}); + ASSERT_EQ(m.Invoke(), kTfLiteError); +} + +TEST(ScatterNdOpTest, OOBWrites) { + ScatterNdOpModel m({TensorType_INT32, {5, 1}}, {TensorType_INT32, {5}}, + {TensorType_INT32, {1}}); + m.SetIndices({4, 3, 1, -0x38, 0x38}); + m.SetUpdates({1, 2, 3, 0x44444444, 0x55555555}); + m.SetShape({1}); + ASSERT_EQ(m.Invoke(), kTfLiteError); +} + } // namespace } // namespace tflite diff --git a/tensorflow/lite/python/interpreter_wrapper/numpy.cc b/tensorflow/lite/python/interpreter_wrapper/numpy.cc index 5fabf660e2e1a9..452d7da60a7971 100644 --- a/tensorflow/lite/python/interpreter_wrapper/numpy.cc +++ b/tensorflow/lite/python/interpreter_wrapper/numpy.cc @@ -162,6 +162,13 @@ bool FillStringBufferFromPyString(PyObject* value, bool FillStringBufferWithPyArray(PyObject* value, DynamicBuffer* dynamic_buffer) { + if (!PyArray_Check(value)) { + PyErr_Format(PyExc_ValueError, + "Passed in value type is not a numpy array, got type %s.", + value->ob_type->tp_name); + return false; + } + PyArrayObject* array = reinterpret_cast(value); switch (PyArray_TYPE(array)) { case NPY_OBJECT: diff --git a/tensorflow/lite/python/lite_v2_test.py b/tensorflow/lite/python/lite_v2_test.py index cf4a3c03420198..89ab738ad58336 100644 --- a/tensorflow/lite/python/lite_v2_test.py +++ b/tensorflow/lite/python/lite_v2_test.py @@ -138,6 +138,35 @@ def testScalarInput(self): actual_value = self._evaluateTFLiteModel(tflite_model, [input_data]) self.assertEqual(expected_value.numpy(), actual_value) + @test_util.run_v2_only + def testStringInput(self): + + class Model(tf.Module): + + @tf.function + def __call__(self, x): + return x + + root = Model() + concrete_func = root.__call__.get_concrete_function( + tf.constant([str(x) for x in range(11)])) + # Convert model. + converter = lite.TFLiteConverterV2.from_concrete_functions([concrete_func], + root) + tflite_model = converter.convert() + input_data = tf.constant([str(x) for x in range(11)], + shape=(11,), + dtype=tf.dtypes.string) + # Check values from converted model. + interpreter = tf.lite.Interpreter(model_content=tflite_model) + interpreter.allocate_tensors() + my_signature = interpreter.get_signature_runner() + + with self.assertRaises(ValueError) as error: + _ = my_signature(x=input_data) + self.assertIn('Passed in value type is not a numpy array, got type ', + str(error.exception)) + @test_util.run_v2_only def testMultiFunctionModel(self): """Convert a single model in a multi-functional model.""" diff --git a/tensorflow/lite/schema/upgrade_schema_test.py b/tensorflow/lite/schema/upgrade_schema_test.py index e55925053e0ae9..99154ccb205b7f 100644 --- a/tensorflow/lite/schema/upgrade_schema_test.py +++ b/tensorflow/lite/schema/upgrade_schema_test.py @@ -254,13 +254,13 @@ class TestSchemaUpgrade(test_util.TensorFlowTestCase): def testNonExistentFile(self): converter = upgrade_schema_lib.Converter() - non_existent = tempfile.mktemp(suffix=".json") + _, non_existent = tempfile.mkstemp(suffix=".json") # safe to ignore fd with self.assertRaisesRegex(IOError, "No such file or directory"): converter.Convert(non_existent, non_existent) def testInvalidExtension(self): converter = upgrade_schema_lib.Converter() - invalid_extension = tempfile.mktemp(suffix=".foo") + _, invalid_extension = tempfile.mkstemp(suffix=".foo") # safe to ignore fd with self.assertRaisesRegex(ValueError, "Invalid extension on input"): converter.Convert(invalid_extension, invalid_extension) with tempfile.NamedTemporaryFile(suffix=".json", mode="w+") as in_json: diff --git a/tensorflow/lite/tools/benchmark/BUILD b/tensorflow/lite/tools/benchmark/BUILD index b6e21763c8e082..e37481c2889872 100644 --- a/tensorflow/lite/tools/benchmark/BUILD +++ b/tensorflow/lite/tools/benchmark/BUILD @@ -185,6 +185,7 @@ cc_library( "@com_google_absl//absl/memory", "//tensorflow/core/util:stats_calculator_portable", "//tensorflow/lite/c:common", + "//tensorflow/lite/delegates/hexagon:hexagon_delegate", "//tensorflow/lite/nnapi:nnapi_util", "//tensorflow/lite/profiling:time", "//tensorflow/lite/tools:command_line_flags", diff --git a/tensorflow/lite/tools/benchmark/benchmark_performance_options.cc b/tensorflow/lite/tools/benchmark/benchmark_performance_options.cc index 438fb485704f9d..adb66b2113bb65 100644 --- a/tensorflow/lite/tools/benchmark/benchmark_performance_options.cc +++ b/tensorflow/lite/tools/benchmark/benchmark_performance_options.cc @@ -33,11 +33,6 @@ limitations under the License. #include "tensorflow/lite/tools/command_line_flags.h" #include "tensorflow/lite/tools/logging.h" -#if (defined(ANDROID) || defined(__ANDROID__)) && \ - (defined(__arm__) || defined(__aarch64__)) -#define TFLITE_ENABLE_HEXAGON -#endif - namespace tflite { namespace benchmark { diff --git a/tensorflow/lite/tools/delegates/hexagon_delegate_provider.cc b/tensorflow/lite/tools/delegates/hexagon_delegate_provider.cc index 2124bec2447c88..5fa1324374ecc3 100644 --- a/tensorflow/lite/tools/delegates/hexagon_delegate_provider.cc +++ b/tensorflow/lite/tools/delegates/hexagon_delegate_provider.cc @@ -18,10 +18,6 @@ limitations under the License. #include "tensorflow/lite/tools/delegates/delegate_provider.h" #include "tensorflow/lite/tools/evaluation/utils.h" -#if !defined(__APPLE__) && (defined(__arm__) || defined(__aarch64__)) -#define TFLITE_ENABLE_HEXAGON -#endif - #if defined(TFLITE_ENABLE_HEXAGON) #include "tensorflow/lite/delegates/hexagon/hexagon_delegate.h" #endif diff --git a/tensorflow/lite/tools/evaluation/utils.cc b/tensorflow/lite/tools/evaluation/utils.cc index 16edffce327dd8..872c0fec56d234 100644 --- a/tensorflow/lite/tools/evaluation/utils.cc +++ b/tensorflow/lite/tools/evaluation/utils.cc @@ -137,16 +137,16 @@ TfLiteDelegatePtr CreateGPUDelegate() { TfLiteDelegatePtr CreateHexagonDelegate( const std::string& library_directory_path, bool profiling) { -#if !defined(__APPLE__) && (defined(__arm__) || defined(__aarch64__)) +#if TFLITE_ENABLE_HEXAGON TfLiteHexagonDelegateOptions options = {0}; options.print_graph_profile = profiling; return CreateHexagonDelegate(&options, library_directory_path); #else return CreateNullDelegate(); -#endif // defined(__arm__) +#endif // TFLITE_ENABLE_HEXAGON } -#if !defined(__APPLE__) && (defined(__arm__) || defined(__aarch64__)) +#if TFLITE_ENABLE_HEXAGON TfLiteDelegatePtr CreateHexagonDelegate( const TfLiteHexagonDelegateOptions* options, const std::string& library_directory_path) { diff --git a/tensorflow/lite/tools/evaluation/utils.h b/tensorflow/lite/tools/evaluation/utils.h index b0e2f057ccc151..4cae2c8102d708 100644 --- a/tensorflow/lite/tools/evaluation/utils.h +++ b/tensorflow/lite/tools/evaluation/utils.h @@ -31,7 +31,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/delegate.h" #endif -#if !defined(__APPLE__) && (defined(__arm__) || defined(__aarch64__)) +#if TFLITE_ENABLE_HEXAGON #include "tensorflow/lite/delegates/hexagon/hexagon_delegate.h" #endif @@ -77,7 +77,7 @@ TfLiteDelegatePtr CreateGPUDelegate(TfLiteGpuDelegateOptionsV2* options); TfLiteDelegatePtr CreateHexagonDelegate( const std::string& library_directory_path, bool profiling); -#if !defined(__APPLE__) && (defined(__arm__) || defined(__aarch64__)) +#if TFLITE_ENABLE_HEXAGON TfLiteDelegatePtr CreateHexagonDelegate( const TfLiteHexagonDelegateOptions* options, const std::string& library_directory_path); diff --git a/tensorflow/lite/tools/pip_package/build_pip_package_with_bazel.sh b/tensorflow/lite/tools/pip_package/build_pip_package_with_bazel.sh index 0c58d489318c35..2712baea396d00 100755 --- a/tensorflow/lite/tools/pip_package/build_pip_package_with_bazel.sh +++ b/tensorflow/lite/tools/pip_package/build_pip_package_with_bazel.sh @@ -107,12 +107,14 @@ chmod u+w "${BUILD_DIR}/tflite_runtime/_pywrap_tensorflow_interpreter_wrapper${L cd "${BUILD_DIR}" case "${TENSORFLOW_TARGET}" in armhf) - ${PYTHON} setup.py bdist --plat-name=linux-armv7l \ - bdist_wheel --plat-name=linux-armv7l + WHEEL_PLATFORM_NAME="${WHEEL_PLATFORM_NAME:-linux-armv7l}" + ${PYTHON} setup.py bdist --plat-name=${WHEEL_PLATFORM_NAME} \ + bdist_wheel --plat-name=${WHEEL_PLATFORM_NAME} ;; aarch64) - ${PYTHON} setup.py bdist --plat-name=linux-aarch64 \ - bdist_wheel --plat-name=linux-aarch64 + WHEEL_PLATFORM_NAME="${WHEEL_PLATFORM_NAME:-linux-aarch64}" + ${PYTHON} setup.py bdist --plat-name=${WHEEL_PLATFORM_NAME} \ + bdist_wheel --plat-name=${WHEEL_PLATFORM_NAME} ;; *) if [[ -n "${TENSORFLOW_TARGET}" ]] && [[ -n "${TENSORFLOW_TARGET_ARCH}" ]]; then diff --git a/tensorflow/lite/tools/pip_package/build_pip_package_with_cmake.sh b/tensorflow/lite/tools/pip_package/build_pip_package_with_cmake.sh index f887375ab9b73c..5c60a6831721a2 100755 --- a/tensorflow/lite/tools/pip_package/build_pip_package_with_cmake.sh +++ b/tensorflow/lite/tools/pip_package/build_pip_package_with_cmake.sh @@ -76,6 +76,7 @@ case "${TENSORFLOW_TARGET}" in -DCMAKE_CXX_FLAGS="${ARMCC_FLAGS}" \ -DCMAKE_SYSTEM_NAME=Linux \ -DCMAKE_SYSTEM_PROCESSOR=armv7 \ + -DTFLITE_ENABLE_XNNPACK=OFF \ "${TENSORFLOW_LITE_DIR}" ;; rpi0) @@ -142,16 +143,19 @@ chmod u+w "${BUILD_DIR}/tflite_runtime/_pywrap_tensorflow_interpreter_wrapper${L cd "${BUILD_DIR}" case "${TENSORFLOW_TARGET}" in armhf) - ${PYTHON} setup.py bdist --plat-name=linux-armv7l \ - bdist_wheel --plat-name=linux-armv7l + WHEEL_PLATFORM_NAME="${WHEEL_PLATFORM_NAME:-linux-armv7l}" + ${PYTHON} setup.py bdist --plat-name=${WHEEL_PLATFORM_NAME} \ + bdist_wheel --plat-name=${WHEEL_PLATFORM_NAME} ;; rpi0) - ${PYTHON} setup.py bdist --plat-name=linux_armv6l \ - bdist_wheel --plat-name=linux-armv6l + WHEEL_PLATFORM_NAME="${WHEEL_PLATFORM_NAME:-linux-armv6l}" + ${PYTHON} setup.py bdist --plat-name=${WHEEL_PLATFORM_NAME} \ + bdist_wheel --plat-name=${WHEEL_PLATFORM_NAME} ;; aarch64) - ${PYTHON} setup.py bdist --plat-name=linux-aarch64 \ - bdist_wheel --plat-name=linux-aarch64 + WHEEL_PLATFORM_NAME="${WHEEL_PLATFORM_NAME:-linux-aarch64}" + ${PYTHON} setup.py bdist --plat-name=${WHEEL_PLATFORM_NAME} \ + bdist_wheel --plat-name=${WHEEL_PLATFORM_NAME} ;; *) if [[ -n "${TENSORFLOW_TARGET}" ]] && [[ -n "${TENSORFLOW_TARGET_ARCH}" ]]; then diff --git a/tensorflow/lite/tools/pip_package/setup_with_binary.py b/tensorflow/lite/tools/pip_package/setup_with_binary.py index 6b9fe534bb2b19..fcfadcf5a20636 100644 --- a/tensorflow/lite/tools/pip_package/setup_with_binary.py +++ b/tensorflow/lite/tools/pip_package/setup_with_binary.py @@ -67,5 +67,6 @@ package_dir={'': '.'}, package_data={'': ['*.so', '*.pyd']}, install_requires=[ - 'numpy ~= 1.19.2', # Higher versions have a compatibility issue. + 'numpy >= 1.19.2', # Better to keep sync with both TF ci_build + # and OpenCV-Python requirement. ]) diff --git a/tensorflow/lite/util.cc b/tensorflow/lite/util.cc index 84dbc16b6079c0..cb2d1ef73a950a 100644 --- a/tensorflow/lite/util.cc +++ b/tensorflow/lite/util.cc @@ -27,6 +27,7 @@ limitations under the License. #include "tensorflow/lite/builtin_ops.h" #include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/core/macros.h" #include "tensorflow/lite/schema/schema_generated.h" namespace tflite { @@ -176,4 +177,19 @@ bool IsValidationSubgraph(const char* name) { // NOLINTNEXTLINE: can't use absl::StartsWith as absl is not allowed. return name && std::string(name).find(kValidationSubgraphNamePrefix) == 0; } + +TfLiteStatus MultiplyAndCheckOverflow(size_t a, size_t b, size_t* product) { + // Multiplying a * b where a and b are size_t cannot result in overflow in a + // size_t accumulator if both numbers have no non-zero bits in their upper + // half. + constexpr size_t size_t_bits = 8 * sizeof(size_t); + constexpr size_t overflow_upper_half_bit_position = size_t_bits / 2; + *product = a * b; + // If neither integers have non-zero bits past 32 bits can't overflow. + // Otherwise check using slow devision. + if (TFLITE_EXPECT_FALSE((a | b) >> overflow_upper_half_bit_position != 0)) { + if (a != 0 && *product / a != b) return kTfLiteError; + } + return kTfLiteOk; +} } // namespace tflite diff --git a/tensorflow/lite/util.h b/tensorflow/lite/util.h index d9d7f7a0a8e673..e6a1aefcd9e5b8 100644 --- a/tensorflow/lite/util.h +++ b/tensorflow/lite/util.h @@ -99,6 +99,12 @@ constexpr char kValidationSubgraphNamePrefix[] = "VALIDATION:"; // Checks whether the prefix of the subgraph name indicates the subgraph is a // validation subgraph. bool IsValidationSubgraph(const char* name); + +// Multiply two sizes and return true if overflow occurred; +// This is based off tensorflow/overflow.h but is simpler as we already +// have unsigned numbers. It is also generalized to work where sizeof(size_t) +// is not 8. +TfLiteStatus MultiplyAndCheckOverflow(size_t a, size_t b, size_t* product); } // namespace tflite #endif // TENSORFLOW_LITE_UTIL_H_ diff --git a/tensorflow/lite/util_test.cc b/tensorflow/lite/util_test.cc index a2e49f05274d6e..8b10de8110fb1a 100644 --- a/tensorflow/lite/util_test.cc +++ b/tensorflow/lite/util_test.cc @@ -22,6 +22,7 @@ limitations under the License. #include #include +#include "tensorflow/lite/c/c_api_types.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/schema/schema_generated.h" @@ -130,5 +131,12 @@ TEST(ValidationSubgraph, NameIsDetected) { EXPECT_TRUE(IsValidationSubgraph("VALIDATION:main")); } +TEST(MultiplyAndCheckOverflow, Validate) { + size_t res = 0; + EXPECT_TRUE(MultiplyAndCheckOverflow(1, 2, &res) == kTfLiteOk); + EXPECT_FALSE(MultiplyAndCheckOverflow(static_cast(123456789023), + 1223423425, &res) == kTfLiteOk); +} + } // namespace } // namespace tflite diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD index 2fb8e112a280e5..f6b64b0b9fe453 100644 --- a/tensorflow/python/data/kernel_tests/BUILD +++ b/tensorflow/python/data/kernel_tests/BUILD @@ -68,6 +68,9 @@ tf_py_test( name = "cache_test", size = "small", srcs = ["cache_test.py"], + tags = [ + "no_oss", # Times out during patch release + ], deps = [ ":checkpoint_test_base", ":test_base", @@ -536,6 +539,9 @@ tf_py_test( size = "small", srcs = ["map_test.py"], shard_count = 19, + tags = [ + "no_oss", # Times out during patch release + ], deps = [ ":checkpoint_test_base", ":test_base", diff --git a/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py index 8f93530010caec..7abe36eaa4528f 100644 --- a/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py +++ b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py @@ -138,6 +138,25 @@ def testEmptySparseTensorSlicesInvalid(self): with self.assertRaises(errors.InvalidArgumentError): sess.run(init_op, feed_dict={st: sparse_feed}) + @combinations.generate(combinations.combine(tf_api_version=1, mode=["graph"])) + def testEmptySparseTensorSlicesInvalid2(self): + """Test a dataset based on invalid `tf.sparse.SparseTensor`.""" + st = array_ops.sparse_placeholder(dtypes.float64) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_sparse_tensor_slices(st)) + init_op = iterator.initializer + + with self.cached_session() as sess: + # Test with an empty sparse tensor but with non empty values. + empty_indices = [[]] + empty_values = [] + dense_shape = [1, 1] + sparse_feed = sparse_tensor.SparseTensorValue(empty_indices, empty_values, + dense_shape) + # Here, we expect the test to fail when running the feed. + with self.assertRaises(errors.InvalidArgumentError): + sess.run(init_op, feed_dict={st: sparse_feed}) + @combinations.generate(combinations.combine(tf_api_version=2, mode=["eager"])) def testFromSparseTensorSlicesError(self): with self.assertRaises(AttributeError): diff --git a/tensorflow/python/debug/cli/curses_ui_test.py b/tensorflow/python/debug/cli/curses_ui_test.py index 3ffa031923dc53..db844f24a9a82f 100644 --- a/tensorflow/python/debug/cli/curses_ui_test.py +++ b/tensorflow/python/debug/cli/curses_ui_test.py @@ -90,8 +90,9 @@ def __init__(self, # Override the default path to the command history file to avoid test # concurrency issues. + _, history_file_path = tempfile.mkstemp() # safe to ignore fd self._command_history_store = debugger_cli_common.CommandHistory( - history_file_path=tempfile.mktemp()) + history_file_path=history_file_path) # Below, override the _screen_ prefixed member methods that interact with the # actual terminal, so that the mock can run in a terminal-less environment. diff --git a/tensorflow/python/debug/cli/debugger_cli_common_test.py b/tensorflow/python/debug/cli/debugger_cli_common_test.py index 93df845c4c585d..6d0bd2bbd906b4 100644 --- a/tensorflow/python/debug/cli/debugger_cli_common_test.py +++ b/tensorflow/python/debug/cli/debugger_cli_common_test.py @@ -253,7 +253,9 @@ def testWriteToFileSucceeds(self): font_attr_segs={0: [(0, 5, "red")], 1: [(0, 7, "blue")]}) - file_path = tempfile.mktemp() + fd, file_path = tempfile.mkstemp() + os.close(fd) # file opened exclusively, so we need to close this + # a better fix would be to make the API take a fd screen_output.write_to_file(file_path) with gfile.Open(file_path, "r") as f: @@ -930,12 +932,13 @@ def testDeregisterNonexistentContext(self): class CommandHistoryTest(test_util.TensorFlowTestCase): def setUp(self): - self._history_file_path = tempfile.mktemp() + self._fd, self._history_file_path = tempfile.mkstemp() self._cmd_hist = debugger_cli_common.CommandHistory( limit=3, history_file_path=self._history_file_path) def tearDown(self): if os.path.isfile(self._history_file_path): + os.close(self._fd) os.remove(self._history_file_path) def _restoreFileReadWritePermissions(self, file_path): @@ -1002,13 +1005,6 @@ def testRepeatingCommandsDoNotGetLoggedRepeatedly(self): self.assertEqual(["help"], self._cmd_hist.most_recent_n(2)) - def testCommandHistoryFileIsCreated(self): - self.assertFalse(os.path.isfile(self._history_file_path)) - self._cmd_hist.add_command("help") - self.assertTrue(os.path.isfile(self._history_file_path)) - with open(self._history_file_path, "rt") as f: - self.assertEqual(["help\n"], f.readlines()) - def testLoadingCommandHistoryFileObeysLimit(self): self._cmd_hist.add_command("help 1") self._cmd_hist.add_command("help 2") diff --git a/tensorflow/python/debug/cli/readline_ui_test.py b/tensorflow/python/debug/cli/readline_ui_test.py index 011ba23fc4d63b..64351ceb6b820a 100644 --- a/tensorflow/python/debug/cli/readline_ui_test.py +++ b/tensorflow/python/debug/cli/readline_ui_test.py @@ -35,9 +35,11 @@ class MockReadlineUI(readline_ui.ReadlineUI): """Test subclass of ReadlineUI that bypasses terminal manipulations.""" def __init__(self, on_ui_exit=None, command_sequence=None): + _, config_file_path = tempfile.mkstemp() # safe to ignore fd readline_ui.ReadlineUI.__init__( - self, on_ui_exit=on_ui_exit, - config=cli_config.CLIConfig(config_file_path=tempfile.mktemp())) + self, + on_ui_exit=on_ui_exit, + config=cli_config.CLIConfig(config_file_path=config_file_path)) self._command_sequence = command_sequence self._command_counter = 0 @@ -168,7 +170,7 @@ def callback_for_test(): self.assertTrue(observer["callback_invoked"]) def testIncompleteRedirectWorks(self): - output_path = tempfile.mktemp() + _, output_path = tempfile.mkstemp() # safe to ignore fd ui = MockReadlineUI( command_sequence=["babble -n 2 > %s" % output_path, "exit"]) diff --git a/tensorflow/python/debug/examples/v1/debug_errors.py b/tensorflow/python/debug/examples/v1/debug_errors.py index 5480a9b6f544e0..83c497999e40eb 100644 --- a/tensorflow/python/debug/examples/v1/debug_errors.py +++ b/tensorflow/python/debug/examples/v1/debug_errors.py @@ -44,9 +44,11 @@ def main(_): z = tf.matmul(m, v, name="z") if FLAGS.debug: - config_file_path = ( - tempfile.mktemp(".tfdbg_config") - if FLAGS.use_random_config_path else None) + if FLAGS.use_random_config_path: + # TODO(mihaimaruseac): Safe to ignore fd here? + _, config_file_path = tempfile.mkstemp(".tfdbg_config") + else: + config_file_path = None sess = tf_debug.LocalCLIDebugWrapperSession( sess, ui_type=FLAGS.ui_type, config_file_path=config_file_path) diff --git a/tensorflow/python/debug/examples/v1/debug_keras.py b/tensorflow/python/debug/examples/v1/debug_keras.py index ffc575776c26d2..4f7405a4deea71 100644 --- a/tensorflow/python/debug/examples/v1/debug_keras.py +++ b/tensorflow/python/debug/examples/v1/debug_keras.py @@ -44,9 +44,11 @@ def main(_): sess = tf.Session() if FLAGS.debug: # Use the command-line interface (CLI) of tfdbg. - config_file_path = ( - tempfile.mktemp(".tfdbg_config") - if FLAGS.use_random_config_path else None) + if FLAGS.use_random_config_path: + # TODO(mihaimaruseac): Safe to ignore fd here? + _, config_file_path = tempfile.mkstemp(".tfdbg_config") + else: + config_file_path = None sess = tf_debug.LocalCLIDebugWrapperSession( sess, ui_type=FLAGS.ui_type, config_file_path=config_file_path) elif FLAGS.tensorboard_debug_address: diff --git a/tensorflow/python/debug/examples/v1/debug_mnist_v1.py b/tensorflow/python/debug/examples/v1/debug_mnist_v1.py index cde1fb97ff280d..d2e67e85b41dd0 100644 --- a/tensorflow/python/debug/examples/v1/debug_mnist_v1.py +++ b/tensorflow/python/debug/examples/v1/debug_mnist_v1.py @@ -214,9 +214,11 @@ def nn_layer(input_tensor, input_dim, output_dim, layer_name, act=tf.nn.relu): "The --debug and --tensorboard_debug_address flags are mutually " "exclusive.") if FLAGS.debug: - config_file_path = ( - tempfile.mktemp(".tfdbg_config") - if FLAGS.use_random_config_path else None) + if FLAGS.use_random_config_path: + # TODO(mihaimaruseac): Safe to ignore fd here? + _, config_file_path = tempfile.mkstemp(".tfdbg_config") + else: + config_file_path = None sess = tf_debug.LocalCLIDebugWrapperSession( sess, ui_type=FLAGS.ui_type, config_file_path=config_file_path) elif FLAGS.tensorboard_debug_address: diff --git a/tensorflow/python/debug/examples/v1/debug_tflearn_iris.py b/tensorflow/python/debug/examples/v1/debug_tflearn_iris.py index 81f41247fd35dc..0de57ef2f81a31 100644 --- a/tensorflow/python/debug/examples/v1/debug_tflearn_iris.py +++ b/tensorflow/python/debug/examples/v1/debug_tflearn_iris.py @@ -62,9 +62,11 @@ def test_input_fn(): "exclusive.") hooks = [] if FLAGS.debug: - config_file_path = ( - tempfile.mktemp(".tfdbg_config") - if FLAGS.use_random_config_path else None) + if FLAGS.use_random_config_path: + # TODO(mihaimaruseac): Safe to ignore fd here? + _, config_file_path = tempfile.mkstemp(".tfdbg_config") + else: + config_file_path = None hooks.append( tf_debug.LocalCLIDebugHook( ui_type=FLAGS.ui_type, diff --git a/tensorflow/python/debug/lib/debug_data_test.py b/tensorflow/python/debug/lib/debug_data_test.py index d7ba5cde1f7dbb..e50c7498c096b7 100644 --- a/tensorflow/python/debug/lib/debug_data_test.py +++ b/tensorflow/python/debug/lib/debug_data_test.py @@ -151,8 +151,7 @@ def testDumpSizeBytesIsNoneForNonexistentFilePath(self): class DebugDumpDirTest(test_util.TensorFlowTestCase): def setUp(self): - self._dump_root = tempfile.mktemp() - os.mkdir(self._dump_root) + self._dump_root = tempfile.mkdtemp() def tearDown(self): # Tear down temporary dump directory. @@ -183,7 +182,7 @@ def _makeDataDirWithMultipleDevicesAndDuplicateNodeNames(self): def testDebugDumpDir_nonexistentDumpRoot(self): with self.assertRaisesRegex(IOError, "does not exist"): - debug_data.DebugDumpDir(tempfile.mktemp() + "_foo") + debug_data.DebugDumpDir(tempfile.mkdtemp() + "_foo") def testDebugDumpDir_invalidFileNamingPattern(self): # File name with too few underscores should lead to an exception. diff --git a/tensorflow/python/debug/lib/source_utils_test.py b/tensorflow/python/debug/lib/source_utils_test.py index 366b25e89ac367..ab0dbe616e3fcd 100644 --- a/tensorflow/python/debug/lib/source_utils_test.py +++ b/tensorflow/python/debug/lib/source_utils_test.py @@ -265,8 +265,8 @@ def testCallingAnnotateSourceWithoutPythonGraphRaisesException(self): def testCallingAnnotateSourceOnUnrelatedSourceFileDoesNotError(self): # Create an unrelated source file. - unrelated_source_path = tempfile.mktemp() - with open(unrelated_source_path, "wt") as source_file: + fd, unrelated_source_path = tempfile.mkstemp() + with open(fd, "wt") as source_file: source_file.write("print('hello, world')\n") self.assertEqual({}, @@ -277,8 +277,8 @@ def testCallingAnnotateSourceOnUnrelatedSourceFileDoesNotError(self): os.remove(unrelated_source_path) def testLoadingPythonSourceFileWithNonAsciiChars(self): - source_path = tempfile.mktemp() - with open(source_path, "wb") as source_file: + fd, source_path = tempfile.mkstemp() + with open(fd, "wb") as source_file: source_file.write(u"print('\U0001f642')\n".encode("utf-8")) source_lines, _ = source_utils.load_source(source_path) self.assertEqual(source_lines, [u"print('\U0001f642')", u""]) diff --git a/tensorflow/python/debug/wrappers/local_cli_wrapper.py b/tensorflow/python/debug/wrappers/local_cli_wrapper.py index 4069bdf1f3ffae..1fb2c1c104fe60 100644 --- a/tensorflow/python/debug/wrappers/local_cli_wrapper.py +++ b/tensorflow/python/debug/wrappers/local_cli_wrapper.py @@ -84,7 +84,7 @@ def __init__(self, self, sess, thread_name_filter=thread_name_filter) if not dump_root: - self._dump_root = tempfile.mktemp(prefix=_DUMP_ROOT_PREFIX) + self._dump_root = tempfile.mkdtemp(prefix=_DUMP_ROOT_PREFIX) else: dump_root = os.path.expanduser(dump_root) if os.path.isfile(dump_root): diff --git a/tensorflow/python/debug/wrappers/local_cli_wrapper_test.py b/tensorflow/python/debug/wrappers/local_cli_wrapper_test.py index 0d930b6e7e08a5..095d0fb4fd093c 100644 --- a/tensorflow/python/debug/wrappers/local_cli_wrapper_test.py +++ b/tensorflow/python/debug/wrappers/local_cli_wrapper_test.py @@ -116,7 +116,7 @@ def _launch_cli(self): config_file_path=os.path.join(tempfile.mkdtemp(), ".tfdbg_config"))) self._register_this_run_info(readline_cli) - while True: + while self._command_pointer < len(self._command_sequence): command = self._command_sequence[self._command_pointer] self._command_pointer += 1 @@ -136,7 +136,7 @@ def _launch_cli(self): class LocalCLIDebugWrapperSessionTest(test_util.TensorFlowTestCase): def setUp(self): - self._tmp_dir = tempfile.mktemp() + self._tmp_dir = tempfile.mkdtemp() self.v = variables.VariableV1(10.0, name="v") self.w = variables.VariableV1(21.0, name="w") @@ -178,15 +178,7 @@ def testConstructWrapper(self): local_cli_wrapper.LocalCLIDebugWrapperSession( session.Session(), log_usage=False) - def testConstructWrapperWithExistingEmptyDumpRoot(self): - os.mkdir(self._tmp_dir) - self.assertTrue(os.path.isdir(self._tmp_dir)) - - local_cli_wrapper.LocalCLIDebugWrapperSession( - session.Session(), dump_root=self._tmp_dir, log_usage=False) - def testConstructWrapperWithExistingNonEmptyDumpRoot(self): - os.mkdir(self._tmp_dir) dir_path = os.path.join(self._tmp_dir, "foo") os.mkdir(dir_path) self.assertTrue(os.path.isdir(dir_path)) @@ -197,7 +189,6 @@ def testConstructWrapperWithExistingNonEmptyDumpRoot(self): session.Session(), dump_root=self._tmp_dir, log_usage=False) def testConstructWrapperWithExistingFileDumpRoot(self): - os.mkdir(self._tmp_dir) file_path = os.path.join(self._tmp_dir, "foo") open(file_path, "a").close() # Create the file self.assertTrue(os.path.isfile(file_path)) @@ -533,16 +524,6 @@ def testRuntimeErrorShouldBeCaught(self): tf_error = wrapped_sess.observers["tf_errors"][0] self.assertEqual("y", tf_error.op.name) - def testRuntimeErrorBeforeGraphExecutionIsRaised(self): - # Use an impossible device name to cause an error before graph execution. - with ops.device("/device:GPU:1337"): - w = variables.VariableV1([1.0] * 10, name="w") - - wrapped_sess = LocalCLIDebuggerWrapperSessionForTest( - [["run"]], self.sess, dump_root=self._tmp_dir) - with self.assertRaisesRegex(errors.OpError, r".*[Dd]evice.*1337.*"): - wrapped_sess.run(w) - def testRunTillFilterPassesShouldLaunchCLIAtCorrectRun(self): wrapped_sess = LocalCLIDebuggerWrapperSessionForTest( [["run", "-f", "v_greater_than_twelve"], diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc index ee83953f590fa0..f5e4fb284783ab 100644 --- a/tensorflow/python/eager/pywrap_tfe_src.cc +++ b/tensorflow/python/eager/pywrap_tfe_src.cc @@ -678,9 +678,12 @@ bool SetOpAttrScalar(TFE_Context* ctx, TFE_Op* op, const char* key, for (int i = 0; i < num_dims; ++i) { tensorflow::Safe_PyObjectPtr inner_py_value( PySequence_ITEM(py_value, i)); + // If an error is generated when iterating through object, we can + // sometimes get a nullptr. if (inner_py_value.get() == Py_None) { dims[i] = -1; - } else if (!ParseDimensionValue(key, inner_py_value.get(), status, + } else if (inner_py_value.get() == nullptr || + !ParseDimensionValue(key, inner_py_value.get(), status, &dims[i])) { return false; } diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py index 1afbeab38982c4..f97a888e4d9139 100644 --- a/tensorflow/python/framework/test_util.py +++ b/tensorflow/python/framework/test_util.py @@ -2373,14 +2373,13 @@ def testMyOperator(self): """ stream.flush() fd = stream.fileno() - tmp_file_path = tempfile.mktemp(dir=self.get_temp_dir()) - tmp_file = open(tmp_file_path, "w") + tmp_file, tmp_file_path = tempfile.mkstemp(dir=self.get_temp_dir()) orig_fd = os.dup(fd) - os.dup2(tmp_file.fileno(), fd) + os.dup2(tmp_file, fd) try: yield CapturedWrites(tmp_file_path) finally: - tmp_file.close() + os.close(tmp_file) os.dup2(orig_fd, fd) def _AssertProtoEquals(self, a, b, msg=None): diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index fc04d89f775e31..bf5af0e75fcc07 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -33,6 +33,7 @@ tf_py_test( name = "attention_ops_test", size = "small", srcs = ["attention_ops_test.py"], + tags = ["no_oss"], deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -594,6 +595,7 @@ tf_py_test( size = "small", srcs = ["fractional_avg_pool_op_test.py"], shard_count = 5, + tags = ["no_oss"], deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -610,6 +612,7 @@ tf_py_test( size = "small", srcs = ["fractional_max_pool_op_test.py"], shard_count = 5, + tags = ["no_oss"], deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py index 8c6492dc53a56d..6b5bc5ad262606 100644 --- a/tensorflow/python/kernel_tests/array_ops_test.py +++ b/tensorflow/python/kernel_tests/array_ops_test.py @@ -353,6 +353,15 @@ def testExpandDimsWithNonScalarDim(self): "must be a tensor with a single value"): array_ops.expand_dims(1, axis=[0, 1]) + def testReshapeWithManyDims(self): + with self.assertRaisesRegex(errors.InvalidArgumentError, + "too many dimensions"): + self.evaluate( + array_ops.reshape( + tensor=[[1]], + shape=constant_op.constant([1 for i in range(254)], + dtype=dtypes.int64))) + class ReverseV2Test(test_util.TensorFlowTestCase): @@ -1580,6 +1589,20 @@ def testUnravelIndexZeroDim(self): dims = constant_op.constant([3, 0], dtype=dtype) self.evaluate(array_ops.unravel_index(indices=indices, dims=dims)) + def testUnravelIndexIntegerOverflow(self): + with self.cached_session(): + for dtype in [dtypes.int32, dtypes.int64]: + with self.assertRaisesRegex( + errors.InvalidArgumentError, + r"Input dims product is causing integer overflow"): + indices = constant_op.constant(-0x100000, dtype=dtype) + if dtype == dtypes.int32: + value = 0x10000000 + else: + value = 0x7FFFFFFFFFFFFFFF + dims = constant_op.constant([value, value], dtype=dtype) + self.evaluate(array_ops.unravel_index(indices=indices, dims=dims)) + class GuaranteeConstOpTest(test_util.TensorFlowTestCase): @@ -1701,6 +1724,21 @@ def f(a): output_grad = gradient_checker_v2.compute_gradient(f, [input_tensor]) self.assertAllClose(output_grad[0], np.zeros([1, 4, 4])) + def testOutOfBoundAxis(self): + input_tensor = constant_op.constant([1., 1.]) + input_min = [0] + input_max = [1] + q_input, _, _ = array_ops.quantize(input_tensor, 0, 1, dtypes.qint32) + error = (errors.InvalidArgumentError, ValueError) + with self.assertRaisesRegex(error, + r".*Axis must be less than input dimension.*"): + self.evaluate( + gen_array_ops.dequantize( + input=q_input, + min_range=input_min, + max_range=input_max, + axis=2**31 - 1)) + @test_util.run_all_in_graph_and_eager_modes class SortedSearchTest(test_util.TensorFlowTestCase): @@ -1921,6 +1959,17 @@ def testZeroValueSize(self): side=side, out_type=dtype), array_ops.zeros([2, 0], dtype)) + def testZeroInputSize(self): + dtype = dtypes.int32 + for side in ("left", "right"): + with self.subTest(side=side): + self.assertAllEqual( + array_ops.searchsorted( + array_ops.ones([2, 0]), + array_ops.ones([2, 3]), + side=side, + out_type=dtype), array_ops.zeros([2, 3], dtype)) + class BatchGatherNdTest(test_util.TensorFlowTestCase): diff --git a/tensorflow/python/kernel_tests/attention_ops_test.py b/tensorflow/python/kernel_tests/attention_ops_test.py index 804a0b20cc9dd4..98ef4e0cd0238e 100644 --- a/tensorflow/python/kernel_tests/attention_ops_test.py +++ b/tensorflow/python/kernel_tests/attention_ops_test.py @@ -301,6 +301,18 @@ def testGlimpseNonNormalizedNonCentered(self): np.asarray([[5, 6, 7], [10, 11, 12], [15, 16, 17]]), self.evaluate(result2)[0, :, :, 0]) + def testGlimpseNegativeInput(self): + img = np.arange(9).reshape([1, 3, 3, 1]) + with self.test_session(): + with self.assertRaises((errors.InvalidArgumentError, ValueError)): + result = image_ops.extract_glimpse_v2( + img, + size=[1023, -63], + offsets=[1023, 63], + centered=False, + normalized=False) + self.evaluate(result) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/kernel_tests/bincount_op_test.py b/tensorflow/python/kernel_tests/bincount_op_test.py index 6197e1e09254c1..b6d6f95b6cec2c 100644 --- a/tensorflow/python/kernel_tests/bincount_op_test.py +++ b/tensorflow/python/kernel_tests/bincount_op_test.py @@ -28,6 +28,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import bincount_ops from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import random_ops from tensorflow.python.ops import sparse_ops from tensorflow.python.ops.ragged import ragged_factory_ops from tensorflow.python.ops.ragged import ragged_tensor @@ -154,6 +155,31 @@ def test_shape_function(self): v2 = gen_math_ops.bincount([1, 2, 3, 1, 6, 8], s, []) self.assertAllEqual(v2.get_shape().as_list(), [None]) + @test_util.run_in_graph_and_eager_modes + def test_invalid_inputs(self): + binary_output = True + inp = random_ops.random_uniform( + shape=[10, 10], + minval=-10000, + maxval=10000, + dtype=dtypes.int32, + seed=-2460) + size = random_ops.random_uniform( + shape=[], minval=-10000, maxval=10000, dtype=dtypes.int32, seed=-10000) + weights = random_ops.random_uniform( + shape=[], + minval=-10000, + maxval=10000, + dtype=dtypes.float32, + seed=-10000) + with self.assertRaises(errors.InvalidArgumentError): + self.evaluate( + gen_math_ops.dense_bincount( + input=inp, + size=size, + weights=weights, + binary_output=binary_output)) + class BincountOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): @@ -348,6 +374,14 @@ def test_invalid_rank(self): gen_math_ops.dense_bincount( input=[[[1, 2, 3], [0, 3, 2]]], weights=[], size=10)) + @test_util.run_in_graph_and_eager_modes + def test_size_is_not_scalar(self): # b/206619828 + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "Shape must be rank 0 but is rank 1"): + self.evaluate( + gen_math_ops.dense_bincount( + input=[0], size=[1, 1], weights=[3], binary_output=False)) + class SparseBincountOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): @@ -362,7 +396,7 @@ def test_sparse_bincount_all_count(self, dtype): num_rows = 128 size = 1000 n_elems = 4096 - inp_indices = np.random.randint(0, num_rows, (n_elems,)) + inp_indices = np.random.randint(0, num_rows, (n_elems, 1)) inp_vals = np.random.randint(0, size, (n_elems,), dtype=dtype) np_out = np.bincount(inp_vals, minlength=size) @@ -386,7 +420,7 @@ def test_sparse_bincount_all_count_with_weights(self, dtype): num_rows = 128 size = 1000 n_elems = 4096 - inp_indices = np.random.randint(0, num_rows, (n_elems,)) + inp_indices = np.random.randint(0, num_rows, (n_elems, 1)) inp_vals = np.random.randint(0, size, (n_elems,), dtype=dtype) inp_weight = np.random.random((n_elems,)) @@ -411,7 +445,7 @@ def test_sparse_bincount_all_binary(self, dtype): num_rows = 128 size = 10 n_elems = 4096 - inp_indices = np.random.randint(0, num_rows, (n_elems,)) + inp_indices = np.random.randint(0, num_rows, (n_elems, 1)) inp_vals = np.random.randint(0, size, (n_elems,), dtype=dtype) np_out = np.ones((size,)) @@ -436,7 +470,7 @@ def test_sparse_bincount_all_binary_weights(self, dtype): num_rows = 128 size = 10 n_elems = 4096 - inp_indices = np.random.randint(0, num_rows, (n_elems,)) + inp_indices = np.random.randint(0, num_rows, (n_elems, 1)) inp_vals = np.random.randint(0, size, (n_elems,), dtype=dtype) inp_weight = np.random.random((n_elems,)) @@ -515,6 +549,40 @@ def test_sparse_bincount_col_reduce_binary(self, dtype): weights=[], binary_output=True))) + @test_util.run_in_graph_and_eager_modes + def test_size_is_not_scalar(self): # b/206619828 + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "Shape must be rank 0 but is rank 1"): + self.evaluate( + gen_math_ops.sparse_bincount( + indices=[[0], [1]], + values=[0, 0], + dense_shape=[1, 1], + size=[1, 1], + weights=[0, 0], + binary_output=False)) + + def test_sparse_bincount_input_validation(self): + np.random.seed(42) + num_rows = 128 + size = 1000 + n_elems = 4096 + inp_indices = np.random.randint(0, num_rows, (n_elems, 1)) + inp_vals = np.random.randint(0, size, (n_elems,)) + + # Insert negative index. + inp_indices[10, 0] = -2 + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "out of bounds"): + self.evaluate( + gen_math_ops.sparse_bincount( + indices=inp_indices, + values=inp_vals, + dense_shape=[num_rows], + size=size, + weights=[])) + class RaggedBincountOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): @@ -654,6 +722,31 @@ def test_ragged_bincount_binary_np_with_weights(self, dtype): size=size, binary_output=True))) + @test_util.run_in_graph_and_eager_modes + def test_size_is_not_scalar(self): # b/206619828 + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "Shape must be rank 0 but is rank 1"): + self.evaluate( + gen_math_ops.ragged_bincount( + splits=[0, 0, 1], + values=[1], + size=[1, 1], + weights=[0, 0, 0], + binary_output=False, + name=None)) + + @test_util.run_in_graph_and_eager_modes + def test_splits_empty(self): # b/238450914 + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "Splits must be non-empty"): + self.evaluate( + gen_math_ops.ragged_bincount( + splits=[], # Invalid splits + values=[1], + size=1, + weights=[1], + binary_output=False, + name=None)) if __name__ == "__main__": googletest.main() diff --git a/tensorflow/python/kernel_tests/checkpoint_ops_test.py b/tensorflow/python/kernel_tests/checkpoint_ops_test.py index 305480c6b4b920..8b50b9b2efae44 100644 --- a/tensorflow/python/kernel_tests/checkpoint_ops_test.py +++ b/tensorflow/python/kernel_tests/checkpoint_ops_test.py @@ -231,6 +231,32 @@ def test_load_and_remap_all_missing_rows_and_cols(self): np.reshape(initializing_values, (num_rows, num_cols)), self.evaluate(remapped_matrix)) + def test_load_and_remap_invalid_dims(self): + ckpt_path = constant_op.constant( + '/tmp/warm_starting_util_test5kl2a3pc/tmpph76tep2/model-0', + shape=[], + dtype=dtypes.string) + old_tensor_name = constant_op.constant( + '/tmp/warm_starting_util_test5kl2a3pc/tmpph76tep2/model-0', + shape=[], + dtype=dtypes.string) + row_remapping = constant_op.constant(0, shape=[], dtype=dtypes.int64) + col_remapping = constant_op.constant(3, shape=[3], dtype=dtypes.int64) + initializing_values = constant_op.constant([], + shape=[0, 1], + dtype=dtypes.float32) + with self.cached_session(), self.assertRaisesRegex( + (ValueError, errors.InvalidArgumentError), 'tensor must be 1-D'): + self.evaluate( + gen_checkpoint_ops.load_and_remap_matrix( + ckpt_path=ckpt_path, + old_tensor_name=old_tensor_name, + row_remapping=row_remapping, + col_remapping=col_remapping, + initializing_values=initializing_values, + num_rows=1, + num_cols=1)) + @test_util.run_deprecated_v1 def test_load_and_remap_invalid_remapping(self): """Tests that errors are raised when an ID maps to multiple new IDs. diff --git a/tensorflow/python/kernel_tests/concat_op_test.py b/tensorflow/python/kernel_tests/concat_op_test.py index da4f4f86b0220d..3f1401baa730ec 100644 --- a/tensorflow/python/kernel_tests/concat_op_test.py +++ b/tensorflow/python/kernel_tests/concat_op_test.py @@ -20,6 +20,7 @@ import numpy as np +from tensorflow.python.eager import def_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl @@ -574,6 +575,17 @@ def testConcatInvalidAxis(self): t2 = [2] gen_array_ops.concat_v2([t1, t2], 1).eval() + def testConcatInvalidAxisInTfFunction(self): + + @def_function.function + def concat_wrapper(): + y = gen_array_ops.concat_v2( + values=[[1, 2, 3], [4, 5, 6]], axis=0xb500005b) + return y + + with self.assertRaises(ValueError): + concat_wrapper() + def testConcatNegativeAxis(self): with test_util.use_gpu(): t1 = [[1, 2, 3], [4, 5, 6]] diff --git a/tensorflow/python/kernel_tests/conv3d_backprop_filter_v2_grad_test.py b/tensorflow/python/kernel_tests/conv3d_backprop_filter_v2_grad_test.py index 7e913febed3dc8..6194bad8fc7a7c 100644 --- a/tensorflow/python/kernel_tests/conv3d_backprop_filter_v2_grad_test.py +++ b/tensorflow/python/kernel_tests/conv3d_backprop_filter_v2_grad_test.py @@ -22,6 +22,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker @@ -62,6 +63,23 @@ def testGradient(self): err_tolerance = 1e-3 self.assertLess(err, err_tolerance) + def testBadFilterShape(self): + strides = [1, 1, 1, 1, 1] + padding = "VALID" + tin = constant_op.constant( + .5053710941, shape=[2, 2, 2, 2, 1], dtype=dtypes.float32) + filter_sizes = constant_op.constant(0, shape=[], dtype=dtypes.int32) + out_backprop = constant_op.constant( + .5053710941, shape=[2, 2, 2, 2, 1], dtype=dtypes.float32) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 1"): + nn_ops.conv3d_backprop_filter_v2( + input=tin, + filter_sizes=filter_sizes, + out_backprop=out_backprop, + strides=strides, + padding=padding) if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py index 986439862afd6b..d597af7766ac91 100644 --- a/tensorflow/python/kernel_tests/conv_ops_test.py +++ b/tensorflow/python/kernel_tests/conv_ops_test.py @@ -37,6 +37,7 @@ from tensorflow.python.layers import convolutional from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import gen_nn_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import math_ops @@ -763,6 +764,15 @@ def testConv2DExplicitPaddingWithDilations(self): padding=[[2, 1], [1, 2]], dilations=[2, 3]) + @test_util.run_in_graph_and_eager_modes() + def testConv2dOnlyPaddingReturnsZeros(self): + self._VerifyValues( + tensor_in_sizes=[1, 0, 2, 1], + filter_in_sizes=[1, 1, 1, 1], + strides=[1, 1], + padding=[[1, 1], [1, 1]], + expected=[0, 0, 0, 0, 0, 0, 0, 0]) + def testConv2DExplicitPaddingWithLayoutOptimizer(self): # Test with Grappler's layout optimizer, to ensure the layout optimizer # handles explicit padding correctly. @@ -1107,6 +1117,23 @@ def testConv2DInputSizesContainsOnlySpatialDimensionsBackpropInput(self): use_gpu=use_gpu, err=1e-5) + @test_util.run_in_graph_and_eager_modes + @test_util.disable_xla("b/239598470") + def testConv2DBackpropInputDegenerateBackpropInput(self): + input_sizes = [3, 1, 1, 2] + expected_output = np.zeros(input_sizes).flatten() + for (data_format, use_gpu) in GetTestConfigs(): + self._RunAndVerifyBackpropInput( + input_sizes=input_sizes, + filter_sizes=[1, 3, 2, 3], + output_sizes=[3, 1, 0, 3], + strides=[1, 2], + padding="VALID", + expected=expected_output, + data_format=data_format, + use_gpu=use_gpu, + err=1e-5) + # Testing for backprops def _RunAndVerifyBackpropFilter(self, input_sizes, @@ -1297,7 +1324,7 @@ def _RunAndVerifyBackpropInputDilation(self, input_sizes, filter_sizes, x2 = self._CreateNumpyTensor(filter_sizes) default_dilations = (dilations[0] == 1 and dilations[1] == 1) if default_dilations or use_gpu: - with self.cached_session(use_gpu=use_gpu) as sess: + with self.cached_session(use_gpu=use_gpu): if data_format == "NCHW": input_sizes = test_util.NHWCToNCHW(input_sizes) t1 = constant_op.constant(x1, shape=input_sizes) @@ -1343,7 +1370,7 @@ def _RunAndVerifyBackpropFilterDilation(self, input_sizes, filter_sizes, x2 = self._CreateNumpyTensor(filter_sizes) default_dilations = (dilations[0] == 1 and dilations[1] == 1) if default_dilations or use_gpu: - with self.cached_session(use_gpu=use_gpu) as sess: + with self.cached_session(use_gpu=use_gpu): if data_format == "NCHW": input_sizes = test_util.NHWCToNCHW(input_sizes) t1 = constant_op.constant(x1, shape=input_sizes) @@ -2606,6 +2633,27 @@ def testOpEdgeCases(self): strides=[1, 1, 1, 1], padding=[[0, 0], [-1, 0], [0, 0], [0, 0]])) + def testConv2DBackpropInputInvalidOutBackpropRaiseError(self): + with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)): + with self.cached_session(): + input_sizes = constant_op.constant([65534, 65534], + shape=[2], + dtype=dtypes.int32) + filters = constant_op.constant( + 0.159749106, shape=[3, 3, 2, 2], dtype=dtypes.float32) + out_backprop = constant_op.constant(0, shape=[], dtype=dtypes.float32) + t = gen_nn_ops.conv2d_backprop_input( + input_sizes=input_sizes, + filter=filters, + out_backprop=out_backprop, + strides=[1, 1, 1, 1], + padding="SAME", + use_cudnn_on_gpu=True, + explicit_paddings=[], + data_format="NHWC", + dilations=[1, 1, 1, 1]) + self.evaluate(t) + class DepthwiseConv2DTest(test.TestCase): @@ -2632,7 +2680,7 @@ def _VerifyValues(self, tensor_in_sizes, filter_in_sizes, stride, padding, # numbers from 1. x1 = [f * 1.0 for f in range(1, total_size_1 + 1)] x2 = [f * 1.0 for f in range(1, total_size_2 + 1)] - with self.cached_session() as sess: + with self.cached_session(): t1 = constant_op.constant(x1, shape=tensor_in_sizes) t1.set_shape(tensor_in_sizes) t2 = constant_op.constant(x2, shape=filter_in_sizes) @@ -2901,7 +2949,7 @@ def _CompareFwdConv2D(self, tensor_in_sizes, filter_in_sizes, conv_strides, x1 = np.random.rand(*tensor_in_sizes).astype(np.float32) x2 = np.random.rand(*filter_in_sizes).astype(np.float32) - with self.cached_session(use_gpu=False) as sess: + with self.cached_session(use_gpu=False): t1 = constant_op.constant(x1, shape=tensor_in_sizes) t2 = constant_op.constant(x2, shape=filter_in_sizes) strides = [1] + conv_strides + [1] diff --git a/tensorflow/python/kernel_tests/distributions/BUILD b/tensorflow/python/kernel_tests/distributions/BUILD index d25196d7961fa6..d1cf84faa32082 100644 --- a/tensorflow/python/kernel_tests/distributions/BUILD +++ b/tensorflow/python/kernel_tests/distributions/BUILD @@ -61,6 +61,7 @@ cuda_py_test( size = "small", srcs = ["beta_test.py"], tags = [ + "no_oss", "nomac", # b/191763315 "notsan", # b/173653918 ], diff --git a/tensorflow/python/kernel_tests/draw_bounding_box_op_test.py b/tensorflow/python/kernel_tests/draw_bounding_box_op_test.py index 5f7c3c9eb51629..d8e0c4613c23a0 100644 --- a/tensorflow/python/kernel_tests/draw_bounding_box_op_test.py +++ b/tensorflow/python/kernel_tests/draw_bounding_box_op_test.py @@ -54,11 +54,16 @@ def _fillBorder(self, image, color): image[height - 1, 0:width, 0:depth] = color return image - def _testDrawBoundingBoxColorCycling(self, img, colors=None): + def _testDrawBoundingBoxColorCycling(self, + img, + dtype=dtypes.float32, + colors=None): """Tests if cycling works appropriately. Args: img: 3-D numpy image on which to draw. + dtype: image dtype (float, half). + colors: color table. """ color_table = colors if colors is None: @@ -86,7 +91,7 @@ def _testDrawBoundingBoxColorCycling(self, img, colors=None): bboxes = math_ops.cast(bboxes, dtypes.float32) bboxes = array_ops.expand_dims(bboxes, 0) image = ops.convert_to_tensor(image) - image = image_ops_impl.convert_image_dtype(image, dtypes.float32) + image = image_ops_impl.convert_image_dtype(image, dtype) image = array_ops.expand_dims(image, 0) image = image_ops.draw_bounding_boxes(image, bboxes, colors=colors) with self.cached_session(use_gpu=False) as sess: @@ -122,6 +127,14 @@ def testDrawBoundingBoxRGBAColorCyclingWithColors(self): [0, 0, 0.5, 1]]) self._testDrawBoundingBoxColorCycling(image, colors=colors) + def testDrawBoundingBoxHalf(self): + """Test if RGBA color cycling works correctly with provided colors.""" + image = np.zeros([10, 10, 4], "float32") + colors = np.asarray([[0.5, 0, 0.5, 1], [0.5, 0.5, 0, 1], [0.5, 0, 0, 1], + [0, 0, 0.5, 1]]) + self._testDrawBoundingBoxColorCycling( + image, dtype=dtypes.half, colors=colors) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/edit_distance_op_test.py b/tensorflow/python/kernel_tests/edit_distance_op_test.py index 4a06ab770aaa07..706c80ec9df949 100644 --- a/tensorflow/python/kernel_tests/edit_distance_op_test.py +++ b/tensorflow/python/kernel_tests/edit_distance_op_test.py @@ -211,6 +211,24 @@ def testEditDistanceZeroLengthHypothesisAndTruth(self): normalize=True, expected_output=expected_output) + def testEditDistanceBadIndices(self): + hypothesis_indices = np.full((3, 3), -1250999896764, dtype=np.int64) + hypothesis_values = np.empty(3, dtype=np.int64) + hypothesis_shape = np.empty(3, dtype=np.int64) + truth_indices = np.full((3, 3), -1250999896764, dtype=np.int64) + truth_values = np.full([3], 2, dtype=np.int64) + truth_shape = np.full([3], 2, dtype=np.int64) + expected_output = [] # dummy; ignored + + self._testEditDistance( + hypothesis=(hypothesis_indices, hypothesis_values, hypothesis_shape), + truth=(truth_indices, truth_values, truth_shape), + normalize=False, + expected_output=expected_output, + expected_err_re=(r"inner product -\d+ which would require writing " + "to outside of the buffer for the output tensor") + ) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/eig_op_test.py b/tensorflow/python/kernel_tests/eig_op_test.py index e9e311ba019161..d3324aeb814164 100644 --- a/tensorflow/python/kernel_tests/eig_op_test.py +++ b/tensorflow/python/kernel_tests/eig_op_test.py @@ -22,8 +22,10 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes as dtypes_lib +from tensorflow.python.framework import errors from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_linalg_ops from tensorflow.python.ops import gradient_checker_v2 from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops @@ -92,6 +94,16 @@ def testMatrixThatFailsWhenFlushingDenormsToZero(self): self.assertAllClose(matrix, np.matmul(np.matmul(v, np.diag(e)), v.transpose())) + def testMismatchedDtypes(self): + tensor = constant_op.constant([[0, 1], [2, 3]], dtype=dtypes_lib.float32) + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "Invalid output dtype"): + self.evaluate( + gen_linalg_ops.eig( + input=tensor, + Tout=dtypes_lib.complex128, # Expected dtype: complex64. + compute_v=True)) + def SortEigenValues(e): perm = np.argsort(e.real + e.imag, -1) diff --git a/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py b/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py index 0d5928aefacf5a..f2b80028bdb88e 100644 --- a/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py +++ b/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py @@ -24,6 +24,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_nn_ops @@ -310,6 +311,32 @@ def testDifferentInputTensorShape(self): input_b, row_seq, col_seq, overlapping) self.assertSequenceEqual(expected.shape, actual.shape) + def testNegativeSeqValuesForGradOp(self): + with self.assertRaisesRegex( + errors.InvalidArgumentError, + r"Row sequence tensor values must not be negative.*"): + y = nn_ops.gen_nn_ops.fractional_avg_pool_grad( + orig_input_tensor_shape=[2, 2, 2, 2], + out_backprop=[[[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [11, + 12]]]], + row_pooling_sequence=[-10, 1, 2, 3], + col_pooling_sequence=[1, 2, 3, 4], + overlapping=True) + + self.evaluate(y) + with self.assertRaisesRegex( + errors.InvalidArgumentError, + r"Column sequence tensor values must not be negative.*"): + z = nn_ops.gen_nn_ops.fractional_avg_pool_grad( + orig_input_tensor_shape=[2, 2, 2, 2], + out_backprop=[[[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [11, + 12]]]], + row_pooling_sequence=[10, 1, 2, 3], + col_pooling_sequence=[1, 2, -3, 4], + overlapping=True) + + self.evaluate(z) + class FractionalAvgPoolGradTest(test.TestCase): """Tests for FractionalAvgPoolGrad. @@ -518,6 +545,27 @@ def testLargePoolingRatioThroughGradientError(self): delta=1e-2) self.assertLess(gradient_error, error_margin) + def testInvalidSeqRaiseErrorForFractionalAvgPoolGrad(self): + with self.assertRaises((errors.InvalidArgumentError, ValueError)): + with self.cached_session() as _: + overlapping = True + orig_input_tensor_shape = constant_op.constant( + -1879048192, shape=[4], dtype=dtypes.int64) + out_backprop = constant_op.constant([], + shape=[0, 0, 0, 0], + dtype=dtypes.float64) + row_pooling_sequence = constant_op.constant( + 1, shape=[4], dtype=dtypes.int64) + col_pooling_sequence = constant_op.constant( + 1, shape=[4], dtype=dtypes.int64) + t = gen_nn_ops.fractional_avg_pool_grad( + orig_input_tensor_shape=orig_input_tensor_shape, + out_backprop=out_backprop, + row_pooling_sequence=row_pooling_sequence, + col_pooling_sequence=col_pooling_sequence, + overlapping=overlapping) + self.evaluate(t) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py b/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py index 2b1e30a8bbd606..b8daa8bb919f7d 100644 --- a/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py +++ b/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py @@ -24,6 +24,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_nn_ops @@ -127,7 +128,7 @@ def _ValidateFractionalMaxPoolResult(self, input_tensor, pooling_ratio, Returns: None """ - with self.cached_session() as sess: + with self.cached_session(): p, r, c = nn_ops.fractional_max_pool_v2( input_tensor, pooling_ratio, @@ -158,7 +159,7 @@ def _testVisually(self): overlapping)) rand_mat = self._PRNG.randint(10, size=tensor_shape) pooling_ratio = [1, math.sqrt(2), math.sqrt(2), 1] - with self.cached_session() as sess: + with self.cached_session(): p, r, c = nn_ops.fractional_max_pool_v2( rand_mat, pooling_ratio, @@ -307,6 +308,22 @@ def testDifferentInputTensorShape(self): input_b, row_seq, col_seq, overlapping) self.assertSequenceEqual(expected.shape, actual.shape) + def testDeterminismExceptionThrowing(self): + tensor_shape = (5, 20, 20, 3) + rand_mat = self._PRNG.random_sample(tensor_shape) * 1000 - 500 + with test_util.deterministic_ops(): + with self.assertRaisesRegex( + ValueError, "requires a non-zero seed to be passed in when " + "determinism is enabled"): + nn_ops.fractional_max_pool_v2(rand_mat, [1, 1.5, 1.5, 1]) + nn_ops.fractional_max_pool_v2(rand_mat, [1, 1.5, 1.5, 1], seed=1) + + with self.assertRaisesRegex(ValueError, + 'requires "seed" and "seed2" to be non-zero'): + nn_ops.fractional_max_pool(rand_mat, [1, 1.5, 1.5, 1]) + nn_ops.fractional_max_pool( + rand_mat, [1, 1.5, 1.5, 1], seed=1, seed2=1, deterministic=True) + class FractionalMaxPoolGradTest(test.TestCase): """Tests for FractionalMaxPoolGrad. @@ -599,6 +616,29 @@ def testWhenRepeatedMaxValueInPoolingRegion(self): self.assertAllClose(expected_input_backprop_overlapping, input_backprop_overlapping) + def testInvalidSeqRaiseErrorForFractionalMaxPoolGrad(self): + with self.assertRaises(errors.InvalidArgumentError): + with self.cached_session() as _: + overlapping = True + orig_input = constant_op.constant( + .453409232, shape=[1, 7, 13, 1], dtype=dtypes.float32) + orig_output = constant_op.constant( + .453409232, shape=[1, 7, 13, 1], dtype=dtypes.float32) + out_backprop = constant_op.constant( + .453409232, shape=[1, 7, 13, 1], dtype=dtypes.float32) + row_pooling_sequence = constant_op.constant( + 0, shape=[5], dtype=dtypes.int64) + col_pooling_sequence = constant_op.constant( + 0, shape=[5], dtype=dtypes.int64) + t = gen_nn_ops.FractionalMaxPoolGrad( + orig_input=orig_input, + orig_output=orig_output, + out_backprop=out_backprop, + row_pooling_sequence=row_pooling_sequence, + col_pooling_sequence=col_pooling_sequence, + overlapping=overlapping) + self.evaluate(t) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py index 77b305572ac4e1..af73908e938ad7 100644 --- a/tensorflow/python/kernel_tests/init_ops_test.py +++ b/tensorflow/python/kernel_tests/init_ops_test.py @@ -553,7 +553,7 @@ def testLargeLimits(self): def testLargeStarts(self): # Test case for GitHub issue 46899. with self.session(): - with self.assertRaises(errors_impl.InternalError): + with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)): v = math_ops.range(start=-1e+38, limit=1) self.evaluate(v) diff --git a/tensorflow/python/kernel_tests/linalg/sparse/BUILD b/tensorflow/python/kernel_tests/linalg/sparse/BUILD index 91c8d0f8cdc356..036f739b2c7154 100644 --- a/tensorflow/python/kernel_tests/linalg/sparse/BUILD +++ b/tensorflow/python/kernel_tests/linalg/sparse/BUILD @@ -44,6 +44,7 @@ cuda_py_test( main = "csr_sparse_matrix_ops_test.py", shard_count = 10, tags = [ + "no_oss", # Times out during patch release "notsan", # b/149115441 ], deps = [ diff --git a/tensorflow/python/kernel_tests/linalg/sparse/csr_sparse_matrix_ops_test.py b/tensorflow/python/kernel_tests/linalg/sparse/csr_sparse_matrix_ops_test.py index b9ea7681f5526c..cc06ed0dd936cd 100644 --- a/tensorflow/python/kernel_tests/linalg/sparse/csr_sparse_matrix_ops_test.py +++ b/tensorflow/python/kernel_tests/linalg/sparse/csr_sparse_matrix_ops_test.py @@ -172,6 +172,25 @@ def testSparseTensorConversion(self): self.assertAllClose(a_values, a_st_rt_value.values) self.assertAllEqual(a_dense_shape, a_st_rt_value.dense_shape) + def testSparseTensorConversionInvalidInputShapes(self): + values = constant_op.constant( + 0.554979503, shape=[5], dtype=dtypes.float32) + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 1"): + indices = constant_op.constant(0, shape=[5, 2], dtype=dtypes.int64) + dense_shape = constant_op.constant(53, shape=[], dtype=dtypes.int64) + csr = sparse_csr_matrix_ops.sparse_tensor_to_csr_sparse_matrix( + indices=indices, values=values, dense_shape=dense_shape) + self.evaluate(csr) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 2"): + indices = constant_op.constant(0, shape=[5], dtype=dtypes.int64) + dense_shape = constant_op.constant(53, shape=[1], dtype=dtypes.int64) + csr = sparse_csr_matrix_ops.sparse_tensor_to_csr_sparse_matrix( + indices=indices, values=values, dense_shape=dense_shape) + self.evaluate(csr) + # TODO(b/139491352): Add handle_data propagation to array_ops.identity. @test_util.run_deprecated_v1 def testCSRSparseMatrixResourceVariable(self): diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py index 5617b07cae8a76..3b7e9784ab98d7 100644 --- a/tensorflow/python/kernel_tests/list_ops_test.py +++ b/tensorflow/python/kernel_tests/list_ops_test.py @@ -98,6 +98,16 @@ def testPopFromEmptyTensorListFails(self, max_num_elements): l = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32) self.evaluate(l) + def testTensorListReserveWithNonScalarNumElements(self): + # list_kernels.cc in tf/core/kernels raises InvalidArgumentError, and + # tf_ops_n_z.cc in tf/compiler/mlir/tf/ir raises UnknownError. + with self.assertRaises((errors.InvalidArgumentError, errors.UnknownError)): + l = list_ops.tensor_list_reserve( + element_dtype=dtypes.float32, + element_shape=[2, 3], + num_elements=constant_op.constant([1, 1])) + self.evaluate(l) + def testPopUninitializedTensorUseListElementShape(self): l = list_ops.tensor_list_reserve( element_dtype=dtypes.float32, element_shape=[2, 3], num_elements=3) @@ -485,6 +495,30 @@ def testScatterOutputListSizeWithNumElementsSpecified(self): # TensorListScatter should return a list with size num_elements. self.assertAllEqual(list_ops.tensor_list_length(l), 5) + def testScatterFailsWhenElementShapeIsNotVector(self): + c0 = constant_op.constant([1.0, 2.0]) + # In Eager mode, InvalidArgumentError is generated by the Compute function. + # In graph mode, ValueError is generated by the shape function. + with self.assertRaisesRegex( + (errors.InvalidArgumentError, ValueError), + "must be at most rank 1"): + l = gen_list_ops.tensor_list_scatter( + # Wrong element_shape. Should be at most rank 1. + c0, [1, 3], element_shape=[[1]]) + self.evaluate(l) + + def testScatterV2FailsWhenElementShapeIsNotVector(self): + c0 = constant_op.constant([1.0, 2.0]) + # In Eager mode, InvalidArgumentError is generated by the Compute function. + # In graph mode, ValueError is generated by the shape function. + with self.assertRaisesRegex( + (errors.InvalidArgumentError, ValueError), + "must be at most rank 1"): + l = gen_list_ops.tensor_list_scatter_v2( + # Wrong element_shape. Should be at most rank 1. + c0, [1, 3], element_shape=[[1]], num_elements=2) + self.evaluate(l) + def testScatterFailsWhenIndexLargerThanNumElements(self): c0 = constant_op.constant([1.0, 2.0]) with self.assertRaisesRegex( @@ -564,6 +598,17 @@ def testTensorListFromTensor(self): self.assertAllEqual(e, 1.0) self.assertAllEqual(list_ops.tensor_list_length(l), 0) + def testTensorListFromTensorFailsWhenElementShapeIsNotVector(self): + t = constant_op.constant([1.0, 2.0]) + # In Eager mode, InvalidArgumentError is generated by the Compute function. + # In graph mode, ValueError is generated by the shape function. + with self.assertRaisesRegex( + (errors.InvalidArgumentError, ValueError), + "must be at most rank 1"): + # Wrong element_shape. Should be at most rank 1. + l = list_ops.tensor_list_from_tensor(t, element_shape=[[1]]) + self.evaluate(l) + @test_util.run_gpu_only def testFromTensorGPU(self): with context.device("gpu:0"): @@ -1462,6 +1507,15 @@ def testConcatWithUninitializedTensorsFailsIfNoInputLengths(self): t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32) self.evaluate(t) + def testEmptyTensorListInvalidShape(self): + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + r"Shape must be at most rank 1 but is rank 2"): + t = gen_list_ops.EmptyTensorList( + element_shape=array_ops.ones(dtype=dtypes.int32, shape=[1, 0]), + max_num_elements=constant_op.constant(1), + element_dtype=dtypes.int32) + self.evaluate(t) + def testEvenSplit(self): def RunTest(input_tensor, lengths, expected_stacked_output): diff --git a/tensorflow/python/kernel_tests/logging_ops_test.py b/tensorflow/python/kernel_tests/logging_ops_test.py index b3c30f07d149b7..cb1645d407d8fc 100644 --- a/tensorflow/python/kernel_tests/logging_ops_test.py +++ b/tensorflow/python/kernel_tests/logging_ops_test.py @@ -274,7 +274,7 @@ def testPrintOneTensorStdout(self): self.assertIn((expected + "\n"), printed.contents()) def testPrintTensorsToFile(self): - tmpfile_name = tempfile.mktemp(".printv2_test") + fd, tmpfile_name = tempfile.mkstemp(".printv2_test") tensor_0 = math_ops.range(0, 10) print_op_0 = logging_ops.print_v2(tensor_0, output_stream="file://"+tmpfile_name) @@ -284,14 +284,14 @@ def testPrintTensorsToFile(self): output_stream="file://"+tmpfile_name) self.evaluate(print_op_1) try: - f = open(tmpfile_name, "r") + f = os.fdopen(fd, "r") line_0 = f.readline() expected_0 = "[0 1 2 ... 7 8 9]" self.assertTrue(expected_0 in line_0) line_1 = f.readline() expected_1 = "[11 12 13 ... 17 18 19]" self.assertTrue(expected_1 in line_1) - f.close() + os.close(fd) os.remove(tmpfile_name) except IOError as e: self.fail(e) diff --git a/tensorflow/python/kernel_tests/lrn_op_test.py b/tensorflow/python/kernel_tests/lrn_op_test.py index f548804852351f..5072b732d2abe6 100644 --- a/tensorflow/python/kernel_tests/lrn_op_test.py +++ b/tensorflow/python/kernel_tests/lrn_op_test.py @@ -24,11 +24,13 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors_impl from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import nn +from tensorflow.python.ops import random_ops import tensorflow.python.ops.nn_grad # pylint: disable=unused-import from tensorflow.python.platform import test @@ -115,6 +117,41 @@ def testGradientsZeroInput(self): self.assertAllClose(r, expected) self.assertShapeEqual(expected, grad) + @test_util.run_in_graph_and_eager_modes + def testIncompatibleInputAndOutputImageShapes(self): + depth_radius = 1 + bias = 1.59018219 + alpha = 0.117728651 + beta = 0.404427052 + input_grads = random_ops.random_uniform( + shape=[4, 4, 4, 4], + minval=-10000, + maxval=10000, + dtype=dtypes.float32, + seed=-2033) + input_image = random_ops.random_uniform( + shape=[4, 4, 4, 4], + minval=-10000, + maxval=10000, + dtype=dtypes.float32, + seed=-2033) + invalid_output_image = random_ops.random_uniform( + shape=[4, 4, 4, 4, 4, 4], + minval=-10000, + maxval=10000, + dtype=dtypes.float32, + seed=-2033) + with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)): + self.evaluate( + nn.lrn_grad( + input_grads=input_grads, + input_image=input_image, + output_image=invalid_output_image, + depth_radius=depth_radius, + bias=bias, + alpha=alpha, + beta=beta)) + def _RunAndVerifyGradients(self, dtype): with self.cached_session(): # random shape diff --git a/tensorflow/python/kernel_tests/map_stage_op_test.py b/tensorflow/python/kernel_tests/map_stage_op_test.py index 516fc37517ca57..8600ad1f8d726b 100644 --- a/tensorflow/python/kernel_tests/map_stage_op_test.py +++ b/tensorflow/python/kernel_tests/map_stage_op_test.py @@ -12,12 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +import numpy as np -from tensorflow.python.framework import errors +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops @@ -32,7 +31,7 @@ class MapStageTest(test.TestCase): @test_util.run_deprecated_v1 def testSimple(self): - with ops.Graph().as_default() as G: + with ops.Graph().as_default() as g: with ops.device('/cpu:0'): x = array_ops.placeholder(dtypes.float32) pi = array_ops.placeholder(dtypes.int64) @@ -44,9 +43,9 @@ def testSimple(self): k, y = stager.get(gi) y = math_ops.reduce_max(math_ops.matmul(y, y)) - G.finalize() + g.finalize() - with self.session(graph=G) as sess: + with self.session(graph=g) as sess: sess.run(stage, feed_dict={x: -1, pi: 0}) for i in range(10): _, yval = sess.run([stage, y], feed_dict={x: i, pi: i + 1, gi: i}) @@ -54,7 +53,7 @@ def testSimple(self): @test_util.run_deprecated_v1 def testMultiple(self): - with ops.Graph().as_default() as G: + with ops.Graph().as_default() as g: with ops.device('/cpu:0'): x = array_ops.placeholder(dtypes.float32) pi = array_ops.placeholder(dtypes.int64) @@ -66,9 +65,9 @@ def testMultiple(self): k, (z, y) = stager.get(gi) y = math_ops.reduce_max(z * math_ops.matmul(y, y)) - G.finalize() + g.finalize() - with self.session(graph=G) as sess: + with self.session(graph=g) as sess: sess.run(stage, feed_dict={x: -1, pi: 0}) for i in range(10): _, yval = sess.run([stage, y], feed_dict={x: i, pi: i + 1, gi: i}) @@ -77,26 +76,25 @@ def testMultiple(self): @test_util.run_deprecated_v1 def testDictionary(self): - with ops.Graph().as_default() as G: + with ops.Graph().as_default() as g: with ops.device('/cpu:0'): x = array_ops.placeholder(dtypes.float32) pi = array_ops.placeholder(dtypes.int64) gi = array_ops.placeholder(dtypes.int64) v = 2. * (array_ops.zeros([128, 128]) + x) with ops.device(test.gpu_device_name()): - stager = data_flow_ops.MapStagingArea( - [dtypes.float32, dtypes.float32], - shapes=[[], [128, 128]], - names=['x', 'v']) + stager = data_flow_ops.MapStagingArea([dtypes.float32, dtypes.float32], + shapes=[[], [128, 128]], + names=['x', 'v']) stage = stager.put(pi, {'x': x, 'v': v}) key, ret = stager.get(gi) z = ret['x'] y = ret['v'] y = math_ops.reduce_max(z * math_ops.matmul(y, y)) - G.finalize() + g.finalize() - with self.session(graph=G) as sess: + with self.session(graph=g) as sess: sess.run(stage, feed_dict={x: -1, pi: 0}) for i in range(10): _, yval = sess.run([stage, y], feed_dict={x: i, pi: i + 1, gi: i}) @@ -106,7 +104,7 @@ def testDictionary(self): def testColocation(self): gpu_dev = test.gpu_device_name() - with ops.Graph().as_default() as G: + with ops.Graph().as_default() as g: with ops.device('/cpu:0'): x = array_ops.placeholder(dtypes.float32) v = 2. * (array_ops.zeros([128, 128]) + x) @@ -123,58 +121,56 @@ def testColocation(self): self.assertEqual(y.device, '/device:CPU:0') self.assertEqual(z[0].device, '/device:CPU:0') - G.finalize() + g.finalize() @test_util.run_deprecated_v1 def testPeek(self): - with ops.Graph().as_default() as G: + with ops.Graph().as_default() as g: with ops.device('/cpu:0'): x = array_ops.placeholder(dtypes.int32, name='x') pi = array_ops.placeholder(dtypes.int64) gi = array_ops.placeholder(dtypes.int64) p = array_ops.placeholder(dtypes.int32, name='p') with ops.device(test.gpu_device_name()): - stager = data_flow_ops.MapStagingArea( - [ - dtypes.int32, - ], shapes=[[]]) + stager = data_flow_ops.MapStagingArea([ + dtypes.int32, + ], shapes=[[]]) stage = stager.put(pi, [x], [0]) peek = stager.peek(gi) size = stager.size() - G.finalize() + g.finalize() n = 10 - with self.session(graph=G) as sess: + with self.session(graph=g) as sess: for i in range(n): sess.run(stage, feed_dict={x: i, pi: i}) for i in range(n): - self.assertTrue(sess.run(peek, feed_dict={gi: i})[0] == i) + self.assertEqual(sess.run(peek, feed_dict={gi: i})[0], i) - self.assertTrue(sess.run(size) == 10) + self.assertEqual(sess.run(size), 10) @test_util.run_deprecated_v1 def testSizeAndClear(self): - with ops.Graph().as_default() as G: + with ops.Graph().as_default() as g: with ops.device('/cpu:0'): x = array_ops.placeholder(dtypes.float32, name='x') pi = array_ops.placeholder(dtypes.int64) gi = array_ops.placeholder(dtypes.int64) v = 2. * (array_ops.zeros([128, 128]) + x) with ops.device(test.gpu_device_name()): - stager = data_flow_ops.MapStagingArea( - [dtypes.float32, dtypes.float32], - shapes=[[], [128, 128]], - names=['x', 'v']) + stager = data_flow_ops.MapStagingArea([dtypes.float32, dtypes.float32], + shapes=[[], [128, 128]], + names=['x', 'v']) stage = stager.put(pi, {'x': x, 'v': v}) size = stager.size() clear = stager.clear() - G.finalize() + g.finalize() - with self.session(graph=G) as sess: + with self.session(graph=g) as sess: sess.run(stage, feed_dict={x: -1, pi: 3}) self.assertEqual(sess.run(size), 1) sess.run(stage, feed_dict={x: -1, pi: 1}) @@ -186,22 +182,23 @@ def testSizeAndClear(self): def testCapacity(self): capacity = 3 - with ops.Graph().as_default() as G: + with ops.Graph().as_default() as g: with ops.device('/cpu:0'): x = array_ops.placeholder(dtypes.int32, name='x') pi = array_ops.placeholder(dtypes.int64, name='pi') gi = array_ops.placeholder(dtypes.int64, name='gi') with ops.device(test.gpu_device_name()): - stager = data_flow_ops.MapStagingArea( - [ - dtypes.int32, - ], capacity=capacity, shapes=[[]]) + stager = data_flow_ops.MapStagingArea([ + dtypes.int32, + ], + capacity=capacity, + shapes=[[]]) stage = stager.put(pi, [x], [0]) get = stager.get() size = stager.size() - G.finalize() + g.finalize() from six.moves import queue as Queue import threading @@ -209,7 +206,7 @@ def testCapacity(self): queue = Queue.Queue() n = 8 - with self.session(graph=G) as sess: + with self.session(graph=g) as sess: # Stage data in a separate thread which will block # when it hits the staging area's capacity and thus # not fill the queue with n tokens @@ -238,13 +235,13 @@ def thread_run(): capacity)) # Should have capacity elements in the staging area - self.assertTrue(sess.run(size) == capacity) + self.assertEqual(sess.run(size), capacity) # Clear the staging area completely for i in range(n): sess.run(get) - self.assertTrue(sess.run(size) == 0) + self.assertEqual(sess.run(size), 0) @test_util.run_deprecated_v1 def testMemoryLimit(self): @@ -252,28 +249,28 @@ def testMemoryLimit(self): chunk = 200 * 1024 # 256K capacity = memory_limit // chunk - with ops.Graph().as_default() as G: + with ops.Graph().as_default() as g: with ops.device('/cpu:0'): x = array_ops.placeholder(dtypes.uint8, name='x') pi = array_ops.placeholder(dtypes.int64, name='pi') gi = array_ops.placeholder(dtypes.int64, name='gi') with ops.device(test.gpu_device_name()): - stager = data_flow_ops.MapStagingArea( - [dtypes.uint8], memory_limit=memory_limit, shapes=[[]]) + stager = data_flow_ops.MapStagingArea([dtypes.uint8], + memory_limit=memory_limit, + shapes=[[]]) stage = stager.put(pi, [x], [0]) get = stager.get() size = stager.size() - G.finalize() + g.finalize() from six.moves import queue as Queue import threading - import numpy as np queue = Queue.Queue() n = 8 - with self.session(graph=G) as sess: + with self.session(graph=g) as sess: # Stage data in a separate thread which will block # when it hits the staging area's capacity and thus # not fill the queue with n tokens @@ -303,56 +300,57 @@ def thread_run(): capacity)) # Should have capacity elements in the staging area - self.assertTrue(sess.run(size) == capacity) + self.assertEqual(sess.run(size), capacity) # Clear the staging area completely for i in range(n): sess.run(get) - self.assertTrue(sess.run(size) == 0) + self.assertEqual(sess.run(size), 0) @test_util.run_deprecated_v1 def testOrdering(self): import six import random - with ops.Graph().as_default() as G: + with ops.Graph().as_default() as g: with ops.device('/cpu:0'): x = array_ops.placeholder(dtypes.int32, name='x') pi = array_ops.placeholder(dtypes.int64, name='pi') gi = array_ops.placeholder(dtypes.int64, name='gi') with ops.device(test.gpu_device_name()): - stager = data_flow_ops.MapStagingArea( - [ - dtypes.int32, - ], shapes=[[]], ordered=True) + stager = data_flow_ops.MapStagingArea([ + dtypes.int32, + ], + shapes=[[]], + ordered=True) stage = stager.put(pi, [x], [0]) get = stager.get() size = stager.size() - G.finalize() + g.finalize() n = 10 - with self.session(graph=G) as sess: + with self.session(graph=g) as sess: # Keys n-1..0 keys = list(reversed(six.moves.range(n))) for i in keys: sess.run(stage, feed_dict={pi: i, x: i}) - self.assertTrue(sess.run(size) == n) + self.assertEqual(sess.run(size), n) # Check that key, values come out in ascending order for i, k in enumerate(reversed(keys)): get_key, values = sess.run(get) self.assertTrue(i == k == get_key == values) - self.assertTrue(sess.run(size) == 0) + self.assertEqual(sess.run(size), 0) @test_util.run_deprecated_v1 def testPartialDictInsert(self): - with ops.Graph().as_default() as G: + with ops.Graph().as_default() as g: with ops.device('/cpu:0'): x = array_ops.placeholder(dtypes.float32) f = array_ops.placeholder(dtypes.float32) @@ -370,41 +368,39 @@ def testPartialDictInsert(self): size = stager.size() isize = stager.incomplete_size() - G.finalize() + g.finalize() - with self.session(graph=G) as sess: + with self.session(graph=g) as sess: # 0 complete and incomplete entries - self.assertTrue(sess.run([size, isize]) == [0, 0]) + self.assertEqual(sess.run([size, isize]), [0, 0]) # Stage key 0, x and f tuple entries sess.run(stage_xf, feed_dict={pi: 0, x: 1, f: 2}) - self.assertTrue(sess.run([size, isize]) == [0, 1]) + self.assertEqual(sess.run([size, isize]), [0, 1]) # Stage key 1, x and f tuple entries sess.run(stage_xf, feed_dict={pi: 1, x: 1, f: 2}) - self.assertTrue(sess.run([size, isize]) == [0, 2]) + self.assertEqual(sess.run([size, isize]), [0, 2]) # Now complete key 0 with tuple entry v sess.run(stage_v, feed_dict={pi: 0, v: 1}) # 1 complete and 1 incomplete entry - self.assertTrue(sess.run([size, isize]) == [1, 1]) + self.assertEqual(sess.run([size, isize]), [1, 1]) # We can now obtain tuple associated with key 0 - self.assertTrue( - sess.run([key, ret], feed_dict={ - gi: 0 - }) == [0, { + self.assertEqual( + sess.run([key, ret], feed_dict={gi: 0}), + [0, { 'x': 1, 'f': 2, 'v': 1 }]) # 0 complete and 1 incomplete entry - self.assertTrue(sess.run([size, isize]) == [0, 1]) + self.assertEqual(sess.run([size, isize]), [0, 1]) # Now complete key 1 with tuple entry v sess.run(stage_v, feed_dict={pi: 1, v: 3}) # We can now obtain tuple associated with key 1 - self.assertTrue( - sess.run([key, ret], feed_dict={ - gi: 1 - }) == [1, { + self.assertEqual( + sess.run([key, ret], feed_dict={gi: 1}), + [1, { 'x': 1, 'f': 2, 'v': 3 @@ -412,7 +408,7 @@ def testPartialDictInsert(self): @test_util.run_deprecated_v1 def testPartialIndexInsert(self): - with ops.Graph().as_default() as G: + with ops.Graph().as_default() as g: with ops.device('/cpu:0'): x = array_ops.placeholder(dtypes.float32) f = array_ops.placeholder(dtypes.float32) @@ -428,35 +424,35 @@ def testPartialIndexInsert(self): size = stager.size() isize = stager.incomplete_size() - G.finalize() + g.finalize() - with self.session(graph=G) as sess: + with self.session(graph=g) as sess: # 0 complete and incomplete entries - self.assertTrue(sess.run([size, isize]) == [0, 0]) + self.assertEqual(sess.run([size, isize]), [0, 0]) # Stage key 0, x and f tuple entries sess.run(stage_xf, feed_dict={pi: 0, x: 1, f: 2}) - self.assertTrue(sess.run([size, isize]) == [0, 1]) + self.assertEqual(sess.run([size, isize]), [0, 1]) # Stage key 1, x and f tuple entries sess.run(stage_xf, feed_dict={pi: 1, x: 1, f: 2}) - self.assertTrue(sess.run([size, isize]) == [0, 2]) + self.assertEqual(sess.run([size, isize]), [0, 2]) # Now complete key 0 with tuple entry v sess.run(stage_v, feed_dict={pi: 0, v: 1}) # 1 complete and 1 incomplete entry - self.assertTrue(sess.run([size, isize]) == [1, 1]) + self.assertEqual(sess.run([size, isize]), [1, 1]) # We can now obtain tuple associated with key 0 - self.assertTrue(sess.run([key, ret], feed_dict={gi: 0}) == [0, [1, 1, 2]]) + self.assertEqual(sess.run([key, ret], feed_dict={gi: 0}), [0, [1, 1, 2]]) # 0 complete and 1 incomplete entry - self.assertTrue(sess.run([size, isize]) == [0, 1]) + self.assertEqual(sess.run([size, isize]), [0, 1]) # Now complete key 1 with tuple entry v sess.run(stage_v, feed_dict={pi: 1, v: 3}) # We can now obtain tuple associated with key 1 - self.assertTrue(sess.run([key, ret], feed_dict={gi: 1}) == [1, [1, 3, 2]]) + self.assertEqual(sess.run([key, ret], feed_dict={gi: 1}), [1, [1, 3, 2]]) @test_util.run_deprecated_v1 def testPartialDictGetsAndPeeks(self): - with ops.Graph().as_default() as G: + with ops.Graph().as_default() as g: with ops.device('/cpu:0'): x = array_ops.placeholder(dtypes.float32) f = array_ops.placeholder(dtypes.float32) @@ -480,40 +476,38 @@ def testPartialDictGetsAndPeeks(self): size = stager.size() isize = stager.incomplete_size() - G.finalize() + g.finalize() - with self.session(graph=G) as sess: + with self.session(graph=g) as sess: # 0 complete and incomplete entries - self.assertTrue(sess.run([size, isize]) == [0, 0]) + self.assertEqual(sess.run([size, isize]), [0, 0]) # Stage key 0, x and f tuple entries sess.run(stage_xf, feed_dict={pi: 0, x: 1, f: 2}) - self.assertTrue(sess.run([size, isize]) == [0, 1]) + self.assertEqual(sess.run([size, isize]), [0, 1]) # Stage key 1, x and f tuple entries sess.run(stage_xf, feed_dict={pi: 1, x: 1, f: 2}) - self.assertTrue(sess.run([size, isize]) == [0, 2]) + self.assertEqual(sess.run([size, isize]), [0, 2]) # Now complete key 0 with tuple entry v sess.run(stage_v, feed_dict={pi: 0, v: 1}) # 1 complete and 1 incomplete entry - self.assertTrue(sess.run([size, isize]) == [1, 1]) + self.assertEqual(sess.run([size, isize]), [1, 1]) # We can now peek at 'x' and 'f' values associated with key 0 - self.assertTrue(sess.run(peek_xf, feed_dict={pei: 0}) == {'x': 1, 'f': 2}) + self.assertEqual(sess.run(peek_xf, feed_dict={pei: 0}), {'x': 1, 'f': 2}) # Peek at 'v' value associated with key 0 - self.assertTrue(sess.run(peek_v, feed_dict={pei: 0}) == {'v': 1}) + self.assertEqual(sess.run(peek_v, feed_dict={pei: 0}), {'v': 1}) # 1 complete and 1 incomplete entry - self.assertTrue(sess.run([size, isize]) == [1, 1]) + self.assertEqual(sess.run([size, isize]), [1, 1]) # We can now obtain 'x' and 'f' values associated with key 0 - self.assertTrue( - sess.run([key_xf, get_xf], feed_dict={ - gi: 0 - }) == [0, { + self.assertEqual( + sess.run([key_xf, get_xf], feed_dict={gi: 0}), [0, { 'x': 1, 'f': 2 }]) # Still have 1 complete and 1 incomplete entry - self.assertTrue(sess.run([size, isize]) == [1, 1]) + self.assertEqual(sess.run([size, isize]), [1, 1]) # We can no longer get 'x' and 'f' from key 0 with self.assertRaises(errors.InvalidArgumentError) as cm: @@ -521,40 +515,36 @@ def testPartialDictGetsAndPeeks(self): exc_str = ("Tensor at index '0' for key '0' " 'has already been removed.') - self.assertTrue(exc_str in cm.exception.message) + self.assertIn(exc_str, cm.exception.message) # Obtain 'v' value associated with key 0 - self.assertTrue( - sess.run([key_v, get_v], feed_dict={ - gi: 0 - }) == [0, { + self.assertEqual( + sess.run([key_v, get_v], feed_dict={gi: 0}), [0, { 'v': 1 }]) # 0 complete and 1 incomplete entry - self.assertTrue(sess.run([size, isize]) == [0, 1]) + self.assertEqual(sess.run([size, isize]), [0, 1]) # Now complete key 1 with tuple entry v sess.run(stage_v, feed_dict={pi: 1, v: 1}) # 1 complete and 1 incomplete entry - self.assertTrue(sess.run([size, isize]) == [1, 0]) + self.assertEqual(sess.run([size, isize]), [1, 0]) # Pop without key to obtain 'x' and 'f' values associated with key 1 - self.assertTrue(sess.run([pop_key_xf, pop_xf]) == [1, {'x': 1, 'f': 2}]) + self.assertEqual(sess.run([pop_key_xf, pop_xf]), [1, {'x': 1, 'f': 2}]) # still 1 complete and 1 incomplete entry - self.assertTrue(sess.run([size, isize]) == [1, 0]) + self.assertEqual(sess.run([size, isize]), [1, 0]) # We can now obtain 'x' and 'f' values associated with key 1 - self.assertTrue( - sess.run([pop_key_v, pop_v], feed_dict={ - pi: 1 - }) == [1, { + self.assertEqual( + sess.run([pop_key_v, pop_v], feed_dict={pi: 1}), [1, { 'v': 1 }]) # Nothing is left - self.assertTrue(sess.run([size, isize]) == [0, 0]) + self.assertEqual(sess.run([size, isize]), [0, 0]) @test_util.run_deprecated_v1 def testPartialIndexGets(self): - with ops.Graph().as_default() as G: + with ops.Graph().as_default() as g: with ops.device('/cpu:0'): x = array_ops.placeholder(dtypes.float32) f = array_ops.placeholder(dtypes.float32) @@ -572,28 +562,72 @@ def testPartialIndexGets(self): size = stager.size() isize = stager.incomplete_size() - G.finalize() + g.finalize() - with self.session(graph=G) as sess: + with self.session(graph=g) as sess: # Stage complete tuple sess.run(stage_xvf, feed_dict={pi: 0, x: 1, f: 2, v: 3}) - self.assertTrue(sess.run([size, isize]) == [1, 0]) + self.assertEqual(sess.run([size, isize]), [1, 0]) # Partial get using indices - self.assertTrue( - sess.run([key_xf, get_xf], feed_dict={ - gi: 0 - }) == [0, [1, 2]]) + self.assertEqual( + sess.run([key_xf, get_xf], feed_dict={gi: 0}), [0, [1, 2]]) # Still some of key 0 left - self.assertTrue(sess.run([size, isize]) == [1, 0]) + self.assertEqual(sess.run([size, isize]), [1, 0]) # Partial get of remaining index - self.assertTrue(sess.run([key_v, get_v], feed_dict={gi: 0}) == [0, [3]]) + self.assertEqual(sess.run([key_v, get_v], feed_dict={gi: 0}), [0, [3]]) # All gone - self.assertTrue(sess.run([size, isize]) == [0, 0]) + self.assertEqual(sess.run([size, isize]), [0, 0]) + + @test_util.run_deprecated_v1 + def testNonScalarKeyOrderedMap(self): + with ops.Graph().as_default() as g: + x = array_ops.placeholder(dtypes.float32) + v = 2. * (array_ops.zeros([128, 128]) + x) + t = data_flow_ops.gen_data_flow_ops.ordered_map_stage( + key=constant_op.constant(value=[1], shape=(1, 3), dtype=dtypes.int64), + indices=np.array([[6]]), + values=[x, v], + dtypes=[dtypes.int64], + capacity=0, + memory_limit=0, + container='container1', + shared_name='', + name=None) + + g.finalize() + + with self.session(graph=g) as sess: + with self.assertRaisesRegex(errors.InvalidArgumentError, + 'key must be an int64 scalar'): + sess.run(t, feed_dict={x: 1}) + + @test_util.run_deprecated_v1 + def testNonScalarKeyUnorderedMap(self): + with ops.Graph().as_default() as g: + x = array_ops.placeholder(dtypes.float32) + v = 2. * (array_ops.zeros([128, 128]) + x) + t = data_flow_ops.gen_data_flow_ops.map_stage( + key=constant_op.constant(value=[1], shape=(1, 3), dtype=dtypes.int64), + indices=np.array([[6]]), + values=[x, v], + dtypes=[dtypes.int64], + capacity=0, + memory_limit=0, + container='container1', + shared_name='', + name=None) + + g.finalize() + + with self.session(graph=g) as sess: + with self.assertRaisesRegex(errors.InvalidArgumentError, + 'key must be an int64 scalar'): + sess.run(t, feed_dict={x: 1}) if __name__ == '__main__': diff --git a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py index c7c64a48ddfacb..0e5f41f3a5961d 100644 --- a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py +++ b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py @@ -22,6 +22,7 @@ from tensorflow.python.eager import context from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import errors_impl from tensorflow.python.framework import test_util @@ -70,7 +71,7 @@ def _VerifyOneTest(self, pool_func, input_sizes, window, strides, padding, # Initializes the input tensor with array containing incrementing # numbers from 1. x = [f * 1.0 for f in range(1, total_size + 1)] - with self.cached_session(use_gpu=use_gpu) as sess: + with self.cached_session(use_gpu=use_gpu): t = constant_op.constant(x, shape=input_sizes) window = [1] + list(window) + [1] strides = [1] + list(strides) + [1] @@ -127,6 +128,23 @@ def testAvgPool3dSamePaddingDifferentStrides(self): padding="SAME", expected=expected_output) + def testMaxPool3dGrad(self): + with self.assertRaises( + (errors.ResourceExhaustedError, errors.InvalidArgumentError)): + with self.cached_session(): + orig_input_shape = constant_op.constant( + 1879048192, shape=[5], dtype=dtypes.int32) + grad = constant_op.constant( + 1, shape=[1, 3, 2, 4, 2], dtype=dtypes.float32) + t = gen_nn_ops.AvgPool3DGrad( + orig_input_shape=orig_input_shape, + grad=grad, + ksize=[1, 1, 1, 1, 1], + strides=[1, 1, 1, 1, 1], + padding="SAME", + data_format="NDHWC") + self.evaluate(t) + def testMaxPool3dValidPadding(self): expected_output = [40.0, 41.0, 42.0] self._VerifyValues( diff --git a/tensorflow/python/kernel_tests/pooling_ops_test.py b/tensorflow/python/kernel_tests/pooling_ops_test.py index d0207671cd6571..ebcbe75ecbc01b 100644 --- a/tensorflow/python/kernel_tests/pooling_ops_test.py +++ b/tensorflow/python/kernel_tests/pooling_ops_test.py @@ -540,6 +540,18 @@ def testAvgPoolEmptyInput(self, **kwargs): expected=[], **kwargs) + @test_util.run_in_graph_and_eager_modes + def testRawAvgPoolLargeKsizeRaiseError(self): + with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)): + with self.cached_session(): + t = gen_nn_ops.avg_pool( + value=np.ones([1, 1, 1, 1]), + ksize=[1, 1e20, 1, 1], + strides=[1, 1, 1, 1], + padding="SAME", + data_format="NHWC") + self.evaluate(t) + @parameterized.parameters( GetTestConfigsDicts(nn_ops.max_pool, gen_nn_ops.max_pool_v2)) @test_util.run_deprecated_v1 @@ -763,6 +775,18 @@ def testMaxPoolEmptyInput(self, **kwargs): expected=[], **kwargs) + @parameterized.parameters( + GetTestConfigsDicts(nn_ops.max_pool, gen_nn_ops.max_pool_v2)) + @test_util.run_deprecated_v1 + def testMaxPoolInvalidFilterSize(self, **kwargs): + with self.cached_session(use_gpu=test.is_gpu_available()): + t = constant_op.constant(1.0, shape=[1, 1, 1, 1]) + with self.assertRaisesRegex( + (errors_impl.InvalidArgumentError, ValueError), + "Negative dimension size"): + t = self.evaluate( + nn_ops.max_pool(t, ksize=[1, 1, 2, 1], strides=1, padding="VALID")) + # Tests for DepthwiseMaxPooling on CPU only. @parameterized.parameters( GetTestConfigsDicts( @@ -2471,6 +2495,22 @@ def testMaxPoolGradWithArgmaxEagerShapeErrors(self): inp, grad, argmax, ksize=[1, 1, 1, 1], strides=[1, 1, 1, 1], padding="VALID") + def testAvgPoolGradInvalidInputShapeRaiseError(self): + with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)): + with self.cached_session(): + orig_input_shape = constant_op.constant( + -536870912, shape=[4], dtype=dtypes.int32) + grad = constant_op.constant( + .0890338004362538, shape=[1, 5, 7, 1], dtype=dtypes.float64) + t = gen_nn_ops.AvgPoolGrad( + orig_input_shape=orig_input_shape, + grad=grad, + ksize=[1, 2, 2, 1], + strides=[1, 2, 2, 1], + padding="VALID", + data_format="NHWC") + self.evaluate(t) + def GetMaxPoolFwdTest(input_size, filter_size, strides, padding): diff --git a/tensorflow/python/kernel_tests/quantization_ops/BUILD b/tensorflow/python/kernel_tests/quantization_ops/BUILD new file mode 100644 index 00000000000000..ff0be9898c601d --- /dev/null +++ b/tensorflow/python/kernel_tests/quantization_ops/BUILD @@ -0,0 +1,24 @@ +# Tests of TensorFlow quantization ops written using the Python API. + +# buildifier: disable=same-origin-load +load("//tensorflow:tensorflow.bzl", "tf_py_test") + +package( + default_visibility = ["//tensorflow:internal"], + licenses = ["notice"], +) + +tf_py_test( + name = "quantization_ops_test", + size = "small", + srcs = ["quantization_ops_test.py"], + deps = [ + "//tensorflow/python:array_ops", + "//tensorflow/python:client", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:math_ops", + "//third_party/py/numpy", + ], +) diff --git a/tensorflow/python/kernel_tests/quantization_ops/quantization_ops_test.py b/tensorflow/python/kernel_tests/quantization_ops/quantization_ops_test.py new file mode 100644 index 00000000000000..354ff385147b16 --- /dev/null +++ b/tensorflow/python/kernel_tests/quantization_ops/quantization_ops_test.py @@ -0,0 +1,467 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for tf.quantize ops.""" +import numpy as np + +from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn_ops +from tensorflow.python.platform import googletest + + +class FakeQuantWithMinMaxVarsOpTest(test_util.TensorFlowTestCase): + + @test_util.run_in_graph_and_eager_modes + def test_invalid_inputs(self): + inputs = constant_op.constant( + value=[[1.0], [2.0], [4.0]], dtype=dtypes.float32) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + array_ops.fake_quant_with_min_max_vars( + inputs=inputs, min=0.0, max=[[1.0], [2.0], [4.0]])) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + array_ops.fake_quant_with_min_max_vars( + inputs=inputs, min=[[1.0], [2.0], [4.0]], max=1.0)) + + +class FakeQuantWithMinMaxVarsPerChannelOpTest(test_util.TensorFlowTestCase): + + @test_util.run_in_graph_and_eager_modes + def test_invalid_inputs(self): + inputs = constant_op.constant( + value=[[1.0], [2.0], [4.0]], dtype=dtypes.float32) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 1"): + self.evaluate( + array_ops.fake_quant_with_min_max_vars_per_channel( + inputs=inputs, min=[[0.0]], max=[1.0])) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "Dimensions must be equal|incorrect size"): + self.evaluate( + array_ops.fake_quant_with_min_max_vars_per_channel( + inputs=inputs, min=[0.0, 0.1], max=[1.0])) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 1"): + self.evaluate( + array_ops.fake_quant_with_min_max_vars_per_channel( + inputs=inputs, min=[1.0], max=[[1.0]])) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "Dimensions must be equal|incorrect size"): + self.evaluate( + array_ops.fake_quant_with_min_max_vars_per_channel( + inputs=inputs, min=[0.0], max=[1.0, 1.1])) + + +class FakeQuantWithMinMaxVarsGradientOpTest(test_util.TensorFlowTestCase): + + @test_util.run_in_graph_and_eager_modes + def test_invalid_inputs(self): + gradients = constant_op.constant( + value=[[1.0], [2.0], [4.0]], dtype=dtypes.float32) + inputs = constant_op.constant( + value=[[1.0], [2.0], [4.0]], dtype=dtypes.float32) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be equal rank|must be rank 0"): + self.evaluate( + array_ops.fake_quant_with_min_max_vars_gradient( + gradients=gradients, + inputs=inputs, + min=0.0, + max=[[1.0], [2.0], [4.0]])) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + array_ops.fake_quant_with_min_max_vars_gradient( + gradients=gradients, + inputs=inputs, + min=[[1.0], [2.0], [4.0]], + max=[[1.0], [2.0], [4.0]])) + + +class FakeQuantWithMinMaxVarsPerChannelGradientOpTest( + test_util.TensorFlowTestCase): + + @test_util.run_in_graph_and_eager_modes + def test_invalid_inputs(self): + gradients = constant_op.constant( + value=[[1.0], [2.0], [4.0]], dtype=dtypes.float32) + inputs = constant_op.constant( + value=[[1.0], [2.0], [4.0]], dtype=dtypes.float32) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "Shapes must be equal rank|must be rank 1"): + self.evaluate( + array_ops.fake_quant_with_min_max_vars_per_channel_gradient( + gradients=gradients, inputs=inputs, min=[[0.0]], max=[1.0])) + + with self.assertRaisesRegex( + (ValueError, errors.InvalidArgumentError), + "Dimension 0 in both shapes must be equal|incorrect size"): + self.evaluate( + array_ops.fake_quant_with_min_max_vars_per_channel_gradient( + gradients=gradients, inputs=inputs, min=[0.0, 0.1], max=[1.0])) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "Shapes must be equal rank|must be rank 1"): + self.evaluate( + array_ops.fake_quant_with_min_max_vars_per_channel_gradient( + gradients=gradients, inputs=inputs, min=[1.0], max=[[1.0]])) + + with self.assertRaisesRegex( + (ValueError, errors.InvalidArgumentError), + "Dimension 0 in both shapes must be equal|incorrect size"): + self.evaluate( + array_ops.fake_quant_with_min_max_vars_per_channel_gradient( + gradients=gradients, inputs=inputs, min=[0.0], max=[1.0, 1.1])) + + +class QuantizedBiasedAddTest(test_util.TensorFlowTestCase): + + @test_util.run_in_graph_and_eager_modes + def test_invalid_inputs(self): + inputs = constant_op.constant( + np.int8(0), shape=[3, 3, 3, 3], dtype=dtypes.qint8) + bias = constant_op.constant(np.int8(0), shape=[3], dtype=dtypes.qint8) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + nn_ops.quantized_bias_add( + input=inputs, + bias=bias, + min_input=[], + max_input=1.0, + min_bias=0.0, + max_bias=1.0, + out_type=dtypes.qint32)) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + nn_ops.quantized_bias_add( + input=inputs, + bias=bias, + min_input=0.0, + max_input=[], + min_bias=0.0, + max_bias=1.0, + out_type=dtypes.qint32)) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + nn_ops.quantized_bias_add( + input=inputs, + bias=bias, + min_input=0.0, + max_input=1.0, + min_bias=[], + max_bias=1.0, + out_type=dtypes.qint32)) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + nn_ops.quantized_bias_add( + input=inputs, + bias=bias, + min_input=0.0, + max_input=1.0, + min_bias=0.0, + max_bias=[], + out_type=dtypes.qint32)) + + +class QuantizedInstanceNormOpTest(test_util.TensorFlowTestCase): + + @test_util.run_in_graph_and_eager_modes + def test_invalid_inputs(self): + inputs = constant_op.constant( + np.uint8(0), shape=[3, 3, 3, 3], dtype=dtypes.quint8) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + array_ops.quantized_instance_norm( + x=inputs, x_min=0.0, x_max=[[1.0], [2.0], [4.0]])) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + array_ops.quantized_instance_norm( + x=inputs, x_min=[[1.0], [2.0], [4.0]], x_max=1.0)) + + +class QuantizedAvgPoolingOpTest(test_util.TensorFlowTestCase): + + @test_util.run_in_graph_and_eager_modes + def test_invalid_inputs(self): + inputs = constant_op.constant( + np.uint8(0), shape=[3, 3, 3, 3], dtype=dtypes.quint8) + ksize = [1, 1, 1, 1] + strides = [1, 1, 1, 1] + padding = "SAME" + + with self.assertRaisesRegex((errors.InvalidArgumentError, ValueError), + "must be.* rank 0"): + self.evaluate( + nn_ops.quantized_avg_pool( + input=inputs, + min_input=[], + max_input=1.0, + ksize=ksize, + strides=strides, + padding=padding)) + + with self.assertRaisesRegex((errors.InvalidArgumentError, ValueError), + "must be.* rank 0"): + self.evaluate( + nn_ops.quantized_avg_pool( + input=inputs, + min_input=0.0, + max_input=[], + ksize=ksize, + strides=strides, + padding=padding)) + + +class QuantizedMaxPoolingOpTest(test_util.TensorFlowTestCase): + + @test_util.run_in_graph_and_eager_modes + def test_invalid_inputs(self): + inputs = constant_op.constant( + np.uint8(0), shape=[3, 3, 3, 3], dtype=dtypes.quint8) + ksize = [1, 1, 1, 1] + strides = [1, 1, 1, 1] + padding = "SAME" + + with self.assertRaisesRegex((errors.InvalidArgumentError, ValueError), + "must be.* rank 0"): + self.evaluate( + nn_ops.quantized_max_pool( + input=inputs, + min_input=[], + max_input=1.0, + ksize=ksize, + strides=strides, + padding=padding)) + + with self.assertRaisesRegex((errors.InvalidArgumentError, ValueError), + "must be.* rank 0"): + self.evaluate( + nn_ops.quantized_max_pool( + input=inputs, + min_input=0.0, + max_input=[], + ksize=ksize, + strides=strides, + padding=padding)) + + +class RequantizeOpTest(test_util.TensorFlowTestCase): + + @test_util.run_in_graph_and_eager_modes + def test_invalid_inputs(self): + inputs = constant_op.constant( + np.int32(0), shape=[3, 3, 3, 3], dtype=dtypes.qint32) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + math_ops.requantize( + input=inputs, + input_min=[], + input_max=1.0, + requested_output_min=0.0, + requested_output_max=1.0, + out_type=dtypes.qint8)) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + math_ops.requantize( + input=inputs, + input_min=0.0, + input_max=[], + requested_output_min=0.0, + requested_output_max=1.0, + out_type=dtypes.qint8)) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + math_ops.requantize( + input=inputs, + input_min=0.0, + input_max=1.0, + requested_output_min=[], + requested_output_max=1.0, + out_type=dtypes.qint8)) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + math_ops.requantize( + input=inputs, + input_min=0.0, + input_max=1.0, + requested_output_min=0.0, + requested_output_max=[], + out_type=dtypes.qint8)) + + +class QuantizedAddOpTest(test_util.TensorFlowTestCase): + + @test_util.run_in_graph_and_eager_modes + def test_invalid_inputs(self): + x = constant_op.constant( + np.int8(0), shape=[3, 3, 3, 3], dtype=dtypes.quint8) + y = constant_op.constant(np.int8(0), shape=[3], dtype=dtypes.quint8) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + math_ops.quantized_add( + x=x, + y=y, + min_x=[], + max_x=1.0, + min_y=0.0, + max_y=1.0, + Toutput=dtypes.qint32)) + + +class QuantizedReluOpTest(test_util.TensorFlowTestCase): + + @test_util.run_in_graph_and_eager_modes + def test_invalid_inputs(self): + inputs = constant_op.constant( + np.int8(0), shape=[3, 3, 3, 3], dtype=dtypes.quint8) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + nn_ops.quantized_relu( + features=inputs, + min_features=[], + max_features=127.0, + out_type=dtypes.quint8)) + + +class QuantizedRelu6OpTest(test_util.TensorFlowTestCase): + + @test_util.run_in_graph_and_eager_modes + def test_invalid_inputs(self): + inputs = constant_op.constant( + np.int8(0), shape=[3, 3, 3, 3], dtype=dtypes.quint8) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + nn_ops.quantized_relu6( + features=inputs, + min_features=[], + max_features=127.0, + out_type=dtypes.quint8)) + + +class QuantizeAndDequantizeV3OpTest(test_util.TensorFlowTestCase): + + @test_util.run_in_graph_and_eager_modes + def test_valid(self): + with ops.Graph().as_default(), context.eager_mode(): + input_value = constant_op.constant([-0.8, -0.5, 0, 0.3, 0.8, -2.0], + shape=(6,), + dtype=dtypes.float32), + input_min = constant_op.constant(-127, shape=(), dtype=dtypes.float32) + input_max = constant_op.constant(127, shape=(), dtype=dtypes.float32) + num_bits = constant_op.constant(8, shape=(), dtype=dtypes.int32) + + quantized = array_ops.quantize_and_dequantize_v3( + input_value, + input_min, + input_max, + num_bits, + signed_input=True, + range_given=False) + self.assertSequenceAlmostEqual( + input_value[0].numpy(), quantized.numpy()[0], delta=0.05) + + @test_util.run_in_graph_and_eager_modes + def test_invalid_inputs(self): + input_value = constant_op.constant([-0.8, -0.5, 0, 0.3, 0.8, -2.0], + shape=(6,), + dtype=dtypes.float32), + input_min = constant_op.constant(-127, shape=(), dtype=dtypes.float32) + input_max = constant_op.constant(127, shape=(), dtype=dtypes.float32) + # Tensor with invalid shape and invalid number of elements. + num_bits = constant_op.constant([], shape=(0,), dtype=dtypes.int32) + + # Test that running the op raises error. It raises different errors + # depending on whether the shape inference is run first or the op's + # Compute() is run first. + try: + array_ops.quantize_and_dequantize_v3( + input_value, input_min, input_max, num_bits, signed_input=True) + except Exception as ex: # pylint: disable=broad-except + if isinstance(ex, errors.InvalidArgumentError): + self.assertRegex(str(ex), "The `num_bits` tensor should be a scalar.") + elif isinstance(ex, ValueError): + self.assertRegex(str(ex), "Shape must be rank 0") + else: + self.fail( + "Raised exception other than expected: %s. " + "Expected exceptions are errors.InvalidArgumentError or ValueError", + ex.__name__) + else: + self.fail( + "Did not raise an exception where it is expected to raise either " + "a ValueError or errors.InvalidArgumentError.") + +class QuantizeDownAndShrinkRangeOpTest(test_util.TensorFlowTestCase): + + @test_util.run_in_graph_and_eager_modes + def test_invalid_inputs(self): + inputs = constant_op.constant( + np.int32(0), shape=[3, 3, 3, 3], dtype=dtypes.qint32) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "must be rank 0"): + self.evaluate( + math_ops.quantize_down_and_shrink_range(input=inputs, + input_min=[], + input_max=4.0, + out_type=dtypes.quint8)) + + +if __name__ == "__main__": + googletest.main() diff --git a/tensorflow/python/kernel_tests/random/parameterized_truncated_normal_op_test.py b/tensorflow/python/kernel_tests/random/parameterized_truncated_normal_op_test.py index 5ec054f6bae5fa..865c1d7c5fa21c 100644 --- a/tensorflow/python/kernel_tests/random/parameterized_truncated_normal_op_test.py +++ b/tensorflow/python/kernel_tests/random/parameterized_truncated_normal_op_test.py @@ -307,6 +307,29 @@ def testSamplingWithSmallStdDevFarFromBound(self): self.assertAllGreater(samples, 0.) self.assertAllGreater(samples_stateless, 0.) + def testShapeTypes(self): + for shape_dtype in [np.int32, np.int64]: + shape = np.array([1000], dtype=shape_dtype) + sample_op = random_ops.parameterized_truncated_normal( + shape=shape, means=0.0, stddevs=0.1, minvals=-1., maxvals=1.) + new_seed = random_ops.random_uniform([2], + seed=1234, + minval=0, + maxval=(2**31 - 1), + dtype=np.int32) + sample_op_stateless = stateless.stateless_parameterized_truncated_normal( + shape=shape, + seed=new_seed, + means=0.0, + stddevs=0.1, + minvals=-1., + maxvals=1.) + + samples = self.evaluate(sample_op) + stateless_samples = self.evaluate(sample_op_stateless) + self.assertAllEqual(samples.shape, shape) + self.assertAllEqual(stateless_samples.shape, shape) + def testStatelessParameterizedTruncatedNormalHasGrads(self): mean = variables.Variable(0.01) stddev = variables.Variable(1.) diff --git a/tensorflow/python/kernel_tests/random/random_gamma_test.py b/tensorflow/python/kernel_tests/random/random_gamma_test.py index c6b2f542d93a9a..a53b81492c2927 100644 --- a/tensorflow/python/kernel_tests/random/random_gamma_test.py +++ b/tensorflow/python/kernel_tests/random/random_gamma_test.py @@ -21,7 +21,10 @@ import numpy as np from six.moves import xrange # pylint: disable=redefined-builtin +from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import random_seed from tensorflow.python.framework import test_util @@ -221,6 +224,16 @@ def testPositive(self): self.assertEqual(0, math_ops.reduce_sum(math_ops.cast( math_ops.less_equal(x, 0.), dtype=dtypes.int64)).eval()) + def testSizeTooLarge(self): + # Grappler asserts on size overflow, so this error is only caught when + # running eagerly. + if context.executing_eagerly(): + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "overflow"): + rate = constant_op.constant(1.0, shape=(4, 4, 4, 4, 4)) + self.evaluate( + random_ops.random_gamma( + shape=[46902, 51188, 34063, 59195], alpha=rate)) if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/random/random_poisson_test.py b/tensorflow/python/kernel_tests/random/random_poisson_test.py index 2d94533078d725..431f3b860fcbfa 100644 --- a/tensorflow/python/kernel_tests/random/random_poisson_test.py +++ b/tensorflow/python/kernel_tests/random/random_poisson_test.py @@ -22,6 +22,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.kernel_tests.random import util @@ -176,6 +177,14 @@ def testInfRate(self): sample = random_ops.random_poisson(shape=[2], lam=np.inf) self.assertAllEqual([np.inf, np.inf], self.evaluate(sample)) + def testSizeTooLarge(self): + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "overflow"): + rate = constant_op.constant(1.0, shape=(4, 4, 4, 4, 4)) + self.evaluate( + random_ops.random_poisson( + shape=[46902, 51188, 34063, 59195], lam=rate)) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/rnn_cell_test.py b/tensorflow/python/kernel_tests/rnn_cell_test.py index c096357d2fbb1d..f71966a80b860d 100644 --- a/tensorflow/python/kernel_tests/rnn_cell_test.py +++ b/tensorflow/python/kernel_tests/rnn_cell_test.py @@ -37,6 +37,7 @@ from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import gen_rnn_ops from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops @@ -1328,6 +1329,88 @@ def testDynamicEquivalentToStaticRNN(self): def testDynamicEquivalentToStaticRNNWithSequenceLength(self): self._testDynamicEquivalentToStaticRNN(use_sequence_length=True) + @test_util.run_in_graph_and_eager_modes + def testLSTMBlockCellErrorHandling(self): + forget_bias = 1 + cell_clip = 0 + use_peephole = False + x = constant_op.constant(0.837607, shape=[28, 29], dtype=dtypes.float32) + cs_prev = constant_op.constant(0, shape=[28, 17], dtype=dtypes.float32) + h_prev = constant_op.constant( + 0.592631638, shape=[28, 17], dtype=dtypes.float32) + w = constant_op.constant(0.887386262, shape=[46, 68], dtype=dtypes.float32) + wci = constant_op.constant(0, shape=[], dtype=dtypes.float32) + wcf = constant_op.constant(0, shape=[17], dtype=dtypes.float32) + wco = constant_op.constant( + 0.592631638, shape=[28, 17], dtype=dtypes.float32) + b = constant_op.constant(0.75259006, shape=[68], dtype=dtypes.float32) + with self.assertRaises(errors_impl.InvalidArgumentError): + self.evaluate( + gen_rnn_ops.lstm_block_cell( + x=x, + cs_prev=cs_prev, + h_prev=h_prev, + w=w, + wci=wci, + wcf=wcf, + wco=wco, + b=b, + forget_bias=forget_bias, + cell_clip=cell_clip, + use_peephole=use_peephole)) + + @test_util.run_in_graph_and_eager_modes + def testLSTMBlockCellGradErrorHandling(self): + use_peephole = False + seq_len_max = constant_op.constant(1, shape=[], dtype=dtypes.int64) + x = constant_op.constant(0.504355371, shape=[1, 1, 1], dtype=dtypes.float32) + cs_prev = constant_op.constant( + 0.504355371, shape=[1, 1, 1], dtype=dtypes.float32) + h_prev = constant_op.constant( + 0.504355371, shape=[1, 1], dtype=dtypes.float32) + w = constant_op.constant(0.504355371, shape=[1, 1], dtype=dtypes.float32) + wci = constant_op.constant(0.504355371, shape=[1], dtype=dtypes.float32) + wcf = constant_op.constant(0.504355371, shape=[1], dtype=dtypes.float32) + wco = constant_op.constant(0.504355371, shape=[1], dtype=dtypes.float32) + b = constant_op.constant(0.504355371, shape=[1], dtype=dtypes.float32) + i = constant_op.constant(0.504355371, shape=[1, 1, 1], dtype=dtypes.float32) + cs = constant_op.constant( + 0.504355371, shape=[1, 1, 1], dtype=dtypes.float32) + f = constant_op.constant(0.504355371, shape=[1, 1, 1], dtype=dtypes.float32) + o = constant_op.constant(0.504355371, shape=[1, 1, 1], dtype=dtypes.float32) + ci = constant_op.constant( + 0.504355371, shape=[1, 1, 1], dtype=dtypes.float32) + co = constant_op.constant( + 0.504355371, shape=[1, 1, 1], dtype=dtypes.float32) + h = constant_op.constant(0.504355371, shape=[1, 1, 1], dtype=dtypes.float32) + cs_grad = constant_op.constant( + 0.504355371, shape=[1, 1, 1], dtype=dtypes.float32) + h_grad = constant_op.constant( + 0.504355371, shape=[1, 1, 1], dtype=dtypes.float32) + with self.assertRaisesRegex((ValueError, errors_impl.InvalidArgumentError), + "must be rank"): + self.evaluate( + gen_rnn_ops.block_lstm_grad_v2( + seq_len_max=seq_len_max, + x=x, + cs_prev=cs_prev, + h_prev=h_prev, + w=w, + wci=wci, + wcf=wcf, + wco=wco, + b=b, + i=i, + cs=cs, + f=f, + o=o, + ci=ci, + co=co, + h=h, + cs_grad=cs_grad, + h_grad=h_grad, + use_peephole=use_peephole)) + class BidirectionalRNNTest(test.TestCase): diff --git a/tensorflow/python/kernel_tests/segment_reduction_ops_test.py b/tensorflow/python/kernel_tests/segment_reduction_ops_test.py index 6b50c60b68d284..1af2c27cd92a35 100644 --- a/tensorflow/python/kernel_tests/segment_reduction_ops_test.py +++ b/tensorflow/python/kernel_tests/segment_reduction_ops_test.py @@ -280,7 +280,7 @@ def testInvalidIds(self): math_ops.segment_prod, ]: with self.cached_session(use_gpu=False): - with self.assertRaises((ValueError, errors_impl.InternalError)): + with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)): s = op(data=np.ones((1, 10, 1)), segment_ids=[1676240524292489355]) self.evaluate(s) @@ -800,6 +800,19 @@ def testSegmentsInvalid7(self): with self.assertRaisesOpError("segment ids must be >= 0"): self.evaluate(s) + @test_util.run_deprecated_v1 + def testSegmentsInvalid8(self): + tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32) + ops_list = [math_ops.sparse_segment_sum, math_ops.sparse_segment_mean] + segment_indices = [2**62 - 1] + tf_indices = [2**62 - 1] + with self.session(use_gpu=False): + for tf_op in ops_list: + s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices) + with self.assertRaisesOpError( + "Encountered overflow when multiplying"): + self.evaluate(s) + def testSegmentWithNumSegmentsValid(self): # Baseline for the test*WithNumSegmentsInvalid* methods below. tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32) diff --git a/tensorflow/python/kernel_tests/sets_test.py b/tensorflow/python/kernel_tests/sets_test.py index 0e9ecf094b34df..05e6ff08cff028 100644 --- a/tensorflow/python/kernel_tests/sets_test.py +++ b/tensorflow/python/kernel_tests/sets_test.py @@ -26,6 +26,7 @@ from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_set_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import sets from tensorflow.python.ops import sparse_ops @@ -1265,6 +1266,18 @@ def _assert_set_operation(self, expected_indices, expected_values, last_indices)) self.assertAllEqual(expected_shape, sparse_tensor_value.dense_shape) + def test_raw_ops_setsize_invalid_shape(self): + with self.assertRaisesRegex(errors_impl.InvalidArgumentError, + "Shape must be a 1D tensor"): + invalid_shape = 1 + self.evaluate( + gen_set_ops.set_size( + set_indices=1, + set_values=[1, 1], + set_shape=invalid_shape, + validate_indices=True, + name="")) + if __name__ == "__main__": googletest.main() diff --git a/tensorflow/python/kernel_tests/shape_ops_test.py b/tensorflow/python/kernel_tests/shape_ops_test.py index cfd216f1d3fbe8..3380d47c88accd 100644 --- a/tensorflow/python/kernel_tests/shape_ops_test.py +++ b/tensorflow/python/kernel_tests/shape_ops_test.py @@ -728,7 +728,7 @@ def testLargeTensor(self): if test_util.is_xla_enabled(): # The following test fails with XLA enabled. return - with self.assertRaises(errors_impl.InternalError): + with self.assertRaises(errors_impl.InvalidArgumentError): with self.cached_session(): tiled = array_ops.tile( np.ones((1, 1, 1)), [100000000, 100000000, 100000000]) diff --git a/tensorflow/python/kernel_tests/signal/fft_ops_test.py b/tensorflow/python/kernel_tests/signal/fft_ops_test.py index 13002e5ee295a7..ad980d579f0e85 100644 --- a/tensorflow/python/kernel_tests/signal/fft_ops_test.py +++ b/tensorflow/python/kernel_tests/signal/fft_ops_test.py @@ -613,6 +613,15 @@ def test_grad_random(self, rank, extra_dims, size, np_rtype): self._tf_ifft_for_rank(rank), re, im, result_is_complex=False, rtol=tol, atol=tol) + def test_invalid_args(self): + # Test case for GitHub issue 55263 + a = np.empty([6, 0]) + b = np.array([1, -1]) + with self.assertRaisesRegex(errors.InvalidArgumentError, "must >= 0"): + with self.session(): + v = fft_ops.rfft2d(input_tensor=a, fft_length=b) + self.evaluate(v) + @test_util.run_all_in_graph_and_eager_modes class FFTShiftTest(test.TestCase, parameterized.TestCase): diff --git a/tensorflow/python/kernel_tests/spacetobatch_op_test.py b/tensorflow/python/kernel_tests/spacetobatch_op_test.py index 97b23b86ae843c..7c3b32919d3111 100644 --- a/tensorflow/python/kernel_tests/spacetobatch_op_test.py +++ b/tensorflow/python/kernel_tests/spacetobatch_op_test.py @@ -20,7 +20,9 @@ import numpy as np +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_util from tensorflow.python.framework import test_util @@ -520,6 +522,27 @@ def testUnknown(self): dtypes.float32, shape=(3, 2, 3, 2)), [2, 3], [[1, 1], [0, 0]]) self.assertEqual([3 * 2 * 3, 2, 1, 2], t.get_shape().as_list()) + @test_util.run_in_graph_and_eager_modes + def testInvalidBlockShape(self): + tf_in = constant_op.constant( + -3.5e+35, shape=[10, 20, 20], dtype=dtypes.float32) + block_shape = constant_op.constant(-10, shape=[2], dtype=dtypes.int64) + paddings = constant_op.constant(0, shape=[2, 2], dtype=dtypes.int32) + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "block_shape must be positive"): + array_ops.space_to_batch_nd(tf_in, block_shape, paddings) + + @test_util.run_in_graph_and_eager_modes + def testOutputSizeOutOfBounds(self): + tf_in = constant_op.constant( + -3.5e+35, shape=[10, 19, 22], dtype=dtypes.float32) + block_shape = constant_op.constant( + 1879048192, shape=[2], dtype=dtypes.int64) + paddings = constant_op.constant(0, shape=[2, 2], dtype=dtypes.int32) + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + "Negative.* dimension size caused by overflow"): + array_ops.space_to_batch_nd(tf_in, block_shape, paddings) + class SpaceToBatchGradientTest(test.TestCase, PythonOpImpl): diff --git a/tensorflow/python/kernel_tests/sparse_add_op_test.py b/tensorflow/python/kernel_tests/sparse_add_op_test.py index 1a43564fec2056..31eb9d2b2bbe12 100644 --- a/tensorflow/python/kernel_tests/sparse_add_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_add_op_test.py @@ -193,7 +193,6 @@ def testSparseTensorDenseAddGradients(self): [(nnz,), (n, m)], s, (n, m)) self.assertLess(err, 1e-3) - @test_util.run_deprecated_v1 def testInvalidSparseTensor(self): with test_util.force_cpu(): shape = [2, 2] @@ -205,12 +204,49 @@ def testInvalidSparseTensor(self): [[1, 3]], # ...so is 3. ]: sparse = sparse_tensor.SparseTensorValue(bad_idx, val, shape) - s = sparse_ops.sparse_add(sparse, dense) - - with self.assertRaisesRegex(errors_impl.InvalidArgumentError, - "invalid index"): + with self.assertRaisesRegex( + (ValueError, errors_impl.InvalidArgumentError), "invalid index"): + s = sparse_ops.sparse_add(sparse, dense) self.evaluate(s) + def _testSparseDenseInvalidInputs(self, + a_indices, + a_values, + a_shape, + b, + expected_error=""): + # Public API call to sparse-dense add. + with self.assertRaisesRegex((ValueError, errors_impl.InvalidArgumentError), + expected_error): + a = sparse_tensor.SparseTensor(a_indices, a_values, a_shape) + self.evaluate(sparse_ops.sparse_add(a, b)) + # Directly call generated kernel, by-passing SparseTensor validation. + with self.assertRaisesRegex((ValueError, errors_impl.InvalidArgumentError), + expected_error): + self.evaluate( + sparse_ops.gen_sparse_ops.sparse_tensor_dense_add( + a_indices, a_values, a_shape, b)) + + def testSparseDenseInvalidInputs(self): + self._testSparseDenseInvalidInputs( + a_indices=constant_op.constant(0, shape=[17, 2], dtype=dtypes.int64), + a_values=constant_op.constant(0, shape=[5], dtype=dtypes.float32), + a_shape=constant_op.constant([3, 4], dtype=dtypes.int64), + b=constant_op.constant(1, shape=[3, 4], dtype=dtypes.float32), + expected_error="Dimensions 17 and 5 are not compatible") + self._testSparseDenseInvalidInputs( + a_indices=constant_op.constant(0, shape=[17, 4], dtype=dtypes.int64), + a_values=constant_op.constant(0, shape=[17], dtype=dtypes.float32), + a_shape=constant_op.constant([3, 4], dtype=dtypes.int64), + b=constant_op.constant(1, shape=[3, 4], dtype=dtypes.float32), + expected_error="Dimensions 4 and 2 are not compatible") + self._testSparseDenseInvalidInputs( + a_indices=constant_op.constant(7, shape=[17, 2], dtype=dtypes.int64), + a_values=constant_op.constant(0, shape=[17], dtype=dtypes.float32), + a_shape=constant_op.constant([3, 4], dtype=dtypes.int64), + b=constant_op.constant(1, shape=[3, 4], dtype=dtypes.float32), + expected_error="invalid index") + ######################## Benchmarking code diff --git a/tensorflow/python/kernel_tests/sparse_cross_op_test.py b/tensorflow/python/kernel_tests/sparse_cross_op_test.py index 9f36e7212e1252..3c1c63f7734db5 100644 --- a/tensorflow/python/kernel_tests/sparse_cross_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_cross_op_test.py @@ -877,6 +877,14 @@ def test_all_columns_empty(self): with self.cached_session(): self._assert_sparse_tensor_empty(self.evaluate(out)) + def testNonScalarInput(self): + with self.assertRaisesRegex(errors.InvalidArgumentError, + 'Input separator should be a scalar.'): + self.evaluate(sparse_ops.sparse_cross( + inputs=[], + name='a', + separator=constant_op.constant(['a', 'b'], dtype=dtypes.string))) + class SparseCrossHashedOpTest(BaseSparseCrossOpTest): diff --git a/tensorflow/python/kernel_tests/sparse_ops_test.py b/tensorflow/python/kernel_tests/sparse_ops_test.py index 1f89806fab0d8d..ae317ec1292aa4 100644 --- a/tensorflow/python/kernel_tests/sparse_ops_test.py +++ b/tensorflow/python/kernel_tests/sparse_ops_test.py @@ -669,7 +669,7 @@ def testInvalidIndices(self): class SparseAddTest(test_util.TensorFlowTestCase): def testValuesInVariable(self): - indices = constant_op.constant([[1]], dtype=dtypes.int64) + indices = constant_op.constant([[0]], dtype=dtypes.int64) values = variables.Variable([1], trainable=False, dtype=dtypes.float32) shape = constant_op.constant([1], dtype=dtypes.int64) @@ -849,6 +849,39 @@ def disabledtestSparseReduceSumOrMaxShape(self): self._testSparseReduceShape(sp_t, [-1], 2, keep_dims, do_sum) self._testSparseReduceShape(sp_t, [1, -2], 2, keep_dims, do_sum) + def testIntegerOverflow(self): + with self.cached_session(use_gpu=False): + with self.assertRaises(errors.InvalidArgumentError): + res = sparse_ops.gen_sparse_ops.sparse_reduce_max( + input_indices=[[1, 2], [3, 4]], + input_shape=[2**32, 2**31], + input_values=[1, 3], + reduction_axes=[0], + keep_dims=False, + name=None) + + self.evaluate(res) + with self.assertRaises(errors.InvalidArgumentError): + res = sparse_ops.gen_sparse_ops.sparse_reduce_max_sparse( + input_indices=[[1, 2], [3, 4]], + input_shape=[2**32, 2**31], + input_values=[1, 3], + reduction_axes=[0], + keep_dims=False, + name=None) + + self.evaluate(res) + with self.assertRaises(errors.InvalidArgumentError): + res = sparse_ops.gen_sparse_ops.sparse_reduce_sum( + input_indices=[[1, 2], [3, 4]], + input_shape=[2**32, 2**31], + input_values=[1, 3], + reduction_axes=[0], + keep_dims=False, + name=None) + + self.evaluate(res) + class SparseMathOpsTest(test_util.TensorFlowTestCase): @@ -1014,6 +1047,25 @@ def testGradient(self): y_tf.values, (nnz,)) self.assertLess(err, 1e-4) + def testIntegerOverflow(self): + with self.cached_session(use_gpu=False): + with self.assertRaises(errors.InvalidArgumentError): + res = sparse_ops.gen_sparse_ops.sparse_softmax( + sp_indices=[[1, 1]], + sp_values=[2.0], + sp_shape=[2**32, 2**31], + name=None) + + self.evaluate(res) + + def testReshapeNegativeShape(self): + with self.cached_session(use_gpu=False): + with self.assertRaises(errors.InvalidArgumentError): + res = sparse_ops.gen_sparse_ops.sparse_softmax( + sp_indices=[[1, 1]], sp_values=[2.0], sp_shape=[-1, 1], name=None) + + self.evaluate(res) + class SparseMinimumMaximumTest(test_util.TensorFlowTestCase): diff --git a/tensorflow/python/kernel_tests/sparse_reshape_op_test.py b/tensorflow/python/kernel_tests/sparse_reshape_op_test.py index ab98c9a3deb718..146549859a7bea 100644 --- a/tensorflow/python/kernel_tests/sparse_reshape_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_reshape_op_test.py @@ -103,6 +103,28 @@ def testSameShape(self): self.assertAllEqual(output_val.values, input_val.values) self.assertAllEqual(output_val.dense_shape, input_val.dense_shape) + def testReshapeIntegeroverflow(self): + with self.session(): + with self.assertRaises(errors.InvalidArgumentError): + sp_output = sparse_ops.gen_sparse_ops.sparse_reshape( + input_indices=[[0, 0]], + input_shape=[2**32, 2**31], + new_shape=[1, 1], + name=None) + + self.evaluate(sp_output) + + def testReshapeNegativeShape(self): + with self.session(): + with self.assertRaises(errors.InvalidArgumentError): + sp_output = sparse_ops.gen_sparse_ops.sparse_reshape( + input_indices=[[0, 0]], + input_shape=[1, -1], + new_shape=[1, 1], + name=None) + + self.evaluate(sp_output) + @test_util.run_deprecated_v1 def testFeedSameShape(self): with self.session() as sess: diff --git a/tensorflow/python/kernel_tests/sparse_slice_op_test.py b/tensorflow/python/kernel_tests/sparse_slice_op_test.py index b24a6a3d4dcfe2..749a87afc8b6ae 100644 --- a/tensorflow/python/kernel_tests/sparse_slice_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_slice_op_test.py @@ -20,6 +20,7 @@ import numpy as np +from tensorflow.python.framework import errors from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker @@ -286,6 +287,27 @@ def testGradients(self): [sp_input.values], [(nnz_in,)], sp_output.values, (nnz_out,)) self.assertLess(err, 1e-3) + def testNegativeSize(self): + with self.session(use_gpu=False): + with self.assertRaises(errors.InvalidArgumentError): + res = sparse_ops.gen_sparse_ops.sparse_slice( + indices=[[0, 0]], + values=[0], + shape=[1, 1], + start=[10, 10], + size=[-100, 100]) + self.evaluate(res) + + def testLargeSize(self): + with self.session(use_gpu=False): + # Confirm potential integer overflow due to size is handled by op. + res = sparse_ops.gen_sparse_ops.sparse_slice( + indices=[[0, 0]], + values=[0], + shape=[1, 1], + start=[2**62, -1], + size=[2**62, 2**62]) + self.evaluate(res) if __name__ == '__main__': test.main() diff --git a/tensorflow/python/kernel_tests/sparse_split_op_test.py b/tensorflow/python/kernel_tests/sparse_split_op_test.py index 31ef1129f1319c..b5cc3f02d9d4c7 100644 --- a/tensorflow/python/kernel_tests/sparse_split_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_split_op_test.py @@ -257,6 +257,15 @@ def testArgumentErrors(self): with self.assertRaisesRegex(ValueError, 'axis is required'): sparse_ops.sparse_split(num_split=2, sp_input=1) + def testInvalidArgumentError(self): + # Test case for GitHub issue 53660. + axis = [1, 2] + with self.assertRaisesRegexp(errors.InvalidArgumentError, + r'axis should be a scalar'): + self.evaluate( + sparse_ops.sparse_split( + sp_input=self._SparseTensor_4x6(), num_split=3, axis=axis)) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/kernel_tests/stage_op_test.py b/tensorflow/python/kernel_tests/stage_op_test.py index 8ea4c5daa2ee27..dd3a46f36168d5 100644 --- a/tensorflow/python/kernel_tests/stage_op_test.py +++ b/tensorflow/python/kernel_tests/stage_op_test.py @@ -17,6 +17,7 @@ from __future__ import print_function from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops @@ -138,6 +139,16 @@ def testPeek(self): for i in range(10): self.assertTrue(sess.run(peek, feed_dict={p: i}) == [i]) + def testPeekBadIndex(self): + stager = data_flow_ops.StagingArea([ + dtypes.int32, + ], shapes=[[10]]) + stager.put([array_ops.zeros([10], dtype=dtypes.int32)]) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + 'must be scalar'): + self.evaluate(stager.peek([])) + @test_util.run_deprecated_v1 def testSizeAndClear(self): with ops.Graph().as_default() as G: diff --git a/tensorflow/python/kernel_tests/summary_ops_test.py b/tensorflow/python/kernel_tests/summary_ops_test.py index 2db2dcd3485980..61305c40316e24 100644 --- a/tensorflow/python/kernel_tests/summary_ops_test.py +++ b/tensorflow/python/kernel_tests/summary_ops_test.py @@ -980,10 +980,12 @@ def testFlushFunction(self): self.assertEqual(3, get_total()) summary_ops.flush(writer=writer) self.assertEqual(4, get_total()) - summary_ops.write('tag', 1, step=0) - self.assertEqual(4, get_total()) - summary_ops.flush(writer=writer._resource) # pylint:disable=protected-access - self.assertEqual(5, get_total()) + + # Regression test for b/228097117. + def testFlushFunction_disallowsInvalidWriterInput(self): + with context.eager_mode(): + with self.assertRaisesRegex(ValueError, 'Invalid argument to flush'): + summary_ops.flush(writer=()) @test_util.assert_no_new_tensors def testNoMemoryLeak_graphMode(self): diff --git a/tensorflow/python/kernel_tests/svd_op_test.py b/tensorflow/python/kernel_tests/svd_op_test.py index 017357f9545414..3770a5b6d35a03 100644 --- a/tensorflow/python/kernel_tests/svd_op_test.py +++ b/tensorflow/python/kernel_tests/svd_op_test.py @@ -112,6 +112,14 @@ def testExecuteMultipleWithoutError(self): for i in range(0, len(val), 2): self.assertAllEqual(val[i], val[i + 1]) + @test_util.run_in_graph_and_eager_modes(use_gpu=True) + def testEmptyBatches(self): + matrices = constant_op.constant(1.0, shape=[0, 2, 2]) + s, u, v = self.evaluate(linalg_ops.svd(matrices)) + self.assertAllEqual(s, np.zeros([0, 2])) + self.assertAllEqual(u, np.zeros([0, 2, 2])) + self.assertAllEqual(v, np.zeros([0, 2, 2])) + def _GetSvdOpTest(dtype_, shape_, use_static_shape_, compute_uv_, full_matrices_): diff --git a/tensorflow/python/ops/batch_ops_test.py b/tensorflow/python/ops/batch_ops_test.py index c29a30600c549d..8d1ac5bb29d662 100644 --- a/tensorflow/python/ops/batch_ops_test.py +++ b/tensorflow/python/ops/batch_ops_test.py @@ -24,7 +24,9 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import function from tensorflow.python.framework import ops from tensorflow.python.framework import test_util @@ -34,6 +36,7 @@ from tensorflow.python.ops import gen_batch_ops from tensorflow.python.ops import gen_functional_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import random_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import script_ops from tensorflow.python.ops import variables @@ -237,6 +240,26 @@ def worker(): self.assertEqual(thread_results[0], [2]) self.assertEqual(main_results[0], [3]) + def testUnbatchInvalidIdArg(self): + """Tests that unbatch work together.""" + if context.executing_eagerly(): + batched_tensor = constant_op.constant( + value=np.random.random(size=(3, 3, 1)), dtype=dtypes.float64) + batched_index = constant_op.constant( + value=np.random.randint(0, 100, size=(3, 3, 1)), dtype=dtypes.int64) + arg_id = constant_op.constant( + value=np.random.randint(0, 100, size=(3, 3, 1)), dtype=dtypes.int64) + + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Input id should be scalar;"): + batch_ops.unbatch( + batched_tensor=batched_tensor, + batch_index=batched_index, + id=arg_id, + timeout_micros=50, + container="", + shared_name="") + def testBatchDecoratedWithCapturedInput(self): """Tests that the batch_function decorator works.""" if context.executing_eagerly(): @@ -561,6 +584,56 @@ def worker(): # The thread's call should hit the timeout, and thus get 0 results. self.assertEqual(len(thread_results), 0) + def testUnbatchGradInvalidId(self): + with self.assertRaises(errors.InvalidArgumentError): + self.evaluate( + gen_batch_ops.unbatch_grad( + original_input=constant_op.constant([1]), + batch_index=constant_op.constant([ + [0, 0, 0], + ], dtype=dtypes.int64), + grad=constant_op.constant([ + 1, + ]), + id=constant_op.constant([ + 1, + 1, + ], dtype=dtypes.int64))) + + def testUnbatchGradInvalidBatchId(self): + with self.assertRaises(errors.InvalidArgumentError): + self.evaluate( + gen_batch_ops.unbatch_grad( + original_input=constant_op.constant([1]), + batch_index=constant_op.constant([ + [0, 0], + ], dtype=dtypes.int64), + grad=constant_op.constant([ + 1, + ]), + id=constant_op.constant([ + 1, + ], dtype=dtypes.int64))) + + def testUnbatchGradInvalidArgs(self): + original_input = random_ops.random_uniform( + shape=(3, 1), dtype=dtypes.float64, maxval=None) + batch_index = random_ops.random_uniform( + shape=(3, 1), dtype=dtypes.int64, maxval=65536) + grad = random_ops.random_uniform( + shape=(3, 1), dtype=dtypes.float64, maxval=None) + batch_id = random_ops.random_uniform( + shape=(3, 1), dtype=dtypes.int64, maxval=65536) + with self.assertRaises(errors.InvalidArgumentError): + self.evaluate( + gen_batch_ops.unbatch_grad( + original_input=original_input, + batch_index=batch_index, + grad=grad, + id=batch_id, + container="", + shared_name="", + name="")) if __name__ == "__main__": test.main() diff --git a/tensorflow/python/ops/collective_ops_test.py b/tensorflow/python/ops/collective_ops_test.py index 5a49c833e92063..85c21ed7636a63 100644 --- a/tensorflow/python/ops/collective_ops_test.py +++ b/tensorflow/python/ops/collective_ops_test.py @@ -455,6 +455,20 @@ def testCollectiveGroupSizeMismatch(self): ]) context.ensure_initialized() + @test_util.run_v2_only + def testCollectiveGatherShapeCheckFailure(self): + with self.assertRaisesRegex(errors.InvalidArgumentError, + 'input should have rank > 0'): + collective_ops.gen_collective_ops.CollectiveGather( + input=1, + group_size=1, + group_key=1, + instance_key=1, + shape=(3, 3, 3), + communication_hint='auto', + timeout_seconds=0, + name='') + @def_function.function def run_all_reduce(): group_key = 10 diff --git a/tensorflow/python/ops/data_flow_ops.py b/tensorflow/python/ops/data_flow_ops.py index 31bc1e393b23ce..a1d1227c42ddcf 100644 --- a/tensorflow/python/ops/data_flow_ops.py +++ b/tensorflow/python/ops/data_flow_ops.py @@ -1741,7 +1741,7 @@ def _check_put_dtypes(self, vals, indices=None): # Sanity check number of values if not len(vals) <= len(self._dtypes): - raise ValueError(f"Unexpected number of inputs {len(vals)} vs" + raise ValueError(f"Unexpected number of inputs {len(vals)} vs " f"{len(self._dtypes)}") tensors = [] diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py index 5744a9acda9431..67d2c24a9b6ca1 100644 --- a/tensorflow/python/ops/image_ops_test.py +++ b/tensorflow/python/ops/image_ops_test.py @@ -2299,7 +2299,7 @@ def testInvalidInput(self): # TODO(b/200850176): test fails with XLA. return with self.session(): - with self.assertRaises(errors_impl.InternalError): + with self.assertRaises(errors_impl.InvalidArgumentError): v = image_ops.pad_to_bounding_box( image=np.ones((1, 1, 1)), target_height=5191549470, @@ -3217,7 +3217,7 @@ def testPreserveAspectRatioSquare(self): def testLargeDim(self): with self.session(): - with self.assertRaises(errors.InternalError): + with self.assertRaises(errors.InvalidArgumentError): x = np.ones((5, 1, 1, 2)) v = image_ops.resize_images_v2(x, [1610637938, 1610637938], image_ops.ResizeMethod.BILINEAR) @@ -6092,7 +6092,7 @@ def testImageCropAndResize(self): def testImageCropAndResizeWithInvalidInput(self): with self.session(): - with self.assertRaises((errors.InternalError, ValueError)): + with self.assertRaises((errors.InvalidArgumentError, ValueError)): op = image_ops_impl.crop_and_resize_v2( image=np.ones((1, 1, 1, 1)), boxes=np.ones((11, 4)), diff --git a/tensorflow/python/ops/parallel_for/BUILD b/tensorflow/python/ops/parallel_for/BUILD index 66455755019e01..2ef92c5bd9882b 100644 --- a/tensorflow/python/ops/parallel_for/BUILD +++ b/tensorflow/python/ops/parallel_for/BUILD @@ -161,6 +161,7 @@ cuda_py_test( name = "array_test", srcs = ["array_test.py"], tags = [ + "no_oss", "notsan", # TODO(b/170999669): Data race ], deps = [ @@ -178,7 +179,10 @@ cuda_py_test( name = "math_test", srcs = ["math_test.py"], shard_count = 5, - tags = ["optonly"], # Too slow in non-opt mode + tags = [ + "no_oss", + "optonly", # Too slow in non-opt mode + ], deps = [ ":control_flow_ops", ":test_util", diff --git a/tensorflow/python/ops/quantized_conv_ops_test.py b/tensorflow/python/ops/quantized_conv_ops_test.py index 6b469a954f6531..dbf352618f13cd 100644 --- a/tensorflow/python/ops/quantized_conv_ops_test.py +++ b/tensorflow/python/ops/quantized_conv_ops_test.py @@ -22,6 +22,8 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors +from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn_ops from tensorflow.python.platform import test @@ -200,6 +202,71 @@ def testConv2D2x2FilterStride2Same(self): padding="SAME", expected=expected_output) + def _testBadInputSize(self, + tin=None, + tfilter=None, + min_input=None, + max_input=None, + min_filter=None, + max_filter=None, + error_regex=""): + strides = [1, 1, 1, 1] + padding = "SAME" + if tin is None: + tin = math_ops.cast( + constant_op.constant(1, shape=[1, 2, 3, 3]), dtype=dtypes.quint8) + + if tfilter is None: + tfilter = math_ops.cast( + constant_op.constant(1, shape=[1, 2, 3, 3]), dtype=dtypes.quint8) + + if min_input is None: + min_input = constant_op.constant(0, shape=[], dtype=dtypes.float32) + + if max_input is None: + max_input = constant_op.constant(0, shape=[], dtype=dtypes.float32) + + if min_filter is None: + min_filter = constant_op.constant(0, shape=[], dtype=dtypes.float32) + + if max_filter is None: + max_filter = constant_op.constant(0, shape=[], dtype=dtypes.float32) + + with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + error_regex): + self.evaluate( + nn_ops.quantized_conv2d( + tin, + tfilter, + out_type=dtypes.qint32, + strides=strides, + padding=padding, + min_input=min_input, + max_input=max_input, + min_filter=min_filter, + max_filter=max_filter)) + + def testBadInputSizes(self): + self._testBadInputSize( + tin=math_ops.cast( + constant_op.constant(1, shape=[1, 2]), dtype=dtypes.quint8), + error_regex="must be rank 4") + self._testBadInputSize( + tfilter=math_ops.cast( + constant_op.constant(1, shape=[1, 2]), dtype=dtypes.quint8), + error_regex="must be rank 4") + self._testBadInputSize( + min_input=constant_op.constant(0, shape=[1], dtype=dtypes.float32), + error_regex="must be rank 0") + self._testBadInputSize( + max_input=constant_op.constant(0, shape=[1], dtype=dtypes.float32), + error_regex="must be rank 0") + self._testBadInputSize( + min_filter=constant_op.constant(0, shape=[1], dtype=dtypes.float32), + error_regex="must be rank 0") + self._testBadInputSize( + max_filter=constant_op.constant(0, shape=[1], dtype=dtypes.float32), + error_regex="must be rank 0") if __name__ == "__main__": test.main() diff --git a/tensorflow/python/ops/ragged/ragged_factory_ops.py b/tensorflow/python/ops/ragged/ragged_factory_ops.py index 0296a185041a50..50f6a4771b126a 100644 --- a/tensorflow/python/ops/ragged/ragged_factory_ops.py +++ b/tensorflow/python/ops/ragged/ragged_factory_ops.py @@ -192,6 +192,9 @@ def _constant_value(ragged_factory, inner_factory, pylist, dtype, ragged_rank, if max_depth > scalar_depth: raise ValueError("Invalid pylist=%r: empty list nesting is greater " "than scalar value nesting" % pylist) + if ragged_rank is not None and max_depth < ragged_rank: + raise ValueError(f"Invalid pylist={pylist}, max depth smaller than " + f"ragged_rank={ragged_rank}") # If both inner_shape and ragged_rank were specified, then check that # they are compatible with pylist. diff --git a/tensorflow/python/ops/ragged/ragged_range_op_test.py b/tensorflow/python/ops/ragged/ragged_range_op_test.py index b655fd1ea84339..ba185f688e7511 100644 --- a/tensorflow/python/ops/ragged/ragged_range_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_range_op_test.py @@ -88,8 +88,7 @@ def testBroadcast(self): list(range(5, 15, 3))]) # Broadcast all arguments. - self.assertAllEqual( - ragged_math_ops.range(0, 5, 1), [list(range(0, 5, 1))]) + self.assertAllEqual(ragged_math_ops.range(0, 5, 1), [list(range(0, 5, 1))]) def testEmptyRanges(self): rt1 = ragged_math_ops.range([0, 5, 3], [0, 3, 5]) @@ -112,6 +111,10 @@ def testKernelErrors(self): r'Requires delta != 0'): self.evaluate(ragged_math_ops.range(0, 0, 0)) + with self.assertRaisesRegex(errors.InvalidArgumentError, + r'Requires \(\(limit - start\) / delta\) <='): + self.evaluate(ragged_math_ops.range(0.1, 1e10, 1e-10)) + def testShape(self): self.assertAllEqual( ragged_math_ops.range(0, 0, 1).shape.as_list(), [1, None]) diff --git a/tensorflow/python/ops/ragged/ragged_tensor_test.py b/tensorflow/python/ops/ragged/ragged_tensor_test.py index 688875a079740b..aec9c2b407ab64 100644 --- a/tensorflow/python/ops/ragged/ragged_tensor_test.py +++ b/tensorflow/python/ops/ragged/ragged_tensor_test.py @@ -1446,6 +1446,21 @@ def testUnbatchVariantInDataset(self): for i in range(3): self.assertAllEqual(sess.run(rt[i]), out) + def testToVariantInvalidParams(self): + self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + r'be rank 1 but is rank 0', + gen_ragged_conversion_ops.ragged_tensor_to_variant, + rt_nested_splits=[0, 1, 2], + rt_dense_values=[0, 1, 2], + batched_input=True) + + self.assertRaisesRegex((ValueError, errors.InvalidArgumentError), + r'be rank 1 but is rank 2', + gen_ragged_conversion_ops.ragged_tensor_to_variant, + rt_nested_splits=[[[0]], [[1]], [[2]]], + rt_dense_values=[0, 1, 2], + batched_input=True) + def testFromVariantInvalidParams(self): rt = ragged_factory_ops.constant([[0], [1], [2], [3]]) batched_variant = rt._to_variant(batched_input=True) diff --git a/tensorflow/python/ops/raw_ops_test.py b/tensorflow/python/ops/raw_ops_test.py index 6706ef194b221a..8891edba481fee 100644 --- a/tensorflow/python/ops/raw_ops_test.py +++ b/tensorflow/python/ops/raw_ops_test.py @@ -32,7 +32,6 @@ @test_util.run_all_in_graph_and_eager_modes -@test_util.disable_tfrt class RawOpsTest(test.TestCase, parameterized.TestCase): def testSimple(self): @@ -67,8 +66,9 @@ def testDefaults(self): @parameterized.parameters([[0, 8]], [[-1, 6]]) def testStringNGramsBadDataSplits(self, splits): data = ["aa", "bb", "cc", "dd", "ee", "ff"] - with self.assertRaisesRegex(errors.InvalidArgumentError, - "Invalid split value"): + with self.assertRaisesRegex( + errors.InvalidArgumentError, + r"Invalid split value|First split value must be 0"): self.evaluate( gen_string_ops.string_n_grams( data=data, @@ -80,6 +80,37 @@ def testStringNGramsBadDataSplits(self, splits): pad_width=0, preserve_short_sequences=False)) + def testStringSplit(self): + data = ["123456"] + data_splits = [0, 1] + separator = "a" * 15 + ngram_widths = [] + pad_width = -5 + left_pad = right_pad = "" + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Pad width should be >= 0"): + self.evaluate(gen_string_ops.string_n_grams( + data=data, + data_splits=data_splits, + separator=separator, + ngram_widths=ngram_widths, + left_pad=left_pad, + right_pad=right_pad, + pad_width=pad_width, + preserve_short_sequences=True)) + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Pad width could lead to integer overflow"): + self.evaluate( + gen_string_ops.string_n_grams( + data=["000.0", "000.0"], + data_splits=[0, 2], + separator="", + ngram_widths=[2**30, 2**30], + left_pad=" ", + right_pad=" ", + pad_width=-2**30, + preserve_short_sequences=False)) + def testGetSessionHandle(self): if context.executing_eagerly(): with self.assertRaisesRegex( diff --git a/tensorflow/python/ops/summary_ops_v2.py b/tensorflow/python/ops/summary_ops_v2.py index 653676a4b2a7ea..c38b5edffbba12 100644 --- a/tensorflow/python/ops/summary_ops_v2.py +++ b/tensorflow/python/ops/summary_ops_v2.py @@ -1113,12 +1113,35 @@ def flush(writer=None, name=None): Returns: The created `tf.Operation`. """ + del name # unused if writer is None: writer = _summary_state.writer if writer is None: return control_flow_ops.no_op() if isinstance(writer, SummaryWriter): return writer.flush() + raise ValueError("Invalid argument to flush(): %r" % (writer,)) + + +def legacy_raw_flush(writer=None, name=None): + """Legacy version of flush() that accepts a raw resource tensor for `writer`. + + Do not use this function in any new code. Not supported and not part of the + public TF APIs. + + Args: + writer: The `tf.summary.SummaryWriter` to flush. If None, the current + default writer will be used instead; if there is no current writer, this + returns `tf.no_op`. For this legacy version only, also accepts a raw + resource tensor pointing to the underlying C++ writer resource. + name: Ignored legacy argument for a name for the operation. + + Returns: + The created `tf.Operation`. + """ + if writer is None or isinstance(writer, SummaryWriter): + # Forward to the TF2 implementation of flush() when possible. + return flush(writer, name) else: # Legacy fallback in case we were passed a raw resource tensor. with ops.device("cpu:0"): diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py index 3368252fd3109b..249447163b7031 100644 --- a/tensorflow/python/saved_model/load_test.py +++ b/tensorflow/python/saved_model/load_test.py @@ -212,8 +212,8 @@ def test_control_outputs(self, cycles): imported_graph.control_outputs) def _make_asset(self, contents): - filename = tempfile.mktemp(prefix=self.get_temp_dir()) - with open(filename, "w") as f: + fd, filename = tempfile.mkstemp(prefix=self.get_temp_dir()) + with os.fdopen(fd, "w") as f: f.write(contents) return filename @@ -2501,8 +2501,8 @@ def layer_variable_creator(next_creator, **kwargs): load_and_run_module(export_dir, weight_size) def _make_asset(self, contents): - filename = tempfile.mktemp(prefix=self.get_temp_dir()) - with open(filename, "w") as f: + fd, filename = tempfile.mkstemp(prefix=self.get_temp_dir()) + with os.fdopen(fd, "w") as f: f.write(contents) return filename diff --git a/tensorflow/python/saved_model/load_v1_in_v2.py b/tensorflow/python/saved_model/load_v1_in_v2.py index 26a26634acdd44..5fce1aaff6838f 100644 --- a/tensorflow/python/saved_model/load_v1_in_v2.py +++ b/tensorflow/python/saved_model/load_v1_in_v2.py @@ -142,7 +142,7 @@ def _extract_signatures(self, wrapped, meta_graph_def): for signature_key, signature_def in meta_graph_def.signature_def.items(): if signature_def.inputs: input_items = sorted( - signature_def.inputs.items(), key=lambda item: item[1].name) + signature_def.inputs.items(), key=lambda item: item[0]) original_input_names, input_specs = zip(*input_items) else: original_input_names = [] diff --git a/tensorflow/python/saved_model/load_v1_in_v2_test.py b/tensorflow/python/saved_model/load_v1_in_v2_test.py index b854e588f71c15..22e212cffa4bd8 100644 --- a/tensorflow/python/saved_model/load_v1_in_v2_test.py +++ b/tensorflow/python/saved_model/load_v1_in_v2_test.py @@ -696,6 +696,33 @@ def test_v1_input_ordered(self): self.assertEqual(imported.signatures["serving_default"].inputs[1].name, "input2:0") + def test_resave_signature(self): + # Tests that signatures saved using TF1 can be resaved with TF2. + # See b/211666001 for context. + export_graph = ops.Graph() + with export_graph.as_default(): + a = array_ops.placeholder( + shape=[None, 1], dtype=dtypes.float32, name="input_2") + b = array_ops.placeholder( + shape=[None, 2], dtype=dtypes.float32, name="input_1") + c = array_ops.identity(a) + with session_lib.Session() as session: + path = os.path.join(self.get_temp_dir(), "saved_model", str(ops.uid())) + simple_save.simple_save( + session, + path, + inputs={"a": a, "b": b}, + outputs={"c": c}) + imported = load.load(path) + path2 = os.path.join(self.get_temp_dir(), "saved_model", str(ops.uid())) + save.save(imported, path2, imported.signatures) + + imported2 = load.load(path2) + self.assertEqual( + imported2.signatures["serving_default"]( + a=constant_op.constant([5.]), + b=constant_op.constant([1., 3.]))["c"].numpy(), 5.) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/summary/summary_test.py b/tensorflow/python/summary/summary_test.py index 6dcafed721d568..86dfe2f014faef 100644 --- a/tensorflow/python/summary/summary_test.py +++ b/tensorflow/python/summary/summary_test.py @@ -28,6 +28,7 @@ from tensorflow.core.framework import summary_pb2 from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops from tensorflow.python.framework import test_util @@ -188,6 +189,11 @@ def testAudioSummaryWithFamily(self): for i in xrange(3)) self.assertEqual(tags, expected) + def testAudioSummaryWithInvalidSampleRate(self): + with self.assertRaises(errors.InvalidArgumentError): + invalid_sample_rate = [22000.0, 22000.0] + self.evaluate(summary_lib.audio('', [[1.0]], invalid_sample_rate)) + @test_util.run_deprecated_v1 def testTextSummary(self): with self.cached_session(): diff --git a/tensorflow/python/tools/saved_model_cli.py b/tensorflow/python/tools/saved_model_cli.py index ba6210c5595728..8a55f79f465fa8 100644 --- a/tensorflow/python/tools/saved_model_cli.py +++ b/tensorflow/python/tools/saved_model_cli.py @@ -673,7 +673,7 @@ def load_inputs_from_input_arg_string(inputs_str, input_exprs_str, tensor_key_feed_dict = {} inputs = preprocess_inputs_arg_string(inputs_str) - input_exprs = preprocess_input_exprs_arg_string(input_exprs_str, safe=False) + input_exprs = preprocess_input_exprs_arg_string(input_exprs_str) input_examples = preprocess_input_examples_arg_string(input_examples_str) for input_tensor_key, (filename, variable_name) in inputs.items(): diff --git a/tensorflow/python/tools/saved_model_cli_test.py b/tensorflow/python/tools/saved_model_cli_test.py index c17d011ac46ef6..482dcebc35a125 100644 --- a/tensorflow/python/tools/saved_model_cli_test.py +++ b/tensorflow/python/tools/saved_model_cli_test.py @@ -490,43 +490,6 @@ def testInputParserPickle(self): self.assertTrue(np.all(feed_dict['y'] == pkl1)) self.assertTrue(np.all(feed_dict['z'] == pkl2)) - def testInputParserPythonExpression(self): - x1 = np.ones([2, 10]) - x2 = np.array([[1], [2], [3]]) - x3 = np.mgrid[0:5, 0:5] - x4 = [[3], [4]] - input_expr_str = ('x1=np.ones([2,10]);x2=np.array([[1],[2],[3]]);' - 'x3=np.mgrid[0:5,0:5];x4=[[3],[4]]') - feed_dict = saved_model_cli.load_inputs_from_input_arg_string( - '', input_expr_str, '') - self.assertTrue(np.all(feed_dict['x1'] == x1)) - self.assertTrue(np.all(feed_dict['x2'] == x2)) - self.assertTrue(np.all(feed_dict['x3'] == x3)) - self.assertTrue(np.all(feed_dict['x4'] == x4)) - - def testInputParserBoth(self): - x0 = np.array([[1], [2]]) - input_path = os.path.join(test.get_temp_dir(), 'input.npz') - np.savez(input_path, a=x0) - x1 = np.ones([2, 10]) - input_str = 'x0=' + input_path + '[a]' - input_expr_str = 'x1=np.ones([2,10])' - feed_dict = saved_model_cli.load_inputs_from_input_arg_string( - input_str, input_expr_str, '') - self.assertTrue(np.all(feed_dict['x0'] == x0)) - self.assertTrue(np.all(feed_dict['x1'] == x1)) - - def testInputParserBothDuplicate(self): - x0 = np.array([[1], [2]]) - input_path = os.path.join(test.get_temp_dir(), 'input.npz') - np.savez(input_path, a=x0) - x1 = np.ones([2, 10]) - input_str = 'x0=' + input_path + '[a]' - input_expr_str = 'x0=np.ones([2,10])' - feed_dict = saved_model_cli.load_inputs_from_input_arg_string( - input_str, input_expr_str, '') - self.assertTrue(np.all(feed_dict['x0'] == x1)) - def testInputParserErrorNoName(self): x0 = np.array([[1], [2]]) x1 = np.array(range(5)) @@ -627,7 +590,7 @@ def testRunCommandInvalidInputKeyError(self): base_path = test.test_src_dir_path(SAVED_MODEL_PATH) args = self.parser.parse_args([ 'run', '--dir', base_path, '--tag_set', 'serve', '--signature_def', - 'regress_x2_to_y3', '--input_exprs', 'x2=np.ones((3,1))' + 'regress_x2_to_y3', '--input_exprs', 'x2=[1,2,3]' ]) with self.assertRaises(ValueError): saved_model_cli.run(args) @@ -637,7 +600,7 @@ def testRunCommandInvalidSignature(self): base_path = test.test_src_dir_path(SAVED_MODEL_PATH) args = self.parser.parse_args([ 'run', '--dir', base_path, '--tag_set', 'serve', '--signature_def', - 'INVALID_SIGNATURE', '--input_exprs', 'x2=np.ones((3,1))' + 'INVALID_SIGNATURE', '--input_exprs', 'x2=[1,2,3]' ]) with self.assertRaisesRegex(ValueError, 'Could not find signature "INVALID_SIGNATURE"'): diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index c72a0d414bd8c8..5200be9efd3291 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -55,7 +55,7 @@ def register_extension_info(**kwargs): # not contain rc or alpha, only numbers. # Also update tensorflow/core/public/version.h # and tensorflow/tools/pip_package/setup.py -VERSION = "2.7.0" +VERSION = "2.7.4" VERSION_MAJOR = VERSION.split(".")[0] two_gpu_tags = ["requires-gpu-nvidia:2", "notap", "manual", "no_pip"] diff --git a/tensorflow/tools/ci_build/builds/pip_new.sh b/tensorflow/tools/ci_build/builds/pip_new.sh index 27208591d50e94..d41d7d6786e737 100755 --- a/tensorflow/tools/ci_build/builds/pip_new.sh +++ b/tensorflow/tools/ci_build/builds/pip_new.sh @@ -255,6 +255,9 @@ PIP_TEST_ROOT=${TF_PIP_TEST_ROOT:-$DEFAULT_PIP_TEST_ROOT} BUILD_BOTH_GPU_PACKAGES=${TF_BUILD_BOTH_GPU_PACKAGES:-$DEFAULT_BUILD_BOTH_GPU_PACKAGES} BUILD_BOTH_CPU_PACKAGES=${TF_BUILD_BOTH_CPU_PACKAGES:-$DEFAULT_BUILD_BOTH_CPU_PACKAGES} +# Override breaking change in setuptools v60 (https://github.com/pypa/setuptools/pull/2896) +export SETUPTOOLS_USE_DISTUTILS=stdlib + # Local variables PIP_WHL_DIR="${KOKORO_ARTIFACTS_DIR}/tensorflow/${PIP_TEST_ROOT}/whl" mkdir -p "${PIP_WHL_DIR}" diff --git a/tensorflow/tools/ci_build/release/requirements_common.txt b/tensorflow/tools/ci_build/release/requirements_common.txt index 6118a8f5066f50..37705231827591 100644 --- a/tensorflow/tools/ci_build/release/requirements_common.txt +++ b/tensorflow/tools/ci_build/release/requirements_common.txt @@ -10,7 +10,7 @@ h5py ~= 3.1.0 # NOTE: not the latest version due to py3.6 keras_preprocessing ~= 1.1.2 numpy ~= 1.19.5 # NOTE: not the latest version due to py3.6 opt_einsum ~= 3.3.0 -protobuf >= 3.17.1 +protobuf ~= 3.19.3 # NOTE: Earliest version for Python 3.10 six ~= 1.16.0 termcolor ~= 1.1.0 typing_extensions ~= 3.10.0.0 diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 26306e43aaf4d7..6982db212aeb52 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -50,7 +50,7 @@ # result for pip. # Also update tensorflow/tensorflow.bzl and # tensorflow/core/public/version.h -_VERSION = '2.7.0' +_VERSION = '2.7.4' # We use the same setup.py for all tensorflow_* packages and for the nightly @@ -88,7 +88,14 @@ 'h5py >= 2.9.0', # capped since 3.3.0 lacks py3.6 'keras_preprocessing >= 1.1.1', # 1.1.0 needs tensorflow==1.7 'opt_einsum >= 2.3.2', # sphinx pin not removed up til 3.3.0 release - 'protobuf >= 3.9.2', + # TODO(b/182876485): Protobuf 3.20 results in linker errors on Windows + # Protobuf 4.0 is binary incompatible with what C++ TF uses. + # We need ~1 quarter to update properly. + # See also: https://github.com/tensorflow/tensorflow/issues/53234 + # See also: https://github.com/protocolbuffers/protobuf/issues/9954 + # See also: https://github.com/tensorflow/tensorflow/issues/56077 + # This is a temporary patch for now, to patch previous TF releases. + 'protobuf >= 3.9.2, < 3.20', 'six >= 1.12.0', 'termcolor >= 1.1.0', 'typing_extensions >= 3.6.6', diff --git a/tensorflow/workspace2.bzl b/tensorflow/workspace2.bzl index eaa2863cfaa819..d2f37131a979a6 100644 --- a/tensorflow/workspace2.bzl +++ b/tensorflow/workspace2.bzl @@ -354,12 +354,12 @@ def _tf_repositories(): tf_http_archive( name = "org_sqlite", build_file = "//third_party:sqlite.BUILD", - sha256 = "999826fe4c871f18919fdb8ed7ec9dd8217180854dd1fe21eea96aed36186729", - strip_prefix = "sqlite-amalgamation-3360000", + sha256 = "87775784f8b22d0d0f1d7811870d39feaa7896319c7c20b849a4181c5a50609b", + strip_prefix = "sqlite-amalgamation-3390200", system_build_file = "//third_party/systemlibs:sqlite.BUILD", urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/sqlite.org/2021/sqlite-amalgamation-3360000.zip", - "https://www.sqlite.org/2021/sqlite-amalgamation-3360000.zip", + "https://storage.googleapis.com/mirror.tensorflow.org/sqlite.org/2022/sqlite-amalgamation-3390200.zip", + "https://www.sqlite.org/2022/sqlite-amalgamation-3390200.zip", ], ) @@ -608,12 +608,12 @@ def _tf_repositories(): tf_http_archive( name = "curl", build_file = "//third_party:curl.BUILD", - sha256 = "ed936c0b02c06d42cf84b39dd12bb14b62d77c7c4e875ade022280df5dcc81d7", - strip_prefix = "curl-7.78.0", + sha256 = "3c6893d38d054d4e378267166858698899e9d87258e8ff1419d020c395384535", + strip_prefix = "curl-7.84.0", system_build_file = "//third_party/systemlibs:curl.BUILD", urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/curl.haxx.se/download/curl-7.78.0.tar.gz", - "https://curl.haxx.se/download/curl-7.78.0.tar.gz", + "https://storage.googleapis.com/mirror.tensorflow.org/curl.haxx.se/download/curl-7.84.0.tar.gz", + "https://curl.haxx.se/download/curl-7.84.0.tar.gz", ], ) @@ -699,15 +699,16 @@ def _tf_repositories(): ], ) + # Note: if you update this, you have to update libpng too. See cl/437813808 tf_http_archive( name = "zlib", build_file = "//third_party:zlib.BUILD", - sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1", - strip_prefix = "zlib-1.2.11", + sha256 = "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9", + strip_prefix = "zlib-1.2.12", system_build_file = "//third_party/systemlibs:zlib.BUILD", urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/zlib.net/zlib-1.2.11.tar.gz", - "https://zlib.net/zlib-1.2.11.tar.gz", + "https://storage.googleapis.com/mirror.tensorflow.org/zlib.net/zlib-1.2.12.tar.gz", + "https://zlib.net/zlib-1.2.12.tar.gz", ], ) diff --git a/third_party/curl.BUILD b/third_party/curl.BUILD index 6a41034e0bd027..508de07e36b246 100644 --- a/third_party/curl.BUILD +++ b/third_party/curl.BUILD @@ -50,8 +50,6 @@ cc_library( "lib/config-os400.h", "lib/config-plan9.h", "lib/config-riscos.h", - "lib/config-tpf.h", - "lib/config-vxworks.h", "lib/config-win32.h", "lib/config-win32ce.h", "lib/conncache.c", @@ -123,12 +121,15 @@ cc_library( "lib/easyif.h", "lib/easyoptions.c", "lib/easyoptions.h", + "lib/easy_lock.h", "lib/escape.c", "lib/escape.h", "lib/file.c", "lib/file.h", "lib/fileinfo.c", "lib/fileinfo.h", + "lib/fopen.c", + "lib/fopen.h", "lib/formdata.c", "lib/formdata.h", "lib/ftp.c", @@ -140,12 +141,14 @@ cc_library( "lib/getinfo.h", "lib/gopher.c", "lib/gopher.h", + "lib/h2h3.c", + "lib/h2h3.h", "lib/hash.c", "lib/hash.h", + "lib/headers.c", + "lib/headers.h", "lib/hmac.c", "lib/hostasyn.c", - "lib/hostcheck.c", - "lib/hostcheck.h", "lib/hostip.c", "lib/hostip.h", "lib/hostip4.c", @@ -195,12 +198,8 @@ cc_library( "lib/multiif.h", "lib/netrc.c", "lib/netrc.h", - "lib/non-ascii.c", - "lib/non-ascii.h", "lib/nonblock.c", "lib/nonblock.h", - #"lib/nwlib.c", - #"lib/nwos.c", "lib/openldap.c", "lib/parsedate.c", "lib/parsedate.h", @@ -262,6 +261,8 @@ cc_library( "lib/telnet.h", "lib/tftp.c", "lib/tftp.h", + "lib/timediff.c", + "lib/timediff.h", "lib/timeval.c", "lib/timeval.h", "lib/transfer.c", @@ -278,8 +279,6 @@ cc_library( "lib/warnless.h", "lib/wildcard.c", "lib/wildcard.h", - "lib/x509asn1.c", - "lib/x509asn1.h", "lib/vauth/cleartext.c", "lib/vauth/cram.c", "lib/vauth/digest.c", @@ -294,6 +293,8 @@ cc_library( "lib/vauth/spnego_sspi.c", "lib/vauth/vauth.c", "lib/vauth/vauth.h", + "lib/vquic/msh3.c", + "lib/vquic/msh3.h", "lib/vquic/ngtcp2.c", "lib/vquic/ngtcp2.h", "lib/vquic/quiche.c", @@ -310,14 +311,14 @@ cc_library( "lib/vtls/gskit.h", "lib/vtls/gtls.c", "lib/vtls/gtls.h", + "lib/vtls/hostcheck.c", + "lib/vtls/hostcheck.h", "lib/vtls/keylog.c", "lib/vtls/keylog.h", "lib/vtls/mbedtls.c", "lib/vtls/mbedtls.h", "lib/vtls/mbedtls_threadlock.c", "lib/vtls/mbedtls_threadlock.h", - "lib/vtls/mesalink.c", - "lib/vtls/mesalink.h", "lib/vtls/nss.c", "lib/vtls/nssg.h", "lib/vtls/openssl.c", @@ -332,6 +333,8 @@ cc_library( "lib/vtls/vtls.h", "lib/vtls/wolfssl.c", "lib/vtls/wolfssl.h", + "lib/vtls/x509asn1.c", + "lib/vtls/x509asn1.h", ] + select({ "@org_tensorflow//tensorflow:macos": [ "lib/vtls/sectransp.c", @@ -347,6 +350,7 @@ cc_library( "include/curl/curl.h", "include/curl/curlver.h", "include/curl/easy.h", + "include/curl/header.h", "include/curl/mprintf.h", "include/curl/multi.h", "include/curl/options.h", @@ -447,8 +451,6 @@ cc_binary( "src/tool_cb_wrt.h", "src/tool_cfgable.c", "src/tool_cfgable.h", - "src/tool_convert.c", - "src/tool_convert.h", "src/tool_dirhie.c", "src/tool_dirhie.h", "src/tool_doswin.c", diff --git a/third_party/icu/data/BUILD.bazel b/third_party/icu/data/BUILD.bazel index 80ea92ce9b47d4..ded85987f911f6 100644 --- a/third_party/icu/data/BUILD.bazel +++ b/third_party/icu/data/BUILD.bazel @@ -19,9 +19,28 @@ exports_files(["LICENSE"]) # $ ICU_DATA_FILTER_FILE=filters.json ./runConfigureICU Linux # $ make clean && make # $ cd data/out/tmp -# $ genccode icudt64l.dat -# $ echo 'U_CAPI const void * U_EXPORT2 uprv_getICUData_conversion() { return icudt64l_dat.bytes; }' >> icudt64l_dat.c -# This creates icudt64l_dat.c, which you can move, rename, gzip, then split. +# $ genccode icudt70l.dat # Note: this number must match version, and below too! +# $ echo 'U_CAPI const void * U_EXPORT2 uprv_getICUData_conversion() { return icudt70l_dat.bytes; }' >> icudt70l_dat.c +# +# This creates icudt70l_dat.c, which you can move, rename, gzip, then split, +# for example (but you can change to other numbers): +# $ cp icudt70l_dat.c icu_conversion_data.c +# $ gzip icu_conversion_data.c +# # Note: make sure you don't forget the last . below! +# $ split -a 3 -b 100000 icu_conversion_data.c.gz icu_conversion_data.c.gz. +# +# Then, copy the generated files to this directory, removing existing ones. +# +# The current files have been generated by this filter (in filters.json): +# { +# "localeFilter": { +# "filterType": "language", +# "includelist": [ +# "en" +# ] +# } +# } +# Please make sure to keep this updated if you change the data files. filegroup( name = "conversion_files", srcs = glob(["icu_conversion_data.c.gz.*"]), diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aa b/third_party/icu/data/icu_conversion_data.c.gz.aa deleted file mode 100644 index 543b6615708830..00000000000000 Binary files a/third_party/icu/data/icu_conversion_data.c.gz.aa and /dev/null differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aaa b/third_party/icu/data/icu_conversion_data.c.gz.aaa new file mode 100644 index 00000000000000..b11bc8e1c2b268 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aaa differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aab b/third_party/icu/data/icu_conversion_data.c.gz.aab new file mode 100644 index 00000000000000..87460f63f97cb7 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aab differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aac b/third_party/icu/data/icu_conversion_data.c.gz.aac new file mode 100644 index 00000000000000..57ca5485de4bde Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aac differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aad b/third_party/icu/data/icu_conversion_data.c.gz.aad new file mode 100644 index 00000000000000..a182512aab6a60 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aad differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aae b/third_party/icu/data/icu_conversion_data.c.gz.aae new file mode 100644 index 00000000000000..4527fa522cec12 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aae differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aaf b/third_party/icu/data/icu_conversion_data.c.gz.aaf new file mode 100644 index 00000000000000..e1dc807b347f85 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aaf differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aag b/third_party/icu/data/icu_conversion_data.c.gz.aag new file mode 100644 index 00000000000000..ed6946008feec8 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aag differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aah b/third_party/icu/data/icu_conversion_data.c.gz.aah new file mode 100644 index 00000000000000..1a474bca1fe728 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aah differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aai b/third_party/icu/data/icu_conversion_data.c.gz.aai new file mode 100644 index 00000000000000..4a78d2f18c6f8b Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aai differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aaj b/third_party/icu/data/icu_conversion_data.c.gz.aaj new file mode 100644 index 00000000000000..5b40d555fdf22e Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aaj differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aak b/third_party/icu/data/icu_conversion_data.c.gz.aak new file mode 100644 index 00000000000000..e43a5cb2b7b7a2 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aak differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aal b/third_party/icu/data/icu_conversion_data.c.gz.aal new file mode 100644 index 00000000000000..8856e1e2cb49da Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aal differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aam b/third_party/icu/data/icu_conversion_data.c.gz.aam new file mode 100644 index 00000000000000..5d0d5e3fae793f Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aam differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aan b/third_party/icu/data/icu_conversion_data.c.gz.aan new file mode 100644 index 00000000000000..9cbff7140acca4 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aan differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aao b/third_party/icu/data/icu_conversion_data.c.gz.aao new file mode 100644 index 00000000000000..b3e8eab98d0e86 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aao differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aap b/third_party/icu/data/icu_conversion_data.c.gz.aap new file mode 100644 index 00000000000000..a3ec92a470fd8c Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aap differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aaq b/third_party/icu/data/icu_conversion_data.c.gz.aaq new file mode 100644 index 00000000000000..cdcdc42024f386 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aaq differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aar b/third_party/icu/data/icu_conversion_data.c.gz.aar new file mode 100644 index 00000000000000..b3d4a2b8396f8c Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aar differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aas b/third_party/icu/data/icu_conversion_data.c.gz.aas new file mode 100644 index 00000000000000..30dd37ff26925e Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aas differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aat b/third_party/icu/data/icu_conversion_data.c.gz.aat new file mode 100644 index 00000000000000..f3e8330204b4aa Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aat differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aau b/third_party/icu/data/icu_conversion_data.c.gz.aau new file mode 100644 index 00000000000000..bd503d27300027 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aau differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aav b/third_party/icu/data/icu_conversion_data.c.gz.aav new file mode 100644 index 00000000000000..7be56870f45656 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aav differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aaw b/third_party/icu/data/icu_conversion_data.c.gz.aaw new file mode 100644 index 00000000000000..40057bbc81905a Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aaw differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aax b/third_party/icu/data/icu_conversion_data.c.gz.aax new file mode 100644 index 00000000000000..e3ec8117d5aa65 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aax differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aay b/third_party/icu/data/icu_conversion_data.c.gz.aay new file mode 100644 index 00000000000000..b0c0b5a171b9a0 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aay differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aaz b/third_party/icu/data/icu_conversion_data.c.gz.aaz new file mode 100644 index 00000000000000..8cdd177cfc5308 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aaz differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ab b/third_party/icu/data/icu_conversion_data.c.gz.ab deleted file mode 100644 index d8cd5108e62fb0..00000000000000 Binary files a/third_party/icu/data/icu_conversion_data.c.gz.ab and /dev/null differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aba b/third_party/icu/data/icu_conversion_data.c.gz.aba new file mode 100644 index 00000000000000..6a892bd60db59d Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aba differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abb b/third_party/icu/data/icu_conversion_data.c.gz.abb new file mode 100644 index 00000000000000..ce05de8084bf6a Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abb differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abc b/third_party/icu/data/icu_conversion_data.c.gz.abc new file mode 100644 index 00000000000000..e42ebce1ded76f Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abc differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abd b/third_party/icu/data/icu_conversion_data.c.gz.abd new file mode 100644 index 00000000000000..04be858c2e71fa Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abd differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abe b/third_party/icu/data/icu_conversion_data.c.gz.abe new file mode 100644 index 00000000000000..f27572bf716a88 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abe differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abf b/third_party/icu/data/icu_conversion_data.c.gz.abf new file mode 100644 index 00000000000000..b1cd4256152abd Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abf differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abg b/third_party/icu/data/icu_conversion_data.c.gz.abg new file mode 100644 index 00000000000000..f071eb404cef13 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abg differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abh b/third_party/icu/data/icu_conversion_data.c.gz.abh new file mode 100644 index 00000000000000..fcbe80a605b523 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abh differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abi b/third_party/icu/data/icu_conversion_data.c.gz.abi new file mode 100644 index 00000000000000..07b5626d49f7a4 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abi differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abj b/third_party/icu/data/icu_conversion_data.c.gz.abj new file mode 100644 index 00000000000000..17db0aebcaa848 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abj differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abk b/third_party/icu/data/icu_conversion_data.c.gz.abk new file mode 100644 index 00000000000000..1df6d71755c019 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abk differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abl b/third_party/icu/data/icu_conversion_data.c.gz.abl new file mode 100644 index 00000000000000..19065efa8bc25b Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abl differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abm b/third_party/icu/data/icu_conversion_data.c.gz.abm new file mode 100644 index 00000000000000..97fbe53304eff2 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abm differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abn b/third_party/icu/data/icu_conversion_data.c.gz.abn new file mode 100644 index 00000000000000..8b47b3c94def78 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abn differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abo b/third_party/icu/data/icu_conversion_data.c.gz.abo new file mode 100644 index 00000000000000..9985a2de553270 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abo differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abp b/third_party/icu/data/icu_conversion_data.c.gz.abp new file mode 100644 index 00000000000000..ae0a812b9db095 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abp differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abq b/third_party/icu/data/icu_conversion_data.c.gz.abq new file mode 100644 index 00000000000000..8b071f0e6a858e Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abq differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abr b/third_party/icu/data/icu_conversion_data.c.gz.abr new file mode 100644 index 00000000000000..f00c95e9246f74 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abr differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abs b/third_party/icu/data/icu_conversion_data.c.gz.abs new file mode 100644 index 00000000000000..c0571dc9adf4fc Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abs differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abt b/third_party/icu/data/icu_conversion_data.c.gz.abt new file mode 100644 index 00000000000000..f6c75209c83128 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abt differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abu b/third_party/icu/data/icu_conversion_data.c.gz.abu new file mode 100644 index 00000000000000..7c049c5550077b Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abu differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abv b/third_party/icu/data/icu_conversion_data.c.gz.abv new file mode 100644 index 00000000000000..a533067e76125a Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abv differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abw b/third_party/icu/data/icu_conversion_data.c.gz.abw new file mode 100644 index 00000000000000..8ad6abb99516e5 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abw differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abx b/third_party/icu/data/icu_conversion_data.c.gz.abx new file mode 100644 index 00000000000000..54e0515a944a09 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abx differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aby b/third_party/icu/data/icu_conversion_data.c.gz.aby new file mode 100644 index 00000000000000..6be26e2dda1f5f Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aby differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.abz b/third_party/icu/data/icu_conversion_data.c.gz.abz new file mode 100644 index 00000000000000..817dd47d5b973d Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.abz differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ac b/third_party/icu/data/icu_conversion_data.c.gz.ac deleted file mode 100644 index bde21d16f57c16..00000000000000 Binary files a/third_party/icu/data/icu_conversion_data.c.gz.ac and /dev/null differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aca b/third_party/icu/data/icu_conversion_data.c.gz.aca new file mode 100644 index 00000000000000..1fac65927fd443 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aca differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acb b/third_party/icu/data/icu_conversion_data.c.gz.acb new file mode 100644 index 00000000000000..f3e6da1f7d0450 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acb differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acc b/third_party/icu/data/icu_conversion_data.c.gz.acc new file mode 100644 index 00000000000000..1fb0cc49281c37 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acc differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acd b/third_party/icu/data/icu_conversion_data.c.gz.acd new file mode 100644 index 00000000000000..60bfeba83255d6 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acd differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ace b/third_party/icu/data/icu_conversion_data.c.gz.ace new file mode 100644 index 00000000000000..7b60fe5a3ac8d9 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.ace differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acf b/third_party/icu/data/icu_conversion_data.c.gz.acf new file mode 100644 index 00000000000000..dd8ebff2963c99 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acf differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acg b/third_party/icu/data/icu_conversion_data.c.gz.acg new file mode 100644 index 00000000000000..c5015757d328e7 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acg differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ach b/third_party/icu/data/icu_conversion_data.c.gz.ach new file mode 100644 index 00000000000000..10c50c1d96a574 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.ach differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aci b/third_party/icu/data/icu_conversion_data.c.gz.aci new file mode 100644 index 00000000000000..75be388aee0c11 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aci differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acj b/third_party/icu/data/icu_conversion_data.c.gz.acj new file mode 100644 index 00000000000000..f55b68e633f400 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acj differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ack b/third_party/icu/data/icu_conversion_data.c.gz.ack new file mode 100644 index 00000000000000..121d97423eb7ea Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.ack differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acl b/third_party/icu/data/icu_conversion_data.c.gz.acl new file mode 100644 index 00000000000000..eafb3b60b47383 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acl differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acm b/third_party/icu/data/icu_conversion_data.c.gz.acm new file mode 100644 index 00000000000000..f7a3b5617bc8c5 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acm differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acn b/third_party/icu/data/icu_conversion_data.c.gz.acn new file mode 100644 index 00000000000000..eff17429e724fd Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acn differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aco b/third_party/icu/data/icu_conversion_data.c.gz.aco new file mode 100644 index 00000000000000..8388dc5c141374 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.aco differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acp b/third_party/icu/data/icu_conversion_data.c.gz.acp new file mode 100644 index 00000000000000..1e9a4bc18ed96a Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acp differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acq b/third_party/icu/data/icu_conversion_data.c.gz.acq new file mode 100644 index 00000000000000..51a5737930a6a7 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acq differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acr b/third_party/icu/data/icu_conversion_data.c.gz.acr new file mode 100644 index 00000000000000..96e27c26624b34 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acr differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acs b/third_party/icu/data/icu_conversion_data.c.gz.acs new file mode 100644 index 00000000000000..30b0970756d7e3 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acs differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.act b/third_party/icu/data/icu_conversion_data.c.gz.act new file mode 100644 index 00000000000000..21b9688e5e774e Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.act differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acu b/third_party/icu/data/icu_conversion_data.c.gz.acu new file mode 100644 index 00000000000000..cea7d355d07ab2 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acu differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acv b/third_party/icu/data/icu_conversion_data.c.gz.acv new file mode 100644 index 00000000000000..8ddf19818ced08 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acv differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acw b/third_party/icu/data/icu_conversion_data.c.gz.acw new file mode 100644 index 00000000000000..c9c2bceaaf1930 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acw differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acx b/third_party/icu/data/icu_conversion_data.c.gz.acx new file mode 100644 index 00000000000000..0ca1d9aaf65aa3 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acx differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acy b/third_party/icu/data/icu_conversion_data.c.gz.acy new file mode 100644 index 00000000000000..fbc2459b6a10ab Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acy differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.acz b/third_party/icu/data/icu_conversion_data.c.gz.acz new file mode 100644 index 00000000000000..862436c9459487 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.acz differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ad b/third_party/icu/data/icu_conversion_data.c.gz.ad deleted file mode 100644 index f476988a0b24fb..00000000000000 Binary files a/third_party/icu/data/icu_conversion_data.c.gz.ad and /dev/null differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ada b/third_party/icu/data/icu_conversion_data.c.gz.ada new file mode 100644 index 00000000000000..6034e047321250 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.ada differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.adb b/third_party/icu/data/icu_conversion_data.c.gz.adb new file mode 100644 index 00000000000000..07b519b21c089a Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.adb differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.adc b/third_party/icu/data/icu_conversion_data.c.gz.adc new file mode 100644 index 00000000000000..12d52c54e02a9e Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.adc differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.add b/third_party/icu/data/icu_conversion_data.c.gz.add new file mode 100644 index 00000000000000..e9995953c924b3 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.add differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ade b/third_party/icu/data/icu_conversion_data.c.gz.ade new file mode 100644 index 00000000000000..292d09cfd1d457 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.ade differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.adf b/third_party/icu/data/icu_conversion_data.c.gz.adf new file mode 100644 index 00000000000000..dc2c28d019b7b3 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.adf differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.adg b/third_party/icu/data/icu_conversion_data.c.gz.adg new file mode 100644 index 00000000000000..c152c80b1d1ac0 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.adg differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.adh b/third_party/icu/data/icu_conversion_data.c.gz.adh new file mode 100644 index 00000000000000..9fcb83e56560b1 Binary files /dev/null and b/third_party/icu/data/icu_conversion_data.c.gz.adh differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ae b/third_party/icu/data/icu_conversion_data.c.gz.ae deleted file mode 100644 index 3388b38c1a2b7a..00000000000000 Binary files a/third_party/icu/data/icu_conversion_data.c.gz.ae and /dev/null differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.af b/third_party/icu/data/icu_conversion_data.c.gz.af deleted file mode 100644 index 344e3925f39d60..00000000000000 Binary files a/third_party/icu/data/icu_conversion_data.c.gz.af and /dev/null differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ag b/third_party/icu/data/icu_conversion_data.c.gz.ag deleted file mode 100644 index 249ffddde77176..00000000000000 Binary files a/third_party/icu/data/icu_conversion_data.c.gz.ag and /dev/null differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ah b/third_party/icu/data/icu_conversion_data.c.gz.ah deleted file mode 100644 index 8893be204197a0..00000000000000 Binary files a/third_party/icu/data/icu_conversion_data.c.gz.ah and /dev/null differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ai b/third_party/icu/data/icu_conversion_data.c.gz.ai deleted file mode 100644 index e6251e3a11c5b2..00000000000000 Binary files a/third_party/icu/data/icu_conversion_data.c.gz.ai and /dev/null differ diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aj b/third_party/icu/data/icu_conversion_data.c.gz.aj deleted file mode 100644 index 3e1dc684c93176..00000000000000 Binary files a/third_party/icu/data/icu_conversion_data.c.gz.aj and /dev/null differ diff --git a/third_party/icu/udata.patch b/third_party/icu/udata.patch index 0b65e4ed3886f1..f31a604c15908a 100644 --- a/third_party/icu/udata.patch +++ b/third_party/icu/udata.patch @@ -41,7 +41,8 @@ diff -ru a/icu4c/source/common/udata.cpp b/icu4c/source/common/udata.cpp } - */ + - #if U_PLATFORM_HAS_WINUWP_API == 0 // Windows UWP Platform does not support dll icu data at this time + #if !defined(ICU_DATA_DIR_WINDOWS) + // When using the Windows system data, we expect only a single data file. setCommonICUDataPointer(&U_ICUDATA_ENTRY_POINT, FALSE, pErrorCode); { diff -ru a/icu4c/source/common/unicode/uconfig.h b/icu4c/source/common/unicode/uconfig.h diff --git a/third_party/icu/workspace.bzl b/third_party/icu/workspace.bzl index e4ed9669e0c7ee..c2ebd557f77a50 100644 --- a/third_party/icu/workspace.bzl +++ b/third_party/icu/workspace.bzl @@ -2,14 +2,16 @@ load("//third_party:repo.bzl", "tf_http_archive") +# NOTE: If you upgrade this, generate the data files by following the +# instructions in third_party/icu/data/BUILD def repo(): tf_http_archive( name = "icu", - strip_prefix = "icu-release-64-2", - sha256 = "dfc62618aa4bd3ca14a3df548cd65fe393155edd213e49c39f3a30ccd618fc27", + strip_prefix = "icu-release-69-1", + sha256 = "3144e17a612dda145aa0e4acb3caa27a5dae4e26edced64bc351c43d5004af53", urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/unicode-org/icu/archive/release-64-2.zip", - "https://github.com/unicode-org/icu/archive/release-64-2.zip", + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/unicode-org/icu/archive/release-69-1.zip", + "https://github.com/unicode-org/icu/archive/release-69-1.zip", ], build_file = "//third_party/icu:BUILD.bazel", system_build_file = "//third_party/icu:BUILD.system", diff --git a/third_party/png.BUILD b/third_party/png.BUILD index 719d4c7c670fc6..145b0dff05e38a 100644 --- a/third_party/png.BUILD +++ b/third_party/png.BUILD @@ -61,7 +61,7 @@ genrule( name = "snappy_stubs_public_h", srcs = ["scripts/pnglibconf.h.prebuilt"], outs = ["pnglibconf.h"], - cmd = "sed -e 's/PNG_ZLIB_VERNUM 0/PNG_ZLIB_VERNUM 0x12b0/' $< >$@", + cmd = "sed -e 's/PNG_ZLIB_VERNUM 0/PNG_ZLIB_VERNUM 0x12c0/' $< >$@", ) config_setting(