diff --git a/.zenodo.json b/.zenodo.json new file mode 100644 index 00000000000000..7161180c51ae3e --- /dev/null +++ b/.zenodo.json @@ -0,0 +1,13 @@ +{ + "description": "TensorFlow is an end-to-end open source platform for machine learning. It has a comprehensive, flexible ecosystem of tools, libraries, and community resources that lets researchers push the state-of-the-art in ML and developers easily build and deploy ML-powered applications.", + "license": "Apache-2.0", + "title": "TensorFlow", + "upload_type": "software", + "creators": [ + { + "name": "TensorFlow Developers" + } + ], + "access_right": "open", + "notes": "Specific TensorFlow versions can be found in the \"Versions\" list on the right side of this page.
See the full list of authors on GitHub." +} diff --git a/RELEASE.md b/RELEASE.md index 87631f9de50d6b..e1ff7153cd8626 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,193 @@ +# Release 2.2.3 + +This release introduces several vulnerability fixes: + + * Fixes a heap buffer overflow in `RaggedBinCount` ([CVE-2021-29512](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29512)) + * Fixes a heap out of bounds write in `RaggedBinCount` ([CVE-2021-29514](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29514)) + * Fixes a type confusion during tensor casts which leads to dereferencing null pointers ([CVE-2021-29513](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29513)) + * Fixes a reference binding to null pointer in `MatrixDiag*` ops ([CVE-2021-29515](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29515)) + * Fixes a null pointer dereference via invalid Ragged Tensors ([CVE-2021-29516](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29516)) + * Fixes a division by zero in `Conv3D` ([CVE-2021-29517](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29517)) + * Fixes vulnerabilities where session operations in eager mode lead to null pointer dereferences ([CVE-2021-29518](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29518)) + * Fixes a `CHECK`-fail in `SparseCross` caused by type confusion ([CVE-2021-29519](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29519)) + * Fixes a segfault in `SparseCountSparseOutput` ([CVE-2021-29521](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29521)) + * Fixes a heap buffer overflow in `Conv3DBackprop*` ([CVE-2021-29520](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29520)) + * Fixes a division by 0 in `Conv3DBackprop*` ([CVE-2021-29522](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29522)) + * Fixes a `CHECK`-fail in `AddManySparseToTensorsMap` ([CVE-2021-29523](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29523)) + * Fixes a division by 0 in `Conv2DBackpropFilter` ([CVE-2021-29524](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29524)) + * Fixes a division by 0 in `Conv2DBackpropInput` ([CVE-2021-29525](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29525)) + * Fixes a division by 0 in `Conv2D` ([CVE-2021-29526](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29526)) + * Fixes a division by 0 in `QuantizedConv2D` ([CVE-2021-29527](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29527)) + * Fixes a division by 0 in `QuantizedMul` ([CVE-2021-29528](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29528)) + * Fixes vulnerabilities caused by invalid validation in `SparseMatrixSparseCholesky` ([CVE-2021-29530](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29530)) + * Fixes a heap buffer overflow caused by rounding ([CVE-2021-29529](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29529)) + * Fixes a `CHECK`-fail in `tf.raw_ops.EncodePng` ([CVE-2021-29531](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29531)) + * Fixes a heap out of bounds read in `RaggedCross` ([CVE-2021-29532](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29532)) + * Fixes a `CHECK`-fail in `DrawBoundingBoxes` ([CVE-2021-29533](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29533)) + * Fixes a heap buffer overflow in `QuantizedMul` ([CVE-2021-29535](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29535)) + * Fixes a `CHECK`-fail in `SparseConcat` ([CVE-2021-29534](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29534)) + * Fixes a heap buffer overflow in `QuantizedResizeBilinear` ([CVE-2021-29537](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29537)) + * Fixes a heap buffer overflow in `QuantizedReshape` ([CVE-2021-29536](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29536)) + * Fixes a division by zero in `Conv2DBackpropFilter` ([CVE-2021-29538](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29538)) + * Fixes a heap buffer overflow in `Conv2DBackpropFilter` ([CVE-2021-29540](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29540)) + * Fixes a heap buffer overflow in `StringNGrams` ([CVE-2021-29542](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29542)) + * Fixes a null pointer dereference in `StringNGrams` ([CVE-2021-29541](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29541)) + * Fixes a `CHECK`-fail in `QuantizeAndDequantizeV4Grad` ([CVE-2021-29544](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29544)) + * Fixes a `CHECK`-fail in `CTCGreedyDecoder` ([CVE-2021-29543](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29543)) + * Fixes a heap buffer overflow in `SparseTensorToCSRSparseMatrix` ([CVE-2021-29545](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29545)) + * Fixes a division by 0 in `QuantizedBiasAdd` ([CVE-2021-29546](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29546)) + * Fixes a heap out of bounds in `QuantizedBatchNormWithGlobalNormalization` ([CVE-2021-29547](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29547)) + * Fixes a division by 0 in `QuantizedBatchNormWithGlobalNormalization` ([CVE-2021-29548](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29548)) + * Fixes a division by 0 in `QuantizedAdd` ([CVE-2021-29549](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29549)) + * Fixes a division by 0 in `FractionalAvgPool` ([CVE-2021-29550](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29550)) + * Fixes an OOB read in `MatrixTriangularSolve` ([CVE-2021-29551](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29551)) + * Fixes a heap OOB in `QuantizeAndDequantizeV3` ([CVE-2021-29553](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29553)) + * Fixes a `CHECK`-failure in `UnsortedSegmentJoin` ([CVE-2021-29552](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29552)) + * Fixes a division by 0 in `DenseCountSparseOutput` ([CVE-2021-29554](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29554)) + * Fixes a division by 0 in `FusedBatchNorm` ([CVE-2021-29555](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29555)) + * Fixes a division by 0 in `SparseMatMul` ([CVE-2021-29557](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29557)) + * Fixes a division by 0 in `Reverse` ([CVE-2021-29556](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29556)) + * Fixes a heap buffer overflow in `SparseSplit` ([CVE-2021-29558](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29558)) + * Fixes a heap OOB access in unicode ops ([CVE-2021-29559](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29559)) + * Fixes a heap buffer overflow in `RaggedTensorToTensor` ([CVE-2021-29560](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29560)) + * Fixes a `CHECK`-fail in `LoadAndRemapMatrix` ([CVE-2021-29561](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29561)) + * Fixes a `CHECK`-fail in `tf.raw_ops.IRFFT` ([CVE-2021-29562](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29562)) + * Fixes a `CHECK`-fail in `tf.raw_ops.RFFT` ([CVE-2021-29563](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29563)) + * Fixes a null pointer dereference in `EditDistance` ([CVE-2021-29564](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29564)) + * Fixes a null pointer dereference in `SparseFillEmptyRows` ([CVE-2021-29565](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29565)) + * Fixes a heap OOB access in `Dilation2DBackpropInput` ([CVE-2021-29566](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29566)) + * Fixes a reference binding to null in `ParameterizedTruncatedNormal` ([CVE-2021-29568](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29568)) + * Fixes a set of vulnerabilities caused by lack of validation in `SparseDenseCwiseMul` ([CVE-2021-29567](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29567)) + * Fixes a heap out of bounds read in `MaxPoolGradWithArgmax` ([CVE-2021-29570](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29570)) + * Fixes a heap out of bounds read in `RequantizationRange` ([CVE-2021-29569](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29569)) + * Fixes a memory corruption in `DrawBoundingBoxesV2` ([CVE-2021-29571](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29571)) + * Fixes a reference binding to nullptr in `SdcaOptimizer` ([CVE-2021-29572](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29572)) + * Fixes an overflow and a denial of service in `tf.raw_ops.ReverseSequence` ([CVE-2021-29575](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29575)) + * Fixes a division by 0 in `MaxPoolGradWithArgmax` ([CVE-2021-29573](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29573)) + * Fixes an undefined behavior in `MaxPool3DGradGrad` ([CVE-2021-29574](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29574)) + * Fixes a heap buffer overflow in `MaxPool3DGradGrad` ([CVE-2021-29576](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29576)) + * Fixes a heap buffer overflow in `AvgPool3DGrad` ([CVE-2021-29577](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29577)) + * Fixes an undefined behavior and a `CHECK`-fail in `FractionalMaxPoolGrad` ([CVE-2021-29580](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29580)) + * Fixes a heap buffer overflow in `FractionalAvgPoolGrad` ([CVE-2021-29578](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29578)) + * Fixes a heap buffer overflow in `MaxPoolGrad` ([CVE-2021-29579](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29579)) + * Fixes a segfault in `CTCBeamSearchDecoder` ([CVE-2021-29581](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29581)) + * Fixes a heap OOB read in `tf.raw_ops.Dequantize` ([CVE-2021-29582](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29582)) + * Fixes a `CHECK`-fail due to integer overflow ([CVE-2021-29584](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29584)) + * Fixes a heap buffer overflow and undefined behavior in `FusedBatchNorm` ([CVE-2021-29583](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29583)) + * Fixes a division by zero in padding computation in TFLite ([CVE-2021-29585](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29585)) + * Fixes a division by zero in optimized pooling implementations in TFLite ([CVE-2021-29586](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29586)) + * Fixes a division by zero in TFLite's implementation of `SpaceToDepth` ([CVE-2021-29587](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29587)) + * Fixes a division by zero in TFLite's implementation of `GatherNd` ([CVE-2021-29589](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29589)) + * Fixes a division by zero in TFLite's implementation of `TransposeConv` ([CVE-2021-29588](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29588)) + * Fixes a heap OOB read in TFLite's implementation of `Minimum` or `Maximum` ([CVE-2021-29590](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29590)) + * Fixes a null pointer dereference in TFLite's `Reshape` operator ([CVE-2021-29592](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29592)) + * Fixes a stack overflow due to looping TFLite subgraph ([CVE-2021-29591](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29591)) + * Fixes a division by zero in TFLite's implementation of `DepthToSpace` ([CVE-2021-29595](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29595)) + * Fixes a division by zero in TFLite's convolution code ([CVE-2021-29594](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29594)) + * Fixes a division by zero in TFLite's implementation of `EmbeddingLookup` ([CVE-2021-29596](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29596)) + * Fixes a division by zero in TFLite's implementation of `BatchToSpaceNd` ([CVE-2021-29593](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29593)) + * Fixes a division by zero in TFLite's implementation of `SpaceToBatchNd` ([CVE-2021-29597](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29597)) + * Fixes a division by zero in TFLite's implementation of `SVDF` ([CVE-2021-29598](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29598)) + * Fixes a division by zero in TFLite's implementation of `Split` ([CVE-2021-29599](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29599)) + * Fixes a division by zero in TFLite's implementation of `OneHot` ([CVE-2021-29600](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29600)) + * Fixes a division by zero in TFLite's implementation of `DepthwiseConv` ([CVE-2021-29602](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29602)) + * Fixes a division by zero in TFLite's implementation of hashtable lookup ([CVE-2021-29604](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29604)) + * Fixes a integer overflow in TFLite concatentation ([CVE-2021-29601](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29601)) + * Fixes a integer overflow in TFLite memory allocation ([CVE-2021-29605](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29605)) + * Fixes a heap OOB write in TFLite ([CVE-2021-29603](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29603)) + * Fixes a heap OOB read in TFLite ([CVE-2021-29606](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29606)) + * Fixes a heap OOB and null pointer dereference in `RaggedTensorToTensor` ([CVE-2021-29608](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29608)) + * Fixes vulnerabilities caused by incomplete validation in `SparseAdd` ([CVE-2021-29609](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29609)) + * Fixes vulnerabilities caused by incomplete validation in `SparseSparseMinimum` ([CVE-2021-29607](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29607)) + * Fixes vulnerabilities caused by incomplete validation in `SparseReshape` ([CVE-2021-29611](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29611)) + * Fixes vulnerabilities caused by invalid validation in `QuantizeAndDequantizeV2` ([CVE-2021-29610](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29610)) + * Fixes a heap buffer overflow in `BandedTriangularSolve` ([CVE-2021-29612](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29612)) + * Fixes vulnerabilities caused by incomplete validation in `tf.raw_ops.CTCLoss` ([CVE-2021-29613](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29613)) + * Fixes an interpreter crash from vulnerabilities in `tf.io.decode_raw` ([CVE-2021-29614](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29614)) + * Fixes a stack overflow in `ParseAttrValue` with nested tensors ([CVE-2021-29615](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29615)) + * Fixes a null dereference in Grappler's `TrySimplify` ([CVE-2021-29616](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29616)) + * Fixes a crash in `tf.transpose` with complex inputs ([CVE-2021-29618](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29618)) + * Fixes a crash in `tf.strings.substr` due to `CHECK`-fail ([CVE-2021-29617](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29617)) + * Fixes a segfault in `tf.raw_ops.SparseCountSparseOutput` ([CVE-2021-29619](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29619)) + * Fixes a segfault in `tf.raw_ops.ImmutableConst` ([CVE-2021-29539](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29539)) + * Updates `curl` to `7.76.0` to handle [CVE-2020-8169](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-8169), [CVE-2020-8177](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-8177), [CVE-2020-8231](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-8231), [CVE-2020-8284](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-8284), [CVE-2020-8285](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-8285) and [CVE-2020-8286](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-8286). + +# Release 2.2.2 + +## Bug Fixes and Other Changes +* Fixes an access to unitialized memory in Eigen code + ([CVE-2020-26266](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26266)) +* Fixes a security vulnerability caused by lack of validation in + `tf.raw_ops.DataFormatVecPermute` and `tf.raw_ops.DataFormatDimMap` + ([CVE-2020-26267](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26267)) +* Fixes a vulnerability caused by attempting to write to immutable memory region in + `tf.raw_ops.ImmutableConst` + ([CVE-2020-26268](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26268) +* Fixes a `CHECK`-fail in LSTM with zero-length input + ([CVE-2020-26270](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26270)) +* Fixes a security vulnerability caused by accessing heap data outside of bounds + when loading a specially crafted `SavedModel` + ([CVE-2020-26271](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26271)) +* Prevents memory leaks in loading `SavedModel`s that import functions +* Updates `libjpeg-turbo` to `2.0.5` to handle + [CVE-2020-13790](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-13790). +* Updates `junit` to `4.13.1` to handle + [CVE-2020-15250](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15250). +* Updates `PCRE` to `8.44` to handle + [CVE-2019-20838](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-20838) + and + [CVE-2020-14155](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-14155). +* Updates `sqlite3` to `3.44.0` to keep in sync with master branch. + +# Release 2.2.1 + +## Bug Fixes and Other Changes +* Fixes an undefined behavior causing a segfault in `tf.raw_ops.Switch` + ([CVE-2020-15190](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15190)) +* Fixes three vulnerabilities in conversion to DLPack format + ([CVE-2020-15191](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15191), + [CVE-2020-15192](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15192), + [CVE-2020-15193](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15193)) +* Fixes two vulnerabilities in `SparseFillEmptyRowsGrad` + ([CVE-2020-15194](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15194), + [CVE-2020-15195](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15195)) +* Fixes an integer truncation vulnerability in code using the work sharder API + ([CVE-2020-15202](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15202)) +* Fixes a format string vulnerability in `tf.strings.as_string` + ([CVE-2020-15203](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15203)) +* Fixes segfault raised by calling session-only ops in eager mode + ([CVE-2020-15204](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15204)) +* Fixes data leak and potential ASLR violation from `tf.raw_ops.StringNGrams` + ([CVE-2020-15205](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15205)) +* Fixes segfaults caused by incomplete `SavedModel` validation + ([CVE-2020-15206](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15206)) +* Fixes a data corruption due to a bug in negative indexing support in TFLite + ([CVE-2020-15207](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15207)) +* Fixes a data corruption due to dimension mismatch in TFLite + ([CVE-2020-15208](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15208)) +* Fixes several vulnerabilities in TFLite saved model format + ([CVE-2020-15209](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15209), + [CVE-2020-15210](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15210), + [CVE-2020-15211](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15211)) +* Fixes several vulnerabilities in TFLite implementation of segment sum + ([CVE-2020-15212](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15212), + [CVE-2020-15213](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15213), + [CVE-2020-15214](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15214)) +* Updates `sqlite3` to `3.33.00` to handle + [CVE-2020-9327](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-9327), + [CVE-2020-11655](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-11655), + [CVE-2020-11656](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-11656), + [CVE-2020-13434](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-13434), + [CVE-2020-13435](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-13435), + [CVE-2020-13630](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-13630), + [CVE-2020-13631](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-13631), + [CVE-2020-13871](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-13871), + and + [CVE-2020-15358](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15358). +* Fixes deprecated usage of `collections` API +* Removes `scipy` dependency from `setup.py` since TensorFlow does not need it + to install the pip package + # Release 2.2.0 TensorFlow 2.2 discontinues support for Python 2, [previously announced](https://groups.google.com/a/tensorflow.org/d/msg/announce/gVwS5RC8mds/dCt1ka2XAAAJ) as following [Python 2's EOL on January 1, 2020](https://www.python.org/dev/peps/pep-0373/#update). diff --git a/tensorflow/c/eager/dlpack.cc b/tensorflow/c/eager/dlpack.cc index f6acf442e6dc94..4a71139627b256 100644 --- a/tensorflow/c/eager/dlpack.cc +++ b/tensorflow/c/eager/dlpack.cc @@ -250,21 +250,36 @@ void TFE_CallDLManagedTensorDeleter(void* dlm_ptr) { } void* TFE_HandleToDLPack(TFE_TensorHandle* h, TF_Status* status) { + auto tf_dlm_context = GetDlContext(h, status); + if (!status->status.ok()) { + return nullptr; + } + + auto* tf_dlm_data = TFE_TensorHandleDevicePointer(h, status); + if (!status->status.ok()) { + return nullptr; + } + const Tensor* tensor = GetTensorFromHandle(h, status); TF_DataType data_type = static_cast(tensor->dtype()); - TensorReference tensor_ref(*tensor); // This will call buf_->Ref() + auto tf_dlm_type = GetDlDataType(data_type, status); + if (!status->status.ok()) { + return nullptr; + } + + TensorReference tensor_ref(*tensor); // This will call buf_->Ref() auto* tf_dlm_tensor_ctx = new TfDlManagedTensorCtx(tensor_ref); tf_dlm_tensor_ctx->reference = tensor_ref; DLManagedTensor* dlm_tensor = &tf_dlm_tensor_ctx->tensor; dlm_tensor->manager_ctx = tf_dlm_tensor_ctx; dlm_tensor->deleter = &DLManagedTensorDeleter; - dlm_tensor->dl_tensor.ctx = GetDlContext(h, status); + dlm_tensor->dl_tensor.ctx = tf_dlm_context; int ndim = tensor->dims(); dlm_tensor->dl_tensor.ndim = ndim; - dlm_tensor->dl_tensor.data = TFE_TensorHandleDevicePointer(h, status); - dlm_tensor->dl_tensor.dtype = GetDlDataType(data_type, status); + dlm_tensor->dl_tensor.data = tf_dlm_data; + dlm_tensor->dl_tensor.dtype = tf_dlm_type; std::vector* shape_arr = &tf_dlm_tensor_ctx->shape; std::vector* stride_arr = &tf_dlm_tensor_ctx->strides; @@ -277,13 +292,14 @@ void* TFE_HandleToDLPack(TFE_TensorHandle* h, TF_Status* status) { (*stride_arr)[i] = (*shape_arr)[i + 1] * (*stride_arr)[i + 1]; } - dlm_tensor->dl_tensor.shape = &(*shape_arr)[0]; + dlm_tensor->dl_tensor.shape = shape_arr->data(); // There are two ways to represent compact row-major data // 1) nullptr indicates tensor is compact and row-majored. // 2) fill in the strides array as the real case for compact row-major data. // Here we choose option 2, since some frameworks didn't handle the strides // argument properly. - dlm_tensor->dl_tensor.strides = &(*stride_arr)[0]; + dlm_tensor->dl_tensor.strides = stride_arr->data(); + dlm_tensor->dl_tensor.byte_offset = 0; // TF doesn't handle the strides and byte_offsets here return static_cast(dlm_tensor); diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc index 6c967dcf464872..9f0a8cd87074a8 100644 --- a/tensorflow/cc/saved_model/loader.cc +++ b/tensorflow/cc/saved_model/loader.cc @@ -20,6 +20,7 @@ limitations under the License. #include "tensorflow/cc/saved_model/constants.h" #include "tensorflow/cc/saved_model/reader.h" #include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/function.pb.h" #include "tensorflow/core/framework/node_def.pb.h" #include "tensorflow/core/framework/tensor.pb.h" #include "tensorflow/core/lib/io/path.h" @@ -72,26 +73,41 @@ uint64 GetLatencyMicroseconds(const uint64 start_microseconds) { // Ensure that constant tensors loaded from the saved model have valid shape. // Also ensure that constant nodes have a value assigned to them. // TODO(b/154763635): this is temporary and will be replaced with a better audit +static Status ValidateNode(const NodeDef& node) { + const auto node_iterator = node.attr().find("value"); + if (node_iterator != node.attr().end()) { + AttrValue node_value = node_iterator->second; + if (node_value.has_tensor()) { + const PartialTensorShape node_shape(node_value.tensor().tensor_shape()); + if (node_shape.num_elements() < 0) { + return errors::FailedPrecondition( + "Saved model contains node \"", node.name(), "\" (op \"", node.op(), + "\") which initializes from a tensor with ", + node_shape.num_elements(), " elements"); + } + } + } else if (node.op() == "Const") { + return errors::FailedPrecondition( + "Saved model contains node \"", node.name(), + "\" which is a constant tensor but no value has been provided"); + } + return Status::OK(); +} + static Status ValidateSavedTensors(const GraphDef& graph_def) { for (const auto& node : graph_def.node()) { - const auto node_iterator = node.attr().find("value"); - if (node_iterator != node.attr().end()) { - AttrValue node_value = node_iterator->second; - if (node_value.has_tensor()) { - const PartialTensorShape node_shape(node_value.tensor().tensor_shape()); - if (node_shape.num_elements() < 0) { - return errors::FailedPrecondition( - "Saved model contains node \"", node.name(), "\" (op \"", - node.op(), "\") which initializes from a tensor with ", - node_shape.num_elements(), " elements"); - } + TF_RETURN_IF_ERROR(ValidateNode(node)); + } + + if (graph_def.has_library()) { + const FunctionDefLibrary& library = graph_def.library(); + for (const auto& function : library.function()) { + for (const auto& node : function.node_def()) { + TF_RETURN_IF_ERROR(ValidateNode(node)); } - } else if (node.op() == "Const") { - return errors::FailedPrecondition( - "Saved model contains node \"", node.name(), - "\" which is a constant tensor but no value has been provided"); } } + return Status::OK(); } diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.cc b/tensorflow/core/common_runtime/eager/kernel_and_device.cc index 7760c2c968f57e..2a2b649ef8cf75 100644 --- a/tensorflow/core/common_runtime/eager/kernel_and_device.cc +++ b/tensorflow/core/common_runtime/eager/kernel_and_device.cc @@ -304,7 +304,12 @@ Status KernelAndDeviceOp::Run( if (outputs != nullptr) { outputs->clear(); for (int i = 0; i < context.num_outputs(); ++i) { - outputs->push_back(Tensor(*context.mutable_output(i))); + const auto* output_tensor = context.mutable_output(i); + if (output_tensor != nullptr) { + outputs->push_back(Tensor(*output_tensor)); + } else { + outputs->push_back(Tensor()); + } } } return Status::OK(); diff --git a/tensorflow/core/framework/attr_value_util.cc b/tensorflow/core/framework/attr_value_util.cc index fb51da9dee26ee..6f6e97788e5a68 100644 --- a/tensorflow/core/framework/attr_value_util.cc +++ b/tensorflow/core/framework/attr_value_util.cc @@ -38,6 +38,9 @@ namespace { // Do not construct large tensors to compute their hash or compare for equality. constexpr int kMaxAttrValueTensorByteSize = 32 * 1024 * 1024; // 32mb +// Limit nesting of tensors to 100 deep to prevent memory overflow. +constexpr int kMaxTensorNestDepth = 100; + // Return the size of the tensor represented by this TensorProto. If shape is // not fully defined return -1. int64 TensorByteSize(const TensorProto& t) { @@ -224,6 +227,54 @@ string SummarizeFunc(const NameAttrList& func) { return strings::StrCat(func.name(), "[", absl::StrJoin(entries, ", "), "]"); } +bool ParseAttrValueHelper_TensorNestsUnderLimit(int limit, string to_parse) { + int nests = 0; + int maxed_out = to_parse.length(); + int open_curly = to_parse.find('{'); + int open_bracket = to_parse.find('<'); + int close_curly = to_parse.find('}'); + int close_bracket = to_parse.find('>'); + if (open_curly == -1) { + open_curly = maxed_out; + } + if (open_bracket == -1) { + open_bracket = maxed_out; + } + int min = std::min(open_curly, open_bracket); + do { + if (open_curly == maxed_out && open_bracket == maxed_out) { + return true; + } + if (min == open_curly) { + nests += 1; + open_curly = to_parse.find('{', open_curly + 1); + if (open_curly == -1) { + open_curly = maxed_out; + } + } else if (min == open_bracket) { + nests += 1; + open_bracket = to_parse.find('<', open_bracket + 1); + if (open_bracket == -1) { + open_bracket = maxed_out; + } + } else if (min == close_curly) { + nests -= 1; + close_curly = to_parse.find('}', close_curly + 1); + if (close_curly == -1) { + close_curly = maxed_out; + } + } else if (min == close_bracket) { + nests -= 1; + close_bracket = to_parse.find('>', close_bracket + 1); + if (close_bracket == -1) { + close_bracket = maxed_out; + } + } + min = std::min({open_curly, open_bracket, close_curly, close_bracket}); + } while (nests < 100); + return false; +} + } // namespace string SummarizeAttrValue(const AttrValue& attr_value) { @@ -448,7 +499,12 @@ bool ParseAttrValue(StringPiece type, StringPiece text, AttrValue* out) { } else { to_parse = strings::StrCat(field_name, ": ", text); } - + if (field_name == "tensor") { + if (!ParseAttrValueHelper_TensorNestsUnderLimit(kMaxTensorNestDepth, + to_parse)) { + return false; + } + } return ProtoParseFromString(to_parse, out); } diff --git a/tensorflow/core/graph/graph_constructor.cc b/tensorflow/core/graph/graph_constructor.cc index 39bb0514c34f4d..ee8f6a093fd6e6 100644 --- a/tensorflow/core/graph/graph_constructor.cc +++ b/tensorflow/core/graph/graph_constructor.cc @@ -1436,6 +1436,17 @@ void GraphConstructor::Undo() { Status GraphConstructor::MakeEdge(Node* src, int output_index, Node* dst, int input_index) { + if (output_index >= src->num_outputs()) { + return errors::InvalidArgument( + "Output ", output_index, " of node ", src->name(), + " does not exist. Node only has ", src->num_outputs(), " outputs."); + } + if (input_index >= dst->num_inputs()) { + return errors::InvalidArgument( + "Input ", input_index, " of node ", dst->name(), + " does not exist. Node only has ", dst->num_inputs(), " inputs."); + } + DataType src_out = src->output_type(output_index); DataType dst_in = dst->input_type(input_index); if (!TypesCompatible(dst_in, src_out)) { diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc index 3281f97457f58b..38e6eba13edde6 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc @@ -1994,6 +1994,12 @@ class ReorderCastLikeAndValuePreserving : public ArithmeticOptimizerStage { Status TrySimplify(NodeDef* consumer, string* simplified_node_name) override { NodeDef* producer; + + if (consumer->input_size() < 1) { + return errors::FailedPrecondition("Node ", simplified_node_name, + " lacks inputs"); + } + TF_RETURN_IF_ERROR(GetInputNode(consumer->input(0), &producer)); const bool producer_is_cast = IsCastLike(*producer); const bool can_optimize = @@ -2393,6 +2399,11 @@ class ReplaceMulWithSquare : public ArithmeticOptimizerStage { ~ReplaceMulWithSquare() override = default; bool IsSupported(const NodeDef* node) const override { + if (!node || node->input_size() < 2) { + // Invalid node + return false; + } + return IsAnyMul(*node) && node->input(0) == node->input(1); } diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc index fed600361370cf..7acad9e679b687 100644 --- a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc @@ -68,6 +68,12 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) const { // The output values of this node may be needed. return false; } + + if (node.input_size() < 1) { + // Node lacks input, is invalid + return false; + } + const NodeDef* input = node_map_->GetNode(NodeName(node.input(0))); CHECK(input != nullptr) << "node = " << node.name() << " input = " << node.input(0); diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 89f50c5d173436..c812b0f4983b36 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -5897,6 +5897,24 @@ tf_kernel_library( deps = STRING_DEPS, ) +tf_cc_test( + name = "as_string_op_test", + size = "small", + srcs = ["as_string_op_test.cc"], + deps = [ + ":as_string_op", + ":ops_testutil", + ":ops_util", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + ], +) + tf_kernel_library( name = "unicode_ops", prefix = "unicode_ops", diff --git a/tensorflow/core/kernels/as_string_op.cc b/tensorflow/core/kernels/as_string_op.cc index 8341909fbc8409..b9af976a654d99 100644 --- a/tensorflow/core/kernels/as_string_op.cc +++ b/tensorflow/core/kernels/as_string_op.cc @@ -65,9 +65,26 @@ class AsStringOp : public OpKernel { OP_REQUIRES(ctx, !(scientific && shortest), errors::InvalidArgument( "Cannot select both scientific and shortest notation")); + format_ = "%"; + if (!fill_string.empty()) { + switch (fill_string[0]) { + case ' ': + case '+': + case '-': + case '0': + case '#': + strings::Appendf(&format_, "%s", fill_string.c_str()); + break; + default: + bool fill_not_supported = true; + OP_REQUIRES(ctx, !fill_not_supported, + errors::InvalidArgument("Fill argument not supported: \"", + fill_string, "\"")); + } + } if (width > -1) { - strings::Appendf(&format_, "%s%d", fill_string.c_str(), width); + strings::Appendf(&format_, "%d", width); } if (precision > -1) { strings::Appendf(&format_, ".%d", precision); diff --git a/tensorflow/core/kernels/as_string_op_test.cc b/tensorflow/core/kernels/as_string_op_test.cc new file mode 100644 index 00000000000000..dff78e25e72025 --- /dev/null +++ b/tensorflow/core/kernels/as_string_op_test.cc @@ -0,0 +1,245 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/fake_input.h" +#include "tensorflow/core/framework/node_def_builder.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/kernels/ops_testutil.h" +#include "tensorflow/core/kernels/ops_util.h" +#include "tensorflow/core/lib/core/status_test_util.h" + +namespace tensorflow { +namespace { + +class AsStringGraphTest : public OpsTestBase { + protected: + Status Init(DataType input_type, const string& fill = "", int width = -1, + int precision = -1, bool scientific = false, + bool shortest = false) { + TF_CHECK_OK(NodeDefBuilder("op", "AsString") + .Input(FakeInput(input_type)) + .Attr("fill", fill) + .Attr("precision", precision) + .Attr("scientific", scientific) + .Attr("shortest", shortest) + .Attr("width", width) + .Finalize(node_def())); + return InitOp(); + } +}; + +TEST_F(AsStringGraphTest, Int8) { + TF_ASSERT_OK(Init(DT_INT8)); + + AddInputFromArray(TensorShape({3}), {-42, 0, 42}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_STRING, TensorShape({3})); + test::FillValues(&expected, {"-42", "0", "42"}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(AsStringGraphTest, Int64) { + TF_ASSERT_OK(Init(DT_INT64)); + + AddInputFromArray(TensorShape({3}), {-42, 0, 42}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_STRING, TensorShape({3})); + test::FillValues(&expected, {"-42", "0", "42"}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(AsStringGraphTest, FloatDefault) { + TF_ASSERT_OK(Init(DT_FLOAT)); + + AddInputFromArray(TensorShape({4}), {-42, 0, 3.14159, 42}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_STRING, TensorShape({4})); + test::FillValues( + &expected, {"-42.000000", "0.000000", "3.141590", "42.000000"}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(AsStringGraphTest, FloatScientific) { + TF_ASSERT_OK(Init(DT_FLOAT, /*fill=*/"", /*width=*/-1, /*precision=*/-1, + /*scientific=*/true)); + + AddInputFromArray(TensorShape({4}), {-42, 0, 3.14159, 42}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_STRING, TensorShape({4})); + test::FillValues(&expected, {"-4.200000e+01", "0.000000e+00", + "3.141590e+00", "4.200000e+01"}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(AsStringGraphTest, FloatShortest) { + TF_ASSERT_OK(Init(DT_FLOAT, /*fill=*/"", /*width=*/-1, /*precision=*/-1, + /*scientific=*/false, /*shortest=*/true)); + + AddInputFromArray(TensorShape({4}), {-42, 0, 3.14159, 42}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_STRING, TensorShape({4})); + test::FillValues(&expected, {"-42", "0", "3.14159", "42"}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(AsStringGraphTest, FloatPrecisionOnly) { + TF_ASSERT_OK(Init(DT_FLOAT, /*fill=*/"", /*width=*/-1, /*precision=*/2)); + + AddInputFromArray(TensorShape({4}), {-42, 0, 3.14159, 42}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_STRING, TensorShape({4})); + test::FillValues(&expected, {"-42.00", "0.00", "3.14", "42.00"}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(AsStringGraphTest, FloatWidthOnly) { + TF_ASSERT_OK(Init(DT_FLOAT, /*fill=*/"", /*width=*/5)); + + AddInputFromArray(TensorShape({4}), {-42, 0, 3.14159, 42}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_STRING, TensorShape({4})); + test::FillValues( + &expected, {"-42.000000", "0.000000", "3.141590", "42.000000"}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(AsStringGraphTest, Float_5_2_Format) { + TF_ASSERT_OK(Init(DT_FLOAT, /*fill=*/"", /*width=*/5, /*precision=*/2)); + + AddInputFromArray(TensorShape({4}), {-42, 0, 3.14159, 42}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_STRING, TensorShape({4})); + test::FillValues(&expected, {"-42.00", " 0.00", " 3.14", "42.00"}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(AsStringGraphTest, Complex) { + TF_ASSERT_OK(Init(DT_COMPLEX64, /*fill=*/"", /*width=*/5, /*precision=*/2)); + + AddInputFromArray(TensorShape({3}), {{-4, 2}, {0}, {3.14159, -1}}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_STRING, TensorShape({3})); + test::FillValues( + &expected, {"(-4.00, 2.00)", "( 0.00, 0.00)", "( 3.14,-1.00)"}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(AsStringGraphTest, Bool) { + TF_ASSERT_OK(Init(DT_BOOL)); + + AddInputFromArray(TensorShape({2}), {true, false}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_STRING, TensorShape({2})); + test::FillValues(&expected, {"true", "false"}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(AsStringGraphTest, String) { + Status s = Init(DT_STRING); + ASSERT_EQ(error::INVALID_ARGUMENT, s.code()); + ASSERT_TRUE(absl::StrContains( + s.error_message(), + "Value for attr 'T' of string is not in the list of allowed values")); +} + +TEST_F(AsStringGraphTest, OnlyOneOfScientificAndShortest) { + Status s = Init(DT_FLOAT, /*fill=*/"", /*width=*/-1, /*precision=*/-1, + /*scientific=*/true, /*shortest=*/true); + ASSERT_EQ(error::INVALID_ARGUMENT, s.code()); + ASSERT_TRUE( + absl::StrContains(s.error_message(), + "Cannot select both scientific and shortest notation")); +} + +TEST_F(AsStringGraphTest, NoShortestForNonFloat) { + Status s = Init(DT_INT32, /*fill=*/"", /*width=*/-1, /*precision=*/-1, + /*scientific=*/false, /*shortest=*/true); + ASSERT_EQ(error::INVALID_ARGUMENT, s.code()); + ASSERT_TRUE(absl::StrContains( + s.error_message(), + "scientific and shortest format not supported for datatype")); +} + +TEST_F(AsStringGraphTest, NoScientificForNonFloat) { + Status s = Init(DT_INT32, /*fill=*/"", /*width=*/-1, /*precision=*/-1, + /*scientific=*/true); + ASSERT_EQ(error::INVALID_ARGUMENT, s.code()); + ASSERT_TRUE(absl::StrContains( + s.error_message(), + "scientific and shortest format not supported for datatype")); +} + +TEST_F(AsStringGraphTest, NoPrecisionForNonFloat) { + Status s = Init(DT_INT32, /*fill=*/"", /*width=*/-1, /*precision=*/5); + ASSERT_EQ(error::INVALID_ARGUMENT, s.code()); + ASSERT_TRUE(absl::StrContains(s.error_message(), + "precision not supported for datatype")); +} + +TEST_F(AsStringGraphTest, LongFill) { + Status s = Init(DT_INT32, /*fill=*/"asdf"); + ASSERT_EQ(error::INVALID_ARGUMENT, s.code()); + ASSERT_TRUE(absl::StrContains(s.error_message(), + "Fill string must be one or fewer characters")); +} + +TEST_F(AsStringGraphTest, FillWithZero) { + TF_ASSERT_OK(Init(DT_INT64, /*fill=*/"0", /*width=*/4)); + + AddInputFromArray(TensorShape({3}), {-42, 0, 42}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_STRING, TensorShape({3})); + test::FillValues(&expected, {"-042", "0000", "0042"}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(AsStringGraphTest, FillWithSpace) { + TF_ASSERT_OK(Init(DT_INT64, /*fill=*/" ", /*width=*/4)); + + AddInputFromArray(TensorShape({3}), {-42, 0, 42}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_STRING, TensorShape({3})); + test::FillValues(&expected, {" -42", " 0", " 42"}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(AsStringGraphTest, FillWithChar1) { + TF_ASSERT_OK(Init(DT_INT64, /*fill=*/"-", /*width=*/4)); + + AddInputFromArray(TensorShape({3}), {-42, 0, 42}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_STRING, TensorShape({3})); + test::FillValues(&expected, {"-42 ", "0 ", "42 "}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(AsStringGraphTest, FillWithChar3) { + Status s = Init(DT_INT32, /*fill=*/"s"); + ASSERT_EQ(error::INVALID_ARGUMENT, s.code()); + ASSERT_TRUE( + absl::StrContains(s.error_message(), "Fill argument not supported")); +} + +TEST_F(AsStringGraphTest, FillWithChar4) { + Status s = Init(DT_INT32, /*fill=*/"n"); + ASSERT_EQ(error::INVALID_ARGUMENT, s.code()); + ASSERT_TRUE( + absl::StrContains(s.error_message(), "Fill argument not supported")); +} + +} // end namespace +} // end namespace tensorflow diff --git a/tensorflow/core/kernels/boosted_trees/prediction_ops.cc b/tensorflow/core/kernels/boosted_trees/prediction_ops.cc index 19be606f184939..e3a908d1b6b20d 100644 --- a/tensorflow/core/kernels/boosted_trees/prediction_ops.cc +++ b/tensorflow/core/kernels/boosted_trees/prediction_ops.cc @@ -121,7 +121,7 @@ class BoostedTreesTrainingPredictOp : public OpKernel { auto do_work = [&resource, &bucketized_features, &cached_tree_ids, &cached_node_ids, &output_partial_logits, &output_node_ids, latest_tree, - this](int32 start, int32 end) { + this](int64 start, int64 end) { for (int32 i = start; i < end; ++i) { int32 tree_id = cached_tree_ids(i); int32 node_id = cached_node_ids(i); @@ -237,7 +237,7 @@ class BoostedTreesPredictOp : public OpKernel { const int32 last_tree = resource->num_trees() - 1; auto do_work = [&resource, &bucketized_features, &output_logits, last_tree, - this](int32 start, int32 end) { + this](int64 start, int64 end) { for (int32 i = start; i < end; ++i) { std::vector tree_logits(logits_dimension_, 0.0); int32 tree_id = 0; @@ -340,7 +340,7 @@ class BoostedTreesExampleDebugOutputsOp : public OpKernel { // path. Note: feature_ids has one less value than logits_path because the // first value of each logit path will be the bias. auto do_work = [&resource, &bucketized_features, &output_debug_info, - last_tree](int32 start, int32 end) { + last_tree](int64 start, int64 end) { for (int32 i = start; i < end; ++i) { // Proto to store debug outputs, per example. boosted_trees::DebugOutput example_debug_info; diff --git a/tensorflow/core/kernels/conv_grad_filter_ops.cc b/tensorflow/core/kernels/conv_grad_filter_ops.cc index f9bf64f2df3a91..220fc3629df476 100644 --- a/tensorflow/core/kernels/conv_grad_filter_ops.cc +++ b/tensorflow/core/kernels/conv_grad_filter_ops.cc @@ -496,6 +496,14 @@ class Conv2DCustomBackpropFilterOp : public OpKernel { const int filter_total_size = dims.spatial_dims[0].filter_size * dims.spatial_dims[1].filter_size * dims.in_depth; + OP_REQUIRES( + context, + filter_total_size * dims.out_depth == filter_backprop->NumElements(), + errors::InvalidArgument( + "filter_size does not have enough elements, requested ", + filter_total_size * dims.out_depth, ", got ", + filter_backprop->NumElements())); + // The output image size is the spatial size of the output. const int output_image_size = dims.spatial_dims[0].output_size * dims.spatial_dims[1].output_size; @@ -519,6 +527,11 @@ class Conv2DCustomBackpropFilterOp : public OpKernel { const size_t work_unit_size = size_A + size_B + size_C; + OP_REQUIRES( + context, work_unit_size != 0, + errors::InvalidArgument( + "Work size for convolution would be 0, which is not acceptable")); + const size_t shard_size = (target_working_set_size + work_unit_size - 1) / work_unit_size; diff --git a/tensorflow/core/kernels/conv_grad_input_ops.cc b/tensorflow/core/kernels/conv_grad_input_ops.cc index be5d821fc32fa6..9fb79fbac68eb0 100644 --- a/tensorflow/core/kernels/conv_grad_input_ops.cc +++ b/tensorflow/core/kernels/conv_grad_input_ops.cc @@ -673,6 +673,11 @@ class Conv2DCustomBackpropInputOp : public OpKernel { dims.batch_size == 1 || thread_work_unit_size >= min_thread_work_unit_size; + OP_REQUIRES( + context, work_unit_size > 0, + errors::InvalidArgument("input, filter_sizes and out_backprop tensors " + "must all have at least 1 element")); + const size_t shard_size = use_parallel_contraction ? 1 diff --git a/tensorflow/core/kernels/conv_grad_ops_3d.cc b/tensorflow/core/kernels/conv_grad_ops_3d.cc index bc6c64963addc4..152f33fe472f7e 100644 --- a/tensorflow/core/kernels/conv_grad_ops_3d.cc +++ b/tensorflow/core/kernels/conv_grad_ops_3d.cc @@ -238,6 +238,28 @@ class Conv3DBackpropInputOp : public OpKernel { input_shape = context->input(0).shape(); } + OP_REQUIRES(context, input_shape.dims() == 5, + errors::InvalidArgument("input tensor must have 5 dimensions")); + OP_REQUIRES( + context, filter_shape.dims() == 5, + errors::InvalidArgument("filter_sizes tensor must have 5 dimensions")); + OP_REQUIRES( + context, out_backprop_shape.dims() == 5, + errors::InvalidArgument("out_backprop tensor must have 5 dimensions")); + OP_REQUIRES( + context, input_shape.dim_size(4) == filter_shape.dim_size(3), + errors::InvalidArgument("input and filter_sizes must have the same " + "number of channels. Got ", + input_shape.dim_size(4), " for input and ", + filter_shape.dim_size(3), " for filter_sizes")); + OP_REQUIRES( + context, out_backprop_shape.dim_size(4) == filter_shape.dim_size(4), + errors::InvalidArgument("out_backprop and filter_sizes must have the " + "same number of channels. Got ", + out_backprop_shape.dim_size(4), + " for out_backprop and ", + filter_shape.dim_size(4), " for filter_sizes")); + ConvBackpropDimensions dims; OP_REQUIRES_OK(context, ConvBackpropComputeDimensions( "Conv3DBackpropInputOp", /*num_spatial_dims=*/3, @@ -345,6 +367,28 @@ class Conv3DCustomBackpropInputOp : public OpKernel { input_shape = context->input(0).shape(); } + OP_REQUIRES(context, input_shape.dims() == 5, + errors::InvalidArgument("input tensor must have 5 dimensions")); + OP_REQUIRES( + context, filter_shape.dims() == 5, + errors::InvalidArgument("filter_sizes tensor must have 5 dimensions")); + OP_REQUIRES( + context, out_backprop_shape.dims() == 5, + errors::InvalidArgument("out_backprop tensor must have 5 dimensions")); + OP_REQUIRES( + context, input_shape.dim_size(4) == filter_shape.dim_size(3), + errors::InvalidArgument("input and filter_sizes must have the same " + "number of channels. Got ", + input_shape.dim_size(4), " for input and ", + filter_shape.dim_size(3), " for filter_sizes")); + OP_REQUIRES( + context, out_backprop_shape.dim_size(4) == filter_shape.dim_size(4), + errors::InvalidArgument("out_backprop and filter_sizes must have the " + "same number of channels. Got ", + out_backprop_shape.dim_size(4), + " for out_backprop and ", + filter_shape.dim_size(4), " for filter_sizes")); + ConvBackpropDimensions dims; OP_REQUIRES_OK(context, ConvBackpropComputeDimensions( "Conv3DBackpropInputOp", /*num_spatial_dims=*/3, @@ -415,6 +459,11 @@ class Conv3DCustomBackpropInputOp : public OpKernel { // contraction compared to sharding and matmuls. const bool use_parallel_contraction = dims.batch_size == 1; + OP_REQUIRES( + context, work_unit_size > 0, + errors::InvalidArgument("input, filter_sizes and out_backprop tensors " + "must all have at least 1 element")); + const size_t shard_size = use_parallel_contraction ? 1 @@ -695,6 +744,28 @@ class Conv3DBackpropFilterOp : public OpKernel { filter_shape = context->input(1).shape(); } + OP_REQUIRES(context, input_shape.dims() == 5, + errors::InvalidArgument("input tensor must have 5 dimensions")); + OP_REQUIRES( + context, filter_shape.dims() == 5, + errors::InvalidArgument("filter_sizes tensor must have 5 dimensions")); + OP_REQUIRES( + context, out_backprop_shape.dims() == 5, + errors::InvalidArgument("out_backprop tensor must have 5 dimensions")); + OP_REQUIRES( + context, input_shape.dim_size(4) == filter_shape.dim_size(3), + errors::InvalidArgument("input and filter_sizes must have the same " + "number of channels. Got ", + input_shape.dim_size(4), " for input and ", + filter_shape.dim_size(3), " for filter_sizes")); + OP_REQUIRES( + context, out_backprop_shape.dim_size(4) == filter_shape.dim_size(4), + errors::InvalidArgument("out_backprop and filter_sizes must have the " + "same number of channels. Got ", + out_backprop_shape.dim_size(4), + " for out_backprop and ", + filter_shape.dim_size(4), " for filter_sizes")); + ConvBackpropDimensions dims; OP_REQUIRES_OK(context, ConvBackpropComputeDimensions( @@ -807,6 +878,28 @@ class Conv3DCustomBackpropFilterOp : public OpKernel { filter_shape = context->input(1).shape(); } + OP_REQUIRES(context, input_shape.dims() == 5, + errors::InvalidArgument("input tensor must have 5 dimensions")); + OP_REQUIRES( + context, filter_shape.dims() == 5, + errors::InvalidArgument("filter_sizes tensor must have 5 dimensions")); + OP_REQUIRES( + context, out_backprop_shape.dims() == 5, + errors::InvalidArgument("out_backprop tensor must have 5 dimensions")); + OP_REQUIRES( + context, input_shape.dim_size(4) == filter_shape.dim_size(3), + errors::InvalidArgument("input and filter_sizes must have the same " + "number of channels. Got ", + input_shape.dim_size(4), " for input and ", + filter_shape.dim_size(3), " for filter_sizes")); + OP_REQUIRES( + context, out_backprop_shape.dim_size(4) == filter_shape.dim_size(4), + errors::InvalidArgument("out_backprop and filter_sizes must have the " + "same number of channels. Got ", + out_backprop_shape.dim_size(4), + " for out_backprop and ", + filter_shape.dim_size(4), " for filter_sizes")); + ConvBackpropDimensions dims; OP_REQUIRES_OK(context, ConvBackpropComputeDimensions( @@ -879,6 +972,11 @@ class Conv3DCustomBackpropFilterOp : public OpKernel { const int64 work_unit_size = size_A + size_B + size_C; + OP_REQUIRES( + context, work_unit_size > 0, + errors::InvalidArgument("input, filter_sizes and out_backprop tensors " + "must all have at least 1 element")); + const size_t shard_size = (target_working_set_size + work_unit_size - 1) / work_unit_size; diff --git a/tensorflow/core/kernels/conv_grad_shape_utils.cc b/tensorflow/core/kernels/conv_grad_shape_utils.cc index 7857257658fcff..ce90e866342311 100644 --- a/tensorflow/core/kernels/conv_grad_shape_utils.cc +++ b/tensorflow/core/kernels/conv_grad_shape_utils.cc @@ -126,6 +126,10 @@ Status ConvBackpropComputeDimensionsV2( // dimensions of the filter Tensor. VLOG(2) << "input vs filter_in depth " << dims->in_depth << " " << filter_shape.dim_size(num_dims - 2); + if (filter_shape.dim_size(num_dims - 2) <= 0) { + return errors ::InvalidArgument( + label, ": filter depth must be strictly greated than zero"); + } if (dims->in_depth % filter_shape.dim_size(num_dims - 2)) { return errors::InvalidArgument( label, ": input depth must be evenly divisible by filter depth"); diff --git a/tensorflow/core/kernels/conv_ops.cc b/tensorflow/core/kernels/conv_ops.cc index d265e9d8f8be95..396a9266e08ee3 100644 --- a/tensorflow/core/kernels/conv_ops.cc +++ b/tensorflow/core/kernels/conv_ops.cc @@ -424,6 +424,9 @@ Status ComputeConv2DDimension(const Conv2DParameters& params, errors::InvalidArgument("Patch depth too large")); const int in_depth = static_cast(in_depth_raw); const int patch_depth = static_cast(patch_depth_raw); + TF_REQUIRES(patch_depth > 0, + errors::InvalidArgument( + "filter depth must be stricly positive, got ", patch_depth)); TF_REQUIRES(in_depth % patch_depth == 0, errors::InvalidArgument( "input depth must be evenly divisible by filter depth: ", diff --git a/tensorflow/core/kernels/conv_ops_3d.cc b/tensorflow/core/kernels/conv_ops_3d.cc index e9e11aebf6191e..5d2ad14c33b3ba 100644 --- a/tensorflow/core/kernels/conv_ops_3d.cc +++ b/tensorflow/core/kernels/conv_ops_3d.cc @@ -67,6 +67,11 @@ struct LaunchConvOp { errors::InvalidArgument("CPU implementation of Conv3D " "currently only supports dilated rates " "of 1.")); + OP_REQUIRES(context, filter.dim_size(3) == input.dim_size(input.dims() - 1), + errors::InvalidArgument( + "Number of channels in filter (", filter.dim_size(3), + ") must match last dimension of input (", + input.dim_size(input.dims() - 1), ")")); functor::CuboidConvolution()( context->eigen_device(), output->tensor(), input.tensor(), filter.tensor(), strides[2], strides[1], @@ -140,6 +145,8 @@ class Conv3DOp : public BinaryOp { const int64 filter_depth = filter.dim_size(3); const int64 out_depth = filter.dim_size(4); + OP_REQUIRES(context, filter_depth != 0, + errors::InvalidArgument("filter_depth must be non-zero")); OP_REQUIRES(context, in_depth % filter_depth == 0, errors::InvalidArgument( "Input depth must be evenly divisible by filter depth: ", diff --git a/tensorflow/core/kernels/ctc_decoder_ops.cc b/tensorflow/core/kernels/ctc_decoder_ops.cc index 517612eecb6057..60b10107537408 100644 --- a/tensorflow/core/kernels/ctc_decoder_ops.cc +++ b/tensorflow/core/kernels/ctc_decoder_ops.cc @@ -70,6 +70,9 @@ class CTCDecodeHelper { if (inputs_shape.dims() != 3) { return errors::InvalidArgument("inputs is not a 3-Tensor"); } + if (inputs_shape.num_elements() == 0) { + return errors::InvalidArgument("inputs must not be empty"); + } const int64 max_time = inputs_shape.dim_size(0); const int64 batch_size = inputs_shape.dim_size(1); @@ -224,6 +227,8 @@ class CTCGreedyDecoderOp : public OpKernel { int prev_indices = -1; for (int t = 0; t < seq_len_t(b); ++t) { int max_class_indices; + OP_REQUIRES(ctx, input_list_t[t].dimension(1) > 0, + errors::InvalidArgument("Invalid input dimensions.")); log_prob_t(b, 0) += -RowMax(input_list_t[t], b, &max_class_indices); if (max_class_indices != blank_index && diff --git a/tensorflow/core/kernels/ctc_loss_op.cc b/tensorflow/core/kernels/ctc_loss_op.cc index 6358e82fdda853..ca505e1db93145 100644 --- a/tensorflow/core/kernels/ctc_loss_op.cc +++ b/tensorflow/core/kernels/ctc_loss_op.cc @@ -100,11 +100,18 @@ class CTCLossOp : public OpKernel { errors::InvalidArgument("sequence_length is not a vector")); OP_REQUIRES(ctx, TensorShapeUtils::IsMatrix(labels_indices->shape()), errors::InvalidArgument("labels_indices is not a matrix")); + OP_REQUIRES(ctx, labels_indices->dim_size(1) > 1, + errors::InvalidArgument( + "labels_indices second dimension must be >= 1. Received ", + labels_indices->dim_size(1))); OP_REQUIRES(ctx, TensorShapeUtils::IsVector(labels_values->shape()), errors::InvalidArgument("labels_values is not a vector")); const TensorShape& inputs_shape = inputs->shape(); const int64 max_time = inputs_shape.dim_size(0); + OP_REQUIRES(ctx, max_time != 0, + errors::InvalidArgument( + "Max time or first dimension of input cannot be 0.")); const int64 batch_size = inputs_shape.dim_size(1); const int64 num_classes_raw = inputs_shape.dim_size(2); OP_REQUIRES( diff --git a/tensorflow/core/kernels/data_format_ops.cc b/tensorflow/core/kernels/data_format_ops.cc index 0b4241dbb9312c..109e6e32d5dd91 100644 --- a/tensorflow/core/kernels/data_format_ops.cc +++ b/tensorflow/core/kernels/data_format_ops.cc @@ -18,16 +18,52 @@ limitations under the License. #define EIGEN_USE_THREADS #include "tensorflow/core/kernels/data_format_ops.h" + +#include + #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/platform/errors.h" namespace tensorflow { typedef Eigen::ThreadPoolDevice CPUDevice; typedef Eigen::GpuDevice GPUDevice; +// Ensure that `src` and `dst` define a valid permutation. +// Ops defined in this file assume that user specifies a permutation via two +// string attributes. This check validates that these attributes properly define +// it to prevent security vulnerabilities. +static bool IsValidPermutation(const std::string& src, const std::string& dst) { + if (src.size() != dst.size()) { + return false; + } + + std::map characters; + + // Every character in `src` must be present only once + for (const auto c : src) { + if (characters[c]) { + return false; + } + characters[c] = true; + } + + // Every character in `dst` must show up in `src` exactly once + for (const auto c : dst) { + if (!characters[c]) { + return false; + } + characters[c] = false; + } + + // At this point, characters[] has been switched to true and false exactly + // once for all character in `src` (and `dst`) so we have a valid permutation + return true; +} + template class DataFormatDimMapOp : public OpKernel { public: @@ -37,15 +73,20 @@ class DataFormatDimMapOp : public OpKernel { OP_REQUIRES_OK(context, context->GetAttr("src_format", &src_format)); string dst_format; OP_REQUIRES_OK(context, context->GetAttr("dst_format", &dst_format)); - OP_REQUIRES(context, src_format.size() == 4, - errors::InvalidArgument(strings::StrCat( - "Source format must of length 4, received src_format = ", - src_format))); + OP_REQUIRES(context, src_format.size() == 4 || src_format.size() == 5, + errors::InvalidArgument( + "Source format must be of length 4 or 5, received " + "src_format = ", + src_format)); + OP_REQUIRES(context, dst_format.size() == 4 || dst_format.size() == 5, + errors::InvalidArgument("Destination format must be of length " + "4 or 5, received dst_format = ", + dst_format)); OP_REQUIRES( - context, dst_format.size() == 4, - errors::InvalidArgument(strings::StrCat( - "Destination format must of length 4, received dst_format = ", - dst_format))); + context, IsValidPermutation(src_format, dst_format), + errors::InvalidArgument( + "Destination and source format must determine a permutation, got ", + src_format, " and ", dst_format)); dst_idx_ = Tensor(DT_INT32, {static_cast(src_format.size())}); for (int i = 0; i < src_format.size(); ++i) { for (int j = 0; j < dst_format.size(); ++j) { @@ -77,8 +118,22 @@ class DataFormatVecPermuteOp : public OpKernel { : OpKernel(context) { string src_format; OP_REQUIRES_OK(context, context->GetAttr("src_format", &src_format)); + OP_REQUIRES(context, src_format.size() == 4 || src_format.size() == 5, + errors::InvalidArgument( + "Source format must be of length 4 or 5, received " + "src_format = ", + src_format)); string dst_format; OP_REQUIRES_OK(context, context->GetAttr("dst_format", &dst_format)); + OP_REQUIRES(context, dst_format.size() == 4 || dst_format.size() == 5, + errors::InvalidArgument("Destination format must be of length " + "4 or 5, received dst_format = ", + dst_format)); + OP_REQUIRES( + context, IsValidPermutation(src_format, dst_format), + errors::InvalidArgument( + "Destination and source format must determine a permutation, got ", + src_format, " and ", dst_format)); src_format_ = src_format; dst_format_ = dst_format; } @@ -112,6 +167,24 @@ class DataFormatVecPermuteOp : public OpKernel { context->allocate_output(0, input.shape(), &output)); // Support 1D and 2D cases. Eigen::DSizes dst_idx; + string src_format_str = src_format_; + string dst_format_str = dst_format_; + if (input.dim_size(0) == 2) { + // If the input is a vector of size 2, treat the two elements as spatial + // dimensions. + auto keep_only_spatial_dimensions = [](string* format_str) -> void { + auto new_end = std::remove_if( + format_str->begin(), format_str->end(), + [](const char dim) { return dim != 'H' && dim != 'W'; }); + format_str->erase(new_end, format_str->end()); + }; + keep_only_spatial_dimensions(&src_format_str); + keep_only_spatial_dimensions(&dst_format_str); + OP_REQUIRES(context, + src_format_str.size() == 2 && dst_format_str.size() == 2, + errors::InvalidArgument( + "Format specifier must contain H and W for 2D case")); + } ComputeDstIndex(input.dims(), &dst_idx); functor::DataFormatVecPermute()(context->eigen_device(), diff --git a/tensorflow/core/kernels/decode_padded_raw_op.cc b/tensorflow/core/kernels/decode_padded_raw_op.cc index 12e8ec6aff0d41..ca7c7104b442d2 100644 --- a/tensorflow/core/kernels/decode_padded_raw_op.cc +++ b/tensorflow/core/kernels/decode_padded_raw_op.cc @@ -83,14 +83,13 @@ class DecodePaddedRawOp : public OpKernel { // can copy the memory directly. if (!convert_data_endianness_ || sizeof(T) == 1) { for (int64 i = 0; i < flat_in.size(); ++i) { - const T* in_data = reinterpret_cast(flat_in(i).data()); - - if (flat_in(i).size() > fixed_length) { - memcpy(out_data, in_data, fixed_length); - } else { - memcpy(out_data, in_data, flat_in(i).size()); - } - out_data += fixed_length; + const auto to_copy = + std::min(flat_in(i).size(), static_cast(fixed_length)); + memcpy(out_data, flat_in(i).data(), to_copy); + // Note: increase out_data by width since it's already of type T* so + // each shift amount is implicitly multiplied by sizeof(T) according to + // pointer arithmetic rules. + out_data += width; } } else { // Otherwise, the data is not in the host's byte order, and rather than a @@ -105,7 +104,10 @@ class DecodePaddedRawOp : public OpKernel { p_in += sizeof(T), p_out += sizeof(T)) { std::reverse_copy(p_in, p_in + sizeof(T), p_out); } - out_data += fixed_length; + // Note: increase out_data by width since it's already of type T* so + // each shift amount is implicitly multiplied by sizeof(T) according to + // pointer arithmetic rules. + out_data += width; } } } diff --git a/tensorflow/core/kernels/dequantize_op.cc b/tensorflow/core/kernels/dequantize_op.cc index 0f5a7019b1f93c..fe748aaf1943ee 100644 --- a/tensorflow/core/kernels/dequantize_op.cc +++ b/tensorflow/core/kernels/dequantize_op.cc @@ -96,6 +96,18 @@ class DequantizeOp : public OpKernel { if (axis_ > -1) { num_slices = input.dim_size(axis_); } + OP_REQUIRES(ctx, input_min_tensor.NumElements() == num_slices, + errors::InvalidArgument( + "input_min_tensor must have as many elements as input on " + "the dequantization axis (", + axis_, "), got ", input_min_tensor.NumElements(), + ", expected ", num_slices)); + OP_REQUIRES(ctx, input_max_tensor.NumElements() == num_slices, + errors::InvalidArgument( + "input_max_tensor must have as many elements as input on " + "the dequantization axis (", + axis_, "), got ", input_max_tensor.NumElements(), + ", expected ", num_slices)); Tensor* output = nullptr; Tensor float_output = tensorflow::Tensor(DT_FLOAT, input.shape()); diff --git a/tensorflow/core/kernels/dilation_ops.cc b/tensorflow/core/kernels/dilation_ops.cc index f2e7b8a857a0f4..fdb2716eff9284 100644 --- a/tensorflow/core/kernels/dilation_ops.cc +++ b/tensorflow/core/kernels/dilation_ops.cc @@ -130,6 +130,7 @@ class DilationOp : public OpKernel { ParseSizes(context, strides_, rates_, padding_, &stride_rows, &stride_cols, &rate_rows, &rate_cols, &pad_top, &pad_left, &out_rows, &out_cols); + if (!context->status().ok()) return; // Output tensor is of the following dimensions: // [ batch, out_rows, out_cols, depth ] @@ -229,6 +230,7 @@ class DilationBackpropInputOp : public OpKernel { ParseSizes(context, strides_, rates_, padding_, &stride_rows, &stride_cols, &rate_rows, &rate_cols, &pad_top, &pad_left, &out_rows, &out_cols); + if (!context->status().ok()) return; // Verify that the incoming gradient tensor has the expected size // [ batch, out_rows, out_cols, depth ] @@ -318,8 +320,10 @@ struct DilationBackpropInput { } } } - in_backprop(b, h_in_max, w_in_max, d) += - out_backprop(b, h_out, w_out, d); + if (h_in_max < input_rows && w_in_max < input_cols) { + in_backprop(b, h_in_max, w_in_max, d) += + out_backprop(b, h_out, w_out, d); + } } } } @@ -349,6 +353,7 @@ class DilationBackpropFilterOp : public OpKernel { ParseSizes(context, strides_, rates_, padding_, &stride_rows, &stride_cols, &rate_rows, &rate_cols, &pad_top, &pad_left, &out_rows, &out_cols); + if (!context->status().ok()) return; // Verify that the incoming gradient tensor has the expected size // [ batch, out_rows, out_cols, depth ] @@ -438,8 +443,10 @@ struct DilationBackpropFilter { } } } - filter_backprop(h_max, w_max, d) += - out_backprop(b, h_out, w_out, d); + if (h_max < filter_rows && w_max < filter_cols) { + filter_backprop(h_max, w_max, d) += + out_backprop(b, h_out, w_out, d); + } } } } diff --git a/tensorflow/core/kernels/draw_bounding_box_op.cc b/tensorflow/core/kernels/draw_bounding_box_op.cc index 30de99b7d560a2..926ea368a58ba8 100644 --- a/tensorflow/core/kernels/draw_bounding_box_op.cc +++ b/tensorflow/core/kernels/draw_bounding_box_op.cc @@ -73,6 +73,12 @@ class DrawBoundingBoxesOp : public OpKernel { errors::InvalidArgument("Channel depth should be either 1 (GRY), " "3 (RGB), or 4 (RGBA)")); + OP_REQUIRES( + context, boxes.dim_size(2) == 4, + errors::InvalidArgument( + "The size of the third dimension of the box must be 4. Received: ", + boxes.dim_size(2))); + const int64 batch_size = images.dim_size(0); const int64 height = images.dim_size(1); const int64 width = images.dim_size(2); @@ -147,22 +153,46 @@ class DrawBoundingBoxesOp : public OpKernel { // At this point, {min,max}_box_{row,col}_clamp are inside the // image. - CHECK_GE(min_box_row_clamp, 0); - CHECK_GE(max_box_row_clamp, 0); - CHECK_LT(min_box_row_clamp, height); - CHECK_LT(max_box_row_clamp, height); - CHECK_GE(min_box_col_clamp, 0); - CHECK_GE(max_box_col_clamp, 0); - CHECK_LT(min_box_col_clamp, width); - CHECK_LT(max_box_col_clamp, width); + OP_REQUIRES( + context, min_box_row_clamp >= 0, + errors::InvalidArgument("Min box row clamp is less than 0.")); + OP_REQUIRES( + context, max_box_row_clamp >= 0, + errors::InvalidArgument("Max box row clamp is less than 0.")); + OP_REQUIRES(context, min_box_row_clamp <= height, + errors::InvalidArgument( + "Min box row clamp is greater than height.")); + OP_REQUIRES(context, max_box_row_clamp <= height, + errors::InvalidArgument( + "Max box row clamp is greater than height.")); + + OP_REQUIRES( + context, min_box_col_clamp >= 0, + errors::InvalidArgument("Min box col clamp is less than 0.")); + OP_REQUIRES( + context, max_box_col_clamp >= 0, + errors::InvalidArgument("Max box col clamp is less than 0.")); + OP_REQUIRES(context, min_box_col_clamp <= width, + errors::InvalidArgument( + "Min box col clamp is greater than width.")); + OP_REQUIRES(context, max_box_col_clamp <= width, + errors::InvalidArgument( + "Max box col clamp is greater than width.")); // At this point, the min_box_row and min_box_col are either // in the image or above/left of it, and max_box_row and // max_box_col are either in the image or below/right or it. - CHECK_LT(min_box_row, height); - CHECK_GE(max_box_row, 0); - CHECK_LT(min_box_col, width); - CHECK_GE(max_box_col, 0); + + OP_REQUIRES( + context, min_box_row <= height, + errors::InvalidArgument("Min box row is greater than height.")); + OP_REQUIRES(context, max_box_row >= 0, + errors::InvalidArgument("Max box row is less than 0.")); + OP_REQUIRES( + context, min_box_col <= width, + errors::InvalidArgument("Min box col is greater than width.")); + OP_REQUIRES(context, max_box_col >= 0, + errors::InvalidArgument("Max box col is less than 0.")); // Draw top line. if (min_box_row >= 0) { diff --git a/tensorflow/core/kernels/edit_distance_op.cc b/tensorflow/core/kernels/edit_distance_op.cc index 4aecdc9e414d36..386a1af08409f6 100644 --- a/tensorflow/core/kernels/edit_distance_op.cc +++ b/tensorflow/core/kernels/edit_distance_op.cc @@ -64,6 +64,12 @@ Status ValidateShapes(OpKernelContext* ctx, const Tensor& hypothesis_indices, return errors::InvalidArgument( "truth_shape should be a vector, but got shape: ", truth_shape.shape().DebugString()); + if (hypothesis_values.NumElements() != hypothesis_indices.dim_size(0)) + return errors::InvalidArgument( + "Expected hypothesis_values.NumElements == " + "#rows(hypothesis_indices), their shapes are: ", + hypothesis_values.shape().DebugString(), " and ", + hypothesis_indices.shape().DebugString()); if (hypothesis_shape.NumElements() != hypothesis_indices.dim_size(1)) return errors::InvalidArgument( "Expected hypothesis_shape.NumElements == " @@ -75,6 +81,12 @@ Status ValidateShapes(OpKernelContext* ctx, const Tensor& hypothesis_indices, "Input SparseTensors must have rank at least 2, but truth_shape " "rank is: ", truth_shape.NumElements()); + if (truth_values.NumElements() != truth_indices.dim_size(0)) + return errors::InvalidArgument( + "Expected truth_values.NumElements == " + "#rows(truth_indices), their shapes are: ", + truth_values.shape().DebugString(), " and ", + truth_indices.shape().DebugString()); if (truth_shape.NumElements() != truth_indices.dim_size(1)) return errors::InvalidArgument( "Expected truth_shape.NumElements == " @@ -153,6 +165,11 @@ class EditDistanceOp : public OpKernel { output_shape.AddDim(std::max(hypothesis_st_shape.dim_size(d), truth_st_shape.dim_size(d))); } + const auto output_elements = output_shape.num_elements(); + OP_REQUIRES( + ctx, output_elements > 0, + errors::InvalidArgument("Got output shape ", output_shape.DebugString(), + " which has 0 elements")); Tensor* output = nullptr; OP_REQUIRES_OK(ctx, ctx->allocate_output("output", output_shape, &output)); @@ -185,6 +202,12 @@ class EditDistanceOp : public OpKernel { if (g_truth == g_hypothesis) { auto loc = std::inner_product(g_truth.begin(), g_truth.end(), output_strides.begin(), int64{0}); + OP_REQUIRES( + ctx, loc < output_elements, + errors::Internal("Got an inner product ", loc, + " which would require in writing to outside of " + "the buffer for the output tensor (max elements ", + output_elements, ")")); output_t(loc) = gtl::LevenshteinDistance(truth_seq, hypothesis_seq, cmp); if (normalize_) output_t(loc) /= truth_seq.size(); @@ -194,6 +217,12 @@ class EditDistanceOp : public OpKernel { } else if (g_truth > g_hypothesis) { // zero-length truth auto loc = std::inner_product(g_hypothesis.begin(), g_hypothesis.end(), output_strides.begin(), int64{0}); + OP_REQUIRES( + ctx, loc < output_elements, + errors::Internal("Got an inner product ", loc, + " which would require in writing to outside of " + "the buffer for the output tensor (max elements ", + output_elements, ")")); output_t(loc) = hypothesis_seq.size(); if (normalize_ && output_t(loc) != 0.0f) { output_t(loc) = std::numeric_limits::infinity(); @@ -202,6 +231,12 @@ class EditDistanceOp : public OpKernel { } else { // zero-length hypothesis auto loc = std::inner_product(g_truth.begin(), g_truth.end(), output_strides.begin(), int64{0}); + OP_REQUIRES( + ctx, loc < output_elements, + errors::Internal("Got an inner product ", loc, + " which would require in writing to outside of " + "the buffer for the output tensor (max elements ", + output_elements, ")")); output_t(loc) = (normalize_) ? 1.0 : truth_seq.size(); ++truth_iter; } @@ -212,6 +247,12 @@ class EditDistanceOp : public OpKernel { auto hypothesis_seq = hypothesis_j.values(); auto loc = std::inner_product(g_hypothesis.begin(), g_hypothesis.end(), output_strides.begin(), int64{0}); + OP_REQUIRES( + ctx, loc < output_elements, + errors::Internal("Got an inner product ", loc, + " which would require in writing to outside of the " + "buffer for the output tensor (max elements ", + output_elements, ")")); output_t(loc) = hypothesis_seq.size(); if (normalize_ && output_t(loc) != 0.0f) { output_t(loc) = std::numeric_limits::infinity(); @@ -224,6 +265,12 @@ class EditDistanceOp : public OpKernel { auto truth_seq = truth_i.values(); auto loc = std::inner_product(g_truth.begin(), g_truth.end(), output_strides.begin(), int64{0}); + OP_REQUIRES( + ctx, loc < output_elements, + errors::Internal("Got an inner product ", loc, + " which would require in writing to outside of the " + "buffer for the output tensor (max elements ", + output_elements, ")")); output_t(loc) = (normalize_) ? 1.0 : truth_seq.size(); ++truth_iter; } diff --git a/tensorflow/core/kernels/encode_png_op.cc b/tensorflow/core/kernels/encode_png_op.cc index 8dbe1d377df5c6..09bcdbe5e3db0b 100644 --- a/tensorflow/core/kernels/encode_png_op.cc +++ b/tensorflow/core/kernels/encode_png_op.cc @@ -54,6 +54,8 @@ class EncodePngOp : public OpKernel { OP_REQUIRES(context, image.dims() == 3, errors::InvalidArgument("image must be 3-dimensional", image.shape().DebugString())); + OP_REQUIRES(context, image.NumElements() > 0, + errors::Internal("Invalid image provided.")); OP_REQUIRES( context, FastBoundsCheck(image.NumElements(), std::numeric_limits::max()), diff --git a/tensorflow/core/kernels/fft_ops.cc b/tensorflow/core/kernels/fft_ops.cc index 058435948394c5..603f9383616399 100644 --- a/tensorflow/core/kernels/fft_ops.cc +++ b/tensorflow/core/kernels/fft_ops.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include "tensorflow/core/platform/errors.h" #define EIGEN_USE_THREADS // See docs in ../ops/fft_ops.cc. @@ -221,6 +222,9 @@ class FFTCPU : public FFTBase { input_slice_sizes[i] = fft_shape[i - 1]; temp_shape.AddDim(fft_shape[i - 1]); } + OP_REQUIRES(ctx, temp_shape.num_elements() > 0, + errors::InvalidArgument("Obtained a FFT shape of 0 elements: ", + temp_shape.DebugString())); auto output = out->flat_inner_dims(); const Eigen::DSizes zero_start_indices; @@ -261,6 +265,9 @@ class FFTCPU : public FFTBase { i == FFTRank ? fft_shape[i - 1] / 2 + 1 : fft_shape[i - 1]; full_fft_shape.AddDim(fft_shape[i - 1]); } + OP_REQUIRES(ctx, full_fft_shape.num_elements() > 0, + errors::InvalidArgument("Obtained a FFT shape of 0 elements: ", + full_fft_shape.DebugString())); Tensor temp; OP_REQUIRES_OK(ctx, ctx->allocate_temp(DataTypeToEnum::v(), diff --git a/tensorflow/core/kernels/fractional_avg_pool_op.cc b/tensorflow/core/kernels/fractional_avg_pool_op.cc index dfc2382624e3fa..0452638a066795 100644 --- a/tensorflow/core/kernels/fractional_avg_pool_op.cc +++ b/tensorflow/core/kernels/fractional_avg_pool_op.cc @@ -80,6 +80,10 @@ class FractionalAvgPoolOp : public OpKernel { std::vector output_size(tensor_in_and_out_dims); for (int i = 0; i < tensor_in_and_out_dims; ++i) { input_size[i] = tensor_in.dim_size(i); + OP_REQUIRES( + context, pooling_ratio_[i] <= input_size[i], + errors::InvalidArgument( + "Pooling ratio cannot be bigger than input tensor dim size.")); } // Output size. for (int i = 0; i < tensor_in_and_out_dims; ++i) { @@ -246,6 +250,19 @@ class FractionalAvgPoolGradOp : public OpKernel { const int64 out_cols = out_backprop.dim_size(2); const int64 out_depth = out_backprop.dim_size(3); + OP_REQUIRES(context, row_seq_tensor.NumElements() > out_rows, + errors::InvalidArgument("Given out_backprop shape ", + out_backprop.shape().DebugString(), + ", row_seq_tensor must have at least ", + out_rows + 1, " elements, but got ", + row_seq_tensor.NumElements())); + OP_REQUIRES(context, col_seq_tensor.NumElements() > out_cols, + errors::InvalidArgument("Given out_backprop shape ", + out_backprop.shape().DebugString(), + ", col_seq_tensor must have at least ", + out_cols + 1, " elements, but got ", + col_seq_tensor.NumElements())); + auto row_seq_tensor_flat = row_seq_tensor.flat(); auto col_seq_tensor_flat = col_seq_tensor.flat(); auto orig_input_tensor_shape_flat = orig_input_tensor_shape.flat(); diff --git a/tensorflow/core/kernels/fractional_max_pool_op.cc b/tensorflow/core/kernels/fractional_max_pool_op.cc index 619a3507ce415f..1a2a783d135c54 100644 --- a/tensorflow/core/kernels/fractional_max_pool_op.cc +++ b/tensorflow/core/kernels/fractional_max_pool_op.cc @@ -235,6 +235,20 @@ class FractionalMaxPoolGradOp : public OpKernel { // Just to make it similar to FractionalMaxPoolOp. constexpr int tensor_in_and_out_dims = 4; + OP_REQUIRES( + context, tensor_in.dims() == tensor_in_and_out_dims, + errors::InvalidArgument("orig_input should be a tensor of rank 4, got ", + tensor_in.DebugString())); + OP_REQUIRES(context, tensor_in.NumElements() > 0, + errors::InvalidArgument("orig_input must not be empty, got ", + tensor_in.DebugString())); + OP_REQUIRES(context, tensor_out.dims() == tensor_in_and_out_dims, + errors::InvalidArgument( + "orig_output should be a tensor of rank 4, got ", + tensor_out.DebugString())); + OP_REQUIRES(context, tensor_out.NumElements() > 0, + errors::InvalidArgument("orig_output must not be empty, got ", + tensor_out.DebugString())); std::vector input_size(tensor_in_and_out_dims); std::vector output_size(tensor_in_and_out_dims); for (int i = 0; i < tensor_in_and_out_dims; ++i) { diff --git a/tensorflow/core/kernels/fused_batch_norm_op.cc b/tensorflow/core/kernels/fused_batch_norm_op.cc index 00ac9be6dcd17f..bd1b94d34b96fc 100644 --- a/tensorflow/core/kernels/fused_batch_norm_op.cc +++ b/tensorflow/core/kernels/fused_batch_norm_op.cc @@ -293,6 +293,9 @@ struct FusedBatchNorm { const CPUDevice& d = context->eigen_device(); const int depth = x.dimension(3); + OP_REQUIRES( + context, depth != 0, + errors::Internal("The 4th element in the input shape cannot be 0.")); const int size = x.size(); const int rest_size = size / depth; Eigen::DSizes rest_by_depth(rest_size, depth); @@ -1264,6 +1267,33 @@ class FusedBatchNormOpBase : public OpKernel { context, estimated_variance.dims() == 1, errors::InvalidArgument("estimated_variance must be 1-dimensional", estimated_variance.shape().DebugString())); + + const auto num_channels = GetTensorDim(x, tensor_format_, 'C'); + OP_REQUIRES( + context, scale.NumElements() == num_channels, + errors::InvalidArgument("scale must have the same number of elements " + "as the channels of x, got ", + scale.NumElements(), " and ", num_channels)); + OP_REQUIRES( + context, offset.NumElements() == num_channels, + errors::InvalidArgument("offset must have the same number of elements " + "as the channels of x, got ", + offset.NumElements(), " and ", num_channels)); + if (estimated_mean.NumElements() != 0) { + OP_REQUIRES(context, estimated_mean.NumElements() == num_channels, + errors::InvalidArgument( + "mean must be empty or have the same number of " + "elements as the channels of x, got ", + estimated_mean.NumElements(), " and ", num_channels)); + } + if (estimated_variance.NumElements() != 0) { + OP_REQUIRES(context, estimated_variance.NumElements() == num_channels, + errors::InvalidArgument( + "variance must be empty or have the same number of " + "elements as the channels of x, got ", + estimated_variance.NumElements(), " and ", num_channels)); + } + if (has_side_input_) { OP_REQUIRES(context, side_input->shape() == x.shape(), errors::InvalidArgument( @@ -1276,7 +1306,7 @@ class FusedBatchNormOpBase : public OpKernel { // NOTE(ezhulenev): This requirement is coming from implementation // details of cudnnBatchNormalizationForwardTrainingEx. OP_REQUIRES( - context, !is_training_ || x.dim_size(3) % 4 == 0, + context, !is_training_ || num_channels % 4 == 0, errors::InvalidArgument("FusedBatchNorm with activation requires " "channel dimension to be a multiple of 4.")); } diff --git a/tensorflow/core/kernels/immutable_constant_op.cc b/tensorflow/core/kernels/immutable_constant_op.cc index 0dd08c694eb6c5..19aa865c1fbe4d 100644 --- a/tensorflow/core/kernels/immutable_constant_op.cc +++ b/tensorflow/core/kernels/immutable_constant_op.cc @@ -17,6 +17,8 @@ limitations under the License. #include +#include "tensorflow/core/framework/types.pb.h" + namespace tensorflow { namespace { @@ -62,6 +64,12 @@ class MemmappedTensorAllocator : public Allocator { void set_delete_on_deallocate() { delete_on_deallocate_ = true; } + // Make sure tensors or complex types (strings, variants, resources) don't get + // their constructor called via a placement new since that would require + // writing to immutable data. + // See also: tensorflow/core/framework/typed_allocator.h + bool AllocatesOpaqueHandle() const override { return true; } + private: std::unique_ptr memory_region_; // If there is an error during allocation we keep it in this status. @@ -80,6 +88,9 @@ ImmutableConstantOp::ImmutableConstantOp(OpKernelConstruction* context) OP_REQUIRES_OK(context, context->GetAttr(kMemoryRegionNameAttr, ®ion_name_)); OP_REQUIRES_OK(context, context->GetAttr(kDTypeAttr, &dtype_)); + OP_REQUIRES(context, dtype_ != DT_RESOURCE && dtype_ != DT_VARIANT, + errors::InvalidArgument( + "Resource and variant dtypes are invalid for this op.")); OP_REQUIRES_OK(context, context->GetAttr(kShapeAttr, &shape_)); } diff --git a/tensorflow/core/kernels/load_and_remap_matrix_op.cc b/tensorflow/core/kernels/load_and_remap_matrix_op.cc index cb0245a9b61261..5ec28c70358132 100644 --- a/tensorflow/core/kernels/load_and_remap_matrix_op.cc +++ b/tensorflow/core/kernels/load_and_remap_matrix_op.cc @@ -123,6 +123,11 @@ class LoadAndRemapMatrixOp : public OpKernel { // Processes the checkpoint source and the provided Tensor name. const Tensor* ckpt_path_t; OP_REQUIRES_OK(context, context->input("ckpt_path", &ckpt_path_t)); + OP_REQUIRES( + context, ckpt_path_t->NumElements() == 1, + errors::InvalidArgument("The `ckpt_path` tensor must have exactly one " + "element, got tensor of shape ", + ckpt_path_t->shape().DebugString())); const string& ckpt_path = ckpt_path_t->scalar()(); const Tensor* old_tensor_name_t; OP_REQUIRES_OK(context, diff --git a/tensorflow/core/kernels/matrix_diag_op.cc b/tensorflow/core/kernels/matrix_diag_op.cc index 9796fd25e39c00..cd1170ca6ada0b 100644 --- a/tensorflow/core/kernels/matrix_diag_op.cc +++ b/tensorflow/core/kernels/matrix_diag_op.cc @@ -192,9 +192,22 @@ class MatrixDiagOp : public OpKernel { upper_diag_index = diag_index.flat()(1); } } - num_rows = context->input(2).flat()(0); - num_cols = context->input(3).flat()(0); - padding_value = context->input(4).flat()(0); + + auto& num_rows_tensor = context->input(2); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(num_rows_tensor.shape()), + errors::InvalidArgument("num_rows must be a scalar")); + num_rows = num_rows_tensor.flat()(0); + + auto& num_cols_tensor = context->input(3); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(num_cols_tensor.shape()), + errors::InvalidArgument("num_cols must be a scalar")); + num_cols = num_cols_tensor.flat()(0); + + auto& padding_value_tensor = context->input(4); + OP_REQUIRES(context, + TensorShapeUtils::IsScalar(padding_value_tensor.shape()), + errors::InvalidArgument("padding_value must be a scalar")); + padding_value = padding_value_tensor.flat()(0); } // Size validations. diff --git a/tensorflow/core/kernels/matrix_triangular_solve_op_impl.h b/tensorflow/core/kernels/matrix_triangular_solve_op_impl.h index fb7e6f0f5ffe1d..d4ffcf18a52290 100644 --- a/tensorflow/core/kernels/matrix_triangular_solve_op_impl.h +++ b/tensorflow/core/kernels/matrix_triangular_solve_op_impl.h @@ -162,6 +162,9 @@ class BaseMatrixTriangularSolveOp : public OpKernel { const Tensor& in1 = ctx->input(1); ValidateInputTensors(ctx, in0, in1); + if (!ctx->status().ok()) { + return; + } MatMulBCast bcast(in0.shape().dim_sizes(), in1.shape().dim_sizes()); OP_REQUIRES( @@ -230,13 +233,22 @@ class MatrixTriangularSolveOp private: void ValidateInputTensors(OpKernelContext* ctx, const Tensor& in0, const Tensor& in1) override { + const auto in0_num_dims = in0.dims(); OP_REQUIRES( - ctx, in0.dims() >= 2, - errors::InvalidArgument("In[0] ndims must be >= 2: ", in0.dims())); + ctx, in0_num_dims >= 2, + errors::InvalidArgument("In[0] ndims must be >= 2: ", in0_num_dims)); + const auto in1_num_dims = in1.dims(); OP_REQUIRES( - ctx, in1.dims() >= 2, - errors::InvalidArgument("In[0] ndims must be >= 2: ", in1.dims())); + ctx, in1_num_dims >= 2, + errors::InvalidArgument("In[1] ndims must be >= 2: ", in1_num_dims)); + + const auto in0_last_dim = in0.dim_size(in0_num_dims - 1); + const auto in0_prev_dim = in0.dim_size(in0_num_dims - 2); + OP_REQUIRES(ctx, in0_last_dim == in0_prev_dim, + errors::InvalidArgument( + "In[0] matrices in the last dimensions must be square (", + in0_last_dim, " =/= ", in0_prev_dim, ")")); } }; diff --git a/tensorflow/core/kernels/maxpooling_op.cc b/tensorflow/core/kernels/maxpooling_op.cc index 5652addd00a957..7accd1a820b0e3 100644 --- a/tensorflow/core/kernels/maxpooling_op.cc +++ b/tensorflow/core/kernels/maxpooling_op.cc @@ -192,7 +192,9 @@ static void SpatialMaxPoolWithArgMaxHelper( // CHECK(input_backprop_index >= in_start && input_backprop_index < // in_end) FastBoundsCheck(input_backprop_index - in_start, in_end - in_start); - input_backprop_flat(input_backprop_index) += out_backprop_flat(index); + if (index < out_backprop.NumElements()) { + input_backprop_flat(input_backprop_index) += out_backprop_flat(index); + } } } }; @@ -984,6 +986,9 @@ struct LaunchMaxPoolingGradWithArgmax { const int input_start = start * input_size_per_batch; const int input_end = limit * input_size_per_batch; for (int64 index = input_start; index < input_end; index++) { + if (index >= argmax.NumElements()) { + break; + } int64 grad_out_index = argmax_flat(index); if (!include_batch_in_index) { const int64 cur_batch = index / input_size_per_batch; @@ -1049,6 +1054,8 @@ class MaxPoolingGradWithArgmaxOp : public OpKernel { OP_REQUIRES_OK(context, context->forward_input_or_allocate_output( {0}, 0, out_shape, &grad_out)); + if (out_shape.num_elements() == 0) return; // nothing to be done + LaunchMaxPoolingGradWithArgmax::launch( context, params, grad_in, argmax, grad_out, include_batch_in_index_); } diff --git a/tensorflow/core/kernels/nth_element_op.cc b/tensorflow/core/kernels/nth_element_op.cc index 0e43cc19aae513..bd523f51e27e2d 100644 --- a/tensorflow/core/kernels/nth_element_op.cc +++ b/tensorflow/core/kernels/nth_element_op.cc @@ -95,7 +95,8 @@ struct NthElementFunctor { const int last_dim = input_tensor.dim_size(input_tensor.dims() - 1); // Allocate each row to different shard. - auto SubNthElement = [&, input, output, last_dim, n](int start, int limit) { + auto SubNthElement = [&, input, output, last_dim, n](int64 start, + int64 limit) { // std::nth_element would rearrange the array, so we need a new buffer. std::vector buf(last_dim); diff --git a/tensorflow/core/kernels/parameterized_truncated_normal_op.cc b/tensorflow/core/kernels/parameterized_truncated_normal_op.cc index 09dc3ffd12910f..62d996ed00ab30 100644 --- a/tensorflow/core/kernels/parameterized_truncated_normal_op.cc +++ b/tensorflow/core/kernels/parameterized_truncated_normal_op.cc @@ -69,8 +69,8 @@ struct TruncatedNormalFunctor { auto DoWork = [samples_per_batch, num_elements, &ctx, &means, &stddevs, &minvals, &maxvals, &gen, &output, - kStdDevsInsideBoundsToUseRandnSampler](int start_batch, - int limit_batch) { + kStdDevsInsideBoundsToUseRandnSampler](int64 start_batch, + int64 limit_batch) { // Capturing "gen" by-value would only make a copy for the _shared_ // lambda. Since we want to let each worker have its own copy, we pass // "gen" by reference and explicitly do a copy assignment here. @@ -336,6 +336,9 @@ class ParameterizedTruncatedNormalOp : public OpKernel { ctx, TensorShapeUtils::IsVector(shape_tensor.shape()), errors::InvalidArgument("Input shape should be a vector, got shape: ", shape_tensor.shape().DebugString())); + OP_REQUIRES(ctx, shape_tensor.NumElements() > 0, + errors::InvalidArgument("Shape tensor must not be empty, got ", + shape_tensor.DebugString())); int32 num_batches = shape_tensor.flat()(0); int32 samples_per_batch = 1; diff --git a/tensorflow/core/kernels/pooling_ops_3d.cc b/tensorflow/core/kernels/pooling_ops_3d.cc index 7345ccf69eeba9..df8237cb54e1e5 100644 --- a/tensorflow/core/kernels/pooling_ops_3d.cc +++ b/tensorflow/core/kernels/pooling_ops_3d.cc @@ -387,6 +387,19 @@ struct LaunchAvgPooling3dGradOp { const std::array& output_shape, const std::array& padding, TensorFormat data_format, Tensor* output) { + OP_REQUIRES( + context, tensor_in_shape.dim_size(0) == out_backprop.dim_size(0), + errors::InvalidArgument( + "Expected first dimension of tensor_in_shape and " + "out_backprop to match, got ", + tensor_in_shape.dim_size(0), " and ", out_backprop.dim_size(0))); + OP_REQUIRES( + context, tensor_in_shape.dim_size(4) == out_backprop.dim_size(4), + errors::InvalidArgument( + "Expected last dimension of tensor_in_shape and " + "out_backprop to match, got ", + tensor_in_shape.dim_size(4), " and ", out_backprop.dim_size(4))); + output->flat().setZero(); std::array input_size = {{tensor_in_shape.dim_size(3), tensor_in_shape.dim_size(2), @@ -697,11 +710,36 @@ class MaxPooling3dGradGradOp : public OpKernel { Pool3dParameters params{context, ksize_, stride_, padding_, data_format_, tensor_in.shape()}; + if (!context->status().ok()) return; // params is invalid Tensor* output = nullptr; OP_REQUIRES_OK(context, context->forward_input_or_allocate_output( {2}, 0, tensor_out.shape(), &output)); + // Given access patterns in LaunchMaxPooling3dGradGradOp, these tensors must + // have elements. + OP_REQUIRES(context, tensor_in.NumElements() > 0, + errors::InvalidArgument("received empty tensor tensor_in: ", + tensor_in.DebugString())); + OP_REQUIRES(context, tensor_out.NumElements() > 0, + errors::InvalidArgument("received empty tensor tensor_out: ", + tensor_out.DebugString())); + OP_REQUIRES( + context, out_grad_backprop.NumElements() > 0, + errors::InvalidArgument("received empty tensor out_grad_backprop: ", + out_grad_backprop.DebugString())); + OP_REQUIRES(context, + tensor_in.NumElements() == out_grad_backprop.NumElements(), + errors::InvalidArgument("tensor_in and out_grad_backprop must " + "have same number of elements, got <", + tensor_in.DebugString(), "> and <", + out_grad_backprop.DebugString(), ">")); + OP_REQUIRES( + context, tensor_out.NumElements() == output->NumElements(), + errors::InvalidArgument( + "tensor_out and output must have same number of elements, got <", + tensor_out.DebugString(), "> and <", output->DebugString(), ">")); + LaunchMaxPooling3dGradGradOp::launch( context, params, tensor_in, tensor_out, out_grad_backprop, output); } diff --git a/tensorflow/core/kernels/quantize_and_dequantize_op.cc b/tensorflow/core/kernels/quantize_and_dequantize_op.cc index 8f71d09c0832e7..28be8a9fa11822 100644 --- a/tensorflow/core/kernels/quantize_and_dequantize_op.cc +++ b/tensorflow/core/kernels/quantize_and_dequantize_op.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include "tensorflow/core/framework/op_requires.h" #define EIGEN_USE_THREADS #if (defined(GOOGLE_CUDA) && GOOGLE_CUDA) || \ @@ -71,6 +72,13 @@ class QuantizeAndDequantizeV2Op : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& input = ctx->input(0); + OP_REQUIRES( + ctx, axis_ >= -1, + errors::InvalidArgument("Axis must be at least -1. Found ", axis_)); + OP_REQUIRES( + ctx, (axis_ == -1 || axis_ < input.shape().dims()), + errors::InvalidArgument("Shape must be at least rank ", axis_ + 1, + " but is rank ", input.shape().dims())); const int depth = (axis_ == -1) ? 1 : input.dim_size(axis_); Tensor input_min_tensor; Tensor input_max_tensor; @@ -151,6 +159,10 @@ class QuantizeAndDequantizeV3Op : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& input = ctx->input(0); + OP_REQUIRES(ctx, axis_ < input.dims(), + errors::InvalidArgument( + "Axis requested is larger than input dimensions. Axis: ", + axis_, " Input Dimensions: ", input.dims())); const int depth = (axis_ == -1) ? 1 : input.dim_size(axis_); Tensor* output = nullptr; OP_REQUIRES_OK(ctx, ctx->allocate_output(0, input.shape(), &output)); diff --git a/tensorflow/core/kernels/quantized_add_op.cc b/tensorflow/core/kernels/quantized_add_op.cc index 55c69de7d3ea6c..b186f00f15c061 100644 --- a/tensorflow/core/kernels/quantized_add_op.cc +++ b/tensorflow/core/kernels/quantized_add_op.cc @@ -538,6 +538,8 @@ class QuantizedAddOp : public OpKernel { tensor_min = min_x; tensor_max = max_x; } + OP_REQUIRES(context, vector_num_elements > 0, + errors::InvalidArgument("Must have some elements to add")); VectorTensorAddition( vector_data, vector_min, vector_max, vector_num_elements, tensor_data, tensor_min, tensor_max, tensor_num_elements, min_z_value, max_z_value, diff --git a/tensorflow/core/kernels/quantized_batch_norm_op.cc b/tensorflow/core/kernels/quantized_batch_norm_op.cc index b03da7ad17fab4..6dfe07f97a4007 100644 --- a/tensorflow/core/kernels/quantized_batch_norm_op.cc +++ b/tensorflow/core/kernels/quantized_batch_norm_op.cc @@ -173,20 +173,50 @@ class QuantizedBatchNormOp : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& input = context->input(0); - const float input_min = context->input(1).flat()(0); - const float input_max = context->input(2).flat()(0); + const auto& input_min_tensor = context->input(1); + OP_REQUIRES(context, input_min_tensor.NumElements() == 1, + errors::InvalidArgument("input_min must have 1 element")); + const float input_min = input_min_tensor.flat()(0); + const auto& input_max_tensor = context->input(2); + OP_REQUIRES(context, input_max_tensor.NumElements() == 1, + errors::InvalidArgument("input_max must have 1 element")); + const float input_max = input_max_tensor.flat()(0); const Tensor& mean = context->input(3); - const float mean_min = context->input(4).flat()(0); - const float mean_max = context->input(5).flat()(0); + const auto& mean_min_tensor = context->input(4); + OP_REQUIRES(context, mean_min_tensor.NumElements() == 1, + errors::InvalidArgument("mean_min must have 1 element")); + const float mean_min = mean_min_tensor.flat()(0); + const auto& mean_max_tensor = context->input(5); + OP_REQUIRES(context, mean_max_tensor.NumElements() == 1, + errors::InvalidArgument("mean_max must have 1 element")); + const float mean_max = mean_max_tensor.flat()(0); const Tensor& var = context->input(6); - const float var_min = context->input(7).flat()(0); - const float var_max = context->input(8).flat()(0); + const auto& var_min_tensor = context->input(7); + OP_REQUIRES(context, var_min_tensor.NumElements() == 1, + errors::InvalidArgument("var_min must have 1 element")); + const float var_min = var_min_tensor.flat()(0); + const auto& var_max_tensor = context->input(8); + OP_REQUIRES(context, var_max_tensor.NumElements() == 1, + errors::InvalidArgument("var_max must have 1 element")); + const float var_max = var_max_tensor.flat()(0); const Tensor& beta = context->input(9); - const float beta_min = context->input(10).flat()(0); - const float beta_max = context->input(11).flat()(0); + const auto& beta_min_tensor = context->input(10); + OP_REQUIRES(context, beta_min_tensor.NumElements() == 1, + errors::InvalidArgument("beta_min must have 1 element")); + const float beta_min = beta_min_tensor.flat()(0); + const auto& beta_max_tensor = context->input(11); + OP_REQUIRES(context, beta_max_tensor.NumElements() == 1, + errors::InvalidArgument("beta_max must have 1 element")); + const float beta_max = beta_max_tensor.flat()(0); const Tensor& gamma = context->input(12); - const float gamma_min = context->input(13).flat()(0); - const float gamma_max = context->input(14).flat()(0); + const auto& gamma_min_tensor = context->input(13); + OP_REQUIRES(context, gamma_min_tensor.NumElements() == 1, + errors::InvalidArgument("gamma_min must have 1 element")); + const float gamma_min = gamma_min_tensor.flat()(0); + const auto& gamma_max_tensor = context->input(14); + OP_REQUIRES(context, gamma_max_tensor.NumElements() == 1, + errors::InvalidArgument("gamma_max must have 1 element")); + const float gamma_max = gamma_max_tensor.flat()(0); OP_REQUIRES(context, input.dims() == 4, errors::InvalidArgument("input must be 4-dimensional", @@ -203,6 +233,33 @@ class QuantizedBatchNormOp : public OpKernel { OP_REQUIRES(context, gamma.dims() == 1, errors::InvalidArgument("gamma must be 1-dimensional", gamma.shape().DebugString())); + OP_REQUIRES(context, mean.NumElements() > 1, + errors::InvalidArgument("Must have at least a mean value", + gamma.shape().DebugString())); + OP_REQUIRES(context, mean.NumElements() > 1, + errors::InvalidArgument("Must have at least a mean value")); + const auto last_dim = input.shape().dims() - 1; + OP_REQUIRES(context, + mean.shape().dim_size(0) == input.shape().dim_size(last_dim), + errors::InvalidArgument("Must provide as many means as the " + "last dimension of the input tensor: ", + mean.shape().DebugString(), " vs. ", + input.shape().DebugString())); + OP_REQUIRES( + context, mean.shape().dim_size(0) == var.shape().dim_size(0), + errors::InvalidArgument( + "Mean and variance tensors must have the same shape: ", + mean.shape().DebugString(), " vs. ", var.shape().DebugString())); + OP_REQUIRES( + context, mean.shape().dim_size(0) == beta.shape().dim_size(0), + errors::InvalidArgument( + "Mean and beta tensors must have the same shape: ", + mean.shape().DebugString(), " vs. ", beta.shape().DebugString())); + OP_REQUIRES( + context, mean.shape().dim_size(0) == gamma.shape().dim_size(0), + errors::InvalidArgument( + "Mean and gamma tensors must have the same shape: ", + mean.shape().DebugString(), " vs. ", gamma.shape().DebugString())); Tensor* output = nullptr; OP_REQUIRES_OK(context, diff --git a/tensorflow/core/kernels/quantized_bias_add_op.cc b/tensorflow/core/kernels/quantized_bias_add_op.cc index 5457d290c2559c..db0e21a498011d 100644 --- a/tensorflow/core/kernels/quantized_bias_add_op.cc +++ b/tensorflow/core/kernels/quantized_bias_add_op.cc @@ -56,6 +56,8 @@ class QuantizedBiasAddOp : public OpKernel { "Must provide as many biases as the last dimension " "of the input tensor: ", bias.shape().DebugString(), " vs. ", input.shape().DebugString())); + OP_REQUIRES(context, bias.NumElements() > 0, + errors::InvalidArgument("Must provide at least 1 bias")); Tensor* output = nullptr; OP_REQUIRES_OK(context, diff --git a/tensorflow/core/kernels/quantized_conv_ops.cc b/tensorflow/core/kernels/quantized_conv_ops.cc index 5b3570edff5fee..1b83c09e1f7f18 100644 --- a/tensorflow/core/kernels/quantized_conv_ops.cc +++ b/tensorflow/core/kernels/quantized_conv_ops.cc @@ -18,6 +18,8 @@ limitations under the License. #include #include +#include "tensorflow/core/platform/errors.h" + #define EIGEN_USE_THREADS #define GEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK @@ -227,8 +229,12 @@ class Im2ColConvFunctor { return; } - CHECK_GT(output_width, 0); - CHECK_GT(output_height, 0); + OP_REQUIRES( + context, output_width > 0, + errors::InvalidArgument("output_width must be strictly positive")); + OP_REQUIRES( + context, output_height > 0, + errors::InvalidArgument("output_height must be strictly positive")); int filter_left_offset; int filter_top_offset; if (padding == VALID) { @@ -255,6 +261,9 @@ class Im2ColConvFunctor { // by the width, then the height. This is the standard memory order in the // image world if it helps to visualize it. const int filter_value_count = filter_width * filter_height * input_depth; + OP_REQUIRES(context, filter_value_count > 0, + errors::InvalidArgument( + "filter patch must contain at least one element")); const int64 patches_per_chunk = kMaxChunkSize / (filter_value_count * sizeof(T1)); const int64 chunk_value_count = diff --git a/tensorflow/core/kernels/quantized_mul_op.cc b/tensorflow/core/kernels/quantized_mul_op.cc index 4e191f162662bb..22cff8939449a6 100644 --- a/tensorflow/core/kernels/quantized_mul_op.cc +++ b/tensorflow/core/kernels/quantized_mul_op.cc @@ -284,10 +284,22 @@ class QuantizedMulOp : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& x = context->input(0); const Tensor& y = context->input(1); - const float min_x = context->input(2).flat()(0); - const float max_x = context->input(3).flat()(0); - const float min_y = context->input(4).flat()(0); - const float max_y = context->input(5).flat()(0); + auto& min_x_tensor = context->input(2); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(min_x_tensor.shape()), + errors::InvalidArgument("min_x must be a scalar")); + const float min_x = min_x_tensor.flat()(0); + auto& max_x_tensor = context->input(3); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(max_x_tensor.shape()), + errors::InvalidArgument("max_x must be a scalar")); + const float max_x = max_x_tensor.flat()(0); + auto& min_y_tensor = context->input(4); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(min_y_tensor.shape()), + errors::InvalidArgument("min_y must be a scalar")); + const float min_y = min_y_tensor.flat()(0); + auto& max_y_tensor = context->input(5); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(max_y_tensor.shape()), + errors::InvalidArgument("max_y must be a scalar")); + const float max_y = max_y_tensor.flat()(0); BCast bcast(BCast::FromShape(x.shape()), BCast::FromShape(y.shape())); if (!bcast.IsValid()) { @@ -347,6 +359,11 @@ class QuantizedMulOp : public OpKernel { tensor_num_elements = x.NumElements(); tensor_offset = offset_x; } + if (vector_num_elements == 0) { + context->SetStatus( + errors::InvalidArgument("vector must have at least 1 element")); + return; + } VectorTensorMultiply( vector_data, vector_offset, vector_num_elements, tensor_data, tensor_offset, tensor_num_elements, z_data); diff --git a/tensorflow/core/kernels/quantized_reshape_op.cc b/tensorflow/core/kernels/quantized_reshape_op.cc index bd76c94edeea7a..682f4aaa1f79e7 100644 --- a/tensorflow/core/kernels/quantized_reshape_op.cc +++ b/tensorflow/core/kernels/quantized_reshape_op.cc @@ -17,6 +17,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/kernels/reshape_op.h" @@ -30,9 +31,29 @@ class QuantizedReshapeOp : public ReshapeOp { void Compute(OpKernelContext* ctx) override { // This call processes inputs 1 and 2 to write output 0. ReshapeOp::Compute(ctx); + if (!ctx->status().ok()) { + return; + } + + const auto& input_min_float_tensor = ctx->input(2); + const auto& input_min_float_shape = input_min_float_tensor.shape(); + OP_REQUIRES(ctx, + TensorShapeUtils::IsScalar(input_min_float_shape) || + (TensorShapeUtils::IsVector(input_min_float_shape) && + (input_min_float_shape.dim_size(0) == 1)), + errors::InvalidArgument( + "input_min must be a scalar or a vector of 1 element")); + const float input_min_float = input_min_float_tensor.flat()(0); + const auto& input_max_float_tensor = ctx->input(3); + const auto& input_max_float_shape = input_max_float_tensor.shape(); + OP_REQUIRES(ctx, + TensorShapeUtils::IsScalar(input_max_float_shape) || + (TensorShapeUtils::IsVector(input_max_float_shape) && + (input_max_float_shape.dim_size(0) == 1)), + errors::InvalidArgument( + "input_max must be a scalar or a vector of 1 element")); + const float input_max_float = input_max_float_tensor.flat()(0); - const float input_min_float = ctx->input(2).flat()(0); - const float input_max_float = ctx->input(3).flat()(0); Tensor* output_min = nullptr; OP_REQUIRES_OK(ctx, ctx->allocate_output(1, TensorShape({}), &output_min)); output_min->flat()(0) = input_min_float; diff --git a/tensorflow/core/kernels/quantized_resize_bilinear_op.cc b/tensorflow/core/kernels/quantized_resize_bilinear_op.cc index 4da56cde5478d5..7569cbc11bdd4e 100644 --- a/tensorflow/core/kernels/quantized_resize_bilinear_op.cc +++ b/tensorflow/core/kernels/quantized_resize_bilinear_op.cc @@ -64,6 +64,8 @@ inline void ComputeInterpolationWeights( std::max(static_cast(in_f), static_cast(0)); interpolation->upper[i] = std::min(static_cast(std::ceil(in)), in_size - 1); + interpolation->lower[i] = + std::min(interpolation->lower[i], interpolation->upper[i]); interpolation->lerp[i] = in - in_f; interpolation->ilerp[i] = static_cast((in - in_f) * (1 << resolution)); @@ -701,8 +703,14 @@ class QuantizedResizeBilinearOp : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& input = context->input(0); - const float in_min = context->input(2).flat()(0); - const float in_max = context->input(3).flat()(0); + const auto& in_min_tensor = context->input(2); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(in_min_tensor.shape()), + errors::InvalidArgument("min must be a scalar")); + const float in_min = in_min_tensor.flat()(0); + const auto& in_max_tensor = context->input(3); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(in_max_tensor.shape()), + errors::InvalidArgument("max must be a scalar")); + const float in_max = in_max_tensor.flat()(0); ImageResizerState st(align_corners_, false); st.ValidateAndCreateOutput(context, input); diff --git a/tensorflow/core/kernels/ragged_tensor_to_tensor_op.cc b/tensorflow/core/kernels/ragged_tensor_to_tensor_op.cc index d729c43f25a4b8..8905647fa6b64f 100644 --- a/tensorflow/core/kernels/ragged_tensor_to_tensor_op.cc +++ b/tensorflow/core/kernels/ragged_tensor_to_tensor_op.cc @@ -207,7 +207,7 @@ class RaggedTensorToTensorBaseOp : public OpKernel { DCHECK_EQ(result->size(), first_dimension); } - void CalculateOutputIndexRowSplit( + Status CalculateOutputIndexRowSplit( const RowPartitionTensor& row_split, const vector& parent_output_index, INDEX_TYPE output_index_multiplier, INDEX_TYPE output_size, @@ -232,9 +232,11 @@ class RaggedTensorToTensorBaseOp : public OpKernel { result->push_back(-1); } } - if (row_split_size > 0) { - DCHECK_EQ(result->size(), row_split(row_split_size - 1)); + if (row_split_size > 0 && result->size() != row_split(row_split_size - 1)) { + return errors::InvalidArgument("Invalid row split size."); } + + return Status::OK(); } // Calculate the output index of the first element of a list. @@ -258,7 +260,7 @@ class RaggedTensorToTensorBaseOp : public OpKernel { // result[6] = -1 because parent_output_index[value_rowids[6]] == -1 // result[7] = -1 because parent_output_index[value_rowids[6]] == -1 // result[8] = parent_output_index[value_rowids[7]] - void CalculateOutputIndexValueRowID( + Status CalculateOutputIndexValueRowID( const RowPartitionTensor& value_rowids, const vector& parent_output_index, INDEX_TYPE output_index_multiplier, INDEX_TYPE output_size, @@ -266,12 +268,18 @@ class RaggedTensorToTensorBaseOp : public OpKernel { const INDEX_TYPE index_size = value_rowids.size(); result->reserve(index_size); if (index_size == 0) { - return; + return Status::OK(); } INDEX_TYPE current_output_column = 0; INDEX_TYPE current_value_rowid = value_rowids(0); - DCHECK_LT(current_value_rowid, parent_output_index.size()); + + if (current_value_rowid >= parent_output_index.size()) { + return errors::InvalidArgument( + "Got current_value_rowid=", current_value_rowid, + " which is not less than ", parent_output_index.size()); + } + INDEX_TYPE current_output_index = parent_output_index[current_value_rowid]; result->push_back(current_output_index); for (INDEX_TYPE i = 1; i < index_size; ++i) { @@ -288,12 +296,23 @@ class RaggedTensorToTensorBaseOp : public OpKernel { } else { current_output_column = 0; current_value_rowid = next_value_rowid; - DCHECK_LT(next_value_rowid, parent_output_index.size()); + + if (next_value_rowid >= parent_output_index.size()) { + return errors::InvalidArgument( + "Got next_value_rowid=", next_value_rowid, + " which is not less than ", parent_output_index.size()); + } + current_output_index = parent_output_index[next_value_rowid]; } result->push_back(current_output_index); } - DCHECK_EQ(result->size(), value_rowids.size()); + + if (result->size() != value_rowids.size()) { + return errors::InvalidArgument("Invalid row ids."); + } + + return Status::OK(); } Status CalculateOutputIndex(OpKernelContext* context, int dimension, @@ -306,15 +325,19 @@ class RaggedTensorToTensorBaseOp : public OpKernel { auto partition_type = GetRowPartitionTypeByDimension(dimension); switch (partition_type) { case RowPartitionType::VALUE_ROWIDS: - CalculateOutputIndexValueRowID( + return CalculateOutputIndexValueRowID( row_partition_tensor, parent_output_index, output_index_multiplier, output_size, result); - return tensorflow::Status::OK(); case RowPartitionType::ROW_SPLITS: - CalculateOutputIndexRowSplit(row_partition_tensor, parent_output_index, - output_index_multiplier, output_size, - result); - return tensorflow::Status::OK(); + if (row_partition_tensor.size() - 1 > parent_output_index.size()) { + return errors::InvalidArgument( + "Row partition size is greater than output size: ", + row_partition_tensor.size() - 1, " > ", + parent_output_index.size()); + } + return CalculateOutputIndexRowSplit( + row_partition_tensor, parent_output_index, output_index_multiplier, + output_size, result); default: return errors::InvalidArgument( "Unsupported partition type:", @@ -345,6 +368,11 @@ class RaggedTensorToTensorBaseOp : public OpKernel { void Compute(OpKernelContext* context) override { INDEX_TYPE first_dimension; + const Tensor first_partition_tensor = + context->input(kFirstPartitionInputIndex); + OP_REQUIRES(context, first_partition_tensor.NumElements() > 0, + errors::InvalidArgument("Invalid first partition input. Tensor " + "requires at least one element.")); OP_REQUIRES_OK(context, GetFirstDimensionSize(context, &first_dimension)); vector output_size; OP_REQUIRES_OK(context, diff --git a/tensorflow/core/kernels/ragged_tensor_to_variant_op.cc b/tensorflow/core/kernels/ragged_tensor_to_variant_op.cc index 7a5ae1c6240b55..49c4a2411b8c3c 100644 --- a/tensorflow/core/kernels/ragged_tensor_to_variant_op.cc +++ b/tensorflow/core/kernels/ragged_tensor_to_variant_op.cc @@ -175,6 +175,11 @@ class RaggedTensorToVariantOp : public OpKernel { // Unbatch the Ragged Tensor and encode the components. std::vector ragged_components; + auto batched_splits_top_vec = + batched_ragged_input.nested_splits[0].vec(); + int num_components = batched_splits_top_vec.size() - 1; + OP_REQUIRES(context, num_components >= 0, + errors::Internal("Invalid split argument.")); OP_REQUIRES_OK(context, UnbatchRaggedZerothDim( batched_ragged_input, &ragged_components)); std::vector encoded_components(ragged_components.size()); diff --git a/tensorflow/core/kernels/random_binomial_op.cc b/tensorflow/core/kernels/random_binomial_op.cc index ea42239e77f8e1..9df6c202b802ff 100644 --- a/tensorflow/core/kernels/random_binomial_op.cc +++ b/tensorflow/core/kernels/random_binomial_op.cc @@ -182,7 +182,7 @@ struct RandomBinomialFunctor { // the sample shape and [H1, ... Hm] for the batch shape of the samples. // We have B1 * ... * Bk samples per batch member we need. auto DoWork = [num_batches, samples_per_batch, &bcast, &counts, &probs, - &gen, &output](int start_output, int limit_output) { + &gen, &output](int64 start_output, int64 limit_output) { // Vectorized intermediate calculations for uniform rejection sampling. // We always generate at most 4 samples. Eigen::array z; diff --git a/tensorflow/core/kernels/random_op.cc b/tensorflow/core/kernels/random_op.cc index 7b7f5153436c6f..60018764ef80a9 100644 --- a/tensorflow/core/kernels/random_op.cc +++ b/tensorflow/core/kernels/random_op.cc @@ -205,7 +205,7 @@ class RandomGammaOp : public OpKernel { // avoid a couple flops which can be done on a per-alpha basis. auto DoWork = [samples_per_alpha, num_alphas, &rng, samples_flat, - alpha_flat](int start_output, int limit_output) { + alpha_flat](int64 start_output, int64 limit_output) { using Eigen::numext::exp; using Eigen::numext::log; using Eigen::numext::pow; diff --git a/tensorflow/core/kernels/random_poisson_op.cc b/tensorflow/core/kernels/random_poisson_op.cc index aa9a0bfe214954..dcb7d6b0f0edfa 100644 --- a/tensorflow/core/kernels/random_poisson_op.cc +++ b/tensorflow/core/kernels/random_poisson_op.cc @@ -97,7 +97,7 @@ struct PoissonFunctor { typedef random::UniformDistribution Uniform; auto DoWork = [num_samples, num_rate, &rng, samples_flat, rate_flat]( - int start_output, int limit_output) { + int64 start_output, int64 limit_output) { // Capturing "rng" by value would only make a copy for the _shared_ // lambda. Since we want to let each worker have its own copy, we pass // "rng" by reference and explicitly do a copy assignment. diff --git a/tensorflow/core/kernels/requantization_range_op.cc b/tensorflow/core/kernels/requantization_range_op.cc index cc6e891a6b352b..f6e217499d1983 100644 --- a/tensorflow/core/kernels/requantization_range_op.cc +++ b/tensorflow/core/kernels/requantization_range_op.cc @@ -46,6 +46,10 @@ class RequantizationRangeOp : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& input = ctx->input(0); + OP_REQUIRES(ctx, ctx->input(1).NumElements() > 0, + errors::InvalidArgument("Input min must not be empty.")); + OP_REQUIRES(ctx, ctx->input(2).NumElements() > 0, + errors::InvalidArgument("Input max must not be empty.")); const float input_min_float = ctx->input(1).flat()(0); const float input_max_float = ctx->input(2).flat()(0); Tensor* output_min = nullptr; diff --git a/tensorflow/core/kernels/reverse_op.cc b/tensorflow/core/kernels/reverse_op.cc index 98bf8bf8e914c1..4a46bffc3e9c00 100644 --- a/tensorflow/core/kernels/reverse_op.cc +++ b/tensorflow/core/kernels/reverse_op.cc @@ -158,6 +158,12 @@ class ReverseOp : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& input = context->input(0); + // If input is provided, check to make sure the first dimension is valid. + if (input.dims() > 0) { + OP_REQUIRES( + context, input.dim_size(0) != 0, + errors::InvalidArgument("Invalid input first dimension. Found 0.")); + } const Tensor& dims = context->input(1); if (TensorShapeUtils::IsScalar(input.shape())) { diff --git a/tensorflow/core/kernels/reverse_sequence_op.cc b/tensorflow/core/kernels/reverse_sequence_op.cc index 0e112133915d5a..490456fddd76b9 100644 --- a/tensorflow/core/kernels/reverse_sequence_op.cc +++ b/tensorflow/core/kernels/reverse_sequence_op.cc @@ -113,6 +113,10 @@ class ReverseSequenceOp : public OpKernel { : OpKernel(context) { OP_REQUIRES_OK(context, context->GetAttr("batch_dim", &batch_dim_)); OP_REQUIRES_OK(context, context->GetAttr("seq_dim", &seq_dim_)); + OP_REQUIRES(context, batch_dim_ >= 0, + errors::InvalidArgument("Invalid batch_dim ", batch_dim_)); + OP_REQUIRES(context, seq_dim_ >= 0, + errors::InvalidArgument("Invalid seq_dim ", seq_dim_)); } void Compute(OpKernelContext* context) override { diff --git a/tensorflow/core/kernels/sdca_internal.cc b/tensorflow/core/kernels/sdca_internal.cc index cbc754af0e9bb1..11a3be8bf46a76 100644 --- a/tensorflow/core/kernels/sdca_internal.cc +++ b/tensorflow/core/kernels/sdca_internal.cc @@ -99,6 +99,10 @@ Status ModelWeights::Initialize(OpKernelContext* const context) { OpInputList sparse_weights_inputs; TF_RETURN_IF_ERROR( context->input_list("sparse_weights", &sparse_weights_inputs)); + if (sparse_indices_inputs.size() != sparse_weights_inputs.size()) + return errors::InvalidArgument( + "sparse_indices and sparse_weights must have the same length, got ", + sparse_indices_inputs.size(), " and ", sparse_weights_inputs.size()); OpInputList dense_weights_inputs; TF_RETURN_IF_ERROR( context->input_list("dense_weights", &dense_weights_inputs)); @@ -106,10 +110,20 @@ Status ModelWeights::Initialize(OpKernelContext* const context) { OpOutputList sparse_weights_outputs; TF_RETURN_IF_ERROR(context->output_list("out_delta_sparse_weights", &sparse_weights_outputs)); + if (sparse_weights_outputs.size() != sparse_weights_inputs.size()) + return errors::InvalidArgument( + "out_delta_sparse_weights and sparse_weights must have the same " + "length, got ", + sparse_weights_outputs.size(), " and ", sparse_weights_inputs.size()); OpOutputList dense_weights_outputs; TF_RETURN_IF_ERROR( context->output_list("out_delta_dense_weights", &dense_weights_outputs)); + if (dense_weights_outputs.size() != dense_weights_inputs.size()) + return errors::InvalidArgument( + "out_delta_dense_weights and dense_weights must have the same length, " + "got ", + dense_weights_outputs.size(), " and ", dense_weights_inputs.size()); for (int i = 0; i < sparse_weights_inputs.size(); ++i) { Tensor* delta_t; @@ -327,13 +341,28 @@ Status Examples::Initialize(OpKernelContext* const context, OpInputList sparse_example_indices_inputs; TF_RETURN_IF_ERROR(context->input_list("sparse_example_indices", &sparse_example_indices_inputs)); + if (sparse_example_indices_inputs.size() != num_sparse_features) + return errors::InvalidArgument( + "Expected ", num_sparse_features, + " tensors in sparse_example_indices but got ", + sparse_example_indices_inputs.size()); OpInputList sparse_feature_indices_inputs; TF_RETURN_IF_ERROR(context->input_list("sparse_feature_indices", &sparse_feature_indices_inputs)); + if (sparse_feature_indices_inputs.size() != num_sparse_features) + return errors::InvalidArgument( + "Expected ", num_sparse_features, + " tensors in sparse_feature_indices but got ", + sparse_feature_indices_inputs.size()); OpInputList sparse_feature_values_inputs; if (num_sparse_features_with_values > 0) { TF_RETURN_IF_ERROR(context->input_list("sparse_feature_values", &sparse_feature_values_inputs)); + if (sparse_feature_values_inputs.size() != num_sparse_features_with_values) + return errors::InvalidArgument( + "Expected ", num_sparse_features_with_values, + " tensors in sparse_feature_values but got ", + sparse_feature_values_inputs.size()); } const Tensor* example_weights_t; @@ -400,6 +429,13 @@ Status Examples::CreateSparseFeatureRepresentation( sparse_example_indices_inputs[i].template flat(); auto feature_indices = sparse_feature_indices_inputs[i].template flat(); + if (example_indices.size() != feature_indices.size()) { + mutex_lock l(mu); + result = errors::InvalidArgument( + "Found mismatched example_indices and feature_indices [", + example_indices, "] vs [", feature_indices, "]"); + return; + } // Parse features for each example. Features for a particular example // are at the offsets (start_id, end_id] diff --git a/tensorflow/core/kernels/session_ops.cc b/tensorflow/core/kernels/session_ops.cc index d83a714452f2af..2403fad55a43ed 100644 --- a/tensorflow/core/kernels/session_ops.cc +++ b/tensorflow/core/kernels/session_ops.cc @@ -16,6 +16,7 @@ limitations under the License. // See docs in ../ops/data_flow_ops.cc. #include + #include #include "tensorflow/core/common_runtime/device.h" @@ -27,6 +28,7 @@ limitations under the License. #include "tensorflow/core/framework/types.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/gtl/map_util.h" +#include "tensorflow/core/platform/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/mutex.h" @@ -42,7 +44,11 @@ class GetSessionHandleOp : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& val = ctx->input(0); - int64 id = ctx->session_state()->GetNewId(); + auto session_state = ctx->session_state(); + OP_REQUIRES(ctx, session_state != nullptr, + errors::FailedPrecondition( + "GetSessionHandle called on null session state")); + int64 id = session_state->GetNewId(); TensorStore::TensorAndKey tk{val, id, requested_device()}; OP_REQUIRES_OK(ctx, ctx->tensor_store()->AddTensor(name(), tk)); @@ -112,7 +118,11 @@ class GetSessionTensorOp : public OpKernel { const Tensor& handle = ctx->input(0); const string& name = handle.scalar()(); Tensor val; - OP_REQUIRES_OK(ctx, ctx->session_state()->GetTensor(name, &val)); + auto session_state = ctx->session_state(); + OP_REQUIRES(ctx, session_state != nullptr, + errors::FailedPrecondition( + "GetSessionTensor called on null session state")); + OP_REQUIRES_OK(ctx, session_state->GetTensor(name, &val)); ctx->set_output(0, val); } @@ -154,7 +164,11 @@ class DeleteSessionTensorOp : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& handle = ctx->input(0); const string& name = handle.scalar()(); - OP_REQUIRES_OK(ctx, ctx->session_state()->DeleteTensor(name)); + auto session_state = ctx->session_state(); + OP_REQUIRES(ctx, session_state != nullptr, + errors::FailedPrecondition( + "DeleteSessionTensor called on null session state")); + OP_REQUIRES_OK(ctx, session_state->DeleteTensor(name)); } TF_DISALLOW_COPY_AND_ASSIGN(DeleteSessionTensorOp); diff --git a/tensorflow/core/kernels/sparse/kernels.cc b/tensorflow/core/kernels/sparse/kernels.cc index 0eea9f1feed5c3..dff9aeb83ccfec 100644 --- a/tensorflow/core/kernels/sparse/kernels.cc +++ b/tensorflow/core/kernels/sparse/kernels.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/errors.h" namespace tensorflow { namespace functor { @@ -63,6 +64,11 @@ Status SparseTensorToCSRSparseMatrixCPUFunctor::operator()( for (int64 i = 0; i < total_nnz; ++i) { // For now, the rows pointers store the corresponding row counts. + int64 ix = indices(i, 0) + 1; + if (ix >= csr_row_ptr.size()) { + return errors::InvalidArgument("Got an index ", ix, + " that is outside of csr_row_ptr"); + } csr_row_ptr(indices(i, 0) + 1) += 1; csr_col_ind(i) = indices(i, 1); } diff --git a/tensorflow/core/kernels/sparse/sparse_cholesky_op.cc b/tensorflow/core/kernels/sparse/sparse_cholesky_op.cc index 9a939276f0b6cb..47ab252317de5e 100644 --- a/tensorflow/core/kernels/sparse/sparse_cholesky_op.cc +++ b/tensorflow/core/kernels/sparse/sparse_cholesky_op.cc @@ -17,6 +17,8 @@ limitations under the License. #include #include +#include "tensorflow/core/framework/op_requires.h" + #define EIGEN_USE_THREADS #include "third_party/eigen3/Eigen/Core" @@ -82,8 +84,8 @@ class CSRSparseCholeskyCPUOp : public OpKernel { int64 num_rows; int batch_size; - ValidateInputs(ctx, *input_matrix, input_permutation_indices, &batch_size, - &num_rows); + OP_REQUIRES_OK(ctx, ValidateInputs(*input_matrix, input_permutation_indices, + &batch_size, &num_rows)); // Allocate batch pointers. Tensor batch_ptr(cpu_allocator(), DT_INT32, TensorShape({batch_size + 1})); @@ -226,49 +228,48 @@ class CSRSparseCholeskyCPUOp : public OpKernel { } private: - void ValidateInputs(OpKernelContext* ctx, - const CSRSparseMatrix& sparse_matrix, - const Tensor& permutation_indices, int* batch_size, - int64* num_rows) { - OP_REQUIRES(ctx, sparse_matrix.dtype() == DataTypeToEnum::value, - errors::InvalidArgument( - "Asked for a CSRSparseMatrix of type ", - DataTypeString(DataTypeToEnum::value), - " but saw dtype: ", DataTypeString(sparse_matrix.dtype()))); + Status ValidateInputs(const CSRSparseMatrix& sparse_matrix, + const Tensor& permutation_indices, int* batch_size, + int64* num_rows) { + if (sparse_matrix.dtype() != DataTypeToEnum::value) + return errors::InvalidArgument( + "Asked for a CSRSparseMatrix of type ", + DataTypeString(DataTypeToEnum::value), + " but saw dtype: ", DataTypeString(sparse_matrix.dtype())); const Tensor& dense_shape = sparse_matrix.dense_shape(); const int rank = dense_shape.dim_size(0); - OP_REQUIRES(ctx, rank == 2 || rank == 3, - errors::InvalidArgument("sparse matrix must have rank 2 or 3; ", - "but dense_shape has size ", rank)); + if (rank < 2 || rank > 3) + return errors::InvalidArgument("sparse matrix must have rank 2 or 3; ", + "but dense_shape has size ", rank); const int row_dim = (rank == 2) ? 0 : 1; auto dense_shape_vec = dense_shape.vec(); *num_rows = dense_shape_vec(row_dim); const int64 num_cols = dense_shape_vec(row_dim + 1); - OP_REQUIRES(ctx, *num_rows == num_cols, - errors::InvalidArgument("sparse matrix must be square; got: ", - *num_rows, " != ", num_cols)); + if (*num_rows != num_cols) + return errors::InvalidArgument( + "sparse matrix must be square; got: ", *num_rows, " != ", num_cols); const TensorShape& perm_shape = permutation_indices.shape(); - OP_REQUIRES( - ctx, perm_shape.dims() + 1 == rank, - errors::InvalidArgument( - "sparse matrix must have the same rank as permutation; got: ", rank, - " != ", perm_shape.dims(), " + 1.")); - OP_REQUIRES( - ctx, perm_shape.dim_size(rank - 2) == *num_rows, - errors::InvalidArgument( - "permutation must have the same number of elements in each batch " - "as the number of rows in sparse matrix; got: ", - perm_shape.dim_size(rank - 2), " != ", *num_rows)); + if (perm_shape.dims() + 1 != rank) + return errors::InvalidArgument( + "sparse matrix must have the same rank as permutation; got: ", rank, + " != ", perm_shape.dims(), " + 1."); + if (perm_shape.dim_size(rank - 2) != *num_rows) + return errors::InvalidArgument( + "permutation must have the same number of elements in each batch " + "as the number of rows in sparse matrix; got: ", + perm_shape.dim_size(rank - 2), " != ", *num_rows); *batch_size = sparse_matrix.batch_size(); if (*batch_size > 1) { - OP_REQUIRES( - ctx, perm_shape.dim_size(0) == *batch_size, - errors::InvalidArgument("permutation must have the same batch size " - "as sparse matrix; got: ", - perm_shape.dim_size(0), " != ", *batch_size)); + if (perm_shape.dim_size(0) != *batch_size) + return errors::InvalidArgument( + "permutation must have the same batch size " + "as sparse matrix; got: ", + perm_shape.dim_size(0), " != ", *batch_size); } + + return Status::OK(); } }; diff --git a/tensorflow/core/kernels/sparse_add_op.cc b/tensorflow/core/kernels/sparse_add_op.cc index d16317af671dd6..e2cec69b14c46b 100644 --- a/tensorflow/core/kernels/sparse_add_op.cc +++ b/tensorflow/core/kernels/sparse_add_op.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/op_requires.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_util.h" @@ -43,6 +44,11 @@ class SparseAddOp : public OpKernel { b_indices->shape().DebugString())); const int64 a_nnz = a_indices->dim_size(0); const int64 b_nnz = b_indices->dim_size(0); + const int num_dims = a_indices->dim_size(1); + OP_REQUIRES(ctx, b_indices->dim_size(1) == num_dims, + errors::InvalidArgument( + "Input indices must have the same dimension, got ", + num_dims, " and ", b_indices->dim_size(1))); OP_REQUIRES_OK(ctx, ctx->input("a_values", &a_values_t)); OP_REQUIRES_OK(ctx, ctx->input("b_values", &b_values_t)); @@ -71,6 +77,13 @@ class SparseAddOp : public OpKernel { "Input shapes should be a vector but received shapes ", a_shape->shape().DebugString(), " and ", b_shape->shape().DebugString())); + OP_REQUIRES( + ctx, a_shape->NumElements() == num_dims, + errors::InvalidArgument("Second dimension of a_indices and length of " + "a_shape must match, got ", + num_dims, " and ", a_shape->NumElements())); + OP_REQUIRES(ctx, num_dims > 0, + errors::InvalidArgument("Tesors must not be empty")); OP_REQUIRES( ctx, a_shape->IsSameSize(*b_shape), errors::InvalidArgument( @@ -99,7 +112,6 @@ class SparseAddOp : public OpKernel { std::vector> entries_to_copy; // from_a?, idx entries_to_copy.reserve(a_nnz + b_nnz); std::vector out_values; - const int num_dims = a_shape->dim_size(0); // The input and output sparse tensors are assumed to be ordered along // increasing dimension number. diff --git a/tensorflow/core/kernels/sparse_concat_op.cc b/tensorflow/core/kernels/sparse_concat_op.cc index 3b2a0cb0f34ed3..d49f92ea556eb2 100644 --- a/tensorflow/core/kernels/sparse_concat_op.cc +++ b/tensorflow/core/kernels/sparse_concat_op.cc @@ -27,6 +27,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_util.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/lib/gtl/inlined_vector.h" +#include "tensorflow/core/util/overflow.h" #include "tensorflow/core/util/sparse/sparse_tensor.h" namespace tensorflow { @@ -66,13 +67,32 @@ class SparseConcatOp : public OpKernel { OP_REQUIRES(context, shapes.size() == N, errors::InvalidArgument("Expected ", N, " input shapes, got ", shapes.size())); + bool overflow_ocurred = false; for (int i = 0; i < N; i++) { + int64 new_num_elements = 1; OP_REQUIRES(context, TensorShapeUtils::IsVector(shapes[i].shape()), errors::InvalidArgument( "Input shapes should be a vector but received shape ", shapes[i].shape().DebugString(), " at position ", i)); + auto input_shape_vector = shapes[i].vec(); + for (int j = 0; j < input_shape_vector.size(); j++) { + new_num_elements = + MultiplyWithoutOverflow(new_num_elements, input_shape_vector(j)); + if (new_num_elements < 0) { + overflow_ocurred = true; + break; + } + } + + if (overflow_ocurred) { + break; + } } + OP_REQUIRES( + context, !overflow_ocurred, + errors::Internal("Encountered overflow from large input shape.")); + const TensorShape input_shape(shapes[0].vec()); const int input_rank = input_shape.dims(); const int concat_dim = (concat_dim_attr_ < 0) diff --git a/tensorflow/core/kernels/sparse_cross_op.cc b/tensorflow/core/kernels/sparse_cross_op.cc index c7c538a945feb1..08d0c21c4655cf 100644 --- a/tensorflow/core/kernels/sparse_cross_op.cc +++ b/tensorflow/core/kernels/sparse_cross_op.cc @@ -26,6 +26,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/fingerprint.h" @@ -295,6 +296,7 @@ class SparseCrossOp : public OpKernel { int64 signed_hash_key_; OP_REQUIRES_OK(context, context->GetAttr("hash_key", &signed_hash_key_)); hash_key_ = static_cast(signed_hash_key_); + OP_REQUIRES_OK(context, context->GetAttr("internal_type", &internal_type_)); } void Compute(OpKernelContext* context) override { @@ -308,8 +310,10 @@ class SparseCrossOp : public OpKernel { OP_REQUIRES_OK(context, context->input_list("dense_inputs", &dense_list_in)); - OP_REQUIRES_OK(context, ValidateInput(indices_list_in, values_list_in, - shapes_list_in, dense_list_in)); + DataType internal_type = internal_type_; + OP_REQUIRES_OK( + context, ValidateInput(indices_list_in, values_list_in, shapes_list_in, + dense_list_in, internal_type)); std::vector>> columns = GenerateColumnsFromInput(indices_list_in, values_list_in, @@ -353,10 +357,19 @@ class SparseCrossOp : public OpKernel { Status ValidateInput(const OpInputList& indices_list_in, const OpInputList& values_list_in, const OpInputList& shapes_list_in, - const OpInputList& dense_list_in) { + const OpInputList& dense_list_in, + const DataType& internal_type) { const auto size = indices_list_in.size(); + // Only perform internal_type check for SparseCrossOp. + // Check if the internal_type is not invalid before doing so. + bool check_type = internal_type != DT_INVALID; // Validates indices_list_in OpInputList. for (int i = 0; i < size; i++) { + if (check_type && indices_list_in[i].dtype() != DT_INT64) { + return errors::InvalidArgument("Input indices should be of type ", + DT_INT64, " but received ", + indices_list_in[i].dtype()); + } if (!TensorShapeUtils::IsMatrix(indices_list_in[i].shape())) { return errors::InvalidArgument( "Input indices should be a matrix but received shape ", @@ -375,6 +388,14 @@ class SparseCrossOp : public OpKernel { values_list_in.size()); } for (int i = 0; i < size; i++) { + // Make sure to avoid the expected type to be string, but input values to be + // int64. + if (check_type && internal_type == DT_STRING && + values_list_in[i].dtype() == DT_INT64) { + return errors::InvalidArgument("Input values should be of internal type ", + internal_type, " but received ", + values_list_in[i].dtype()); + } if (!TensorShapeUtils::IsVector(values_list_in[i].shape())) { return errors::InvalidArgument( "Input values should be a vector but received shape ", @@ -395,6 +416,11 @@ class SparseCrossOp : public OpKernel { shapes_list_in.size()); } for (int i = 0; i < size; i++) { + if (check_type && shapes_list_in[i].dtype() != DT_INT64) { + return errors::InvalidArgument("Input shape should be of type ", DT_INT64, + " but received ", + shapes_list_in[i].dtype()); + } if (!TensorShapeUtils::IsVector(shapes_list_in[i].shape())) { return errors::InvalidArgument( "Input shapes should be a vector but received shape ", @@ -410,6 +436,14 @@ class SparseCrossOp : public OpKernel { // Validates dense_list_in OpInputList for (int i = 0; i < dense_list_in.size(); ++i) { + // Make sure to avoid the expected type to be string, but input values to be + // int64. + if (check_type && internal_type == DT_STRING && + dense_list_in[i].dtype() == DT_INT64) { + return errors::InvalidArgument("Dense inputs should be of internal type ", + internal_type, " but received ", + dense_list_in[i].dtype()); + } if (!TensorShapeUtils::IsMatrix(dense_list_in[i].shape())) { return errors::InvalidArgument( "Dense inputs should be a matrix but received shape ", @@ -563,6 +597,7 @@ class SparseCrossOp : public OpKernel { } int64 num_buckets_; uint64 hash_key_; + DataType internal_type_; }; REGISTER_KERNEL_BUILDER(Name("SparseCross") diff --git a/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc b/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc index 3a5e66a0e73ea6..dac4a3d3e6bfcd 100644 --- a/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc +++ b/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc @@ -78,6 +78,11 @@ class SparseDenseBinaryOpShared : public OpKernel { "but received shapes: ", values_t->shape().DebugString(), " and ", shape_t->shape().DebugString())); + OP_REQUIRES( + ctx, values_t->dim_size(0) == indices_t->dim_size(0), + errors::InvalidArgument( + "The first dimension of values and indices should match. (", + values_t->dim_size(0), " vs. ", indices_t->dim_size(0), ")")); const auto indices_mat = indices_t->matrix(); const auto shape_vec = shape_t->vec(); diff --git a/tensorflow/core/kernels/sparse_fill_empty_rows_op.cc b/tensorflow/core/kernels/sparse_fill_empty_rows_op.cc index 8de93cf9b30d74..2b80903a9210b1 100644 --- a/tensorflow/core/kernels/sparse_fill_empty_rows_op.cc +++ b/tensorflow/core/kernels/sparse_fill_empty_rows_op.cc @@ -69,7 +69,9 @@ class SparseFillEmptyRowsOp : public OpKernel { default_value_t.shape().DebugString())); // TODO(ebrevdo): add shape checks between values, indices, // dense_shape. Also add check that dense rank > 0. - + // Also add check that dense rank > 0. + OP_REQUIRES(context, dense_shape_t.NumElements() != 0, + errors::InvalidArgument("Dense shape cannot be empty.")); const T& default_value = default_value_t.scalar()(); const auto indices = indices_t.matrix(); const auto values = values_t.vec(); @@ -232,6 +234,9 @@ class SparseFillEmptyRowsGradOp : public OpKernel { context, TensorShapeUtils::IsVector(reverse_index_map_t->shape()), errors::InvalidArgument("reverse_index_map must be a vector, saw: ", reverse_index_map_t->shape().DebugString())); + OP_REQUIRES(context, TensorShapeUtils::IsVector(grad_values_t->shape()), + errors::InvalidArgument("grad_values must be a vector, saw: ", + grad_values_t->shape().DebugString())); const auto reverse_index_map = reverse_index_map_t->vec(); const auto grad_values = grad_values_t->vec(); @@ -260,8 +265,13 @@ class SparseFillEmptyRowsGradOp : public OpKernel { // Locate the index of the output of the forward prop associated // with this location in the input of the forward prop. Copy // the gradient into it. Mark it as visited. - d_values(i) = grad_values(reverse_index_map(i)); - visited(reverse_index_map(i)) = true; + int64 reverse_index = reverse_index_map(i); + OP_REQUIRES( + context, 0 <= reverse_index && reverse_index < N_full, + errors::InvalidArgument("Elements in reverse index must be in [0, ", + N_full, ") but got ", reverse_index)); + d_values(i) = grad_values(reverse_index); + visited(reverse_index) = true; } for (int j = 0; j < N_full; ++j) { // The default value gradient gets the accumulated remainder of diff --git a/tensorflow/core/kernels/sparse_matmul_op.cc b/tensorflow/core/kernels/sparse_matmul_op.cc index eb460147d71f39..1f66f8058a613a 100644 --- a/tensorflow/core/kernels/sparse_matmul_op.cc +++ b/tensorflow/core/kernels/sparse_matmul_op.cc @@ -1039,6 +1039,10 @@ class SparseMatMulOp : public OpKernel { if (transpose_b) { // TODO(agarwal): avoid transposing the matrix here and directly handle // transpose in CreateDenseSlices. + OP_REQUIRES(ctx, right->dim_size(0) != 0, + errors::InvalidArgument("b has an entry 0 in it's shape.")); + OP_REQUIRES(ctx, right->dim_size(1) != 0, + errors::InvalidArgument("b has an entry 0 in it's shape.")); right_tr.reset( new Tensor(right->dtype(), TensorShape({right->dim_size(1), right->dim_size(0)}))); diff --git a/tensorflow/core/kernels/sparse_reshape_op.cc b/tensorflow/core/kernels/sparse_reshape_op.cc index 059519a913b7e7..23c595da354daa 100644 --- a/tensorflow/core/kernels/sparse_reshape_op.cc +++ b/tensorflow/core/kernels/sparse_reshape_op.cc @@ -26,6 +26,7 @@ limitations under the License. #include "tensorflow/core/framework/types.h" #include "tensorflow/core/kernels/reshape_util.h" #include "tensorflow/core/lib/gtl/inlined_vector.h" +#include "tensorflow/core/platform/errors.h" namespace tensorflow { @@ -34,6 +35,17 @@ class SparseReshapeOp : public OpKernel { explicit SparseReshapeOp(OpKernelConstruction* context) : OpKernel(context) {} void Compute(OpKernelContext* context) override { + const Tensor& input_indices_in = context->input(0); + const Tensor& input_shape_in = context->input(1); + + OP_REQUIRES(context, TensorShapeUtils::IsMatrix(input_indices_in.shape()), + errors::InvalidArgument("Input must be a matrix.")); + OP_REQUIRES(context, TensorShapeUtils::IsVector(input_shape_in.shape()), + errors::InvalidArgument("Input shape must be a vector.")); + OP_REQUIRES(context, + input_indices_in.dim_size(1) == input_shape_in.dim_size(0), + errors::InvalidArgument( + "Input tensor rank must match input shape length.")); Reshape(context, context->input(0), context->input(1), context->input(2), 0 /* output indices index */, 1 /* output shape index */); } diff --git a/tensorflow/core/kernels/sparse_sparse_binary_op_shared.cc b/tensorflow/core/kernels/sparse_sparse_binary_op_shared.cc index 43dc9ae70cd627..eb993a5965043b 100644 --- a/tensorflow/core/kernels/sparse_sparse_binary_op_shared.cc +++ b/tensorflow/core/kernels/sparse_sparse_binary_op_shared.cc @@ -150,6 +150,7 @@ class SparseSparseBinaryOpShared : public OpKernel { const int64 a_nnz = a_indices_t->dim_size(0); const int64 b_nnz = b_indices_t->dim_size(0); + const auto a_values = a_values_t->vec(); const auto b_values = b_values_t->vec(); @@ -166,6 +167,14 @@ class SparseSparseBinaryOpShared : public OpKernel { "Input shapes should be a vector but received shapes ", a_shape_t->shape().DebugString(), " and ", b_shape_t->shape().DebugString())); + const int num_dims = a_indices_t->dim_size(1); + OP_REQUIRES( + ctx, a_shape_t->NumElements() == num_dims, + errors::InvalidArgument("Second dimension of a_indices and length of " + "a_shape must match, got ", + num_dims, " and ", a_shape_t->NumElements())); + OP_REQUIRES(ctx, num_dims > 0, + errors::InvalidArgument("Tensors must not be empty")); OP_REQUIRES(ctx, a_shape_t->IsSameSize(*b_shape_t), errors::InvalidArgument( "Operands do not have the same ranks; got shapes: ", @@ -180,7 +189,6 @@ class SparseSparseBinaryOpShared : public OpKernel { " for dimension ", i)); } - const int num_dims = a_indices_t->dim_size(1); const auto a_indices_mat = a_indices_t->matrix(); const auto b_indices_mat = b_indices_t->matrix(); std::vector a_augmented_values, b_augmented_values; diff --git a/tensorflow/core/kernels/sparse_split_op.cc b/tensorflow/core/kernels/sparse_split_op.cc index 3d02be47cbbef5..b0c147da8a8344 100644 --- a/tensorflow/core/kernels/sparse_split_op.cc +++ b/tensorflow/core/kernels/sparse_split_op.cc @@ -18,6 +18,7 @@ limitations under the License. #include #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/util/overflow.h" #include "tensorflow/core/util/sparse/sparse_tensor.h" namespace tensorflow { @@ -63,6 +64,16 @@ class SparseSplitOp : public OpKernel { input_shape.vec()(split_dim), "), got ", num_split_)); + // Prevent overflow by constructing the dense shape separately + int64 total_elements = 1; + const auto input_shape_flat = input_shape.flat(); + for (int i = 0; i < input_shape.NumElements(); i++) { + total_elements = + MultiplyWithoutOverflow(total_elements, input_shape_flat(i)); + OP_REQUIRES(context, total_elements >= 0, + errors::Internal("Encountered overflow in dense shape")); + } + sparse::SparseTensor sparse_tensor; OP_REQUIRES_OK(context, sparse::SparseTensor::Create( diff --git a/tensorflow/core/kernels/sparse_tensors_map_ops.cc b/tensorflow/core/kernels/sparse_tensors_map_ops.cc index c2c0e43ca2ba8d..5ea5fca544d3e9 100644 --- a/tensorflow/core/kernels/sparse_tensors_map_ops.cc +++ b/tensorflow/core/kernels/sparse_tensors_map_ops.cc @@ -21,9 +21,6 @@ limitations under the License. #include #include -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/register_types.h" - #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/resource_mgr.h" @@ -31,6 +28,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_util.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/lib/gtl/inlined_vector.h" +#include "tensorflow/core/util/overflow.h" #include "tensorflow/core/util/sparse/sparse_tensor.h" namespace tensorflow { @@ -254,7 +252,22 @@ class AddManySparseToTensorsMapOp : public SparseTensorAccessingOp { errors::InvalidArgument( "Rank of input SparseTensor should be > 1, but saw rank: ", rank)); - TensorShape tensor_input_shape(input_shape->vec()); + auto input_shape_vec = input_shape->vec(); + int new_num_elements = 1; + bool overflow_ocurred = false; + for (int i = 0; i < input_shape_vec.size(); i++) { + new_num_elements = + MultiplyWithoutOverflow(new_num_elements, input_shape_vec(i)); + if (new_num_elements < 0) { + overflow_ocurred = true; + } + } + + OP_REQUIRES( + context, !overflow_ocurred, + errors::Internal("Encountered overflow from large input shape.")); + + TensorShape tensor_input_shape(input_shape_vec); gtl::InlinedVector std_order(rank); std::iota(std_order.begin(), std_order.end(), 0); SparseTensor input_st; @@ -262,8 +275,7 @@ class AddManySparseToTensorsMapOp : public SparseTensorAccessingOp { tensor_input_shape, std_order, &input_st)); - auto input_shape_t = input_shape->vec(); - const int64 N = input_shape_t(0); + const int64 N = input_shape_vec(0); Tensor sparse_handles(DT_INT64, TensorShape({N})); auto sparse_handles_t = sparse_handles.vec(); @@ -274,7 +286,7 @@ class AddManySparseToTensorsMapOp : public SparseTensorAccessingOp { // minibatch entries. TensorShape output_shape; OP_REQUIRES_OK(context, TensorShapeUtils::MakeShape( - input_shape_t.data() + 1, + input_shape_vec.data() + 1, input_shape->NumElements() - 1, &output_shape)); // Get groups by minibatch dimension diff --git a/tensorflow/core/kernels/stateless_random_ops.cc b/tensorflow/core/kernels/stateless_random_ops.cc index 167daf2ff9e9d3..93e34170860429 100644 --- a/tensorflow/core/kernels/stateless_random_ops.cc +++ b/tensorflow/core/kernels/stateless_random_ops.cc @@ -252,7 +252,7 @@ class StatelessRandomGammaOp : public StatelessRandomOpBase { // avoid a couple flops which can be done on a per-alpha basis. auto DoWork = [samples_per_alpha, num_alphas, &random, samples_flat, - alpha_flat](int start_output, int limit_output) { + alpha_flat](int64 start_output, int64 limit_output) { // Capturing "random" by-value would only make a copy for the _shared_ // lambda. Since we want to let each worker have its own copy, we pass // "random" by reference and explicitly do a copy assignment. diff --git a/tensorflow/core/kernels/string_ngrams_op.cc b/tensorflow/core/kernels/string_ngrams_op.cc index 97b32c4242ccdc..7008a1d766af25 100644 --- a/tensorflow/core/kernels/string_ngrams_op.cc +++ b/tensorflow/core/kernels/string_ngrams_op.cc @@ -19,6 +19,7 @@ limitations under the License. #include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/platform/errors.h" namespace tensorflow { namespace text { @@ -60,6 +61,30 @@ class StringNGramsOp : public tensorflow::OpKernel { OP_REQUIRES_OK(context, context->input("data_splits", &splits)); const auto& splits_vec = splits->flat(); + // Validate that the splits are valid indices into data, only if there are + // splits specified. + const int input_data_size = data->flat().size(); + const int splits_vec_size = splits_vec.size(); + if (splits_vec_size > 0) { + int prev_split = splits_vec(0); + OP_REQUIRES(context, prev_split == 0, + errors::InvalidArgument("First split value must be 0, got ", + prev_split)); + for (int i = 1; i < splits_vec_size; ++i) { + bool valid_splits = splits_vec(i) >= prev_split; + valid_splits = valid_splits && (splits_vec(i) <= input_data_size); + OP_REQUIRES(context, valid_splits, + errors::InvalidArgument( + "Invalid split value ", splits_vec(i), ", must be in [", + prev_split, ", ", input_data_size, "]")); + prev_split = splits_vec(i); + } + OP_REQUIRES(context, prev_split == input_data_size, + errors::InvalidArgument( + "Last split value must be data size. Expected ", + input_data_size, ", got ", prev_split)); + } + int num_batch_items = splits_vec.size() - 1; tensorflow::Tensor* ngrams_splits; OP_REQUIRES_OK( @@ -161,13 +186,31 @@ class StringNGramsOp : public tensorflow::OpKernel { ngram->append(left_pad_); ngram->append(separator_); } + // Only output first num_tokens - 1 pairs of data and separator for (int n = 0; n < num_tokens - 1; ++n) { ngram->append(data[data_start_index + n]); ngram->append(separator_); } - ngram->append(data[data_start_index + num_tokens - 1]); - for (int n = 0; n < right_padding; ++n) { - ngram->append(separator_); + // Handle case when there are no tokens or no right padding as these can + // result in consecutive separators. + if (num_tokens > 0) { + // If we have tokens, then output last and then pair each separator with + // the right padding that follows, to ensure ngram ends either with the + // token or with the right pad. + ngram->append(data[data_start_index + num_tokens - 1]); + for (int n = 0; n < right_padding; ++n) { + ngram->append(separator_); + ngram->append(right_pad_); + } + } else { + // If we don't have tokens, then the last item inserted into the ngram + // has been the separator from the left padding loop above. Hence, + // output right pad and separator and make sure to finish with a + // padding, not a separator. + for (int n = 0; n < right_padding - 1; ++n) { + ngram->append(right_pad_); + ngram->append(separator_); + } ngram->append(right_pad_); } diff --git a/tensorflow/core/kernels/string_ngrams_op_test.cc b/tensorflow/core/kernels/string_ngrams_op_test.cc index b89de9ad16dab8..0d52283bd8fb9d 100644 --- a/tensorflow/core/kernels/string_ngrams_op_test.cc +++ b/tensorflow/core/kernels/string_ngrams_op_test.cc @@ -542,6 +542,40 @@ TEST_F(NgramKernelTest, TestEmptyInput) { assert_int64_equal(expected_splits, *GetOutput(1)); } +TEST_F(NgramKernelTest, TestNoTokens) { + MakeOp("|", {3}, "L", "R", -1, false); + // Batch items are: + // 0: + // 1: "a" + AddInputFromArray(TensorShape({1}), {"a"}); + AddInputFromArray(TensorShape({3}), {0, 0, 1}); + TF_ASSERT_OK(RunOpKernel()); + + std::vector expected_values( + {"L|L|R", "L|R|R", // no input in first split + "L|L|a", "L|a|R", "a|R|R"}); // second split + std::vector expected_splits({0, 2, 5}); + + assert_string_equal(expected_values, *GetOutput(0)); + assert_int64_equal(expected_splits, *GetOutput(1)); +} + +TEST_F(NgramKernelTest, TestNoTokensNoPad) { + MakeOp("|", {3}, "", "", 0, false); + // Batch items are: + // 0: + // 1: "a" + AddInputFromArray(TensorShape({1}), {"a"}); + AddInputFromArray(TensorShape({3}), {0, 0, 1}); + TF_ASSERT_OK(RunOpKernel()); + + std::vector expected_values({}); + std::vector expected_splits({0, 0, 0}); + + assert_string_equal(expected_values, *GetOutput(0)); + assert_int64_equal(expected_splits, *GetOutput(1)); +} + TEST_F(NgramKernelTest, ShapeFn) { ShapeInferenceTestOp op("StringNGrams"); INFER_OK(op, "?;?", "[?];[?]"); diff --git a/tensorflow/core/kernels/substr_op.cc b/tensorflow/core/kernels/substr_op.cc index e382381e122324..ab83efda2a2e17 100644 --- a/tensorflow/core/kernels/substr_op.cc +++ b/tensorflow/core/kernels/substr_op.cc @@ -51,6 +51,11 @@ class SubstrOp : public OpKernel { const Tensor& len_tensor = context->input(2); const TensorShape& input_shape = input_tensor.shape(); const TensorShape& pos_shape = pos_tensor.shape(); + const TensorShape& len_shape = len_tensor.shape(); + OP_REQUIRES(context, (pos_shape == len_shape), + errors::InvalidArgument( + "pos and len should have the same shape, got: ", + pos_shape.DebugString(), " vs. ", len_shape.DebugString())); bool is_scalar = TensorShapeUtils::IsScalar(pos_shape); diff --git a/tensorflow/core/kernels/topk_op.cc b/tensorflow/core/kernels/topk_op.cc index 02b99e44880a56..327ed5bfa25f29 100644 --- a/tensorflow/core/kernels/topk_op.cc +++ b/tensorflow/core/kernels/topk_op.cc @@ -136,7 +136,7 @@ struct TopKFunctor { return Status::OK(); } - auto SortIndices = [&](int start_batch, int limit_batch) { + auto SortIndices = [&](int64 start_batch, int64 limit_batch) { for (int32 b = start_batch; b < limit_batch; ++b) { const T* input_data = &input(b, 0); const auto stable_comp = [input_data](const int32 a, const int32 b) { diff --git a/tensorflow/core/kernels/transpose_functor.h b/tensorflow/core/kernels/transpose_functor.h index a89fc40d772594..4b07a076296aa4 100644 --- a/tensorflow/core/kernels/transpose_functor.h +++ b/tensorflow/core/kernels/transpose_functor.h @@ -19,6 +19,7 @@ limitations under the License. #include #include #include + #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/platform/logging.h" @@ -166,7 +167,6 @@ template Status DoTransposeImpl(const Device& d, const Tensor& in, const gtl::ArraySlice perm, bool conjugate, Tensor* out) { - CHECK_GE(in.dims(), 2); CHECK_EQ(in.dims(), out->dims()); CHECK_EQ(in.dims(), perm.size()); CHECK_EQ(in.dtype(), out->dtype()); diff --git a/tensorflow/core/kernels/unicode_ops.cc b/tensorflow/core/kernels/unicode_ops.cc index 14ae49c837fef9..0e7a24ddd9be79 100644 --- a/tensorflow/core/kernels/unicode_ops.cc +++ b/tensorflow/core/kernels/unicode_ops.cc @@ -533,6 +533,17 @@ class UnicodeEncodeOp : public OpKernel { const Tensor& input_splits = context->input(1); const auto input_splits_flat = input_splits.flat(); + // Operation will treat first argument in input_splits as if it were zero + // regardless of its actual value since splits should begin with zero and + // end with the length of the input values vector. + OP_REQUIRES( + context, input_splits_flat(0) == 0, + errors::InvalidArgument("First value in input_splits must be zero.")); + OP_REQUIRES(context, + input_splits_flat(input_splits_flat.size() - 1) == + input_tensor_flat.size(), + errors::InvalidArgument("Last value in input_splits must be " + "equal to length of input_tensor.")); // Since we limit to a 2-D input (flat_values of rank 1 and a single splits // tensor), our output dimension will be 1 with it's size equal to the // number of splits (outer dimension or ragged tensor). @@ -548,6 +559,14 @@ class UnicodeEncodeOp : public OpKernel { for (int i = 1; i < input_splits_flat.size(); ++i) { icu::UnicodeString unicode_string; icu::UnicodeStringAppendable appendable_unicode_string(unicode_string); + OP_REQUIRES( + context, input_splits_flat(i - 1) <= input_splits_flat(i), + errors::InvalidArgument( + "Values in input_splits must be equal or in ascending order.")); + OP_REQUIRES( + context, input_splits_flat(i) <= input_tensor_flat.size(), + errors::InvalidArgument("Values in input_splits must be less than or " + "equal to input_tensor length.")); for (; idx < input_splits_flat(i); ++idx) { int32 code_point = input_tensor_flat(idx); // Check for invalid code point diff --git a/tensorflow/core/kernels/unsorted_segment_join_op.cc b/tensorflow/core/kernels/unsorted_segment_join_op.cc index 7464e165e46c8b..9acfe7fb1e4952 100644 --- a/tensorflow/core/kernels/unsorted_segment_join_op.cc +++ b/tensorflow/core/kernels/unsorted_segment_join_op.cc @@ -90,6 +90,8 @@ class UnsortedSegmentJoinOp : public OpKernel { const int32 segment_dims = segment_id_shape.dims(); const Tensor& num_segments_tensor = context->input(2); + OP_REQUIRES(context, num_segments_tensor.NumElements() != 0, + errors::InvalidArgument("Number of segments cannot be empty.")); auto num_segments = num_segments_tensor.scalar()(); OP_REQUIRES(context, segment_dims != 0, diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 921248e8aa777b..334cbafa2bdcc7 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -22,7 +22,7 @@ limitations under the License. // tensorflow/tools/pip_package/setup.py #define TF_MAJOR_VERSION 2 #define TF_MINOR_VERSION 2 -#define TF_PATCH_VERSION 0 +#define TF_PATCH_VERSION 3 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", // "-beta", "-rc", "-rc.1") diff --git a/tensorflow/core/util/sparse/sparse_tensor.h b/tensorflow/core/util/sparse/sparse_tensor.h index 2654d126e861cb..1d57e4c310c926 100644 --- a/tensorflow/core/util/sparse/sparse_tensor.h +++ b/tensorflow/core/util/sparse/sparse_tensor.h @@ -525,6 +525,10 @@ inline Status SparseTensor::Split(const SparseTensor& input_tensor, for (int i = 0; i < input_tensor.indices().dim_size(0); ++i) { const int dim = input_tensor.indices().matrix()(i, split_dim); int slice_index = GetSliceIndex(dim, split_size, residual); + if (slice_index >= num_values.size()) { + return errors::InvalidArgument("Slice index ", slice_index, + " is larger than num_split."); + } num_values[slice_index]++; } diff --git a/tensorflow/java/maven/spark-tensorflow-connector/pom.xml b/tensorflow/java/maven/spark-tensorflow-connector/pom.xml index f40090ac45d6d9..19f5e29da2bf38 100644 --- a/tensorflow/java/maven/spark-tensorflow-connector/pom.xml +++ b/tensorflow/java/maven/spark-tensorflow-connector/pom.xml @@ -35,7 +35,7 @@ 1.8 2.4.5 2.7.3 - 4.11 + 4.13.1 diff --git a/tensorflow/java/maven/tensorflow-hadoop/pom.xml b/tensorflow/java/maven/tensorflow-hadoop/pom.xml index e900d81e5dab50..675a3369cf1ff3 100644 --- a/tensorflow/java/maven/tensorflow-hadoop/pom.xml +++ b/tensorflow/java/maven/tensorflow-hadoop/pom.xml @@ -16,7 +16,7 @@ 1.6 2.6.0 3.5.1 - 4.11 + 4.13.1 diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD index e9539d42f7540f..85a22251602c7d 100644 --- a/tensorflow/lite/BUILD +++ b/tensorflow/lite/BUILD @@ -357,6 +357,7 @@ cc_test( "testdata/test_min_runtime.bin", "testdata/test_model.bin", "testdata/test_model_broken.bin", + "testdata/unsupported_recursion.bin", ], tags = [ "tflite_not_portable", diff --git a/tensorflow/lite/c/common.c b/tensorflow/lite/c/common.c index 7196f32b62abad..e0880df188d362 100644 --- a/tensorflow/lite/c/common.c +++ b/tensorflow/lite/c/common.c @@ -43,8 +43,10 @@ int TfLiteIntArrayEqualsArray(const TfLiteIntArray* a, int b_size, #ifndef TF_LITE_STATIC_MEMORY TfLiteIntArray* TfLiteIntArrayCreate(int size) { - TfLiteIntArray* ret = - (TfLiteIntArray*)malloc(TfLiteIntArrayGetSizeInBytes(size)); + int alloc_size = TfLiteIntArrayGetSizeInBytes(size); + if (alloc_size <= 0) return NULL; + TfLiteIntArray* ret = (TfLiteIntArray*)malloc(alloc_size); + if (!ret) return ret; ret->size = size; return ret; } diff --git a/tensorflow/lite/core/subgraph.cc b/tensorflow/lite/core/subgraph.cc index 09e70390f0f560..62f52479ec9270 100644 --- a/tensorflow/lite/core/subgraph.cc +++ b/tensorflow/lite/core/subgraph.cc @@ -18,6 +18,7 @@ limitations under the License. #include #include "tensorflow/lite/arena_planner.h" +#include "tensorflow/lite/builtin_ops.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/context_util.h" #include "tensorflow/lite/core/api/tensor_utils.h" @@ -138,6 +139,38 @@ const char* GetTFLiteOpName(const TfLiteRegistration& op_reg) { return tflite::EnumNamesBuiltinOperator()[op_reg.builtin_code]; } +// An utility test to detect if the subgraph is abused: +// 1. Detects if recursion exists in the graph (recursion is not currently +// supported. +// 2. Detects if the interpreter / subgraph is used in multiple subgraphs. +// Note: It's clearly documented that the interpreter / subgraph are not +// thread-safe. This serves as a check with possible false negatives +// unless we switch to atomic boolean flags. +class SubgraphGuard { + public: + SubgraphGuard(TfLiteContext* context, bool* is_subgraph_in_use) + : is_subgraph_in_use_(is_subgraph_in_use) { + if (*is_subgraph_in_use_) { + context->ReportError(context, "Subgraph is already in use."); + status_ = kTfLiteError; + } else { + *is_subgraph_in_use_ = true; + } + } + ~SubgraphGuard() { + // If tht original status was OK, recover the boolean flag. + if (status_ == kTfLiteOk) { + *is_subgraph_in_use_ = false; + } + } + + TfLiteStatus status() const { return status_; } + + private: + TfLiteStatus status_ = kTfLiteOk; + bool* is_subgraph_in_use_; +}; + } // namespace // A trivial implementation of GraphInfo around the Interpreter. @@ -560,6 +593,33 @@ TfLiteStatus Subgraph::CheckTensorIndices(const char* label, const int* indices, return kTfLiteOk; } +// We have two arrays and we need to check that elements from one array don't +// show up in the other. We could sort both arrays and then iterate with two +// pointers from start to finish always increasing the smaller one but since +// these arrays are usually short (<25 elements for inputs, usually <3 for +// outputs), this might be slower than the naive approach (if arrays have size n +// and m, with n >> m ~ O(1), first approach is O(nlogn) whereas the other is +// O(n)). Plus, sorting the input and output arrays might not be something we +// want as it destroys ordering of elements. +// +// If it turns out that this is an issue, we can switch to the other algorithm. +TfLiteStatus Subgraph::CheckInputAndOutputForOverlap(const int* input_indices, + int num_inputs, + const int* output_indices, + int num_outputs) { + for (int i = 0; i < num_inputs; i++) { + for (int j = 0; j < num_outputs; j++) { + if (input_indices[i] == output_indices[j]) { + ReportError("Tensor %d is both input %d and output %d\n", + input_indices[i], i, j); + consistent_ = false; + return kTfLiteError; + } + } + } + return kTfLiteOk; +} + namespace { // Multiply two sizes and return true if overflow occurred; // This is based off tensorflow/overflow.h but is simpler as we already @@ -602,6 +662,7 @@ TfLiteStatus Subgraph::BytesRequired(TfLiteType type, const int* dims, TfLiteStatus Subgraph::AllocateTensors() { TFLITE_SCOPED_TAGGED_DEFAULT_PROFILE(profiler_.get(), "AllocateTensors"); + if (!consistent_) { ReportError("AllocateTensors() called on inconsistent model."); return kTfLiteError; @@ -625,6 +686,12 @@ TfLiteStatus Subgraph::AllocateTensors() { return kTfLiteOk; } + // Note `AllocateTensors` sometimes calls itself recursively above + // for delegates. Therefore only the logic below need to be guarded + // by `SubgraphGuard`. + SubgraphGuard guard(&context_, &is_subgraph_in_use_); + TF_LITE_ENSURE_OK(&context_, guard.status()); + next_execution_plan_index_to_prepare_ = 0; next_execution_plan_index_to_plan_allocation_ = 0; if (memory_planner_) { @@ -681,6 +748,16 @@ TfLiteStatus Subgraph::AddNodeWithParameters( &context_, CheckTensorIndices("node outputs", outputs.data(), outputs.size())); + // For builtin ops, inputs and outputs must not overlap. Custom ops must do + // this check by themselves if they don't support overlapping tensors. This + // distinction is to allow custom ops to just forward a tensor, reusing it as + // both input and output. + if (builtin_data != nullptr) { + TF_LITE_ENSURE_OK(&context_, CheckInputAndOutputForOverlap( + inputs.data(), inputs.size(), + outputs.data(), outputs.size())); + } + int new_node_index = nodes_and_registration_.size(); if (node_index) *node_index = new_node_index; nodes_and_registration_.resize(nodes_and_registration_.size() + 1); @@ -842,6 +919,9 @@ TfLiteStatus Subgraph::PrepareOpsAndTensors() { } TfLiteStatus Subgraph::Invoke() { + SubgraphGuard guard(&context_, &is_subgraph_in_use_); + TF_LITE_ENSURE_OK(&context_, guard.status()); + if (!consistent_) { ReportError("Invoke called on model that is not consistent."); return kTfLiteError; @@ -897,6 +977,26 @@ TfLiteStatus Subgraph::Invoke() { tensor->data_is_stale) { TF_LITE_ENSURE_STATUS(EnsureTensorDataIsReadable(tensor_index)); } + if (tensor->data.raw == nullptr && tensor->bytes > 0) { + if (registration.builtin_code == kTfLiteBuiltinReshape && i == 1 && + tensor->dims->size != 1) { + // In general, having a tensor here with no buffer will be an error. + // However, for the reshape operator, the second input tensor is + // sometimes only used for the shape, not for the data. Thus, null + // buffer is ok in this situation. + // The situation where null buffer is not ok for reshape operator is + // only when there are 2 inputs given to the node and the one + // corresponding to the shape (i == 1) is a vector that contains all + // dimensions. See `GetOutputShape()` function in + // `tensorflow/lite/kernels/reshape.cc` + continue; + } else { + // In all other cases, we need to return an error as otherwise we will + // trigger a null pointer dereference (likely). + ReportError("Input tensor %d lacks data", tensor_index); + return kTfLiteError; + } + } } if (check_cancelled_func_ != nullptr && diff --git a/tensorflow/lite/core/subgraph.h b/tensorflow/lite/core/subgraph.h index 4380feda283a2c..b2091ed9e31e36 100644 --- a/tensorflow/lite/core/subgraph.h +++ b/tensorflow/lite/core/subgraph.h @@ -415,6 +415,15 @@ class Subgraph { TfLiteStatus CheckTensorIndices(const char* label, const int* indices, int length); + // Check that the input indices and the output indices don't overlap. + // This is needed because same tensor must not be used both as input and + // output for an operator. + // NOTE: this changes consistent_ to be false if indices are out of bounds. + TfLiteStatus CheckInputAndOutputForOverlap(const int* input_indices, + int num_inputs, + const int* output_indices, + int num_outputs); + // Compute the number of bytes required to represent a tensor with dimensions // specified by the array dims (of length dims_size). Returns the status code // and bytes. @@ -673,6 +682,10 @@ class Subgraph { // A map of resources. Owned by interpreter and shared by multiple subgraphs. resource::ResourceMap* resources_ = nullptr; + + // Whether the subgraph is currently in use (e.g. running the `Invoke` + // or `AllocateTensors` functions). + bool is_subgraph_in_use_ = false; }; } // namespace tflite diff --git a/tensorflow/lite/kernels/arg_min_max.cc b/tensorflow/lite/kernels/arg_min_max.cc index e99f59ba703ee2..0264b313f2f647 100644 --- a/tensorflow/lite/kernels/arg_min_max.cc +++ b/tensorflow/lite/kernels/arg_min_max.cc @@ -36,6 +36,9 @@ TfLiteStatus ResizeOutput(TfLiteContext* context, const TfLiteTensor* input, axis_value += NumDimensions(input); } + TF_LITE_ENSURE(context, axis_value >= 0); + TF_LITE_ENSURE(context, axis_value < NumDimensions(input)); + // Copy the input dimensions to output except the axis dimension. TfLiteIntArray* output_dims = TfLiteIntArrayCreate(NumDimensions(input) - 1); int j = 0; diff --git a/tensorflow/lite/kernels/concatenation.cc b/tensorflow/lite/kernels/concatenation.cc index 18bd2786a29553..7bf5338058b6f8 100644 --- a/tensorflow/lite/kernels/concatenation.cc +++ b/tensorflow/lite/kernels/concatenation.cc @@ -19,6 +19,8 @@ limitations under the License. #include #include +#include + #include "tensorflow/lite/c/builtin_op_data.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h" @@ -69,6 +71,10 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, t->type, input_type); for (int d = 0; d < t0->dims->size; ++d) { if (d == axis) { + // Avoid integer overflow in sum_axis below + TF_LITE_ENSURE(context, t->dims->data[axis] >= 0); + TF_LITE_ENSURE(context, t->dims->data[axis] <= + std::numeric_limits::max() - sum_axis); sum_axis += t->dims->data[axis]; } else { TF_LITE_ENSURE_EQ(context, t->dims->data[d], t0->dims->data[d]); diff --git a/tensorflow/lite/kernels/conv.cc b/tensorflow/lite/kernels/conv.cc index 8a3539df8d5c1e..2d11632363fe00 100644 --- a/tensorflow/lite/kernels/conv.cc +++ b/tensorflow/lite/kernels/conv.cc @@ -496,6 +496,7 @@ TfLiteStatus Prepare(KernelType kernel_type, TfLiteContext* context, // Only one scale factor per batch is typically necessary. See optimized // implementation for why we need to allocate for the height of the inputs // flattened to 2D. + TF_LITE_ENSURE(context, channels_in != 0); const int height = NumElements(input) / channels_in; int scaling_dims[1] = {height}; if (!TfLiteIntArrayEqualsArray(scaling_factors->dims, 1, scaling_dims)) { @@ -534,6 +535,7 @@ TfLiteStatus Prepare(KernelType kernel_type, TfLiteContext* context, input_offsets->type = kTfLiteInt32; input_offsets->allocation_type = kTfLiteArenaRw; // See above comment for the need to allocate for height of inputs. + TF_LITE_ENSURE(context, channels_in != 0); const int height = NumElements(input) / channels_in; const int input_offset_dims[1] = {height}; if (!TfLiteIntArrayEqualsArray(input_offsets->dims, 1, @@ -747,17 +749,19 @@ void EvalFloat(TfLiteContext* context, TfLiteNode* node, } template -void EvalHybridPerChannel(TfLiteContext* context, TfLiteNode* node, - TfLiteConvParams* params, OpData* data, - TfLiteTensor* input, TfLiteTensor* filter, - TfLiteTensor* bias, TfLiteTensor* im2col, - TfLiteTensor* output) { +TfLiteStatus EvalHybridPerChannel(TfLiteContext* context, TfLiteNode* node, + TfLiteConvParams* params, OpData* data, + TfLiteTensor* input, + TfLiteTensor* filter, + TfLiteTensor* bias, + TfLiteTensor* im2col, TfLiteTensor* output) { float output_activation_min, output_activation_max; CalculateActivationRange(params->activation, &output_activation_min, &output_activation_max); - const int input_size = NumElements(input) / SizeOfDimension(input, 0); const int batch_size = SizeOfDimension(input, 0); + TF_LITE_ENSURE(context, batch_size != 0); + const int input_size = NumElements(input) / batch_size; int8_t* quantized_input_ptr_batch = GetTensorData( GetTemporary(context, node, data->input_quantized_index)); float* scaling_factors_ptr = GetTensorData( @@ -825,16 +829,18 @@ void EvalHybridPerChannel(TfLiteContext* context, TfLiteNode* node, } template -void EvalHybrid(TfLiteContext* context, TfLiteNode* node, - TfLiteConvParams* params, OpData* data, TfLiteTensor* input, - TfLiteTensor* filter, TfLiteTensor* bias, TfLiteTensor* im2col, - TfLiteTensor* accum_scratch, TfLiteTensor* output) { +TfLiteStatus EvalHybrid(TfLiteContext* context, TfLiteNode* node, + TfLiteConvParams* params, OpData* data, + TfLiteTensor* input, TfLiteTensor* filter, + TfLiteTensor* bias, TfLiteTensor* im2col, + TfLiteTensor* accum_scratch, TfLiteTensor* output) { float output_activation_min, output_activation_max; CalculateActivationRange(params->activation, &output_activation_min, &output_activation_max); - const int input_size = NumElements(input) / SizeOfDimension(input, 0); const int batch_size = SizeOfDimension(input, 0); + TF_LITE_ENSURE(context, batch_size != 0); + const int input_size = NumElements(input) / batch_size; float* input_ptr = GetTensorData(input); int8_t* quantized_input_ptr_batch = GetTensorData( @@ -913,14 +919,17 @@ TfLiteStatus EvalImpl(TfLiteContext* context, TfLiteNode* node) { case kTfLiteFloat32: if (filter->type == kTfLiteUInt8 || filter->type == kTfLiteInt8) { if (data->is_hybrid_per_channel) { - EvalHybridPerChannel(context, node, params, data, input, - filter, bias, im2col, output); + TF_LITE_ENSURE_OK(context, EvalHybridPerChannel( + context, node, params, data, input, + filter, bias, im2col, output)); } else { TfLiteTensor* accum_scratch = &context->tensors[node->temporaries ->data[data->accum_scratch_index]]; - EvalHybrid(context, node, params, data, input, filter, - bias, im2col, accum_scratch, output); + TF_LITE_ENSURE_OK(context, + EvalHybrid(context, node, params, data, + input, filter, bias, im2col, + accum_scratch, output)); } } else { EvalFloat(context, node, params, data, input, filter, bias, diff --git a/tensorflow/lite/kernels/depth_to_space.cc b/tensorflow/lite/kernels/depth_to_space.cc index d6fe8c7ab1ceed..c9ba027781847b 100644 --- a/tensorflow/lite/kernels/depth_to_space.cc +++ b/tensorflow/lite/kernels/depth_to_space.cc @@ -55,6 +55,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, input->type, output->type); const int block_size = params->block_size; + TF_LITE_ENSURE(context, block_size > 0); const int input_height = input->dims->data[1]; const int input_width = input->dims->data[2]; const int input_channels = input->dims->data[3]; diff --git a/tensorflow/lite/kernels/depth_to_space_test.cc b/tensorflow/lite/kernels/depth_to_space_test.cc index 8d59a1ad82f418..8a73676fa1c6ad 100644 --- a/tensorflow/lite/kernels/depth_to_space_test.cc +++ b/tensorflow/lite/kernels/depth_to_space_test.cc @@ -55,6 +55,11 @@ TEST(DepthToSpaceOpModel, BadBlockSize) { EXPECT_DEATH(DepthToSpaceOpModel({TensorType_FLOAT32, {1, 1, 1, 4}}, 4), "Cannot allocate tensors"); } + +TEST(DepthToSpaceOpModel, NoBlockSize) { + EXPECT_DEATH(DepthToSpaceOpModel({TensorType_FLOAT32, {1, 1, 1, 4}}, 0), + "Cannot allocate tensors"); +} #endif TEST(DepthToSpaceOpModel, Float32) { diff --git a/tensorflow/lite/kernels/depthwise_conv.cc b/tensorflow/lite/kernels/depthwise_conv.cc index 34d0556e7bd493..d71a88178d720e 100644 --- a/tensorflow/lite/kernels/depthwise_conv.cc +++ b/tensorflow/lite/kernels/depthwise_conv.cc @@ -273,8 +273,8 @@ TfLiteStatus ComputeDepthMultiplier(TfLiteContext* context, int16* depth_multiplier) { int num_filter_channels = SizeOfDimension(filter, 3); int num_input_channels = SizeOfDimension(input, 3); + TF_LITE_ENSURE(context, num_input_channels != 0); TF_LITE_ENSURE_EQ(context, num_filter_channels % num_input_channels, 0); - *depth_multiplier = num_filter_channels / num_input_channels; return kTfLiteOk; } @@ -444,10 +444,11 @@ TfLiteStatus EvalHybridPerChannel(TfLiteContext* context, TfLiteNode* node, float output_activation_min, output_activation_max; CalculateActivationRange(params->activation, &output_activation_min, &output_activation_max); - const int input_size = NumElements(input) / SizeOfDimension(input, 0); const int batch_size = SizeOfDimension(input, 0); const TfLiteTensor* input_quantized = GetTemporary(context, node, data->input_quantized_index); + TF_LITE_ENSURE(context, batch_size != 0); + const int input_size = NumElements(input) / batch_size; int8_t* quantized_input_ptr_batch = input_quantized->data.int8; float* scaling_factors_ptr = GetTensorData( GetTemporary(context, node, data->scaling_factors_index)); diff --git a/tensorflow/lite/kernels/embedding_lookup.cc b/tensorflow/lite/kernels/embedding_lookup.cc index 0334c0daf8400a..50830865ee5858 100644 --- a/tensorflow/lite/kernels/embedding_lookup.cc +++ b/tensorflow/lite/kernels/embedding_lookup.cc @@ -74,6 +74,10 @@ TfLiteStatus EvalSimple(TfLiteContext* context, TfLiteNode* node, const TfLiteTensor* lookup, const TfLiteTensor* value, TfLiteTensor* output) { const int row_size = SizeOfDimension(value, 0); + if (row_size == 0) { + // Propagate empty tensor if input is empty + return kTfLiteOk; + } const int row_bytes = value->bytes / row_size; char* output_raw = GetTensorData(output); diff --git a/tensorflow/lite/kernels/embedding_lookup_sparse.cc b/tensorflow/lite/kernels/embedding_lookup_sparse.cc index 8ecf427d4c121c..0b631acd2e4445 100644 --- a/tensorflow/lite/kernels/embedding_lookup_sparse.cc +++ b/tensorflow/lite/kernels/embedding_lookup_sparse.cc @@ -160,6 +160,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { // Resize output tensor. TfLiteIntArray* output_shape = TfLiteIntArrayCreate(output_rank); + TF_LITE_ENSURE(context, output_shape != nullptr); int k = 0; int embedding_size = 1; int lookup_size = 1; diff --git a/tensorflow/lite/kernels/gather_nd.cc b/tensorflow/lite/kernels/gather_nd.cc index 7332d6dfd47502..8ed4f3d87e9152 100644 --- a/tensorflow/lite/kernels/gather_nd.cc +++ b/tensorflow/lite/kernels/gather_nd.cc @@ -130,6 +130,9 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* indices = GetInput(context, node, kIndices); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + // Prevent division by 0 in the helper + TF_LITE_ENSURE(context, NumElements(params) > 0); + switch (indices->type) { case kTfLiteInt32: return EvalGatherNd(context, params, indices, output); diff --git a/tensorflow/lite/kernels/hashtable_lookup.cc b/tensorflow/lite/kernels/hashtable_lookup.cc index 62a15c68e29769..929de4ef9509ab 100644 --- a/tensorflow/lite/kernels/hashtable_lookup.cc +++ b/tensorflow/lite/kernels/hashtable_lookup.cc @@ -106,6 +106,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* value = GetInput(context, node, 2); const int num_rows = SizeOfDimension(value, 0); + TF_LITE_ENSURE(context, num_rows != 0); const int row_bytes = value->bytes / num_rows; void* pointer = nullptr; DynamicBuffer buf; diff --git a/tensorflow/lite/kernels/internal/reference/reduce.h b/tensorflow/lite/kernels/internal/reference/reduce.h index 8bfe66cdd485d4..6fbe8f33eb41e9 100644 --- a/tensorflow/lite/kernels/internal/reference/reduce.h +++ b/tensorflow/lite/kernels/internal/reference/reduce.h @@ -67,6 +67,9 @@ inline bool ResolveAxis(const int num_dims, const int* axis, // eg: For num_dims=3, [0, 1, 2] is the same as [-3, -2, -1] */ int current = axis[idx] < 0 ? (axis[idx] + num_dims) : axis[idx]; TFLITE_DCHECK(current >= 0 && current < num_dims); + if (current < 0 || current >= num_dims) { + return false; + } bool is_dup = false; for (int j = 0; j < *out_num_axis; ++j) { if (out_axis[j] == current) { diff --git a/tensorflow/lite/kernels/internal/types.h b/tensorflow/lite/kernels/internal/types.h index f4d2b5c0dad7a9..fa0f749901fe1c 100644 --- a/tensorflow/lite/kernels/internal/types.h +++ b/tensorflow/lite/kernels/internal/types.h @@ -432,7 +432,7 @@ int MatchingArraySize(const ArrayType1& array1, int index1, inline int MatchingDim(const RuntimeShape& shape1, int index1, const RuntimeShape& shape2, int index2) { TFLITE_DCHECK_EQ(shape1.Dims(index1), shape2.Dims(index2)); - return shape1.Dims(index1); + return std::min(shape1.Dims(index1), shape2.Dims(index2)); } template diff --git a/tensorflow/lite/kernels/kernel_util.h b/tensorflow/lite/kernels/kernel_util.h index ad068ddd3fddaf..299b81bbf9af04 100644 --- a/tensorflow/lite/kernels/kernel_util.h +++ b/tensorflow/lite/kernels/kernel_util.h @@ -30,31 +30,48 @@ inline int SizeOfDimension(const TfLiteTensor* t, int dim) { } inline const TfLiteTensor* GetInput(TfLiteContext* context, const TfLiteNode* node, int index) { - return &context - ->tensors[flatbuffers::EndianScalar(node->inputs->data[index])]; + const int tensor_index = flatbuffers::EndianScalar(node->inputs->data[index]); + if (tensor_index < 0) { + return nullptr; + } + return &context->tensors[tensor_index]; } // Note: You must check if result is not null: // TfLiteTensor* my_tensor = GetVariableInput(context, node, kMyTensorIdx); // TF_LITE_ENSURE(context, my_tensor != nullptr); inline TfLiteTensor* GetVariableInput(TfLiteContext* context, const TfLiteNode* node, int index) { - TfLiteTensor* tensor = - &context->tensors[flatbuffers::EndianScalar(node->inputs->data[index])]; + const int tensor_index = flatbuffers::EndianScalar(node->inputs->data[index]); + if (tensor_index < 0) { + return nullptr; + } + TfLiteTensor* tensor = &context->tensors[tensor_index]; return (tensor->is_variable) ? tensor : nullptr; } inline TfLiteTensor* GetOutput(TfLiteContext* context, const TfLiteNode* node, int index) { - return &context - ->tensors[flatbuffers::EndianScalar(node->outputs->data[index])]; + const int tensor_index = flatbuffers::EndianScalar(node->outputs->data[index]); + if (tensor_index < 0) { + return nullptr; + } + return &context->tensors[tensor_index]; } inline TfLiteTensor* GetTemporary(TfLiteContext* context, const TfLiteNode* node, int index) { - return &context->tensors[flatbuffers::EndianScalar( - node->temporaries->data[index])]; + const int tensor_index = flatbuffers::EndianScalar(node->temporaries->data[index]); + if (tensor_index < 0) { + return nullptr; + } + return &context->tensors[tensor_index]; } + inline const TfLiteTensor* GetIntermediates(TfLiteContext* context, const TfLiteNode* node, int index) { - return &context->tensors[node->intermediates->data[index]]; + const int tensor_index = flatbuffers::EndianScalar(node->intermediates->data[index]); + if (tensor_index < 0) { + return nullptr; + } + return &context->tensors[tensor_index]; } inline int NumInputs(const TfLiteNode* node) { return node->inputs->size; } inline int NumOutputs(const TfLiteNode* node) { return node->outputs->size; } @@ -77,13 +94,7 @@ inline int64_t NumElements(const TfLiteTensor* t) { inline const TfLiteTensor* GetOptionalInputTensor(TfLiteContext* context, const TfLiteNode* node, int index) { - const bool use_tensor = index < node->inputs->size && - node->inputs->data[index] != kTfLiteOptionalTensor; - if (use_tensor) { - return &context - ->tensors[flatbuffers::EndianScalar(node->inputs->data[index])]; - } - return nullptr; + return GetInput(context, node, index); } // Determines whether tensor is constant. diff --git a/tensorflow/lite/kernels/maximum_minimum.cc b/tensorflow/lite/kernels/maximum_minimum.cc index c51d7d07aff347..3b8f684a461042 100644 --- a/tensorflow/lite/kernels/maximum_minimum.cc +++ b/tensorflow/lite/kernels/maximum_minimum.cc @@ -102,6 +102,12 @@ template TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { OpContext op_context(context, node); + // If inputs have no element, shortcircuit. + if (NumElements(op_context.input1) == 0 || + NumElements(op_context.input2) == 0) { + return kTfLiteOk; + } + if (kernel_type == kReference) { switch (op_context.output->type) { case kTfLiteFloat32: diff --git a/tensorflow/lite/kernels/one_hot.cc b/tensorflow/lite/kernels/one_hot.cc index c52fdbd212d0e5..e254cca32c3df8 100644 --- a/tensorflow/lite/kernels/one_hot.cc +++ b/tensorflow/lite/kernels/one_hot.cc @@ -67,6 +67,11 @@ void OneHotComputeImpl(const OneHotContext& op_context) { for (int i = 0; i < op_context.axis; ++i) { prefix_dim_size *= op_context.indices->dims->data[i]; } + if (prefix_dim_size == 0) { + // If indices tensor is degenerate, return a degenerate tensor, just like + // TensorFlow does. + return; + } const int suffix_dim_size = NumElements(op_context.indices) / prefix_dim_size; const int depth = *op_context.depth->data.i32; diff --git a/tensorflow/lite/kernels/padding.h b/tensorflow/lite/kernels/padding.h index 1116b1da852cf6..6b4ab7fa58d1aa 100644 --- a/tensorflow/lite/kernels/padding.h +++ b/tensorflow/lite/kernels/padding.h @@ -44,6 +44,11 @@ inline int ComputePaddingWithOffset(int stride, int dilation_rate, int in_size, inline int ComputeOutSize(TfLitePadding padding, int image_size, int filter_size, int stride, int dilation_rate = 1) { int effective_filter_size = (filter_size - 1) * dilation_rate + 1; + + // TODO(b/186448822): This uses 0 since the function has no other way to + // report error case + if (stride == 0) return 0; + switch (padding) { case kTfLitePaddingSame: return (image_size + stride - 1) / stride; diff --git a/tensorflow/lite/kernels/pooling.cc b/tensorflow/lite/kernels/pooling.cc index e871b72f4a1957..28614aa4aa7624 100644 --- a/tensorflow/lite/kernels/pooling.cc +++ b/tensorflow/lite/kernels/pooling.cc @@ -84,6 +84,10 @@ TfLiteStatus GenericPrepare(TfLiteContext* context, TfLiteNode* node) { auto padding = params->padding; int out_width, out_height; + // Prevent division by 0 in optimized pooling implementations + TF_LITE_ENSURE(context, params->stride_height > 0); + TF_LITE_ENSURE(context, params->stride_width > 0); + data->padding = ComputePaddingHeightWidth( params->stride_height, params->stride_width, 1, 1, height, width, params->filter_height, params->filter_width, padding, &out_height, diff --git a/tensorflow/lite/kernels/pooling_test.cc b/tensorflow/lite/kernels/pooling_test.cc index 1b371361a4d875..ec3ba072e7c9c3 100644 --- a/tensorflow/lite/kernels/pooling_test.cc +++ b/tensorflow/lite/kernels/pooling_test.cc @@ -1083,5 +1083,18 @@ TEST(FloatPoolingOpTest, L2PoolPaddingValidSlide1) { EXPECT_THAT(m.GetOutput(), ElementsAreArray({3.5, 6.0, 6.5})); } +#ifdef GTEST_HAS_DEATH_TEST +TEST(FloatPoolingOpTest, MaxPoolWithZeroStride) { + EXPECT_DEATH( + FloatPoolingOpModel m(BuiltinOperator_MAX_POOL_2D, + /*input=*/{TensorType_FLOAT32, {1, 2, 4, 1}}, + /*filter_width=*/2, /*filter_height=*/2, + /*output=*/{TensorType_FLOAT32, {}}, + /*padding=*/Padding_VALID, + /*stride_w=*/0, /*stride_h=*/0), + "Cannot allocate tensors"); +} +#endif + } // namespace } // namespace tflite diff --git a/tensorflow/lite/kernels/segment_sum.cc b/tensorflow/lite/kernels/segment_sum.cc index db8aa688ebe833..6e582b4f1d84f5 100644 --- a/tensorflow/lite/kernels/segment_sum.cc +++ b/tensorflow/lite/kernels/segment_sum.cc @@ -32,11 +32,24 @@ TfLiteStatus ResizeOutputTensor(TfLiteContext* context, const TfLiteTensor* data, const TfLiteTensor* segment_ids, TfLiteTensor* output) { - int max_index = -1; + // Segment ids should be of same cardinality as first input dimension and they + // should be increasing by at most 1, from 0 (e.g., [0, 0, 1, 2, 3] is valid) const int segment_id_size = segment_ids->dims->data[0]; - if (segment_id_size > 0) { - max_index = segment_ids->data.i32[segment_id_size - 1]; + TF_LITE_ENSURE_EQ(context, segment_id_size, data->dims->data[0]); + int previous_segment_id = -1; + for (int i = 0; i < segment_id_size; i++) { + const int current_segment_id = GetTensorData(segment_ids)[i]; + if (i == 0) { + TF_LITE_ENSURE_EQ(context, current_segment_id, 0); + } else { + int delta = current_segment_id - previous_segment_id; + TF_LITE_ENSURE(context, delta == 0 || delta == 1); + } + previous_segment_id = current_segment_id; } + + const int max_index = previous_segment_id; + const int data_rank = NumDimensions(data); TfLiteIntArray* output_shape = TfLiteIntArrayCreate(NumDimensions(data)); output_shape->data[0] = max_index + 1; diff --git a/tensorflow/lite/kernels/segment_sum_test.cc b/tensorflow/lite/kernels/segment_sum_test.cc index d083feb44aa2d6..de07991d09f237 100644 --- a/tensorflow/lite/kernels/segment_sum_test.cc +++ b/tensorflow/lite/kernels/segment_sum_test.cc @@ -108,5 +108,37 @@ TEST(SegmentSumOpModelTest, Float32Test_ThreeDimensions) { EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 2, 1})); } +TEST(SegmentSumOpModelTest, TestFailIfSegmentsAreNotSorted) { + SegmentSumOpModel model({TensorType_INT32, {3, 2}}, + {TensorType_INT32, {3}}); + model.PopulateTensor(model.data(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor(model.segment_ids(), {0, 3, 1}); + ASSERT_EQ(model.InvokeUnchecked(), kTfLiteError); +} + +TEST(SegmentSumOpModelTest, TestFailIfSegmentsAreNotConsecutive) { + SegmentSumOpModel model({TensorType_INT32, {3, 2}}, + {TensorType_INT32, {3}}); + model.PopulateTensor(model.data(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor(model.segment_ids(), {0, 3, 5}); + ASSERT_EQ(model.InvokeUnchecked(), kTfLiteError); +} + +TEST(SegmentSumOpModelTest, TestFailIfSegmentsAreNegative) { + SegmentSumOpModel model({TensorType_INT32, {3, 2}}, + {TensorType_INT32, {3}}); + model.PopulateTensor(model.data(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor(model.segment_ids(), {-1, 0, 1}); + ASSERT_EQ(model.InvokeUnchecked(), kTfLiteError); +} + +TEST(SegmentSumOpModelTest, TestFailIfSegmentsAreNotTheRightCardinality) { + SegmentSumOpModel model({TensorType_INT32, {3, 2}}, + {TensorType_INT32, {2}}); + model.PopulateTensor(model.data(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor(model.segment_ids(), {0, 1}); + ASSERT_EQ(model.InvokeUnchecked(), kTfLiteError); +} + } // namespace } // namespace tflite diff --git a/tensorflow/lite/kernels/space_to_batch_nd.cc b/tensorflow/lite/kernels/space_to_batch_nd.cc index dabbf1395c8610..7ec713449242c7 100644 --- a/tensorflow/lite/kernels/space_to_batch_nd.cc +++ b/tensorflow/lite/kernels/space_to_batch_nd.cc @@ -75,6 +75,7 @@ TfLiteStatus ResizeOutputTensor(TfLiteContext* context, for (int dim = 0; dim < kSpatialDimensionNum; ++dim) { int final_dim_size = (input_size->data[dim + 1] + paddings_data[dim * 2] + paddings_data[dim * 2 + 1]); + TF_LITE_ENSURE(context, block_shape[dim] != 0); TF_LITE_ENSURE_EQ(context, final_dim_size % block_shape[dim], 0); output_size->data[dim + 1] = final_dim_size / block_shape[dim]; } diff --git a/tensorflow/lite/kernels/space_to_depth.cc b/tensorflow/lite/kernels/space_to_depth.cc index 527b7c83adba9c..1549ef01aabcc6 100644 --- a/tensorflow/lite/kernels/space_to_depth.cc +++ b/tensorflow/lite/kernels/space_to_depth.cc @@ -55,6 +55,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, input->type, output->type); const int block_size = params->block_size; + TF_LITE_ENSURE(context, block_size > 0); const int input_height = input->dims->data[1]; const int input_width = input->dims->data[2]; int output_height = input_height / block_size; diff --git a/tensorflow/lite/kernels/split.cc b/tensorflow/lite/kernels/split.cc index da239e6ecab391..f5ddd2cc8fe387 100644 --- a/tensorflow/lite/kernels/split.cc +++ b/tensorflow/lite/kernels/split.cc @@ -59,6 +59,7 @@ TfLiteStatus ResizeOutputTensors(TfLiteContext* context, TfLiteNode* node, TF_LITE_ENSURE(context, axis_value < NumDimensions(input)); const int input_size = SizeOfDimension(input, axis_value); + TF_LITE_ENSURE(context, num_splits != 0); TF_LITE_ENSURE_MSG(context, input_size % num_splits == 0, "Not an even split"); const int slice_size = input_size / num_splits; diff --git a/tensorflow/lite/kernels/split_v.cc b/tensorflow/lite/kernels/split_v.cc index 69a2e47508b11a..74b8344198df42 100644 --- a/tensorflow/lite/kernels/split_v.cc +++ b/tensorflow/lite/kernels/split_v.cc @@ -91,6 +91,8 @@ TfLiteStatus ResizeOutputTensors(TfLiteContext* context, TfLiteNode* node, } } + TF_LITE_ENSURE(context, axis_value >= 0); + TF_LITE_ENSURE(context, axis_value < NumDimensions(input)); const int input_size = SizeOfDimension(input, axis_value); if (minus_one_index != -1) { diff --git a/tensorflow/lite/kernels/svdf.cc b/tensorflow/lite/kernels/svdf.cc index bcbd06e8a673d4..809a2688fcbeac 100644 --- a/tensorflow/lite/kernels/svdf.cc +++ b/tensorflow/lite/kernels/svdf.cc @@ -94,6 +94,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { const int rank = params->rank; const int batch_size = input->dims->data[0]; const int num_filters = weights_feature->dims->data[0]; + TF_LITE_ENSURE(context, rank != 0); TF_LITE_ENSURE_EQ(context, num_filters % rank, 0); const int num_units = num_filters / rank; const int memory_size = weights_time->dims->data[1]; diff --git a/tensorflow/lite/kernels/transpose_conv.cc b/tensorflow/lite/kernels/transpose_conv.cc index 114b9ae48f4f20..5f448d9ab54d51 100644 --- a/tensorflow/lite/kernels/transpose_conv.cc +++ b/tensorflow/lite/kernels/transpose_conv.cc @@ -492,6 +492,10 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { const auto* params = reinterpret_cast(node->builtin_data); + // Prevent divisions by 0 + TF_LITE_ENSURE(context, params->stride_height > 0); + TF_LITE_ENSURE(context, params->stride_width > 0); + // Resize any deferred dynamic tensors if (IsDynamicTensor(output)) { TF_LITE_ENSURE_OK(context, ResizeTensor(context, output_shape, output)); diff --git a/tensorflow/lite/model.cc b/tensorflow/lite/model.cc index d232f70f2fe2cf..b91ceacb25c50a 100644 --- a/tensorflow/lite/model.cc +++ b/tensorflow/lite/model.cc @@ -728,6 +728,11 @@ TfLiteStatus InterpreterBuilder::operator()( return cleanup_and_error(); } + if (!buffers) { + error_reporter_->Report("No buffers in the model.\n"); + return cleanup_and_error(); + } + interpreter->reset(new Interpreter(error_reporter_)); (*interpreter)->SetNumThreads(num_threads); if (subgraphs->Length() > 1) { @@ -745,9 +750,9 @@ TfLiteStatus InterpreterBuilder::operator()( (*interpreter)->subgraph(subgraph_index); auto operators = subgraph->operators(); auto tensors = subgraph->tensors(); - if (!operators || !tensors || !buffers) { + if (!operators || !tensors) { error_reporter_->Report( - "Did not get operators, tensors, or buffers in subgraph %d.\n", + "Did not get operators or tensors in subgraph %d.\n", subgraph_index); return cleanup_and_error(); } diff --git a/tensorflow/lite/model_test.cc b/tensorflow/lite/model_test.cc index b9efdf676a893c..9efe76d15b4171 100644 --- a/tensorflow/lite/model_test.cc +++ b/tensorflow/lite/model_test.cc @@ -442,6 +442,24 @@ TEST(BasicFlatBufferModel, TestParseModelWithSparseTensor) { } // TODO(b/150072943): Add malformed model with sparse tensor tests. +// Recursion & reentrant are not supported in TFLite. +// The test ensures it fails gracefullly instead of crashing with +// a stack overflow. +TEST(BasicFlatBufferModel, TestUnsupportedRecursion) { + const auto model_path = + "tensorflow/lite/testdata/unsupported_recursion.bin"; + + std::unique_ptr model = + FlatBufferModel::BuildFromFile(model_path); + ASSERT_NE(model, nullptr); + + tflite::ops::builtin::BuiltinOpResolver resolver; + InterpreterBuilder builder(*model, resolver); + std::unique_ptr interpreter; + ASSERT_EQ(builder(&interpreter), kTfLiteOk); + ASSERT_NE(interpreter, nullptr); + ASSERT_NE(interpreter->AllocateTensors(), kTfLiteOk); +} // TODO(aselle): Add tests for serialization of builtin op data types. // These tests will occur with the evaluation tests of individual operators, diff --git a/tensorflow/lite/testdata/unsupported_recursion.bin b/tensorflow/lite/testdata/unsupported_recursion.bin new file mode 100644 index 00000000000000..525c5383ab4ef6 Binary files /dev/null and b/tensorflow/lite/testdata/unsupported_recursion.bin differ diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 1d4db0a5ee9072..e64573cc25a94c 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -4761,6 +4761,7 @@ cuda_py_test( srcs = ["ops/control_flow_ops_test.py"], python_version = "PY3", shard_count = 2, + tags = ["no_pip"], deps = [ ":array_ops", ":cond_v2", @@ -6480,6 +6481,8 @@ tf_py_test( # The multiprocessing module behaves differently on # windows, so we disable this test on windows. "no_windows", + # Also disable on mac + "no_mac", ], deps = [ ":client_testlib", diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD index d34e34d230789e..fe5a99929e61b3 100644 --- a/tensorflow/python/debug/BUILD +++ b/tensorflow/python/debug/BUILD @@ -872,6 +872,7 @@ py_test( python_version = "PY3", srcs_version = "PY2AND3", tags = [ + "no_mac", "no_oss_py38", #TODO(b/151449908) "no_windows", ], diff --git a/tensorflow/python/dlpack/dlpack_test.py b/tensorflow/python/dlpack/dlpack_test.py index af91da8051284a..df53220849cbd5 100644 --- a/tensorflow/python/dlpack/dlpack_test.py +++ b/tensorflow/python/dlpack/dlpack_test.py @@ -20,9 +20,11 @@ from absl.testing import parameterized import numpy as np + from tensorflow.python.dlpack import dlpack from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.platform import test @@ -95,6 +97,12 @@ def UnsupportedComplex64(): self.assertRaisesRegex(Exception, ".* is not supported by dlpack", UnsupportedComplex64) + def testMustPassTensorArgumentToDLPack(self): + with self.assertRaisesRegex( + errors.InvalidArgumentError, + "The argument to `to_dlpack` must be a TF tensor, not Python object"): + dlpack.to_dlpack([1]) + if __name__ == "__main__": ops.enable_eager_execution() diff --git a/tensorflow/python/feature_column/BUILD b/tensorflow/python/feature_column/BUILD index b29214f3f07c56..905a44b76cec5a 100644 --- a/tensorflow/python/feature_column/BUILD +++ b/tensorflow/python/feature_column/BUILD @@ -245,7 +245,10 @@ py_test( srcs = ["sequence_feature_column_integration_test.py"], python_version = "PY3", srcs_version = "PY2AND3", - tags = ["no_pip"], + tags = [ + "no_mac", + "no_pip", + ], deps = [ ":feature_column_v2", "//tensorflow/python:client_testlib", diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 1e6d5862720464..f437a36bbca07b 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -718,6 +718,7 @@ cuda_py_test( name = "matrix_solve_ls_op_test", size = "medium", srcs = ["matrix_solve_ls_op_test.py"], + tags = ["no_mac"], deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -787,6 +788,7 @@ tf_py_test( name = "parsing_ops_test", size = "medium", srcs = ["parsing_ops_test.py"], + tags = ["no_mac"], deps = [ "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", diff --git a/tensorflow/python/kernel_tests/boosted_trees/BUILD b/tensorflow/python/kernel_tests/boosted_trees/BUILD index 5b318324d4cd2e..68b27849773bf7 100644 --- a/tensorflow/python/kernel_tests/boosted_trees/BUILD +++ b/tensorflow/python/kernel_tests/boosted_trees/BUILD @@ -24,6 +24,7 @@ tf_py_test( name = "resource_ops_test", size = "small", srcs = ["resource_ops_test.py"], + tags = ["no_mac"], deps = [ "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_py", "//tensorflow/python:boosted_trees_ops", @@ -39,6 +40,7 @@ tf_py_test( name = "prediction_ops_test", size = "small", srcs = ["prediction_ops_test.py"], + tags = ["no_mac"], deps = [ "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_py", "//tensorflow/python:array_ops", @@ -69,6 +71,7 @@ tf_py_test( name = "training_ops_test", size = "small", srcs = ["training_ops_test.py"], + tags = ["no_mac"], deps = [ "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_py", "//tensorflow/python:array_ops", diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py index ec9d97c4bcca9c..10866d380b4e48 100644 --- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py +++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py @@ -4554,6 +4554,14 @@ def testUInt64SwitchMerge(self): result = control_flow_ops.merge([v_f, v_t]) self.evaluate(result) + def testSwitchEagerMode(self): + if not context.executing_eagerly(): + return + input_data = [1, 2, 3, 4] + vf, vt = control_flow_ops.switch(input_data, False) + self.assertAllEqual(vf, input_data) + self.assertAllEqual(vt, []) + @test_util.run_deprecated_v1 def testQIntArgAndRet(self): diff --git a/tensorflow/python/kernel_tests/substr_op_test.py b/tensorflow/python/kernel_tests/substr_op_test.py index 9302152e82bfa9..eae4e10f378567 100644 --- a/tensorflow/python/kernel_tests/substr_op_test.py +++ b/tensorflow/python/kernel_tests/substr_op_test.py @@ -492,6 +492,15 @@ def testInvalidUnit(self): with self.assertRaises(ValueError): string_ops.substr(b"test", 3, 1, unit="UTF8") + def testInvalidPos(self): + # Test case for GitHub issue 46900. + with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)): + x = string_ops.substr(b"abc", len=1, pos=[1, -1]) + self.evaluate(x) + + with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)): + x = string_ops.substr(b"abc", len=1, pos=[1, 2]) + self.evaluate(x) if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/transpose_op_test.py b/tensorflow/python/kernel_tests/transpose_op_test.py index 8d1fe388c55be1..6ae67bef8b86a1 100644 --- a/tensorflow/python/kernel_tests/transpose_op_test.py +++ b/tensorflow/python/kernel_tests/transpose_op_test.py @@ -373,6 +373,8 @@ def testDouble(self): @test_util.run_v1_only("b/120545219") def testComplex64(self): + self._testBoth(np.array(np.complex(1, 2)).astype(np.complex64)) + self._testBoth(np.complex(1, 2) * np.arange(0, 21).astype(np.complex64)) self._testBoth( np.complex(1, 2) * np.arange(0, 21).reshape([3, 7]).astype(np.complex64)) @@ -385,6 +387,8 @@ def testComplex64(self): @test_util.run_v1_only("b/120545219") def testComplex128(self): + self._testBoth(np.array(np.complex(1, 2)).astype(np.complex128)) + self._testBoth(np.complex(1, 2) * np.arange(0, 21).astype(np.complex128)) self._testBoth( np.complex(1, 2) * np.arange(0, 21).reshape([3, 7]).astype(np.complex128)) diff --git a/tensorflow/python/lib/core/ndarray_tensor.cc b/tensorflow/python/lib/core/ndarray_tensor.cc index 2f9972c81bfeb0..9bc84ed1157e98 100644 --- a/tensorflow/python/lib/core/ndarray_tensor.cc +++ b/tensorflow/python/lib/core/ndarray_tensor.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/python/lib/core/ndarray_tensor.h" #include +#include #include "tensorflow/core/lib/core/coding.h" #include "tensorflow/core/lib/core/errors.h" @@ -72,6 +73,13 @@ Status PyArrayDescr_to_TF_DataType(PyArray_Descr* descr, PyObject* key; PyObject* value; Py_ssize_t pos = 0; + + // Return an error if the fields attribute is null. + // Occurs with an improper conversion attempt to resource. + if (descr->fields == nullptr) { + return errors::Internal("Unexpected numpy data type"); + } + if (PyDict_Next(descr->fields, &pos, &key, &value)) { // In Python 3, the keys of numpy custom struct types are unicode, unlike // Python 2, where the keys are bytes. diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py index 860bdc60387653..6ef4a96d5cf23e 100644 --- a/tensorflow/python/ops/nn_test.py +++ b/tensorflow/python/ops/nn_test.py @@ -27,6 +27,7 @@ from tensorflow.python.eager import def_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_spec from tensorflow.python.framework import test_util @@ -1188,6 +1189,46 @@ def testArbitraryASCII(self): y_val = self.evaluate(y) self.assertAllEqual(y_val, y_val_expected) + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def testInvalidLength(self): + x = [-4, -3, -2, -1, 0, 1, 2, 3] + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Source format must be of length 4 or 5"): + op = nn_ops.data_format_dim_map( + x, src_format="12345678", dst_format="87654321") + with test_util.use_gpu(): + self.evaluate(op) + + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def testDuplicateSrc(self): + x = [-4, -3, -2, -1, 0, 1, 2, 3] + with self.assertRaisesRegex( + errors.InvalidArgumentError, + "Destination and source format must determine a permutation"): + op = nn_ops.data_format_dim_map(x, src_format="1233", dst_format="4321") + with test_util.use_gpu(): + self.evaluate(op) + + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def testDuplicateDst(self): + x = [-4, -3, -2, -1, 0, 1, 2, 3] + with self.assertRaisesRegex( + errors.InvalidArgumentError, + "Destination and source format must determine a permutation"): + op = nn_ops.data_format_dim_map(x, src_format="1234", dst_format="3321") + with test_util.use_gpu(): + self.evaluate(op) + + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def testExtraSpecifiers(self): + x = [-4, -3, -2, -1, 0, 1, 2, 3] + with self.assertRaisesRegex( + errors.InvalidArgumentError, + "Destination and source format must determine a permutation"): + op = nn_ops.data_format_dim_map(x, src_format="1234", dst_format="5321") + with test_util.use_gpu(): + self.evaluate(op) + class DataFormatVectorPermuteTest(test_lib.TestCase): @@ -1255,6 +1296,49 @@ def testNCHWToNHWC2D(self): y_val = self.evaluate(y) self.assertAllEqual(y_val, [[7, 4], [4, 5], [5, 1], [9, 3]]) + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def testInvalidLength(self): + x = [0, 1, 2, 3] + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Source format must be of length 4 or 5"): + op = nn_ops.data_format_vec_permute( + x, src_format="12345678", dst_format="87654321") + with test_util.use_gpu(): + self.evaluate(op) + + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def testDuplicateSrc(self): + x = [0, 1, 2, 3] + with self.assertRaisesRegex( + errors.InvalidArgumentError, + "Destination and source format must determine a permutation"): + op = nn_ops.data_format_vec_permute( + x, src_format="1233", dst_format="4321") + with test_util.use_gpu(): + self.evaluate(op) + + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def testDuplicateDst(self): + x = [0, 1, 2, 3] + with self.assertRaisesRegex( + errors.InvalidArgumentError, + "Destination and source format must determine a permutation"): + op = nn_ops.data_format_vec_permute( + x, src_format="1234", dst_format="3321") + with test_util.use_gpu(): + self.evaluate(op) + + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def testExtraSpecifiers(self): + x = [0, 1, 2, 3] + with self.assertRaisesRegex( + errors.InvalidArgumentError, + "Destination and source format must determine a permutation"): + op = nn_ops.data_format_vec_permute( + x, src_format="1234", dst_format="5321") + with test_util.use_gpu(): + self.evaluate(op) + @test_util.run_all_in_graph_and_eager_modes class AvgPoolTest(test_lib.TestCase): diff --git a/tensorflow/python/ops/raw_ops_test.py b/tensorflow/python/ops/raw_ops_test.py index fff94f5c25ae8c..3f67260c00eca4 100644 --- a/tensorflow/python/ops/raw_ops_test.py +++ b/tensorflow/python/ops/raw_ops_test.py @@ -18,16 +18,20 @@ from __future__ import division from __future__ import print_function +from absl.testing import parameterized + from tensorflow.python.eager import context from tensorflow.python.framework import constant_op +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import test_util +from tensorflow.python.ops import gen_data_flow_ops from tensorflow.python.ops import gen_math_ops from tensorflow.python.platform import test @test_util.run_all_in_graph_and_eager_modes -class RawOpsTest(test.TestCase): +class RawOpsTest(test.TestCase, parameterized.TestCase): def testSimple(self): x = constant_op.constant(1) @@ -57,6 +61,13 @@ def testDefaults(self): self.assertAllClose( gen_math_ops.Any(input=x, axis=0), gen_math_ops.Any(input=x, axis=0, keep_dims=False)) + + def testGetSessionHandle(self): + if context.executing_eagerly(): + with self.assertRaisesRegex( + errors.FailedPreconditionError, + "GetSessionHandle called on null session state"): + gen_data_flow_ops.GetSessionHandle(value=[1]) if __name__ == "__main__": diff --git a/tensorflow/python/ops/sparse_ops_test.py b/tensorflow/python/ops/sparse_ops_test.py index 91151ba8461c53..3a48a6aefd9c36 100644 --- a/tensorflow/python/ops/sparse_ops_test.py +++ b/tensorflow/python/ops/sparse_ops_test.py @@ -21,14 +21,17 @@ from absl.testing import parameterized import numpy as np +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import test_util # Need array_grad to register gradient for Identity. from tensorflow.python.ops import array_grad # pylint: disable=unused-import from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_sparse_ops from tensorflow.python.ops import gradient_checker_v2 as gradient_checker from tensorflow.python.ops import math_ops # Need sparse_grad to register gradient for SparseToDense. @@ -181,5 +184,57 @@ def testDenseSparseTensorMatMul(self): self.assertAllEqual(expected, result) +@test_util.run_all_in_graph_and_eager_modes +class RawOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): + + def testSparseFillEmptyRowsGrad(self): + reverse_index_map = [2, 1] + grad_values = [0, 1, 2, 3] + d_values, d_default_value = self.evaluate( + gen_sparse_ops.SparseFillEmptyRowsGrad( + reverse_index_map=reverse_index_map, grad_values=grad_values)) + self.assertAllEqual([2, 1], d_values) + self.assertEqual(3, d_default_value) + + def testSparseFillEmptyRowsGradNegativeIndexMapValue(self): + reverse_index_map = [2, -1] + grad_values = [0, 1, 2, 3] + with self.assertRaisesRegex( + errors.InvalidArgumentError, + r'Elements in reverse index must be in \[0, 4\)'): + self.evaluate( + gen_sparse_ops.SparseFillEmptyRowsGrad( + reverse_index_map=reverse_index_map, grad_values=grad_values)) + + def testSparseFillEmptyRowsGradLargeIndexMapValue(self): + reverse_index_map = [2, 10] + grad_values = [0, 1, 2, 3] + with self.assertRaisesRegex( + errors.InvalidArgumentError, + r'Elements in reverse index must be in \[0, 4\)'): + self.evaluate( + gen_sparse_ops.SparseFillEmptyRowsGrad( + reverse_index_map=reverse_index_map, grad_values=grad_values)) + + def testSparseFillEmptyRowsGradMatrix(self): + reverse_index_map = [0, 1] + grad_values = [[0, 1], [2, 3]] + # Note: Eager mode and graph mode throw different errors here. Graph mode + # will fail with a ValueError from the shape checking logic, while Eager + # will fail with an InvalidArgumentError from the kernel itself. + if context.executing_eagerly(): + with self.assertRaisesRegex(errors.InvalidArgumentError, + r'grad_values must be a vector'): + self.evaluate( + gen_sparse_ops.SparseFillEmptyRowsGrad( + reverse_index_map=reverse_index_map, grad_values=grad_values)) + else: + with self.assertRaisesRegex(ValueError, + r'Shape must be rank 1 but is rank 2'): + self.evaluate( + gen_sparse_ops.SparseFillEmptyRowsGrad( + reverse_index_map=reverse_index_map, grad_values=grad_values)) + + if __name__ == '__main__': googletest.main() diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py index b25247aa912561..c524914dc7b63a 100644 --- a/tensorflow/python/saved_model/function_deserialization.py +++ b/tensorflow/python/saved_model/function_deserialization.py @@ -22,7 +22,6 @@ import re from tensorflow.core.framework import function_pb2 -from tensorflow.python.eager import context from tensorflow.python.eager import def_function from tensorflow.python.eager import function as function_lib from tensorflow.python.framework import func_graph as func_graph_lib @@ -298,6 +297,19 @@ def load_function_def_library(library, load_shared_name_suffix=None): functions = {} renamed_functions = {} + # Our graph building code currently requires functions to be registered with + # some tf.Graph in order to import functions using the + # op-name-is-function-name calling convention. To avoid leaking memory into + # the global default graph when executing eagerly, we create a temporary + # Graph. + # + # TODO(allenl): Make this Graph creation unnecessary when executing eagerly by + # fixing function_def_to_graph_def. + if ops.executing_eagerly_outside_functions(): + graph = ops.Graph() + else: + graph = ops.get_default_graph() + if load_shared_name_suffix is None: load_shared_name_suffix = "_load_{}".format(ops.uid()) for fdef in _sort_function_defs(library, library_function_names): @@ -308,18 +320,22 @@ def load_function_def_library(library, load_shared_name_suffix=None): # extra function definitions are a no-op since they already imported as a # function before and passed in explicitly (due to the topologic sort # import). - func_graph = function_def_lib.function_def_to_graph(copy) + with graph.as_default(): + func_graph = function_def_lib.function_def_to_graph(copy) _restore_gradient_functions(func_graph, renamed_functions) for dep in _list_function_deps(fdef, library_function_names): functions[dep].add_to_graph(func_graph) func = function_lib.ConcreteFunction(func_graph) - func.add_to_graph() - if context.executing_eagerly(): - func.add_to_graph(ops.get_default_graph()) + func.add_to_graph(graph) functions[fdef.signature.name] = func renamed_functions[func.name] = func + if any(op.type == "TRTEngineOp" for op in func_graph.get_operations()): + # TODO(b/150708051): Remove this hack once TensorRT SavedModel integration + # is fixed. Currently it's leaking memory to maintain bug compatibility + # with previous behavior. + func.add_to_graph(ops.get_default_graph()) return functions diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py index fc77270651b032..69ce35609c640a 100644 --- a/tensorflow/python/saved_model/load_test.py +++ b/tensorflow/python/saved_model/load_test.py @@ -1992,6 +1992,21 @@ def increment_v(x): self.assertAllClose({"output_0": 13}, imported.signatures["serving_default"]()) + # TODO(allenl, kkb): Use the new memory checker here once it's fast enough (3 + # iterations took hundreds of seconds). It would be really nice to check + # allocations at a lower level. + @test_util.assert_no_new_pyobjects_executing_eagerly + def test_functions_cleaned(self): + if sys.version_info.major < 3: + self.skipTest("Not working in Python 2") + root = module.Module() + root.v = variables.Variable(1.) + root.f = def_function.function( + lambda x: x + root.v, + input_signature=[ + tensor_spec.TensorSpec(shape=[], dtype=dtypes.float32)]) + cycle(root, 1) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/tfe_wrapper.cc b/tensorflow/python/tfe_wrapper.cc index e3682e9613e163..98d36c72d3cf31 100644 --- a/tensorflow/python/tfe_wrapper.cc +++ b/tensorflow/python/tfe_wrapper.cc @@ -1051,9 +1051,16 @@ PYBIND11_MODULE(_pywrap_tfe, m) { // DLPack functions m.def("TFE_ToDlpackCapsule", [](py::handle& o) { PyObject* eager_tensor_pyobject_ptr = o.ptr(); - TFE_TensorHandle* thandle = EagerTensor_Handle(eager_tensor_pyobject_ptr); tensorflow::Safe_TF_StatusPtr status = tensorflow::make_safe(TF_NewStatus()); + + if (!EagerTensor_CheckExact(eager_tensor_pyobject_ptr)) { + status->status = tensorflow::errors::InvalidArgument( + "The argument to `to_dlpack` must be a TF tensor, not Python object"); + tensorflow::MaybeRaiseRegisteredFromTFStatus(status.get()); + } + + TFE_TensorHandle* thandle = EagerTensor_Handle(eager_tensor_pyobject_ptr); void* dlm_ptr = tensorflow::TFE_HandleToDLPack(thandle, status.get()); tensorflow::MaybeRaiseRegisteredFromTFStatus(status.get()); diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc index 2768777426b807..ea26fb064e795c 100755 --- a/tensorflow/stream_executor/cuda/cuda_dnn.cc +++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc @@ -1449,7 +1449,9 @@ class CudnnRnnSequenceTensorDescriptor static port::StatusOr Create( GpuExecutor* parent, int max_seq_length, int batch_size, int data_size, cudnnDataType_t data_type) { - CHECK_GT(max_seq_length, 0); + if (max_seq_length <= 0) { + return port::Status(port::error::INVALID_ARGUMENT, "max_seq_length <= 0"); + } int dims[] = {batch_size, data_size, 1}; int strides[] = {dims[1] * dims[2], dims[2], 1}; TensorDescriptor tensor_desc = CreateTensorDescriptor(); @@ -1470,7 +1472,9 @@ class CudnnRnnSequenceTensorDescriptor const absl::Span& seq_lengths, bool time_major, cudnnDataType_t data_type) { #if CUDNN_VERSION >= 7201 - CHECK_GT(max_seq_length, 0); + if (max_seq_length <= 0) { + return port::Status(port::error::INVALID_ARGUMENT, "max_seq_length <= 0"); + } int dims[] = {batch_size, data_size, 1}; int strides[] = {dims[1] * dims[2], dims[2], 1}; TensorDescriptor tensor_desc = CreateTensorDescriptor(); diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 796143ab2bda21..c422feda4dfc72 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -58,7 +58,7 @@ load( # not contain rc or alpha, only numbers. # Also update tensorflow/core/public/version.h # and tensorflow/tools/pip_package/setup.py -VERSION = "2.2.0" +VERSION = "2.2.3" VERSION_MAJOR = VERSION.split(".")[0] # Sanitize a dependency so that it works correctly from code that includes @@ -1652,6 +1652,7 @@ def _py_wrap_cc_impl(ctx): args.append(src.path) outputs = [ctx.outputs.cc_out, ctx.outputs.py_out] ctx.actions.run( + use_default_shell_env = True, executable = ctx.executable._swig, arguments = args, inputs = inputs, diff --git a/tensorflow/tools/api/tests/BUILD b/tensorflow/tools/api/tests/BUILD index 76d3324d27ccba..e2d0e4ecf070b4 100644 --- a/tensorflow/tools/api/tests/BUILD +++ b/tensorflow/tools/api/tests/BUILD @@ -28,6 +28,7 @@ py_test( python_version = "PY3", srcs_version = "PY2AND3", tags = [ + "no_mac", "no_oss_py38", "no_pip", "no_rocm", diff --git a/tensorflow/tools/ci_build/rel/macos/cpu_libtensorflow.sh b/tensorflow/tools/ci_build/rel/macos/cpu_libtensorflow.sh new file mode 100644 index 00000000000000..348778b5f158ee --- /dev/null +++ b/tensorflow/tools/ci_build/rel/macos/cpu_libtensorflow.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +echo "chmod go+w lib_package/*" >> tensorflow/tools/ci_build/linux/libtensorflow.sh +echo "bazel clean --expunge" >> tensorflow/tools/ci_build/linux/libtensorflow.sh + +# Install latest bazel +source tensorflow/tools/ci_build/release/common.sh +install_bazelisk + +# Pick a version of xcode +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" + +tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh diff --git a/tensorflow/tools/ci_build/rel/macos/cpu_py35_nonpip.sh b/tensorflow/tools/ci_build/rel/macos/cpu_py35_nonpip.sh new file mode 100644 index 00000000000000..63b614dd68739e --- /dev/null +++ b/tensorflow/tools/ci_build/rel/macos/cpu_py35_nonpip.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh +install_bazelisk + +# Pick a more recent version of xcode +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" +python3.5 -m virtualenv tf_build_env --system-site-packages +source tf_build_env/bin/activate + +# Install macos pip dependencies +install_macos_pip_deps sudo pip3.5 + +# Run configure. +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export TF2_BEHAVIOR=1 +export PYTHON_BIN_PATH=$(which python3.5) +yes "" | "$PYTHON_BIN_PATH" configure.py + +tag_filters="-no_oss,-oss_serial,-nomac,-no_mac,-no_oss_py35,-v1only,-gpu,-tpu,-benchmark-test" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Run tests +bazel test --test_output=errors --config=opt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} \ + -//tensorflow/lite/... diff --git a/tensorflow/tools/ci_build/rel/macos/cpu_py35_pip.sh b/tensorflow/tools/ci_build/rel/macos/cpu_py35_pip.sh new file mode 100644 index 00000000000000..8c9b91dd55e04f --- /dev/null +++ b/tensorflow/tools/ci_build/rel/macos/cpu_py35_pip.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh +install_bazelisk + +# Pick a more recent version of xcode +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" + +# Install macos pip dependencies +install_macos_pip_deps sudo pip3.5 + +# Export required variables for running pip_new.sh +export OS_TYPE="MACOS" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.5' +export TF_BUILD_BOTH_CPU_PACKAGES=1 + +# Run configure. +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=opt --config=v2" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="//tensorflow/python/..." +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-nomac,-no_mac,-no_oss,-oss_serial,-no_oss_py35,-gpu,-tpu,-benchmark-test' +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME="tensorflow" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/macos/cpu_py36_nonpip.sh b/tensorflow/tools/ci_build/rel/macos/cpu_py36_nonpip.sh new file mode 100644 index 00000000000000..a80cdd88ddcd40 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/macos/cpu_py36_nonpip.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh +install_bazelisk + +# Pick a more recent version of xcode +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" +python3.6 -m virtualenv tf_build_env --system-site-packages +source tf_build_env/bin/activate + +# Install macos pip dependencies +install_macos_pip_deps sudo pip3.6 + +# Run configure. +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export TF2_BEHAVIOR=1 +export PYTHON_BIN_PATH=$(which python3.6) +yes "" | "$PYTHON_BIN_PATH" configure.py + +tag_filters="-no_oss,-oss_serial,-nomac,-no_mac,-no_oss_py36,-v1only,-gpu,-tpu,-benchmark-test" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Run tests +bazel test --test_output=errors --config=opt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} \ + -//tensorflow/lite/... + diff --git a/tensorflow/tools/ci_build/rel/macos/cpu_py36_pip.sh b/tensorflow/tools/ci_build/rel/macos/cpu_py36_pip.sh new file mode 100644 index 00000000000000..a66dca3885e2b3 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/macos/cpu_py36_pip.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh +install_bazelisk + +# Pick a more recent version of xcode +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" + +# Install macos pip dependencies +install_macos_pip_deps sudo pip3.6 + +# Export required variables for running pip_new.sh +export OS_TYPE="MACOS" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.6' +export TF_BUILD_BOTH_CPU_PACKAGES=1 + +# Run configure. +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=opt --config=v2" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="//tensorflow/python/..." +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-nomac,-no_mac,-no_oss,-oss_serial,-no_oss_py35,-v1only,-gpu,-tpu,-benchmark-test' +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME="tensorflow" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/macos/cpu_py37_nonpip.sh b/tensorflow/tools/ci_build/rel/macos/cpu_py37_nonpip.sh new file mode 100644 index 00000000000000..e7234024ca543f --- /dev/null +++ b/tensorflow/tools/ci_build/rel/macos/cpu_py37_nonpip.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh +install_bazelisk + +# Pick a more recent version of xcode +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" +python -m virtualenv tf_build_env --system-site-packages +source tf_build_env/bin/activate + +# Install macos pip dependencies +install_macos_pip_deps sudo pip3.7 + +# Run configure. +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export TF2_BEHAVIOR=1 +export PYTHON_BIN_PATH=$(which python3.7) +yes "" | "$PYTHON_BIN_PATH" configure.py + +tag_filters="-no_oss,-oss_serial,-nomac,-no_mac$(maybe_skip_v1),-gpu,-tpu,-benchmark-test" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Run tests +bazel test --test_output=errors --config=opt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} \ + -//tensorflow/lite/... diff --git a/tensorflow/tools/ci_build/rel/macos/cpu_py37_pip.sh b/tensorflow/tools/ci_build/rel/macos/cpu_py37_pip.sh new file mode 100644 index 00000000000000..5d75224a45c478 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/macos/cpu_py37_pip.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh +install_bazelisk + +# Pick a more recent version of xcode +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" + +# Install macos pip dependencies +install_macos_pip_deps sudo pip3.7 + +# Export required variables for running pip_new.sh +export OS_TYPE="MACOS" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.7' +export TF_BUILD_BOTH_CPU_PACKAGES=1 + +# Run configure. +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=opt --config=v2" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="//tensorflow/python/..." +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-nomac,-no_mac,-no_oss,-oss_serial,-no_oss_py37,-v1only,-gpu,-tpu,-benchmark-test' +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME="tensorflow" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/macos/cpu_py38_nonpip.sh b/tensorflow/tools/ci_build/rel/macos/cpu_py38_nonpip.sh new file mode 100644 index 00000000000000..b9a4157577db7d --- /dev/null +++ b/tensorflow/tools/ci_build/rel/macos/cpu_py38_nonpip.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh +install_bazelisk + +# Pick a more recent version of xcode +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" +python -m virtualenv tf_build_env --system-site-packages +source tf_build_env/bin/activate + +# Install macos pip dependencies +install_macos_pip_deps sudo pip3.8 + +# Run configure. +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export TF2_BEHAVIOR=1 +export PYTHON_BIN_PATH=$(which python3.8) +yes "" | "$PYTHON_BIN_PATH" configure.py + +tag_filters="-no_oss,-oss_serial,-nomac,-no_mac$(maybe_skip_v1),-gpu,-tpu,-benchmark-test" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Run tests +bazel test --test_output=errors --config=opt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} \ + -//tensorflow/lite/... diff --git a/tensorflow/tools/ci_build/rel/macos/cpu_py38_pip.sh b/tensorflow/tools/ci_build/rel/macos/cpu_py38_pip.sh new file mode 100644 index 00000000000000..a5a5b6a34c436c --- /dev/null +++ b/tensorflow/tools/ci_build/rel/macos/cpu_py38_pip.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh +install_bazelisk + +# Pick a more recent version of xcode +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" + +# Install macos pip dependencies +install_macos_pip_deps sudo pip3.8 + +# Export required variables for running pip_new.sh +export OS_TYPE="MACOS" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.8' +export TF_BUILD_BOTH_CPU_PACKAGES=1 + +# Run configure. +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=opt --config=v2" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="//tensorflow/python/..." +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-nomac,-no_mac,-no_oss,-oss_serial,-no_oss_py38,-v1only,-gpu,-tpu,-benchmark-test' +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME="tensorflow" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu/cpu_py35_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu/cpu_py35_nonpip.sh new file mode 100644 index 00000000000000..5b161b2f53b86d --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/cpu_py35_nonpip.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.5 +# Update bazel +install_bazelisk + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.5) +export TF2_BEHAVIOR=1 +yes "" | "$PYTHON_BIN_PATH" configure.py +tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-no_oss_py35,-v1only" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Run tests +bazel test --test_output=errors --config=opt --test_lang_filters=py \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... diff --git a/tensorflow/tools/ci_build/rel/ubuntu/cpu_py35_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu/cpu_py35_pip.sh new file mode 100644 index 00000000000000..3842410edb2db3 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/cpu_py35_pip.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.5 +# Update bazel +install_bazelisk + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.5' + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=opt --config=v2 --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-no_oss,-oss_serial,-no_oss_py35,-v1only' +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME="tensorflow_cpu" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu/cpu_py36_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu/cpu_py36_nonpip.sh new file mode 100644 index 00000000000000..2b621a5d8ca83e --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/cpu_py36_nonpip.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.6 +# Update bazel +install_bazelisk + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.6) +export TF2_BEHAVIOR=1 +yes "" | "$PYTHON_BIN_PATH" configure.py +tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-no_oss_py36,-v1only" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Run tests +bazel test --test_output=errors --config=opt --test_lang_filters=py \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... diff --git a/tensorflow/tools/ci_build/rel/ubuntu/cpu_py36_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu/cpu_py36_pip.sh new file mode 100644 index 00000000000000..d23ce0160803c5 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/cpu_py36_pip.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.6 +# Update bazel +install_bazelisk + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.6' + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=opt --config=v2 --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-no_oss,-oss_serial,-no_oss_py36,-v1only' +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME="tensorflow_cpu" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu/cpu_py37_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu/cpu_py37_nonpip.sh new file mode 100644 index 00000000000000..3fd9fd66afd821 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/cpu_py37_nonpip.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.7 +# Update bazel +install_bazelisk + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.7) +export TF2_BEHAVIOR=1 +yes "" | "$PYTHON_BIN_PATH" configure.py +tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-no_oss_py37,-v1only" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Run tests +bazel test --test_output=errors --config=opt --test_lang_filters=py \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... diff --git a/tensorflow/tools/ci_build/rel/ubuntu/cpu_py37_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu/cpu_py37_pip.sh new file mode 100644 index 00000000000000..9cded426bde2e1 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/cpu_py37_pip.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.7 +# Update bazel +install_bazelisk + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.7' + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=opt --config=v2 --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-no_oss,-oss_serial,-no_oss_py37,-v1only' +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME="tensorflow_cpu" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu/cpu_py38_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu/cpu_py38_nonpip.sh new file mode 100644 index 00000000000000..205c488847d86b --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/cpu_py38_nonpip.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.8 +# Update bazel +install_bazelisk + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.8) +export TF2_BEHAVIOR=1 +yes "" | "$PYTHON_BIN_PATH" configure.py +tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-no_oss_py38,-v1only" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Run tests +bazel test --test_output=errors --config=opt --test_lang_filters=py \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... diff --git a/tensorflow/tools/ci_build/rel/ubuntu/cpu_py38_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu/cpu_py38_pip.sh new file mode 100644 index 00000000000000..366f2464612809 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/cpu_py38_pip.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.8 +# Update bazel +install_bazelisk + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.8' + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=opt --config=v2 --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-no_oss,-oss_serial,-no_oss_py38,-v1only' +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME="tensorflow_cpu" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu/gpu_pip_on_cpu.sh b/tensorflow/tools/ci_build/rel/ubuntu/gpu_pip_on_cpu.sh new file mode 100755 index 00000000000000..6e913071043c9f --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/gpu_pip_on_cpu.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.6 +# Update Bazel to the desired version +install_bazelisk + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.6) +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +######################## +## Build GPU pip package +######################## +bazel build --config=opt \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + tensorflow/tools/pip_package:build_pip_package + +PIP_WHL_DIR=whl +mkdir -p ${PIP_WHL_DIR} +PIP_WHL_DIR=$(readlink -f ${PIP_WHL_DIR}) # Get absolute path +bazel-bin/tensorflow/tools/pip_package/build_pip_package "${PIP_WHL_DIR}" +WHL_PATH=$(ls "${PIP_WHL_DIR}"/*.whl) + +cp "${WHL_PATH}" "$(pwd)"/. +chmod +x tensorflow/tools/ci_build/builds/docker_cpu_pip.sh +docker run -e "BAZEL_VERSION=${BAZEL_VERSION}" -e "CI_BUILD_USER=$(id -u -n)" -e "CI_BUILD_UID=$(id -u)" -e "CI_BUILD_GROUP=$(id -g -n)" -e "CI_BUILD_GID=$(id -g)" -e "CI_BUILD_HOME=/bazel_pip" -v "$(pwd)":/bazel_pip tensorflow/tensorflow:devel-py3 "./bazel_pip/tensorflow/tools/ci_build/builds/with_the_same_user" "./bazel_pip/tensorflow/tools/ci_build/builds/docker_cpu_pip.sh" diff --git a/tensorflow/tools/ci_build/rel/ubuntu/gpu_py35_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu/gpu_py35_nonpip.sh new file mode 100644 index 00000000000000..c576be51bd7a47 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/gpu_py35_nonpip.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.5 +# Update bazel +install_bazelisk + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.5) +export TF2_BEHAVIOR=1 +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +tag_filters="gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py35" + +bazel test --config=cuda --config=opt \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --test_lang_filters=py \ + --test_tag_filters=${tag_filters} \ + --build_tag_filters=${tag_filters} \ + --test_timeout="300,450,1200,3600" --local_test_jobs=4 \ + --test_output=errors --verbose_failures=true --keep_going \ + --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ + -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... diff --git a/tensorflow/tools/ci_build/rel/ubuntu/gpu_py35_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu/gpu_py35_pip.sh new file mode 100644 index 00000000000000..be97cc4bfa8cfb --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/gpu_py35_pip.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.5 +# Update bazel +install_bazelisk + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="GPU" +export TF_PYTHON_VERSION='python3.5' + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py35' +export TF_BUILD_FLAGS="--config=opt --config=v2 --config=cuda --distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION --action_env=TF_CUDNN_VERSION --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain " +export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \ +--distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION --action_env=TF_CUDNN_VERSION --test_env=TF2_BEHAVIOR=1 \ +--config=cuda --test_output=errors --local_test_jobs=4 --test_lang_filters=py \ +--verbose_failures=true --keep_going --define=no_tensorflow_py_deps=true \ +--run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute " +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME=${PROJECT_NAME} +export TF_PIP_TEST_ROOT="pip_test" + +# To build both tensorflow and tensorflow-gpu pip packages +export TF_BUILD_BOTH_GPU_PACKAGES=1 + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu/gpu_py36_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu/gpu_py36_nonpip.sh new file mode 100644 index 00000000000000..deb0dcafbb00f5 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/gpu_py36_nonpip.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.6 +# Update bazel +install_bazelisk + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.6) +export TF2_BEHAVIOR=1 +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +tag_filters="gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py36" + +bazel test --config=cuda --config=opt \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --test_lang_filters=py \ + --test_tag_filters=${tag_filters} \ + --build_tag_filters=${tag_filters} \ + --test_timeout="300,450,1200,3600" --local_test_jobs=4 \ + --test_output=errors --verbose_failures=true --keep_going \ + --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ + -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... diff --git a/tensorflow/tools/ci_build/rel/ubuntu/gpu_py36_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu/gpu_py36_pip.sh new file mode 100644 index 00000000000000..15f7db11a87c1a --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/gpu_py36_pip.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.6 +# Update bazel +install_bazelisk + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="GPU" +export TF_PYTHON_VERSION='python3.6' + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py36' +export TF_BUILD_FLAGS="--config=opt --config=v2 --config=cuda --distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION --action_env=TF_CUDNN_VERSION --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain " +export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \ +--distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION --action_env=TF_CUDNN_VERSION --test_env=TF2_BEHAVIOR=1 \ +--config=cuda --test_output=errors --local_test_jobs=4 --test_lang_filters=py \ +--verbose_failures=true --keep_going --define=no_tensorflow_py_deps=true \ +--run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute " +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME=${PROJECT_NAME} +export TF_PIP_TEST_ROOT="pip_test" + +# To build both tensorflow and tensorflow-gpu pip packages +export TF_BUILD_BOTH_GPU_PACKAGES=1 + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu/gpu_py37_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu/gpu_py37_nonpip.sh new file mode 100644 index 00000000000000..7c85556b722148 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/gpu_py37_nonpip.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.7 +# Update bazel +install_bazelisk + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.7) +export TF2_BEHAVIOR=1 +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +tag_filters="gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py37" + +bazel test --config=cuda --config=opt \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --test_lang_filters=py \ + --build_tag_filters=${tag_filters} \ + --test_tag_filters=${tag_filters} \ + --test_timeout="300,450,1200,3600" --local_test_jobs=4 \ + --test_output=errors --verbose_failures=true --keep_going \ + --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ + -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... diff --git a/tensorflow/tools/ci_build/rel/ubuntu/gpu_py37_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu/gpu_py37_pip.sh new file mode 100644 index 00000000000000..56f2a7f66e989a --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/gpu_py37_pip.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.7 +# Update bazel +install_bazelisk + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="GPU" +export TF_PYTHON_VERSION='python3.7' + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py37' +export TF_BUILD_FLAGS="--config=opt --config=v2 --config=cuda --distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION --action_env=TF_CUDNN_VERSION --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain " +export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \ +--distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION --action_env=TF_CUDNN_VERSION --test_env=TF2_BEHAVIOR=1 \ +--config=cuda --test_output=errors --local_test_jobs=4 --test_lang_filters=py \ +--verbose_failures=true --keep_going --define=no_tensorflow_py_deps=true \ +--run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute " +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME=${PROJECT_NAME} +export TF_PIP_TEST_ROOT="pip_test" + +# To build both tensorflow and tensorflow-gpu pip packages +export TF_BUILD_BOTH_GPU_PACKAGES=1 + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu/gpu_py38_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu/gpu_py38_nonpip.sh new file mode 100644 index 00000000000000..639ba9edb5ae30 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/gpu_py38_nonpip.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.8 +# Update bazel +update_bazel_linux + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.8) +export TF2_BEHAVIOR=1 +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +tag_filters="gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py38" + +bazel test --config=cuda --config=opt \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --test_lang_filters=py \ + --build_tag_filters=${tag_filters} \ + --test_tag_filters=${tag_filters} \ + --test_timeout="300,450,1200,3600" --local_test_jobs=4 \ + --test_output=errors --verbose_failures=true --keep_going \ + --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ + -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... diff --git a/tensorflow/tools/ci_build/rel/ubuntu/gpu_py38_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu/gpu_py38_pip.sh new file mode 100644 index 00000000000000..28b633c390e111 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/gpu_py38_pip.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.8 +# Update bazel +update_bazel_linux + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="GPU" +export TF_PYTHON_VERSION='python3.8' + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py38' +export TF_BUILD_FLAGS="--config=opt --config=v2 --config=cuda --distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION --action_env=TF_CUDNN_VERSION --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain " +export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \ +--distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION --action_env=TF_CUDNN_VERSION --test_env=TF2_BEHAVIOR=1 \ +--config=cuda --test_output=errors --local_test_jobs=4 --test_lang_filters=py \ +--verbose_failures=true --keep_going --define=no_tensorflow_py_deps=true \ +--run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute " +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME=${PROJECT_NAME} +export TF_PIP_TEST_ROOT="pip_test" + +# To build both tensorflow and tensorflow-gpu pip packages +export TF_BUILD_BOTH_GPU_PACKAGES=1 + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu/sanity.sh b/tensorflow/tools/ci_build/rel/ubuntu/sanity.sh new file mode 100644 index 00000000000000..4fc600de867e50 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu/sanity.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e + +# Install latest bazel +source tensorflow/tools/ci_build/release/common.sh +install_bazelisk +which bazel + +# We need py3 lint +sudo pip3 install pep8 + +# TODO(gunan): figure out why we get stuck with later versions of pylint. +# Install pylint. +sudo python3 -m pip install setuptools --upgrade +sudo python2 -m pip install pylint==1.6.4 +sudo python3 -m pip install pylint==1.6.4 + +# TODO(yifeif): print pylint version for debug. remove later. +python3 -m pylint --version + +# Run tensorflow sanity checks. +tensorflow/tools/ci_build/ci_sanity.sh diff --git a/tensorflow/tools/ci_build/rel/windows/cpu_libtensorflow.bat b/tensorflow/tools/ci_build/rel/windows/cpu_libtensorflow.bat new file mode 100644 index 00000000000000..67941234b155c0 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows/cpu_libtensorflow.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\cpu\bazel\run_libtensorflow.bat || exit /b 1 + +copy lib_package %TF_ARTIFACTS_DIR%\lib_package diff --git a/tensorflow/tools/ci_build/rel/windows/cpu_py35.bat b/tensorflow/tools/ci_build/rel/windows/cpu_py35.bat new file mode 100644 index 00000000000000..bd8c217ddefe77 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows/cpu_py35.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python35 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows/cpu_py36.bat b/tensorflow/tools/ci_build/rel/windows/cpu_py36.bat new file mode 100644 index 00000000000000..0a81a90a43164c --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows/cpu_py36.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python36 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows/cpu_py37.bat b/tensorflow/tools/ci_build/rel/windows/cpu_py37.bat new file mode 100644 index 00000000000000..9591d7aac343bd --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows/cpu_py37.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python37 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows/cpu_py38.bat b/tensorflow/tools/ci_build/rel/windows/cpu_py38.bat new file mode 100644 index 00000000000000..7a7435b3713927 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows/cpu_py38.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python38 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows/gpu_libtensorflow.bat b/tensorflow/tools/ci_build/rel/windows/gpu_libtensorflow.bat new file mode 100644 index 00000000000000..8ab78bef3ca0af --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows/gpu_libtensorflow.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\gpu\bazel\run_libtensorflow.bat || exit /b + +copy lib_package %TF_ARTIFACTS_DIR%\lib_package diff --git a/tensorflow/tools/ci_build/rel/windows/gpu_pip_on_cpu.bat b/tensorflow/tools/ci_build/rel/windows/gpu_pip_on_cpu.bat new file mode 100644 index 00000000000000..213de532069244 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows/gpu_pip_on_cpu.bat @@ -0,0 +1,21 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python36 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\integration\gpu_pip_on_cpu\run.bat + diff --git a/tensorflow/tools/ci_build/rel/windows/gpu_py35.bat b/tensorflow/tools/ci_build/rel/windows/gpu_py35.bat new file mode 100644 index 00000000000000..8a21961fdef3db --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows/gpu_py35.bat @@ -0,0 +1,22 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python35 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" + +bash -l tensorflow\tools\ci_build\release\windows\gpu_py35_full\release_pip_rename.sh diff --git a/tensorflow/tools/ci_build/rel/windows/gpu_py36.bat b/tensorflow/tools/ci_build/rel/windows/gpu_py36.bat new file mode 100644 index 00000000000000..7c4a395f62dd11 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows/gpu_py36.bat @@ -0,0 +1,22 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python36 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" + +bash -l tensorflow\tools\ci_build\release\windows\gpu_py36_full\release_pip_rename.sh diff --git a/tensorflow/tools/ci_build/rel/windows/gpu_py37.bat b/tensorflow/tools/ci_build/rel/windows/gpu_py37.bat new file mode 100644 index 00000000000000..97eb1168d1ce0d --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows/gpu_py37.bat @@ -0,0 +1,22 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python37 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" + +bash -l tensorflow\tools\ci_build\release\windows\gpu_py37_full\release_pip_rename.sh diff --git a/tensorflow/tools/ci_build/rel/windows/gpu_py38.bat b/tensorflow/tools/ci_build/rel/windows/gpu_py38.bat new file mode 100644 index 00000000000000..f980d311a5be5c --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows/gpu_py38.bat @@ -0,0 +1,22 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python38 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" + +bash -l tensorflow\tools\ci_build\release\windows\gpu_py38_full\release_pip_rename.sh diff --git a/tensorflow/tools/ci_build/release/common.sh b/tensorflow/tools/ci_build/release/common.sh index 3e3a22f23582ce..bafb36deebda43 100644 --- a/tensorflow/tools/ci_build/release/common.sh +++ b/tensorflow/tools/ci_build/release/common.sh @@ -144,7 +144,7 @@ function install_pip_deps { ${SUDO_CMD} ${PIP_CMD} install scikit-learn ${SUDO_CMD} ${PIP_CMD} install --upgrade tb-nightly ${PIP_CMD} install --user --upgrade attrs - ${PIP_CMD} install --user --upgrade tf-estimator-nightly + ${PIP_CMD} install --user --upgrade tf-estimator-nightly==2.3.0.dev2020041501 ${PIP_CMD} install --user --upgrade "future>=0.17.1" # LINT.ThenChange(:ubuntu_16_pip_installations) } @@ -176,7 +176,7 @@ function install_ubuntu_16_pip_deps { "${PIP_CMD}" install scipy --user "${PIP_CMD}" install scikit-learn --user "${PIP_CMD}" install PyYAML==3.13 --user - "${PIP_CMD}" install --user --upgrade tf-estimator-nightly + "${PIP_CMD}" install --user --upgrade tf-estimator-nightly==2.3.0.dev2020041501 "${PIP_CMD}" install --user --upgrade tb-nightly # LINT.ThenChange(:ubuntu_pip_installations) } @@ -218,7 +218,7 @@ function install_macos_pip_deps { ${SUDO_CMD} ${PIP_CMD} install --upgrade grpcio ${SUDO_CMD} ${PIP_CMD} install --upgrade tb-nightly ${PIP_CMD} install --user --upgrade attrs - ${PIP_CMD} install --user --upgrade tf-estimator-nightly + ${PIP_CMD} install --user --upgrade tf-estimator-nightly==2.3.0.dev2020041501 ${PIP_CMD} install --user --upgrade "future>=0.17.1" } diff --git a/tensorflow/tools/ci_build/release/common_win.bat b/tensorflow/tools/ci_build/release/common_win.bat index 85f22c1e4cb5dd..a593f8cbcdbffa 100644 --- a/tensorflow/tools/ci_build/release/common_win.bat +++ b/tensorflow/tools/ci_build/release/common_win.bat @@ -28,9 +28,9 @@ SET PATH=%PATH%;C:\%PYTHON_DIRECTORY% %PIP_EXE% install setuptools --upgrade %PIP_EXE% install future>=0.17.1 --no-deps -%PIP_EXE% install tf-estimator-nightly --no-deps +%PIP_EXE% install tf-estimator-nightly==2.3.0.dev2020041501 --no-deps %PIP_EXE% install tb-nightly --no-deps -%PIP_EXE% install numpy --upgrade --no-deps +%PIP_EXE% install numpy==1.16.0 --upgrade --no-deps %PIP_EXE% install opt_einsum --upgrade %PIP_EXE% install pandas --upgrade --no-deps %PIP_EXE% install protobuf --upgrade --no-deps diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile index d07d701b9b7e62..afc9b78b4d359f 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile @@ -39,7 +39,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -67,9 +67,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile index 6e8b09de45a608..8c9a46c724e7fb 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile @@ -39,7 +39,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile index ea38e5389458f6..b46d3119009753 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile @@ -68,7 +68,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -119,9 +119,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile index 6c731f24b8d4f4..39104876b67e59 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile @@ -68,7 +68,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile index 5644d536751b11..dbb19a7f97199f 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile @@ -110,7 +110,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -161,9 +161,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile index eb75d905462eb6..6f824dc836231b 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile @@ -110,7 +110,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile index f9ec2b603cf445..e993d903594262 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile @@ -88,7 +88,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -116,9 +116,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile index 33fdc44626e7cd..56886b3b05ec76 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile @@ -88,7 +88,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/devel-horovod-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/devel-horovod-jupyter.Dockerfile index 129fc78db5425b..f22599ba338464 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/devel-horovod-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/devel-horovod-jupyter.Dockerfile @@ -68,7 +68,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -100,7 +100,7 @@ RUN ${PIP} --no-cache-dir install \ enum34 # Install bazel -ARG BAZEL_VERSION=1.2.1 +ARG BAZEL_VERSION=2.0.0 RUN mkdir /bazel && \ wget -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \ wget -O /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \ @@ -171,9 +171,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/devel-horovod.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/devel-horovod.Dockerfile index 245b8f6ee68b64..1e8c5f1fba4779 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/devel-horovod.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/devel-horovod.Dockerfile @@ -68,7 +68,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -100,7 +100,7 @@ RUN ${PIP} --no-cache-dir install \ enum34 # Install bazel -ARG BAZEL_VERSION=1.2.1 +ARG BAZEL_VERSION=2.0.0 RUN mkdir /bazel && \ wget -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \ wget -O /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \ diff --git a/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/horovod-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/horovod-jupyter.Dockerfile index a00bf8365e0706..0931711790db65 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/horovod-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/horovod-jupyter.Dockerfile @@ -39,7 +39,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -119,9 +119,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/horovod.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/horovod.Dockerfile index 4208ea8ffac3d0..b9e97438435815 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/horovod.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/horovod.Dockerfile @@ -39,7 +39,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile index 4dae9f50c4b6b8..7a70e8f4fe5c4a 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile @@ -39,7 +39,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -85,9 +85,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le.Dockerfile index e6a5184c8e77a7..babe5ea86972ea 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le.Dockerfile @@ -39,7 +39,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile index 0870cfc83f9a37..80015c45980061 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile @@ -68,7 +68,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -120,9 +120,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile index 560eca225086a9..6b12dd8790ac65 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile @@ -68,7 +68,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile index cec115afea9aea..45763773f54833 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile @@ -110,7 +110,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -162,9 +162,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile index 7b4507730913d4..dc87869850b32b 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile @@ -110,7 +110,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile index 93ac1fc1bbca91..5d2e2e1996adac 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile @@ -88,7 +88,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -134,9 +134,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile index d174615487ffb4..a661f55be8e38d 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile @@ -88,7 +88,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile index 3ffc295f09b4cd..33c417f0930f0e 100644 --- a/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile +++ b/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile @@ -6,9 +6,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/python.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/python.partial.Dockerfile index 804f8102e522e3..2e948b83da4b1f 100644 --- a/tensorflow/tools/dockerfiles/partials/ubuntu/python.partial.Dockerfile +++ b/tensorflow/tools/dockerfiles/partials/ubuntu/python.partial.Dockerfile @@ -12,7 +12,7 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-pip RUN ${PIP} --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/docs/BUILD b/tensorflow/tools/docs/BUILD index 5d349da84bfda3..6fef052a6f73fe 100644 --- a/tensorflow/tools/docs/BUILD +++ b/tensorflow/tools/docs/BUILD @@ -24,6 +24,7 @@ py_test( srcs = ["tf_doctest.py"], python_version = "PY3", tags = [ + "no_mac", "no_oss_py2", "no_pip", "no_windows", # numpy prints differently on windows. diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index b2ce642994c274..1321ccd8d559fb 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -47,7 +47,7 @@ # result for pip. # Also update tensorflow/tensorflow.bzl and # tensorflow/core/public/version.h -_VERSION = '2.2.0' +_VERSION = '2.2.3' REQUIRED_PACKAGES = [ 'absl-py >= 0.7.0', @@ -58,7 +58,7 @@ 'google_pasta >= 0.1.8', 'h5py >= 2.10.0, < 2.11.0', 'keras_preprocessing >= 1.1.0', - 'numpy >= 1.16.0, < 2.0', + 'numpy >= 1.16.0, < 1.19.0', 'opt_einsum >= 2.3.2', 'protobuf >= 3.8.0', 'tensorboard >= 2.2.0, < 2.3.0', @@ -73,10 +73,6 @@ # functools comes with python3, need to install the backport for python2 'functools32 >= 3.2.3;python_version<"3"', 'six >= 1.12.0', - # scipy < 1.4.1 causes segfaults due to pybind11 - # Latest scipy pip for py2 is scipy==1.2.2 - 'scipy == 1.4.1;python_version>="3"', - 'scipy == 1.2.2;python_version<"3"', ] if sys.byteorder == 'little': diff --git a/tensorflow/tools/pip_package/setup.py.orig b/tensorflow/tools/pip_package/setup.py.orig new file mode 100644 index 00000000000000..6f158a8c84db91 --- /dev/null +++ b/tensorflow/tools/pip_package/setup.py.orig @@ -0,0 +1,313 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""TensorFlow is an open source machine learning framework for everyone. + +TensorFlow is an open source software library for high performance numerical +computation. Its flexible architecture allows easy deployment of computation +across a variety of platforms (CPUs, GPUs, TPUs), and from desktops to clusters +of servers to mobile and edge devices. + +Originally developed by researchers and engineers from the Google Brain team +within Google's AI organization, it comes with strong support for machine +learning and deep learning and the flexible numerical computation core is used +across many other scientific domains. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import fnmatch +import os +import re +import sys + +from setuptools import Command +from setuptools import find_packages +from setuptools import setup +from setuptools.command.install import install as InstallCommandBase +from setuptools.dist import Distribution + +DOCLINES = __doc__.split('\n') + +# This version string is semver compatible, but incompatible with pip. +# For pip, we will remove all '-' characters from this string, and use the +# result for pip. +# Also update tensorflow/tensorflow.bzl and +# tensorflow/core/public/version.h +_VERSION = '2.1.2' + +REQUIRED_PACKAGES = [ + 'absl-py >= 0.7.0', + 'astor >= 0.6.0', + 'backports.weakref >= 1.0rc1;python_version<"3.4"', + 'enum34 >= 1.1.6;python_version<"3.4"', + 'gast == 0.2.2', + 'google_pasta >= 0.1.6', + 'keras_applications >= 1.0.8', + 'keras_preprocessing == 1.1.0', + 'numpy >= 1.16.0, < 1.19.0', + 'opt_einsum >= 2.3.2', + 'protobuf >= 3.8.0', + 'tensorboard >= 2.1.0, < 2.2.0', + 'tensorflow_estimator >= 2.1.0rc0, < 2.2.0', + 'termcolor >= 1.1.0', + 'wrapt >= 1.11.1', + # python3 requires wheel 0.26 + 'wheel >= 0.26;python_version>="3"', + 'wheel;python_version<"3"', +<<<<<<< HEAD + # mock comes with unittest.mock for python3, need to install for python2 + 'mock >= 2.0.0;python_version<"3"', + # functools comes with python3, need to install the backport for python2 + 'functools32 >= 3.2.3;python_version<"3"', + 'six >= 1.12.0', +======= + 'wrapt >= 1.11.1', + # Pin h5py to at most 2.10.0 as newer versions break old keras tests + 'h5py <= 2.10.0', +>>>>>>> 03d7ca7871b (Add upper bound to `h5py`.) +] + +if sys.byteorder == 'little': + # grpcio does not build correctly on big-endian machines due to lack of + # BoringSSL support. + # See https://github.com/tensorflow/tensorflow/issues/17882. + REQUIRED_PACKAGES.append('grpcio >= 1.8.6') + +project_name = 'tensorflow' +if '--project_name' in sys.argv: + project_name_idx = sys.argv.index('--project_name') + project_name = sys.argv[project_name_idx + 1] + sys.argv.remove('--project_name') + sys.argv.pop(project_name_idx) + +# tf-nightly should depend on tb-nightly +if 'tf_nightly' in project_name: + for i, pkg in enumerate(REQUIRED_PACKAGES): + if 'tensorboard' in pkg: + REQUIRED_PACKAGES[i] = 'tb-nightly >= 2.1.0a0, < 2.2.0a0' + elif 'tensorflow_estimator' in pkg and '2.0' in project_name: + REQUIRED_PACKAGES[i] = 'tensorflow-estimator-2.0-preview' + elif 'tensorflow_estimator' in pkg: + REQUIRED_PACKAGES[i] = 'tf-estimator-nightly' + +# pylint: disable=line-too-long +CONSOLE_SCRIPTS = [ + 'toco_from_protos = tensorflow.lite.toco.python.toco_from_protos:main', + 'tflite_convert = tensorflow.lite.python.tflite_convert:main', + 'toco = tensorflow.lite.python.tflite_convert:main', + 'saved_model_cli = tensorflow.python.tools.saved_model_cli:main', + # We need to keep the TensorBoard command, even though the console script + # is now declared by the tensorboard pip package. If we remove the + # TensorBoard command, pip will inappropriately remove it during install, + # even though the command is not removed, just moved to a different wheel. + 'tensorboard = tensorboard.main:run_main', + 'tf_upgrade_v2 = tensorflow.tools.compatibility.tf_upgrade_v2_main:main', + 'estimator_ckpt_converter = tensorflow_estimator.python.estimator.tools.checkpoint_converter:main', +] +# pylint: enable=line-too-long + +# Only keep freeze_graph console script in 1.X. +if _VERSION.startswith('1.') and '_2.0' not in project_name: + CONSOLE_SCRIPTS.append( + 'freeze_graph = tensorflow.python.tools.freeze_graph:run_main') + +# remove the tensorboard console script if building tf_nightly +if 'tf_nightly' in project_name: + CONSOLE_SCRIPTS.remove('tensorboard = tensorboard.main:run_main') + +TEST_PACKAGES = [ + 'scipy >= 0.15.1', +] + + +class BinaryDistribution(Distribution): + + def has_ext_modules(self): + return True + + +class InstallCommand(InstallCommandBase): + """Override the dir where the headers go.""" + + def finalize_options(self): + ret = InstallCommandBase.finalize_options(self) + self.install_headers = os.path.join(self.install_purelib, 'tensorflow_core', + 'include') + self.install_lib = self.install_platlib + return ret + + +class InstallHeaders(Command): + """Override how headers are copied. + + The install_headers that comes with setuptools copies all files to + the same directory. But we need the files to be in a specific directory + hierarchy for -I to work correctly. + """ + description = 'install C/C++ header files' + + user_options = [('install-dir=', 'd', + 'directory to install header files to'), + ('force', 'f', + 'force installation (overwrite existing files)'), + ] + + boolean_options = ['force'] + + def initialize_options(self): + self.install_dir = None + self.force = 0 + self.outfiles = [] + + def finalize_options(self): + self.set_undefined_options('install', + ('install_headers', 'install_dir'), + ('force', 'force')) + + def mkdir_and_copy_file(self, header): + install_dir = os.path.join(self.install_dir, os.path.dirname(header)) + # Get rid of some extra intervening directories so we can have fewer + # directories for -I + install_dir = re.sub('/google/protobuf_archive/src', '', install_dir) + install_dir = re.sub('/include/tensorflow_core/', '/include/tensorflow/', + install_dir) + + # Copy external code headers into tensorflow_core/include. + # A symlink would do, but the wheel file that gets created ignores + # symlink within the directory hierarchy. + # NOTE(keveman): Figure out how to customize bdist_wheel package so + # we can do the symlink. + external_header_locations = [ + 'tensorflow_core/include/external/eigen_archive/', + 'tensorflow_core/include/external/com_google_absl/', + ] + for location in external_header_locations: + if location in install_dir: + extra_dir = install_dir.replace(location, '') + if not os.path.exists(extra_dir): + self.mkpath(extra_dir) + self.copy_file(header, extra_dir) + + if not os.path.exists(install_dir): + self.mkpath(install_dir) + return self.copy_file(header, install_dir) + + def run(self): + hdrs = self.distribution.headers + if not hdrs: + return + + self.mkpath(self.install_dir) + for header in hdrs: + (out, _) = self.mkdir_and_copy_file(header) + self.outfiles.append(out) + + def get_inputs(self): + return self.distribution.headers or [] + + def get_outputs(self): + return self.outfiles + + +def find_files(pattern, root): + """Return all the files matching pattern below root dir.""" + for dirpath, _, files in os.walk(root): + for filename in fnmatch.filter(files, pattern): + yield os.path.join(dirpath, filename) + + +so_lib_paths = [ + i for i in os.listdir('.') + if os.path.isdir(i) and fnmatch.fnmatch(i, '_solib_*') +] + +matches = [] +for path in so_lib_paths: + matches.extend( + ['../' + x for x in find_files('*', path) if '.py' not in x] + ) + +if os.name == 'nt': + EXTENSION_NAME = 'python/_pywrap_tensorflow_internal.pyd' +else: + EXTENSION_NAME = 'python/_pywrap_tensorflow_internal.so' + +headers = ( + list(find_files('*.h', 'tensorflow_core/core')) + + list(find_files('*.h', 'tensorflow_core/stream_executor')) + + list(find_files('*.h', 'google/com_google_protobuf/src')) + + list(find_files('*.inc', 'google/com_google_protobuf/src')) + + list(find_files('*', 'third_party/eigen3')) + list( + find_files('*.h', 'tensorflow_core/include/external/com_google_absl')) + + list( + find_files('*.inc', 'tensorflow_core/include/external/com_google_absl')) + + list(find_files('*', 'tensorflow_core/include/external/eigen_archive'))) + +setup( + name=project_name, + version=_VERSION.replace('-', ''), + description=DOCLINES[0], + long_description='\n'.join(DOCLINES[2:]), + url='https://www.tensorflow.org/', + download_url='https://github.com/tensorflow/tensorflow/tags', + author='Google Inc.', + author_email='packages@tensorflow.org', + # Contained modules and scripts. + packages=find_packages(), + entry_points={ + 'console_scripts': CONSOLE_SCRIPTS, + }, + headers=headers, + install_requires=REQUIRED_PACKAGES, + tests_require=REQUIRED_PACKAGES + TEST_PACKAGES, + # Add in any packaged data. + include_package_data=True, + package_data={ + 'tensorflow': [ + EXTENSION_NAME, + ] + matches, + }, + zip_safe=False, + distclass=BinaryDistribution, + cmdclass={ + 'install_headers': InstallHeaders, + 'install': InstallCommand, + }, + # PyPI package information. + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Mathematics', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + license='Apache 2.0', + keywords='tensorflow tensor machine learning', +) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 262da77e0c22aa..6f0de953a837f8 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -305,12 +305,12 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "org_sqlite", build_file = clean_dep("//third_party:sqlite.BUILD"), - sha256 = "f3c79bc9f4162d0b06fa9fe09ee6ccd23bb99ce310b792c5145f87fbcc30efca", - strip_prefix = "sqlite-amalgamation-3310100", + sha256 = "8ff0b79fd9118af7a760f1f6a98cac3e69daed325c8f9f0a581ecb62f797fd64", + strip_prefix = "sqlite-amalgamation-3340000", system_build_file = clean_dep("//third_party/systemlibs:sqlite.BUILD"), urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/www.sqlite.org/2020/sqlite-amalgamation-3310100.zip", - "https://www.sqlite.org/2020/sqlite-amalgamation-3310100.zip", + "https://storage.googleapis.com/mirror.tensorflow.org/www.sqlite.org/2020/sqlite-amalgamation-3340000.zip", + "https://www.sqlite.org/2020/sqlite-amalgamation-3340000.zip", ], ) @@ -528,12 +528,12 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "pcre", build_file = clean_dep("//third_party:pcre.BUILD"), - sha256 = "69acbc2fbdefb955d42a4c606dfde800c2885711d2979e356c0636efde9ec3b5", - strip_prefix = "pcre-8.42", + sha256 = "aecafd4af3bd0f3935721af77b889d9024b2e01d96b58471bd91a3063fb47728", + strip_prefix = "pcre-8.44", system_build_file = clean_dep("//third_party/systemlibs:pcre.BUILD"), urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/ftp.exim.org/pub/pcre/pcre-8.42.tar.gz", - "https://ftp.exim.org/pub/pcre/pcre-8.42.tar.gz", + "https://storage.googleapis.com/mirror.tensorflow.org/ftp.exim.org/pub/pcre/pcre-8.44.tar.gz", + "https://ftp.exim.org/pub/pcre/pcre-8.44.tar.gz", ], ) @@ -553,12 +553,12 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "curl", build_file = clean_dep("//third_party:curl.BUILD"), - sha256 = "01ae0c123dee45b01bbaef94c0bc00ed2aec89cb2ee0fd598e0d302a6b5e0a98", - strip_prefix = "curl-7.69.1", + sha256 = "3b4378156ba09e224008e81dcce854b7ce4d182b1f9cfb97fe5ed9e9c18c6bd3", + strip_prefix = "curl-7.76.0", system_build_file = clean_dep("//third_party/systemlibs:curl.BUILD"), urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/curl.haxx.se/download/curl-7.69.1.tar.gz", - "https://curl.haxx.se/download/curl-7.69.1.tar.gz", + "https://storage.googleapis.com/mirror.tensorflow.org/curl.haxx.se/download/curl-7.76.0.tar.gz", + "https://curl.haxx.se/download/curl-7.76.0.tar.gz", ], ) @@ -635,12 +635,12 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "jsoncpp_git", build_file = clean_dep("//third_party:jsoncpp.BUILD"), - sha256 = "c49deac9e0933bcb7044f08516861a2d560988540b23de2ac1ad443b219afdb6", - strip_prefix = "jsoncpp-1.8.4", + sha256 = "e34a628a8142643b976c7233ef381457efad79468c67cb1ae0b83a33d7493999", + strip_prefix = "jsoncpp-1.9.4", system_build_file = clean_dep("//third_party/systemlibs:jsoncpp.BUILD"), urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/open-source-parsers/jsoncpp/archive/1.8.4.tar.gz", - "https://github.com/open-source-parsers/jsoncpp/archive/1.8.4.tar.gz", + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/open-source-parsers/jsoncpp/archive/1.9.4.tar.gz", + "https://github.com/open-source-parsers/jsoncpp/archive/1.9.4.tar.gz", ], ) diff --git a/third_party/curl.BUILD b/third_party/curl.BUILD index 62fc946956da85..3b73ac7d29fa8f 100644 --- a/third_party/curl.BUILD +++ b/third_party/curl.BUILD @@ -25,22 +25,33 @@ CURL_WIN_SRCS = [ "lib/asyn-thread.c", "lib/inet_ntop.c", "lib/system_win32.c", - "lib/x509asn1.c", - "lib/vtls/schannel.c", - "lib/vtls/schannel_verify.c", - "lib/idn_win32.c", + "lib/setup-win32.h", ] cc_library( name = "curl", srcs = [ "include/curl_config.h", + "lib/altsvc.c", + "lib/altsvc.h", + "lib/amigaos.c", "lib/amigaos.h", "lib/arpa_telnet.h", - "lib/asyn.h", "lib/asyn-ares.c", + "lib/asyn.h", "lib/base64.c", + "lib/c-hyper.c", + "lib/c-hyper.h", + "lib/config-amigaos.h", + "lib/config-dos.h", + "lib/config-mac.h", + "lib/config-os400.h", + "lib/config-plan9.h", + "lib/config-riscos.h", + "lib/config-tpf.h", + "lib/config-vxworks.h", "lib/config-win32.h", + "lib/config-win32ce.h", "lib/conncache.c", "lib/conncache.h", "lib/connect.c", @@ -54,14 +65,20 @@ cc_library( "lib/curl_base64.h", "lib/curl_ctype.c", "lib/curl_ctype.h", + "lib/curl_des.c", "lib/curl_des.h", + "lib/curl_endian.c", "lib/curl_endian.h", "lib/curl_fnmatch.c", "lib/curl_fnmatch.h", + "lib/curl_get_line.c", + "lib/curl_get_line.h", "lib/curl_gethostname.c", "lib/curl_gethostname.h", + "lib/curl_gssapi.c", "lib/curl_gssapi.h", "lib/curl_hmac.h", + "lib/curl_krb5.h", "lib/curl_ldap.h", "lib/curl_md4.h", "lib/curl_md5.h", @@ -70,14 +87,19 @@ cc_library( "lib/curl_memrchr.h", "lib/curl_multibyte.c", "lib/curl_multibyte.h", + "lib/curl_ntlm_core.c", "lib/curl_ntlm_core.h", + "lib/curl_ntlm_wb.c", "lib/curl_ntlm_wb.h", + "lib/curl_path.c", + "lib/curl_path.h", "lib/curl_printf.h", + "lib/curl_range.c", + "lib/curl_range.h", "lib/curl_rtmp.c", "lib/curl_rtmp.h", "lib/curl_sasl.c", "lib/curl_sasl.h", - "lib/curl_sec.h", "lib/curl_setup.h", "lib/curl_setup_once.h", "lib/curl_sha256.h", @@ -86,23 +108,35 @@ cc_library( "lib/curl_threads.c", "lib/curl_threads.h", "lib/curlx.h", + "lib/dict.c", "lib/dict.h", + "lib/doh.c", + "lib/doh.h", "lib/dotdot.c", "lib/dotdot.h", + "lib/dynbuf.c", + "lib/dynbuf.h", "lib/easy.c", + "lib/easygetopt.c", "lib/easyif.h", + "lib/easyoptions.c", + "lib/easyoptions.h", "lib/escape.c", "lib/escape.h", + "lib/file.c", "lib/file.h", "lib/fileinfo.c", "lib/fileinfo.h", "lib/formdata.c", "lib/formdata.h", + "lib/ftp.c", "lib/ftp.h", + "lib/ftplistparser.c", "lib/ftplistparser.h", "lib/getenv.c", "lib/getinfo.c", "lib/getinfo.h", + "lib/gopher.c", "lib/gopher.h", "lib/hash.c", "lib/hash.h", @@ -115,6 +149,8 @@ cc_library( "lib/hostip4.c", "lib/hostip6.c", "lib/hostsyn.c", + "lib/hsts.c", + "lib/hsts.h", "lib/http.c", "lib/http.h", "lib/http2.c", @@ -123,17 +159,24 @@ cc_library( "lib/http_chunks.h", "lib/http_digest.c", "lib/http_digest.h", + "lib/http_negotiate.c", "lib/http_negotiate.h", + "lib/http_ntlm.c", "lib/http_ntlm.h", "lib/http_proxy.c", "lib/http_proxy.h", + "lib/http_aws_sigv4.c", + "lib/http_aws_sigv4.h", + "lib/idn_win32.c", "lib/if2ip.c", "lib/if2ip.h", + "lib/imap.c", "lib/imap.h", "lib/inet_ntop.h", "lib/inet_pton.c", "lib/inet_pton.h", "lib/krb5.c", + "lib/ldap.c", "lib/llist.c", "lib/llist.h", "lib/md4.c", @@ -143,38 +186,43 @@ cc_library( "lib/mime.c", "lib/mime.h", "lib/mprintf.c", + "lib/mqtt.c", + "lib/mqtt.h", "lib/multi.c", "lib/multihandle.h", "lib/multiif.h", "lib/netrc.c", "lib/netrc.h", + "lib/non-ascii.c", "lib/non-ascii.h", "lib/nonblock.c", "lib/nonblock.h", - "lib/nwlib.c", - "lib/nwos.c", + #"lib/nwlib.c", + #"lib/nwos.c", + "lib/openldap.c", "lib/parsedate.c", "lib/parsedate.h", - "lib/pingpong.h", "lib/pingpong.c", + "lib/pingpong.h", + "lib/pop3.c", "lib/pop3.h", "lib/progress.c", "lib/progress.h", + "lib/psl.c", + "lib/psl.h", "lib/quic.h", "lib/rand.c", "lib/rand.h", - "lib/rename.h", "lib/rename.c", + "lib/rename.h", "lib/rtsp.c", "lib/rtsp.h", - "lib/security.c", "lib/select.c", "lib/select.h", "lib/sendf.c", "lib/sendf.h", "lib/setopt.c", "lib/setopt.h", - "lib/setup-os400.h", "lib/setup-vms.h", "lib/sha256.c", "lib/share.c", @@ -182,13 +230,17 @@ cc_library( "lib/sigpipe.h", "lib/slist.c", "lib/slist.h", + "lib/smb.c", "lib/smb.h", + "lib/smtp.c", "lib/smtp.h", "lib/sockaddr.h", - "lib/socketpair.h", "lib/socketpair.c", + "lib/socketpair.h", "lib/socks.c", "lib/socks.h", + "lib/socks_gssapi.c", + "lib/socks_sspi.c", "lib/speedcheck.c", "lib/speedcheck.h", "lib/splay.c", @@ -204,7 +256,9 @@ cc_library( "lib/strtoofft.c", "lib/strtoofft.h", "lib/system_win32.h", + "lib/telnet.c", "lib/telnet.h", + "lib/tftp.c", "lib/tftp.h", "lib/timeval.c", "lib/timeval.h", @@ -213,44 +267,69 @@ cc_library( "lib/url.c", "lib/url.h", "lib/urldata.h", + "lib/urlapi-int.h", + "lib/urlapi.c", + "lib/version.c", + "lib/version_win32.c", + "lib/version_win32.h", + "lib/warnless.c", + "lib/warnless.h", + "lib/wildcard.c", + "lib/wildcard.h", + "lib/x509asn1.c", + "lib/x509asn1.h", "lib/vauth/cleartext.c", "lib/vauth/cram.c", "lib/vauth/digest.c", "lib/vauth/digest.h", + "lib/vauth/digest_sspi.c", + "lib/vauth/krb5_gssapi.c", + "lib/vauth/krb5_sspi.c", + "lib/vauth/ntlm.c", "lib/vauth/ntlm.h", + "lib/vauth/ntlm_sspi.c", "lib/vauth/oauth2.c", + "lib/vauth/spnego_sspi.c", "lib/vauth/vauth.c", "lib/vauth/vauth.h", - "lib/version.c", + "lib/vquic/ngtcp2.c", + "lib/vquic/ngtcp2.h", + "lib/vquic/quiche.c", + "lib/vquic/quiche.h", + "lib/vquic/vquic.c", + "lib/vquic/vquic.h", + "lib/vssh/libssh.c", + "lib/vssh/libssh2.c", "lib/vssh/ssh.h", + "lib/vssh/wolfssh.c", + "lib/vtls/bearssl.c", "lib/vtls/bearssl.h", + "lib/vtls/gskit.c", "lib/vtls/gskit.h", + "lib/vtls/gtls.c", "lib/vtls/gtls.h", + "lib/vtls/keylog.c", + "lib/vtls/keylog.h", + "lib/vtls/mbedtls.c", "lib/vtls/mbedtls.h", + "lib/vtls/mbedtls_threadlock.c", + "lib/vtls/mbedtls_threadlock.h", + "lib/vtls/mesalink.c", + "lib/vtls/mesalink.h", + "lib/vtls/nss.c", "lib/vtls/nssg.h", + "lib/vtls/openssl.c", "lib/vtls/openssl.h", + "lib/vtls/rustls.c", + "lib/vtls/rustls.h", + "lib/vtls/schannel.c", "lib/vtls/schannel.h", + "lib/vtls/schannel_verify.c", + "lib/vtls/sectransp.h", "lib/vtls/vtls.c", "lib/vtls/vtls.h", + "lib/vtls/wolfssl.c", "lib/vtls/wolfssl.h", - "lib/warnless.c", - "lib/warnless.h", - "lib/wildcard.c", - "lib/wildcard.h", - "lib/x509asn1.h", - "lib/psl.h", - "lib/psl.c", - "lib/vtls/sectransp.h", - "lib/vtls/mesalink.h", - "lib/vtls/mesalink.c", - "lib/curl_get_line.h", - "lib/curl_get_line.c", - "lib/urlapi-int.h", - "lib/urlapi.c", - "lib/altsvc.h", - "lib/altsvc.c", - "lib/doh.h", - "lib/doh.c", ] + select({ "@org_tensorflow//tensorflow:macos": [ "lib/vtls/sectransp.c", @@ -260,7 +339,6 @@ cc_library( ], "@org_tensorflow//tensorflow:windows": CURL_WIN_SRCS, "//conditions:default": [ - "lib/vtls/openssl.c", ], }), hdrs = [ @@ -269,6 +347,7 @@ cc_library( "include/curl/easy.h", "include/curl/mprintf.h", "include/curl/multi.h", + "include/curl/options.h", "include/curl/stdcheaders.h", "include/curl/system.h", "include/curl/typecheck-gcc.h", @@ -372,6 +451,8 @@ cc_binary( "src/tool_doswin.h", "src/tool_easysrc.c", "src/tool_easysrc.h", + "src/tool_filetime.c", + "src/tool_filetime.h", "src/tool_formparse.c", "src/tool_formparse.h", "src/tool_getparam.c", @@ -406,6 +487,8 @@ cc_binary( "src/tool_paramhlp.h", "src/tool_parsecfg.c", "src/tool_parsecfg.h", + "src/tool_progress.c", + "src/tool_progress.h", "src/tool_sdecls.h", "src/tool_setopt.c", "src/tool_setopt.h", @@ -425,6 +508,8 @@ cc_binary( "src/tool_writeenv.h", "src/tool_writeout.c", "src/tool_writeout.h", + "src/tool_writeout_json.c", + "src/tool_writeout_json.h", "src/tool_xattr.c", "src/tool_xattr.h", ], diff --git a/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h b/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h index ff359cedced961..fd35360da28208 100644 --- a/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h +++ b/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h @@ -49,7 +49,7 @@ struct scalar_product_traits { // the compiler from silently type cast the mantissa into a bigger or a smaller // representation. struct QInt8 { - QInt8() {} + QInt8() : value(0) {} QInt8(const int8_t v) : value(v) {} QInt8(const QInt32 v); @@ -59,7 +59,7 @@ struct QInt8 { }; struct QUInt8 { - QUInt8() {} + QUInt8() : value(0) {} QUInt8(const uint8_t v) : value(v) {} QUInt8(const QInt32 v); @@ -69,7 +69,7 @@ struct QUInt8 { }; struct QInt16 { - QInt16() {} + QInt16() : value(0) {} QInt16(const int16_t v) : value(v) {} QInt16(const QInt32 v); operator int() const { return static_cast(value); } @@ -78,7 +78,7 @@ struct QInt16 { }; struct QUInt16 { - QUInt16() {} + QUInt16() : value(0) {} QUInt16(const uint16_t v) : value(v) {} QUInt16(const QInt32 v); operator int() const { return static_cast(value); } @@ -87,7 +87,7 @@ struct QUInt16 { }; struct QInt32 { - QInt32() {} + QInt32() : value(0) {} QInt32(const int8_t v) : value(v) {} QInt32(const int32_t v) : value(v) {} QInt32(const uint32_t v) : value(static_cast(v)) {} diff --git a/third_party/jpeg/workspace.bzl b/third_party/jpeg/workspace.bzl index c458ff12ba8248..60f989df722152 100644 --- a/third_party/jpeg/workspace.bzl +++ b/third_party/jpeg/workspace.bzl @@ -6,11 +6,11 @@ def repo(): third_party_http_archive( name = "libjpeg_turbo", urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/libjpeg-turbo/libjpeg-turbo/archive/2.0.4.tar.gz", - "https://github.com/libjpeg-turbo/libjpeg-turbo/archive/2.0.4.tar.gz", + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/libjpeg-turbo/libjpeg-turbo/archive/2.0.5.tar.gz", + "https://github.com/libjpeg-turbo/libjpeg-turbo/archive/2.0.5.tar.gz", ], - sha256 = "7777c3c19762940cff42b3ba4d7cd5c52d1671b39a79532050c85efb99079064", - strip_prefix = "libjpeg-turbo-2.0.4", + sha256 = "b3090cd37b5a8b3e4dbd30a1311b3989a894e5d3c668f14cbc6739d77c9402b7", + strip_prefix = "libjpeg-turbo-2.0.5", build_file = "//third_party/jpeg:BUILD.bazel", system_build_file = "//third_party/jpeg:BUILD.system", ) diff --git a/third_party/jsoncpp.BUILD b/third_party/jsoncpp.BUILD index cf3cba05556a0b..ee3abd70502400 100644 --- a/third_party/jsoncpp.BUILD +++ b/third_party/jsoncpp.BUILD @@ -12,11 +12,10 @@ cc_library( "src/lib_json/json_writer.cpp", ], hdrs = [ - "include/json/autolink.h", "include/json/config.h", - "include/json/features.h", "include/json/forwards.h", "include/json/json.h", + "include/json/json_features.h", "include/json/reader.h", "include/json/value.h", "include/json/version.h",