From b95885ec68bad8d3616d03a1983183d74a5e5e29 Mon Sep 17 00:00:00 2001 From: Yeqing Li Date: Wed, 26 May 2021 21:43:16 -0700 Subject: [PATCH 01/50] Adds yaml file for training on k600. PiperOrigin-RevId: 376094072 --- .../k600_3d-resnet50g_tpu.yaml | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50g_tpu.yaml diff --git a/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50g_tpu.yaml b/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50g_tpu.yaml new file mode 100644 index 00000000000..3ae54c41564 --- /dev/null +++ b/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50g_tpu.yaml @@ -0,0 +1,112 @@ +# 3D ResNet-50g video classification on Kinetics-600. +# +# --experiment_type=video_classification_kinetics600 +# Expected accuracy: 78.7% accuracy, 93.6% top-5. +# Train on TPU: v3-128, eval on TPU: v3-32 +runtime: + distribution_strategy: 'tpu' + mixed_precision_dtype: 'bfloat16' +task: + init_checkpoint: null + init_checkpoint_modules: all + losses: + l2_weight_decay: 0.0001 + label_smoothing: 0.0 + model: + aggregate_endpoints: false + backbone: + resnet_3d: + block_specs: !!python/tuple + - temporal_kernel_sizes: !!python/tuple + - 3 + - 3 + - 3 + temporal_strides: 1 + use_self_gating: true + - temporal_kernel_sizes: !!python/tuple + - 3 + - 1 + - 3 + - 1 + temporal_strides: 1 + use_self_gating: true + - temporal_kernel_sizes: !!python/tuple + - 3 + - 1 + - 3 + - 1 + - 3 + - 1 + temporal_strides: 1 + use_self_gating: true + - temporal_kernel_sizes: !!python/tuple + - 1 + - 3 + - 1 + temporal_strides: 1 + use_self_gating: true + model_id: 50 + stem_conv_temporal_kernel_size: 5 + stem_conv_temporal_stride: 2 + stem_pool_temporal_stride: 2 + stem_type: v0 + stochastic_depth_drop_rate: 0.0 + type: resnet_3d + dropout_rate: 0.2 + model_type: video_classification + norm_activation: + activation: relu + norm_epsilon: 1.0e-05 + norm_momentum: 0.9 + use_sync_bn: false + train_data: + aug_max_area_ratio: 1.0 + aug_max_aspect_ratio: 2.0 + aug_min_area_ratio: 0.49 + aug_min_aspect_ratio: 0.5 + drop_remainder: true + dtype: 'bfloat16' + feature_shape: !!python/tuple + - 64 + - 224 + - 224 + - 3 + global_batch_size: 1024 + min_image_size: 256 + name: kinetics600 + num_classes: 600 + split: train + validation_data: + dtype: 'bfloat16' + feature_shape: !!python/tuple + - 250 + - 224 + - 224 + - 3 + global_batch_size: 64 + min_image_size: 256 + name: kinetics600 + num_classes: 600 + num_examples: 27780 + num_test_clips: 1 + num_test_crops: 1 + one_hot: true +trainer: + optimizer_config: + learning_rate: + cosine: + alpha: 0.0 + decay_steps: 71400 + initial_learning_rate: 1.6 + name: CosineDecay + type: cosine + warmup: + linear: + name: linear + warmup_learning_rate: 0 + warmup_steps: 1785 + type: linear + train_steps: 71400 + steps_per_loop: 500 + summary_interval: 500 + validation_interval: 500 From 87a04344a961427dd7fb3b852c3396ffba73cf7e Mon Sep 17 00:00:00 2001 From: Woojung Kim Date: Thu, 27 May 2021 14:28:08 +0900 Subject: [PATCH 02/50] Correct typo --- official/nlp/projects/mobilebert/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official/nlp/projects/mobilebert/README.md b/official/nlp/projects/mobilebert/README.md index 9209b4720d6..ef7ec1d62e4 100644 --- a/official/nlp/projects/mobilebert/README.md +++ b/official/nlp/projects/mobilebert/README.md @@ -22,7 +22,7 @@ modeling library: * [mobile_bert_encoder.py](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/networks/mobile_bert_encoder.py) contains `MobileBERTEncoder` implementation. * [mobile_bert_layers.py](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/layers/mobile_bert_layers.py) - contains `MobileBertEmbedding`, `MobileBertMaskedLM` and `MobileBertMaskedLM` + contains `MobileBertEmbedding`, `MobileBertTransformer` and `MobileBertMaskedLM` implementation. ## Pre-trained Models From e39572bb557acc5cc01e10d38941ed62cd255a2f Mon Sep 17 00:00:00 2001 From: Rebecca Chen Date: Thu, 27 May 2021 13:40:25 -0700 Subject: [PATCH 03/50] Internal change PiperOrigin-RevId: 376239468 --- orbit/controller.py | 2 +- orbit/standard_runner.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/orbit/controller.py b/orbit/controller.py index 5242a7a7e42..688d0260796 100644 --- a/orbit/controller.py +++ b/orbit/controller.py @@ -338,7 +338,7 @@ def evaluate_continuously(self, self.restore_checkpoint(checkpoint_path) self.evaluate(steps) - def restore_checkpoint(self, checkpoint_path: str = None): + def restore_checkpoint(self, checkpoint_path: Optional[str] = None): """Restores the model from a checkpoint. Args: diff --git a/orbit/standard_runner.py b/orbit/standard_runner.py index ac03707a0f7..d6ea757af00 100644 --- a/orbit/standard_runner.py +++ b/orbit/standard_runner.py @@ -83,7 +83,9 @@ class StandardTrainer(runner.AbstractTrainer, metaclass=abc.ABCMeta): `tf.function`, as determined by the `options` passed to `__init__`. """ - def __init__(self, train_dataset, options: StandardTrainerOptions = None): + def __init__(self, + train_dataset, + options: Optional[StandardTrainerOptions] = None): """Initializes the `StandardTrainer` instance. Args: @@ -256,7 +258,9 @@ class StandardEvaluator(runner.AbstractEvaluator, metaclass=abc.ABCMeta): is recommended in this case. """ - def __init__(self, eval_dataset, options: StandardEvaluatorOptions = None): + def __init__(self, + eval_dataset, + options: Optional[StandardEvaluatorOptions] = None): """Initializes the `StandardEvaluator` instance. Args: @@ -403,7 +407,7 @@ def eval_end(self, *args) -> Optional[runner.Output]: pass def eval_reduce(self, - state: Any = None, + state: Optional[Any] = None, step_outputs: Optional[runner.Output] = None) -> Any: """A function to perform per-step reduction on the evaluation outputs. From 636fd747a380766ae7ef0b6c339caa603d744296 Mon Sep 17 00:00:00 2001 From: Dan Holtmann-Rice Date: Thu, 27 May 2021 17:07:38 -0700 Subject: [PATCH 04/50] Internal change PiperOrigin-RevId: 376280713 --- orbit/__init__.py | 2 + orbit/actions.py | 429 +++++++++++++++++++++++++++++++++++++++ orbit/actions_test.py | 218 ++++++++++++++++++++ orbit/controller.py | 49 ++++- orbit/controller_test.py | 49 ++++- 5 files changed, 738 insertions(+), 9 deletions(-) create mode 100644 orbit/actions.py create mode 100644 orbit/actions_test.py diff --git a/orbit/__init__.py b/orbit/__init__.py index a97bb719d7a..01442a565d5 100644 --- a/orbit/__init__.py +++ b/orbit/__init__.py @@ -14,8 +14,10 @@ """Defines exported symbols for the `orbit` package.""" +from orbit import actions from orbit import utils +from orbit.controller import Action from orbit.controller import Controller from orbit.runner import AbstractEvaluator diff --git a/orbit/actions.py b/orbit/actions.py new file mode 100644 index 00000000000..d1258134ec9 --- /dev/null +++ b/orbit/actions.py @@ -0,0 +1,429 @@ +# Copyright 2021 The Orbit Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Defines an "action" abstraction for use with `orbit.Controller`. + +"Actions" are simply arbitrary callables that are applied by the `Controller` +to the output of train steps (after each inner loop of `steps_per_loop` steps) +or an evaluation. This provides a hook mechanism, enabling things like reporting +metrics to Vizier, model exporting, additional logging, etc. + +The basic `Action` abstraction (just a type alias) is defined in the +`controller` module. This `actions` module adds a `ConditionalAction` utility +class to make it easy to trigger actions conditionally based on reusable +predicates, as well as a small handful of predefined conditions/actions (in +particular, a `NewBestMetric` condition and an `ExportSavedModel` action). + +One example of using actions to do metric-conditional export: + + new_best_metric = orbit.actions.NewBestMetric('accuracy') + export_action = orbit.actions.ConditionalAction( + condition=lambda x: x['accuracy'] > 0.9 and new_best_metric(x), + action=orbit.actions.ExportSavedModel( + model, + orbit.actions.ExportFileManager( + base_name=f'{FLAGS.model_dir}/saved_model', + next_id_fn=trainer.global_step.numpy), + signatures=model.infer)) + + controller = orbit.Controller( + strategy=strategy, + trainer=trainer, + evaluator=evaluator, + eval_actions=[export_action], + global_step=trainer.global_step, + steps_per_loop=FLAGS.steps_per_loop, + checkpoint_manager=checkpoint_manager, + summary_interval=1000) + +Note: In multi-client settings where each client runs its own `Controller` +instance, some care should be taken in deciding which clients should run certain +actions. Isolating actions to an individual client (say client 0) can be +achieved using `ConditionalAction` as follows: + + client_0_actions = orbit.actions.ConditionalAction( + condition=lambda _: client_id() == 0, + action=[ + ... + ]) + +In particular, the `NewBestMetric` condition may be used in multi-client +settings if all clients are guaranteed to compute the same metric (ensuring this +is up to client code, not Orbit). However, when saving metrics it may be helpful +to avoid unnecessary writes by setting the `write_value` parameter to `False` +for most clients. +""" + +import json +import os +import sys +from typing import Any, Callable, Optional, Sequence, Union +import uuid + +from orbit import controller +from orbit import runner +from orbit import utils + +import tensorflow as tf + +Condition = Callable[[runner.Output], Union[bool, tf.Tensor]] + + +def _as_sequence(maybe_sequence: Union[Any, Sequence[Any]]) -> Sequence[Any]: + if isinstance(maybe_sequence, Sequence): + return maybe_sequence + return [maybe_sequence] + + +class ConditionalAction: + """Represents an action that is only taken when a given condition is met. + + This class is itself an `Action` (a callable that can be applied to train or + eval outputs), but is intended to make it easier to write modular and reusable + conditions by decoupling "when" something whappens (the condition) from "what" + happens (the action). + """ + + def __init__( + self, + condition: Condition, + action: Union[controller.Action, Sequence[controller.Action]], + ): + """Initializes the instance. + + Args: + condition: A callable accepting train or eval outputs and returing a bool. + action: The action (or optionally sequence of actions) to perform when + `condition` is met. + """ + self.condition = condition + self.action = action + + def __call__(self, output: runner.Output) -> None: + if self.condition(output): + for action in _as_sequence(self.action): + action(output) + + +MetricFn = Callable[[runner.Output], Union[float, tf.Tensor]] + + +class NewBestMetric: + """Condition that is satisfied when a new best metric is achieved. + + This class keeps track of the best metric value seen so far, optionally in a + persistent (preemption-safe) way. + + Two methods are provided, which each satisfy the `Action` protocol: `test` for + only testing whether a new best metric is achieved by a given train/eval + output, and `commit`, which both tests and records the new best metric value + if it is achieved. These separate methods enable the same `NewBestMetric` + instance to be reused as a condition multiple times, and can also provide + additional preemption/failure safety. For example, to avoid updating the best + metric if a model export fails or is pre-empted: + + new_best_metric = orbit.actions.NewBestMetric( + 'accuracy', filename='/model/dir/best_metric') + action = orbit.actions.ConditionalAction( + condition=new_best_metric.test, + action=[ + orbit.actions.ExportSavedModel(...), + new_best_metric.commit + ]) + + The default `__call__` implementation is equivalent to `commit`. + + This class is safe to use in multi-client settings if all clients can be + guaranteed to compute the same metric. However when saving metrics it may be + helpful to avoid unnecessary writes by setting the `write_value` parameter to + `False` for most clients. + + Attributes: + metric: The metric passed to __init__ (may be a string key or a callable + that can be applied to train/eval output). + higher_is_better: Whether higher metric values are better. + """ + + def __init__(self, + metric: Union[str, MetricFn], + higher_is_better: bool = True, + filename: Optional[str] = None, + write_metric=True): + """Initializes the instance. + + Args: + metric: Either a string key name to use to look up a metric (assuming the + train/eval output is a dictionary), or a callable that accepts the + train/eval output and returns a metric value. + higher_is_better: Whether higher metric values are better. If `True`, a + new best metric is achieved when the metric value is strictly greater + than the previous best metric. If `False`, a new best metric is achieved + when the metric value is strictly less than the previous best metric. + filename: A filename to use for storage of the best metric value seen so + far, to allow peristence of the value across preemptions. If `None` + (default), values aren't persisted. + write_metric: If `filename` is set, this controls whether this instance + will write new best metric values to the file, or just read from the + file to obtain the initial value. Setting this to `False` for most + clients in some multi-client setups can avoid unnecessary file writes. + Has no effect if `filename` is `None`. + """ + self.metric = metric + self.higher_is_better = higher_is_better + float_max = sys.float_info.max + self._best_value = JSONPersistedValue( + initial_value=-float_max if higher_is_better else float_max, + filename=filename, + write_value=write_metric) + + def __call__(self, output: runner.Output) -> bool: + """Tests `output` and updates the current best value if necessary. + + This is equivalent to `commit` below. + + Args: + output: The train or eval output to test. + + Returns: + `True` if `output` contains a new best metric value, `False` otherwise. + """ + return self.commit(output) + + def metric_value(self, output: runner.Output) -> float: + """Computes the metric value for the given `output`.""" + if callable(self.metric): + value = self.metric(output) + else: + value = output[self.metric] + return float(utils.get_value(value)) + + @property + def best_value(self) -> float: + """Returns the best metric value seen so far.""" + return self._best_value.read() + + def test(self, output: runner.Output) -> bool: + """Tests `output` to see if it contains a new best metric value. + + If `output` does contain a new best metric value, this method does *not* + save it (i.e., calling this method multiple times in a row with the same + `output` will continue to return `True`). + + Args: + output: The train or eval output to test. + + Returns: + `True` if `output` contains a new best metric value, `False` otherwise. + """ + metric_value = self.metric_value(output) + if self.higher_is_better: + if metric_value > self.best_value: + return True + else: # Lower is better. + if metric_value < self.best_value: + return True + return False + + def commit(self, output: runner.Output) -> bool: + """Tests `output` and updates the current best value if necessary. + + Unlike `test` above, if `output` does contain a new best metric value, this + method *does* save it (i.e., subsequent calls to this method with the same + `output` will return `False`). + + Args: + output: The train or eval output to test. + + Returns: + `True` if `output` contains a new best metric value, `False` otherwise. + """ + + if self.test(output): + self._best_value.write(self.metric_value(output)) + return True + return False + + +class JSONPersistedValue: + """Represents a value that is persisted via a file-based backing store. + + The value must be JSON-serializable. Each time the value is updated, it will + be written to the backing file. It is only read from the file at + initialization. + """ + + def __init__(self, + initial_value: Any, + filename: str, + write_value: bool = True): + """Initializes the instance. + + Args: + initial_value: The initial value to use if no backing file exists or was + given. This must be a JSON-serializable value (possibly nested + combination of lists, dicts, and primitive values). + filename: The path to use for persistent storage of the value. This may be + `None`, in which case the value is not stable across preemptions. + write_value: If `True`, new values will be written to `filename` on calls + to `write()`. If `False`, `filename` is only read once to restore any + persisted value, and new values will not be written to it. This can be + useful in certain multi-client settings to avoid race conditions or + excessive file writes. If `filename` is `None`, this parameter has no + effect. + """ + self._value = None + self._filename = filename + self._write_value = write_value + + if self._filename is not None: + if tf.io.gfile.exists(self._filename): + if tf.io.gfile.stat(self._filename).length > 0: + with tf.io.gfile.GFile(self._filename, 'r') as f: + self._value = json.loads(f.read()) + elif self._write_value: + tf.io.gfile.makedirs(os.path.dirname(self._filename)) + + if self._value is None: + self.write(initial_value) + + def read(self): + """Returns the value.""" + return self._value + + def write(self, value): + """Writes the value, updating the backing store if one was provided.""" + self._value = value + if self._filename is not None and self._write_value: + # To achieve atomic writes, we first write to a temporary file, and then + # rename it to `self._filename`. + tmp_filename = f'{self._filename}.tmp.{uuid.uuid4().hex}' + with tf.io.gfile.GFile(tmp_filename, 'w') as f: + json.dump(self._value, f) + tf.io.gfile.rename(tmp_filename, self._filename, overwrite=True) + + +class _CounterIdFn: + """Implements a counter-based ID function for `ExportFileManager`.""" + + def __init__(self, base_name: str): + filenames = tf.io.gfile.glob(f'{base_name}-*') + max_counter = -1 + for filename in filenames: + try: + _, file_number = filename.rsplit('-', maxsplit=1) + max_counter = max(max_counter, int(file_number)) + except ValueError: + continue + self.value = max_counter + 1 + + def __call__(self): + output = self.value + self.value += 1 + return output + + +class ExportFileManager: + """Utility class that manages a group of files with a shared base name. + + For actions like SavedModel exporting, there are potentially many different + file naming and cleanup strategies that may be desirable. This class provides + a basic interface allowing SavedModel export to be decoupled from these + details, and a default implementation that should work for many basic + scenarios. Users may subclass this class to alter behavior and define more + customized naming and cleanup strategies. + """ + + def __init__(self, + base_name: str, + max_to_keep: int = 5, + next_id_fn: Optional[Callable[[], int]] = None): + """Initializes the instance. + + Args: + base_name: A shared base name for file names generated by this class. + max_to_keep: The maximum number of files matching `base_name` to keep + after each call to `cleanup`. The most recent (as determined by file + modification time) `max_to_keep` files are preserved; the rest are + deleted. If < 0, all files are preserved. + next_id_fn: An optional callable that returns integer IDs to append to + base name (formatted as `'{base_name}-{id}'`). The order of integers is + used to sort files to determine the oldest ones deleted by `clean_up`. + If not supplied, a default ID based on an incrementing counter is used. + One common alternative maybe be to use the current global step count, + for instance passing `next_id_fn=global_step.numpy`. + """ + self._base_name = base_name + self._max_to_keep = max_to_keep + self._next_id_fn = next_id_fn or _CounterIdFn(base_name) + + @property + def managed_files(self): + """Returns all files managed by this instance, in sorted order. + + Returns: + The list of files matching the `base_name` provided when constructing this + `ExportFileManager` instance, sorted in increasing integer order of the + IDs returned by `next_id_fn`. + """ + + def id_key(name): + _, id_num = name.rsplit('-', maxsplit=1) + return int(id_num) + + filenames = tf.io.gfile.glob(f'{self._base_name}-*') + return sorted(filenames, key=id_key) + + def clean_up(self): + """Cleans up old files matching `{base_name}-*`. + + The most recent `max_to_keep` files are preserved. + """ + if self._max_to_keep < 0: + return + + for filename in self.managed_files[:-self._max_to_keep]: + tf.io.gfile.rmtree(filename) + + def next_name(self) -> str: + """Returns a new file name based on `base_name` and `next_id_fn()`.""" + return f'{self._base_name}-{self._next_id_fn()}' + + +class ExportSavedModel: + """Action that exports the given model as a SavedModel.""" + + def __init__(self, + model: tf.Module, + file_manager: ExportFileManager, + signatures, + options: Optional[tf.saved_model.SaveOptions] = None): + """Initializes the instance. + + Args: + model: The model to export. + file_manager: An instance of `ExportFileManager` (or a subclass), that + provides file naming and cleanup functionality. + signatures: The signatures to forward to `tf.saved_model.save()`. + options: Optional options to forward to `tf.saved_model.save()`. + """ + self.model = model + self.file_manager = file_manager + self.signatures = signatures + self.options = options + + def __call__(self, _): + """Exports the SavedModel.""" + export_dir = self.file_manager.next_name() + tf.saved_model.save(self.model, export_dir, self.signatures, self.options) + self.file_manager.clean_up() diff --git a/orbit/actions_test.py b/orbit/actions_test.py new file mode 100644 index 00000000000..5d15724431a --- /dev/null +++ b/orbit/actions_test.py @@ -0,0 +1,218 @@ +# Copyright 2021 The Orbit Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for orbit.actions.""" + +import os + +from orbit import actions + +import tensorflow as tf + + +def _id_key(name): + _, id_num = name.rsplit('-', maxsplit=1) + return int(id_num) + + +def _id_sorted_file_base_names(dir_path): + return sorted(tf.io.gfile.listdir(dir_path), key=_id_key) + + +class TestModel(tf.Module): + + def __init__(self): + self.value = tf.Variable(0) + + @tf.function(input_signature=[]) + def __call__(self): + return self.value + + +class ActionsTest(tf.test.TestCase): + + def test_conditional_action(self): + # Define a function to raise an AssertionError, since we can't in a lambda. + def raise_assertion(arg): + raise AssertionError(str(arg)) + + conditional_action = actions.ConditionalAction( + condition=lambda x: x, action=raise_assertion) + + conditional_action(False) # Nothing is raised. + with self.assertRaises(AssertionError) as ctx: + conditional_action(True) + self.assertEqual(ctx.exception.message, 'True') + + def test_new_best_metric_higher_is_better(self): + new_best_metric = actions.NewBestMetric(lambda x: x, higher_is_better=True) + self.assertTrue(new_best_metric.test(0.0)) + self.assertTrue(new_best_metric.commit(0.0)) + self.assertFalse(new_best_metric.test(0.0)) + self.assertTrue(new_best_metric.test(1.0)) + + def test_new_best_metric_lower_is_better(self): + new_best_metric = actions.NewBestMetric(lambda x: x, higher_is_better=False) + self.assertTrue(new_best_metric.test(0.0)) + self.assertTrue(new_best_metric.commit(0.0)) + self.assertFalse(new_best_metric.test(0.0)) + self.assertTrue(new_best_metric.test(-1.0)) + + def test_new_best_metric_persistence(self): + backing_file = self.create_tempfile() + new_best_metric = actions.NewBestMetric( + lambda x: x, + higher_is_better=True, + filename=backing_file.full_path, + write_metric=False) + self.assertTrue(new_best_metric.test(0.0)) + self.assertTrue(new_best_metric.commit(0.0)) + self.assertFalse(new_best_metric.test(0.0)) + new_best_metric = actions.NewBestMetric( + lambda x: x, higher_is_better=True, filename=backing_file.full_path) + self.assertLess(new_best_metric.best_value, 0.0) + self.assertTrue(new_best_metric.commit(5.0)) + self.assertEqual(new_best_metric.best_value, 5.0) + new_best_metric = actions.NewBestMetric( + lambda x: x, higher_is_better=True, filename=backing_file.full_path) + self.assertEqual(new_best_metric.best_value, 5.0) + + def test_json_persisted_value(self): + tempfile = self.create_tempfile().full_path + value = {'a': 1, 'b': 2} + persisted_value = actions.JSONPersistedValue(value, tempfile) + # The inital value is used since tempfile is empty. + self.assertEqual(persisted_value.read(), value) + persisted_value = actions.JSONPersistedValue('ignored', tempfile) + # Initial value of 'ignored' is ignored, since there's a value in tempfile. + self.assertEqual(persisted_value.read(), value) + value = [1, 2, 3] + persisted_value.write(value) + # Now that a new value is written, it gets read on initialization. + persisted_value = actions.JSONPersistedValue(['also ignored'], tempfile) + self.assertEqual(persisted_value.read(), value) + # Writes can be disabled. + persisted_value = actions.JSONPersistedValue( + 'ignored', tempfile, write_value=False) + self.assertEqual(persisted_value.read(), value) + persisted_value.write("won't get persisted") + persisted_value = actions.JSONPersistedValue( + 'ignored', tempfile, write_value=False) + self.assertEqual(persisted_value.read(), value) + + def test_json_persisted_value_create_dirs(self): + tempfile = os.path.join(self.create_tempdir().full_path, 'subdir/value') + value = {'a': 1, 'b': 2} + # The directory is not created if write_value=False. + actions.JSONPersistedValue(value, tempfile, write_value=False) + self.assertFalse(tf.io.gfile.exists(os.path.dirname(tempfile))) + actions.JSONPersistedValue(value, tempfile) + self.assertTrue(tf.io.gfile.exists(tempfile)) + + def test_export_file_manager_default_ids(self): + directory = self.create_tempdir() + base_name = os.path.join(directory.full_path, 'basename') + manager = actions.ExportFileManager(base_name, max_to_keep=3) + self.assertLen(tf.io.gfile.listdir(directory.full_path), 0) + directory.create_file(manager.next_name()) + manager.clean_up() # Shouldn't do anything... + self.assertLen(tf.io.gfile.listdir(directory.full_path), 1) + directory.create_file(manager.next_name()) + manager.clean_up() # Shouldn't do anything... + self.assertLen(tf.io.gfile.listdir(directory.full_path), 2) + directory.create_file(manager.next_name()) + manager.clean_up() # Shouldn't do anything... + self.assertLen(tf.io.gfile.listdir(directory.full_path), 3) + directory.create_file(manager.next_name()) + self.assertLen(tf.io.gfile.listdir(directory.full_path), 4) + self.assertEqual( + _id_sorted_file_base_names(directory.full_path), + ['basename-0', 'basename-1', 'basename-2', 'basename-3']) + manager.clean_up() # Should delete file with lowest ID. + self.assertEqual( + _id_sorted_file_base_names(directory.full_path), + ['basename-1', 'basename-2', 'basename-3']) + manager = actions.ExportFileManager(base_name, max_to_keep=3) + self.assertEqual(os.path.basename(manager.next_name()), 'basename-4') + + def test_export_file_manager_custom_ids(self): + directory = self.create_tempdir() + base_name = os.path.join(directory.full_path, 'basename') + + id_num = 0 + + def next_id(): + return id_num + + manager = actions.ExportFileManager( + base_name, max_to_keep=2, next_id_fn=next_id) + self.assertLen(tf.io.gfile.listdir(directory.full_path), 0) + id_num = 30 + directory.create_file(manager.next_name()) + self.assertLen(tf.io.gfile.listdir(directory.full_path), 1) + manager.clean_up() # Shouldn't do anything... + self.assertEqual( + _id_sorted_file_base_names(directory.full_path), ['basename-30']) + id_num = 200 + directory.create_file(manager.next_name()) + self.assertLen(tf.io.gfile.listdir(directory.full_path), 2) + manager.clean_up() # Shouldn't do anything... + self.assertEqual( + _id_sorted_file_base_names(directory.full_path), + ['basename-30', 'basename-200']) + id_num = 1000 + directory.create_file(manager.next_name()) + self.assertLen(tf.io.gfile.listdir(directory.full_path), 3) + self.assertEqual( + _id_sorted_file_base_names(directory.full_path), + ['basename-30', 'basename-200', 'basename-1000']) + manager.clean_up() # Should delete file with lowest ID. + self.assertLen(tf.io.gfile.listdir(directory.full_path), 2) + self.assertEqual( + _id_sorted_file_base_names(directory.full_path), + ['basename-200', 'basename-1000']) + + def test_export_saved_model(self): + directory = self.create_tempdir() + base_name = os.path.join(directory.full_path, 'basename') + file_manager = actions.ExportFileManager(base_name, max_to_keep=2) + model = TestModel() + export_action = actions.ExportSavedModel( + model, file_manager=file_manager, signatures=model.__call__) + + model.value.assign(3) + self.assertEqual(model(), 3) + self.assertEmpty(file_manager.managed_files) + export_action({}) + self.assertLen(file_manager.managed_files, 1) + reloaded_model = tf.saved_model.load(file_manager.managed_files[-1]) + self.assertEqual(reloaded_model(), 3) + + model.value.assign(5) + self.assertEqual(model(), 5) + export_action({}) + self.assertLen(file_manager.managed_files, 2) + reloaded_model = tf.saved_model.load(file_manager.managed_files[-1]) + self.assertEqual(reloaded_model(), 5) + + model.value.assign(7) + self.assertEqual(model(), 7) + export_action({}) + self.assertLen(file_manager.managed_files, 2) # Still 2, due to clean up. + reloaded_model = tf.saved_model.load(file_manager.managed_files[-1]) + self.assertEqual(reloaded_model(), 7) + + +if __name__ == '__main__': + tf.test.main() diff --git a/orbit/controller.py b/orbit/controller.py index 688d0260796..525331c7ec2 100644 --- a/orbit/controller.py +++ b/orbit/controller.py @@ -17,7 +17,7 @@ import pprint import time -from typing import Callable, Optional, Union +from typing import Callable, List, Optional, Union from absl import logging @@ -46,6 +46,9 @@ def _format_output(output, indent=4): return "\n" + "\n".join(lines) +Action = Callable[[runner.Output], None] + + class Controller: """Class that controls the outer loop of model training and evaluation. @@ -53,10 +56,9 @@ class Controller: loops are implemented by users in the form of `AbstractTrainer` and `AbstractEvaluator` subclasses, and define how to run a given number of training or evaluation steps. The outer loop is provided by this `Controller`, - and interleaves calls to the user provided inner loops with additional actions - such as saving checkpoints, running evaluations, and writing summaries - (depending on the arguments passed to `Controller.__init__` and the method - being called). + and interleaves calls to the user-provided inner loops with additional actions + such as saving checkpoints, running evaluations, writing summaries, as well as + (optionally) user provided `Action`s (see below). There are four top-level "outer loops" provided: @@ -70,6 +72,15 @@ class Controller: training and evaluation use cases, the internal details and method implementations are also intended to be simple enough to make subclassing or other custom outer loop implementations easy to achieve. + + Some additional customization can be achieved by supplying `train_actions` or + `eval_actions` when constructing the `Controller`. These are just lists of + arbitrary callables that are applied by the `Controller` to the output of + train steps (after each inner loop of `steps_per_loop` steps) or an + evaluation. This provides a hook mechanism, enabling things like reporting + metrics to Vizier, model exporting, additional logging, etc. See the + `orbit.actions` package for a small handful of predefined actions and some + utility classes that may be useful in defining your own. """ def __init__( @@ -79,6 +90,9 @@ def __init__( trainer: Optional[runner.AbstractTrainer] = None, evaluator: Optional[runner.AbstractEvaluator] = None, strategy: Optional[tf.distribute.Strategy] = None, + # Actions + train_actions: Optional[List[Action]] = None, + eval_actions: Optional[List[Action]] = None, # Train related steps_per_loop: Optional[int] = None, checkpoint_manager: Optional[tf.train.CheckpointManager] = None, @@ -86,7 +100,8 @@ def __init__( summary_interval: Optional[int] = None, summary_dir: Optional[str] = None, # Evaluation related - eval_summary_dir: Optional[str] = None): + eval_summary_dir: Optional[str] = None, + ): """Initializes a `Controller` instance. Note that if `checkpoint_manager` is provided and there are checkpoints in @@ -110,6 +125,12 @@ def __init__( strategy: An instance of `tf.distribute.Strategy`. If not provided, the strategy will be initialized from the current in-scope strategy using `tf.distribute.get_strategy()`. + train_actions: An optional list of `orbit.Action`s to call after each + block of `steps_per_loop` training steps are run. These will be called + with the output of `trainer.train`. + eval_actions: An optional list of `orbit.Action`s to call after each + evaluation. These will be called with the output of + `evaluator.evaluate`. steps_per_loop: The number of steps to run in each inner loop of training (passed as the `num_steps` parameter of `trainer.train`). checkpoint_manager: An instance of `tf.train.CheckpointManager`. If @@ -138,6 +159,7 @@ def __init__( """ if trainer is None and evaluator is None: raise ValueError("`trainer` and `evaluator` should not both be `None`.") + if trainer is not None: if steps_per_loop is None: raise ValueError( @@ -163,6 +185,9 @@ def __init__( self.strategy = strategy or tf.distribute.get_strategy() + self.train_actions = train_actions or [] + self.eval_actions = eval_actions or [] + self.global_step = global_step self.checkpoint_manager = checkpoint_manager @@ -255,9 +280,13 @@ def evaluate(self, steps: int = -1) -> Optional[runner.Output]: with self.eval_summary_manager.summary_writer().as_default(): steps_tensor = tf.convert_to_tensor(steps, dtype=tf.int32) eval_output = self.evaluator.evaluate(steps_tensor) - eval_output = tf.nest.map_structure(utils.get_value, eval_output or {}) elapsed = time.time() - start + eval_output = eval_output or {} + for action in self.eval_actions: + action(eval_output) + eval_output = tf.nest.map_structure(utils.get_value, eval_output) + _log(f" eval | step: {current_step: 6d} | " f"eval time: {elapsed: 6.1f} sec | " f"output: {_format_output(eval_output)}") @@ -408,7 +437,6 @@ def _train_n_steps(self, num_steps: int): with tf.summary.record_if(should_record): num_steps_tensor = tf.convert_to_tensor(num_steps, dtype=tf.int32) train_output = self.trainer.train(num_steps_tensor) - train_output = tf.nest.map_structure(utils.get_value, train_output or {}) # Verify that global_step was updated properly, then update current_step. expected_step = current_step + num_steps @@ -420,6 +448,11 @@ def _train_n_steps(self, num_steps: int): logging.warning(message) return + train_output = train_output or {} + for action in self.train_actions: + action(train_output) + train_output = tf.nest.map_structure(utils.get_value, train_output) + current_step = expected_step steps_per_second = self.step_timer.steps_per_second() _log(f"train | step: {current_step: 6d} | " diff --git a/orbit/controller_test.py b/orbit/controller_test.py index b4620b83bd7..fd1d1b8b87c 100644 --- a/orbit/controller_test.py +++ b/orbit/controller_test.py @@ -583,7 +583,7 @@ def test_early_stop_on_eval_loss(self): test_runner = TestRunner() class EarlyStopController(controller.Controller): - """A subclass of Controller supports early stopping.""" + """A subclass of Controller that supports early stopping.""" def train_and_evaluate(self, train_steps: int = None, @@ -724,5 +724,52 @@ def test_evaluate_with_nested_summaries(self): summaries_with_matching_keyword( "accuracy", os.path.join(self.model_dir, "dataset2"))) + def test_actions(self): + test_runner = TestRunner() + checkpoint = tf.train.Checkpoint( + model=test_runner.model, optimizer=test_runner.optimizer) + checkpoint_manager = tf.train.CheckpointManager( + checkpoint, + self.model_dir, + max_to_keep=None, + step_counter=test_runner.global_step, + checkpoint_interval=10) + + class OutputRecorderAction: + """Simple `Action` that just saves the outputs passed to `__call__`.""" + + def __init__(self): + self.outputs = [] + + def __call__(self, output): + self.outputs.append(output) + + train_output_recorder = OutputRecorderAction() + eval_output_recorder = OutputRecorderAction() + + test_controller = controller.Controller( + trainer=test_runner, + evaluator=test_runner, + train_actions=[train_output_recorder], + eval_actions=[eval_output_recorder], + global_step=test_runner.global_step, + steps_per_loop=2, + summary_dir=os.path.join(self.model_dir, "summaries/train"), + checkpoint_manager=checkpoint_manager, + eval_summary_dir=os.path.join(self.model_dir, "summaries/eval")) + test_controller.train_and_evaluate( + train_steps=10, eval_steps=2, eval_interval=6) + + self.assertLen(train_output_recorder.outputs, 5) + for output in train_output_recorder.outputs: + self.assertIn("loss", output) + self.assertGreaterEqual(output["loss"], 0) + + self.assertLen(eval_output_recorder.outputs, 2) + for output in eval_output_recorder.outputs: + self.assertIn("eval_loss", output) + self.assertGreaterEqual(output["eval_loss"], 0) + + if __name__ == "__main__": tf.test.main() From b3fa67e07354180fa123c42214e520639f23918a Mon Sep 17 00:00:00 2001 From: Rebecca Chen Date: Thu, 27 May 2021 19:44:43 -0700 Subject: [PATCH 05/50] Internal change PiperOrigin-RevId: 376298243 --- official/core/base_task.py | 5 ++- .../modeling/optimization/ema_optimizer.py | 4 +-- .../optimization/optimizer_factory.py | 6 ++-- official/nlp/modeling/models/xlnet.py | 4 +-- .../modeling/heads/dense_prediction_heads.py | 2 +- .../modeling/layers/detection_generator.py | 2 +- .../assemblenet/modeling/assemblenet.py | 14 ++++---- .../movinet/modeling/movinet_model.py | 25 +++++++------- .../executor/distributed_executor.py | 34 +++++++++---------- .../efficientnet/efficientnet_model.py | 14 ++++---- 10 files changed, 58 insertions(+), 52 deletions(-) diff --git a/official/core/base_task.py b/official/core/base_task.py index 3ef5d0d5984..f7bba16f2cf 100644 --- a/official/core/base_task.py +++ b/official/core/base_task.py @@ -38,7 +38,10 @@ class Task(tf.Module, metaclass=abc.ABCMeta): # Special keys in train/validate step returned logs. loss = "loss" - def __init__(self, params, logging_dir: str = None, name: str = None): + def __init__(self, + params, + logging_dir: Optional[str] = None, + name: Optional[str] = None): """Task initialization. Args: diff --git a/official/modeling/optimization/ema_optimizer.py b/official/modeling/optimization/ema_optimizer.py index 3bf3c3607df..c4f44d7124d 100644 --- a/official/modeling/optimization/ema_optimizer.py +++ b/official/modeling/optimization/ema_optimizer.py @@ -14,7 +14,7 @@ """Exponential moving average optimizer.""" -from typing import Text, List +from typing import List, Optional, Text import tensorflow as tf @@ -106,7 +106,7 @@ def has_shadow_copy(self): def _create_slots(self, var_list): self._optimizer._create_slots(var_list=var_list) # pylint: disable=protected-access - def apply_gradients(self, grads_and_vars, name: Text = None): + def apply_gradients(self, grads_and_vars, name: Optional[Text] = None): result = self._optimizer.apply_gradients(grads_and_vars, name) self.update_average(self.iterations) return result diff --git a/official/modeling/optimization/optimizer_factory.py b/official/modeling/optimization/optimizer_factory.py index c5080989642..09bb5deda6f 100644 --- a/official/modeling/optimization/optimizer_factory.py +++ b/official/modeling/optimization/optimizer_factory.py @@ -13,7 +13,7 @@ # limitations under the License. """Optimizer factory class.""" -from typing import Callable, Union +from typing import Callable, Optional, Union import gin import tensorflow as tf @@ -134,8 +134,8 @@ def build_learning_rate(self): def build_optimizer( self, lr: Union[tf.keras.optimizers.schedules.LearningRateSchedule, float], - postprocessor: Callable[[tf.keras.optimizers.Optimizer], - tf.keras.optimizers.Optimizer] = None): + postprocessor: Optional[Callable[[tf.keras.optimizers.Optimizer], + tf.keras.optimizers.Optimizer]] = None): """Build optimizer. Builds optimizer from config. It takes learning rate as input, and builds diff --git a/official/nlp/modeling/models/xlnet.py b/official/nlp/modeling/models/xlnet.py index 4b5a54e7b8f..1e932ce21af 100644 --- a/official/nlp/modeling/models/xlnet.py +++ b/official/nlp/modeling/models/xlnet.py @@ -15,7 +15,7 @@ """XLNet models.""" # pylint: disable=g-classes-have-attributes -from typing import Any, Mapping, Union +from typing import Any, Mapping, Optional, Union import tensorflow as tf @@ -99,7 +99,7 @@ def __init__( network: Union[tf.keras.layers.Layer, tf.keras.Model], mlm_activation=None, mlm_initializer='glorot_uniform', - name: str = None, + name: Optional[str] = None, **kwargs): super().__init__(name=name, **kwargs) self._config = { diff --git a/official/vision/beta/modeling/heads/dense_prediction_heads.py b/official/vision/beta/modeling/heads/dense_prediction_heads.py index a9da2d3b32f..fc9fdf0b38c 100644 --- a/official/vision/beta/modeling/heads/dense_prediction_heads.py +++ b/official/vision/beta/modeling/heads/dense_prediction_heads.py @@ -36,7 +36,7 @@ def __init__( num_anchors_per_location: int, num_convs: int = 4, num_filters: int = 256, - attribute_heads: List[Dict[str, Any]] = None, + attribute_heads: Optional[List[Dict[str, Any]]] = None, use_separable_conv: bool = False, activation: str = 'relu', use_sync_bn: bool = False, diff --git a/official/vision/beta/modeling/layers/detection_generator.py b/official/vision/beta/modeling/layers/detection_generator.py index b069a199ea8..2c2948714f6 100644 --- a/official/vision/beta/modeling/layers/detection_generator.py +++ b/official/vision/beta/modeling/layers/detection_generator.py @@ -593,7 +593,7 @@ def __call__(self, raw_scores: Mapping[str, tf.Tensor], anchor_boxes: tf.Tensor, image_shape: tf.Tensor, - raw_attributes: Mapping[str, tf.Tensor] = None): + raw_attributes: Optional[Mapping[str, tf.Tensor]] = None): """Generates final detections. Args: diff --git a/official/vision/beta/projects/assemblenet/modeling/assemblenet.py b/official/vision/beta/projects/assemblenet/modeling/assemblenet.py index ea6c2cef367..beb127bd258 100644 --- a/official/vision/beta/projects/assemblenet/modeling/assemblenet.py +++ b/official/vision/beta/projects/assemblenet/modeling/assemblenet.py @@ -411,7 +411,7 @@ class _ApplyEdgeWeight(layers.Layer): def __init__(self, weights_shape, - index: int = None, + index: Optional[int] = None, use_5d_mode: bool = False, model_edge_weights: Optional[List[Any]] = None, **kwargs): @@ -471,7 +471,7 @@ def build(self, input_shape: tf.TensorShape): def call(self, inputs: List[tf.Tensor], - training: bool = None) -> Mapping[Any, List[tf.Tensor]]: + training: Optional[bool] = None) -> Mapping[Any, List[tf.Tensor]]: use_5d_mode = self._use_5d_mode dtype = inputs[0].dtype assert len(inputs) > 1 @@ -517,7 +517,7 @@ def call(self, def multi_connection_fusion(inputs: List[tf.Tensor], - index: int = None, + index: Optional[int] = None, use_5d_mode: bool = False, model_edge_weights: Optional[List[Any]] = None): """Do weighted summation of multiple different sized tensors. @@ -893,7 +893,8 @@ def __init__(self, num_classes, num_frames: int, model_structure: List[Any], - input_specs: Mapping[str, tf.keras.layers.InputSpec] = None, + input_specs: Optional[Mapping[str, + tf.keras.layers.InputSpec]] = None, max_pool_preditions: bool = False, **kwargs): if not input_specs: @@ -1018,7 +1019,8 @@ def build_assemblenet_v1( input_specs: tf.keras.layers.InputSpec, backbone_config: hyperparams.Config, norm_activation_config: hyperparams.Config, - l2_regularizer: tf.keras.regularizers.Regularizer = None) -> tf.keras.Model: + l2_regularizer: Optional[tf.keras.regularizers.Regularizer] = None +) -> tf.keras.Model: """Builds assemblenet backbone.""" del l2_regularizer @@ -1058,7 +1060,7 @@ def build_assemblenet_model( input_specs: tf.keras.layers.InputSpec, model_config: cfg.AssembleNetModel, num_classes: int, - l2_regularizer: tf.keras.regularizers.Regularizer = None): + l2_regularizer: Optional[tf.keras.regularizers.Regularizer] = None): """Builds assemblenet model.""" input_specs_dict = {'image': input_specs} backbone = build_assemblenet_v1(input_specs, model_config.backbone, diff --git a/official/vision/beta/projects/movinet/modeling/movinet_model.py b/official/vision/beta/projects/movinet/modeling/movinet_model.py index 552880a8b77..0fc74b4765c 100644 --- a/official/vision/beta/projects/movinet/modeling/movinet_model.py +++ b/official/vision/beta/projects/movinet/modeling/movinet_model.py @@ -16,7 +16,7 @@ Reference: https://arxiv.org/pdf/2103.11511.pdf """ -from typing import Mapping +from typing import Mapping, Optional from absl import logging import tensorflow as tf @@ -31,16 +31,17 @@ class MovinetClassifier(tf.keras.Model): """A video classification class builder.""" - def __init__(self, - backbone: tf.keras.Model, - num_classes: int, - input_specs: Mapping[str, tf.keras.layers.InputSpec] = None, - dropout_rate: float = 0.0, - kernel_initializer: str = 'HeNormal', - kernel_regularizer: tf.keras.regularizers.Regularizer = None, - bias_regularizer: tf.keras.regularizers.Regularizer = None, - output_states: bool = False, - **kwargs): + def __init__( + self, + backbone: tf.keras.Model, + num_classes: int, + input_specs: Optional[Mapping[str, tf.keras.layers.InputSpec]] = None, + dropout_rate: float = 0.0, + kernel_initializer: str = 'HeNormal', + kernel_regularizer: Optional[tf.keras.regularizers.Regularizer] = None, + bias_regularizer: Optional[tf.keras.regularizers.Regularizer] = None, + output_states: bool = False, + **kwargs): """Movinet initialization function. Args: @@ -144,7 +145,7 @@ def build_movinet_model( input_specs: tf.keras.layers.InputSpec, model_config: cfg.MovinetModel, num_classes: int, - l2_regularizer: tf.keras.regularizers.Regularizer = None): + l2_regularizer: Optional[tf.keras.regularizers.Regularizer] = None): """Builds movinet model.""" logging.info('Building movinet model with num classes: %s', num_classes) if l2_regularizer is not None: diff --git a/official/vision/detection/executor/distributed_executor.py b/official/vision/detection/executor/distributed_executor.py index 8f8c861c99f..128271e73ec 100644 --- a/official/vision/detection/executor/distributed_executor.py +++ b/official/vision/detection/executor/distributed_executor.py @@ -322,21 +322,21 @@ def _test_step_fn(inputs): return test_step - def train(self, - train_input_fn: Callable[[params_dict.ParamsDict], tf.data.Dataset], - eval_input_fn: Callable[[params_dict.ParamsDict], - tf.data.Dataset] = None, - model_dir: Text = None, - total_steps: int = 1, - iterations_per_loop: int = 1, - train_metric_fn: Callable[[], Any] = None, - eval_metric_fn: Callable[[], Any] = None, - summary_writer_fn: Callable[[Text, Text], - SummaryWriter] = SummaryWriter, - init_checkpoint: Callable[[tf.keras.Model], Any] = None, - custom_callbacks: List[tf.keras.callbacks.Callback] = None, - continuous_eval: bool = False, - save_config: bool = True): + def train( + self, + train_input_fn: Callable[[params_dict.ParamsDict], tf.data.Dataset], + eval_input_fn: Optional[Callable[[params_dict.ParamsDict], + tf.data.Dataset]] = None, + model_dir: Optional[Text] = None, + total_steps: int = 1, + iterations_per_loop: int = 1, + train_metric_fn: Optional[Callable[[], Any]] = None, + eval_metric_fn: Optional[Callable[[], Any]] = None, + summary_writer_fn: Callable[[Text, Text], SummaryWriter] = SummaryWriter, + init_checkpoint: Optional[Callable[[tf.keras.Model], Any]] = None, + custom_callbacks: Optional[List[tf.keras.callbacks.Callback]] = None, + continuous_eval: bool = False, + save_config: bool = True): """Runs distributed training. Args: @@ -590,7 +590,7 @@ def evaluate_from_model_dir( eval_input_fn: Callable[[params_dict.ParamsDict], tf.data.Dataset], eval_metric_fn: Callable[[], Any], total_steps: int = -1, - eval_timeout: int = None, + eval_timeout: Optional[int] = None, min_eval_interval: int = 180, summary_writer_fn: Callable[[Text, Text], SummaryWriter] = SummaryWriter): """Runs distributed evaluation on model folder. @@ -646,7 +646,7 @@ def evaluate_checkpoint(self, eval_input_fn: Callable[[params_dict.ParamsDict], tf.data.Dataset], eval_metric_fn: Callable[[], Any], - summary_writer: SummaryWriter = None): + summary_writer: Optional[SummaryWriter] = None): """Runs distributed evaluation on the one checkpoint. Args: diff --git a/official/vision/image_classification/efficientnet/efficientnet_model.py b/official/vision/image_classification/efficientnet/efficientnet_model.py index e5f2c2c69fd..c331b080f97 100644 --- a/official/vision/image_classification/efficientnet/efficientnet_model.py +++ b/official/vision/image_classification/efficientnet/efficientnet_model.py @@ -160,9 +160,9 @@ def conv2d_block(inputs: tf.Tensor, strides: Any = (1, 1), use_batch_norm: bool = True, use_bias: bool = False, - activation: Any = None, + activation: Optional[Any] = None, depthwise: bool = False, - name: Text = None): + name: Optional[Text] = None): """A conv2d followed by batch norm and an activation.""" batch_norm = common_modules.get_batch_norm(config.batch_norm) bn_momentum = config.bn_momentum @@ -212,7 +212,7 @@ def conv2d_block(inputs: tf.Tensor, def mb_conv_block(inputs: tf.Tensor, block: BlockConfig, config: ModelConfig, - prefix: Text = None): + prefix: Optional[Text] = None): """Mobile Inverted Residual Bottleneck. Args: @@ -432,8 +432,8 @@ class EfficientNet(tf.keras.Model): """ def __init__(self, - config: ModelConfig = None, - overrides: Dict[Text, Any] = None): + config: Optional[ModelConfig] = None, + overrides: Optional[Dict[Text, Any]] = None): """Create an EfficientNet model. Args: @@ -463,9 +463,9 @@ def __init__(self, @classmethod def from_name(cls, model_name: Text, - model_weights_path: Text = None, + model_weights_path: Optional[Text] = None, weights_format: Text = 'saved_model', - overrides: Dict[Text, Any] = None): + overrides: Optional[Dict[Text, Any]] = None): """Construct an EfficientNet model from a predefined model name. E.g., `EfficientNet.from_name('efficientnet-b0')`. From 0c9253b4a0b34935cf78bd13e6520bbeee2f5f92 Mon Sep 17 00:00:00 2001 From: Rebecca Chen Date: Thu, 27 May 2021 19:44:51 -0700 Subject: [PATCH 06/50] Internal change PiperOrigin-RevId: 376298252 --- .../vision/beta/modeling/backbones/mobilenet.py | 5 +++-- .../vision/beta/modeling/backbones/resnet_3d.py | 8 +++++--- official/vision/beta/modeling/maskrcnn_model.py | 8 ++++---- .../beta/modeling/video_classification_model.py | 2 +- .../vision/image_classification/callbacks.py | 16 ++++++++-------- .../image_classification/dataset_factory.py | 7 +++++-- .../image_classification/optimizer_factory.py | 10 +++++----- .../vision/image_classification/preprocessing.py | 2 +- 8 files changed, 32 insertions(+), 26 deletions(-) diff --git a/official/vision/beta/modeling/backbones/mobilenet.py b/official/vision/beta/modeling/backbones/mobilenet.py index 84647962c6f..0d77c8facdd 100644 --- a/official/vision/beta/modeling/backbones/mobilenet.py +++ b/official/vision/beta/modeling/backbones/mobilenet.py @@ -502,7 +502,7 @@ def __init__( kernel_regularizer: Optional[tf.keras.regularizers.Regularizer] = None, bias_regularizer: Optional[tf.keras.regularizers.Regularizer] = None, # The followings should be kept the same most of the times. - output_stride: int = None, + output_stride: Optional[int] = None, min_depth: int = 8, # divisible is not used in MobileNetV1. divisible_by: int = 8, @@ -768,7 +768,8 @@ def build_mobilenet( input_specs: tf.keras.layers.InputSpec, backbone_config: hyperparams.Config, norm_activation_config: hyperparams.Config, - l2_regularizer: tf.keras.regularizers.Regularizer = None) -> tf.keras.Model: + l2_regularizer: Optional[tf.keras.regularizers.Regularizer] = None +) -> tf.keras.Model: """Builds MobileNet backbone from a config.""" backbone_type = backbone_config.type backbone_cfg = backbone_config.get() diff --git a/official/vision/beta/modeling/backbones/resnet_3d.py b/official/vision/beta/modeling/backbones/resnet_3d.py index b9207a4a317..f1876df24bd 100644 --- a/official/vision/beta/modeling/backbones/resnet_3d.py +++ b/official/vision/beta/modeling/backbones/resnet_3d.py @@ -81,7 +81,7 @@ def __init__( model_id: int, temporal_strides: List[int], temporal_kernel_sizes: List[Tuple[int]], - use_self_gating: List[int] = None, + use_self_gating: Optional[List[int]] = None, input_specs: tf.keras.layers.InputSpec = layers.InputSpec( shape=[None, None, None, None, 3]), stem_type: str = 'v0', @@ -380,7 +380,8 @@ def build_resnet3d( input_specs: tf.keras.layers.InputSpec, backbone_config: hyperparams.Config, norm_activation_config: hyperparams.Config, - l2_regularizer: tf.keras.regularizers.Regularizer = None) -> tf.keras.Model: + l2_regularizer: Optional[tf.keras.regularizers.Regularizer] = None +) -> tf.keras.Model: """Builds ResNet 3d backbone from a config.""" backbone_cfg = backbone_config.get() @@ -418,7 +419,8 @@ def build_resnet3d_rs( input_specs: tf.keras.layers.InputSpec, backbone_config: hyperparams.Config, norm_activation_config: hyperparams.Config, - l2_regularizer: tf.keras.regularizers.Regularizer = None) -> tf.keras.Model: + l2_regularizer: Optional[tf.keras.regularizers.Regularizer] = None +) -> tf.keras.Model: """Builds ResNet-3D-RS backbone from a config.""" backbone_cfg = backbone_config.get() diff --git a/official/vision/beta/modeling/maskrcnn_model.py b/official/vision/beta/modeling/maskrcnn_model.py index e85d0e57547..2a18ccb3df9 100644 --- a/official/vision/beta/modeling/maskrcnn_model.py +++ b/official/vision/beta/modeling/maskrcnn_model.py @@ -140,10 +140,10 @@ def call(self, images: tf.Tensor, image_shape: tf.Tensor, anchor_boxes: Optional[Mapping[str, tf.Tensor]] = None, - gt_boxes: tf.Tensor = None, - gt_classes: tf.Tensor = None, - gt_masks: tf.Tensor = None, - training: bool = None) -> Mapping[str, tf.Tensor]: + gt_boxes: Optional[tf.Tensor] = None, + gt_classes: Optional[tf.Tensor] = None, + gt_masks: Optional[tf.Tensor] = None, + training: Optional[bool] = None) -> Mapping[str, tf.Tensor]: model_outputs = {} # Feature extraction. diff --git a/official/vision/beta/modeling/video_classification_model.py b/official/vision/beta/modeling/video_classification_model.py index 34a2edeca0a..f65df6c228a 100644 --- a/official/vision/beta/modeling/video_classification_model.py +++ b/official/vision/beta/modeling/video_classification_model.py @@ -27,7 +27,7 @@ def __init__( self, backbone: tf.keras.Model, num_classes: int, - input_specs: Mapping[str, tf.keras.layers.InputSpec] = None, + input_specs: Optional[Mapping[str, tf.keras.layers.InputSpec]] = None, dropout_rate: float = 0.0, aggregate_endpoints: bool = False, kernel_initializer: str = 'random_uniform', diff --git a/official/vision/image_classification/callbacks.py b/official/vision/image_classification/callbacks.py index 033a2dd714f..cffe605c9fd 100644 --- a/official/vision/image_classification/callbacks.py +++ b/official/vision/image_classification/callbacks.py @@ -20,7 +20,7 @@ from __future__ import print_function import os -from typing import Any, List, MutableMapping, Text +from typing import Any, List, MutableMapping, Optional, Text from absl import logging import tensorflow as tf @@ -39,7 +39,7 @@ def get_callbacks( initial_step: int = 0, batch_size: int = 0, log_steps: int = 0, - model_dir: str = None, + model_dir: Optional[str] = None, backup_and_restore: bool = False) -> List[tf.keras.callbacks.Callback]: """Get all callbacks.""" model_dir = model_dir or '' @@ -120,7 +120,7 @@ def __init__(self, def on_batch_begin(self, epoch: int, - logs: MutableMapping[str, Any] = None) -> None: + logs: Optional[MutableMapping[str, Any]] = None) -> None: self.step += 1 if logs is None: logs = {} @@ -129,7 +129,7 @@ def on_batch_begin(self, def on_epoch_begin(self, epoch: int, - logs: MutableMapping[str, Any] = None) -> None: + logs: Optional[MutableMapping[str, Any]] = None) -> None: if logs is None: logs = {} metrics = self._calculate_metrics() @@ -140,7 +140,7 @@ def on_epoch_begin(self, def on_epoch_end(self, epoch: int, - logs: MutableMapping[str, Any] = None) -> None: + logs: Optional[MutableMapping[str, Any]] = None) -> None: if logs is None: logs = {} metrics = self._calculate_metrics() @@ -195,13 +195,13 @@ def set_model(self, model: tf.keras.Model): optimization.ExponentialMovingAverage) self.model.optimizer.shadow_copy(self.model) - def on_test_begin(self, logs: MutableMapping[Text, Any] = None): + def on_test_begin(self, logs: Optional[MutableMapping[Text, Any]] = None): self.model.optimizer.swap_weights() - def on_test_end(self, logs: MutableMapping[Text, Any] = None): + def on_test_end(self, logs: Optional[MutableMapping[Text, Any]] = None): self.model.optimizer.swap_weights() - def on_train_end(self, logs: MutableMapping[Text, Any] = None): + def on_train_end(self, logs: Optional[MutableMapping[Text, Any]] = None): if self.overwrite_weights_on_train_end: self.model.optimizer.assign_average_vars(self.model.variables) diff --git a/official/vision/image_classification/dataset_factory.py b/official/vision/image_classification/dataset_factory.py index 463de95c77e..1b8a67fd5fd 100644 --- a/official/vision/image_classification/dataset_factory.py +++ b/official/vision/image_classification/dataset_factory.py @@ -280,7 +280,9 @@ def info(self) -> tfds.core.DatasetInfo: raise e return self.builder_info - def build(self, strategy: tf.distribute.Strategy = None) -> tf.data.Dataset: + def build( + self, + strategy: Optional[tf.distribute.Strategy] = None) -> tf.data.Dataset: """Construct a dataset end-to-end and return it using an optional strategy. Args: @@ -305,7 +307,8 @@ def build(self, strategy: tf.distribute.Strategy = None) -> tf.data.Dataset: def _build( self, - input_context: tf.distribute.InputContext = None) -> tf.data.Dataset: + input_context: Optional[tf.distribute.InputContext] = None + ) -> tf.data.Dataset: """Construct a dataset end-to-end and return it. Args: diff --git a/official/vision/image_classification/optimizer_factory.py b/official/vision/image_classification/optimizer_factory.py index e3eaba944b5..a0f6c929d57 100644 --- a/official/vision/image_classification/optimizer_factory.py +++ b/official/vision/image_classification/optimizer_factory.py @@ -18,7 +18,7 @@ # from __future__ import google_type_annotations from __future__ import print_function -from typing import Any, Dict, Text +from typing import Any, Dict, Optional, Text from absl import logging import tensorflow as tf @@ -35,7 +35,7 @@ def build_optimizer( optimizer_name: Text, base_learning_rate: tf.keras.optimizers.schedules.LearningRateSchedule, params: Dict[Text, Any], - model: tf.keras.Model = None): + model: Optional[tf.keras.Model] = None): """Build the optimizer based on name. Args: @@ -124,9 +124,9 @@ def build_optimizer( def build_learning_rate(params: base_configs.LearningRateConfig, - batch_size: int = None, - train_epochs: int = None, - train_steps: int = None): + batch_size: Optional[int] = None, + train_epochs: Optional[int] = None, + train_steps: Optional[int] = None): """Build the learning rate given the provided configuration.""" decay_type = params.name base_lr = params.initial_lr diff --git a/official/vision/image_classification/preprocessing.py b/official/vision/image_classification/preprocessing.py index dece1fbc119..6c7f88d61b6 100644 --- a/official/vision/image_classification/preprocessing.py +++ b/official/vision/image_classification/preprocessing.py @@ -329,7 +329,7 @@ def load_eval_image(filename: Text, image_size: int = IMAGE_SIZE) -> tf.Tensor: def build_eval_dataset(filenames: List[Text], - labels: List[int] = None, + labels: Optional[List[int]] = None, image_size: int = IMAGE_SIZE, batch_size: int = 1) -> tf.Tensor: """Builds a tf.data.Dataset from a list of filenames and labels. From c6eb983b2ee124c1e980f2f2f14dda5a54448881 Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Tue, 1 Jun 2021 12:25:25 -0700 Subject: [PATCH 07/50] Internal change. PiperOrigin-RevId: 376892317 --- official/vision/beta/serving/detection.py | 4 ++-- official/vision/beta/serving/detection_test.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/official/vision/beta/serving/detection.py b/official/vision/beta/serving/detection.py index 7061048e4b6..e8bb5edf98f 100644 --- a/official/vision/beta/serving/detection.py +++ b/official/vision/beta/serving/detection.py @@ -34,9 +34,9 @@ class DetectionModule(export_base.ExportModule): def _build_model(self): if self._batch_size is None: - ValueError("batch_size can't be None for detection models") + raise ValueError('batch_size cannot be None for detection models.') if not self.params.task.model.detection_generator.use_batched_nms: - ValueError('Only batched_nms is supported.') + raise ValueError('Only batched_nms is supported.') input_specs = tf.keras.layers.InputSpec(shape=[self._batch_size] + self._input_image_size + [3]) diff --git a/official/vision/beta/serving/detection_test.py b/official/vision/beta/serving/detection_test.py index 26ec504cfa7..a4d761eb17d 100644 --- a/official/vision/beta/serving/detection_test.py +++ b/official/vision/beta/serving/detection_test.py @@ -118,6 +118,20 @@ def test_export(self, input_type, experiment_name, image_size): self.assertAllClose(outputs['num_detections'].numpy(), expected_outputs['num_detections'].numpy()) + def test_build_model_fail_with_none_batch_size(self): + params = exp_factory.get_exp_config('retinanet_resnetfpn_coco') + with self.assertRaisesRegex( + ValueError, 'batch_size cannot be None for detection models.'): + detection.DetectionModule( + params, batch_size=None, input_image_size=[640, 640]) + + def test_build_model_fail_with_batched_nms_false(self): + params = exp_factory.get_exp_config('retinanet_resnetfpn_coco') + params.task.model.detection_generator.use_batched_nms = False + with self.assertRaisesRegex(ValueError, 'Only batched_nms is supported.'): + detection.DetectionModule( + params, batch_size=1, input_image_size=[640, 640]) + if __name__ == '__main__': tf.test.main() From aaaf721f3992e9c885886bd4de72463078b8a02b Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Tue, 1 Jun 2021 15:12:23 -0700 Subject: [PATCH 08/50] Internal change. PiperOrigin-RevId: 376928064 --- official/core/base_task.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/official/core/base_task.py b/official/core/base_task.py index f7bba16f2cf..598c7235859 100644 --- a/official/core/base_task.py +++ b/official/core/base_task.py @@ -297,11 +297,38 @@ def inference_step(self, inputs, model: tf.keras.Model): return model(inputs, training=False) def aggregate_logs(self, state, step_logs): - """Optional aggregation over logs returned from a validation step.""" + """Optional aggregation over logs returned from a validation step. + + Given step_logs from a validation step, this function aggregates the logs + after each eval_step() (see eval_reduce() function in + official/core/base_trainer.py). It runs on CPU and can be used to aggregate + metrics during validation, when there are too many metrics that cannot fit + into TPU memory. Note that this may increase latency due to data transfer + between TPU and CPU. Also, the step output from a validation step may be a + tuple with elements from replicas, and a concatenation of the elements is + needed in such case. + + Args: + state: The current state of training, for example, it can be a sequence of + metrics. + step_logs: Logs from a validation step. Can be a dictionary. + """ pass def reduce_aggregated_logs(self, aggregated_logs, global_step: Optional[tf.Tensor] = None): - """Optional reduce of aggregated logs over validation steps.""" + """Optional reduce of aggregated logs over validation steps. + + This function reduces aggregated logs at the end of validation, and can be + used to compute the final metrics. It runs on CPU and in each eval_end() in + base trainer (see eval_end() function in official/core/base_trainer.py). + + Args: + aggregated_logs: Aggregated logs over multiple validation steps. + global_step: An optional variable of global step. + + Returns: + A dictionary of reduced results. + """ return {} From 67686d4c8ea4802c3efaceef692d99d5b49271cd Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 1 Jun 2021 18:15:54 -0700 Subject: [PATCH 09/50] Internal change PiperOrigin-RevId: 376960380 --- official/nlp/projects/teams/README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 official/nlp/projects/teams/README.md diff --git a/official/nlp/projects/teams/README.md b/official/nlp/projects/teams/README.md new file mode 100644 index 00000000000..f57aa266d06 --- /dev/null +++ b/official/nlp/projects/teams/README.md @@ -0,0 +1,21 @@ +# TEAMS (Training ELECTRA Augmented with Multi-word Selection) + +**Note:** This project is working in progress and please stay tuned. + +TEAMS is a text encoder pre-training method that simultaneously learns a +generator and a discriminator using multi-task learning. We propose a new +pre-training task, multi-word selection, and combine it with previous +pre-training tasks for efficient encoder pre-training. We also develop two +techniques, attention-based task-specific heads and partial layer sharing, +to further improve pre-training effectiveness. + + +Our academic paper [[1]](#1) which describes TEAMS in detail can be found here: +https://arxiv.org/abs/2106.00139. + +## References + +[1] +Jiaming Shen, Jialu Liu, Tianqi Liu, Cong Yu and Jiawei Han, "Training ELECTRA +Augmented with Multi-word Selection", Findings of the Association for +Computational Linguistics: ACL 2021. From 0b9a2a74f6dfae3cb1a6ad5857694eea622a069b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 2 Jun 2021 10:21:37 -0700 Subject: [PATCH 10/50] Internal change PiperOrigin-RevId: 377088653 --- official/nlp/tasks/sentence_prediction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/official/nlp/tasks/sentence_prediction.py b/official/nlp/tasks/sentence_prediction.py index 64b9835fa6d..e79651e8e80 100644 --- a/official/nlp/tasks/sentence_prediction.py +++ b/official/nlp/tasks/sentence_prediction.py @@ -206,10 +206,10 @@ def reduce_aggregated_logs(self, aggregated_logs, global_step=None): def initialize(self, model): """Load a pretrained checkpoint (if exists) and then train from iter 0.""" ckpt_dir_or_file = self.task_config.init_checkpoint - if tf.io.gfile.isdir(ckpt_dir_or_file): - ckpt_dir_or_file = tf.train.latest_checkpoint(ckpt_dir_or_file) if not ckpt_dir_or_file: return + if tf.io.gfile.isdir(ckpt_dir_or_file): + ckpt_dir_or_file = tf.train.latest_checkpoint(ckpt_dir_or_file) pretrain2finetune_mapping = { 'encoder': model.checkpoint_items['encoder'], From 213a9649949f5ed8d85e9ec49122cb7b1260f4e5 Mon Sep 17 00:00:00 2001 From: Ronny Votel Date: Wed, 2 Jun 2021 10:54:58 -0700 Subject: [PATCH 11/50] Introducing groundtruth instance mask weights. PiperOrigin-RevId: 377096964 --- .../object_detection/core/preprocessor.py | 31 ++++++++- .../core/preprocessor_test.py | 43 +++++++++++- .../object_detection/core/standard_fields.py | 2 + .../data_decoders/tf_example_decoder.py | 24 +++++++ .../data_decoders/tf_example_decoder_test.py | 68 +++++++++++++++++++ research/object_detection/inputs.py | 11 +++ research/object_detection/inputs_test.py | 11 ++- 7 files changed, 182 insertions(+), 8 deletions(-) diff --git a/research/object_detection/core/preprocessor.py b/research/object_detection/core/preprocessor.py index 50c37786b86..6d63d86131f 100644 --- a/research/object_detection/core/preprocessor.py +++ b/research/object_detection/core/preprocessor.py @@ -1414,6 +1414,7 @@ def _strict_random_crop_image(image, label_confidences=None, multiclass_scores=None, masks=None, + mask_weights=None, keypoints=None, keypoint_visibilities=None, densepose_num_points=None, @@ -1451,6 +1452,8 @@ def _strict_random_crop_image(image, masks: (optional) rank 3 float32 tensor with shape [num_instances, height, width] containing instance masks. The masks are of the same height, width as the input `image`. + mask_weights: (optional) rank 1 float32 tensor with shape [num_instances] + with instance masks weights. keypoints: (optional) rank 3 float32 tensor with shape [num_instances, num_keypoints, 2]. The keypoints are in y-x normalized coordinates. @@ -1488,7 +1491,7 @@ def _strict_random_crop_image(image, Boxes are in normalized form. labels: new labels. - If label_weights, multiclass_scores, masks, keypoints, + If label_weights, multiclass_scores, masks, mask_weights, keypoints, keypoint_visibilities, densepose_num_points, densepose_part_ids, or densepose_surface_coords is not None, the function also returns: label_weights: rank 1 float32 tensor with shape [num_instances]. @@ -1496,6 +1499,8 @@ def _strict_random_crop_image(image, [num_instances, num_classes] masks: rank 3 float32 tensor with shape [num_instances, height, width] containing instance masks. + mask_weights: rank 1 float32 tensor with shape [num_instances] with mask + weights. keypoints: rank 3 float32 tensor with shape [num_instances, num_keypoints, 2] keypoint_visibilities: rank 2 bool tensor with shape @@ -1605,6 +1610,12 @@ def _strict_random_crop_image(image, 0]:im_box_end[0], im_box_begin[1]:im_box_end[1]] result.append(new_masks) + if mask_weights is not None: + mask_weights_inside_window = tf.gather(mask_weights, inside_window_ids) + mask_weights_completely_inside_window = tf.gather( + mask_weights_inside_window, keep_ids) + result.append(mask_weights_completely_inside_window) + if keypoints is not None: keypoints_of_boxes_inside_window = tf.gather(keypoints, inside_window_ids) keypoints_of_boxes_completely_inside_window = tf.gather( @@ -1654,6 +1665,7 @@ def random_crop_image(image, label_confidences=None, multiclass_scores=None, masks=None, + mask_weights=None, keypoints=None, keypoint_visibilities=None, densepose_num_points=None, @@ -1701,6 +1713,8 @@ def random_crop_image(image, masks: (optional) rank 3 float32 tensor with shape [num_instances, height, width] containing instance masks. The masks are of the same height, width as the input `image`. + mask_weights: (optional) rank 1 float32 tensor with shape [num_instances] + containing weights for each instance mask. keypoints: (optional) rank 3 float32 tensor with shape [num_instances, num_keypoints, 2]. The keypoints are in y-x normalized coordinates. @@ -1751,6 +1765,7 @@ def random_crop_image(image, [num_instances, num_classes] masks: rank 3 float32 tensor with shape [num_instances, height, width] containing instance masks. + mask_weights: rank 1 float32 tensor with shape [num_instances]. keypoints: rank 3 float32 tensor with shape [num_instances, num_keypoints, 2] keypoint_visibilities: rank 2 bool tensor with shape @@ -1771,6 +1786,7 @@ def strict_random_crop_image_fn(): label_confidences=label_confidences, multiclass_scores=multiclass_scores, masks=masks, + mask_weights=mask_weights, keypoints=keypoints, keypoint_visibilities=keypoint_visibilities, densepose_num_points=densepose_num_points, @@ -1803,6 +1819,8 @@ def strict_random_crop_image_fn(): outputs.append(multiclass_scores) if masks is not None: outputs.append(masks) + if mask_weights is not None: + outputs.append(mask_weights) if keypoints is not None: outputs.append(keypoints) if keypoint_visibilities is not None: @@ -4388,6 +4406,7 @@ def get_default_func_arg_map(include_label_weights=True, include_label_confidences=False, include_multiclass_scores=False, include_instance_masks=False, + include_instance_mask_weights=False, include_keypoints=False, include_keypoint_visibilities=False, include_dense_pose=False, @@ -4403,6 +4422,8 @@ def get_default_func_arg_map(include_label_weights=True, multiclass scores, too. include_instance_masks: If True, preprocessing functions will modify the instance masks, too. + include_instance_mask_weights: If True, preprocessing functions will modify + the instance mask weights. include_keypoints: If True, preprocessing functions will modify the keypoints, too. include_keypoint_visibilities: If True, preprocessing functions will modify @@ -4434,6 +4455,11 @@ def get_default_func_arg_map(include_label_weights=True, groundtruth_instance_masks = ( fields.InputDataFields.groundtruth_instance_masks) + groundtruth_instance_mask_weights = None + if include_instance_mask_weights: + groundtruth_instance_mask_weights = ( + fields.InputDataFields.groundtruth_instance_mask_weights) + groundtruth_keypoints = None if include_keypoints: groundtruth_keypoints = fields.InputDataFields.groundtruth_keypoints @@ -4503,7 +4529,8 @@ def get_default_func_arg_map(include_label_weights=True, fields.InputDataFields.groundtruth_boxes, fields.InputDataFields.groundtruth_classes, groundtruth_label_weights, groundtruth_label_confidences, - multiclass_scores, groundtruth_instance_masks, groundtruth_keypoints, + multiclass_scores, groundtruth_instance_masks, + groundtruth_instance_mask_weights, groundtruth_keypoints, groundtruth_keypoint_visibilities, groundtruth_dp_num_points, groundtruth_dp_part_ids, groundtruth_dp_surface_coords), random_pad_image: diff --git a/research/object_detection/core/preprocessor_test.py b/research/object_detection/core/preprocessor_test.py index f08de2c5b88..b844a17164b 100644 --- a/research/object_detection/core/preprocessor_test.py +++ b/research/object_detection/core/preprocessor_test.py @@ -1894,6 +1894,37 @@ def graph_fn(): self.assertAllClose( new_boxes.flatten(), expected_boxes.flatten()) + def testStrictRandomCropImageWithMaskWeights(self): + def graph_fn(): + image = self.createColorfulTestImage()[0] + boxes = self.createTestBoxes() + labels = self.createTestLabels() + weights = self.createTestGroundtruthWeights() + masks = tf.random_uniform([2, 200, 400], dtype=tf.float32) + mask_weights = tf.constant([1.0, 0.0], dtype=tf.float32) + with mock.patch.object( + tf.image, + 'sample_distorted_bounding_box' + ) as mock_sample_distorted_bounding_box: + mock_sample_distorted_bounding_box.return_value = ( + tf.constant([6, 143, 0], dtype=tf.int32), + tf.constant([190, 237, -1], dtype=tf.int32), + tf.constant([[[0.03, 0.3575, 0.98, 0.95]]], dtype=tf.float32)) + results = preprocessor._strict_random_crop_image( + image, boxes, labels, weights, masks=masks, + mask_weights=mask_weights) + return results + (new_image, new_boxes, _, _, + new_masks, new_mask_weights) = self.execute_cpu(graph_fn, []) + expected_boxes = np.array( + [[0.0, 0.0, 0.75789469, 1.0], + [0.23157893, 0.24050637, 0.75789469, 1.0]], dtype=np.float32) + self.assertAllEqual(new_image.shape, [190, 237, 3]) + self.assertAllEqual(new_masks.shape, [2, 190, 237]) + self.assertAllClose(new_mask_weights, [1.0, 0.0]) + self.assertAllClose( + new_boxes.flatten(), expected_boxes.flatten()) + def testStrictRandomCropImageWithKeypoints(self): def graph_fn(): image = self.createColorfulTestImage()[0] @@ -1947,6 +1978,7 @@ def graph_fn(): labels = self.createTestLabels() weights = self.createTestGroundtruthWeights() masks = tf.random_uniform([2, 200, 400], dtype=tf.float32) + mask_weights = tf.constant([1.0, 0.0], dtype=tf.float32) tensor_dict = { fields.InputDataFields.image: image, @@ -1954,10 +1986,12 @@ def graph_fn(): fields.InputDataFields.groundtruth_classes: labels, fields.InputDataFields.groundtruth_weights: weights, fields.InputDataFields.groundtruth_instance_masks: masks, + fields.InputDataFields.groundtruth_instance_mask_weights: + mask_weights } preprocessor_arg_map = preprocessor.get_default_func_arg_map( - include_instance_masks=True) + include_instance_masks=True, include_instance_mask_weights=True) preprocessing_options = [(preprocessor.random_crop_image, {})] @@ -1980,16 +2014,19 @@ def graph_fn(): fields.InputDataFields.groundtruth_classes] distorted_masks = distorted_tensor_dict[ fields.InputDataFields.groundtruth_instance_masks] + distorted_mask_weights = distorted_tensor_dict[ + fields.InputDataFields.groundtruth_instance_mask_weights] return [distorted_image, distorted_boxes, distorted_labels, - distorted_masks] + distorted_masks, distorted_mask_weights] (distorted_image_, distorted_boxes_, distorted_labels_, - distorted_masks_) = self.execute_cpu(graph_fn, []) + distorted_masks_, distorted_mask_weights_) = self.execute_cpu(graph_fn, []) expected_boxes = np.array([ [0.0, 0.0, 0.75789469, 1.0], [0.23157893, 0.24050637, 0.75789469, 1.0], ], dtype=np.float32) self.assertAllEqual(distorted_image_.shape, [1, 190, 237, 3]) self.assertAllEqual(distorted_masks_.shape, [2, 190, 237]) + self.assertAllClose(distorted_mask_weights_, [1.0, 0.0]) self.assertAllEqual(distorted_labels_, [1, 2]) self.assertAllClose( distorted_boxes_.flatten(), expected_boxes.flatten()) diff --git a/research/object_detection/core/standard_fields.py b/research/object_detection/core/standard_fields.py index 1925c550615..2cb895cb822 100644 --- a/research/object_detection/core/standard_fields.py +++ b/research/object_detection/core/standard_fields.py @@ -64,6 +64,7 @@ class InputDataFields(object): proposal_boxes: coordinates of object proposal boxes. proposal_objectness: objectness score of each proposal. groundtruth_instance_masks: ground truth instance masks. + groundtruth_instance_mask_weights: ground truth instance masks weights. groundtruth_instance_boundaries: ground truth instance boundaries. groundtruth_instance_classes: instance mask-level class labels. groundtruth_keypoints: ground truth keypoints. @@ -122,6 +123,7 @@ class InputDataFields(object): proposal_boxes = 'proposal_boxes' proposal_objectness = 'proposal_objectness' groundtruth_instance_masks = 'groundtruth_instance_masks' + groundtruth_instance_mask_weights = 'groundtruth_instance_mask_weights' groundtruth_instance_boundaries = 'groundtruth_instance_boundaries' groundtruth_instance_classes = 'groundtruth_instance_classes' groundtruth_keypoints = 'groundtruth_keypoints' diff --git a/research/object_detection/data_decoders/tf_example_decoder.py b/research/object_detection/data_decoders/tf_example_decoder.py index acd48750fd9..0a2060972dd 100644 --- a/research/object_detection/data_decoders/tf_example_decoder.py +++ b/research/object_detection/data_decoders/tf_example_decoder.py @@ -373,6 +373,11 @@ def __init__(self, self._decode_png_instance_masks)) else: raise ValueError('Did not recognize the `instance_mask_type` option.') + self.keys_to_features['image/object/mask/weight'] = ( + tf.VarLenFeature(tf.float32)) + self.items_to_handlers[ + fields.InputDataFields.groundtruth_instance_mask_weights] = ( + slim_example_decoder.Tensor('image/object/mask/weight')) if load_dense_pose: self.keys_to_features['image/object/densepose/num'] = ( tf.VarLenFeature(tf.int64)) @@ -491,6 +496,10 @@ def decode(self, tf_example_string_tensor): tensor of shape [None, num_keypoints] containing keypoint visibilites. fields.InputDataFields.groundtruth_instance_masks - 3D float32 tensor of shape [None, None, None] containing instance masks. + fields.InputDataFields.groundtruth_instance_mask_weights - 1D float32 + tensor of shape [None] containing weights. These are typically values + in {0.0, 1.0} which indicate whether to consider the mask related to an + object. fields.InputDataFields.groundtruth_image_classes - 1D int64 of shape [None] containing classes for the boxes. fields.InputDataFields.multiclass_scores - 1D float32 tensor of shape @@ -531,6 +540,21 @@ def default_groundtruth_weights(): 0), lambda: tensor_dict[fields.InputDataFields.groundtruth_weights], default_groundtruth_weights) + if fields.InputDataFields.groundtruth_instance_masks in tensor_dict: + gt_instance_masks = tensor_dict[ + fields.InputDataFields.groundtruth_instance_masks] + num_gt_instance_masks = tf.shape(gt_instance_masks)[0] + gt_instance_mask_weights = tensor_dict[ + fields.InputDataFields.groundtruth_instance_mask_weights] + num_gt_instance_mask_weights = tf.shape(gt_instance_mask_weights)[0] + def default_groundtruth_instance_mask_weights(): + return tf.ones([num_gt_instance_masks], dtype=tf.float32) + + tensor_dict[fields.InputDataFields.groundtruth_instance_mask_weights] = ( + tf.cond(tf.greater(num_gt_instance_mask_weights, 0), + lambda: gt_instance_mask_weights, + default_groundtruth_instance_mask_weights)) + if fields.InputDataFields.groundtruth_keypoints in tensor_dict: # Set all keypoints that are not labeled to NaN. gt_kpt_fld = fields.InputDataFields.groundtruth_keypoints diff --git a/research/object_detection/data_decoders/tf_example_decoder_test.py b/research/object_detection/data_decoders/tf_example_decoder_test.py index 5311bdf4dfe..f91863e165c 100644 --- a/research/object_detection/data_decoders/tf_example_decoder_test.py +++ b/research/object_detection/data_decoders/tf_example_decoder_test.py @@ -1225,6 +1225,9 @@ def graph_fn(): self.assertAllEqual( instance_masks.astype(np.float32), tensor_dict[fields.InputDataFields.groundtruth_instance_masks]) + self.assertAllEqual( + tensor_dict[fields.InputDataFields.groundtruth_instance_mask_weights], + [1, 1, 1, 1]) self.assertAllEqual(object_classes, tensor_dict[fields.InputDataFields.groundtruth_classes]) @@ -1272,6 +1275,71 @@ def graph_fn(): self.assertNotIn(fields.InputDataFields.groundtruth_instance_masks, tensor_dict) + def testDecodeInstanceSegmentationWithWeights(self): + num_instances = 4 + image_height = 5 + image_width = 3 + + # Randomly generate image. + image_tensor = np.random.randint( + 256, size=(image_height, image_width, 3)).astype(np.uint8) + encoded_jpeg, _ = self._create_encoded_and_decoded_data( + image_tensor, 'jpeg') + + # Randomly generate instance segmentation masks. + instance_masks = ( + np.random.randint(2, size=(num_instances, image_height, + image_width)).astype(np.float32)) + instance_masks_flattened = np.reshape(instance_masks, [-1]) + instance_mask_weights = np.array([1, 1, 0, 1], dtype=np.float32) + + # Randomly generate class labels for each instance. + object_classes = np.random.randint( + 100, size=(num_instances)).astype(np.int64) + + def graph_fn(): + example = tf.train.Example( + features=tf.train.Features( + feature={ + 'image/encoded': + dataset_util.bytes_feature(encoded_jpeg), + 'image/format': + dataset_util.bytes_feature(six.b('jpeg')), + 'image/height': + dataset_util.int64_feature(image_height), + 'image/width': + dataset_util.int64_feature(image_width), + 'image/object/mask': + dataset_util.float_list_feature(instance_masks_flattened), + 'image/object/mask/weight': + dataset_util.float_list_feature(instance_mask_weights), + 'image/object/class/label': + dataset_util.int64_list_feature(object_classes) + })).SerializeToString() + example_decoder = tf_example_decoder.TfExampleDecoder( + load_instance_masks=True) + output = example_decoder.decode(tf.convert_to_tensor(example)) + + self.assertAllEqual( + (output[fields.InputDataFields.groundtruth_instance_masks].get_shape( + ).as_list()), [4, 5, 3]) + self.assertAllEqual( + output[fields.InputDataFields.groundtruth_instance_mask_weights], + [1, 1, 0, 1]) + + self.assertAllEqual((output[ + fields.InputDataFields.groundtruth_classes].get_shape().as_list()), + [4]) + return output + + tensor_dict = self.execute_cpu(graph_fn, []) + + self.assertAllEqual( + instance_masks.astype(np.float32), + tensor_dict[fields.InputDataFields.groundtruth_instance_masks]) + self.assertAllEqual(object_classes, + tensor_dict[fields.InputDataFields.groundtruth_classes]) + def testDecodeImageLabels(self): image_tensor = np.random.randint(256, size=(4, 5, 3)).astype(np.uint8) encoded_jpeg, _ = self._create_encoded_and_decoded_data( diff --git a/research/object_detection/inputs.py b/research/object_detection/inputs.py index bdb219b08cc..e944a7f5e11 100644 --- a/research/object_detection/inputs.py +++ b/research/object_detection/inputs.py @@ -479,6 +479,7 @@ def pad_input_data_to_static_shapes(tensor_dict, input_fields.groundtruth_instance_masks: [ max_num_boxes, height, width ], + input_fields.groundtruth_instance_mask_weights: [max_num_boxes], input_fields.groundtruth_is_crowd: [max_num_boxes], input_fields.groundtruth_group_of: [max_num_boxes], input_fields.groundtruth_area: [max_num_boxes], @@ -601,6 +602,8 @@ def augment_input_data(tensor_dict, data_augmentation_options): include_instance_masks = (fields.InputDataFields.groundtruth_instance_masks in tensor_dict) + include_instance_mask_weights = ( + fields.InputDataFields.groundtruth_instance_mask_weights in tensor_dict) include_keypoints = (fields.InputDataFields.groundtruth_keypoints in tensor_dict) include_keypoint_visibilities = ( @@ -624,6 +627,7 @@ def augment_input_data(tensor_dict, data_augmentation_options): include_label_confidences=include_label_confidences, include_multiclass_scores=include_multiclass_scores, include_instance_masks=include_instance_masks, + include_instance_mask_weights=include_instance_mask_weights, include_keypoints=include_keypoints, include_keypoint_visibilities=include_keypoint_visibilities, include_dense_pose=include_dense_pose, @@ -652,6 +656,7 @@ def _get_labels_dict(input_dict): fields.InputDataFields.groundtruth_keypoint_depths, fields.InputDataFields.groundtruth_keypoint_depth_weights, fields.InputDataFields.groundtruth_instance_masks, + fields.InputDataFields.groundtruth_instance_mask_weights, fields.InputDataFields.groundtruth_area, fields.InputDataFields.groundtruth_is_crowd, fields.InputDataFields.groundtruth_group_of, @@ -804,6 +809,9 @@ def train_input(train_config, train_input_config, labels[fields.InputDataFields.groundtruth_instance_masks] is a [batch_size, num_boxes, H, W] float32 tensor containing only binary values, which represent instance masks for objects. + labels[fields.InputDataFields.groundtruth_instance_mask_weights] is a + [batch_size, num_boxes] float32 tensor containing groundtruth weights + for each instance mask. labels[fields.InputDataFields.groundtruth_keypoints] is a [batch_size, num_boxes, num_keypoints, 2] float32 tensor containing keypoints for each box. @@ -961,6 +969,9 @@ def eval_input(eval_config, eval_input_config, model_config, labels[fields.InputDataFields.groundtruth_instance_masks] is a [1, num_boxes, H, W] float32 tensor containing only binary values, which represent instance masks for objects. + labels[fields.InputDataFields.groundtruth_instance_mask_weights] is a + [1, num_boxes] float32 tensor containing groundtruth weights for each + instance mask. labels[fields.InputDataFields.groundtruth_weights] is a [batch_size, num_boxes, num_keypoints] float32 tensor containing groundtruth weights for the keypoints. diff --git a/research/object_detection/inputs_test.py b/research/object_detection/inputs_test.py index 4716882e9a3..ea69717a478 100644 --- a/research/object_detection/inputs_test.py +++ b/research/object_detection/inputs_test.py @@ -795,15 +795,20 @@ def graph_fn(): fields.InputDataFields.image: tf.constant(np.random.rand(10, 10, 3).astype(np.float32)), fields.InputDataFields.groundtruth_instance_masks: - tf.constant(np.zeros([2, 10, 10], np.uint8)) + tf.constant(np.zeros([2, 10, 10], np.uint8)), + fields.InputDataFields.groundtruth_instance_mask_weights: + tf.constant([1.0, 0.0], np.float32) } augmented_tensor_dict = data_augmentation_fn(tensor_dict=tensor_dict) return (augmented_tensor_dict[fields.InputDataFields.image], augmented_tensor_dict[fields.InputDataFields. - groundtruth_instance_masks]) - image, masks = self.execute_cpu(graph_fn, []) + groundtruth_instance_masks], + augmented_tensor_dict[fields.InputDataFields. + groundtruth_instance_mask_weights]) + image, masks, mask_weights = self.execute_cpu(graph_fn, []) self.assertAllEqual(image.shape, [20, 20, 3]) self.assertAllEqual(masks.shape, [2, 20, 20]) + self.assertAllClose(mask_weights, [1.0, 0.0]) def test_include_keypoints_in_data_augmentation(self): data_augmentation_options = [ From 677aaa11b11f41e65c750815cbb06054e73aa5f1 Mon Sep 17 00:00:00 2001 From: Austin Myers Date: Wed, 2 Jun 2021 11:23:01 -0700 Subject: [PATCH 12/50] Enables combined static and dynamic shape for the batch size in 'MultiLevelRoIAlign' PiperOrigin-RevId: 377103499 --- research/object_detection/utils/spatial_transform_ops.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/research/object_detection/utils/spatial_transform_ops.py b/research/object_detection/utils/spatial_transform_ops.py index 1880dffea1a..26122dbccb1 100644 --- a/research/object_detection/utils/spatial_transform_ops.py +++ b/research/object_detection/utils/spatial_transform_ops.py @@ -19,6 +19,7 @@ from __future__ import print_function import tensorflow.compat.v1 as tf +from object_detection.utils import shape_utils def _coordinate_vector_1d(start, end, size, align_endpoints): @@ -322,7 +323,7 @@ def multilevel_roi_align(features, boxes, box_levels, output_size, """ with tf.name_scope(scope, 'MultiLevelRoIAlign'): features, true_feature_shapes = pad_to_max_size(features) - batch_size = tf.shape(features)[0] + batch_size = shape_utils.combined_static_and_dynamic_shape(features)[0] num_levels = features.get_shape().as_list()[1] max_feature_height = tf.shape(features)[2] max_feature_width = tf.shape(features)[3] From 6bccc202a40fb06bdef92d818b3886c3bde1bc04 Mon Sep 17 00:00:00 2001 From: Ronny Votel Date: Wed, 2 Jun 2021 11:28:12 -0700 Subject: [PATCH 13/50] Plumbing groundtruth instance mask weights through the model codebase. PiperOrigin-RevId: 377104676 --- research/object_detection/core/model.py | 10 ++++++++-- research/object_detection/core/standard_fields.py | 2 ++ research/object_detection/model_lib.py | 6 ++++++ research/object_detection/model_lib_v2.py | 5 +++++ 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/research/object_detection/core/model.py b/research/object_detection/core/model.py index bb96038dabf..46bcb82e670 100644 --- a/research/object_detection/core/model.py +++ b/research/object_detection/core/model.py @@ -101,7 +101,7 @@ def groundtruth_lists(self, field): Args: field: a string key, options are - fields.BoxListFields.{boxes,classes,masks,keypoints, + fields.BoxListFields.{boxes,classes,masks,mask_weights,keypoints, keypoint_visibilities, densepose_*, track_ids, temporal_offsets, track_match_flags} fields.InputDataFields.is_annotated. @@ -123,7 +123,7 @@ def groundtruth_has_field(self, field): Args: field: a string key, options are - fields.BoxListFields.{boxes,classes,masks,keypoints, + fields.BoxListFields.{boxes,classes,masks,mask_weights,keypoints, keypoint_visibilities, densepose_*, track_ids} or fields.InputDataFields.is_annotated. @@ -299,6 +299,7 @@ def provide_groundtruth( groundtruth_boxes_list, groundtruth_classes_list, groundtruth_masks_list=None, + groundtruth_mask_weights_list=None, groundtruth_keypoints_list=None, groundtruth_keypoint_visibilities_list=None, groundtruth_dp_num_points_list=None, @@ -334,6 +335,8 @@ def provide_groundtruth( masks with values in {0, 1}. If None, no masks are provided. Mask resolution `height_in`x`width_in` must agree with the resolution of the input image tensor provided to the `preprocess` function. + groundtruth_mask_weights_list: a list of 1-D tf.float32 tensors of shape + [num_boxes] with weights for each instance mask. groundtruth_keypoints_list: a list of 3-D tf.float32 tensors of shape [num_boxes, num_keypoints, 2] containing keypoints. Keypoints are assumed to be provided in normalized coordinates and @@ -399,6 +402,9 @@ def provide_groundtruth( if groundtruth_masks_list: self._groundtruth_lists[ fields.BoxListFields.masks] = groundtruth_masks_list + if groundtruth_mask_weights_list: + self._groundtruth_lists[ + fields.BoxListFields.mask_weights] = groundtruth_mask_weights_list if groundtruth_keypoints_list: self._groundtruth_lists[ fields.BoxListFields.keypoints] = groundtruth_keypoints_list diff --git a/research/object_detection/core/standard_fields.py b/research/object_detection/core/standard_fields.py index 2cb895cb822..2267dff52f8 100644 --- a/research/object_detection/core/standard_fields.py +++ b/research/object_detection/core/standard_fields.py @@ -210,6 +210,7 @@ class BoxListFields(object): weights: sample weights per bounding box. objectness: objectness score per bounding box. masks: masks per bounding box. + mask_weights: mask weights for each bounding box. boundaries: boundaries per bounding box. keypoints: keypoints per bounding box. keypoint_visibilities: keypoint visibilities per bounding box. @@ -230,6 +231,7 @@ class BoxListFields(object): confidences = 'confidences' objectness = 'objectness' masks = 'masks' + mask_weights = 'mask_weights' boundaries = 'boundaries' keypoints = 'keypoints' keypoint_visibilities = 'keypoint_visibilities' diff --git a/research/object_detection/model_lib.py b/research/object_detection/model_lib.py index 111be9cb4a7..1a92c469f4c 100644 --- a/research/object_detection/model_lib.py +++ b/research/object_detection/model_lib.py @@ -266,6 +266,7 @@ def unstack_batch(tensor_dict, unpad_groundtruth_tensors=True): # dimension. This list has to be kept in sync with InputDataFields in # standard_fields.py. fields.InputDataFields.groundtruth_instance_masks, + fields.InputDataFields.groundtruth_instance_mask_weights, fields.InputDataFields.groundtruth_classes, fields.InputDataFields.groundtruth_boxes, fields.InputDataFields.groundtruth_keypoints, @@ -319,6 +320,10 @@ def provide_groundtruth(model, labels): if fields.InputDataFields.groundtruth_instance_masks in labels: gt_masks_list = labels[ fields.InputDataFields.groundtruth_instance_masks] + gt_mask_weights_list = None + if fields.InputDataFields.groundtruth_instance_mask_weights in labels: + gt_mask_weights_list = labels[ + fields.InputDataFields.groundtruth_instance_mask_weights] gt_keypoints_list = None if fields.InputDataFields.groundtruth_keypoints in labels: gt_keypoints_list = labels[fields.InputDataFields.groundtruth_keypoints] @@ -383,6 +388,7 @@ def provide_groundtruth(model, labels): groundtruth_confidences_list=gt_confidences_list, groundtruth_labeled_classes=gt_labeled_classes, groundtruth_masks_list=gt_masks_list, + groundtruth_mask_weights_list=gt_mask_weights_list, groundtruth_keypoints_list=gt_keypoints_list, groundtruth_keypoint_visibilities_list=gt_keypoint_visibilities_list, groundtruth_dp_num_points_list=gt_dp_num_points_list, diff --git a/research/object_detection/model_lib_v2.py b/research/object_detection/model_lib_v2.py index 45d600da779..81b75366634 100644 --- a/research/object_detection/model_lib_v2.py +++ b/research/object_detection/model_lib_v2.py @@ -87,6 +87,8 @@ def _compute_losses_and_predictions_dicts( labels[fields.InputDataFields.groundtruth_instance_masks] is a float32 tensor containing only binary values, which represent instance masks for objects. + labels[fields.InputDataFields.groundtruth_instance_mask_weights] is a + float32 tensor containing weights for the instance masks. labels[fields.InputDataFields.groundtruth_keypoints] is a float32 tensor containing keypoints for each box. labels[fields.InputDataFields.groundtruth_dp_num_points] is an int32 @@ -237,6 +239,9 @@ def eager_train_step(detection_model, labels[fields.InputDataFields.groundtruth_instance_masks] is a [batch_size, num_boxes, H, W] float32 tensor containing only binary values, which represent instance masks for objects. + labels[fields.InputDataFields.groundtruth_instance_mask_weights] is a + [batch_size, num_boxes] float32 tensor containing weights for the + instance masks. labels[fields.InputDataFields.groundtruth_keypoints] is a [batch_size, num_boxes, num_keypoints, 2] float32 tensor containing keypoints for each box. From 5247a17e446e4883979da58f0d58438986ffaab3 Mon Sep 17 00:00:00 2001 From: Dan Holtmann-Rice Date: Wed, 2 Jun 2021 13:26:16 -0700 Subject: [PATCH 14/50] Internal change PiperOrigin-RevId: 377131555 --- orbit/actions/__init__.py | 74 ++++++ orbit/actions/conditional_action.py | 60 +++++ orbit/actions/conditional_action_test.py | 39 ++++ orbit/actions/export_saved_model.py | 135 +++++++++++ .../export_saved_model_test.py} | 82 +------ .../new_best_metric.py} | 215 +----------------- orbit/actions/new_best_metric_test.py | 94 ++++++++ 7 files changed, 408 insertions(+), 291 deletions(-) create mode 100644 orbit/actions/__init__.py create mode 100644 orbit/actions/conditional_action.py create mode 100644 orbit/actions/conditional_action_test.py create mode 100644 orbit/actions/export_saved_model.py rename orbit/{actions_test.py => actions/export_saved_model_test.py} (58%) rename orbit/{actions.py => actions/new_best_metric.py} (51%) create mode 100644 orbit/actions/new_best_metric_test.py diff --git a/orbit/actions/__init__.py b/orbit/actions/__init__.py new file mode 100644 index 00000000000..5c3eab2d8b0 --- /dev/null +++ b/orbit/actions/__init__.py @@ -0,0 +1,74 @@ +# Copyright 2021 The Orbit Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Defines an "action" abstraction for use with `orbit.Controller`. + +"Actions" are simply arbitrary callables that are applied by the `Controller` +to the output of train steps (after each inner loop of `steps_per_loop` steps) +or an evaluation. This provides a hook mechanism, enabling things like reporting +metrics to Vizier, model exporting, additional logging, etc. + +The basic `Action` abstraction (just a type alias) is defined in the +`controller` module. This `actions` module adds a `ConditionalAction` utility +class to make it easy to trigger actions conditionally based on reusable +predicates, as well as a small handful of predefined conditions/actions (in +particular, a `NewBestMetric` condition and an `ExportSavedModel` action). + +One example of using actions to do metric-conditional export: + + new_best_metric = orbit.actions.NewBestMetric('accuracy') + export_action = orbit.actions.ConditionalAction( + condition=lambda x: x['accuracy'] > 0.9 and new_best_metric(x), + action=orbit.actions.ExportSavedModel( + model, + orbit.actions.ExportFileManager( + base_name=f'{FLAGS.model_dir}/saved_model', + next_id_fn=trainer.global_step.numpy), + signatures=model.infer)) + + controller = orbit.Controller( + strategy=strategy, + trainer=trainer, + evaluator=evaluator, + eval_actions=[export_action], + global_step=trainer.global_step, + steps_per_loop=FLAGS.steps_per_loop, + checkpoint_manager=checkpoint_manager, + summary_interval=1000) + +Note: In multi-client settings where each client runs its own `Controller` +instance, some care should be taken in deciding which clients should run certain +actions. Isolating actions to an individual client (say client 0) can be +achieved using `ConditionalAction` as follows: + + client_0_actions = orbit.actions.ConditionalAction( + condition=lambda _: client_id() == 0, + action=[ + ... + ]) + +In particular, the `NewBestMetric` condition may be used in multi-client +settings if all clients are guaranteed to compute the same metric (ensuring this +is up to client code, not Orbit). However, when saving metrics it may be helpful +to avoid unnecessary writes by setting the `write_value` parameter to `False` +for most clients. +""" + +from orbit.actions.conditional_action import ConditionalAction + +from orbit.actions.export_saved_model import ExportFileManager +from orbit.actions.export_saved_model import ExportSavedModel + +from orbit.actions.new_best_metric import JSONPersistedValue +from orbit.actions.new_best_metric import NewBestMetric diff --git a/orbit/actions/conditional_action.py b/orbit/actions/conditional_action.py new file mode 100644 index 00000000000..e4b8122270f --- /dev/null +++ b/orbit/actions/conditional_action.py @@ -0,0 +1,60 @@ +# Copyright 2021 The Orbit Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provides a `ConditionalAction` abstraction.""" + +from typing import Any, Callable, Sequence, Union + +from orbit import controller +from orbit import runner + +import tensorflow as tf + +Condition = Callable[[runner.Output], Union[bool, tf.Tensor]] + + +def _as_sequence(maybe_sequence: Union[Any, Sequence[Any]]) -> Sequence[Any]: + if isinstance(maybe_sequence, Sequence): + return maybe_sequence + return [maybe_sequence] + + +class ConditionalAction: + """Represents an action that is only taken when a given condition is met. + + This class is itself an `Action` (a callable that can be applied to train or + eval outputs), but is intended to make it easier to write modular and reusable + conditions by decoupling "when" something whappens (the condition) from "what" + happens (the action). + """ + + def __init__( + self, + condition: Condition, + action: Union[controller.Action, Sequence[controller.Action]], + ): + """Initializes the instance. + + Args: + condition: A callable accepting train or eval outputs and returing a bool. + action: The action (or optionally sequence of actions) to perform when + `condition` is met. + """ + self.condition = condition + self.action = action + + def __call__(self, output: runner.Output) -> None: + if self.condition(output): + for action in _as_sequence(self.action): + action(output) diff --git a/orbit/actions/conditional_action_test.py b/orbit/actions/conditional_action_test.py new file mode 100644 index 00000000000..cfcfd0f541b --- /dev/null +++ b/orbit/actions/conditional_action_test.py @@ -0,0 +1,39 @@ +# Copyright 2021 The Orbit Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for orbit.actions.conditional_action.""" + +from orbit import actions + +import tensorflow as tf + + +class ConditionalActionTest(tf.test.TestCase): + + def test_conditional_action(self): + # Define a function to raise an AssertionError, since we can't in a lambda. + def raise_assertion(arg): + raise AssertionError(str(arg)) + + conditional_action = actions.ConditionalAction( + condition=lambda x: x['value'], action=raise_assertion) + + conditional_action({'value': False}) # Nothing is raised. + with self.assertRaises(AssertionError) as ctx: + conditional_action({'value': True}) + self.assertEqual(ctx.exception.message, "{'value': True}") + + +if __name__ == '__main__': + tf.test.main() diff --git a/orbit/actions/export_saved_model.py b/orbit/actions/export_saved_model.py new file mode 100644 index 00000000000..dd6d74fb8b2 --- /dev/null +++ b/orbit/actions/export_saved_model.py @@ -0,0 +1,135 @@ +# Copyright 2021 The Orbit Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provides the `ExportSavedModel` action and associated helper classes.""" + +from typing import Callable, Optional + +import tensorflow as tf + + +class _CounterIdFn: + """Implements a counter-based ID function for `ExportFileManager`.""" + + def __init__(self, base_name: str): + filenames = tf.io.gfile.glob(f'{base_name}-*') + max_counter = -1 + for filename in filenames: + try: + _, file_number = filename.rsplit('-', maxsplit=1) + max_counter = max(max_counter, int(file_number)) + except ValueError: + continue + self.value = max_counter + 1 + + def __call__(self): + output = self.value + self.value += 1 + return output + + +class ExportFileManager: + """Utility class that manages a group of files with a shared base name. + + For actions like SavedModel exporting, there are potentially many different + file naming and cleanup strategies that may be desirable. This class provides + a basic interface allowing SavedModel export to be decoupled from these + details, and a default implementation that should work for many basic + scenarios. Users may subclass this class to alter behavior and define more + customized naming and cleanup strategies. + """ + + def __init__(self, + base_name: str, + max_to_keep: int = 5, + next_id_fn: Optional[Callable[[], int]] = None): + """Initializes the instance. + + Args: + base_name: A shared base name for file names generated by this class. + max_to_keep: The maximum number of files matching `base_name` to keep + after each call to `cleanup`. The most recent (as determined by file + modification time) `max_to_keep` files are preserved; the rest are + deleted. If < 0, all files are preserved. + next_id_fn: An optional callable that returns integer IDs to append to + base name (formatted as `'{base_name}-{id}'`). The order of integers is + used to sort files to determine the oldest ones deleted by `clean_up`. + If not supplied, a default ID based on an incrementing counter is used. + One common alternative maybe be to use the current global step count, + for instance passing `next_id_fn=global_step.numpy`. + """ + self._base_name = base_name + self._max_to_keep = max_to_keep + self._next_id_fn = next_id_fn or _CounterIdFn(base_name) + + @property + def managed_files(self): + """Returns all files managed by this instance, in sorted order. + + Returns: + The list of files matching the `base_name` provided when constructing this + `ExportFileManager` instance, sorted in increasing integer order of the + IDs returned by `next_id_fn`. + """ + + def id_key(name): + _, id_num = name.rsplit('-', maxsplit=1) + return int(id_num) + + filenames = tf.io.gfile.glob(f'{self._base_name}-*') + return sorted(filenames, key=id_key) + + def clean_up(self): + """Cleans up old files matching `{base_name}-*`. + + The most recent `max_to_keep` files are preserved. + """ + if self._max_to_keep < 0: + return + + for filename in self.managed_files[:-self._max_to_keep]: + tf.io.gfile.rmtree(filename) + + def next_name(self) -> str: + """Returns a new file name based on `base_name` and `next_id_fn()`.""" + return f'{self._base_name}-{self._next_id_fn()}' + + +class ExportSavedModel: + """Action that exports the given model as a SavedModel.""" + + def __init__(self, + model: tf.Module, + file_manager: ExportFileManager, + signatures, + options: Optional[tf.saved_model.SaveOptions] = None): + """Initializes the instance. + + Args: + model: The model to export. + file_manager: An instance of `ExportFileManager` (or a subclass), that + provides file naming and cleanup functionality. + signatures: The signatures to forward to `tf.saved_model.save()`. + options: Optional options to forward to `tf.saved_model.save()`. + """ + self.model = model + self.file_manager = file_manager + self.signatures = signatures + self.options = options + + def __call__(self, _): + """Exports the SavedModel.""" + export_dir = self.file_manager.next_name() + tf.saved_model.save(self.model, export_dir, self.signatures, self.options) + self.file_manager.clean_up() diff --git a/orbit/actions_test.py b/orbit/actions/export_saved_model_test.py similarity index 58% rename from orbit/actions_test.py rename to orbit/actions/export_saved_model_test.py index 5d15724431a..7ac3f611259 100644 --- a/orbit/actions_test.py +++ b/orbit/actions/export_saved_model_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for orbit.actions.""" +"""Tests for orbit.actions.export_saved_model.""" import os @@ -40,85 +40,7 @@ def __call__(self): return self.value -class ActionsTest(tf.test.TestCase): - - def test_conditional_action(self): - # Define a function to raise an AssertionError, since we can't in a lambda. - def raise_assertion(arg): - raise AssertionError(str(arg)) - - conditional_action = actions.ConditionalAction( - condition=lambda x: x, action=raise_assertion) - - conditional_action(False) # Nothing is raised. - with self.assertRaises(AssertionError) as ctx: - conditional_action(True) - self.assertEqual(ctx.exception.message, 'True') - - def test_new_best_metric_higher_is_better(self): - new_best_metric = actions.NewBestMetric(lambda x: x, higher_is_better=True) - self.assertTrue(new_best_metric.test(0.0)) - self.assertTrue(new_best_metric.commit(0.0)) - self.assertFalse(new_best_metric.test(0.0)) - self.assertTrue(new_best_metric.test(1.0)) - - def test_new_best_metric_lower_is_better(self): - new_best_metric = actions.NewBestMetric(lambda x: x, higher_is_better=False) - self.assertTrue(new_best_metric.test(0.0)) - self.assertTrue(new_best_metric.commit(0.0)) - self.assertFalse(new_best_metric.test(0.0)) - self.assertTrue(new_best_metric.test(-1.0)) - - def test_new_best_metric_persistence(self): - backing_file = self.create_tempfile() - new_best_metric = actions.NewBestMetric( - lambda x: x, - higher_is_better=True, - filename=backing_file.full_path, - write_metric=False) - self.assertTrue(new_best_metric.test(0.0)) - self.assertTrue(new_best_metric.commit(0.0)) - self.assertFalse(new_best_metric.test(0.0)) - new_best_metric = actions.NewBestMetric( - lambda x: x, higher_is_better=True, filename=backing_file.full_path) - self.assertLess(new_best_metric.best_value, 0.0) - self.assertTrue(new_best_metric.commit(5.0)) - self.assertEqual(new_best_metric.best_value, 5.0) - new_best_metric = actions.NewBestMetric( - lambda x: x, higher_is_better=True, filename=backing_file.full_path) - self.assertEqual(new_best_metric.best_value, 5.0) - - def test_json_persisted_value(self): - tempfile = self.create_tempfile().full_path - value = {'a': 1, 'b': 2} - persisted_value = actions.JSONPersistedValue(value, tempfile) - # The inital value is used since tempfile is empty. - self.assertEqual(persisted_value.read(), value) - persisted_value = actions.JSONPersistedValue('ignored', tempfile) - # Initial value of 'ignored' is ignored, since there's a value in tempfile. - self.assertEqual(persisted_value.read(), value) - value = [1, 2, 3] - persisted_value.write(value) - # Now that a new value is written, it gets read on initialization. - persisted_value = actions.JSONPersistedValue(['also ignored'], tempfile) - self.assertEqual(persisted_value.read(), value) - # Writes can be disabled. - persisted_value = actions.JSONPersistedValue( - 'ignored', tempfile, write_value=False) - self.assertEqual(persisted_value.read(), value) - persisted_value.write("won't get persisted") - persisted_value = actions.JSONPersistedValue( - 'ignored', tempfile, write_value=False) - self.assertEqual(persisted_value.read(), value) - - def test_json_persisted_value_create_dirs(self): - tempfile = os.path.join(self.create_tempdir().full_path, 'subdir/value') - value = {'a': 1, 'b': 2} - # The directory is not created if write_value=False. - actions.JSONPersistedValue(value, tempfile, write_value=False) - self.assertFalse(tf.io.gfile.exists(os.path.dirname(tempfile))) - actions.JSONPersistedValue(value, tempfile) - self.assertTrue(tf.io.gfile.exists(tempfile)) +class ExportSavedModelTest(tf.test.TestCase): def test_export_file_manager_default_ids(self): directory = self.create_tempdir() diff --git a/orbit/actions.py b/orbit/actions/new_best_metric.py similarity index 51% rename from orbit/actions.py rename to orbit/actions/new_best_metric.py index d1258134ec9..f2a01c80f55 100644 --- a/orbit/actions.py +++ b/orbit/actions/new_best_metric.py @@ -12,110 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Defines an "action" abstraction for use with `orbit.Controller`. - -"Actions" are simply arbitrary callables that are applied by the `Controller` -to the output of train steps (after each inner loop of `steps_per_loop` steps) -or an evaluation. This provides a hook mechanism, enabling things like reporting -metrics to Vizier, model exporting, additional logging, etc. - -The basic `Action` abstraction (just a type alias) is defined in the -`controller` module. This `actions` module adds a `ConditionalAction` utility -class to make it easy to trigger actions conditionally based on reusable -predicates, as well as a small handful of predefined conditions/actions (in -particular, a `NewBestMetric` condition and an `ExportSavedModel` action). - -One example of using actions to do metric-conditional export: - - new_best_metric = orbit.actions.NewBestMetric('accuracy') - export_action = orbit.actions.ConditionalAction( - condition=lambda x: x['accuracy'] > 0.9 and new_best_metric(x), - action=orbit.actions.ExportSavedModel( - model, - orbit.actions.ExportFileManager( - base_name=f'{FLAGS.model_dir}/saved_model', - next_id_fn=trainer.global_step.numpy), - signatures=model.infer)) - - controller = orbit.Controller( - strategy=strategy, - trainer=trainer, - evaluator=evaluator, - eval_actions=[export_action], - global_step=trainer.global_step, - steps_per_loop=FLAGS.steps_per_loop, - checkpoint_manager=checkpoint_manager, - summary_interval=1000) - -Note: In multi-client settings where each client runs its own `Controller` -instance, some care should be taken in deciding which clients should run certain -actions. Isolating actions to an individual client (say client 0) can be -achieved using `ConditionalAction` as follows: - - client_0_actions = orbit.actions.ConditionalAction( - condition=lambda _: client_id() == 0, - action=[ - ... - ]) - -In particular, the `NewBestMetric` condition may be used in multi-client -settings if all clients are guaranteed to compute the same metric (ensuring this -is up to client code, not Orbit). However, when saving metrics it may be helpful -to avoid unnecessary writes by setting the `write_value` parameter to `False` -for most clients. -""" +"""Provides the `NewBestMetric` condition and associated helper classes.""" import json import os import sys -from typing import Any, Callable, Optional, Sequence, Union +from typing import Any, Callable, Optional, Union import uuid -from orbit import controller from orbit import runner from orbit import utils import tensorflow as tf -Condition = Callable[[runner.Output], Union[bool, tf.Tensor]] - - -def _as_sequence(maybe_sequence: Union[Any, Sequence[Any]]) -> Sequence[Any]: - if isinstance(maybe_sequence, Sequence): - return maybe_sequence - return [maybe_sequence] - - -class ConditionalAction: - """Represents an action that is only taken when a given condition is met. - - This class is itself an `Action` (a callable that can be applied to train or - eval outputs), but is intended to make it easier to write modular and reusable - conditions by decoupling "when" something whappens (the condition) from "what" - happens (the action). - """ - - def __init__( - self, - condition: Condition, - action: Union[controller.Action, Sequence[controller.Action]], - ): - """Initializes the instance. - - Args: - condition: A callable accepting train or eval outputs and returing a bool. - action: The action (or optionally sequence of actions) to perform when - `condition` is met. - """ - self.condition = condition - self.action = action - - def __call__(self, output: runner.Output) -> None: - if self.condition(output): - for action in _as_sequence(self.action): - action(output) - - MetricFn = Callable[[runner.Output], Union[float, tf.Tensor]] @@ -151,7 +60,7 @@ class NewBestMetric: Attributes: metric: The metric passed to __init__ (may be a string key or a callable - that can be applied to train/eval output). + that can be applied to train/eval output). higher_is_better: Whether higher metric values are better. """ @@ -290,7 +199,7 @@ def __init__(self, if tf.io.gfile.exists(self._filename): if tf.io.gfile.stat(self._filename).length > 0: with tf.io.gfile.GFile(self._filename, 'r') as f: - self._value = json.loads(f.read()) + self._value = json.load(f) elif self._write_value: tf.io.gfile.makedirs(os.path.dirname(self._filename)) @@ -311,119 +220,3 @@ def write(self, value): with tf.io.gfile.GFile(tmp_filename, 'w') as f: json.dump(self._value, f) tf.io.gfile.rename(tmp_filename, self._filename, overwrite=True) - - -class _CounterIdFn: - """Implements a counter-based ID function for `ExportFileManager`.""" - - def __init__(self, base_name: str): - filenames = tf.io.gfile.glob(f'{base_name}-*') - max_counter = -1 - for filename in filenames: - try: - _, file_number = filename.rsplit('-', maxsplit=1) - max_counter = max(max_counter, int(file_number)) - except ValueError: - continue - self.value = max_counter + 1 - - def __call__(self): - output = self.value - self.value += 1 - return output - - -class ExportFileManager: - """Utility class that manages a group of files with a shared base name. - - For actions like SavedModel exporting, there are potentially many different - file naming and cleanup strategies that may be desirable. This class provides - a basic interface allowing SavedModel export to be decoupled from these - details, and a default implementation that should work for many basic - scenarios. Users may subclass this class to alter behavior and define more - customized naming and cleanup strategies. - """ - - def __init__(self, - base_name: str, - max_to_keep: int = 5, - next_id_fn: Optional[Callable[[], int]] = None): - """Initializes the instance. - - Args: - base_name: A shared base name for file names generated by this class. - max_to_keep: The maximum number of files matching `base_name` to keep - after each call to `cleanup`. The most recent (as determined by file - modification time) `max_to_keep` files are preserved; the rest are - deleted. If < 0, all files are preserved. - next_id_fn: An optional callable that returns integer IDs to append to - base name (formatted as `'{base_name}-{id}'`). The order of integers is - used to sort files to determine the oldest ones deleted by `clean_up`. - If not supplied, a default ID based on an incrementing counter is used. - One common alternative maybe be to use the current global step count, - for instance passing `next_id_fn=global_step.numpy`. - """ - self._base_name = base_name - self._max_to_keep = max_to_keep - self._next_id_fn = next_id_fn or _CounterIdFn(base_name) - - @property - def managed_files(self): - """Returns all files managed by this instance, in sorted order. - - Returns: - The list of files matching the `base_name` provided when constructing this - `ExportFileManager` instance, sorted in increasing integer order of the - IDs returned by `next_id_fn`. - """ - - def id_key(name): - _, id_num = name.rsplit('-', maxsplit=1) - return int(id_num) - - filenames = tf.io.gfile.glob(f'{self._base_name}-*') - return sorted(filenames, key=id_key) - - def clean_up(self): - """Cleans up old files matching `{base_name}-*`. - - The most recent `max_to_keep` files are preserved. - """ - if self._max_to_keep < 0: - return - - for filename in self.managed_files[:-self._max_to_keep]: - tf.io.gfile.rmtree(filename) - - def next_name(self) -> str: - """Returns a new file name based on `base_name` and `next_id_fn()`.""" - return f'{self._base_name}-{self._next_id_fn()}' - - -class ExportSavedModel: - """Action that exports the given model as a SavedModel.""" - - def __init__(self, - model: tf.Module, - file_manager: ExportFileManager, - signatures, - options: Optional[tf.saved_model.SaveOptions] = None): - """Initializes the instance. - - Args: - model: The model to export. - file_manager: An instance of `ExportFileManager` (or a subclass), that - provides file naming and cleanup functionality. - signatures: The signatures to forward to `tf.saved_model.save()`. - options: Optional options to forward to `tf.saved_model.save()`. - """ - self.model = model - self.file_manager = file_manager - self.signatures = signatures - self.options = options - - def __call__(self, _): - """Exports the SavedModel.""" - export_dir = self.file_manager.next_name() - tf.saved_model.save(self.model, export_dir, self.signatures, self.options) - self.file_manager.clean_up() diff --git a/orbit/actions/new_best_metric_test.py b/orbit/actions/new_best_metric_test.py new file mode 100644 index 00000000000..aff21fda2c7 --- /dev/null +++ b/orbit/actions/new_best_metric_test.py @@ -0,0 +1,94 @@ +# Copyright 2021 The Orbit Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for orbit.actions.new_best_metric.""" + +import os + +from orbit import actions + +import tensorflow as tf + + +class NewBestMetricTest(tf.test.TestCase): + + def test_new_best_metric_higher_is_better(self): + new_best_metric = actions.NewBestMetric( + lambda x: x['value'], higher_is_better=True) + self.assertTrue(new_best_metric.test({'value': 0.0})) + self.assertTrue(new_best_metric.commit({'value': 0.0})) + self.assertFalse(new_best_metric.test({'value': 0.0})) + self.assertTrue(new_best_metric.test({'value': 1.0})) + + def test_new_best_metric_lower_is_better(self): + new_best_metric = actions.NewBestMetric('value', higher_is_better=False) + self.assertTrue(new_best_metric.test({'value': 0.0})) + self.assertTrue(new_best_metric.commit({'value': 0.0})) + self.assertFalse(new_best_metric.test({'value': 0.0})) + self.assertTrue(new_best_metric.test({'value': -1.0})) + + def test_new_best_metric_persistence(self): + backing_file = self.create_tempfile() + new_best_metric = actions.NewBestMetric( + 'value', + higher_is_better=True, + filename=backing_file.full_path, + write_metric=False) + self.assertTrue(new_best_metric.test({'value': 0.0})) + self.assertTrue(new_best_metric.commit({'value': 0.0})) + self.assertFalse(new_best_metric.test({'value': 0.0})) + new_best_metric = actions.NewBestMetric( + 'value', higher_is_better=True, filename=backing_file.full_path) + self.assertLess(new_best_metric.best_value, 0.0) + self.assertTrue(new_best_metric.commit({'value': 5.0})) + self.assertEqual(new_best_metric.best_value, 5.0) + new_best_metric = actions.NewBestMetric( + 'value', higher_is_better=True, filename=backing_file.full_path) + self.assertEqual(new_best_metric.best_value, 5.0) + + def test_json_persisted_value(self): + tempfile = self.create_tempfile().full_path + value = {'a': 1, 'b': 2} + persisted_value = actions.JSONPersistedValue(value, tempfile) + # The inital value is used since tempfile is empty. + self.assertEqual(persisted_value.read(), value) + persisted_value = actions.JSONPersistedValue('ignored', tempfile) + # Initial value of 'ignored' is ignored, since there's a value in tempfile. + self.assertEqual(persisted_value.read(), value) + value = [1, 2, 3] + persisted_value.write(value) + # Now that a new value is written, it gets read on initialization. + persisted_value = actions.JSONPersistedValue(['also ignored'], tempfile) + self.assertEqual(persisted_value.read(), value) + # Writes can be disabled. + persisted_value = actions.JSONPersistedValue( + 'ignored', tempfile, write_value=False) + self.assertEqual(persisted_value.read(), value) + persisted_value.write("won't get persisted") + persisted_value = actions.JSONPersistedValue( + 'ignored', tempfile, write_value=False) + self.assertEqual(persisted_value.read(), value) + + def test_json_persisted_value_create_dirs(self): + tempfile = os.path.join(self.create_tempdir().full_path, 'subdir/value') + value = {'a': 1, 'b': 2} + # The directory is not created if write_value=False. + actions.JSONPersistedValue(value, tempfile, write_value=False) + self.assertFalse(tf.io.gfile.exists(os.path.dirname(tempfile))) + actions.JSONPersistedValue(value, tempfile) + self.assertTrue(tf.io.gfile.exists(tempfile)) + + +if __name__ == '__main__': + tf.test.main() From e8b0e95055398d2f1401216303c3323255c34078 Mon Sep 17 00:00:00 2001 From: Vighnesh Birodkar Date: Wed, 2 Jun 2021 18:02:20 -0700 Subject: [PATCH 15/50] Log loss tensor summaries after they are reduced on the coordinator. PiperOrigin-RevId: 377187393 --- research/object_detection/model_lib_v2.py | 112 +++++++++++--------- research/object_detection/model_main_tf2.py | 6 +- 2 files changed, 66 insertions(+), 52 deletions(-) diff --git a/research/object_detection/model_lib_v2.py b/research/object_detection/model_lib_v2.py index 81b75366634..012cae74e36 100644 --- a/research/object_detection/model_lib_v2.py +++ b/research/object_detection/model_lib_v2.py @@ -20,11 +20,11 @@ import copy import os +import pprint import time -import numpy as np +import numpy as np import tensorflow.compat.v1 as tf -import tensorflow.compat.v2 as tf2 from object_detection import eval_util from object_detection import inputs @@ -183,6 +183,22 @@ def _dummy_computation_fn(features, labels): )) +def normalize_dict(values_dict, num_replicas): + + num_replicas = tf.constant(num_replicas, dtype=tf.float32) + return {key: tf.math.divide(loss, num_replicas) for key, loss + in values_dict.items()} + + +def reduce_dict(strategy, reduction_dict, reduction_op): + # TODO(anjalisridhar): explore if it is safe to remove the # num_replicas + # scaling of the loss and switch this to a ReduceOp.Mean + return { + name: strategy.reduce(reduction_op, loss, axis=None) + for name, loss in reduction_dict.items() + } + + # TODO(kaftan): Explore removing learning_rate from this method & returning ## The full losses dict instead of just total_loss, then doing all summaries ## saving in a utility method called by the outer training loop. @@ -192,10 +208,8 @@ def eager_train_step(detection_model, labels, unpad_groundtruth_tensors, optimizer, - learning_rate, add_regularization_loss=True, clip_gradients_value=None, - global_step=None, num_replicas=1.0): """Process a single training batch. @@ -266,16 +280,10 @@ def eager_train_step(detection_model, float32 tensor containing the weights of the keypoint depth feature. unpad_groundtruth_tensors: A parameter passed to unstack_batch. optimizer: The training optimizer that will update the variables. - learning_rate: The learning rate tensor for the current training step. - This is used only for TensorBoard logging purposes, it does not affect - model training. add_regularization_loss: Whether or not to include the model's regularization loss in the losses dictionary. clip_gradients_value: If this is present, clip the gradients global norm at this value using `tf.clip_by_global_norm`. - global_step: The current training step. Used for TensorBoard logging - purposes. This step is not updated by this function and must be - incremented separately. num_replicas: The number of replicas in the current distribution strategy. This is used to scale the total loss so that training in a distribution strategy works correctly. @@ -296,31 +304,18 @@ def eager_train_step(detection_model, losses_dict, _ = _compute_losses_and_predictions_dicts( detection_model, features, labels, add_regularization_loss) - total_loss = losses_dict['Loss/total_loss'] - - # Normalize loss for num replicas - total_loss = tf.math.divide(total_loss, - tf.constant(num_replicas, dtype=tf.float32)) - losses_dict['Loss/normalized_total_loss'] = total_loss - - for loss_type in losses_dict: - tf.compat.v2.summary.scalar( - loss_type, losses_dict[loss_type], step=global_step) + losses_dict = normalize_dict(losses_dict, num_replicas) trainable_variables = detection_model.trainable_variables + total_loss = losses_dict['Loss/total_loss'] gradients = tape.gradient(total_loss, trainable_variables) if clip_gradients_value: gradients, _ = tf.clip_by_global_norm(gradients, clip_gradients_value) optimizer.apply_gradients(zip(gradients, trainable_variables)) - tf.compat.v2.summary.scalar('learning_rate', learning_rate, step=global_step) - tf.compat.v2.summary.image( - name='train_input_images', - step=global_step, - data=features[fields.InputDataFields.image], - max_outputs=3) - return total_loss + + return losses_dict def validate_tf_v2_checkpoint_restore_map(checkpoint_restore_map): @@ -479,7 +474,12 @@ def train_loop( Checkpoint every n training steps. checkpoint_max_to_keep: int, the number of most recent checkpoints to keep in the model directory. - record_summaries: Boolean, whether or not to record summaries. + record_summaries: Boolean, whether or not to record summaries defined by + the model or the training pipeline. This does not impact the summaries + of the loss values which are always recorded. Examples of summaries + that are controlled by this flag include: + - Image summaries of training images. + - Intermediate tensors which maybe logged by meta architectures. performance_summary_exporter: function for exporting performance metrics. num_steps_per_iteration: int, The number of training steps to perform in each iteration. @@ -538,7 +538,8 @@ def train_loop( strategy = tf.compat.v2.distribute.get_strategy() with strategy.scope(): detection_model = MODEL_BUILD_UTIL_MAP['detection_model_fn_base']( - model_config=model_config, is_training=True) + model_config=model_config, is_training=True, + add_summaries=record_summaries) def train_dataset_fn(input_context): """Callable to create train input.""" @@ -581,11 +582,9 @@ def train_dataset_fn(input_context): # is the chief. summary_writer_filepath = get_filepath(strategy, os.path.join(model_dir, 'train')) - if record_summaries: - summary_writer = tf.compat.v2.summary.create_file_writer( - summary_writer_filepath) - else: - summary_writer = tf2.summary.create_noop_writer() + + summary_writer = tf.compat.v2.summary.create_file_writer( + summary_writer_filepath) with summary_writer.as_default(): with strategy.scope(): @@ -619,32 +618,37 @@ def train_dataset_fn(input_context): def train_step_fn(features, labels): """Single train step.""" - loss = eager_train_step( + + if record_summaries: + tf.compat.v2.summary.image( + name='train_input_images', + step=global_step, + data=features[fields.InputDataFields.image], + max_outputs=3) + losses_dict = eager_train_step( detection_model, features, labels, unpad_groundtruth_tensors, optimizer, - learning_rate=learning_rate_fn(), add_regularization_loss=add_regularization_loss, clip_gradients_value=clip_gradients_value, - global_step=global_step, num_replicas=strategy.num_replicas_in_sync) global_step.assign_add(1) - return loss + return losses_dict def _sample_and_train(strategy, train_step_fn, data_iterator): features, labels = data_iterator.next() if hasattr(tf.distribute.Strategy, 'run'): - per_replica_losses = strategy.run( + per_replica_losses_dict = strategy.run( train_step_fn, args=(features, labels)) else: - per_replica_losses = strategy.experimental_run_v2( - train_step_fn, args=(features, labels)) - # TODO(anjalisridhar): explore if it is safe to remove the - ## num_replicas scaling of the loss and switch this to a ReduceOp.Mean - return strategy.reduce(tf.distribute.ReduceOp.SUM, - per_replica_losses, axis=None) + per_replica_losses_dict = ( + strategy.experimental_run_v2( + train_step_fn, args=(features, labels))) + + return reduce_dict( + strategy, per_replica_losses_dict, tf.distribute.ReduceOp.SUM) @tf.function def _dist_train_step(data_iterator): @@ -670,7 +674,7 @@ def _dist_train_step(data_iterator): for _ in range(global_step.value(), train_steps, num_steps_per_iteration): - loss = _dist_train_step(train_input_iter) + losses_dict = _dist_train_step(train_input_iter) time_taken = time.time() - last_step_time last_step_time = time.time() @@ -681,11 +685,19 @@ def _dist_train_step(data_iterator): steps_per_sec_list.append(steps_per_sec) + logged_dict = losses_dict.copy() + logged_dict['learning_rate'] = learning_rate_fn() + + for key, val in logged_dict.items(): + tf.compat.v2.summary.scalar(key, val, step=global_step) + if global_step.value() - logged_step >= 100: + logged_dict_np = {name: value.numpy() for name, value in + logged_dict.items()} tf.logging.info( - 'Step {} per-step time {:.3f}s loss={:.3f}'.format( - global_step.value(), time_taken / num_steps_per_iteration, - loss)) + 'Step {} per-step time {:.3f}s'.format( + global_step.value(), time_taken / num_steps_per_iteration)) + tf.logging.info(pprint.pformat(logged_dict_np, width=40)) logged_step = global_step.value() if ((int(global_step.value()) - checkpointed_step) >= @@ -704,7 +716,7 @@ def _dist_train_step(data_iterator): 'steps_per_sec': np.mean(steps_per_sec_list), 'steps_per_sec_p50': np.median(steps_per_sec_list), 'steps_per_sec_max': max(steps_per_sec_list), - 'last_batch_loss': float(loss) + 'last_batch_loss': float(losses_dict['Loss/total_loss']) } mixed_precision = 'bf16' if kwargs['use_bfloat16'] else 'fp32' performance_summary_exporter(metrics, mixed_precision) diff --git a/research/object_detection/model_main_tf2.py b/research/object_detection/model_main_tf2.py index 0cf053039ec..a97bd5901e0 100644 --- a/research/object_detection/model_main_tf2.py +++ b/research/object_detection/model_main_tf2.py @@ -65,8 +65,10 @@ flags.DEFINE_integer( 'checkpoint_every_n', 1000, 'Integer defining how often we checkpoint.') flags.DEFINE_boolean('record_summaries', True, - ('Whether or not to record summaries during' - ' training.')) + ('Whether or not to record summaries defined by the model' + ' or the training pipeline. This does not impact the' + ' summaries of the loss values which are always' + ' recorded.')) FLAGS = flags.FLAGS From 7004ce3b7b9bb200167c1f744bbc5dacb2f4774f Mon Sep 17 00:00:00 2001 From: Ronny Votel Date: Wed, 2 Jun 2021 20:56:25 -0700 Subject: [PATCH 16/50] Updating target assigner utils to allow for weights applied in box regions. PiperOrigin-RevId: 377208217 --- .../core/target_assigner_test.py | 6 +- .../utils/target_assigner_utils.py | 83 ++++++++++++++----- .../utils/target_assigner_utils_test.py | 35 ++++++-- 3 files changed, 96 insertions(+), 28 deletions(-) diff --git a/research/object_detection/core/target_assigner_test.py b/research/object_detection/core/target_assigner_test.py index ad0eaa82006..07e6889d320 100644 --- a/research/object_detection/core/target_assigner_test.py +++ b/research/object_detection/core/target_assigner_test.py @@ -1734,9 +1734,9 @@ def graph_fn(): # the region of the 1st instance boxing box should be blacked out # (0.0, 0.0, 0.5, 0.5), transfering to (0, 0, 15, 10) in absolute output # space. - self.assertAlmostEqual(np.sum(valid_mask[:, 0:16, 0:11]), 0.0) - # All other values are 1.0 so the sum is: 30 * 20 - 16 * 11 = 424. - self.assertAlmostEqual(np.sum(valid_mask), 424.0) + self.assertAlmostEqual(np.sum(valid_mask[:, 0:15, 0:10]), 0.0) + # All other values are 1.0 so the sum is: 30 * 20 - 15 * 10 = 450. + self.assertAlmostEqual(np.sum(valid_mask), 450.0) def test_assign_keypoints_offset_targets(self): def graph_fn(): diff --git a/research/object_detection/utils/target_assigner_utils.py b/research/object_detection/utils/target_assigner_utils.py index 7ac61e8a84d..dd8c69b9c12 100644 --- a/research/object_detection/utils/target_assigner_utils.py +++ b/research/object_detection/utils/target_assigner_utils.py @@ -289,12 +289,38 @@ def get_valid_keypoint_mask_for_class(keypoint_coordinates, return mask, keypoints_nan_to_zeros -def blackout_pixel_weights_by_box_regions(height, width, boxes, blackout): - """Blackout the pixel weights in the target box regions. +def blackout_pixel_weights_by_box_regions(height, width, boxes, blackout, + weights=None): + """Apply weights at pixel locations. This function is used to generate the pixel weight mask (usually in the output image dimension). The mask is to ignore some regions when computing loss. + Weights are applied as follows: + - Any region outside of a box gets the default weight 1.0 + - Any box for which an explicit weight is specifed gets that weight. If + multiple boxes overlap, the maximum of the weights is applied. + - Any box for which blackout=True is specified will get a weight of 0.0, + regardless of whether an equivalent non-zero weight is specified. Also, the + blackout region takes precedence over other boxes which may overlap with + non-zero weight. + + Example: + height = 4 + width = 4 + boxes = [[0., 0., 2., 2.], + [0., 0., 4., 2.], + [3., 0., 4., 4.]] + blackout = [False, False, True] + weights = [4.0, 3.0, 2.0] + blackout_pixel_weights_by_box_regions(height, width, boxes, blackout, + weights) + >> [[4.0, 4.0, 1.0, 1.0], + [4.0, 4.0, 1.0, 1.0], + [3.0, 3.0, 1.0, 1.0], + [0.0, 0.0, 0.0, 0.0]] + + Args: height: int, height of the (output) image. width: int, width of the (output) image. @@ -302,10 +328,15 @@ def blackout_pixel_weights_by_box_regions(height, width, boxes, blackout): coordinates of the four corners of the boxes. blackout: A boolean tensor with shape [num_instances] indicating whether to blackout (zero-out) the weights within the box regions. + weights: An optional float32 tensor with shape [num_instances] indicating + a value to apply in each box region. Note that if blackout=True for a + given box, the weight will be zero. If None, all weights are assumed to be + 1. Returns: A float tensor with shape [height, width] where all values within the - regions of the blackout boxes are 0.0 and 1.0 else where. + regions of the blackout boxes are 0.0 and 1.0 (or weights if supplied) + elsewhere. """ num_instances, _ = shape_utils.combined_static_and_dynamic_shape(boxes) # If no annotation instance is provided, return all ones (instead of @@ -323,22 +354,36 @@ def blackout_pixel_weights_by_box_regions(height, width, boxes, blackout): # Make the mask with all 1.0 in the box regions. # Shape: [num_instances, height, width] - in_boxes = tf.cast( - tf.logical_and( - tf.logical_and(y_grid >= y_min, y_grid <= y_max), - tf.logical_and(x_grid >= x_min, x_grid <= x_max)), - dtype=tf.float32) - - # Shape: [num_instances, height, width] - blackout = tf.tile( - tf.expand_dims(tf.expand_dims(blackout, axis=-1), axis=-1), - [1, height, width]) - - # Select only the boxes specified by blackout. - selected_in_boxes = tf.where(blackout, in_boxes, tf.zeros_like(in_boxes)) - out_boxes = tf.reduce_max(selected_in_boxes, axis=0) - out_boxes = tf.ones_like(out_boxes) - out_boxes - return out_boxes + in_boxes = tf.math.logical_and( + tf.math.logical_and(y_grid >= y_min, y_grid < y_max), + tf.math.logical_and(x_grid >= x_min, x_grid < x_max)) + + if weights is None: + weights = tf.ones_like(blackout, dtype=tf.float32) + + # Compute a [height, width] tensor with the maximum weight in each box, and + # 0.0 elsewhere. + weights_tiled = tf.tile( + weights[:, tf.newaxis, tf.newaxis], [1, height, width]) + weights_3d = tf.where(in_boxes, weights_tiled, + tf.zeros_like(weights_tiled)) + weights_2d = tf.math.maximum( + tf.math.reduce_max(weights_3d, axis=0), 0.0) + + # Add 1.0 to all regions outside a box. + weights_2d = tf.where( + tf.math.reduce_any(in_boxes, axis=0), + weights_2d, + tf.ones_like(weights_2d)) + + # Now enforce that blackout regions all have zero weights. + keep_region = tf.cast(tf.math.logical_not(blackout), tf.float32) + keep_region_tiled = tf.tile( + keep_region[:, tf.newaxis, tf.newaxis], [1, height, width]) + keep_region_3d = tf.where(in_boxes, keep_region_tiled, + tf.ones_like(keep_region_tiled)) + keep_region_2d = tf.math.reduce_min(keep_region_3d, axis=0) + return weights_2d * keep_region_2d def _get_yx_indices_offset_by_radius(radius): diff --git a/research/object_detection/utils/target_assigner_utils_test.py b/research/object_detection/utils/target_assigner_utils_test.py index ef0f3420e01..4f35a4463f5 100644 --- a/research/object_detection/utils/target_assigner_utils_test.py +++ b/research/object_detection/utils/target_assigner_utils_test.py @@ -196,13 +196,36 @@ def graph_fn(): return output output = self.execute(graph_fn, []) - # All zeros in region [0:6, 0:6]. - self.assertAlmostEqual(np.sum(output[0:6, 0:6]), 0.0) - # All zeros in region [12:19, 6:9]. - self.assertAlmostEqual(np.sum(output[6:9, 12:19]), 0.0) + # All zeros in region [0:5, 0:5]. + self.assertAlmostEqual(np.sum(output[0:5, 0:5]), 0.0) + # All zeros in region [12:18, 6:8]. + self.assertAlmostEqual(np.sum(output[6:8, 12:18]), 0.0) # All other pixel weights should be 1.0. - # 20 * 10 - 6 * 6 - 3 * 7 = 143.0 - self.assertAlmostEqual(np.sum(output), 143.0) + # 20 * 10 - 5 * 5 - 2 * 6 = 163.0 + self.assertAlmostEqual(np.sum(output), 163.0) + + def test_blackout_pixel_weights_by_box_regions_with_weights(self): + def graph_fn(): + boxes = tf.constant( + [[0.0, 0.0, 2.0, 2.0], + [0.0, 0.0, 4.0, 2.0], + [3.0, 0.0, 4.0, 4.0]], + dtype=tf.float32) + blackout = tf.constant([False, False, True], dtype=tf.bool) + weights = tf.constant([0.4, 0.3, 0.2], tf.float32) + blackout_pixel_weights_by_box_regions = tf.function( + ta_utils.blackout_pixel_weights_by_box_regions) + output = blackout_pixel_weights_by_box_regions( + 4, 4, boxes, blackout, weights) + return output + + output = self.execute(graph_fn, []) + expected_weights = [ + [0.4, 0.4, 1.0, 1.0], + [0.4, 0.4, 1.0, 1.0], + [0.3, 0.3, 1.0, 1.0], + [0.0, 0.0, 0.0, 0.0]] + np.testing.assert_array_almost_equal(expected_weights, output) def test_blackout_pixel_weights_by_box_regions_zero_instance(self): def graph_fn(): From b185135fe6241ef5d84e2547c17e66abdb47d7e2 Mon Sep 17 00:00:00 2001 From: Xianzhi Du Date: Wed, 2 Jun 2021 22:13:02 -0700 Subject: [PATCH 17/50] Internal change PiperOrigin-RevId: 377216027 --- official/vision/beta/tasks/retinanet.py | 46 +++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/official/vision/beta/tasks/retinanet.py b/official/vision/beta/tasks/retinanet.py index a1dca4205c5..a2e05a8be64 100644 --- a/official/vision/beta/tasks/retinanet.py +++ b/official/vision/beta/tasks/retinanet.py @@ -133,12 +133,54 @@ def build_inputs(self, return dataset + def build_attribute_loss(self, + attribute_heads: List[exp_cfg.AttributeHead], + outputs: Mapping[str, Any], + labels: Mapping[str, Any], + box_sample_weight: tf.Tensor) -> float: + """Computes attribute loss. + + Args: + attribute_heads: a list of attribute head configs. + outputs: RetinaNet model outputs. + labels: RetinaNet labels. + box_sample_weight: normalized bounding box sample weights. + + Returns: + Attribute loss of all attribute heads. + """ + attribute_loss = 0.0 + for head in attribute_heads: + if head.name not in labels['attribute_targets']: + raise ValueError(f'Attribute {head.name} not found in label targets.') + if head.name not in outputs['attribute_outputs']: + raise ValueError(f'Attribute {head.name} not found in model outputs.') + + y_true_att = keras_cv.losses.multi_level_flatten( + labels['attribute_targets'][head.name], last_dim=head.size) + y_pred_att = keras_cv.losses.multi_level_flatten( + outputs['attribute_outputs'][head.name], last_dim=head.size) + if head.type == 'regression': + att_loss_fn = tf.keras.losses.Huber( + 1.0, reduction=tf.keras.losses.Reduction.SUM) + att_loss = att_loss_fn( + y_true=y_true_att, + y_pred=y_pred_att, + sample_weight=box_sample_weight) + else: + raise ValueError(f'Attribute type {head.type} not supported.') + attribute_loss += att_loss + + return attribute_loss + def build_losses(self, outputs: Mapping[str, Any], labels: Mapping[str, Any], aux_losses: Optional[Any] = None): """Build RetinaNet losses.""" params = self.task_config + attribute_heads = self.task_config.model.head.attribute_heads + cls_loss_fn = keras_cv.losses.FocalLoss( alpha=params.losses.focal_loss_alpha, gamma=params.losses.focal_loss_gamma, @@ -170,6 +212,10 @@ def build_losses(self, model_loss = cls_loss + params.losses.box_loss_weight * box_loss + if attribute_heads: + model_loss += self.build_attribute_loss(attribute_heads, outputs, labels, + box_sample_weight) + total_loss = model_loss if aux_losses: reg_loss = tf.reduce_sum(aux_losses) From ec52ef2dc2b0e749c9f5a7b6238daf6ce67567bd Mon Sep 17 00:00:00 2001 From: Ronny Votel Date: Thu, 3 Jun 2021 07:07:21 -0700 Subject: [PATCH 18/50] Adding head customization parameters to `ObjectDetection` and `MaskEstimation` CenterNet proto messages. PiperOrigin-RevId: 377280733 --- .../builders/model_builder.py | 28 ++++++++++- .../builders/model_builder_tf2_test.py | 42 ++++++++++++++-- .../center_net_meta_arch.py | 50 ++++++++++++++++--- .../center_net_meta_arch_tf2_test.py | 4 +- .../object_detection/protos/center_net.proto | 12 +++++ 5 files changed, 121 insertions(+), 15 deletions(-) diff --git a/research/object_detection/builders/model_builder.py b/research/object_detection/builders/model_builder.py index bf2eeebb790..b1d97a9a790 100644 --- a/research/object_detection/builders/model_builder.py +++ b/research/object_detection/builders/model_builder.py @@ -926,11 +926,27 @@ def object_detection_proto_to_params(od_config): losses_pb2.WeightedSigmoidClassificationLoss()) loss.localization_loss.CopyFrom(od_config.localization_loss) _, localization_loss, _, _, _, _, _ = (losses_builder.build(loss)) + if od_config.HasField('scale_head_params'): + scale_head_num_filters = list(od_config.scale_head_params.num_filters) + scale_head_kernel_sizes = list(od_config.scale_head_params.kernel_sizes) + else: + scale_head_num_filters = [256] + scale_head_kernel_sizes = [3] + if od_config.HasField('offset_head_params'): + offset_head_num_filters = list(od_config.offset_head_params.num_filters) + offset_head_kernel_sizes = list(od_config.offset_head_params.kernel_sizes) + else: + offset_head_num_filters = [256] + offset_head_kernel_sizes = [3] return center_net_meta_arch.ObjectDetectionParams( localization_loss=localization_loss, scale_loss_weight=od_config.scale_loss_weight, offset_loss_weight=od_config.offset_loss_weight, - task_loss_weight=od_config.task_loss_weight) + task_loss_weight=od_config.task_loss_weight, + scale_head_num_filters=scale_head_num_filters, + scale_head_kernel_sizes=scale_head_kernel_sizes, + offset_head_num_filters=offset_head_num_filters, + offset_head_kernel_sizes=offset_head_kernel_sizes) def object_center_proto_to_params(oc_config): @@ -973,13 +989,21 @@ def mask_proto_to_params(mask_config): losses_pb2.WeightedL2LocalizationLoss()) loss.classification_loss.CopyFrom(mask_config.classification_loss) classification_loss, _, _, _, _, _, _ = (losses_builder.build(loss)) + if mask_config.HasField('mask_head_params'): + mask_head_num_filters = list(mask_config.mask_head_params.num_filters) + mask_head_kernel_sizes = list(mask_config.mask_head_params.kernel_sizes) + else: + mask_head_num_filters = [256] + mask_head_kernel_sizes = [3] return center_net_meta_arch.MaskParams( classification_loss=classification_loss, task_loss_weight=mask_config.task_loss_weight, mask_height=mask_config.mask_height, mask_width=mask_config.mask_width, score_threshold=mask_config.score_threshold, - heatmap_bias_init=mask_config.heatmap_bias_init) + heatmap_bias_init=mask_config.heatmap_bias_init, + mask_head_num_filters=mask_head_num_filters, + mask_head_kernel_sizes=mask_head_kernel_sizes) def densepose_proto_to_params(densepose_config): diff --git a/research/object_detection/builders/model_builder_tf2_test.py b/research/object_detection/builders/model_builder_tf2_test.py index 5b3aa302bea..4c55dad67b5 100644 --- a/research/object_detection/builders/model_builder_tf2_test.py +++ b/research/object_detection/builders/model_builder_tf2_test.py @@ -188,7 +188,7 @@ def get_fake_object_center_from_keypoints_proto(self): return text_format.Merge(proto_txt, center_net_pb2.CenterNet.ObjectCenterParams()) - def get_fake_object_detection_proto(self): + def get_fake_object_detection_proto(self, customize_head_params=False): proto_txt = """ task_loss_weight: 0.5 offset_loss_weight: 0.1 @@ -198,10 +198,19 @@ def get_fake_object_detection_proto(self): } } """ + if customize_head_params: + proto_txt += """ + scale_head_params { + num_filters: 128 + num_filters: 64 + kernel_sizes: 5 + kernel_sizes: 3 + } + """ return text_format.Merge(proto_txt, center_net_pb2.CenterNet.ObjectDetection()) - def get_fake_mask_proto(self): + def get_fake_mask_proto(self, customize_head_params=False): proto_txt = """ task_loss_weight: 0.7 classification_loss { @@ -212,6 +221,15 @@ def get_fake_mask_proto(self): score_threshold: 0.7 heatmap_bias_init: -2.0 """ + if customize_head_params: + proto_txt += """ + mask_head_params { + num_filters: 128 + num_filters: 64 + kernel_sizes: 5 + kernel_sizes: 3 + } + """ return text_format.Merge(proto_txt, center_net_pb2.CenterNet.MaskEstimation()) @@ -266,14 +284,16 @@ def test_create_center_net_model(self, customize_head_params): self.get_fake_object_center_proto( customize_head_params=customize_head_params)) config.center_net.object_detection_task.CopyFrom( - self.get_fake_object_detection_proto()) + self.get_fake_object_detection_proto( + customize_head_params=customize_head_params)) config.center_net.keypoint_estimation_task.append( self.get_fake_keypoint_proto( customize_head_params=customize_head_params)) config.center_net.keypoint_label_map_path = ( self.get_fake_label_map_file_path()) config.center_net.mask_estimation_task.CopyFrom( - self.get_fake_mask_proto()) + self.get_fake_mask_proto( + customize_head_params=customize_head_params)) config.center_net.densepose_estimation_task.CopyFrom( self.get_fake_densepose_proto()) @@ -303,6 +323,14 @@ def test_create_center_net_model(self, customize_head_params): self.assertAlmostEqual(model._od_params.task_loss_weight, 0.5) self.assertIsInstance(model._od_params.localization_loss, losses.L1LocalizationLoss) + self.assertEqual(model._od_params.offset_head_num_filters, [256]) + self.assertEqual(model._od_params.offset_head_kernel_sizes, [3]) + if customize_head_params: + self.assertEqual(model._od_params.scale_head_num_filters, [128, 64]) + self.assertEqual(model._od_params.scale_head_kernel_sizes, [5, 3]) + else: + self.assertEqual(model._od_params.scale_head_num_filters, [256]) + self.assertEqual(model._od_params.scale_head_kernel_sizes, [3]) # Check keypoint estimation related parameters. kp_params = model._kp_params_dict['human_pose'] @@ -352,6 +380,12 @@ def test_create_center_net_model(self, customize_head_params): self.assertAlmostEqual(model._mask_params.score_threshold, 0.7) self.assertAlmostEqual( model._mask_params.heatmap_bias_init, -2.0, places=4) + if customize_head_params: + self.assertEqual(model._mask_params.mask_head_num_filters, [128, 64]) + self.assertEqual(model._mask_params.mask_head_kernel_sizes, [5, 3]) + else: + self.assertEqual(model._mask_params.mask_head_num_filters, [256]) + self.assertEqual(model._mask_params.mask_head_kernel_sizes, [3]) # Check DensePose related parameters. self.assertEqual(model._densepose_params.class_id, 0) diff --git a/research/object_detection/meta_architectures/center_net_meta_arch.py b/research/object_detection/meta_architectures/center_net_meta_arch.py index e9e84e39644..79dc7186f22 100644 --- a/research/object_detection/meta_architectures/center_net_meta_arch.py +++ b/research/object_detection/meta_architectures/center_net_meta_arch.py @@ -1668,7 +1668,9 @@ def predicted_embeddings_at_object_centers(embedding_predictions, class ObjectDetectionParams( collections.namedtuple('ObjectDetectionParams', [ 'localization_loss', 'scale_loss_weight', 'offset_loss_weight', - 'task_loss_weight' + 'task_loss_weight', 'scale_head_num_filters', + 'scale_head_kernel_sizes', 'offset_head_num_filters', + 'offset_head_kernel_sizes' ])): """Namedtuple to host object detection related parameters. @@ -1684,7 +1686,11 @@ def __new__(cls, localization_loss, scale_loss_weight, offset_loss_weight, - task_loss_weight=1.0): + task_loss_weight=1.0, + scale_head_num_filters=(256), + scale_head_kernel_sizes=(3), + offset_head_num_filters=(256), + offset_head_kernel_sizes=(3)): """Constructor with default values for ObjectDetectionParams. Args: @@ -1697,13 +1703,23 @@ def __new__(cls, depending on the input size. offset_loss_weight: float, The weight for localizing center offsets. task_loss_weight: float, the weight of the object detection loss. + scale_head_num_filters: filter numbers of the convolutional layers used + by the object detection box scale prediction head. + scale_head_kernel_sizes: kernel size of the convolutional layers used + by the object detection box scale prediction head. + offset_head_num_filters: filter numbers of the convolutional layers used + by the object detection box offset prediction head. + offset_head_kernel_sizes: kernel size of the convolutional layers used + by the object detection box offset prediction head. Returns: An initialized ObjectDetectionParams namedtuple. """ return super(ObjectDetectionParams, cls).__new__(cls, localization_loss, scale_loss_weight, - offset_loss_weight, task_loss_weight) + offset_loss_weight, task_loss_weight, + scale_head_num_filters, scale_head_kernel_sizes, + offset_head_num_filters, offset_head_kernel_sizes) class KeypointEstimationParams( @@ -1937,7 +1953,8 @@ def __new__(cls, class MaskParams( collections.namedtuple('MaskParams', [ 'classification_loss', 'task_loss_weight', 'mask_height', 'mask_width', - 'score_threshold', 'heatmap_bias_init' + 'score_threshold', 'heatmap_bias_init', 'mask_head_num_filters', + 'mask_head_kernel_sizes' ])): """Namedtuple to store mask prediction related parameters.""" @@ -1949,7 +1966,9 @@ def __new__(cls, mask_height=256, mask_width=256, score_threshold=0.5, - heatmap_bias_init=-2.19): + heatmap_bias_init=-2.19, + mask_head_num_filters=(256), + mask_head_kernel_sizes=(3)): """Constructor with default values for MaskParams. Args: @@ -1963,6 +1982,10 @@ def __new__(cls, heatmap_bias_init: float, the initial value of bias in the convolutional kernel of the semantic segmentation prediction head. If set to None, the bias is initialized with zeros. + mask_head_num_filters: filter numbers of the convolutional layers used + by the mask prediction head. + mask_head_kernel_sizes: kernel size of the convolutional layers used + by the mask prediction head. Returns: An initialized MaskParams namedtuple. @@ -1970,7 +1993,8 @@ def __new__(cls, return super(MaskParams, cls).__new__(cls, classification_loss, task_loss_weight, mask_height, mask_width, - score_threshold, heatmap_bias_init) + score_threshold, heatmap_bias_init, + mask_head_num_filters, mask_head_kernel_sizes) class DensePoseParams( @@ -2312,10 +2336,18 @@ def _construct_prediction_heads(self, num_classes, num_feature_outputs, if self._od_params is not None: prediction_heads[BOX_SCALE] = self._make_prediction_net_list( - num_feature_outputs, NUM_SIZE_CHANNELS, name='box_scale', + num_feature_outputs, + NUM_SIZE_CHANNELS, + kernel_sizes=self._od_params.scale_head_kernel_sizes, + num_filters=self._od_params.scale_head_num_filters, + name='box_scale', unit_height_conv=unit_height_conv) prediction_heads[BOX_OFFSET] = self._make_prediction_net_list( - num_feature_outputs, NUM_OFFSET_CHANNELS, name='box_offset', + num_feature_outputs, + NUM_OFFSET_CHANNELS, + kernel_sizes=self._od_params.offset_head_kernel_sizes, + num_filters=self._od_params.offset_head_num_filters, + name='box_offset', unit_height_conv=unit_height_conv) if self._kp_params_dict is not None: @@ -2370,6 +2402,8 @@ def _construct_prediction_heads(self, num_classes, num_feature_outputs, prediction_heads[SEGMENTATION_HEATMAP] = self._make_prediction_net_list( num_feature_outputs, num_classes, + kernel_sizes=self._mask_params.mask_head_kernel_sizes, + num_filters=self._mask_params.mask_head_num_filters, bias_fill=self._mask_params.heatmap_bias_init, name='seg_heatmap', unit_height_conv=unit_height_conv) diff --git a/research/object_detection/meta_architectures/center_net_meta_arch_tf2_test.py b/research/object_detection/meta_architectures/center_net_meta_arch_tf2_test.py index 6880f51984f..c88790a720d 100644 --- a/research/object_detection/meta_architectures/center_net_meta_arch_tf2_test.py +++ b/research/object_detection/meta_architectures/center_net_meta_arch_tf2_test.py @@ -1539,7 +1539,9 @@ def get_fake_mask_params(): classification_loss=losses.WeightedSoftmaxClassificationLoss(), task_loss_weight=1.0, mask_height=4, - mask_width=4) + mask_width=4, + mask_head_num_filters=[96], + mask_head_kernel_sizes=[3]) def get_fake_densepose_params(): diff --git a/research/object_detection/protos/center_net.proto b/research/object_detection/protos/center_net.proto index 9e58bf1ad93..fd138e42ce7 100644 --- a/research/object_detection/protos/center_net.proto +++ b/research/object_detection/protos/center_net.proto @@ -65,6 +65,14 @@ message CenterNet { // Localization loss configuration for object scale and offset losses. optional LocalizationLoss localization_loss = 8; + + // Parameters to determine the architecture of the object scale prediction + // head. + optional PredictionHeadParams scale_head_params = 9; + + // Parameters to determine the architecture of the object offset prediction + // head. + optional PredictionHeadParams offset_head_params = 10; } optional ObjectDetection object_detection_task = 4; @@ -268,6 +276,10 @@ message CenterNet { // prediction head. -2.19 corresponds to predicting foreground with // a probability of 0.1. optional float heatmap_bias_init = 3 [default = -2.19]; + + // Parameters to determine the architecture of the segmentation mask + // prediction head. + optional PredictionHeadParams mask_head_params = 7; } optional MaskEstimation mask_estimation_task = 8; From 55d04bbe8a0d133295cf7b51463ae84c434e51b3 Mon Sep 17 00:00:00 2001 From: Abdullah Rashwan Date: Thu, 3 Jun 2021 11:57:53 -0700 Subject: [PATCH 19/50] Internal change PiperOrigin-RevId: 377340715 --- .../beta/configs/image_classification.py | 1 + .../beta/dataloaders/classification_input.py | 54 ++++++++++++++----- .../beta/dataloaders/tfexample_utils.py | 4 +- .../vision/beta/tasks/image_classification.py | 1 + 4 files changed, 45 insertions(+), 15 deletions(-) diff --git a/official/vision/beta/configs/image_classification.py b/official/vision/beta/configs/image_classification.py index 7044a4c0004..b98354a8587 100644 --- a/official/vision/beta/configs/image_classification.py +++ b/official/vision/beta/configs/image_classification.py @@ -43,6 +43,7 @@ class DataConfig(cfg.DataConfig): file_type: str = 'tfrecord' image_field_key: str = 'image/encoded' label_field_key: str = 'image/class/label' + decode_jpeg_only: bool = True # Keep for backward compatibility. aug_policy: Optional[str] = None # None, 'autoaug', or 'randaug'. diff --git a/official/vision/beta/dataloaders/classification_input.py b/official/vision/beta/dataloaders/classification_input.py index 3f12d043ee7..734d84dd6dd 100644 --- a/official/vision/beta/dataloaders/classification_input.py +++ b/official/vision/beta/dataloaders/classification_input.py @@ -66,6 +66,7 @@ def __init__(self, num_classes: float, image_field_key: str = DEFAULT_IMAGE_FIELD_KEY, label_field_key: str = DEFAULT_LABEL_FIELD_KEY, + decode_jpeg_only: bool = True, aug_rand_hflip: bool = True, aug_type: Optional[common.Augmentation] = None, is_multilabel: bool = False, @@ -78,6 +79,8 @@ def __init__(self, num_classes: `float`, number of classes. image_field_key: `str`, the key name to encoded image in tf.Example. label_field_key: `str`, the key name to label in tf.Example. + decode_jpeg_only: `bool`, if True, only JPEG format is decoded, this is + faster than decoding other types. Default is True. aug_rand_hflip: `bool`, if True, augment training with random horizontal flip. aug_type: An optional Augmentation object to choose from AutoAugment and @@ -118,6 +121,7 @@ def __init__(self, self._augmenter = None self._label_field_key = label_field_key self._is_multilabel = is_multilabel + self._decode_jpeg_only = decode_jpeg_only def _parse_train_data(self, decoded_tensors): """Parses data for training.""" @@ -142,16 +146,29 @@ def _parse_eval_data(self, decoded_tensors): def _parse_train_image(self, decoded_tensors): """Parses image data for training.""" image_bytes = decoded_tensors[self._image_field_key] - image_shape = tf.image.extract_jpeg_shape(image_bytes) - # Crops image. - # TODO(pengchong): support image format other than JPEG. - cropped_image = preprocess_ops.random_crop_image_v2( - image_bytes, image_shape) - image = tf.cond( - tf.reduce_all(tf.equal(tf.shape(cropped_image), image_shape)), - lambda: preprocess_ops.center_crop_image_v2(image_bytes, image_shape), - lambda: cropped_image) + if self._decode_jpeg_only: + image_shape = tf.image.extract_jpeg_shape(image_bytes) + + # Crops image. + cropped_image = preprocess_ops.random_crop_image_v2( + image_bytes, image_shape) + image = tf.cond( + tf.reduce_all(tf.equal(tf.shape(cropped_image), image_shape)), + lambda: preprocess_ops.center_crop_image_v2(image_bytes, image_shape), + lambda: cropped_image) + else: + # Decodes image. + image = tf.io.decode_image(image_bytes, channels=3) + image.set_shape([None, None, 3]) + + # Crops image. + cropped_image = preprocess_ops.random_crop_image(image) + + image = tf.cond( + tf.reduce_all(tf.equal(tf.shape(cropped_image), tf.shape(image))), + lambda: preprocess_ops.center_crop_image(image), + lambda: cropped_image) if self._aug_rand_hflip: image = tf.image.random_flip_left_right(image) @@ -159,6 +176,7 @@ def _parse_train_image(self, decoded_tensors): # Resizes image. image = tf.image.resize( image, self._output_size, method=tf.image.ResizeMethod.BILINEAR) + image.set_shape([self._output_size[0], self._output_size[1], 3]) # Apply autoaug or randaug. if self._augmenter is not None: @@ -177,15 +195,23 @@ def _parse_train_image(self, decoded_tensors): def _parse_eval_image(self, decoded_tensors): """Parses image data for evaluation.""" image_bytes = decoded_tensors[self._image_field_key] - image_shape = tf.image.extract_jpeg_shape(image_bytes) - # Center crops and resizes image. - image = preprocess_ops.center_crop_image_v2(image_bytes, image_shape) + if self._decode_jpeg_only: + image_shape = tf.image.extract_jpeg_shape(image_bytes) + + # Center crops. + image = preprocess_ops.center_crop_image_v2(image_bytes, image_shape) + else: + # Decodes image. + image = tf.io.decode_image(image_bytes, channels=3) + image.set_shape([None, None, 3]) + + # Center crops. + image = preprocess_ops.center_crop_image(image) image = tf.image.resize( image, self._output_size, method=tf.image.ResizeMethod.BILINEAR) - - image = tf.reshape(image, [self._output_size[0], self._output_size[1], 3]) + image.set_shape([self._output_size[0], self._output_size[1], 3]) # Normalizes image with mean and std pixel values. image = preprocess_ops.normalize_image(image, diff --git a/official/vision/beta/dataloaders/tfexample_utils.py b/official/vision/beta/dataloaders/tfexample_utils.py index b64d24ff35b..8e55e3c55ff 100644 --- a/official/vision/beta/dataloaders/tfexample_utils.py +++ b/official/vision/beta/dataloaders/tfexample_utils.py @@ -127,10 +127,12 @@ def _encode_image(image_array: np.ndarray, fmt: str) -> bytes: def create_classification_example( image_height: int, image_width: int, + image_format: str = 'JPEG', is_multilabel: bool = False) -> tf.train.Example: """Creates image and labels for image classification input pipeline.""" image = _encode_image( - np.uint8(np.random.rand(image_height, image_width, 3) * 255), fmt='JPEG') + np.uint8(np.random.rand(image_height, image_width, 3) * 255), + fmt=image_format) labels = [0, 1] if is_multilabel else [0] serialized_example = tf.train.Example( features=tf.train.Features( diff --git a/official/vision/beta/tasks/image_classification.py b/official/vision/beta/tasks/image_classification.py index 9f8f2edc884..5a2a0eb7b2d 100644 --- a/official/vision/beta/tasks/image_classification.py +++ b/official/vision/beta/tasks/image_classification.py @@ -104,6 +104,7 @@ def build_inputs( num_classes=num_classes, image_field_key=image_field_key, label_field_key=label_field_key, + decode_jpeg_only=params.decode_jpeg_only, aug_rand_hflip=params.aug_rand_hflip, aug_type=params.aug_type, is_multilabel=is_multilabel, From 104a7e80b26763aa051f8fa36a40860adbb0b5c6 Mon Sep 17 00:00:00 2001 From: Le Hou Date: Thu, 3 Jun 2021 15:01:25 -0700 Subject: [PATCH 20/50] Internal change PiperOrigin-RevId: 377381104 --- official/modeling/progressive/trainer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/official/modeling/progressive/trainer.py b/official/modeling/progressive/trainer.py index c3bebbdfeb5..caf8e27c5cc 100644 --- a/official/modeling/progressive/trainer.py +++ b/official/modeling/progressive/trainer.py @@ -284,8 +284,11 @@ def _maybe_export_non_progressive_checkpoint(self, export_ckpt_dir): checkpoint_interval=checkpoint_interval, ) + # Make sure we export the last checkpoint. + last_checkpoint = ( + self.global_step.numpy() == self._config.trainer.train_steps) checkpoint_path = self._export_ckpt_manager.save( checkpoint_number=self.global_step.numpy(), - check_interval=True) + check_interval=not last_checkpoint) if checkpoint_path: logging.info('Checkpoints exported: %s.', checkpoint_path) From 0b555eed127cb6769cb57493fe6e88dbf6f5a3c7 Mon Sep 17 00:00:00 2001 From: Jaeyoun Kim Date: Thu, 3 Jun 2021 16:52:32 -0700 Subject: [PATCH 21/50] Update CODEOWNERS Add a new code owner for ranking --- CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/CODEOWNERS b/CODEOWNERS index 9dd84ad290b..dcb769487b1 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,6 +1,7 @@ * @tensorflow/tf-garden-team @tensorflow/tf-model-garden-team /official/ @rachellj218 @saberkun @jaeyounkim /official/nlp/ @saberkun @lehougoogle @rachellj218 @jaeyounkim +/official/recommendation/ranking/ @gagika /official/vision/ @xianzhidu @yeqingli @arashwan @saberkun @rachellj218 @jaeyounkim /official/vision/beta/projects/assemblenet/ @mryoo /official/vision/beta/projects/deepmac_maskrcnn/ @vighneshbirodkar From 28ce46963e9fbe05a8e253cf45539976026811da Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 3 Jun 2021 19:09:17 -0700 Subject: [PATCH 22/50] Internal change PiperOrigin-RevId: 377422408 --- official/modeling/activations/sigmoid.py | 2 +- official/modeling/activations/swish.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/official/modeling/activations/sigmoid.py b/official/modeling/activations/sigmoid.py index e815f7ee8c5..277463040e7 100644 --- a/official/modeling/activations/sigmoid.py +++ b/official/modeling/activations/sigmoid.py @@ -28,4 +28,4 @@ def hard_sigmoid(features): The activation value. """ features = tf.convert_to_tensor(features) - return tf.nn.relu6(features + tf.constant(3.)) * 0.16667 + return tf.nn.relu6(features + tf.cast(3., features.dtype)) * 0.16667 diff --git a/official/modeling/activations/swish.py b/official/modeling/activations/swish.py index 7fcac2b2bff..ea79985e300 100644 --- a/official/modeling/activations/swish.py +++ b/official/modeling/activations/swish.py @@ -52,7 +52,8 @@ def hard_swish(features): The activation value. """ features = tf.convert_to_tensor(features) - return features * tf.nn.relu6(features + tf.constant(3.)) * (1. / 6.) + fdtype = features.dtype + return features * tf.nn.relu6(features + tf.cast(3., fdtype)) * (1. / 6.) @tf.keras.utils.register_keras_serializable(package='Text') From 30851764f098e03ee23513896e30a1be54be7006 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 3 Jun 2021 19:25:36 -0700 Subject: [PATCH 23/50] Internal change PiperOrigin-RevId: 377424053 --- official/vision/beta/modeling/layers/nn_layers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/official/vision/beta/modeling/layers/nn_layers.py b/official/vision/beta/modeling/layers/nn_layers.py index f44b17a25ca..60759ab019c 100644 --- a/official/vision/beta/modeling/layers/nn_layers.py +++ b/official/vision/beta/modeling/layers/nn_layers.py @@ -132,8 +132,7 @@ def __init__(self, def build(self, input_shape): num_reduced_filters = make_divisible( - max(1, int(self._in_filters * self._se_ratio)), - divisor=self._divisible_by) + self._in_filters * self._se_ratio, divisor=self._divisible_by) self._se_reduce = tf.keras.layers.Conv2D( filters=num_reduced_filters, From 33d1ce8380971fe6cb5e10e514ad8d9ba879ed05 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 3 Jun 2021 19:37:11 -0700 Subject: [PATCH 24/50] Internal change PiperOrigin-RevId: 377425143 --- .../experiments/cifar_simclr_pretrain.yaml | 2 +- .../imagenet_simclr_finetune_gpu.yaml | 4 +- .../imagenet_simclr_finetune_tpu.yaml | 70 ++++++++++++++++++ .../imagenet_simclr_pretrain_gpu.yaml | 16 ++--- .../imagenet_simclr_pretrain_tpu.yaml | 71 +++++++++++++++++++ 5 files changed, 152 insertions(+), 11 deletions(-) create mode 100644 official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_tpu.yaml create mode 100644 official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_tpu.yaml diff --git a/official/vision/beta/projects/simclr/configs/experiments/cifar_simclr_pretrain.yaml b/official/vision/beta/projects/simclr/configs/experiments/cifar_simclr_pretrain.yaml index 5d5bd642efa..07d319a6929 100644 --- a/official/vision/beta/projects/simclr/configs/experiments/cifar_simclr_pretrain.yaml +++ b/official/vision/beta/projects/simclr/configs/experiments/cifar_simclr_pretrain.yaml @@ -72,7 +72,7 @@ trainer: type: 'cosine' cosine: initial_learning_rate: 0.6 # 0.3 × BatchSize / 256 - decay_steps: 43200 # train_steps - warmup_steps + decay_steps: 48000 warmup: type: 'linear' linear: diff --git a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_gpu.yaml b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_gpu.yaml index 06973e207f4..13b02cdf113 100644 --- a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_gpu.yaml +++ b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_gpu.yaml @@ -1,4 +1,4 @@ -# ImageNet classification. +# SimCLR Imagenet 10% finetuning. runtime: distribution_strategy: 'mirrored' mixed_precision_dtype: 'float16' @@ -55,7 +55,7 @@ trainer: train_steps: 12500 # 100 epochs validation_steps: 49 # NUM_EXAMPLES (50000) // global_batch_size validation_interval: 125 - steps_per_loop: 125 # NUM_EXAMPLES (1281167) // global_batch_size + steps_per_loop: 125 # NUM_EXAMPLES (128116) // global_batch_size summary_interval: 125 checkpoint_interval: 125 optimizer_config: diff --git a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_tpu.yaml b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_tpu.yaml new file mode 100644 index 00000000000..45cceb5fcd4 --- /dev/null +++ b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_tpu.yaml @@ -0,0 +1,70 @@ +# SimCLR Imagenet 10% finetuning. +runtime: + distribution_strategy: 'tpu' + mixed_precision_dtype: 'bfloat16' +task: + model: + mode: 'finetune' + input_size: [224, 224, 3] + backbone: + type: 'resnet' + resnet: + model_id: 50 + backbone_trainable: true + projection_head: + proj_output_dim: 128 + num_proj_layers: 3 + ft_proj_idx: 1 + supervised_head: + num_classes: 1001 + zero_init: true + norm_activation: + use_sync_bn: false + norm_momentum: 0.9 + norm_epsilon: 0.00001 + loss: + label_smoothing: 0.0 + one_hot: true + evaluation: + top_k: 5 + one_hot: true + init_checkpoint: gs://tf_model_garden/vision/simclr/r50_1x + init_checkpoint_modules: 'backbone_projection' + train_data: + tfds_name: 'imagenet2012_subset/10pct' + tfds_split: 'train' + input_path: '' + is_training: true + global_batch_size: 1024 + dtype: 'bfloat16' + parser: + mode: 'finetune' + validation_data: + tfds_name: 'imagenet2012_subset/10pct' + tfds_split: 'validation' + input_path: '' + is_training: false + global_batch_size: 1024 + dtype: 'bfloat16' + drop_remainder: false + parser: + mode: 'finetune' +trainer: + train_steps: 12500 # 100 epochs + validation_steps: 49 # NUM_EXAMPLES (50000) // global_batch_size + validation_interval: 125 + steps_per_loop: 125 # NUM_EXAMPLES (128116) // global_batch_size + summary_interval: 125 + checkpoint_interval: 125 + optimizer_config: + optimizer: + type: 'lars' + lars: + momentum: 0.9 + weight_decay_rate: 0.0 + exclude_from_weight_decay: ['batch_normalization', 'bias'] + learning_rate: + type: 'cosine' + cosine: + initial_learning_rate: 0.04 # 0.01 × BatchSize / 512 + decay_steps: 12500 # train_steps diff --git a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_gpu.yaml b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_gpu.yaml index 9e7c326c3d6..f2fa25ef8e7 100644 --- a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_gpu.yaml +++ b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_gpu.yaml @@ -1,4 +1,4 @@ -# ImageNet classification. +# SimCLR Imagenet pretraining. runtime: distribution_strategy: 'mirrored' mixed_precision_dtype: 'float16' @@ -49,12 +49,12 @@ task: decoder: decode_label: true trainer: - train_steps: 187200 # 300 epochs + train_steps: 500000 # 800 epochs validation_steps: 24 # NUM_EXAMPLES (50000) // global_batch_size - validation_interval: 624 - steps_per_loop: 624 # NUM_EXAMPLES (1281167) // global_batch_size - summary_interval: 624 - checkpoint_interval: 624 + validation_interval: 625 + steps_per_loop: 625 # NUM_EXAMPLES (1281167) // global_batch_size + summary_interval: 625 + checkpoint_interval: 625 optimizer_config: optimizer: type: 'lars' @@ -66,8 +66,8 @@ trainer: type: 'cosine' cosine: initial_learning_rate: 1.6 # 0.2 * BatchSize / 256 - decay_steps: 177840 # train_steps - warmup_steps + decay_steps: 500000 warmup: type: 'linear' linear: - warmup_steps: 9360 # 5% of total epochs + warmup_steps: 25000 # 5% of total epochs diff --git a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_tpu.yaml b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_tpu.yaml new file mode 100644 index 00000000000..f5c8045483b --- /dev/null +++ b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_tpu.yaml @@ -0,0 +1,71 @@ +# SimCLR Imagenet pretraining. +runtime: + distribution_strategy: 'tpu' + mixed_precision_dtype: 'bfloat16' +task: + model: + mode: 'pretrain' + input_size: [224, 224, 3] + backbone: + type: 'resnet' + resnet: + model_id: 50 + backbone_trainable: true + projection_head: + proj_output_dim: 128 + num_proj_layers: 3 + ft_proj_idx: 0 + supervised_head: + num_classes: 1001 + norm_activation: + use_sync_bn: true + norm_momentum: 0.9 + norm_epsilon: 0.00001 + loss: + projection_norm: true + temperature: 0.1 + evaluation: + top_k: 5 + one_hot: true + train_data: + input_path: '/readahead/200M/placer/prod/home/distbelief/imagenet-tensorflow/imagenet-2012-tfrecord/train*' + is_training: true + global_batch_size: 2048 + dtype: 'bfloat16' + parser: + mode: 'pretrain' + decoder: + decode_label: true + validation_data: + input_path: '/readahead/200M/placer/prod/home/distbelief/imagenet-tensorflow/imagenet-2012-tfrecord/valid*' + is_training: false + global_batch_size: 2048 + dtype: 'bfloat16' + drop_remainder: false + parser: + mode: 'pretrain' + decoder: + decode_label: true +trainer: + train_steps: 500000 # 800 epochs + validation_steps: 24 # NUM_EXAMPLES (50000) // global_batch_size + validation_interval: 625 + steps_per_loop: 625 # NUM_EXAMPLES (1281167) // global_batch_size + summary_interval: 625 + checkpoint_interval: 625 + optimizer_config: + optimizer: + type: 'lars' + lars: + momentum: 0.9 + weight_decay_rate: 0.000001 + exclude_from_weight_decay: ['batch_normalization', 'bias'] + learning_rate: + type: 'cosine' + cosine: + initial_learning_rate: 1.6 # 0.2 * BatchSize / 256 + decay_steps: 500000 + warmup: + type: 'linear' + linear: + warmup_steps: 25000 # 5% of total epochs From 4bf492a87083e84a1cd520cf49e1cbf279a839ec Mon Sep 17 00:00:00 2001 From: Ronny Votel Date: Fri, 4 Jun 2021 07:52:32 -0700 Subject: [PATCH 25/50] Updating the centernet mask target assigner. PiperOrigin-RevId: 377511299 --- .../object_detection/core/target_assigner.py | 48 +++++++++++++++++-- .../core/target_assigner_test.py | 37 +++++++++++--- .../center_net_meta_arch.py | 18 +++++-- 3 files changed, 88 insertions(+), 15 deletions(-) diff --git a/research/object_detection/core/target_assigner.py b/research/object_detection/core/target_assigner.py index e491bfcfb59..43b54e6327d 100644 --- a/research/object_detection/core/target_assigner.py +++ b/research/object_detection/core/target_assigner.py @@ -2001,8 +2001,8 @@ def __init__(self, stride): self._stride = stride def assign_segmentation_targets( - self, gt_masks_list, gt_classes_list, - mask_resize_method=ResizeMethod.BILINEAR): + self, gt_masks_list, gt_classes_list, gt_boxes_list=None, + gt_mask_weights_list=None, mask_resize_method=ResizeMethod.BILINEAR): """Computes the segmentation targets. This utility produces a semantic segmentation mask for each class, starting @@ -2016,15 +2016,25 @@ def assign_segmentation_targets( gt_classes_list: A list of float tensors with shape [num_boxes, num_classes] representing the one-hot encoded class labels for each box in the gt_boxes_list. + gt_boxes_list: An optional list of float tensors with shape [num_boxes, 4] + with normalized boxes corresponding to each mask. The boxes are used to + spatially allocate mask weights. + gt_mask_weights_list: An optional list of float tensors with shape + [num_boxes] with weights for each mask. If a mask has a zero weight, it + indicates that the box region associated with the mask should not + contribute to the loss. If not provided, will use a per-pixel weight of + 1. mask_resize_method: A `tf.compat.v2.image.ResizeMethod`. The method to use when resizing masks from input resolution to output resolution. + Returns: segmentation_targets: An int32 tensor of size [batch_size, output_height, output_width, num_classes] representing the class of each location in the output space. + segmentation_weight: A float32 tensor of size [batch_size, output_height, + output_width] indicating the loss weight to apply at each location. """ - # TODO(ronnyvotel): Handle groundtruth weights. _, num_classes = shape_utils.combined_static_and_dynamic_shape( gt_classes_list[0]) @@ -2033,8 +2043,35 @@ def assign_segmentation_targets( output_height = tf.maximum(input_height // self._stride, 1) output_width = tf.maximum(input_width // self._stride, 1) + if gt_boxes_list is None: + gt_boxes_list = [None] * len(gt_masks_list) + if gt_mask_weights_list is None: + gt_mask_weights_list = [None] * len(gt_masks_list) + segmentation_targets_list = [] - for gt_masks, gt_classes in zip(gt_masks_list, gt_classes_list): + segmentation_weights_list = [] + + for gt_boxes, gt_masks, gt_mask_weights, gt_classes in zip( + gt_boxes_list, gt_masks_list, gt_mask_weights_list, gt_classes_list): + + if gt_boxes is not None and gt_mask_weights is not None: + boxes = box_list.BoxList(gt_boxes) + # Convert the box coordinates to absolute output image dimension space. + boxes_absolute = box_list_ops.to_absolute_coordinates( + boxes, output_height, output_width) + + # Generate a segmentation weight that applies mask weights in object + # regions. + blackout = gt_mask_weights <= 0 + segmentation_weight_for_image = ( + ta_utils.blackout_pixel_weights_by_box_regions( + output_height, output_width, boxes_absolute.get(), blackout, + weights=gt_mask_weights)) + segmentation_weights_list.append(segmentation_weight_for_image) + else: + segmentation_weights_list.append(tf.ones((output_height, output_width), + dtype=tf.float32)) + gt_masks = _resize_masks(gt_masks, output_height, output_width, mask_resize_method) gt_masks = gt_masks[:, :, :, tf.newaxis] @@ -2047,7 +2084,8 @@ def assign_segmentation_targets( segmentation_targets_list.append(segmentations_for_image) segmentation_target = tf.stack(segmentation_targets_list, axis=0) - return segmentation_target + segmentation_weight = tf.stack(segmentation_weights_list, axis=0) + return segmentation_target, segmentation_weight class CenterNetDensePoseTargetAssigner(object): diff --git a/research/object_detection/core/target_assigner_test.py b/research/object_detection/core/target_assigner_test.py index 07e6889d320..510ebd9da99 100644 --- a/research/object_detection/core/target_assigner_test.py +++ b/research/object_detection/core/target_assigner_test.py @@ -2090,13 +2090,31 @@ def graph_fn(): tf.constant([[0., 1., 0.], [0., 1., 0.]], dtype=tf.float32) ] + gt_boxes_list = [ + # Example 0. + tf.constant([[0.0, 0.0, 0.5, 0.5], + [0.0, 0.5, 0.5, 1.0], + [0.0, 0.0, 1.0, 1.0]], dtype=tf.float32), + # Example 1. + tf.constant([[0.0, 0.0, 1.0, 1.0], + [0.5, 0.0, 1.0, 0.5]], dtype=tf.float32) + ] + gt_mask_weights_list = [ + # Example 0. + tf.constant([0.0, 1.0, 1.0], dtype=tf.float32), + # Example 1. + tf.constant([1.0, 1.0], dtype=tf.float32) + ] cn_assigner = targetassigner.CenterNetMaskTargetAssigner(stride=2) - segmentation_target = cn_assigner.assign_segmentation_targets( - gt_masks_list=gt_masks_list, - gt_classes_list=gt_classes_list, - mask_resize_method=targetassigner.ResizeMethod.NEAREST_NEIGHBOR) - return segmentation_target - segmentation_target = self.execute(graph_fn, []) + segmentation_target, segmentation_weight = ( + cn_assigner.assign_segmentation_targets( + gt_masks_list=gt_masks_list, + gt_classes_list=gt_classes_list, + gt_boxes_list=gt_boxes_list, + gt_mask_weights_list=gt_mask_weights_list, + mask_resize_method=targetassigner.ResizeMethod.NEAREST_NEIGHBOR)) + return segmentation_target, segmentation_weight + segmentation_target, segmentation_weight = self.execute(graph_fn, []) expected_seg_target = np.array([ # Example 0 [[class 0, class 1], [background, class 0]] @@ -2108,13 +2126,18 @@ def graph_fn(): ], dtype=np.float32) np.testing.assert_array_almost_equal( expected_seg_target, segmentation_target) + expected_seg_weight = np.array([ + [[0, 1], [1, 1]], + [[1, 1], [1, 1]]], dtype=np.float32) + np.testing.assert_array_almost_equal( + expected_seg_weight, segmentation_weight) def test_assign_segmentation_targets_no_objects(self): def graph_fn(): gt_masks_list = [tf.zeros((0, 5, 5))] gt_classes_list = [tf.zeros((0, 10))] cn_assigner = targetassigner.CenterNetMaskTargetAssigner(stride=1) - segmentation_target = cn_assigner.assign_segmentation_targets( + segmentation_target, _ = cn_assigner.assign_segmentation_targets( gt_masks_list=gt_masks_list, gt_classes_list=gt_classes_list, mask_resize_method=targetassigner.ResizeMethod.NEAREST_NEIGHBOR) diff --git a/research/object_detection/meta_architectures/center_net_meta_arch.py b/research/object_detection/meta_architectures/center_net_meta_arch.py index 79dc7186f22..7c6014059e2 100644 --- a/research/object_detection/meta_architectures/center_net_meta_arch.py +++ b/research/object_detection/meta_architectures/center_net_meta_arch.py @@ -2979,20 +2979,32 @@ def _compute_mask_loss(self, segmentation_predictions, Returns: A float scalar tensor representing the mask loss. """ + gt_boxes_list = self.groundtruth_lists(fields.BoxListFields.boxes) gt_masks_list = self.groundtruth_lists(fields.BoxListFields.masks) + gt_mask_weights_list = None + if self.groundtruth_has_field(fields.BoxListFields.mask_weights): + gt_mask_weights_list = self.groundtruth_lists( + fields.BoxListFields.mask_weights) gt_classes_list = self.groundtruth_lists(fields.BoxListFields.classes) # Convert the groundtruth to targets. assigner = self._target_assigner_dict[SEGMENTATION_TASK] - heatmap_targets = assigner.assign_segmentation_targets( + heatmap_targets, heatmap_weight = assigner.assign_segmentation_targets( gt_masks_list=gt_masks_list, - gt_classes_list=gt_classes_list) + gt_classes_list=gt_classes_list, + gt_boxes_list=gt_boxes_list, + gt_mask_weights_list=gt_mask_weights_list) flattened_heatmap_targets = _flatten_spatial_dimensions(heatmap_targets) + flattened_heatmap_mask = _flatten_spatial_dimensions( + heatmap_weight[:, :, :, tf.newaxis]) + per_pixel_weights *= flattened_heatmap_mask loss = 0.0 mask_loss_fn = self._mask_params.classification_loss - total_pixels_in_loss = tf.reduce_sum(per_pixel_weights) + + total_pixels_in_loss = tf.math.maximum( + tf.reduce_sum(per_pixel_weights), 1) # Loop through each feature output head. for pred in segmentation_predictions: From e9f041c09dfdf82c12bc8cd28986fbd20a370776 Mon Sep 17 00:00:00 2001 From: Dan Kondratyuk Date: Fri, 4 Jun 2021 12:16:15 -0700 Subject: [PATCH 26/50] Improve MoViNet stream interface, fix state propagation. PiperOrigin-RevId: 377562613 --- .../vision/beta/modeling/layers/nn_layers.py | 112 +++++--- .../beta/modeling/layers/nn_layers_test.py | 31 ++- .../vision/beta/projects/movinet/README.md | 187 +++++++++++-- .../beta/projects/movinet/configs/movinet.py | 1 + .../beta/projects/movinet/modeling/movinet.py | 247 +++++++++++++++--- .../movinet/modeling/movinet_layers.py | 161 ++++++------ .../movinet/modeling/movinet_layers_test.py | 4 - .../movinet/modeling/movinet_model.py | 103 ++++++-- .../movinet/modeling/movinet_model_test.py | 77 +++++- .../projects/movinet/modeling/movinet_test.py | 41 +-- 10 files changed, 727 insertions(+), 237 deletions(-) diff --git a/official/vision/beta/modeling/layers/nn_layers.py b/official/vision/beta/modeling/layers/nn_layers.py index 60759ab019c..96ef9005ae4 100644 --- a/official/vision/beta/modeling/layers/nn_layers.py +++ b/official/vision/beta/modeling/layers/nn_layers.py @@ -281,9 +281,6 @@ class Scale(tf.keras.layers.Layer): This is useful for applying ReZero to layers, which improves convergence speed. This implements the paper: - - Thomas Bachlechner, Bodhisattwa Prasad Majumder, Huanru Henry Mao, - Garrison W. Cottrell, Julian McAuley. ReZero is All You Need: Fast Convergence at Large Depth. (https://arxiv.org/pdf/2003.04887.pdf). """ @@ -371,6 +368,7 @@ class PositionalEncoding(tf.keras.layers.Layer): def __init__(self, initializer: tf.keras.initializers.Initializer = 'zeros', cache_encoding: bool = False, + state_prefix: Optional[str] = None, **kwargs): """Initializes positional encoding. @@ -380,6 +378,7 @@ def __init__(self, after calling build. Otherwise, rebuild the tensor for every call. Setting this to False can be useful when we want to input a variable number of frames, so the positional encoding tensor can change shape. + state_prefix: a prefix string to identify states. **kwargs: Additional keyword arguments to be passed to this layer. Returns: @@ -390,33 +389,43 @@ def __init__(self, self._cache_encoding = cache_encoding self._pos_encoding = None self._rezero = Scale(initializer=initializer, name='rezero') + state_prefix = state_prefix if state_prefix is not None else '' + self._state_prefix = state_prefix + self._frame_count_name = f'{state_prefix}/pos_enc_frame_count' def get_config(self): """Returns a dictionary containing the config used for initialization.""" config = { 'initializer': self._initializer, 'cache_encoding': self._cache_encoding, + 'state_prefix': self._state_prefix, } base_config = super(PositionalEncoding, self).get_config() return dict(list(base_config.items()) + list(config.items())) def _positional_encoding(self, - num_positions: int, - hidden_size: int, - dtype: tf.DType = tf.float32): + num_positions: Union[int, tf.Tensor], + hidden_size: Union[int, tf.Tensor], + start_position: Union[int, tf.Tensor] = 0, + dtype: str = 'float32') -> tf.Tensor: """Creates a sequence of sinusoidal positional encoding vectors. Args: - num_positions: An `int` of number of positions (frames). - hidden_size: An `int` of number of channels used for the hidden vectors. - dtype: The dtype of the output tensor. + num_positions: the total number of positions (frames). + hidden_size: the number of channels used for the hidden vectors. + start_position: the start position. + dtype: the dtype of the output tensor. Returns: The positional encoding tensor with shape [num_positions, hidden_size]. """ + if isinstance(start_position, tf.Tensor) and start_position.shape.rank == 1: + start_position = start_position[0] + # Calling `tf.range` with `dtype=tf.bfloat16` results in an error, # so we cast afterward. - positions = tf.cast(tf.range(num_positions)[:, tf.newaxis], dtype) + positions = tf.range(start_position, start_position + num_positions) + positions = tf.cast(positions, dtype)[:, tf.newaxis] idx = tf.range(hidden_size)[tf.newaxis, :] power = tf.cast(2 * (idx // 2), dtype) @@ -430,11 +439,24 @@ def _positional_encoding(self, return pos_encoding - def _get_pos_encoding(self, input_shape): - """Calculates the positional encoding from the input shape.""" + def _get_pos_encoding(self, + input_shape: tf.Tensor, + frame_count: int = 0) -> tf.Tensor: + """Calculates the positional encoding from the input shape. + + Args: + input_shape: the shape of the input. + frame_count: a count of frames that indicates the index of the first + frame. + + Returns: + The positional encoding tensor with shape [num_positions, hidden_size]. + + """ frames = input_shape[1] channels = input_shape[-1] - pos_encoding = self._positional_encoding(frames, channels, dtype=self.dtype) + pos_encoding = self._positional_encoding( + frames, channels, start_position=frame_count, dtype=self.dtype) pos_encoding = tf.reshape(pos_encoding, [1, frames, 1, 1, channels]) return pos_encoding @@ -455,16 +477,46 @@ def build(self, input_shape): super(PositionalEncoding, self).build(input_shape) - def call(self, inputs): - """Calls the layer with the given inputs.""" + def call( + self, + inputs: tf.Tensor, + states: Optional[States] = None, + output_states: bool = True, + ) -> Union[tf.Tensor, Tuple[tf.Tensor, States]]: + """Calls the layer with the given inputs. + + Args: + inputs: An input `tf.Tensor`. + states: A `dict` of states such that, if any of the keys match for this + layer, will overwrite the contents of the buffer(s). Expected keys + include `state_prefix + '/pos_enc_frame_count'`. + output_states: A `bool`. If True, returns the output tensor and output + states. Returns just the output tensor otherwise. + + Returns: + An output `tf.Tensor` (and optionally the states if `output_states=True`). + + Raises: + ValueError: If using 'channels_first' data format. + """ + states = dict(states) if states is not None else {} + + # Keep a count of frames encountered across input iterations in + # num_frames to be able to accurately update the positional encoding. + num_frames = tf.shape(inputs)[1] + frame_count = tf.cast(states.get(self._frame_count_name, [0]), tf.int32) + states[self._frame_count_name] = frame_count + num_frames + if self._cache_encoding: pos_encoding = self._pos_encoding else: - pos_encoding = self._get_pos_encoding(tf.shape(inputs)) + pos_encoding = self._get_pos_encoding( + tf.shape(inputs), frame_count=frame_count) pos_encoding = tf.cast(pos_encoding, inputs.dtype) - pos_encoding = tf.stop_gradient(pos_encoding) pos_encoding = self._rezero(pos_encoding) - return inputs + pos_encoding + outputs = inputs + pos_encoding + + return (outputs, states) if output_states else outputs @tf.keras.utils.register_keras_serializable(package='Vision') @@ -480,6 +532,7 @@ class GlobalAveragePool3D(tf.keras.layers.Layer): def __init__(self, keepdims: bool = False, causal: bool = False, + state_prefix: Optional[str] = None, **kwargs): """Initializes a global average pool layer. @@ -487,6 +540,7 @@ def __init__(self, keepdims: A `bool`. If True, keep the averaged dimensions. causal: A `bool` of whether to run in causal mode with a cumulative sum across frames. + state_prefix: a prefix string to identify states. **kwargs: Additional keyword arguments to be passed to this layer. Returns: @@ -496,29 +550,22 @@ def __init__(self, self._keepdims = keepdims self._causal = causal + state_prefix = state_prefix if state_prefix is not None else '' + self._state_prefix = state_prefix - self._frame_count = None + self._state_name = f'{state_prefix}/pool_buffer' + self._frame_count_name = f'{state_prefix}/pool_frame_count' def get_config(self): """Returns a dictionary containing the config used for initialization.""" config = { 'keepdims': self._keepdims, 'causal': self._causal, + 'state_prefix': self._state_prefix, } base_config = super(GlobalAveragePool3D, self).get_config() return dict(list(base_config.items()) + list(config.items())) - def build(self, input_shape): - """Builds the layer with the given input shape.""" - # Here we define strings that will uniquely reference the buffer states - # in the TF graph. These will be used for passing in a mapping of states - # for streaming mode. To do this, we can use a name scope. - with tf.name_scope('buffer') as state_name: - self._state_name = state_name - self._frame_count_name = state_name + '_frame_count' - - super(GlobalAveragePool3D, self).build(input_shape) - def call(self, inputs: tf.Tensor, states: Optional[States] = None, @@ -530,6 +577,8 @@ def call(self, inputs: An input `tf.Tensor`. states: A `dict` of states such that, if any of the keys match for this layer, will overwrite the contents of the buffer(s). + Expected keys include `state_prefix + '/pool_buffer'` and + `state_prefix + '/pool_frame_count'`. output_states: A `bool`. If True, returns the output tensor and output states. Returns just the output tensor otherwise. @@ -561,7 +610,8 @@ def call(self, # num_frames to be able to accurately take a cumulative average across # all frames when running in streaming mode num_frames = tf.shape(inputs)[1] - frame_count = states.get(self._frame_count_name, 0) + frame_count = states.get(self._frame_count_name, tf.constant([0])) + frame_count = tf.cast(frame_count, tf.int32) states[self._frame_count_name] = frame_count + num_frames if self._causal: diff --git a/official/vision/beta/modeling/layers/nn_layers_test.py b/official/vision/beta/modeling/layers/nn_layers_test.py index 50af2b10057..979355bcfe4 100644 --- a/official/vision/beta/modeling/layers/nn_layers_test.py +++ b/official/vision/beta/modeling/layers/nn_layers_test.py @@ -48,8 +48,8 @@ def test_positional_encoding(self): initializer='ones', cache_encoding=True) inputs = tf.ones([1, 4, 1, 1, 3]) - outputs = pos_encoding(inputs) - outputs_cached = pos_encoding_cached(inputs) + outputs, _ = pos_encoding(inputs) + outputs_cached, _ = pos_encoding_cached(inputs) expected = tf.constant( [[[[[1.0000000, 1.0000000, 2.0000000]]], @@ -70,7 +70,7 @@ def test_positional_encoding_bfloat16(self): pos_encoding = nn_layers.PositionalEncoding(initializer='ones') inputs = tf.ones([1, 4, 1, 1, 3], dtype=tf.bfloat16) - outputs = pos_encoding(inputs) + outputs, _ = pos_encoding(inputs) expected = tf.constant( [[[[[1.0000000, 1.0000000, 2.0000000]]], @@ -92,6 +92,31 @@ def test_global_average_pool_basic(self): self.assertEqual(outputs.shape, expected.shape) self.assertAllEqual(outputs, expected) + def test_positional_encoding_stream(self): + pos_encoding = nn_layers.PositionalEncoding( + initializer='ones', cache_encoding=False) + + inputs = tf.range(4, dtype=tf.float32) + 1. + inputs = tf.reshape(inputs, [1, 4, 1, 1, 1]) + inputs = tf.tile(inputs, [1, 1, 1, 1, 3]) + expected, _ = pos_encoding(inputs) + + for num_splits in [1, 2, 4]: + frames = tf.split(inputs, num_splits, axis=1) + states = {} + predicted = [] + for frame in frames: + output, states = pos_encoding(frame, states=states) + predicted.append(output) + predicted = tf.concat(predicted, axis=1) + + self.assertEqual(predicted.shape, expected.shape) + self.assertAllClose(predicted, expected) + self.assertAllClose(predicted, [[[[[1.0000000, 1.0000000, 2.0000000]]], + [[[2.8414710, 2.0021544, 2.5403023]]], + [[[3.9092975, 3.0043090, 2.5838532]]], + [[[4.1411200, 4.0064630, 3.0100074]]]]]) + def test_global_average_pool_keras(self): pool = nn_layers.GlobalAveragePool3D(keepdims=False) keras_pool = tf.keras.layers.GlobalAveragePooling3D() diff --git a/official/vision/beta/projects/movinet/README.md b/official/vision/beta/projects/movinet/README.md index 804dd5491df..5ccf1d3e838 100644 --- a/official/vision/beta/projects/movinet/README.md +++ b/official/vision/beta/projects/movinet/README.md @@ -8,16 +8,27 @@ This repository is the official implementation of [MoViNets: Mobile Video Networks for Efficient Video Recognition](https://arxiv.org/abs/2103.11511). +

+ +

+ ## Description Mobile Video Networks (MoViNets) are efficient video classification models runnable on mobile devices. MoViNets demonstrate state-of-the-art accuracy and efficiency on several large-scale video action recognition datasets. +On [Kinetics 600](https://deepmind.com/research/open-source/kinetics), +MoViNet-A6 achieves 84.8% top-1 accuracy, outperforming recent +Vision Transformer models like [ViViT](https://arxiv.org/abs/2103.15691) (83.0%) +and [VATT](https://arxiv.org/abs/2104.11178) (83.6%) without any additional +training data, while using 10x fewer FLOPs. And streaming MoViNet-A0 achieves +72% accuracy while using 3x fewer FLOPs than MobileNetV3-large (68%). + There is a large gap between video model performance of accurate models and efficient models for video action recognition. On the one hand, 2D MobileNet CNNs are fast and can operate on streaming video in real time, but are prone to -be noisy and are inaccurate. On the other hand, 3D CNNs are accurate, but are +be noisy and inaccurate. On the other hand, 3D CNNs are accurate, but are memory and computation intensive and cannot operate on streaming video. MoViNets bridge this gap, producing: @@ -28,19 +39,22 @@ to A6). usage. - Temporal ensembles of models to boost efficiency even higher. -Small MoViNets demonstrate higher efficiency and accuracy than MobileNetV3 for -video action recognition (Kinetics 600). +MoViNets also improve computational efficiency by outputting high-quality +predictions frame by frame, as opposed to the traditional multi-clip evaluation +approach that performs redundant computation and limits temporal scope. -MoViNets also improve efficiency by outputting high-quality predictions with a -single frame, as opposed to the traditional multi-clip evaluation approach. +

+ +

-[![Multi-Clip Eval](https://storage.googleapis.com/tf_model_garden/vision/movinet/artifacts/movinet_multi_clip_eval.png)](https://arxiv.org/pdf/2103.11511.pdf) - -[![Streaming Eval](https://storage.googleapis.com/tf_model_garden/vision/movinet/artifacts/movinet_stream_eval.png)](https://arxiv.org/pdf/2103.11511.pdf) +

+ +

## History -- Initial Commit. +- **2021-05-30** Add streaming MoViNet checkpoints and examples. +- **2021-05-11** Initial Commit. ## Authors and Maintainers @@ -53,6 +67,7 @@ single frame, as opposed to the traditional multi-clip evaluation approach. - [Requirements](#requirements) - [Results and Pretrained Weights](#results-and-pretrained-weights) - [Kinetics 600](#kinetics-600) +- [Prediction Examples](#prediction-examples) - [Training and Evaluation](#training-and-evaluation) - [References](#references) - [License](#license) @@ -76,33 +91,154 @@ pip install -r requirements.txt ### Kinetics 600 -[![MoViNet Comparison](https://storage.googleapis.com/tf_model_garden/vision/movinet/artifacts/movinet_comparison.png)](https://arxiv.org/pdf/2103.11511.pdf) +

+ +

[tensorboard.dev summary](https://tensorboard.dev/experiment/Q07RQUlVRWOY4yDw3SnSkA/) of training runs across all models. -The table below summarizes the performance of each model and provides links to -download pretrained models. All models are evaluated on single clips with the -same resolution as training. +The table below summarizes the performance of each model on +[Kinetics 600](https://deepmind.com/research/open-source/kinetics) +and provides links to download pretrained models. All models are evaluated on +single clips with the same resolution as training. + +Note: MoViNet-A6 can be constructed as an ensemble of MoViNet-A4 and +MoViNet-A5. -Streaming MoViNets will be added in the future. +#### Base Models -| Model Name | Top-1 Accuracy | Top-5 Accuracy | GFLOPs\* | Checkpoint | TF Hub SavedModel | -|------------|----------------|----------------|----------|------------|-------------------| -| MoViNet-A0-Base | 71.41 | 90.91 | 2.7 | [checkpoint (12 MiB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a0_base.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a0/base/kinetics-600/classification/) | -| MoViNet-A1-Base | 76.01 | 93.28 | 6.0 | [checkpoint (18 MiB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a1_base.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a1/base/kinetics-600/classification/) | -| MoViNet-A2-Base | 78.03 | 93.99 | 10 | [checkpoint (20 MiB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a2_base.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a2/base/kinetics-600/classification/) | -| MoViNet-A3-Base | 81.22 | 95.35 | 57 | [checkpoint (29 MiB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a3_base.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a3/base/kinetics-600/classification/) | -| MoViNet-A4-Base | 82.96 | 95.98 | 110 | [checkpoint (44 MiB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a4_base.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a4/base/kinetics-600/classification/) | -| MoViNet-A5-Base | 84.22 | 96.36 | 280 | [checkpoint (72 MiB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a5_base.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a5/base/kinetics-600/classification/) | +Base models implement standard 3D convolutions without stream buffers. + +| Model Name | Top-1 Accuracy | Top-5 Accuracy | Input Shape | GFLOPs\* | Chekpoint | TF Hub SavedModel | +|------------|----------------|----------------|-------------|----------|-----------|-------------------| +| MoViNet-A0-Base | 72.28 | 90.92 | 50 x 172 x 172 | 2.7 | [checkpoint (12 MB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a0_base.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a0/base/kinetics-600/classification/) | +| MoViNet-A1-Base | 76.69 | 93.40 | 50 x 172 x 172 | 6.0 | [checkpoint (18 MB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a1_base.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a1/base/kinetics-600/classification/) | +| MoViNet-A2-Base | 78.62 | 94.17 | 50 x 224 x 224 | 10 | [checkpoint (20 MB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a2_base.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a2/base/kinetics-600/classification/) | +| MoViNet-A3-Base | 81.79 | 95.67 | 120 x 256 x 256 | 57 | [checkpoint (29 MB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a3_base.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a3/base/kinetics-600/classification/) | +| MoViNet-A4-Base | 83.48 | 96.16 | 80 x 290 x 290 | 110 | [checkpoint (44 MB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a4_base.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a4/base/kinetics-600/classification/) | +| MoViNet-A5-Base | 84.27 | 96.39 | 120 x 320 x 320 | 280 | [checkpoint (72 MB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a5_base.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a5/base/kinetics-600/classification/) | \*GFLOPs per video on Kinetics 600. -## Training and Evaluation +#### Streaming Models + +Streaming models implement causal 3D convolutions with stream buffers. + +| Model Name | Top-1 Accuracy | Top-5 Accuracy | Input Shape\* | GFLOPs\*\* | Chekpoint | TF Hub SavedModel | +|------------|----------------|----------------|---------------|------------|-----------|-------------------| +| MoViNet-A0-Stream | 72.05 | 90.63 | 50 x 172 x 172 | 2.7 | [checkpoint (12 MB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a0_stream.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a0/stream/kinetics-600/classification/) | +| MoViNet-A1-Stream | 76.45 | 93.25 | 50 x 172 x 172 | 6.0 | [checkpoint (18 MB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a1_stream.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a1/stream/kinetics-600/classification/) | +| MoViNet-A2-Stream | 78.40 | 94.05 | 50 x 224 x 224 | 10 | [checkpoint (20 MB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a2_stream.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a2/stream/kinetics-600/classification/) | +| MoViNet-A3-Stream | 80.09 | 94.84 | 120 x 256 x 256 | 57 | [checkpoint (29 MB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a3_stream.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a3/stream/kinetics-600/classification/) | +| MoViNet-A4-Stream | 81.49 | 95.66 | 80 x 290 x 290 | 110 | [checkpoint (44 MB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a4_stream.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a4/stream/kinetics-600/classification/) | +| MoViNet-A5-Stream | 82.37 | 95.79 | 120 x 320 x 320 | 280 | [checkpoint (72 MB)](https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a5_stream.tar.gz) | [tfhub](https://tfhub.dev/tensorflow/movinet/a5/stream/kinetics-600/classification/) | + +\*In streaming mode, the number of frames correspond to the total accumulated +duration of the 10-second clip. + +\*\*GFLOPs per video on Kinetics 600. + +## Prediction Examples Please check out our [Colab Notebook](https://colab.research.google.com/github/tensorflow/models/tree/master/official/vision/beta/projects/movinet/movinet_tutorial.ipynb) to get started with MoViNets. +This section provides examples on how to run prediction. + +For base models, run the following: + +```python +import tensorflow as tf + +from official.vision.beta.projects.movinet.modeling import movinet +from official.vision.beta.projects.movinet.modeling import movinet_model + +# Create backbone and model. +backbone = movinet.Movinet( + model_id='a0', + causal=True, + use_external_states=True, +) +model = movinet_model.MovinetClassifier( + backbone, num_classes=600, output_states=True) + +# Create your example input here. +# Refer to the paper for recommended input shapes. +inputs = tf.ones([1, 8, 172, 172, 3]) + +# [Optional] Build the model and load a pretrained checkpoint +model.build(inputs.shape) + +checkpoint_dir = '/path/to/checkpoint' +checkpoint_path = tf.train.latest_checkpoint(checkpoint_dir) +checkpoint = tf.train.Checkpoint(model=model) +status = checkpoint.restore(checkpoint_path) +status.assert_existing_objects_matched() + +# Run the model prediction. +output = model(inputs) +prediction = tf.argmax(output, -1) +``` + +For streaming models, run the following: + +```python +import tensorflow as tf + +from official.vision.beta.projects.movinet.modeling import movinet +from official.vision.beta.projects.movinet.modeling import movinet_model + +# Create backbone and model. +backbone = movinet.Movinet( + model_id='a0', + causal=True, + use_external_states=True, +) +model = movinet_model.MovinetClassifier( + backbone, num_classes=600, output_states=True) + +# Create your example input here. +# Refer to the paper for recommended input shapes. +inputs = tf.ones([1, 8, 172, 172, 3]) + +# [Optional] Build the model and load a pretrained checkpoint +model.build(inputs.shape) + +checkpoint_dir = '/path/to/checkpoint' +checkpoint_path = tf.train.latest_checkpoint(checkpoint_dir) +checkpoint = tf.train.Checkpoint(model=model) +status = checkpoint.restore(checkpoint_path) +status.assert_existing_objects_matched() + +# Split the video into individual frames. +# Note: we can also split into larger clips as well (e.g., 8-frame clips). +# Running on larger clips will slightly reduce latency overhead, but +# will consume more memory. +frames = tf.split(inputs, inputs.shape[1], axis=1) + +# Initialize the dict of states. All state tensors are initially zeros. +init_states = model.init_states(tf.shape(inputs)) + +# Run the model prediction by looping over each frame. +states = init_states +predictions = [] +for frame in frames: + output, states = model({**states, 'image': frame}) + predictions.append(output) + +# The video classification will simply be the last output of the model. +final_prediction = tf.argmax(predictions[-1], -1) + +# Alternatively, we can run the network on the entire input video. +# The output should be effectively the same +# (but it may differ a small amount due to floating point errors). +non_streaming_output, _ = model({**init_states, 'image': inputs}) +non_streaming_prediction = tf.argmax(non_streaming_output, -1) +``` + +## Training and Evaluation + Run this command line for continuous training and evaluation. ```shell @@ -137,11 +273,6 @@ python3 official/vision/beta/projects/movinet/train.py \ --tf_data_service="" ``` -## References - -- [Kinetics Datasets](https://deepmind.com/research/open-source/kinetics) -- [MoViNets (Mobile Video Networks)](https://arxiv.org/abs/2103.11511) - ## License [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) diff --git a/official/vision/beta/projects/movinet/configs/movinet.py b/official/vision/beta/projects/movinet/configs/movinet.py index 2ed59595b42..97cbef09036 100644 --- a/official/vision/beta/projects/movinet/configs/movinet.py +++ b/official/vision/beta/projects/movinet/configs/movinet.py @@ -45,6 +45,7 @@ class Movinet(hyperparams.Config): # 3d_2plus1d: (2+1)D convolution with Conv3D (no 2D reshaping) conv_type: str = '3d' stochastic_depth_drop_rate: float = 0.2 + use_external_states: bool = False @dataclasses.dataclass diff --git a/official/vision/beta/projects/movinet/modeling/movinet.py b/official/vision/beta/projects/movinet/modeling/movinet.py index beb9e021022..ddde7c1a416 100644 --- a/official/vision/beta/projects/movinet/modeling/movinet.py +++ b/official/vision/beta/projects/movinet/modeling/movinet.py @@ -17,7 +17,8 @@ Reference: https://arxiv.org/pdf/2103.11511.pdf """ -from typing import Optional, Sequence, Tuple +import math +from typing import Dict, Mapping, Optional, Sequence, Tuple, Union import dataclasses import tensorflow as tf @@ -71,8 +72,6 @@ class HeadSpec(BlockSpec): """Configuration of a Movinet block.""" project_filters: int = 0 head_filters: int = 0 - output_per_frame: bool = False - max_pool_predictions: bool = False # Block specs specify the architecture of each model @@ -317,6 +316,7 @@ def __init__(self, kernel_regularizer: Optional[str] = None, bias_regularizer: Optional[str] = None, stochastic_depth_drop_rate: float = 0., + use_external_states: bool = False, **kwargs): """MoViNet initialization function. @@ -344,6 +344,8 @@ def __init__(self, bias_regularizer: tf.keras.regularizers.Regularizer object for Conv2d. Defaults to None. stochastic_depth_drop_rate: the base rate for stochastic depth. + use_external_states: if True, expects states to be passed as additional + input. **kwargs: keyword arguments to be passed. """ block_specs = BLOCK_SPECS[model_id] @@ -371,7 +373,10 @@ def __init__(self, self._kernel_regularizer = kernel_regularizer self._bias_regularizer = bias_regularizer self._stochastic_depth_drop_rate = stochastic_depth_drop_rate + self._use_external_states = use_external_states + if self._use_external_states and not self._causal: + raise ValueError('External states should be used with causal mode.') if not isinstance(block_specs[0], StemSpec): raise ValueError( 'Expected first spec to be StemSpec, got {}'.format(block_specs[0])) @@ -380,22 +385,55 @@ def __init__(self, 'Expected final spec to be HeadSpec, got {}'.format(block_specs[-1])) self._head_filters = block_specs[-1].head_filters - if tf.keras.backend.image_data_format() == 'channels_last': - bn_axis = -1 - else: - bn_axis = 1 + state_specs = None + if use_external_states: + self._set_dtype_policy(input_specs.dtype) + state_specs = self.initial_state_specs(input_specs.shape) - # Build MoViNet backbone. - inputs = tf.keras.Input(shape=input_specs.shape[1:], name='inputs') + inputs, outputs = self._build_network(input_specs, state_specs=state_specs) - x = inputs - states = {} + super(Movinet, self).__init__(inputs=inputs, outputs=outputs, **kwargs) + + self._state_specs = state_specs + + def _build_network( + self, + input_specs: tf.keras.layers.InputSpec, + state_specs: Optional[Mapping[str, tf.keras.layers.InputSpec]] = None, + ) -> Tuple[Mapping[str, tf.keras.Input], Tuple[Mapping[str, tf.Tensor], + Mapping[str, tf.Tensor]]]: + """Builds the model network. + + Args: + input_specs: the model input spec to use. + state_specs: a dict mapping a state name to the corresponding state spec. + State names should match with the `state` input/output dict. + + Returns: + Inputs and outputs as a tuple. Inputs are expected to be a dict with + base input and states. Outputs are expected to be a dict of endpoints + and output states. + """ + state_specs = state_specs if state_specs is not None else {} + + image_input = tf.keras.Input(shape=input_specs.shape[1:], name='inputs') + + states = { + name: tf.keras.Input(shape=spec.shape[1:], dtype=spec.dtype, name=name) + for name, spec in state_specs.items() + } + + inputs = {**states, 'image': image_input} endpoints = {} - num_layers = sum(len(block.expand_filters) for block in block_specs - if isinstance(block, MovinetBlockSpec)) + x = image_input + + num_layers = sum( + len(block.expand_filters) + for block in self._block_specs + if isinstance(block, MovinetBlockSpec)) stochastic_depth_idx = 1 - for block_idx, block in enumerate(block_specs): + for block_idx, block in enumerate(self._block_specs): if isinstance(block, StemSpec): x, states = movinet_layers.Stem( block.filters, @@ -404,12 +442,14 @@ def __init__(self, conv_type=self._conv_type, causal=self._causal, activation=self._activation, - kernel_initializer=kernel_initializer, - kernel_regularizer=kernel_regularizer, + kernel_initializer=self._kernel_initializer, + kernel_regularizer=self._kernel_regularizer, batch_norm_layer=self._norm, batch_norm_momentum=self._norm_momentum, batch_norm_epsilon=self._norm_epsilon, - name='stem')(x, states=states) + state_prefix='state/stem', + name='stem')( + x, states=states) endpoints['stem'] = x elif isinstance(block, MovinetBlockSpec): if not (len(block.expand_filters) == len(block.kernel_sizes) == @@ -437,14 +477,16 @@ def __init__(self, activation=self._activation, stochastic_depth_drop_rate=stochastic_depth_drop_rate, conv_type=self._conv_type, - use_positional_encoding= - self._use_positional_encoding and self._causal, - kernel_initializer=kernel_initializer, - kernel_regularizer=kernel_regularizer, + use_positional_encoding=self._use_positional_encoding and + self._causal, + kernel_initializer=self._kernel_initializer, + kernel_regularizer=self._kernel_regularizer, batch_norm_layer=self._norm, batch_norm_momentum=self._norm_momentum, batch_norm_epsilon=self._norm_epsilon, - name=name)(x, states=states) + state_prefix=f'state/{name}', + name=name)( + x, states=states) endpoints[name] = x stochastic_depth_idx += 1 elif isinstance(block, HeadSpec): @@ -452,27 +494,154 @@ def __init__(self, project_filters=block.project_filters, conv_type=self._conv_type, activation=self._activation, - kernel_initializer=kernel_initializer, - kernel_regularizer=kernel_regularizer, + kernel_initializer=self._kernel_initializer, + kernel_regularizer=self._kernel_regularizer, batch_norm_layer=self._norm, batch_norm_momentum=self._norm_momentum, - batch_norm_epsilon=self._norm_epsilon)(x, states=states) + batch_norm_epsilon=self._norm_epsilon, + state_prefix='state/head', + name='head')( + x, states=states) endpoints['head'] = x else: raise ValueError('Unknown block type {}'.format(block)) - self._output_specs = {l: endpoints[l].get_shape() for l in endpoints} + outputs = (endpoints, states) + + return inputs, outputs + + def _get_initial_state_shapes( + self, + block_specs: Sequence[BlockSpec], + input_shape: Union[Sequence[int], tf.Tensor], + use_positional_encoding: bool = False) -> Dict[str, Sequence[int]]: + """Generates names and shapes for all input states. + + Args: + block_specs: sequence of specs used for creating a model. + input_shape: the expected 5D shape of the image input. + use_positional_encoding: whether the model will use positional encoding. + + Returns: + A dict mapping state names to state shapes. + """ + + def divide_resolution(shape, num_downsamples): + """Downsamples the dimension to calculate strided convolution shape.""" + if shape is None: + return None + if isinstance(shape, tf.Tensor): + # Avoid using div and ceil to support tf lite + shape = tf.cast(shape, tf.float32) + resolution_divisor = 2 ** num_downsamples + resolution_multiplier = 0.5 ** num_downsamples + shape = ((shape + resolution_divisor - 1) * resolution_multiplier) + return tf.cast(shape, tf.int32) + else: + resolution_divisor = 2 ** num_downsamples + return math.ceil(shape / resolution_divisor) + + states = {} + num_downsamples = 0 + + for block_idx, block in enumerate(block_specs): + if isinstance(block, StemSpec): + if block.kernel_size[0] > 1: + states['state/stem/stream_buffer'] = ( + input_shape[0], + input_shape[1], + divide_resolution(input_shape[2], num_downsamples), + divide_resolution(input_shape[3], num_downsamples), + block.filters, + ) + num_downsamples += 1 + elif isinstance(block, MovinetBlockSpec): + block_idx -= 1 + params = list(zip( + block.expand_filters, + block.kernel_sizes, + block.strides)) + for layer_idx, layer in enumerate(params): + expand_filters, kernel_size, strides = layer - inputs = { - 'image': inputs, - 'states': { - name: tf.keras.Input(shape=state.shape[1:], name=f'states/{name}') - for name, state in states.items() - }, + if kernel_size[0] > 1: + states[f'state/b{block_idx}/l{layer_idx}/stream_buffer'] = ( + input_shape[0], + kernel_size[0] - 1, + divide_resolution(input_shape[2], num_downsamples), + divide_resolution(input_shape[3], num_downsamples), + expand_filters, + ) + + states[f'state/b{block_idx}/l{layer_idx}/pool_buffer'] = ( + input_shape[0], 1, 1, 1, expand_filters, + ) + states[f'state/b{block_idx}/l{layer_idx}/pool_frame_count'] = (1,) + + if use_positional_encoding: + name = f'state/b{block_idx}/l{layer_idx}/pos_enc_frame_count' + states[name] = (1,) + + if strides[1] != strides[2]: + raise ValueError('Strides must match in the spatial dimensions, ' + 'got {}'.format(strides)) + if strides[1] != 1 or strides[2] != 1: + num_downsamples += 1 + elif isinstance(block, HeadSpec): + states['state/head/pool_buffer'] = ( + input_shape[0], 1, 1, 1, block.project_filters, + ) + states['state/head/pool_frame_count'] = (1,) + + return states + + def _get_state_dtype(self, name: str) -> str: + """Returns the dtype associated with a state.""" + if 'frame_count' in name: + return 'int32' + return self.dtype + + def initial_state_specs( + self, input_shape: Sequence[int]) -> Dict[str, tf.keras.layers.InputSpec]: + """Creates a mapping of state name to InputSpec from the input shape.""" + state_shapes = self._get_initial_state_shapes( + self._block_specs, + input_shape, + use_positional_encoding=self._use_positional_encoding) + + return { + name: tf.keras.layers.InputSpec( + shape=shape, dtype=self._get_state_dtype(name)) + for name, shape in state_shapes.items() } - outputs = (endpoints, states) - super(Movinet, self).__init__(inputs=inputs, outputs=outputs, **kwargs) + def init_states(self, input_shape: Sequence[int]) -> Dict[str, tf.Tensor]: + """Returns initial states for the first call in steaming mode.""" + state_shapes = self._get_initial_state_shapes( + self._block_specs, + input_shape, + use_positional_encoding=self._use_positional_encoding) + + states = { + name: tf.zeros(shape, dtype=self._get_state_dtype(name)) + for name, shape in state_shapes.items() + } + return states + + @property + def use_external_states(self) -> bool: + """Whether this model is expecting input states as additional input.""" + return self._use_external_states + + @property + def head_filters(self): + """The number of filters expected to be in the head classifer layer.""" + return self._head_filters + + @property + def conv_type(self): + """The expected convolution type (see __init__ for more details).""" + return self._conv_type def get_config(self): config_dict = { @@ -495,11 +664,6 @@ def get_config(self): def from_config(cls, config, custom_objects=None): return cls(**config) - @property - def output_specs(self): - """A dict of {level: TensorShape} pairs for the model output.""" - return self._output_specs - @factory.register_backbone_builder('movinet') def build_movinet( @@ -508,8 +672,6 @@ def build_movinet( norm_activation_config: hyperparams.Config, l2_regularizer: tf.keras.regularizers.Regularizer = None) -> tf.keras.Model: """Builds MoViNet backbone from a config.""" - l2_regularizer = l2_regularizer or tf.keras.regularizers.L2(1.5e-5) - backbone_type = backbone_config.type backbone_cfg = backbone_config.get() assert backbone_type == 'movinet', ('Inconsistent backbone type ' @@ -526,4 +688,5 @@ def build_movinet( norm_momentum=norm_activation_config.norm_momentum, norm_epsilon=norm_activation_config.norm_epsilon, kernel_regularizer=l2_regularizer, - stochastic_depth_drop_rate=backbone_cfg.stochastic_depth_drop_rate) + stochastic_depth_drop_rate=backbone_cfg.stochastic_depth_drop_rate, + use_external_states=backbone_cfg.use_external_states) diff --git a/official/vision/beta/projects/movinet/modeling/movinet_layers.py b/official/vision/beta/projects/movinet/modeling/movinet_layers.py index 171660754d2..acde9bff559 100644 --- a/official/vision/beta/projects/movinet/modeling/movinet_layers.py +++ b/official/vision/beta/projects/movinet/modeling/movinet_layers.py @@ -18,7 +18,7 @@ Reference: https://arxiv.org/pdf/2103.11511.pdf """ -from typing import Any, Optional, Sequence, Tuple, Union, Dict +from typing import Any, Mapping, Optional, Sequence, Tuple, Union import tensorflow as tf @@ -270,7 +270,6 @@ def __init__( batch_norm_epsilon: float = 1e-3, activation: Optional[Any] = None, conv_type: str = '3d', - use_positional_encoding: bool = False, use_buffered_input: bool = False, **kwargs): """Initializes a conv block. @@ -293,9 +292,6 @@ def __init__( ops. '2plus1d' split any 3D ops into two sequential 2D ops with their own batch norm and activation. '3d_2plus1d' is like '2plus1d', but uses two sequential 3D ops instead. - use_positional_encoding: add a positional encoding before the temporal - convolution. Assumes `kernel_size[0] > 1`. Otherwise, this argument - is ignored. use_buffered_input: if True, the input is expected to be padded beforehand. In effect, calling this layer will use 'valid' padding on the temporal dimension to simulate 'causal' padding. @@ -324,7 +320,6 @@ def __init__( self._batch_norm_epsilon = batch_norm_epsilon self._activation = activation self._conv_type = conv_type - self._use_positional_encoding = use_positional_encoding self._use_buffered_input = use_buffered_input if activation is not None: @@ -350,7 +345,6 @@ def get_config(self): 'batch_norm_epsilon': self._batch_norm_epsilon, 'activation': self._activation, 'conv_type': self._conv_type, - 'use_positional_encoding': self._use_positional_encoding, 'use_buffered_input': self._use_buffered_input, } base_config = super(ConvBlock, self).get_config() @@ -426,11 +420,6 @@ def build(self, input_shape): use_buffered_input=self._use_buffered_input, name='conv3d') - if self._use_positional_encoding and self._kernel_size[0] > 1: - self._pos_encoding = nn_layers.PositionalEncoding() - else: - self._pos_encoding = None - self._batch_norm = None self._batch_norm_temporal = None @@ -451,9 +440,6 @@ def call(self, inputs): """Calls the layer with the given inputs.""" x = inputs - if self._pos_encoding is not None and self._conv_temporal is None: - x = self._pos_encoding(x) - x = self._conv(x) if self._batch_norm is not None: x = self._batch_norm(x) @@ -461,9 +447,6 @@ def call(self, inputs): x = self._activation_layer(x) if self._conv_temporal is not None: - if self._pos_encoding is not None: - x = self._pos_encoding(x) - x = self._conv_temporal(x) if self._batch_norm_temporal is not None: x = self._batch_norm_temporal(x) @@ -477,11 +460,15 @@ def call(self, inputs): class StreamBuffer(tf.keras.layers.Layer): """Stream buffer wrapper which caches activations of previous frames.""" - def __init__(self, buffer_size: int, **kwargs): + def __init__(self, + buffer_size: int, + state_prefix: Optional[str] = None, + **kwargs): """Initializes a stream buffer. Args: buffer_size: the number of input frames to cache. + state_prefix: a prefix string to identify states. **kwargs: keyword arguments to be passed to this layer. Returns: @@ -489,36 +476,32 @@ def __init__(self, buffer_size: int, **kwargs): """ super(StreamBuffer, self).__init__(**kwargs) + state_prefix = state_prefix if state_prefix is not None else '' + self._state_prefix = state_prefix + self._state_name = f'{state_prefix}/stream_buffer' self._buffer_size = buffer_size - def build(self, input_shape): - """Builds the layer with the given input shape.""" - # Here we define strings that will uniquely reference the buffer states - # in the TF graph. These will be used for passing in a mapping of states - # for streaming mode. To do this, we can use a name scope. - with tf.name_scope('buffer') as state_name: - self._state_name = state_name - - super(StreamBuffer, self).build(input_shape) - def get_config(self): """Returns a dictionary containing the config used for initialization.""" config = { 'buffer_size': self._buffer_size, + 'state_prefix': self._state_prefix, } base_config = super(StreamBuffer, self).get_config() return dict(list(base_config.items()) + list(config.items())) - def call(self, - inputs: tf.Tensor, - states: Optional[nn_layers.States] = None - ) -> Tuple[Any, nn_layers.States]: + def call( + self, + inputs: tf.Tensor, + states: Optional[nn_layers.States] = None, + ) -> Tuple[Any, nn_layers.States]: """Calls the layer with the given inputs. Args: inputs: the input tensor. states: a dict of states such that, if any of the keys match for this layer, will overwrite the contents of the buffer(s). + Expected keys include `state_prefix + '/stream_buffer'`. Returns: the output tensor and states @@ -526,12 +509,16 @@ def call(self, states = dict(states) if states is not None else {} buffer = states.get(self._state_name, None) - # `tf.pad` has limited support for tf lite, so use tf.concat instead + # Create the buffer if it does not exist in the states. + # Output buffer shape: + # [batch_size, buffer_size, input_height, input_width, num_channels] if buffer is None: shape = tf.shape(inputs) buffer = tf.zeros( [shape[0], self._buffer_size, shape[2], shape[3], shape[4]], dtype=inputs.dtype) + + # tf.pad has limited support for tf lite, so use tf.concat instead. full_inputs = tf.concat([buffer, inputs], axis=1) # Cache the last b frames of the input where b is the buffer size and f @@ -557,16 +544,16 @@ def __init__( causal: bool = False, use_bias: bool = False, kernel_initializer: tf.keras.initializers.Initializer = 'HeNormal', - kernel_regularizer: Optional[tf.keras.regularizers.Regularizer] = - tf.keras.regularizers.L2(KERNEL_WEIGHT_DECAY), + kernel_regularizer: Optional[tf.keras.regularizers.Regularizer] = tf.keras + .regularizers.L2(KERNEL_WEIGHT_DECAY), use_batch_norm: bool = True, - batch_norm_layer: tf.keras.layers.Layer = - tf.keras.layers.experimental.SyncBatchNormalization, + batch_norm_layer: tf.keras.layers.Layer = tf.keras.layers.experimental + .SyncBatchNormalization, batch_norm_momentum: float = 0.99, batch_norm_epsilon: float = 1e-3, activation: Optional[Any] = None, conv_type: str = '3d', - use_positional_encoding: bool = False, + state_prefix: Optional[str] = None, **kwargs): """Initializes a stream conv block. @@ -588,7 +575,7 @@ def __init__( ops. '2plus1d' split any 3D ops into two sequential 2D ops with their own batch norm and activation. '3d_2plus1d' is like '2plus1d', but uses two sequential 3D ops instead. - use_positional_encoding: add a positional encoding before the convolution. + state_prefix: a prefix string to identify states. **kwargs: keyword arguments to be passed to this layer. Returns: @@ -598,6 +585,8 @@ def __init__( buffer_size = kernel_size[0] - 1 use_buffer = buffer_size > 0 and causal + self._state_prefix = state_prefix + super(StreamConvBlock, self).__init__( filters, kernel_size, @@ -613,18 +602,17 @@ def __init__( batch_norm_epsilon=batch_norm_epsilon, activation=activation, conv_type=conv_type, - use_positional_encoding=use_positional_encoding, use_buffered_input=use_buffer, **kwargs) self._stream_buffer = None if use_buffer: self._stream_buffer = StreamBuffer( - buffer_size=buffer_size) + buffer_size=buffer_size, state_prefix=state_prefix) def get_config(self): """Returns a dictionary containing the config used for initialization.""" - config = {} + config = {'state_prefix': self._state_prefix} base_config = super(StreamConvBlock, self).get_config() return dict(list(base_config.items()) + list(config.items())) @@ -667,9 +655,10 @@ def __init__( causal: bool = False, conv_type: str = '3d', kernel_initializer: tf.keras.initializers.Initializer = 'HeNormal', - kernel_regularizer: Optional[tf.keras.regularizers.Regularizer] = - tf.keras.regularizers.L2(KERNEL_WEIGHT_DECAY), + kernel_regularizer: Optional[tf.keras.regularizers.Regularizer] = tf.keras + .regularizers.L2(KERNEL_WEIGHT_DECAY), use_positional_encoding: bool = False, + state_prefix: Optional[str] = None, **kwargs): """Implementation for squeeze and excitation. @@ -686,6 +675,7 @@ def __init__( kernel_regularizer: kernel regularizer for the conv operation. use_positional_encoding: add a positional encoding after the (cumulative) global average pooling layer. + state_prefix: a prefix string to identify states. **kwargs: keyword arguments to be passed to this layer. """ super(StreamSqueezeExcitation, self).__init__(**kwargs) @@ -698,13 +688,15 @@ def __init__( self._kernel_initializer = kernel_initializer self._kernel_regularizer = kernel_regularizer self._use_positional_encoding = use_positional_encoding + self._state_prefix = state_prefix - self._pool = nn_layers.GlobalAveragePool3D(keepdims=True, causal=causal) + self._pool = nn_layers.GlobalAveragePool3D( + keepdims=True, causal=causal, state_prefix=state_prefix) + self._pos_encoding = None if use_positional_encoding: - self._pos_encoding = nn_layers.PositionalEncoding() - else: - self._pos_encoding = None + self._pos_encoding = nn_layers.PositionalEncoding( + initializer='zeros', state_prefix=state_prefix) def get_config(self): """Returns a dictionary containing the config used for initialization.""" @@ -717,6 +709,7 @@ def get_config(self): 'kernel_initializer': self._kernel_initializer, 'kernel_regularizer': self._kernel_regularizer, 'use_positional_encoding': self._use_positional_encoding, + 'state_prefix': self._state_prefix, } base_config = super(StreamSqueezeExcitation, self).get_config() return dict(list(base_config.items()) + list(config.items())) @@ -768,7 +761,7 @@ def call(self, x, states = self._pool(inputs, states=states) if self._pos_encoding is not None: - x = self._pos_encoding(x) + x, states = self._pos_encoding(x, states=states) x = self._se_reduce(x) x = self._se_expand(x) @@ -992,12 +985,13 @@ def __init__( conv_type: str = '3d', use_positional_encoding: bool = False, kernel_initializer: tf.keras.initializers.Initializer = 'HeNormal', - kernel_regularizer: Optional[tf.keras.regularizers.Regularizer] = - tf.keras.regularizers.L2(KERNEL_WEIGHT_DECAY), - batch_norm_layer: tf.keras.layers.Layer = - tf.keras.layers.experimental.SyncBatchNormalization, + kernel_regularizer: Optional[tf.keras.regularizers.Regularizer] = tf.keras + .regularizers.L2(KERNEL_WEIGHT_DECAY), + batch_norm_layer: tf.keras.layers.Layer = tf.keras.layers.experimental + .SyncBatchNormalization, batch_norm_momentum: float = 0.99, batch_norm_epsilon: float = 1e-3, + state_prefix: Optional[str] = None, **kwargs): """Implementation for MoViNet block. @@ -1021,6 +1015,7 @@ def __init__( batch_norm_layer: class to use for batch norm. batch_norm_momentum: momentum of the batch norm operation. batch_norm_epsilon: epsilon of the batch norm operation. + state_prefix: a prefix string to identify states. **kwargs: keyword arguments to be passed to this layer. """ super(MovinetBlock, self).__init__(**kwargs) @@ -1045,6 +1040,7 @@ def __init__( self._batch_norm_layer = batch_norm_layer self._batch_norm_momentum = batch_norm_momentum self._batch_norm_epsilon = batch_norm_epsilon + self._state_prefix = state_prefix self._expansion = ConvBlock( expand_filters, @@ -1066,15 +1062,14 @@ def __init__( causal=self._causal, activation=activation, conv_type=conv_type, - use_positional_encoding=use_positional_encoding, kernel_initializer=kernel_initializer, kernel_regularizer=kernel_regularizer, use_batch_norm=True, batch_norm_layer=self._batch_norm_layer, batch_norm_momentum=self._batch_norm_momentum, batch_norm_epsilon=self._batch_norm_epsilon, + state_prefix=state_prefix, name='feature') - self._projection = ConvBlock( out_filters, (1, 1, 1), @@ -1095,6 +1090,7 @@ def __init__( use_positional_encoding=use_positional_encoding, kernel_initializer=kernel_initializer, kernel_regularizer=kernel_regularizer, + state_prefix=state_prefix, name='se') def get_config(self): @@ -1114,6 +1110,7 @@ def get_config(self): 'kernel_regularizer': self._kernel_regularizer, 'batch_norm_momentum': self._batch_norm_momentum, 'batch_norm_epsilon': self._batch_norm_epsilon, + 'state_prefix': self._state_prefix, } base_config = super(MovinetBlock, self).get_config() return dict(list(base_config.items()) + list(config.items())) @@ -1176,12 +1173,13 @@ def __init__( conv_type: str = '3d', activation: nn_layers.Activation = 'swish', kernel_initializer: tf.keras.initializers.Initializer = 'HeNormal', - kernel_regularizer: Optional[tf.keras.regularizers.Regularizer] = - tf.keras.regularizers.L2(KERNEL_WEIGHT_DECAY), - batch_norm_layer: tf.keras.layers.Layer = - tf.keras.layers.experimental.SyncBatchNormalization, + kernel_regularizer: Optional[tf.keras.regularizers.Regularizer] = tf.keras + .regularizers.L2(KERNEL_WEIGHT_DECAY), + batch_norm_layer: tf.keras.layers.Layer = tf.keras.layers.experimental + .SyncBatchNormalization, batch_norm_momentum: float = 0.99, batch_norm_epsilon: float = 1e-3, + state_prefix: Optional[str] = None, **kwargs): """Implementation for video model stem. @@ -1200,35 +1198,38 @@ def __init__( batch_norm_layer: class to use for batch norm. batch_norm_momentum: momentum of the batch norm operation. batch_norm_epsilon: epsilon of the batch norm operation. + state_prefix: a prefix string to identify states. **kwargs: keyword arguments to be passed to this layer. """ super(Stem, self).__init__(**kwargs) + self._out_filters = out_filters self._kernel_size = normalize_tuple(kernel_size, 3, 'kernel_size') self._strides = normalize_tuple(strides, 3, 'strides') - - self._out_filters = out_filters - self._conv_type = conv_type self._causal = causal + self._conv_type = conv_type + self._activation = activation self._kernel_initializer = kernel_initializer self._kernel_regularizer = kernel_regularizer self._batch_norm_layer = batch_norm_layer self._batch_norm_momentum = batch_norm_momentum self._batch_norm_epsilon = batch_norm_epsilon + self._state_prefix = state_prefix self._stem = StreamConvBlock( filters=self._out_filters, kernel_size=self._kernel_size, strides=self._strides, causal=self._causal, - activation=activation, + activation=self._activation, conv_type=self._conv_type, - kernel_initializer=kernel_initializer, - kernel_regularizer=kernel_regularizer, + kernel_initializer=self._kernel_initializer, + kernel_regularizer=self._kernel_regularizer, use_batch_norm=True, batch_norm_layer=self._batch_norm_layer, batch_norm_momentum=self._batch_norm_momentum, batch_norm_epsilon=self._batch_norm_epsilon, + state_prefix=self._state_prefix, name='stem') def get_config(self): @@ -1238,11 +1239,13 @@ def get_config(self): 'kernel_size': self._kernel_size, 'strides': self._strides, 'causal': self._causal, + 'activation': self._activation, 'conv_type': self._conv_type, 'kernel_initializer': self._kernel_initializer, 'kernel_regularizer': self._kernel_regularizer, 'batch_norm_momentum': self._batch_norm_momentum, 'batch_norm_epsilon': self._batch_norm_epsilon, + 'state_prefix': self._state_prefix, } base_config = super(Stem, self).get_config() return dict(list(base_config.items()) + list(config.items())) @@ -1278,12 +1281,13 @@ def __init__( conv_type: str = '3d', activation: nn_layers.Activation = 'swish', kernel_initializer: tf.keras.initializers.Initializer = 'HeNormal', - kernel_regularizer: Optional[tf.keras.regularizers.Regularizer] = - tf.keras.regularizers.L2(KERNEL_WEIGHT_DECAY), - batch_norm_layer: tf.keras.layers.Layer = - tf.keras.layers.experimental.SyncBatchNormalization, + kernel_regularizer: Optional[tf.keras.regularizers.Regularizer] = tf.keras + .regularizers.L2(KERNEL_WEIGHT_DECAY), + batch_norm_layer: tf.keras.layers.Layer = tf.keras.layers.experimental + .SyncBatchNormalization, batch_norm_momentum: float = 0.99, batch_norm_epsilon: float = 1e-3, + state_prefix: Optional[str] = None, **kwargs): """Implementation for video model head. @@ -1299,17 +1303,20 @@ def __init__( batch_norm_layer: class to use for batch norm. batch_norm_momentum: momentum of the batch norm operation. batch_norm_epsilon: epsilon of the batch norm operation. + state_prefix: a prefix string to identify states. **kwargs: keyword arguments to be passed to this layer. """ super(Head, self).__init__(**kwargs) self._project_filters = project_filters self._conv_type = conv_type + self._activation = activation self._kernel_initializer = kernel_initializer self._kernel_regularizer = kernel_regularizer self._batch_norm_layer = batch_norm_layer self._batch_norm_momentum = batch_norm_momentum self._batch_norm_epsilon = batch_norm_epsilon + self._state_prefix = state_prefix self._project = ConvBlock( filters=project_filters, @@ -1322,25 +1329,29 @@ def __init__( batch_norm_momentum=self._batch_norm_momentum, batch_norm_epsilon=self._batch_norm_epsilon, name='project') - self._pool = nn_layers.GlobalAveragePool3D(keepdims=True, causal=False) + self._pool = nn_layers.GlobalAveragePool3D( + keepdims=True, causal=False, state_prefix=state_prefix) def get_config(self): """Returns a dictionary containing the config used for initialization.""" config = { 'project_filters': self._project_filters, 'conv_type': self._conv_type, + 'activation': self._activation, 'kernel_initializer': self._kernel_initializer, 'kernel_regularizer': self._kernel_regularizer, 'batch_norm_momentum': self._batch_norm_momentum, 'batch_norm_epsilon': self._batch_norm_epsilon, + 'state_prefix': self._state_prefix, } base_config = super(Head, self).get_config() return dict(list(base_config.items()) + list(config.items())) - def call(self, - inputs: Union[tf.Tensor, Dict[str, tf.Tensor]], - states: Optional[nn_layers.States] = None, - ) -> Tuple[tf.Tensor, nn_layers.States]: + def call( + self, + inputs: Union[tf.Tensor, Mapping[str, tf.Tensor]], + states: Optional[nn_layers.States] = None, + ) -> Tuple[tf.Tensor, nn_layers.States]: """Calls the layer with the given inputs. Args: diff --git a/official/vision/beta/projects/movinet/modeling/movinet_layers_test.py b/official/vision/beta/projects/movinet/modeling/movinet_layers_test.py index 4095966fcd8..bb804f38d61 100644 --- a/official/vision/beta/projects/movinet/modeling/movinet_layers_test.py +++ b/official/vision/beta/projects/movinet/modeling/movinet_layers_test.py @@ -146,7 +146,6 @@ def test_stream_conv_block_2plus1d(self): use_bias=False, activation='relu', conv_type='2plus1d', - use_positional_encoding=True, ) stream_conv_block = movinet_layers.StreamConvBlock( @@ -158,7 +157,6 @@ def test_stream_conv_block_2plus1d(self): use_bias=False, activation='relu', conv_type='2plus1d', - use_positional_encoding=True, ) inputs = tf.ones([1, 4, 2, 2, 3]) @@ -197,7 +195,6 @@ def test_stream_conv_block_3d_2plus1d(self): use_bias=False, activation='relu', conv_type='3d_2plus1d', - use_positional_encoding=True, ) stream_conv_block = movinet_layers.StreamConvBlock( @@ -209,7 +206,6 @@ def test_stream_conv_block_3d_2plus1d(self): use_bias=False, activation='relu', conv_type='3d_2plus1d', - use_positional_encoding=True, ) inputs = tf.ones([1, 4, 2, 2, 3]) diff --git a/official/vision/beta/projects/movinet/modeling/movinet_model.py b/official/vision/beta/projects/movinet/modeling/movinet_model.py index 0fc74b4765c..f95b690e8f7 100644 --- a/official/vision/beta/projects/movinet/modeling/movinet_model.py +++ b/official/vision/beta/projects/movinet/modeling/movinet_model.py @@ -16,7 +16,7 @@ Reference: https://arxiv.org/pdf/2103.11511.pdf """ -from typing import Mapping, Optional +from typing import Any, Dict, Mapping, Optional, Sequence, Tuple, Union from absl import logging import tensorflow as tf @@ -71,47 +71,94 @@ def __init__( self._bias_regularizer = bias_regularizer self._output_states = output_states - # Keras model variable that excludes @property.setters from tracking - self._self_setattr_tracking = False + state_specs = None + if backbone.use_external_states: + state_specs = backbone.initial_state_specs( + input_shape=input_specs['image'].shape) - inputs = { - name: tf.keras.Input(shape=state.shape[1:], name=f'states/{name}') - for name, state in input_specs.items() + inputs, outputs = self._build_network( + backbone, input_specs, state_specs=state_specs) + + super(MovinetClassifier, self).__init__( + inputs=inputs, outputs=outputs, **kwargs) + + # Move backbone after super() call so Keras is happy + self._backbone = backbone + + def _build_network( + self, + backbone: tf.keras.Model, + input_specs: Mapping[str, tf.keras.layers.InputSpec], + state_specs: Optional[Mapping[str, tf.keras.layers.InputSpec]] = None, + ) -> Tuple[Mapping[str, tf.keras.Input], Union[Tuple[Mapping[ + str, tf.Tensor], Mapping[str, tf.Tensor]], Mapping[str, tf.Tensor]]]: + """Builds the model network. + + Args: + backbone: the model backbone. + input_specs: the model input spec to use. + state_specs: a dict of states such that, if any of the keys match for a + layer, will overwrite the contents of the buffer(s). + + Returns: + Inputs and outputs as a tuple. Inputs are expected to be a dict with + base input and states. Outputs are expected to be a dict of endpoints + and (optionally) output states. + """ + state_specs = state_specs if state_specs is not None else {} + + states = { + name: tf.keras.Input(shape=spec.shape[1:], dtype=spec.dtype, name=name) + for name, spec in state_specs.items() } - states = inputs.get('states', {}) + image = tf.keras.Input(shape=input_specs['image'].shape[1:], name='image') + inputs = {**states, 'image': image} + + if backbone.use_external_states: + before_states = set(states) + endpoints, states = backbone(inputs) + after_states = set(states) + + new_states = after_states - before_states + if new_states: + raise AttributeError('Expected input and output states to be the same. ' + 'Got extra states {}, expected {}'.format( + new_states, before_states)) + else: + endpoints, states = backbone(inputs) - endpoints, states = backbone(dict(image=inputs['image'], states=states)) x = endpoints['head'] x = movinet_layers.ClassifierHead( - head_filters=backbone._head_filters, - num_classes=num_classes, - dropout_rate=dropout_rate, - kernel_initializer=kernel_initializer, - kernel_regularizer=kernel_regularizer, - conv_type=backbone._conv_type)(x) - - if output_states: - inputs['states'] = { - k: tf.keras.Input(shape=v.shape[1:], name=k) - for k, v in states.items() - } + head_filters=backbone.head_filters, + num_classes=self._num_classes, + dropout_rate=self._dropout_rate, + kernel_initializer=self._kernel_initializer, + kernel_regularizer=self._kernel_regularizer, + conv_type=backbone.conv_type)( + x) - outputs = (x, states) if output_states else x + outputs = (x, states) if self._output_states else x - super(MovinetClassifier, self).__init__( - inputs=inputs, outputs=outputs, **kwargs) + return inputs, outputs - # Move backbone after super() call so Keras is happy - self._backbone = backbone + def initial_state_specs( + self, input_shape: Sequence[int]) -> Dict[str, tf.keras.layers.InputSpec]: + return self._backbone.initial_state_specs(input_shape=input_shape) + + @tf.function + def init_states(self, input_shape: Sequence[int]) -> Dict[str, tf.Tensor]: + """Returns initial states for the first call in steaming mode.""" + return self._backbone.init_states(input_shape) @property - def checkpoint_items(self): + def checkpoint_items(self) -> Dict[str, Any]: """Returns a dictionary of items to be additionally checkpointed.""" return dict(backbone=self.backbone) @property - def backbone(self): + def backbone(self) -> tf.keras.Model: + """Returns the backbone of the model.""" return self._backbone def get_config(self): @@ -142,7 +189,7 @@ def from_config(cls, config, custom_objects=None): @model_factory.register_model_builder('movinet') def build_movinet_model( - input_specs: tf.keras.layers.InputSpec, + input_specs: Mapping[str, tf.keras.layers.InputSpec], model_config: cfg.MovinetModel, num_classes: int, l2_regularizer: Optional[tf.keras.regularizers.Regularizer] = None): diff --git a/official/vision/beta/projects/movinet/modeling/movinet_model_test.py b/official/vision/beta/projects/movinet/modeling/movinet_model_test.py index ba2b6dd6dbf..7d77f703504 100644 --- a/official/vision/beta/projects/movinet/modeling/movinet_model_test.py +++ b/official/vision/beta/projects/movinet/modeling/movinet_model_test.py @@ -48,28 +48,85 @@ def test_movinet_classifier_creation(self, is_training): self.assertAllEqual([2, num_classes], logits.shape) def test_movinet_classifier_stream(self): + """Test if the classifier can be run in streaming mode.""" tf.keras.backend.set_image_data_format('channels_last') - model = movinet.Movinet( + backbone = movinet.Movinet( model_id='a0', causal=True, + use_external_states=True, ) - inputs = tf.ones([1, 5, 128, 128, 3]) + model = movinet_model.MovinetClassifier( + backbone, num_classes=600, output_states=True) + + inputs = tf.ones([1, 8, 172, 172, 3]) + + init_states = model.init_states(tf.shape(inputs)) + expected, _ = model({**init_states, 'image': inputs}) + + frames = tf.split(inputs, inputs.shape[1], axis=1) + + states = init_states + for frame in frames: + output, states = model({**states, 'image': frame}) + predicted = output + + self.assertEqual(predicted.shape, expected.shape) + self.assertAllClose(predicted, expected, 1e-5, 1e-5) + + def test_movinet_classifier_stream_pos_enc(self): + """Test if the classifier can be run in streaming mode with pos encoding.""" + tf.keras.backend.set_image_data_format('channels_last') + + backbone = movinet.Movinet( + model_id='a0', + causal=True, + use_external_states=True, + use_positional_encoding=True, + ) + model = movinet_model.MovinetClassifier( + backbone, num_classes=600, output_states=True) + + inputs = tf.ones([1, 8, 172, 172, 3]) - expected_endpoints, _ = model(dict(image=inputs, states={})) + init_states = model.init_states(tf.shape(inputs)) + expected, _ = model({**init_states, 'image': inputs}) frames = tf.split(inputs, inputs.shape[1], axis=1) - output, states = None, {} + states = init_states for frame in frames: - output, states = model(dict(image=frame, states=states)) - predicted_endpoints = output + output, states = model({**states, 'image': frame}) + predicted = output + + self.assertEqual(predicted.shape, expected.shape) + self.assertAllClose(predicted, expected, 1e-5, 1e-5) + + def test_movinet_classifier_stream_pos_enc_2plus1d(self): + """Test if the model can run in streaming mode with pos encoding, (2+1)D.""" + tf.keras.backend.set_image_data_format('channels_last') + + backbone = movinet.Movinet( + model_id='a0', + causal=True, + use_external_states=True, + use_positional_encoding=True, + conv_type='2plus1d', + ) + model = movinet_model.MovinetClassifier( + backbone, num_classes=600, output_states=True) - predicted = predicted_endpoints['head'] + inputs = tf.ones([1, 8, 172, 172, 3]) - # The expected final output is simply the mean across frames - expected = expected_endpoints['head'] - expected = tf.reduce_mean(expected, 1, keepdims=True) + init_states = model.init_states(tf.shape(inputs)) + expected, _ = model({**init_states, 'image': inputs}) + + frames = tf.split(inputs, inputs.shape[1], axis=1) + + states = init_states + for frame in frames: + output, states = model({**states, 'image': frame}) + predicted = output self.assertEqual(predicted.shape, expected.shape) self.assertAllClose(predicted, expected, 1e-5, 1e-5) diff --git a/official/vision/beta/projects/movinet/modeling/movinet_test.py b/official/vision/beta/projects/movinet/modeling/movinet_test.py index 6467af18b32..a0b3ba35f4b 100644 --- a/official/vision/beta/projects/movinet/modeling/movinet_test.py +++ b/official/vision/beta/projects/movinet/modeling/movinet_test.py @@ -48,14 +48,15 @@ def test_network_with_states(self): """Test creation of MoViNet family models with states.""" tf.keras.backend.set_image_data_format('channels_last') - network = movinet.Movinet( + backbone = movinet.Movinet( model_id='a0', causal=True, + use_external_states=True, ) inputs = tf.ones([1, 8, 128, 128, 3]) - _, states = network(inputs) - endpoints, new_states = network(dict(image=inputs, states=states)) + init_states = backbone.init_states(tf.shape(inputs)) + endpoints, new_states = backbone({**init_states, 'image': inputs}) self.assertAllEqual(endpoints['stem'].shape, [1, 8, 64, 64, 8]) self.assertAllEqual(endpoints['b0/l0'].shape, [1, 8, 32, 32, 8]) @@ -65,25 +66,28 @@ def test_network_with_states(self): self.assertAllEqual(endpoints['b4/l0'].shape, [1, 8, 4, 4, 104]) self.assertAllEqual(endpoints['head'].shape, [1, 1, 1, 1, 480]) - self.assertNotEmpty(states) + self.assertNotEmpty(init_states) self.assertNotEmpty(new_states) def test_movinet_stream(self): + """Test if the backbone can be run in streaming mode.""" tf.keras.backend.set_image_data_format('channels_last') - model = movinet.Movinet( + backbone = movinet.Movinet( model_id='a0', causal=True, + use_external_states=True, ) inputs = tf.ones([1, 5, 128, 128, 3]) - expected_endpoints, _ = model(dict(image=inputs, states={})) + init_states = backbone.init_states(tf.shape(inputs)) + expected_endpoints, _ = backbone({**init_states, 'image': inputs}) frames = tf.split(inputs, inputs.shape[1], axis=1) - output, states = None, {} + states = init_states for frame in frames: - output, states = model(dict(image=frame, states=states)) + output, states = backbone({**states, 'image': frame}) predicted_endpoints = output predicted = predicted_endpoints['head'] @@ -98,20 +102,22 @@ def test_movinet_stream(self): def test_movinet_2plus1d_stream(self): tf.keras.backend.set_image_data_format('channels_last') - model = movinet.Movinet( + backbone = movinet.Movinet( model_id='a0', causal=True, conv_type='2plus1d', + use_external_states=True, ) inputs = tf.ones([1, 5, 128, 128, 3]) - expected_endpoints, _ = model(dict(image=inputs, states={})) + init_states = backbone.init_states(tf.shape(inputs)) + expected_endpoints, _ = backbone({**init_states, 'image': inputs}) frames = tf.split(inputs, inputs.shape[1], axis=1) - output, states = None, {} + states = init_states for frame in frames: - output, states = model(dict(image=frame, states=states)) + output, states = backbone({**states, 'image': frame}) predicted_endpoints = output predicted = predicted_endpoints['head'] @@ -126,20 +132,22 @@ def test_movinet_2plus1d_stream(self): def test_movinet_3d_2plus1d_stream(self): tf.keras.backend.set_image_data_format('channels_last') - model = movinet.Movinet( + backbone = movinet.Movinet( model_id='a0', causal=True, conv_type='3d_2plus1d', + use_external_states=True, ) inputs = tf.ones([1, 5, 128, 128, 3]) - expected_endpoints, _ = model(dict(image=inputs, states={})) + init_states = backbone.init_states(tf.shape(inputs)) + expected_endpoints, _ = backbone({**init_states, 'image': inputs}) frames = tf.split(inputs, inputs.shape[1], axis=1) - output, states = None, {} + states = init_states for frame in frames: - output, states = model(dict(image=frame, states=states)) + output, states = backbone({**states, 'image': frame}) predicted_endpoints = output predicted = predicted_endpoints['head'] @@ -157,6 +165,7 @@ def test_serialize_deserialize(self): model_id='a0', causal=True, use_positional_encoding=True, + use_external_states=True, ) network = movinet.Movinet(**kwargs) From 6a9839f2cd30a42fa28cd75afd566f4479acb0e8 Mon Sep 17 00:00:00 2001 From: Dan Kondratyuk Date: Fri, 4 Jun 2021 15:46:42 -0700 Subject: [PATCH 27/50] Add MoViNet-A5-Stream config. PiperOrigin-RevId: 377605316 --- .../yaml/movinet_a5_stream_k600_8x8.yaml | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 official/vision/beta/projects/movinet/configs/yaml/movinet_a5_stream_k600_8x8.yaml diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_stream_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_stream_k600_8x8.yaml new file mode 100644 index 00000000000..1983937679f --- /dev/null +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_stream_k600_8x8.yaml @@ -0,0 +1,75 @@ +# Video classification on Kinetics-600 using MoViNet-A5-Stream backbone. +# --experiment_type=movinet_kinetics600 +# Achieves 82.37% Top-1 accuracy. +# http://mldash/experiments/7675567202035803461 + +runtime: + distribution_strategy: 'tpu' + mixed_precision_dtype: 'bfloat16' +task: + losses: + l2_weight_decay: 0.00003 + label_smoothing: 0.1 + model: + backbone: + movinet: + model_id: 'a5' + causal: true + use_positional_encoding: true + stochastic_depth_drop_rate: 0.2 + norm_activation: + use_sync_bn: true + dropout_rate: 0.5 + train_data: + name: kinetics600 + variant_name: rgb + feature_shape: !!python/tuple + - 32 + - 320 + - 320 + - 3 + temporal_stride: 2 + random_stride_range: 1 + global_batch_size: 1024 + dtype: 'bfloat16' + shuffle_buffer_size: 1024 + min_image_size: 368 + aug_max_area_ratio: 1.0 + aug_max_aspect_ratio: 2.0 + aug_min_area_ratio: 0.08 + aug_min_aspect_ratio: 0.5 + aug_type: 'autoaug' + validation_data: + name: kinetics600 + feature_shape: !!python/tuple + - 120 + - 320 + - 320 + - 3 + temporal_stride: 2 + num_test_clips: 1 + num_test_crops: 1 + global_batch_size: 32 + min_image_size: 368 + dtype: 'bfloat16' + drop_remainder: false +trainer: + optimizer_config: + learning_rate: + cosine: + initial_learning_rate: 1.8 + decay_steps: 85785 + warmup: + linear: + warmup_steps: 2145 + optimizer: + type: 'rmsprop' + rmsprop: + rho: 0.9 + momentum: 0.9 + epsilon: 1.0 + clipnorm: 1.0 + train_steps: 85785 + steps_per_loop: 500 + summary_interval: 500 + validation_interval: 500 From ed7d404f4780565884d2344de6aa3bce59bb5a2c Mon Sep 17 00:00:00 2001 From: Hongkun Yu Date: Sun, 6 Jun 2021 13:34:36 -0700 Subject: [PATCH 28/50] Internal change PiperOrigin-RevId: 377801393 --- official/LICENSE | 14 ++++++++++++++ official/README-TPU.md | 14 ++++++++++++++ official/README.md | 14 ++++++++++++++ official/__init__.py | 14 ++++++++++++++ official/colab/decoding_api_in_tf_nlp.ipynb | 14 ++++++++++++++ official/colab/nlp/customize_encoder.ipynb | 14 ++++++++++++++ .../colab/nlp/nlp_modeling_library_intro.ipynb | 14 ++++++++++++++ official/nlp/README.md | 14 ++++++++++++++ official/nlp/albert/README.md | 14 ++++++++++++++ official/nlp/bert/README.md | 14 ++++++++++++++ official/nlp/bert/bert_cloud_tpu.md | 14 ++++++++++++++ .../nlp/configs/experiments/glue_mnli_matched.yaml | 14 ++++++++++++++ official/nlp/configs/experiments/squad_v1.yaml | 14 ++++++++++++++ .../nlp/configs/models/bert_en_uncased_base.yaml | 14 ++++++++++++++ official/nlp/docs/pretrained_models.md | 14 ++++++++++++++ official/nlp/docs/tfhub.md | 14 ++++++++++++++ official/nlp/docs/train.md | 14 ++++++++++++++ official/nlp/keras_nlp/README.md | 14 ++++++++++++++ official/nlp/keras_nlp/contributing.md | 14 ++++++++++++++ official/nlp/keras_nlp/requirements.txt | 14 ++++++++++++++ official/nlp/modeling/README.md | 14 ++++++++++++++ official/nlp/modeling/layers/README.md | 14 ++++++++++++++ official/nlp/modeling/losses/README.md | 14 ++++++++++++++ official/nlp/modeling/models/README.md | 14 ++++++++++++++ official/nlp/modeling/networks/README.md | 14 ++++++++++++++ official/nlp/nhnet/README.md | 14 ++++++++++++++ .../crawled_articles/domain_0.com/url_000.html | 14 ++++++++++++++ .../crawled_articles/domain_0.com/url_000.json | 14 ++++++++++++++ .../crawled_articles/domain_1.com/url_001.html | 14 ++++++++++++++ .../crawled_articles/domain_1.com/url_001.json | 14 ++++++++++++++ official/nlp/nhnet/testdata/stories.json | 14 ++++++++++++++ official/nlp/nhnet/testdata/vocab.txt | 14 ++++++++++++++ official/nlp/projects/bigbird/README.md | 14 ++++++++++++++ .../bigbird/experiments/glue_mnli_matched.yaml | 14 ++++++++++++++ .../nlp/projects/bigbird/experiments/squad_v1.yaml | 14 ++++++++++++++ official/nlp/projects/mobilebert/README.md | 14 ++++++++++++++ .../mobilebert/experiments/en_uncased_student.yaml | 14 ++++++++++++++ .../mobilebert/experiments/en_uncased_teacher.yaml | 14 ++++++++++++++ .../mobilebert_distillation_en_uncased.yaml | 14 ++++++++++++++ official/nlp/projects/teams/README.md | 14 ++++++++++++++ official/nlp/projects/tn_bert/README.md | 14 ++++++++++++++ official/nlp/transformer/README.md | 14 ++++++++++++++ official/nlp/xlnet/README.md | 14 ++++++++++++++ official/pip_package/setup.py | 4 ++-- official/recommendation/README.md | 14 ++++++++++++++ official/recommendation/ranking/README.md | 14 ++++++++++++++ .../ranking/configs/yaml/dcn_v2_criteo_tpu.yaml | 14 ++++++++++++++ .../ranking/configs/yaml/dlrm_criteo_tpu.yaml | 14 ++++++++++++++ official/recommendation/run.sh | 14 ++++++++++++++ official/requirements.txt | 14 ++++++++++++++ official/staging/__init__.py | 14 ++++++++++++++ official/staging/training/__init__.py | 4 ++-- official/staging/training/grad_utils.py | 4 ++-- official/utils/flags/README.md | 14 ++++++++++++++ official/utils/flags/guidelines.md | 14 ++++++++++++++ official/utils/testing/pylint.rcfile | 14 ++++++++++++++ official/utils/testing/scripts/builds_common.sh | 14 ++++++++++++++ official/utils/testing/scripts/ci_sanity.sh | 14 ++++++++++++++ official/utils/testing/scripts/presubmit.sh | 14 ++++++++++++++ official/vision/beta/MODEL_GARDEN.md | 14 ++++++++++++++ official/vision/beta/README.md | 14 ++++++++++++++ .../imagenet_mobilenetv2_gpu.yaml | 14 ++++++++++++++ .../imagenet_mobilenetv2_tpu.yaml | 14 ++++++++++++++ .../imagenet_resnet101_deeplab_tpu.yaml | 14 ++++++++++++++ .../imagenet_resnet101_tpu.yaml | 14 ++++++++++++++ .../imagenet_resnet152_tpu.yaml | 14 ++++++++++++++ .../imagenet_resnet50_deeplab_tpu.yaml | 14 ++++++++++++++ .../imagenet_resnet50_gpu.yaml | 14 ++++++++++++++ .../imagenet_resnet50_tfds_tpu.yaml | 14 ++++++++++++++ .../imagenet_resnet50_tpu.yaml | 14 ++++++++++++++ .../imagenet_resnetrs101_i160.yaml | 14 ++++++++++++++ .../imagenet_resnetrs101_i192.yaml | 14 ++++++++++++++ .../imagenet_resnetrs152_i192.yaml | 14 ++++++++++++++ .../imagenet_resnetrs152_i224.yaml | 14 ++++++++++++++ .../imagenet_resnetrs152_i256.yaml | 14 ++++++++++++++ .../imagenet_resnetrs200_i256.yaml | 14 ++++++++++++++ .../imagenet_resnetrs270_i256.yaml | 14 ++++++++++++++ .../imagenet_resnetrs350_i256.yaml | 14 ++++++++++++++ .../imagenet_resnetrs350_i320.yaml | 14 ++++++++++++++ .../imagenet_resnetrs420_i320.yaml | 14 ++++++++++++++ .../imagenet_resnetrs50_i160.yaml | 14 ++++++++++++++ .../maskrcnn/coco_spinenet143_cascadercnn_tpu.yaml | 14 ++++++++++++++ .../maskrcnn/coco_spinenet143_mrcnn_tpu.yaml | 14 ++++++++++++++ .../maskrcnn/coco_spinenet49_cascadercnn_tpu.yaml | 14 ++++++++++++++ .../maskrcnn/coco_spinenet49_mrcnn_tpu.yaml | 14 ++++++++++++++ .../maskrcnn/coco_spinenet96_casrcnn_tpu.yaml | 14 ++++++++++++++ .../maskrcnn/coco_spinenet96_mrcnn_tpu.yaml | 14 ++++++++++++++ .../maskrcnn/r50fpn_640_coco_scratch_tpu4x4.yaml | 14 ++++++++++++++ .../retinanet/coco_spinenet143_tpu.yaml | 14 ++++++++++++++ .../retinanet/coco_spinenet190_tpu.yaml | 14 ++++++++++++++ .../retinanet/coco_spinenet49_mobile_tpu.yaml | 14 ++++++++++++++ .../experiments/retinanet/coco_spinenet49_tpu.yaml | 14 ++++++++++++++ .../retinanet/coco_spinenet49s_mobile_tpu.yaml | 14 ++++++++++++++ .../retinanet/coco_spinenet49xs_mobile_tpu.yaml | 14 ++++++++++++++ .../experiments/retinanet/coco_spinenet96_tpu.yaml | 14 ++++++++++++++ .../retinanet/resnet50fpn_coco_tfds_tpu.yaml | 14 ++++++++++++++ .../resnet50fpn_coco_tpu4x4_benchmark.yaml | 14 ++++++++++++++ ...eeplabv3plus_resnet101_cityscapes_tfds_tpu.yaml | 14 ++++++++++++++ .../video_classification/k400_3d-resnet50_tpu.yaml | 14 ++++++++++++++ .../k400_resnet3drs_50_tpu.yaml | 14 ++++++++++++++ .../k400_slowonly16x4_tpu.yaml | 14 ++++++++++++++ .../video_classification/k400_slowonly8x8_tpu.yaml | 14 ++++++++++++++ .../video_classification/k600_3d-resnet50_tpu.yaml | 14 ++++++++++++++ .../k600_3d-resnet50g_tpu.yaml | 14 ++++++++++++++ .../video_classification/k600_slowonly8x8_tpu.yaml | 14 ++++++++++++++ official/vision/beta/projects/README.md | 14 ++++++++++++++ .../vision/beta/projects/assemblenet/README.md | 14 ++++++++++++++ .../beta/projects/deepmac_maskrcnn/README.md | 14 ++++++++++++++ .../deep_mask_head_rcnn_voc_r101_hg52.yaml | 14 ++++++++++++++ .../experiments/deep_mask_head_rcnn_voc_r50.yaml | 14 ++++++++++++++ .../deep_mask_head_rcnn_voc_r50_hg52.yaml | 14 ++++++++++++++ official/vision/beta/projects/movinet/README.md | 14 ++++++++++++++ .../movinet/configs/yaml/movinet_a0_k600_8x8.yaml | 14 ++++++++++++++ .../configs/yaml/movinet_a0_k600_cpu_local.yaml | 14 ++++++++++++++ .../configs/yaml/movinet_a0_stream_k600_8x8.yaml | 14 ++++++++++++++ .../movinet/configs/yaml/movinet_a1_k600_8x8.yaml | 14 ++++++++++++++ .../configs/yaml/movinet_a1_stream_k600_8x8.yaml | 14 ++++++++++++++ .../movinet/configs/yaml/movinet_a2_k600_8x8.yaml | 14 ++++++++++++++ .../configs/yaml/movinet_a2_stream_k600_8x8.yaml | 14 ++++++++++++++ .../movinet/configs/yaml/movinet_a3_k600_8x8.yaml | 14 ++++++++++++++ .../configs/yaml/movinet_a3_stream_k600_8x8.yaml | 14 ++++++++++++++ .../movinet/configs/yaml/movinet_a4_k600_8x8.yaml | 14 ++++++++++++++ .../configs/yaml/movinet_a4_stream_k600_8x8.yaml | 14 ++++++++++++++ .../movinet/configs/yaml/movinet_a5_k600_8x8.yaml | 14 ++++++++++++++ .../configs/yaml/movinet_a5_stream_k600_8x8.yaml | 14 ++++++++++++++ .../movinet/configs/yaml/movinet_t0_k600_8x8.yaml | 14 ++++++++++++++ .../configs/yaml/movinet_t0_stream_k600_8x8.yaml | 14 ++++++++++++++ .../beta/projects/movinet/movinet_tutorial.ipynb | 14 ++++++++++++++ .../vision/beta/projects/movinet/requirements.txt | 14 ++++++++++++++ official/vision/beta/projects/simclr/README.md | 14 ++++++++++++++ .../configs/experiments/cifar_simclr_pretrain.yaml | 14 ++++++++++++++ .../experiments/imagenet_simclr_finetune_gpu.yaml | 14 ++++++++++++++ .../experiments/imagenet_simclr_finetune_tpu.yaml | 14 ++++++++++++++ .../experiments/imagenet_simclr_pretrain_gpu.yaml | 14 ++++++++++++++ .../experiments/imagenet_simclr_pretrain_tpu.yaml | 14 ++++++++++++++ official/vision/beta/projects/yolo/README.md | 14 ++++++++++++++ .../yolo/configs/experiments/csp_darknet53.yaml | 14 ++++++++++++++ .../configs/experiments/csp_darknet53_tfds.yaml | 14 ++++++++++++++ .../yolo/configs/experiments/darknet53.yaml | 14 ++++++++++++++ .../yolo/configs/experiments/darknet53_tfds.yaml | 14 ++++++++++++++ official/vision/beta/projects/yt8m/README.md | 14 ++++++++++++++ .../beta/projects/yt8m/experiments/yt8m.yaml | 14 ++++++++++++++ .../beta/projects/yt8m/experiments/yt8m_test.yaml | 14 ++++++++++++++ official/vision/detection/README.md | 14 ++++++++++++++ official/vision/image_classification/README.md | 14 ++++++++++++++ .../efficientnet/imagenet/efficientnet-b0-gpu.yaml | 14 ++++++++++++++ .../efficientnet/imagenet/efficientnet-b0-tpu.yaml | 14 ++++++++++++++ .../efficientnet/imagenet/efficientnet-b1-gpu.yaml | 14 ++++++++++++++ .../efficientnet/imagenet/efficientnet-b1-tpu.yaml | 14 ++++++++++++++ .../configs/examples/resnet/imagenet/gpu.yaml | 14 ++++++++++++++ .../configs/examples/resnet/imagenet/tpu.yaml | 14 ++++++++++++++ .../vision/image_classification/resnet/README.md | 14 ++++++++++++++ official/vision/keras_cv/LICENSE | 14 ++++++++++++++ official/vision/keras_cv/README.md | 14 ++++++++++++++ official/vision/keras_cv/contributing.md | 14 ++++++++++++++ official/vision/keras_cv/requirements.txt | 14 ++++++++++++++ 156 files changed, 2148 insertions(+), 6 deletions(-) diff --git a/official/LICENSE b/official/LICENSE index d3da228420e..b1143d7b536 100644 --- a/official/LICENSE +++ b/official/LICENSE @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + Copyright 2015 The TensorFlow Authors. All rights reserved. Apache License diff --git a/official/README-TPU.md b/official/README-TPU.md index a6031c44f03..435079d1c83 100644 --- a/official/README-TPU.md +++ b/official/README-TPU.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Offically Supported TensorFlow 2.1+ Models on Cloud TPU ## Natural Language Processing diff --git a/official/README.md b/official/README.md index 188a94ca532..52ca8f3d9bc 100644 --- a/official/README.md +++ b/official/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + ![Logo](https://storage.googleapis.com/model_garden_artifacts/TF_Model_Garden.png) # TensorFlow Official Models diff --git a/official/__init__.py b/official/__init__.py index e69de29bb2d..e419af524b5 100644 --- a/official/__init__.py +++ b/official/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/official/colab/decoding_api_in_tf_nlp.ipynb b/official/colab/decoding_api_in_tf_nlp.ipynb index 726b382e228..155a18422f0 100644 --- a/official/colab/decoding_api_in_tf_nlp.ipynb +++ b/official/colab/decoding_api_in_tf_nlp.ipynb @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + { "cells": [ { diff --git a/official/colab/nlp/customize_encoder.ipynb b/official/colab/nlp/customize_encoder.ipynb index aeddb29f963..809ffd74786 100644 --- a/official/colab/nlp/customize_encoder.ipynb +++ b/official/colab/nlp/customize_encoder.ipynb @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + { "nbformat": 4, "nbformat_minor": 0, diff --git a/official/colab/nlp/nlp_modeling_library_intro.ipynb b/official/colab/nlp/nlp_modeling_library_intro.ipynb index e4ce780c96b..ae6ad9e5f76 100644 --- a/official/colab/nlp/nlp_modeling_library_intro.ipynb +++ b/official/colab/nlp/nlp_modeling_library_intro.ipynb @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + { "cells": [ { diff --git a/official/nlp/README.md b/official/nlp/README.md index dfa047b4ed3..733a218bcce 100644 --- a/official/nlp/README.md +++ b/official/nlp/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # TensorFlow NLP Modelling Toolkit This codebase provides a Natrual Language Processing modeling toolkit written in diff --git a/official/nlp/albert/README.md b/official/nlp/albert/README.md index 69620e0579f..33d92c81ab4 100644 --- a/official/nlp/albert/README.md +++ b/official/nlp/albert/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ALBERT (ALBERT: A Lite BERT for Self-supervised Learning of Language Representations) The academic paper which describes ALBERT in detail and provides full results on diff --git a/official/nlp/bert/README.md b/official/nlp/bert/README.md index 037ff0b1ff8..c2c545308c3 100644 --- a/official/nlp/bert/README.md +++ b/official/nlp/bert/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # BERT (Bidirectional Encoder Representations from Transformers) **WARNING**: We are on the way to deprecate most of the code in this directory. diff --git a/official/nlp/bert/bert_cloud_tpu.md b/official/nlp/bert/bert_cloud_tpu.md index baf6f9bdc0c..60f2e77acd8 100644 --- a/official/nlp/bert/bert_cloud_tpu.md +++ b/official/nlp/bert/bert_cloud_tpu.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # BERT FineTuning with Cloud TPU: Sentence and Sentence-Pair Classification Tasks (TF 2.1) This tutorial shows you how to train the Bidirectional Encoder Representations from Transformers (BERT) model on Cloud TPU. diff --git a/official/nlp/configs/experiments/glue_mnli_matched.yaml b/official/nlp/configs/experiments/glue_mnli_matched.yaml index 29dfcb68b9c..6580a5e2898 100644 --- a/official/nlp/configs/experiments/glue_mnli_matched.yaml +++ b/official/nlp/configs/experiments/glue_mnli_matched.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + task: hub_module_url: '' model: diff --git a/official/nlp/configs/experiments/squad_v1.yaml b/official/nlp/configs/experiments/squad_v1.yaml index a69710a58f7..89d1ff90d89 100644 --- a/official/nlp/configs/experiments/squad_v1.yaml +++ b/official/nlp/configs/experiments/squad_v1.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + task: hub_module_url: '' max_answer_length: 30 diff --git a/official/nlp/configs/models/bert_en_uncased_base.yaml b/official/nlp/configs/models/bert_en_uncased_base.yaml index 1e49bc5430e..22085984a75 100644 --- a/official/nlp/configs/models/bert_en_uncased_base.yaml +++ b/official/nlp/configs/models/bert_en_uncased_base.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + task: model: encoder: diff --git a/official/nlp/docs/pretrained_models.md b/official/nlp/docs/pretrained_models.md index 0c836b33b7d..3c64efd4876 100644 --- a/official/nlp/docs/pretrained_models.md +++ b/official/nlp/docs/pretrained_models.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Pre-trained Models We provide a large collection of baselines and checkpoints for NLP pre-trained diff --git a/official/nlp/docs/tfhub.md b/official/nlp/docs/tfhub.md index c6fe9a2f8f4..505051db5a8 100644 --- a/official/nlp/docs/tfhub.md +++ b/official/nlp/docs/tfhub.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Exporting a pre-trained Encoder to TF Hub ## Overview diff --git a/official/nlp/docs/train.md b/official/nlp/docs/train.md index d2ad9d7622d..69bef3cc5d4 100644 --- a/official/nlp/docs/train.md +++ b/official/nlp/docs/train.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Model Garden NLP Common Training Driver [train.py](https://github.com/tensorflow/models/blob/master/official/nlp/train.py) is the common training driver that supports multiple diff --git a/official/nlp/keras_nlp/README.md b/official/nlp/keras_nlp/README.md index 1c5bbb13182..993d8684ca6 100644 --- a/official/nlp/keras_nlp/README.md +++ b/official/nlp/keras_nlp/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # keras-nlp ## Layers diff --git a/official/nlp/keras_nlp/contributing.md b/official/nlp/keras_nlp/contributing.md index b9ec1716d96..6974ef6666c 100644 --- a/official/nlp/keras_nlp/contributing.md +++ b/official/nlp/keras_nlp/contributing.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + ## Contributing to KerasNLP Patches to KerasNLP are welcome! diff --git a/official/nlp/keras_nlp/requirements.txt b/official/nlp/keras_nlp/requirements.txt index c765b1ead22..822aed5d9d5 100644 --- a/official/nlp/keras_nlp/requirements.txt +++ b/official/nlp/keras_nlp/requirements.txt @@ -1 +1,15 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + numpy>=1.15.4 diff --git a/official/nlp/modeling/README.md b/official/nlp/modeling/README.md index 99c7c361f97..fe192621493 100644 --- a/official/nlp/modeling/README.md +++ b/official/nlp/modeling/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # NLP Modeling Library This library provides a set of Keras primitives (`tf.keras.Layer` and diff --git a/official/nlp/modeling/layers/README.md b/official/nlp/modeling/layers/README.md index 79e142a0887..43a7e271ef7 100644 --- a/official/nlp/modeling/layers/README.md +++ b/official/nlp/modeling/layers/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Layers Layers are the fundamental building blocks for NLP models. They can be used to diff --git a/official/nlp/modeling/losses/README.md b/official/nlp/modeling/losses/README.md index a2607b1dab7..94a5808cbf0 100644 --- a/official/nlp/modeling/losses/README.md +++ b/official/nlp/modeling/losses/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Losses Losses contains common loss computation used in NLP tasks. diff --git a/official/nlp/modeling/models/README.md b/official/nlp/modeling/models/README.md index 22fd8193c29..b7fbc7ab74f 100644 --- a/official/nlp/modeling/models/README.md +++ b/official/nlp/modeling/models/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Models Models are combinations of `tf.keras` layers and models that can be trained. diff --git a/official/nlp/modeling/networks/README.md b/official/nlp/modeling/networks/README.md index b192399a727..25547a7449c 100644 --- a/official/nlp/modeling/networks/README.md +++ b/official/nlp/modeling/networks/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Networks Networks are combinations of `tf.keras` layers (and possibly other networks). diff --git a/official/nlp/nhnet/README.md b/official/nlp/nhnet/README.md index f838d120fb8..7536133fcd7 100644 --- a/official/nlp/nhnet/README.md +++ b/official/nlp/nhnet/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Multi-doc News Headline Generation Model: NHNet This repository contains TensorFlow 2.x implementation for NHNet [[1]](#1) as diff --git a/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.html b/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.html index 0a8549c1d27..c3085f2cf1d 100644 --- a/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.html +++ b/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.html @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + Page Title 0 diff --git a/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.json b/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.json index b7308592b77..0cfb775b567 100644 --- a/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.json +++ b/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.json @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + { "title": "title for 0", "maintext": "text snippet for 0", diff --git a/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.html b/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.html index 7c8bb8d285c..9f13aee1864 100644 --- a/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.html +++ b/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.html @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + Page Title 1 diff --git a/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.json b/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.json index dbc2322c7de..cb51d6ce321 100644 --- a/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.json +++ b/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.json @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + { "title": "title for 1", "maintext": "text snippet for 1", diff --git a/official/nlp/nhnet/testdata/stories.json b/official/nlp/nhnet/testdata/stories.json index 0618f3d5c8a..b20c6cc5ae1 100644 --- a/official/nlp/nhnet/testdata/stories.json +++ b/official/nlp/nhnet/testdata/stories.json @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + [ { "urls": [ diff --git a/official/nlp/nhnet/testdata/vocab.txt b/official/nlp/nhnet/testdata/vocab.txt index dd708d71c2f..20f9567cd48 100644 --- a/official/nlp/nhnet/testdata/vocab.txt +++ b/official/nlp/nhnet/testdata/vocab.txt @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + [UNK] [CLS] [SEP] diff --git a/official/nlp/projects/bigbird/README.md b/official/nlp/projects/bigbird/README.md index 7c5435f154d..20849d79d87 100644 --- a/official/nlp/projects/bigbird/README.md +++ b/official/nlp/projects/bigbird/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # BigBird: Transformers for Longer Sequences [BigBird](https://arxiv.org/abs/2007.14062) diff --git a/official/nlp/projects/bigbird/experiments/glue_mnli_matched.yaml b/official/nlp/projects/bigbird/experiments/glue_mnli_matched.yaml index f38bdfc8b32..6b0556998a4 100644 --- a/official/nlp/projects/bigbird/experiments/glue_mnli_matched.yaml +++ b/official/nlp/projects/bigbird/experiments/glue_mnli_matched.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + task: hub_module_url: '' model: diff --git a/official/nlp/projects/bigbird/experiments/squad_v1.yaml b/official/nlp/projects/bigbird/experiments/squad_v1.yaml index 7971f1b2472..af1dfb74269 100644 --- a/official/nlp/projects/bigbird/experiments/squad_v1.yaml +++ b/official/nlp/projects/bigbird/experiments/squad_v1.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + task: hub_module_url: '' model: diff --git a/official/nlp/projects/mobilebert/README.md b/official/nlp/projects/mobilebert/README.md index 9209b4720d6..2ff549e3303 100644 --- a/official/nlp/projects/mobilebert/README.md +++ b/official/nlp/projects/mobilebert/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # MobileBERT (MobileBERT: A Compact Task-Agnostic BERT for Resource-Limited Devices) [MobileBERT](https://arxiv.org/abs/2004.02984) diff --git a/official/nlp/projects/mobilebert/experiments/en_uncased_student.yaml b/official/nlp/projects/mobilebert/experiments/en_uncased_student.yaml index cfcf40c2b89..0f989dc4cdc 100644 --- a/official/nlp/projects/mobilebert/experiments/en_uncased_student.yaml +++ b/official/nlp/projects/mobilebert/experiments/en_uncased_student.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + task: model: encoder: diff --git a/official/nlp/projects/mobilebert/experiments/en_uncased_teacher.yaml b/official/nlp/projects/mobilebert/experiments/en_uncased_teacher.yaml index eeee8537da5..15db0a3dc21 100644 --- a/official/nlp/projects/mobilebert/experiments/en_uncased_teacher.yaml +++ b/official/nlp/projects/mobilebert/experiments/en_uncased_teacher.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + task: model: encoder: diff --git a/official/nlp/projects/mobilebert/experiments/mobilebert_distillation_en_uncased.yaml b/official/nlp/projects/mobilebert/experiments/mobilebert_distillation_en_uncased.yaml index 74e6adc3c47..8c2a9044d2d 100644 --- a/official/nlp/projects/mobilebert/experiments/mobilebert_distillation_en_uncased.yaml +++ b/official/nlp/projects/mobilebert/experiments/mobilebert_distillation_en_uncased.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + task: train_data: drop_remainder: true diff --git a/official/nlp/projects/teams/README.md b/official/nlp/projects/teams/README.md index f57aa266d06..8faa6e2ead6 100644 --- a/official/nlp/projects/teams/README.md +++ b/official/nlp/projects/teams/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # TEAMS (Training ELECTRA Augmented with Multi-word Selection) **Note:** This project is working in progress and please stay tuned. diff --git a/official/nlp/projects/tn_bert/README.md b/official/nlp/projects/tn_bert/README.md index 50928155807..c513972fc23 100644 --- a/official/nlp/projects/tn_bert/README.md +++ b/official/nlp/projects/tn_bert/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # TN-BERT (TensorNetwork BERT) TN-BERT is a modification of the BERT-base architecture that greatly compresses diff --git a/official/nlp/transformer/README.md b/official/nlp/transformer/README.md index a3aec5f9a05..d67e3df71ee 100644 --- a/official/nlp/transformer/README.md +++ b/official/nlp/transformer/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Transformer Translation Model This is an implementation of the Transformer translation model as described in the [Attention is All You Need](https://arxiv.org/abs/1706.03762) paper. The diff --git a/official/nlp/xlnet/README.md b/official/nlp/xlnet/README.md index 546d1128e2d..417c37d5a0f 100644 --- a/official/nlp/xlnet/README.md +++ b/official/nlp/xlnet/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # XLNet: Generalized Autoregressive Pretraining for Language Understanding The academic paper which describes XLNet in detail and provides full results on diff --git a/official/pip_package/setup.py b/official/pip_package/setup.py index 0478191f5c5..cfc7a751f29 100644 --- a/official/pip_package/setup.py +++ b/official/pip_package/setup.py @@ -1,4 +1,4 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# ============================================================================== + """Sets up TensorFlow Official Models.""" import datetime import os diff --git a/official/recommendation/README.md b/official/recommendation/README.md index ea2abfadcab..7e9f784240c 100644 --- a/official/recommendation/README.md +++ b/official/recommendation/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Recommendation Model ## Overview This is an implementation of the Neural Collaborative Filtering (NCF) framework with Neural Matrix Factorization (NeuMF) model as described in the [Neural Collaborative Filtering](https://arxiv.org/abs/1708.05031) paper. Current implementation is based on the code from the authors' [NCF code](https://github.com/hexiangnan/neural_collaborative_filtering) and the Stanford implementation in the [MLPerf Repo](https://github.com/mlperf/reference/tree/master/recommendation/pytorch). diff --git a/official/recommendation/ranking/README.md b/official/recommendation/ranking/README.md index 9c2ca21039f..b2d891848f7 100644 --- a/official/recommendation/ranking/README.md +++ b/official/recommendation/ranking/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # TF Model Garden Ranking Models ## Overview diff --git a/official/recommendation/ranking/configs/yaml/dcn_v2_criteo_tpu.yaml b/official/recommendation/ranking/configs/yaml/dcn_v2_criteo_tpu.yaml index a281e03259b..6532d59e8c1 100644 --- a/official/recommendation/ranking/configs/yaml/dcn_v2_criteo_tpu.yaml +++ b/official/recommendation/ranking/configs/yaml/dcn_v2_criteo_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + runtime: distribution_strategy: 'tpu' task: diff --git a/official/recommendation/ranking/configs/yaml/dlrm_criteo_tpu.yaml b/official/recommendation/ranking/configs/yaml/dlrm_criteo_tpu.yaml index aaaadf58e60..79ea6fb6e33 100644 --- a/official/recommendation/ranking/configs/yaml/dlrm_criteo_tpu.yaml +++ b/official/recommendation/ranking/configs/yaml/dlrm_criteo_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + runtime: distribution_strategy: 'tpu' task: diff --git a/official/recommendation/run.sh b/official/recommendation/run.sh index b8e1143a38b..935a7d01830 100755 --- a/official/recommendation/run.sh +++ b/official/recommendation/run.sh @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + #!/bin/bash set -e diff --git a/official/requirements.txt b/official/requirements.txt index 74028adcb55..fc95b6dae90 100644 --- a/official/requirements.txt +++ b/official/requirements.txt @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + six google-api-python-client>=1.6.7 google-cloud-bigquery>=0.31.0 diff --git a/official/staging/__init__.py b/official/staging/__init__.py index e69de29bb2d..e419af524b5 100644 --- a/official/staging/__init__.py +++ b/official/staging/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/official/staging/training/__init__.py b/official/staging/training/__init__.py index 931c2ef11db..e419af524b5 100644 --- a/official/staging/training/__init__.py +++ b/official/staging/training/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,4 +11,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# ============================================================================== + diff --git a/official/staging/training/grad_utils.py b/official/staging/training/grad_utils.py index 48e7566ed9a..1113d39d5e6 100644 --- a/official/staging/training/grad_utils.py +++ b/official/staging/training/grad_utils.py @@ -1,4 +1,4 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# ============================================================================== + """Some gradient util functions to help users writing custom training loop.""" from absl import logging diff --git a/official/utils/flags/README.md b/official/utils/flags/README.md index beb3b2a1e1d..49619bdbf2d 100644 --- a/official/utils/flags/README.md +++ b/official/utils/flags/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Adding Abseil (absl) flags quickstart **WARNING** This module is deprecated. We no long use it in new models and diff --git a/official/utils/flags/guidelines.md b/official/utils/flags/guidelines.md index db963aabebc..fd3720c8596 100644 --- a/official/utils/flags/guidelines.md +++ b/official/utils/flags/guidelines.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Using flags in official models 1. **All common flags must be incorporated in the models.** diff --git a/official/utils/testing/pylint.rcfile b/official/utils/testing/pylint.rcfile index b872802a811..33ce61e4c39 100644 --- a/official/utils/testing/pylint.rcfile +++ b/official/utils/testing/pylint.rcfile @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + [MESSAGES CONTROL] disable=R,W,bad-option-value,trailing-newlines,no-name-in-module diff --git a/official/utils/testing/scripts/builds_common.sh b/official/utils/testing/scripts/builds_common.sh index 3cf08bb510d..7bea3ef09b3 100644 --- a/official/utils/testing/scripts/builds_common.sh +++ b/official/utils/testing/scripts/builds_common.sh @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + #!/usr/bin/env bash # Copyright 2019 The TensorFlow Authors. All Rights Reserved. # diff --git a/official/utils/testing/scripts/ci_sanity.sh b/official/utils/testing/scripts/ci_sanity.sh index 0646c87a943..4e73b6288fa 100755 --- a/official/utils/testing/scripts/ci_sanity.sh +++ b/official/utils/testing/scripts/ci_sanity.sh @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + #!/bin/bash # Copyright 2019 The TensorFlow Authors. All Rights Reserved. # diff --git a/official/utils/testing/scripts/presubmit.sh b/official/utils/testing/scripts/presubmit.sh index 33eca3cbb41..6ef7f5a4ab2 100755 --- a/official/utils/testing/scripts/presubmit.sh +++ b/official/utils/testing/scripts/presubmit.sh @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + #!/bin/bash # Copyright 2018 The TensorFlow Authors. All Rights Reserved. # diff --git a/official/vision/beta/MODEL_GARDEN.md b/official/vision/beta/MODEL_GARDEN.md index 42f7eea546b..da804662a39 100644 --- a/official/vision/beta/MODEL_GARDEN.md +++ b/official/vision/beta/MODEL_GARDEN.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # TF Vision Model Garden ## Introduction diff --git a/official/vision/beta/README.md b/official/vision/beta/README.md index 7d8f84fd5bd..5e163bebf7d 100644 --- a/official/vision/beta/README.md +++ b/official/vision/beta/README.md @@ -1,2 +1,16 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + This directory contains the new design of TF model garden vision framework. Stay tuned. diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_mobilenetv2_gpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_mobilenetv2_gpu.yaml index ff1a0719e6f..ef1f2f1b5a3 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_mobilenetv2_gpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_mobilenetv2_gpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # MobileNetV2_1.0 ImageNet classification. 71.0% top-1 and 90.0% top-5 accuracy. runtime: distribution_strategy: 'mirrored' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_mobilenetv2_tpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_mobilenetv2_tpu.yaml index b5df9d6e74a..f16c6315576 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_mobilenetv2_tpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_mobilenetv2_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # MobileNetV2_1.0 ImageNet classification. 72.72% top-1 and 91.05% top-5 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet101_deeplab_tpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet101_deeplab_tpu.yaml index 5d7d2959637..3846a6c4b12 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet101_deeplab_tpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet101_deeplab_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Top-1 accuracy 81.6% on ImageNet runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet101_tpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet101_tpu.yaml index 2600f58faa5..a777f0c9130 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet101_tpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet101_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ResNet-101 ImageNet classification. 79.1% top-1 and 94.5% top-5 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet152_tpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet152_tpu.yaml index 1c81953e2f6..700b71317c0 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet152_tpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet152_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ResNet-152 ImageNet classification. 79.4% top-1 and 94.7% top-5 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_deeplab_tpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_deeplab_tpu.yaml index 11bdafbc35d..6bcdde4a616 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_deeplab_tpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_deeplab_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + runtime: distribution_strategy: 'tpu' mixed_precision_dtype: 'bfloat16' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_gpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_gpu.yaml index dd6a4dc1618..e70ba6ce198 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_gpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_gpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + runtime: distribution_strategy: 'mirrored' mixed_precision_dtype: 'float16' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_tfds_tpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_tfds_tpu.yaml index 1506b48f903..382e3fbc700 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_tfds_tpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_tfds_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ResNet-50 ImageNet classification. 78.1% top-1 and 93.9% top-5 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_tpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_tpu.yaml index 7fd10535aa8..3ba1b2b7771 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_tpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ResNet-50 ImageNet classification. 78.1% top-1 and 93.9% top-5 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs101_i160.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs101_i160.yaml index 7c9e7b80a02..cff3d32b3d4 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs101_i160.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs101_i160.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ResNet-RS-101 ImageNet classification. 80.2% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs101_i192.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs101_i192.yaml index 576c4862505..28c5c75388b 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs101_i192.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs101_i192.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ResNet-RS-101 ImageNet classification. 81.3% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i192.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i192.yaml index b1c8edc463f..6c763f33184 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i192.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i192.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ResNet-RS-152 ImageNet classification. 81.9% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i224.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i224.yaml index 2ec14bae5ab..5a3c4d1c399 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i224.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i224.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ResNet-RS-152 ImageNet classification. 82.5% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i256.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i256.yaml index 91b53d6217f..548c5c5330d 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i256.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i256.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ResNet-RS-152 ImageNet classification. 83.1% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs200_i256.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs200_i256.yaml index 9d76c010170..79968298ce8 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs200_i256.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs200_i256.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ResNet-RS-200 ImageNet classification. 83.5% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs270_i256.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs270_i256.yaml index b7c6a644e2c..4849a457e14 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs270_i256.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs270_i256.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ResNet-RS-270 ImageNet classification. 83.6% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs350_i256.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs350_i256.yaml index 3b2d3fe261c..a025dd481a0 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs350_i256.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs350_i256.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ResNet-RS-350 ImageNet classification. 83.7% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs350_i320.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs350_i320.yaml index 36cdba7bb43..5110ad6459d 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs350_i320.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs350_i320.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ResNet-RS-350 ImageNet classification. 84.2% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs420_i320.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs420_i320.yaml index 9b02b7e006a..7d249604175 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs420_i320.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs420_i320.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + runtime: distribution_strategy: 'tpu' mixed_precision_dtype: 'bfloat16' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs50_i160.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs50_i160.yaml index a57f41f3908..8303553e3b9 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs50_i160.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs50_i160.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ResNet-RS-50 ImageNet classification. 79.1% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet143_cascadercnn_tpu.yaml b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet143_cascadercnn_tpu.yaml index 1f8b245da76..0121ef53148 100644 --- a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet143_cascadercnn_tpu.yaml +++ b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet143_cascadercnn_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # --experiment_type=cascadercnn_spinenet_coco # Expect to reach: box mAP: 51.9%, mask mAP: 45.0% on COCO runtime: diff --git a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet143_mrcnn_tpu.yaml b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet143_mrcnn_tpu.yaml index 4d5ec8ae481..0d83e409379 100644 --- a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet143_mrcnn_tpu.yaml +++ b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet143_mrcnn_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Expect to reach: box mAP: 49.3%, mask mAP: 43.4% on COCO runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet49_cascadercnn_tpu.yaml b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet49_cascadercnn_tpu.yaml index a28dd4bb0fc..26df9523e82 100644 --- a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet49_cascadercnn_tpu.yaml +++ b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet49_cascadercnn_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # --experiment_type=cascadercnn_spinenet_coco # Expect to reach: box mAP: 46.4%, mask mAP: 40.0% on COCO runtime: diff --git a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet49_mrcnn_tpu.yaml b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet49_mrcnn_tpu.yaml index 4ac1ae428bf..674ceac933a 100644 --- a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet49_mrcnn_tpu.yaml +++ b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet49_mrcnn_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Expect to reach: box mAP: 43.2%, mask mAP: 38.3% on COCO runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet96_casrcnn_tpu.yaml b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet96_casrcnn_tpu.yaml index 612608333c3..2ec84bd7dad 100644 --- a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet96_casrcnn_tpu.yaml +++ b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet96_casrcnn_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + runtime: distribution_strategy: 'tpu' mixed_precision_dtype: 'bfloat16' diff --git a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet96_mrcnn_tpu.yaml b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet96_mrcnn_tpu.yaml index 9609b7eee34..d30a532a098 100644 --- a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet96_mrcnn_tpu.yaml +++ b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet96_mrcnn_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Expect to reach: box mAP: 48.1%, mask mAP: 42.4% on COCO runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/maskrcnn/r50fpn_640_coco_scratch_tpu4x4.yaml b/official/vision/beta/configs/experiments/maskrcnn/r50fpn_640_coco_scratch_tpu4x4.yaml index 218f0451058..1ffc6f9e641 100644 --- a/official/vision/beta/configs/experiments/maskrcnn/r50fpn_640_coco_scratch_tpu4x4.yaml +++ b/official/vision/beta/configs/experiments/maskrcnn/r50fpn_640_coco_scratch_tpu4x4.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Expect to reach: box mAP: 42.3%, mask mAP: 37.6% on COCO task: init_checkpoint: null diff --git a/official/vision/beta/configs/experiments/retinanet/coco_spinenet143_tpu.yaml b/official/vision/beta/configs/experiments/retinanet/coco_spinenet143_tpu.yaml index 438fe031a8b..dc3bc2b1710 100644 --- a/official/vision/beta/configs/experiments/retinanet/coco_spinenet143_tpu.yaml +++ b/official/vision/beta/configs/experiments/retinanet/coco_spinenet143_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # SpineNet-143 COCO detection with protocal C config. Expecting 50.0% AP. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/retinanet/coco_spinenet190_tpu.yaml b/official/vision/beta/configs/experiments/retinanet/coco_spinenet190_tpu.yaml index bc0ea1f94ec..653ef5fd791 100644 --- a/official/vision/beta/configs/experiments/retinanet/coco_spinenet190_tpu.yaml +++ b/official/vision/beta/configs/experiments/retinanet/coco_spinenet190_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + runtime: distribution_strategy: 'tpu' mixed_precision_dtype: 'bfloat16' diff --git a/official/vision/beta/configs/experiments/retinanet/coco_spinenet49_mobile_tpu.yaml b/official/vision/beta/configs/experiments/retinanet/coco_spinenet49_mobile_tpu.yaml index 116dd008493..094e067040e 100644 --- a/official/vision/beta/configs/experiments/retinanet/coco_spinenet49_mobile_tpu.yaml +++ b/official/vision/beta/configs/experiments/retinanet/coco_spinenet49_mobile_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # --experiment_type=retinanet_spinenet_mobile_coco runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/retinanet/coco_spinenet49_tpu.yaml b/official/vision/beta/configs/experiments/retinanet/coco_spinenet49_tpu.yaml index 725e1fc9b84..1587130e978 100644 --- a/official/vision/beta/configs/experiments/retinanet/coco_spinenet49_tpu.yaml +++ b/official/vision/beta/configs/experiments/retinanet/coco_spinenet49_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # SpineNet-49 COCO detection with protocal C config. Expecting 44.2% AP. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/retinanet/coco_spinenet49s_mobile_tpu.yaml b/official/vision/beta/configs/experiments/retinanet/coco_spinenet49s_mobile_tpu.yaml index 3b89a626759..f62ec9b348f 100644 --- a/official/vision/beta/configs/experiments/retinanet/coco_spinenet49s_mobile_tpu.yaml +++ b/official/vision/beta/configs/experiments/retinanet/coco_spinenet49s_mobile_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # --experiment_type=retinanet_spinenet_mobile_coco runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/retinanet/coco_spinenet49xs_mobile_tpu.yaml b/official/vision/beta/configs/experiments/retinanet/coco_spinenet49xs_mobile_tpu.yaml index 8c9bf3a0b1f..028a84beebf 100644 --- a/official/vision/beta/configs/experiments/retinanet/coco_spinenet49xs_mobile_tpu.yaml +++ b/official/vision/beta/configs/experiments/retinanet/coco_spinenet49xs_mobile_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # --experiment_type=retinanet_spinenet_mobile_coco runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/retinanet/coco_spinenet96_tpu.yaml b/official/vision/beta/configs/experiments/retinanet/coco_spinenet96_tpu.yaml index c75d667753f..7b794cd1651 100644 --- a/official/vision/beta/configs/experiments/retinanet/coco_spinenet96_tpu.yaml +++ b/official/vision/beta/configs/experiments/retinanet/coco_spinenet96_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # SpineNet-96 COCO detection with protocol C config. Expecting 48.5% AP. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/retinanet/resnet50fpn_coco_tfds_tpu.yaml b/official/vision/beta/configs/experiments/retinanet/resnet50fpn_coco_tfds_tpu.yaml index 0f9a30a3443..0d6cf883907 100644 --- a/official/vision/beta/configs/experiments/retinanet/resnet50fpn_coco_tfds_tpu.yaml +++ b/official/vision/beta/configs/experiments/retinanet/resnet50fpn_coco_tfds_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + runtime: distribution_strategy: 'tpu' mixed_precision_dtype: 'bfloat16' diff --git a/official/vision/beta/configs/experiments/retinanet/resnet50fpn_coco_tpu4x4_benchmark.yaml b/official/vision/beta/configs/experiments/retinanet/resnet50fpn_coco_tpu4x4_benchmark.yaml index 46b1f3cad7c..1523946cb1b 100644 --- a/official/vision/beta/configs/experiments/retinanet/resnet50fpn_coco_tpu4x4_benchmark.yaml +++ b/official/vision/beta/configs/experiments/retinanet/resnet50fpn_coco_tpu4x4_benchmark.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Benchmarks runs on same instnace, change eval batch size to fit on 4x4 tpu task: validation_data: diff --git a/official/vision/beta/configs/experiments/semantic_segmentation/deeplabv3plus_resnet101_cityscapes_tfds_tpu.yaml b/official/vision/beta/configs/experiments/semantic_segmentation/deeplabv3plus_resnet101_cityscapes_tfds_tpu.yaml index 4ffc7689d2f..bf929ee4ee4 100644 --- a/official/vision/beta/configs/experiments/semantic_segmentation/deeplabv3plus_resnet101_cityscapes_tfds_tpu.yaml +++ b/official/vision/beta/configs/experiments/semantic_segmentation/deeplabv3plus_resnet101_cityscapes_tfds_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Use your own cityscapes preprocessed dataset. 79% meanIoU. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/video_classification/k400_3d-resnet50_tpu.yaml b/official/vision/beta/configs/experiments/video_classification/k400_3d-resnet50_tpu.yaml index d9158c2bd17..704c727ffe6 100644 --- a/official/vision/beta/configs/experiments/video_classification/k400_3d-resnet50_tpu.yaml +++ b/official/vision/beta/configs/experiments/video_classification/k400_3d-resnet50_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # 3D ResNet-50 video classification on Kinetics-400. # # --experiment_type=video_classification_kinetics400 diff --git a/official/vision/beta/configs/experiments/video_classification/k400_resnet3drs_50_tpu.yaml b/official/vision/beta/configs/experiments/video_classification/k400_resnet3drs_50_tpu.yaml index 83875d1273a..9e1977297ec 100644 --- a/official/vision/beta/configs/experiments/video_classification/k400_resnet3drs_50_tpu.yaml +++ b/official/vision/beta/configs/experiments/video_classification/k400_resnet3drs_50_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # 3D ResNet-RS-50 video classification on Kinetics-400. # # --experiment_type=video_classification_kinetics400 diff --git a/official/vision/beta/configs/experiments/video_classification/k400_slowonly16x4_tpu.yaml b/official/vision/beta/configs/experiments/video_classification/k400_slowonly16x4_tpu.yaml index 8e6793a374e..b6d20bfb4c1 100644 --- a/official/vision/beta/configs/experiments/video_classification/k400_slowonly16x4_tpu.yaml +++ b/official/vision/beta/configs/experiments/video_classification/k400_slowonly16x4_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # SlowOnly 16x4 video classification on Kinetics-400. # # --experiment_type=video_classification_kinetics400 diff --git a/official/vision/beta/configs/experiments/video_classification/k400_slowonly8x8_tpu.yaml b/official/vision/beta/configs/experiments/video_classification/k400_slowonly8x8_tpu.yaml index c0bcd881ef3..c711f2e105c 100644 --- a/official/vision/beta/configs/experiments/video_classification/k400_slowonly8x8_tpu.yaml +++ b/official/vision/beta/configs/experiments/video_classification/k400_slowonly8x8_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # SlowOnly 8x8 video classification on Kinetics-400. # # --experiment_type=video_classification_kinetics400 diff --git a/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50_tpu.yaml b/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50_tpu.yaml index ceb38608d6d..348a2466f6b 100644 --- a/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50_tpu.yaml +++ b/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # 3D ResNet-50 video classification on Kinetics-600. # # --experiment_type=video_classification_kinetics600 diff --git a/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50g_tpu.yaml b/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50g_tpu.yaml index 3ae54c41564..38627270718 100644 --- a/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50g_tpu.yaml +++ b/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50g_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # 3D ResNet-50g video classification on Kinetics-600. # # --experiment_type=video_classification_kinetics600 diff --git a/official/vision/beta/configs/experiments/video_classification/k600_slowonly8x8_tpu.yaml b/official/vision/beta/configs/experiments/video_classification/k600_slowonly8x8_tpu.yaml index 43f656ce3f0..6e7e3e1fdb2 100644 --- a/official/vision/beta/configs/experiments/video_classification/k600_slowonly8x8_tpu.yaml +++ b/official/vision/beta/configs/experiments/video_classification/k600_slowonly8x8_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # SlowOnly 8x8 video classification on Kinetics-600. # # --experiment_type=video_classification_kinetics600 diff --git a/official/vision/beta/projects/README.md b/official/vision/beta/projects/README.md index 9c20f07fc60..93cc973ce8f 100644 --- a/official/vision/beta/projects/README.md +++ b/official/vision/beta/projects/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + Here are a few projects that are built on tf.vision. They are build and maintain by different parties. They can be used as examples of how to build your own projects based on tf.vision. diff --git a/official/vision/beta/projects/assemblenet/README.md b/official/vision/beta/projects/assemblenet/README.md index fb19541f9fc..66b9c56537e 100644 --- a/official/vision/beta/projects/assemblenet/README.md +++ b/official/vision/beta/projects/assemblenet/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # AssembleNet and AssembleNet++ This repository is the official implementations of the following papers. diff --git a/official/vision/beta/projects/deepmac_maskrcnn/README.md b/official/vision/beta/projects/deepmac_maskrcnn/README.md index 1e241f2cf9b..ed0b1be8b2a 100644 --- a/official/vision/beta/projects/deepmac_maskrcnn/README.md +++ b/official/vision/beta/projects/deepmac_maskrcnn/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Mask R-CNN with deep mask heads This project brings insights from the DeepMAC model into the Mask-RCNN diff --git a/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r101_hg52.yaml b/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r101_hg52.yaml index 1a097653808..a3e4d443ae3 100644 --- a/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r101_hg52.yaml +++ b/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r101_hg52.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + task: # VOC class taken from # models/official/vision/detection/utils/class_utils.py diff --git a/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r50.yaml b/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r50.yaml index 792bc9c1f7c..cb925d38fea 100644 --- a/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r50.yaml +++ b/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r50.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + task: # VOC class taken from # models/official/vision/detection/utils/class_utils.py diff --git a/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r50_hg52.yaml b/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r50_hg52.yaml index 060bfb61b2e..79f25ec5aae 100644 --- a/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r50_hg52.yaml +++ b/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r50_hg52.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + task: # VOC class taken from # models/official/vision/detection/utils/class_utils.py diff --git a/official/vision/beta/projects/movinet/README.md b/official/vision/beta/projects/movinet/README.md index 5ccf1d3e838..a19576cf97e 100644 --- a/official/vision/beta/projects/movinet/README.md +++ b/official/vision/beta/projects/movinet/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Mobile Video Networks (MoViNets) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tensorflow/models/blob/master/official/vision/beta/projects/movinet/movinet_tutorial.ipynb) diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_8x8.yaml index bd7d3ce92a9..0c3ea819ebc 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_8x8.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Video classification on Kinetics-600 using MoViNet-A0 backbone. # --experiment_type=movinet_kinetics600 # Achieves 72.28% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_cpu_local.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_cpu_local.yaml index a144ac56e4d..4424f422139 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_cpu_local.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_cpu_local.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Video classification on Kinetics-600 using MoViNet-A0 backbone. # --experiment_type=movinet_kinetics600 diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_stream_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_stream_k600_8x8.yaml index df4afcc4dab..4ed49a1be98 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_stream_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_stream_k600_8x8.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Video classification on Kinetics-600 using MoViNet-A0-Stream backbone. # --experiment_type=movinet_kinetics600 # Achieves 72.03% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a1_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a1_k600_8x8.yaml index 8c097f49b4a..e05c3c411a9 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a1_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a1_k600_8x8.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Video classification on Kinetics-600 using MoViNet-A1 backbone. # --experiment_type=movinet_kinetics600 # Achieves 76.69% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a1_stream_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a1_stream_k600_8x8.yaml index 3452067ad9c..6abf33886cb 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a1_stream_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a1_stream_k600_8x8.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Video classification on Kinetics-600 using MoViNet-A1-Stream backbone. # --experiment_type=movinet_kinetics600 # Achieves 76.45% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a2_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a2_k600_8x8.yaml index 575772b9f3e..8e9581d7d86 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a2_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a2_k600_8x8.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Video classification on Kinetics-600 using MoViNet-A2 backbone. # --experiment_type=movinet_kinetics600 # Achieves 78.62% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a2_stream_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a2_stream_k600_8x8.yaml index 206c7940311..83a6cd04ea1 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a2_stream_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a2_stream_k600_8x8.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Video classification on Kinetics-600 using MoViNet-A2-Stream backbone. # --experiment_type=movinet_kinetics600 # Achieves 78.40% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a3_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a3_k600_8x8.yaml index a4d34314695..b0f503aa670 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a3_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a3_k600_8x8.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Video classification on Kinetics-600 using MoViNet-A3 backbone. # --experiment_type=movinet_kinetics600 # Achieves 81.79% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a3_stream_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a3_stream_k600_8x8.yaml index d503e3dc6f9..62ec0e0ae5c 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a3_stream_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a3_stream_k600_8x8.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Video classification on Kinetics-600 using MoViNet-A3-Stream backbone. # --experiment_type=movinet_kinetics600 # Achieves 80.09% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a4_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a4_k600_8x8.yaml index 102ccad4f55..6b689bb41ff 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a4_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a4_k600_8x8.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Video classification on Kinetics-600 using MoViNet-A4 backbone. # --experiment_type=movinet_kinetics600 # Achieves 83.48% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a4_stream_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a4_stream_k600_8x8.yaml index 63b9865d366..2eab748d043 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a4_stream_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a4_stream_k600_8x8.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Video classification on Kinetics-600 using MoViNet-A4 backbone. # --experiment_type=movinet_kinetics600 # Achieves 81.33% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_k600_8x8.yaml index 79c9d209d91..a6aa551e71f 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_k600_8x8.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Video classification on Kinetics-600 using MoViNet-A5 backbone. # --experiment_type=movinet_kinetics600 # Achieves 84.00% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_stream_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_stream_k600_8x8.yaml index 1983937679f..d1bcc87a967 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_stream_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_stream_k600_8x8.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Video classification on Kinetics-600 using MoViNet-A5-Stream backbone. # --experiment_type=movinet_kinetics600 # Achieves 82.37% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_t0_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_t0_k600_8x8.yaml index b6b190c8acb..67495e7f042 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_t0_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_t0_k600_8x8.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Video classification on Kinetics-600 using MoViNet-T0 backbone. # --experiment_type=movinet_kinetics600 # Achieves 68.40% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_t0_stream_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_t0_stream_k600_8x8.yaml index 320a51ea681..ebf0cc93111 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_t0_stream_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_t0_stream_k600_8x8.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Video classification on Kinetics-600 using MoViNet-T0-Stream backbone. # --experiment_type=movinet_kinetics600 # Achieves 67.17% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/movinet_tutorial.ipynb b/official/vision/beta/projects/movinet/movinet_tutorial.ipynb index 319489d7a78..7ad97fb722e 100644 --- a/official/vision/beta/projects/movinet/movinet_tutorial.ipynb +++ b/official/vision/beta/projects/movinet/movinet_tutorial.ipynb @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + { "cells": [ { diff --git a/official/vision/beta/projects/movinet/requirements.txt b/official/vision/beta/projects/movinet/requirements.txt index 55b985f2258..190cec2dcb7 100644 --- a/official/vision/beta/projects/movinet/requirements.txt +++ b/official/vision/beta/projects/movinet/requirements.txt @@ -1 +1,15 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + mediapy diff --git a/official/vision/beta/projects/simclr/README.md b/official/vision/beta/projects/simclr/README.md index 91b4375bd60..b381c5c4718 100644 --- a/official/vision/beta/projects/simclr/README.md +++ b/official/vision/beta/projects/simclr/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Simple Framework for Contrastive Learning [![Paper](http://img.shields.io/badge/Paper-arXiv.2002.05709-B3181B?logo=arXiv)](https://arxiv.org/abs/2002.05709) diff --git a/official/vision/beta/projects/simclr/configs/experiments/cifar_simclr_pretrain.yaml b/official/vision/beta/projects/simclr/configs/experiments/cifar_simclr_pretrain.yaml index 07d319a6929..f4b5d006701 100644 --- a/official/vision/beta/projects/simclr/configs/experiments/cifar_simclr_pretrain.yaml +++ b/official/vision/beta/projects/simclr/configs/experiments/cifar_simclr_pretrain.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Cifar classification. runtime: distribution_strategy: 'mirrored' diff --git a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_gpu.yaml b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_gpu.yaml index 13b02cdf113..49db1edb081 100644 --- a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_gpu.yaml +++ b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_gpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # SimCLR Imagenet 10% finetuning. runtime: distribution_strategy: 'mirrored' diff --git a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_tpu.yaml b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_tpu.yaml index 45cceb5fcd4..933736fd509 100644 --- a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_tpu.yaml +++ b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # SimCLR Imagenet 10% finetuning. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_gpu.yaml b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_gpu.yaml index f2fa25ef8e7..31ce98d7ee5 100644 --- a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_gpu.yaml +++ b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_gpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # SimCLR Imagenet pretraining. runtime: distribution_strategy: 'mirrored' diff --git a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_tpu.yaml b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_tpu.yaml index f5c8045483b..8d7f48e9374 100644 --- a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_tpu.yaml +++ b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # SimCLR Imagenet pretraining. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/projects/yolo/README.md b/official/vision/beta/projects/yolo/README.md index 0a1e27fbe90..707ad2f0b2a 100644 --- a/official/vision/beta/projects/yolo/README.md +++ b/official/vision/beta/projects/yolo/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # YOLO Object Detectors, You Only Look Once [![Paper](http://img.shields.io/badge/Paper-arXiv.1804.02767-B3181B?logo=arXiv)](https://arxiv.org/abs/1804.02767) diff --git a/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53.yaml b/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53.yaml index 10dbdc56855..60feb5fc0f2 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + runtime: distribution_strategy: 'mirrored' mixed_precision_dtype: 'float32' diff --git a/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53_tfds.yaml b/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53_tfds.yaml index b27ff015708..778bca2e408 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53_tfds.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53_tfds.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + runtime: distribution_strategy: 'mirrored' mixed_precision_dtype: 'float16' diff --git a/official/vision/beta/projects/yolo/configs/experiments/darknet53.yaml b/official/vision/beta/projects/yolo/configs/experiments/darknet53.yaml index a3333b599e3..bf2f37a984c 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/darknet53.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/darknet53.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + runtime: distribution_strategy: 'mirrored' mixed_precision_dtype: 'float32' diff --git a/official/vision/beta/projects/yolo/configs/experiments/darknet53_tfds.yaml b/official/vision/beta/projects/yolo/configs/experiments/darknet53_tfds.yaml index 8f9fb2dfc6b..32d935a7cb0 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/darknet53_tfds.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/darknet53_tfds.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + runtime: distribution_strategy: 'mirrored' mixed_precision_dtype: 'float16' diff --git a/official/vision/beta/projects/yt8m/README.md b/official/vision/beta/projects/yt8m/README.md index 2f4cf2ab4b0..2acaef2b906 100644 --- a/official/vision/beta/projects/yt8m/README.md +++ b/official/vision/beta/projects/yt8m/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # YouTube-8M Tensorflow Starter Code DISCLAIMER: This project is still under development. diff --git a/official/vision/beta/projects/yt8m/experiments/yt8m.yaml b/official/vision/beta/projects/yt8m/experiments/yt8m.yaml index c099f23f90b..40e7c784587 100644 --- a/official/vision/beta/projects/yt8m/experiments/yt8m.yaml +++ b/official/vision/beta/projects/yt8m/experiments/yt8m.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # yt8m config file task: model: diff --git a/official/vision/beta/projects/yt8m/experiments/yt8m_test.yaml b/official/vision/beta/projects/yt8m/experiments/yt8m_test.yaml index 9a7ef94cc73..343629aa710 100644 --- a/official/vision/beta/projects/yt8m/experiments/yt8m_test.yaml +++ b/official/vision/beta/projects/yt8m/experiments/yt8m_test.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # yt8m test config file task: model: diff --git a/official/vision/detection/README.md b/official/vision/detection/README.md index 2633f86d5dc..6985f3a79c7 100644 --- a/official/vision/detection/README.md +++ b/official/vision/detection/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Object Detection Models on TensorFlow 2 **WARNING**: This repository will be deprecated and replaced by the solid diff --git a/official/vision/image_classification/README.md b/official/vision/image_classification/README.md index 78bfe1f27e6..6137e9ccce4 100644 --- a/official/vision/image_classification/README.md +++ b/official/vision/image_classification/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Image Classification This folder contains TF 2.0 model examples for image classification: diff --git a/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-gpu.yaml b/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-gpu.yaml index 6f40ffb1e30..756b489fef5 100644 --- a/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-gpu.yaml +++ b/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-gpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Training configuration for EfficientNet-b0 trained on ImageNet on GPUs. # Takes ~32 minutes per epoch for 8 V100s. # Reaches ~76.1% within 350 epochs. diff --git a/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-tpu.yaml b/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-tpu.yaml index c5be7e9ba32..b406fbe0f6c 100644 --- a/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-tpu.yaml +++ b/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Training configuration for EfficientNet-b0 trained on ImageNet on TPUs. # Takes ~2 minutes, 50 seconds per epoch for v3-32. # Reaches ~76.1% within 350 epochs. diff --git a/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-gpu.yaml b/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-gpu.yaml index 2f3dce01a46..f4ecba89b8e 100644 --- a/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-gpu.yaml +++ b/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-gpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Note: This configuration uses a scaled per-replica batch size based on the number of devices. runtime: distribution_strategy: 'mirrored' diff --git a/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-tpu.yaml b/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-tpu.yaml index 0bb6a9fe6f0..4a96ce232e8 100644 --- a/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-tpu.yaml +++ b/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Training configuration for EfficientNet-b1 trained on ImageNet on TPUs. # Takes ~3 minutes, 15 seconds per epoch for v3-32. # Note: This configuration uses a scaled per-replica batch size based on the number of devices. diff --git a/official/vision/image_classification/configs/examples/resnet/imagenet/gpu.yaml b/official/vision/image_classification/configs/examples/resnet/imagenet/gpu.yaml index 2037d6b5d1c..5fc88552caa 100644 --- a/official/vision/image_classification/configs/examples/resnet/imagenet/gpu.yaml +++ b/official/vision/image_classification/configs/examples/resnet/imagenet/gpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Training configuration for ResNet trained on ImageNet on GPUs. # Reaches > 76.1% within 90 epochs. # Note: This configuration uses a scaled per-replica batch size based on the number of devices. diff --git a/official/vision/image_classification/configs/examples/resnet/imagenet/tpu.yaml b/official/vision/image_classification/configs/examples/resnet/imagenet/tpu.yaml index 0a3030333bb..cda4cd6df9d 100644 --- a/official/vision/image_classification/configs/examples/resnet/imagenet/tpu.yaml +++ b/official/vision/image_classification/configs/examples/resnet/imagenet/tpu.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Training configuration for ResNet trained on ImageNet on TPUs. # Takes ~4 minutes, 30 seconds seconds per epoch for a v3-32. # Reaches > 76.1% within 90 epochs. diff --git a/official/vision/image_classification/resnet/README.md b/official/vision/image_classification/resnet/README.md index 5064523fbdc..a5243db2e01 100644 --- a/official/vision/image_classification/resnet/README.md +++ b/official/vision/image_classification/resnet/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + This folder contains a [custom training loop (CTL)](#resnet-custom-training-loop) implementation for ResNet50. diff --git a/official/vision/keras_cv/LICENSE b/official/vision/keras_cv/LICENSE index 0b1ba442980..945e036f578 100644 --- a/official/vision/keras_cv/LICENSE +++ b/official/vision/keras_cv/LICENSE @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + Copyright 2020 The TensorFlow Authors. All rights reserved. Apache License diff --git a/official/vision/keras_cv/README.md b/official/vision/keras_cv/README.md index 1132d521cd7..4c9917964f5 100644 --- a/official/vision/keras_cv/README.md +++ b/official/vision/keras_cv/README.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # keras-cv ## Losses diff --git a/official/vision/keras_cv/contributing.md b/official/vision/keras_cv/contributing.md index d9efe9b0691..729528ae38a 100644 --- a/official/vision/keras_cv/contributing.md +++ b/official/vision/keras_cv/contributing.md @@ -1,3 +1,17 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + ## Contributing to KerasCV Patches to KerasCV are welcome! diff --git a/official/vision/keras_cv/requirements.txt b/official/vision/keras_cv/requirements.txt index 6bad10388ec..6d50ed5b314 100644 --- a/official/vision/keras_cv/requirements.txt +++ b/official/vision/keras_cv/requirements.txt @@ -1,2 +1,16 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + numpy scipy From 1fa648a753b877f18ca3a1de9bb921c3f024c11d Mon Sep 17 00:00:00 2001 From: Hongkun Yu Date: Sun, 6 Jun 2021 14:09:42 -0700 Subject: [PATCH 29/50] Internal change PiperOrigin-RevId: 377803367 --- official/LICENSE | 14 -------------- official/README-TPU.md | 14 -------------- official/README.md | 14 -------------- official/colab/decoding_api_in_tf_nlp.ipynb | 14 -------------- official/colab/nlp/customize_encoder.ipynb | 14 -------------- .../colab/nlp/nlp_modeling_library_intro.ipynb | 14 -------------- official/nlp/README.md | 14 -------------- official/nlp/albert/README.md | 14 -------------- official/nlp/bert/README.md | 14 -------------- official/nlp/bert/bert_cloud_tpu.md | 14 -------------- .../nlp/configs/experiments/glue_mnli_matched.yaml | 14 -------------- official/nlp/configs/experiments/squad_v1.yaml | 14 -------------- .../nlp/configs/models/bert_en_uncased_base.yaml | 14 -------------- official/nlp/docs/pretrained_models.md | 14 -------------- official/nlp/docs/tfhub.md | 14 -------------- official/nlp/docs/train.md | 14 -------------- official/nlp/keras_nlp/README.md | 14 -------------- official/nlp/keras_nlp/contributing.md | 14 -------------- official/nlp/keras_nlp/requirements.txt | 14 -------------- official/nlp/modeling/README.md | 14 -------------- official/nlp/modeling/layers/README.md | 14 -------------- official/nlp/modeling/losses/README.md | 14 -------------- official/nlp/modeling/models/README.md | 14 -------------- official/nlp/modeling/networks/README.md | 14 -------------- official/nlp/nhnet/README.md | 14 -------------- .../crawled_articles/domain_0.com/url_000.html | 14 -------------- .../crawled_articles/domain_0.com/url_000.json | 14 -------------- .../crawled_articles/domain_1.com/url_001.html | 14 -------------- .../crawled_articles/domain_1.com/url_001.json | 14 -------------- official/nlp/nhnet/testdata/stories.json | 14 -------------- official/nlp/nhnet/testdata/vocab.txt | 14 -------------- official/nlp/projects/bigbird/README.md | 14 -------------- .../bigbird/experiments/glue_mnli_matched.yaml | 14 -------------- .../nlp/projects/bigbird/experiments/squad_v1.yaml | 14 -------------- official/nlp/projects/mobilebert/README.md | 14 -------------- .../mobilebert/experiments/en_uncased_student.yaml | 14 -------------- .../mobilebert/experiments/en_uncased_teacher.yaml | 14 -------------- .../mobilebert_distillation_en_uncased.yaml | 14 -------------- official/nlp/projects/teams/README.md | 14 -------------- official/nlp/projects/tn_bert/README.md | 14 -------------- official/nlp/transformer/README.md | 14 -------------- official/nlp/xlnet/README.md | 14 -------------- official/recommendation/README.md | 14 -------------- official/recommendation/ranking/README.md | 14 -------------- .../ranking/configs/yaml/dcn_v2_criteo_tpu.yaml | 14 -------------- .../ranking/configs/yaml/dlrm_criteo_tpu.yaml | 14 -------------- official/recommendation/run.sh | 14 -------------- official/requirements.txt | 14 -------------- official/utils/flags/README.md | 14 -------------- official/utils/flags/guidelines.md | 14 -------------- official/utils/testing/pylint.rcfile | 14 -------------- official/utils/testing/scripts/builds_common.sh | 14 -------------- official/utils/testing/scripts/ci_sanity.sh | 14 -------------- official/utils/testing/scripts/presubmit.sh | 14 -------------- official/vision/beta/MODEL_GARDEN.md | 14 -------------- official/vision/beta/README.md | 14 -------------- .../imagenet_mobilenetv2_gpu.yaml | 14 -------------- .../imagenet_mobilenetv2_tpu.yaml | 14 -------------- .../imagenet_resnet101_deeplab_tpu.yaml | 14 -------------- .../imagenet_resnet101_tpu.yaml | 14 -------------- .../imagenet_resnet152_tpu.yaml | 14 -------------- .../imagenet_resnet50_deeplab_tpu.yaml | 14 -------------- .../imagenet_resnet50_gpu.yaml | 14 -------------- .../imagenet_resnet50_tfds_tpu.yaml | 14 -------------- .../imagenet_resnet50_tpu.yaml | 14 -------------- .../imagenet_resnetrs101_i160.yaml | 14 -------------- .../imagenet_resnetrs101_i192.yaml | 14 -------------- .../imagenet_resnetrs152_i192.yaml | 14 -------------- .../imagenet_resnetrs152_i224.yaml | 14 -------------- .../imagenet_resnetrs152_i256.yaml | 14 -------------- .../imagenet_resnetrs200_i256.yaml | 14 -------------- .../imagenet_resnetrs270_i256.yaml | 14 -------------- .../imagenet_resnetrs350_i256.yaml | 14 -------------- .../imagenet_resnetrs350_i320.yaml | 14 -------------- .../imagenet_resnetrs420_i320.yaml | 14 -------------- .../imagenet_resnetrs50_i160.yaml | 14 -------------- .../maskrcnn/coco_spinenet143_cascadercnn_tpu.yaml | 14 -------------- .../maskrcnn/coco_spinenet143_mrcnn_tpu.yaml | 14 -------------- .../maskrcnn/coco_spinenet49_cascadercnn_tpu.yaml | 14 -------------- .../maskrcnn/coco_spinenet49_mrcnn_tpu.yaml | 14 -------------- .../maskrcnn/coco_spinenet96_casrcnn_tpu.yaml | 14 -------------- .../maskrcnn/coco_spinenet96_mrcnn_tpu.yaml | 14 -------------- .../maskrcnn/r50fpn_640_coco_scratch_tpu4x4.yaml | 14 -------------- .../retinanet/coco_spinenet143_tpu.yaml | 14 -------------- .../retinanet/coco_spinenet190_tpu.yaml | 14 -------------- .../retinanet/coco_spinenet49_mobile_tpu.yaml | 14 -------------- .../experiments/retinanet/coco_spinenet49_tpu.yaml | 14 -------------- .../retinanet/coco_spinenet49s_mobile_tpu.yaml | 14 -------------- .../retinanet/coco_spinenet49xs_mobile_tpu.yaml | 14 -------------- .../experiments/retinanet/coco_spinenet96_tpu.yaml | 14 -------------- .../retinanet/resnet50fpn_coco_tfds_tpu.yaml | 14 -------------- .../resnet50fpn_coco_tpu4x4_benchmark.yaml | 14 -------------- ...eeplabv3plus_resnet101_cityscapes_tfds_tpu.yaml | 14 -------------- .../video_classification/k400_3d-resnet50_tpu.yaml | 14 -------------- .../k400_resnet3drs_50_tpu.yaml | 14 -------------- .../k400_slowonly16x4_tpu.yaml | 14 -------------- .../video_classification/k400_slowonly8x8_tpu.yaml | 14 -------------- .../video_classification/k600_3d-resnet50_tpu.yaml | 14 -------------- .../k600_3d-resnet50g_tpu.yaml | 14 -------------- .../video_classification/k600_slowonly8x8_tpu.yaml | 14 -------------- official/vision/beta/projects/README.md | 14 -------------- .../vision/beta/projects/assemblenet/README.md | 14 -------------- .../beta/projects/deepmac_maskrcnn/README.md | 14 -------------- .../deep_mask_head_rcnn_voc_r101_hg52.yaml | 14 -------------- .../experiments/deep_mask_head_rcnn_voc_r50.yaml | 14 -------------- .../deep_mask_head_rcnn_voc_r50_hg52.yaml | 14 -------------- official/vision/beta/projects/movinet/README.md | 14 -------------- .../movinet/configs/yaml/movinet_a0_k600_8x8.yaml | 14 -------------- .../configs/yaml/movinet_a0_k600_cpu_local.yaml | 14 -------------- .../configs/yaml/movinet_a0_stream_k600_8x8.yaml | 14 -------------- .../movinet/configs/yaml/movinet_a1_k600_8x8.yaml | 14 -------------- .../configs/yaml/movinet_a1_stream_k600_8x8.yaml | 14 -------------- .../movinet/configs/yaml/movinet_a2_k600_8x8.yaml | 14 -------------- .../configs/yaml/movinet_a2_stream_k600_8x8.yaml | 14 -------------- .../movinet/configs/yaml/movinet_a3_k600_8x8.yaml | 14 -------------- .../configs/yaml/movinet_a3_stream_k600_8x8.yaml | 14 -------------- .../movinet/configs/yaml/movinet_a4_k600_8x8.yaml | 14 -------------- .../configs/yaml/movinet_a4_stream_k600_8x8.yaml | 14 -------------- .../movinet/configs/yaml/movinet_a5_k600_8x8.yaml | 14 -------------- .../configs/yaml/movinet_a5_stream_k600_8x8.yaml | 14 -------------- .../movinet/configs/yaml/movinet_t0_k600_8x8.yaml | 14 -------------- .../configs/yaml/movinet_t0_stream_k600_8x8.yaml | 14 -------------- .../beta/projects/movinet/movinet_tutorial.ipynb | 14 -------------- .../vision/beta/projects/movinet/requirements.txt | 14 -------------- official/vision/beta/projects/simclr/README.md | 14 -------------- .../configs/experiments/cifar_simclr_pretrain.yaml | 14 -------------- .../experiments/imagenet_simclr_finetune_gpu.yaml | 14 -------------- .../experiments/imagenet_simclr_finetune_tpu.yaml | 14 -------------- .../experiments/imagenet_simclr_pretrain_gpu.yaml | 14 -------------- .../experiments/imagenet_simclr_pretrain_tpu.yaml | 14 -------------- official/vision/beta/projects/yolo/README.md | 14 -------------- .../yolo/configs/experiments/csp_darknet53.yaml | 14 -------------- .../configs/experiments/csp_darknet53_tfds.yaml | 14 -------------- .../yolo/configs/experiments/darknet53.yaml | 14 -------------- .../yolo/configs/experiments/darknet53_tfds.yaml | 14 -------------- official/vision/beta/projects/yt8m/README.md | 14 -------------- .../beta/projects/yt8m/experiments/yt8m.yaml | 14 -------------- .../beta/projects/yt8m/experiments/yt8m_test.yaml | 14 -------------- official/vision/detection/README.md | 14 -------------- official/vision/image_classification/README.md | 14 -------------- .../efficientnet/imagenet/efficientnet-b0-gpu.yaml | 14 -------------- .../efficientnet/imagenet/efficientnet-b0-tpu.yaml | 14 -------------- .../efficientnet/imagenet/efficientnet-b1-gpu.yaml | 14 -------------- .../efficientnet/imagenet/efficientnet-b1-tpu.yaml | 14 -------------- .../configs/examples/resnet/imagenet/gpu.yaml | 14 -------------- .../configs/examples/resnet/imagenet/tpu.yaml | 14 -------------- .../vision/image_classification/resnet/README.md | 14 -------------- official/vision/keras_cv/LICENSE | 14 -------------- official/vision/keras_cv/README.md | 14 -------------- official/vision/keras_cv/contributing.md | 14 -------------- official/vision/keras_cv/requirements.txt | 14 -------------- 151 files changed, 2114 deletions(-) diff --git a/official/LICENSE b/official/LICENSE index b1143d7b536..d3da228420e 100644 --- a/official/LICENSE +++ b/official/LICENSE @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - Copyright 2015 The TensorFlow Authors. All rights reserved. Apache License diff --git a/official/README-TPU.md b/official/README-TPU.md index 435079d1c83..a6031c44f03 100644 --- a/official/README-TPU.md +++ b/official/README-TPU.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Offically Supported TensorFlow 2.1+ Models on Cloud TPU ## Natural Language Processing diff --git a/official/README.md b/official/README.md index 52ca8f3d9bc..188a94ca532 100644 --- a/official/README.md +++ b/official/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ![Logo](https://storage.googleapis.com/model_garden_artifacts/TF_Model_Garden.png) # TensorFlow Official Models diff --git a/official/colab/decoding_api_in_tf_nlp.ipynb b/official/colab/decoding_api_in_tf_nlp.ipynb index 155a18422f0..726b382e228 100644 --- a/official/colab/decoding_api_in_tf_nlp.ipynb +++ b/official/colab/decoding_api_in_tf_nlp.ipynb @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - { "cells": [ { diff --git a/official/colab/nlp/customize_encoder.ipynb b/official/colab/nlp/customize_encoder.ipynb index 809ffd74786..aeddb29f963 100644 --- a/official/colab/nlp/customize_encoder.ipynb +++ b/official/colab/nlp/customize_encoder.ipynb @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - { "nbformat": 4, "nbformat_minor": 0, diff --git a/official/colab/nlp/nlp_modeling_library_intro.ipynb b/official/colab/nlp/nlp_modeling_library_intro.ipynb index ae6ad9e5f76..e4ce780c96b 100644 --- a/official/colab/nlp/nlp_modeling_library_intro.ipynb +++ b/official/colab/nlp/nlp_modeling_library_intro.ipynb @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - { "cells": [ { diff --git a/official/nlp/README.md b/official/nlp/README.md index 733a218bcce..dfa047b4ed3 100644 --- a/official/nlp/README.md +++ b/official/nlp/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # TensorFlow NLP Modelling Toolkit This codebase provides a Natrual Language Processing modeling toolkit written in diff --git a/official/nlp/albert/README.md b/official/nlp/albert/README.md index 33d92c81ab4..69620e0579f 100644 --- a/official/nlp/albert/README.md +++ b/official/nlp/albert/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # ALBERT (ALBERT: A Lite BERT for Self-supervised Learning of Language Representations) The academic paper which describes ALBERT in detail and provides full results on diff --git a/official/nlp/bert/README.md b/official/nlp/bert/README.md index c2c545308c3..037ff0b1ff8 100644 --- a/official/nlp/bert/README.md +++ b/official/nlp/bert/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # BERT (Bidirectional Encoder Representations from Transformers) **WARNING**: We are on the way to deprecate most of the code in this directory. diff --git a/official/nlp/bert/bert_cloud_tpu.md b/official/nlp/bert/bert_cloud_tpu.md index 60f2e77acd8..baf6f9bdc0c 100644 --- a/official/nlp/bert/bert_cloud_tpu.md +++ b/official/nlp/bert/bert_cloud_tpu.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # BERT FineTuning with Cloud TPU: Sentence and Sentence-Pair Classification Tasks (TF 2.1) This tutorial shows you how to train the Bidirectional Encoder Representations from Transformers (BERT) model on Cloud TPU. diff --git a/official/nlp/configs/experiments/glue_mnli_matched.yaml b/official/nlp/configs/experiments/glue_mnli_matched.yaml index 6580a5e2898..29dfcb68b9c 100644 --- a/official/nlp/configs/experiments/glue_mnli_matched.yaml +++ b/official/nlp/configs/experiments/glue_mnli_matched.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - task: hub_module_url: '' model: diff --git a/official/nlp/configs/experiments/squad_v1.yaml b/official/nlp/configs/experiments/squad_v1.yaml index 89d1ff90d89..a69710a58f7 100644 --- a/official/nlp/configs/experiments/squad_v1.yaml +++ b/official/nlp/configs/experiments/squad_v1.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - task: hub_module_url: '' max_answer_length: 30 diff --git a/official/nlp/configs/models/bert_en_uncased_base.yaml b/official/nlp/configs/models/bert_en_uncased_base.yaml index 22085984a75..1e49bc5430e 100644 --- a/official/nlp/configs/models/bert_en_uncased_base.yaml +++ b/official/nlp/configs/models/bert_en_uncased_base.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - task: model: encoder: diff --git a/official/nlp/docs/pretrained_models.md b/official/nlp/docs/pretrained_models.md index 3c64efd4876..0c836b33b7d 100644 --- a/official/nlp/docs/pretrained_models.md +++ b/official/nlp/docs/pretrained_models.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Pre-trained Models We provide a large collection of baselines and checkpoints for NLP pre-trained diff --git a/official/nlp/docs/tfhub.md b/official/nlp/docs/tfhub.md index 505051db5a8..c6fe9a2f8f4 100644 --- a/official/nlp/docs/tfhub.md +++ b/official/nlp/docs/tfhub.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Exporting a pre-trained Encoder to TF Hub ## Overview diff --git a/official/nlp/docs/train.md b/official/nlp/docs/train.md index 69bef3cc5d4..d2ad9d7622d 100644 --- a/official/nlp/docs/train.md +++ b/official/nlp/docs/train.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Model Garden NLP Common Training Driver [train.py](https://github.com/tensorflow/models/blob/master/official/nlp/train.py) is the common training driver that supports multiple diff --git a/official/nlp/keras_nlp/README.md b/official/nlp/keras_nlp/README.md index 993d8684ca6..1c5bbb13182 100644 --- a/official/nlp/keras_nlp/README.md +++ b/official/nlp/keras_nlp/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # keras-nlp ## Layers diff --git a/official/nlp/keras_nlp/contributing.md b/official/nlp/keras_nlp/contributing.md index 6974ef6666c..b9ec1716d96 100644 --- a/official/nlp/keras_nlp/contributing.md +++ b/official/nlp/keras_nlp/contributing.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ## Contributing to KerasNLP Patches to KerasNLP are welcome! diff --git a/official/nlp/keras_nlp/requirements.txt b/official/nlp/keras_nlp/requirements.txt index 822aed5d9d5..c765b1ead22 100644 --- a/official/nlp/keras_nlp/requirements.txt +++ b/official/nlp/keras_nlp/requirements.txt @@ -1,15 +1 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - numpy>=1.15.4 diff --git a/official/nlp/modeling/README.md b/official/nlp/modeling/README.md index fe192621493..99c7c361f97 100644 --- a/official/nlp/modeling/README.md +++ b/official/nlp/modeling/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # NLP Modeling Library This library provides a set of Keras primitives (`tf.keras.Layer` and diff --git a/official/nlp/modeling/layers/README.md b/official/nlp/modeling/layers/README.md index 43a7e271ef7..79e142a0887 100644 --- a/official/nlp/modeling/layers/README.md +++ b/official/nlp/modeling/layers/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Layers Layers are the fundamental building blocks for NLP models. They can be used to diff --git a/official/nlp/modeling/losses/README.md b/official/nlp/modeling/losses/README.md index 94a5808cbf0..a2607b1dab7 100644 --- a/official/nlp/modeling/losses/README.md +++ b/official/nlp/modeling/losses/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Losses Losses contains common loss computation used in NLP tasks. diff --git a/official/nlp/modeling/models/README.md b/official/nlp/modeling/models/README.md index b7fbc7ab74f..22fd8193c29 100644 --- a/official/nlp/modeling/models/README.md +++ b/official/nlp/modeling/models/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Models Models are combinations of `tf.keras` layers and models that can be trained. diff --git a/official/nlp/modeling/networks/README.md b/official/nlp/modeling/networks/README.md index 25547a7449c..b192399a727 100644 --- a/official/nlp/modeling/networks/README.md +++ b/official/nlp/modeling/networks/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Networks Networks are combinations of `tf.keras` layers (and possibly other networks). diff --git a/official/nlp/nhnet/README.md b/official/nlp/nhnet/README.md index 7536133fcd7..f838d120fb8 100644 --- a/official/nlp/nhnet/README.md +++ b/official/nlp/nhnet/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Multi-doc News Headline Generation Model: NHNet This repository contains TensorFlow 2.x implementation for NHNet [[1]](#1) as diff --git a/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.html b/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.html index c3085f2cf1d..0a8549c1d27 100644 --- a/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.html +++ b/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.html @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - Page Title 0 diff --git a/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.json b/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.json index 0cfb775b567..b7308592b77 100644 --- a/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.json +++ b/official/nlp/nhnet/testdata/crawled_articles/domain_0.com/url_000.json @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - { "title": "title for 0", "maintext": "text snippet for 0", diff --git a/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.html b/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.html index 9f13aee1864..7c8bb8d285c 100644 --- a/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.html +++ b/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.html @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - Page Title 1 diff --git a/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.json b/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.json index cb51d6ce321..dbc2322c7de 100644 --- a/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.json +++ b/official/nlp/nhnet/testdata/crawled_articles/domain_1.com/url_001.json @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - { "title": "title for 1", "maintext": "text snippet for 1", diff --git a/official/nlp/nhnet/testdata/stories.json b/official/nlp/nhnet/testdata/stories.json index b20c6cc5ae1..0618f3d5c8a 100644 --- a/official/nlp/nhnet/testdata/stories.json +++ b/official/nlp/nhnet/testdata/stories.json @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - [ { "urls": [ diff --git a/official/nlp/nhnet/testdata/vocab.txt b/official/nlp/nhnet/testdata/vocab.txt index 20f9567cd48..dd708d71c2f 100644 --- a/official/nlp/nhnet/testdata/vocab.txt +++ b/official/nlp/nhnet/testdata/vocab.txt @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - [UNK] [CLS] [SEP] diff --git a/official/nlp/projects/bigbird/README.md b/official/nlp/projects/bigbird/README.md index 20849d79d87..7c5435f154d 100644 --- a/official/nlp/projects/bigbird/README.md +++ b/official/nlp/projects/bigbird/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # BigBird: Transformers for Longer Sequences [BigBird](https://arxiv.org/abs/2007.14062) diff --git a/official/nlp/projects/bigbird/experiments/glue_mnli_matched.yaml b/official/nlp/projects/bigbird/experiments/glue_mnli_matched.yaml index 6b0556998a4..f38bdfc8b32 100644 --- a/official/nlp/projects/bigbird/experiments/glue_mnli_matched.yaml +++ b/official/nlp/projects/bigbird/experiments/glue_mnli_matched.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - task: hub_module_url: '' model: diff --git a/official/nlp/projects/bigbird/experiments/squad_v1.yaml b/official/nlp/projects/bigbird/experiments/squad_v1.yaml index af1dfb74269..7971f1b2472 100644 --- a/official/nlp/projects/bigbird/experiments/squad_v1.yaml +++ b/official/nlp/projects/bigbird/experiments/squad_v1.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - task: hub_module_url: '' model: diff --git a/official/nlp/projects/mobilebert/README.md b/official/nlp/projects/mobilebert/README.md index 2ff549e3303..9209b4720d6 100644 --- a/official/nlp/projects/mobilebert/README.md +++ b/official/nlp/projects/mobilebert/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # MobileBERT (MobileBERT: A Compact Task-Agnostic BERT for Resource-Limited Devices) [MobileBERT](https://arxiv.org/abs/2004.02984) diff --git a/official/nlp/projects/mobilebert/experiments/en_uncased_student.yaml b/official/nlp/projects/mobilebert/experiments/en_uncased_student.yaml index 0f989dc4cdc..cfcf40c2b89 100644 --- a/official/nlp/projects/mobilebert/experiments/en_uncased_student.yaml +++ b/official/nlp/projects/mobilebert/experiments/en_uncased_student.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - task: model: encoder: diff --git a/official/nlp/projects/mobilebert/experiments/en_uncased_teacher.yaml b/official/nlp/projects/mobilebert/experiments/en_uncased_teacher.yaml index 15db0a3dc21..eeee8537da5 100644 --- a/official/nlp/projects/mobilebert/experiments/en_uncased_teacher.yaml +++ b/official/nlp/projects/mobilebert/experiments/en_uncased_teacher.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - task: model: encoder: diff --git a/official/nlp/projects/mobilebert/experiments/mobilebert_distillation_en_uncased.yaml b/official/nlp/projects/mobilebert/experiments/mobilebert_distillation_en_uncased.yaml index 8c2a9044d2d..74e6adc3c47 100644 --- a/official/nlp/projects/mobilebert/experiments/mobilebert_distillation_en_uncased.yaml +++ b/official/nlp/projects/mobilebert/experiments/mobilebert_distillation_en_uncased.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - task: train_data: drop_remainder: true diff --git a/official/nlp/projects/teams/README.md b/official/nlp/projects/teams/README.md index 8faa6e2ead6..f57aa266d06 100644 --- a/official/nlp/projects/teams/README.md +++ b/official/nlp/projects/teams/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # TEAMS (Training ELECTRA Augmented with Multi-word Selection) **Note:** This project is working in progress and please stay tuned. diff --git a/official/nlp/projects/tn_bert/README.md b/official/nlp/projects/tn_bert/README.md index c513972fc23..50928155807 100644 --- a/official/nlp/projects/tn_bert/README.md +++ b/official/nlp/projects/tn_bert/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # TN-BERT (TensorNetwork BERT) TN-BERT is a modification of the BERT-base architecture that greatly compresses diff --git a/official/nlp/transformer/README.md b/official/nlp/transformer/README.md index d67e3df71ee..a3aec5f9a05 100644 --- a/official/nlp/transformer/README.md +++ b/official/nlp/transformer/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Transformer Translation Model This is an implementation of the Transformer translation model as described in the [Attention is All You Need](https://arxiv.org/abs/1706.03762) paper. The diff --git a/official/nlp/xlnet/README.md b/official/nlp/xlnet/README.md index 417c37d5a0f..546d1128e2d 100644 --- a/official/nlp/xlnet/README.md +++ b/official/nlp/xlnet/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # XLNet: Generalized Autoregressive Pretraining for Language Understanding The academic paper which describes XLNet in detail and provides full results on diff --git a/official/recommendation/README.md b/official/recommendation/README.md index 7e9f784240c..ea2abfadcab 100644 --- a/official/recommendation/README.md +++ b/official/recommendation/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Recommendation Model ## Overview This is an implementation of the Neural Collaborative Filtering (NCF) framework with Neural Matrix Factorization (NeuMF) model as described in the [Neural Collaborative Filtering](https://arxiv.org/abs/1708.05031) paper. Current implementation is based on the code from the authors' [NCF code](https://github.com/hexiangnan/neural_collaborative_filtering) and the Stanford implementation in the [MLPerf Repo](https://github.com/mlperf/reference/tree/master/recommendation/pytorch). diff --git a/official/recommendation/ranking/README.md b/official/recommendation/ranking/README.md index b2d891848f7..9c2ca21039f 100644 --- a/official/recommendation/ranking/README.md +++ b/official/recommendation/ranking/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # TF Model Garden Ranking Models ## Overview diff --git a/official/recommendation/ranking/configs/yaml/dcn_v2_criteo_tpu.yaml b/official/recommendation/ranking/configs/yaml/dcn_v2_criteo_tpu.yaml index 6532d59e8c1..a281e03259b 100644 --- a/official/recommendation/ranking/configs/yaml/dcn_v2_criteo_tpu.yaml +++ b/official/recommendation/ranking/configs/yaml/dcn_v2_criteo_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - runtime: distribution_strategy: 'tpu' task: diff --git a/official/recommendation/ranking/configs/yaml/dlrm_criteo_tpu.yaml b/official/recommendation/ranking/configs/yaml/dlrm_criteo_tpu.yaml index 79ea6fb6e33..aaaadf58e60 100644 --- a/official/recommendation/ranking/configs/yaml/dlrm_criteo_tpu.yaml +++ b/official/recommendation/ranking/configs/yaml/dlrm_criteo_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - runtime: distribution_strategy: 'tpu' task: diff --git a/official/recommendation/run.sh b/official/recommendation/run.sh index 935a7d01830..b8e1143a38b 100755 --- a/official/recommendation/run.sh +++ b/official/recommendation/run.sh @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - #!/bin/bash set -e diff --git a/official/requirements.txt b/official/requirements.txt index fc95b6dae90..74028adcb55 100644 --- a/official/requirements.txt +++ b/official/requirements.txt @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - six google-api-python-client>=1.6.7 google-cloud-bigquery>=0.31.0 diff --git a/official/utils/flags/README.md b/official/utils/flags/README.md index 49619bdbf2d..beb3b2a1e1d 100644 --- a/official/utils/flags/README.md +++ b/official/utils/flags/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Adding Abseil (absl) flags quickstart **WARNING** This module is deprecated. We no long use it in new models and diff --git a/official/utils/flags/guidelines.md b/official/utils/flags/guidelines.md index fd3720c8596..db963aabebc 100644 --- a/official/utils/flags/guidelines.md +++ b/official/utils/flags/guidelines.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Using flags in official models 1. **All common flags must be incorporated in the models.** diff --git a/official/utils/testing/pylint.rcfile b/official/utils/testing/pylint.rcfile index 33ce61e4c39..b872802a811 100644 --- a/official/utils/testing/pylint.rcfile +++ b/official/utils/testing/pylint.rcfile @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - [MESSAGES CONTROL] disable=R,W,bad-option-value,trailing-newlines,no-name-in-module diff --git a/official/utils/testing/scripts/builds_common.sh b/official/utils/testing/scripts/builds_common.sh index 7bea3ef09b3..3cf08bb510d 100644 --- a/official/utils/testing/scripts/builds_common.sh +++ b/official/utils/testing/scripts/builds_common.sh @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - #!/usr/bin/env bash # Copyright 2019 The TensorFlow Authors. All Rights Reserved. # diff --git a/official/utils/testing/scripts/ci_sanity.sh b/official/utils/testing/scripts/ci_sanity.sh index 4e73b6288fa..0646c87a943 100755 --- a/official/utils/testing/scripts/ci_sanity.sh +++ b/official/utils/testing/scripts/ci_sanity.sh @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - #!/bin/bash # Copyright 2019 The TensorFlow Authors. All Rights Reserved. # diff --git a/official/utils/testing/scripts/presubmit.sh b/official/utils/testing/scripts/presubmit.sh index 6ef7f5a4ab2..33eca3cbb41 100755 --- a/official/utils/testing/scripts/presubmit.sh +++ b/official/utils/testing/scripts/presubmit.sh @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - #!/bin/bash # Copyright 2018 The TensorFlow Authors. All Rights Reserved. # diff --git a/official/vision/beta/MODEL_GARDEN.md b/official/vision/beta/MODEL_GARDEN.md index da804662a39..42f7eea546b 100644 --- a/official/vision/beta/MODEL_GARDEN.md +++ b/official/vision/beta/MODEL_GARDEN.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # TF Vision Model Garden ## Introduction diff --git a/official/vision/beta/README.md b/official/vision/beta/README.md index 5e163bebf7d..7d8f84fd5bd 100644 --- a/official/vision/beta/README.md +++ b/official/vision/beta/README.md @@ -1,16 +1,2 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - This directory contains the new design of TF model garden vision framework. Stay tuned. diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_mobilenetv2_gpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_mobilenetv2_gpu.yaml index ef1f2f1b5a3..ff1a0719e6f 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_mobilenetv2_gpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_mobilenetv2_gpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # MobileNetV2_1.0 ImageNet classification. 71.0% top-1 and 90.0% top-5 accuracy. runtime: distribution_strategy: 'mirrored' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_mobilenetv2_tpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_mobilenetv2_tpu.yaml index f16c6315576..b5df9d6e74a 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_mobilenetv2_tpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_mobilenetv2_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # MobileNetV2_1.0 ImageNet classification. 72.72% top-1 and 91.05% top-5 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet101_deeplab_tpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet101_deeplab_tpu.yaml index 3846a6c4b12..5d7d2959637 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet101_deeplab_tpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet101_deeplab_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Top-1 accuracy 81.6% on ImageNet runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet101_tpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet101_tpu.yaml index a777f0c9130..2600f58faa5 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet101_tpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet101_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # ResNet-101 ImageNet classification. 79.1% top-1 and 94.5% top-5 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet152_tpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet152_tpu.yaml index 700b71317c0..1c81953e2f6 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet152_tpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet152_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # ResNet-152 ImageNet classification. 79.4% top-1 and 94.7% top-5 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_deeplab_tpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_deeplab_tpu.yaml index 6bcdde4a616..11bdafbc35d 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_deeplab_tpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_deeplab_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - runtime: distribution_strategy: 'tpu' mixed_precision_dtype: 'bfloat16' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_gpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_gpu.yaml index e70ba6ce198..dd6a4dc1618 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_gpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_gpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - runtime: distribution_strategy: 'mirrored' mixed_precision_dtype: 'float16' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_tfds_tpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_tfds_tpu.yaml index 382e3fbc700..1506b48f903 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_tfds_tpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_tfds_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # ResNet-50 ImageNet classification. 78.1% top-1 and 93.9% top-5 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_tpu.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_tpu.yaml index 3ba1b2b7771..7fd10535aa8 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_tpu.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnet50_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # ResNet-50 ImageNet classification. 78.1% top-1 and 93.9% top-5 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs101_i160.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs101_i160.yaml index cff3d32b3d4..7c9e7b80a02 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs101_i160.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs101_i160.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # ResNet-RS-101 ImageNet classification. 80.2% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs101_i192.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs101_i192.yaml index 28c5c75388b..576c4862505 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs101_i192.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs101_i192.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # ResNet-RS-101 ImageNet classification. 81.3% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i192.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i192.yaml index 6c763f33184..b1c8edc463f 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i192.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i192.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # ResNet-RS-152 ImageNet classification. 81.9% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i224.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i224.yaml index 5a3c4d1c399..2ec14bae5ab 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i224.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i224.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # ResNet-RS-152 ImageNet classification. 82.5% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i256.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i256.yaml index 548c5c5330d..91b53d6217f 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i256.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs152_i256.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # ResNet-RS-152 ImageNet classification. 83.1% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs200_i256.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs200_i256.yaml index 79968298ce8..9d76c010170 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs200_i256.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs200_i256.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # ResNet-RS-200 ImageNet classification. 83.5% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs270_i256.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs270_i256.yaml index 4849a457e14..b7c6a644e2c 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs270_i256.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs270_i256.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # ResNet-RS-270 ImageNet classification. 83.6% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs350_i256.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs350_i256.yaml index a025dd481a0..3b2d3fe261c 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs350_i256.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs350_i256.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # ResNet-RS-350 ImageNet classification. 83.7% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs350_i320.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs350_i320.yaml index 5110ad6459d..36cdba7bb43 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs350_i320.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs350_i320.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # ResNet-RS-350 ImageNet classification. 84.2% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs420_i320.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs420_i320.yaml index 7d249604175..9b02b7e006a 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs420_i320.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs420_i320.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - runtime: distribution_strategy: 'tpu' mixed_precision_dtype: 'bfloat16' diff --git a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs50_i160.yaml b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs50_i160.yaml index 8303553e3b9..a57f41f3908 100644 --- a/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs50_i160.yaml +++ b/official/vision/beta/configs/experiments/image_classification/imagenet_resnetrs50_i160.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # ResNet-RS-50 ImageNet classification. 79.1% top-1 accuracy. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet143_cascadercnn_tpu.yaml b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet143_cascadercnn_tpu.yaml index 0121ef53148..1f8b245da76 100644 --- a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet143_cascadercnn_tpu.yaml +++ b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet143_cascadercnn_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # --experiment_type=cascadercnn_spinenet_coco # Expect to reach: box mAP: 51.9%, mask mAP: 45.0% on COCO runtime: diff --git a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet143_mrcnn_tpu.yaml b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet143_mrcnn_tpu.yaml index 0d83e409379..4d5ec8ae481 100644 --- a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet143_mrcnn_tpu.yaml +++ b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet143_mrcnn_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Expect to reach: box mAP: 49.3%, mask mAP: 43.4% on COCO runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet49_cascadercnn_tpu.yaml b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet49_cascadercnn_tpu.yaml index 26df9523e82..a28dd4bb0fc 100644 --- a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet49_cascadercnn_tpu.yaml +++ b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet49_cascadercnn_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # --experiment_type=cascadercnn_spinenet_coco # Expect to reach: box mAP: 46.4%, mask mAP: 40.0% on COCO runtime: diff --git a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet49_mrcnn_tpu.yaml b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet49_mrcnn_tpu.yaml index 674ceac933a..4ac1ae428bf 100644 --- a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet49_mrcnn_tpu.yaml +++ b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet49_mrcnn_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Expect to reach: box mAP: 43.2%, mask mAP: 38.3% on COCO runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet96_casrcnn_tpu.yaml b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet96_casrcnn_tpu.yaml index 2ec84bd7dad..612608333c3 100644 --- a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet96_casrcnn_tpu.yaml +++ b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet96_casrcnn_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - runtime: distribution_strategy: 'tpu' mixed_precision_dtype: 'bfloat16' diff --git a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet96_mrcnn_tpu.yaml b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet96_mrcnn_tpu.yaml index d30a532a098..9609b7eee34 100644 --- a/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet96_mrcnn_tpu.yaml +++ b/official/vision/beta/configs/experiments/maskrcnn/coco_spinenet96_mrcnn_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Expect to reach: box mAP: 48.1%, mask mAP: 42.4% on COCO runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/maskrcnn/r50fpn_640_coco_scratch_tpu4x4.yaml b/official/vision/beta/configs/experiments/maskrcnn/r50fpn_640_coco_scratch_tpu4x4.yaml index 1ffc6f9e641..218f0451058 100644 --- a/official/vision/beta/configs/experiments/maskrcnn/r50fpn_640_coco_scratch_tpu4x4.yaml +++ b/official/vision/beta/configs/experiments/maskrcnn/r50fpn_640_coco_scratch_tpu4x4.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Expect to reach: box mAP: 42.3%, mask mAP: 37.6% on COCO task: init_checkpoint: null diff --git a/official/vision/beta/configs/experiments/retinanet/coco_spinenet143_tpu.yaml b/official/vision/beta/configs/experiments/retinanet/coco_spinenet143_tpu.yaml index dc3bc2b1710..438fe031a8b 100644 --- a/official/vision/beta/configs/experiments/retinanet/coco_spinenet143_tpu.yaml +++ b/official/vision/beta/configs/experiments/retinanet/coco_spinenet143_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # SpineNet-143 COCO detection with protocal C config. Expecting 50.0% AP. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/retinanet/coco_spinenet190_tpu.yaml b/official/vision/beta/configs/experiments/retinanet/coco_spinenet190_tpu.yaml index 653ef5fd791..bc0ea1f94ec 100644 --- a/official/vision/beta/configs/experiments/retinanet/coco_spinenet190_tpu.yaml +++ b/official/vision/beta/configs/experiments/retinanet/coco_spinenet190_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - runtime: distribution_strategy: 'tpu' mixed_precision_dtype: 'bfloat16' diff --git a/official/vision/beta/configs/experiments/retinanet/coco_spinenet49_mobile_tpu.yaml b/official/vision/beta/configs/experiments/retinanet/coco_spinenet49_mobile_tpu.yaml index 094e067040e..116dd008493 100644 --- a/official/vision/beta/configs/experiments/retinanet/coco_spinenet49_mobile_tpu.yaml +++ b/official/vision/beta/configs/experiments/retinanet/coco_spinenet49_mobile_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # --experiment_type=retinanet_spinenet_mobile_coco runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/retinanet/coco_spinenet49_tpu.yaml b/official/vision/beta/configs/experiments/retinanet/coco_spinenet49_tpu.yaml index 1587130e978..725e1fc9b84 100644 --- a/official/vision/beta/configs/experiments/retinanet/coco_spinenet49_tpu.yaml +++ b/official/vision/beta/configs/experiments/retinanet/coco_spinenet49_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # SpineNet-49 COCO detection with protocal C config. Expecting 44.2% AP. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/retinanet/coco_spinenet49s_mobile_tpu.yaml b/official/vision/beta/configs/experiments/retinanet/coco_spinenet49s_mobile_tpu.yaml index f62ec9b348f..3b89a626759 100644 --- a/official/vision/beta/configs/experiments/retinanet/coco_spinenet49s_mobile_tpu.yaml +++ b/official/vision/beta/configs/experiments/retinanet/coco_spinenet49s_mobile_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # --experiment_type=retinanet_spinenet_mobile_coco runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/retinanet/coco_spinenet49xs_mobile_tpu.yaml b/official/vision/beta/configs/experiments/retinanet/coco_spinenet49xs_mobile_tpu.yaml index 028a84beebf..8c9bf3a0b1f 100644 --- a/official/vision/beta/configs/experiments/retinanet/coco_spinenet49xs_mobile_tpu.yaml +++ b/official/vision/beta/configs/experiments/retinanet/coco_spinenet49xs_mobile_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # --experiment_type=retinanet_spinenet_mobile_coco runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/retinanet/coco_spinenet96_tpu.yaml b/official/vision/beta/configs/experiments/retinanet/coco_spinenet96_tpu.yaml index 7b794cd1651..c75d667753f 100644 --- a/official/vision/beta/configs/experiments/retinanet/coco_spinenet96_tpu.yaml +++ b/official/vision/beta/configs/experiments/retinanet/coco_spinenet96_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # SpineNet-96 COCO detection with protocol C config. Expecting 48.5% AP. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/retinanet/resnet50fpn_coco_tfds_tpu.yaml b/official/vision/beta/configs/experiments/retinanet/resnet50fpn_coco_tfds_tpu.yaml index 0d6cf883907..0f9a30a3443 100644 --- a/official/vision/beta/configs/experiments/retinanet/resnet50fpn_coco_tfds_tpu.yaml +++ b/official/vision/beta/configs/experiments/retinanet/resnet50fpn_coco_tfds_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - runtime: distribution_strategy: 'tpu' mixed_precision_dtype: 'bfloat16' diff --git a/official/vision/beta/configs/experiments/retinanet/resnet50fpn_coco_tpu4x4_benchmark.yaml b/official/vision/beta/configs/experiments/retinanet/resnet50fpn_coco_tpu4x4_benchmark.yaml index 1523946cb1b..46b1f3cad7c 100644 --- a/official/vision/beta/configs/experiments/retinanet/resnet50fpn_coco_tpu4x4_benchmark.yaml +++ b/official/vision/beta/configs/experiments/retinanet/resnet50fpn_coco_tpu4x4_benchmark.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Benchmarks runs on same instnace, change eval batch size to fit on 4x4 tpu task: validation_data: diff --git a/official/vision/beta/configs/experiments/semantic_segmentation/deeplabv3plus_resnet101_cityscapes_tfds_tpu.yaml b/official/vision/beta/configs/experiments/semantic_segmentation/deeplabv3plus_resnet101_cityscapes_tfds_tpu.yaml index bf929ee4ee4..4ffc7689d2f 100644 --- a/official/vision/beta/configs/experiments/semantic_segmentation/deeplabv3plus_resnet101_cityscapes_tfds_tpu.yaml +++ b/official/vision/beta/configs/experiments/semantic_segmentation/deeplabv3plus_resnet101_cityscapes_tfds_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Use your own cityscapes preprocessed dataset. 79% meanIoU. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/configs/experiments/video_classification/k400_3d-resnet50_tpu.yaml b/official/vision/beta/configs/experiments/video_classification/k400_3d-resnet50_tpu.yaml index 704c727ffe6..d9158c2bd17 100644 --- a/official/vision/beta/configs/experiments/video_classification/k400_3d-resnet50_tpu.yaml +++ b/official/vision/beta/configs/experiments/video_classification/k400_3d-resnet50_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # 3D ResNet-50 video classification on Kinetics-400. # # --experiment_type=video_classification_kinetics400 diff --git a/official/vision/beta/configs/experiments/video_classification/k400_resnet3drs_50_tpu.yaml b/official/vision/beta/configs/experiments/video_classification/k400_resnet3drs_50_tpu.yaml index 9e1977297ec..83875d1273a 100644 --- a/official/vision/beta/configs/experiments/video_classification/k400_resnet3drs_50_tpu.yaml +++ b/official/vision/beta/configs/experiments/video_classification/k400_resnet3drs_50_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # 3D ResNet-RS-50 video classification on Kinetics-400. # # --experiment_type=video_classification_kinetics400 diff --git a/official/vision/beta/configs/experiments/video_classification/k400_slowonly16x4_tpu.yaml b/official/vision/beta/configs/experiments/video_classification/k400_slowonly16x4_tpu.yaml index b6d20bfb4c1..8e6793a374e 100644 --- a/official/vision/beta/configs/experiments/video_classification/k400_slowonly16x4_tpu.yaml +++ b/official/vision/beta/configs/experiments/video_classification/k400_slowonly16x4_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # SlowOnly 16x4 video classification on Kinetics-400. # # --experiment_type=video_classification_kinetics400 diff --git a/official/vision/beta/configs/experiments/video_classification/k400_slowonly8x8_tpu.yaml b/official/vision/beta/configs/experiments/video_classification/k400_slowonly8x8_tpu.yaml index c711f2e105c..c0bcd881ef3 100644 --- a/official/vision/beta/configs/experiments/video_classification/k400_slowonly8x8_tpu.yaml +++ b/official/vision/beta/configs/experiments/video_classification/k400_slowonly8x8_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # SlowOnly 8x8 video classification on Kinetics-400. # # --experiment_type=video_classification_kinetics400 diff --git a/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50_tpu.yaml b/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50_tpu.yaml index 348a2466f6b..ceb38608d6d 100644 --- a/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50_tpu.yaml +++ b/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # 3D ResNet-50 video classification on Kinetics-600. # # --experiment_type=video_classification_kinetics600 diff --git a/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50g_tpu.yaml b/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50g_tpu.yaml index 38627270718..3ae54c41564 100644 --- a/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50g_tpu.yaml +++ b/official/vision/beta/configs/experiments/video_classification/k600_3d-resnet50g_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # 3D ResNet-50g video classification on Kinetics-600. # # --experiment_type=video_classification_kinetics600 diff --git a/official/vision/beta/configs/experiments/video_classification/k600_slowonly8x8_tpu.yaml b/official/vision/beta/configs/experiments/video_classification/k600_slowonly8x8_tpu.yaml index 6e7e3e1fdb2..43f656ce3f0 100644 --- a/official/vision/beta/configs/experiments/video_classification/k600_slowonly8x8_tpu.yaml +++ b/official/vision/beta/configs/experiments/video_classification/k600_slowonly8x8_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # SlowOnly 8x8 video classification on Kinetics-600. # # --experiment_type=video_classification_kinetics600 diff --git a/official/vision/beta/projects/README.md b/official/vision/beta/projects/README.md index 93cc973ce8f..9c20f07fc60 100644 --- a/official/vision/beta/projects/README.md +++ b/official/vision/beta/projects/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - Here are a few projects that are built on tf.vision. They are build and maintain by different parties. They can be used as examples of how to build your own projects based on tf.vision. diff --git a/official/vision/beta/projects/assemblenet/README.md b/official/vision/beta/projects/assemblenet/README.md index 66b9c56537e..fb19541f9fc 100644 --- a/official/vision/beta/projects/assemblenet/README.md +++ b/official/vision/beta/projects/assemblenet/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # AssembleNet and AssembleNet++ This repository is the official implementations of the following papers. diff --git a/official/vision/beta/projects/deepmac_maskrcnn/README.md b/official/vision/beta/projects/deepmac_maskrcnn/README.md index ed0b1be8b2a..1e241f2cf9b 100644 --- a/official/vision/beta/projects/deepmac_maskrcnn/README.md +++ b/official/vision/beta/projects/deepmac_maskrcnn/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Mask R-CNN with deep mask heads This project brings insights from the DeepMAC model into the Mask-RCNN diff --git a/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r101_hg52.yaml b/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r101_hg52.yaml index a3e4d443ae3..1a097653808 100644 --- a/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r101_hg52.yaml +++ b/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r101_hg52.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - task: # VOC class taken from # models/official/vision/detection/utils/class_utils.py diff --git a/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r50.yaml b/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r50.yaml index cb925d38fea..792bc9c1f7c 100644 --- a/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r50.yaml +++ b/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r50.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - task: # VOC class taken from # models/official/vision/detection/utils/class_utils.py diff --git a/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r50_hg52.yaml b/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r50_hg52.yaml index 79f25ec5aae..060bfb61b2e 100644 --- a/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r50_hg52.yaml +++ b/official/vision/beta/projects/deepmac_maskrcnn/configs/experiments/deep_mask_head_rcnn_voc_r50_hg52.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - task: # VOC class taken from # models/official/vision/detection/utils/class_utils.py diff --git a/official/vision/beta/projects/movinet/README.md b/official/vision/beta/projects/movinet/README.md index a19576cf97e..5ccf1d3e838 100644 --- a/official/vision/beta/projects/movinet/README.md +++ b/official/vision/beta/projects/movinet/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Mobile Video Networks (MoViNets) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tensorflow/models/blob/master/official/vision/beta/projects/movinet/movinet_tutorial.ipynb) diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_8x8.yaml index 0c3ea819ebc..bd7d3ce92a9 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_8x8.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Video classification on Kinetics-600 using MoViNet-A0 backbone. # --experiment_type=movinet_kinetics600 # Achieves 72.28% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_cpu_local.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_cpu_local.yaml index 4424f422139..a144ac56e4d 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_cpu_local.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_cpu_local.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Video classification on Kinetics-600 using MoViNet-A0 backbone. # --experiment_type=movinet_kinetics600 diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_stream_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_stream_k600_8x8.yaml index 4ed49a1be98..df4afcc4dab 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_stream_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a0_stream_k600_8x8.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Video classification on Kinetics-600 using MoViNet-A0-Stream backbone. # --experiment_type=movinet_kinetics600 # Achieves 72.03% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a1_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a1_k600_8x8.yaml index e05c3c411a9..8c097f49b4a 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a1_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a1_k600_8x8.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Video classification on Kinetics-600 using MoViNet-A1 backbone. # --experiment_type=movinet_kinetics600 # Achieves 76.69% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a1_stream_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a1_stream_k600_8x8.yaml index 6abf33886cb..3452067ad9c 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a1_stream_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a1_stream_k600_8x8.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Video classification on Kinetics-600 using MoViNet-A1-Stream backbone. # --experiment_type=movinet_kinetics600 # Achieves 76.45% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a2_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a2_k600_8x8.yaml index 8e9581d7d86..575772b9f3e 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a2_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a2_k600_8x8.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Video classification on Kinetics-600 using MoViNet-A2 backbone. # --experiment_type=movinet_kinetics600 # Achieves 78.62% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a2_stream_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a2_stream_k600_8x8.yaml index 83a6cd04ea1..206c7940311 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a2_stream_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a2_stream_k600_8x8.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Video classification on Kinetics-600 using MoViNet-A2-Stream backbone. # --experiment_type=movinet_kinetics600 # Achieves 78.40% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a3_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a3_k600_8x8.yaml index b0f503aa670..a4d34314695 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a3_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a3_k600_8x8.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Video classification on Kinetics-600 using MoViNet-A3 backbone. # --experiment_type=movinet_kinetics600 # Achieves 81.79% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a3_stream_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a3_stream_k600_8x8.yaml index 62ec0e0ae5c..d503e3dc6f9 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a3_stream_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a3_stream_k600_8x8.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Video classification on Kinetics-600 using MoViNet-A3-Stream backbone. # --experiment_type=movinet_kinetics600 # Achieves 80.09% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a4_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a4_k600_8x8.yaml index 6b689bb41ff..102ccad4f55 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a4_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a4_k600_8x8.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Video classification on Kinetics-600 using MoViNet-A4 backbone. # --experiment_type=movinet_kinetics600 # Achieves 83.48% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a4_stream_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a4_stream_k600_8x8.yaml index 2eab748d043..63b9865d366 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a4_stream_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a4_stream_k600_8x8.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Video classification on Kinetics-600 using MoViNet-A4 backbone. # --experiment_type=movinet_kinetics600 # Achieves 81.33% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_k600_8x8.yaml index a6aa551e71f..79c9d209d91 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_k600_8x8.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Video classification on Kinetics-600 using MoViNet-A5 backbone. # --experiment_type=movinet_kinetics600 # Achieves 84.00% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_stream_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_stream_k600_8x8.yaml index d1bcc87a967..1983937679f 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_stream_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_a5_stream_k600_8x8.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Video classification on Kinetics-600 using MoViNet-A5-Stream backbone. # --experiment_type=movinet_kinetics600 # Achieves 82.37% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_t0_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_t0_k600_8x8.yaml index 67495e7f042..b6b190c8acb 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_t0_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_t0_k600_8x8.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Video classification on Kinetics-600 using MoViNet-T0 backbone. # --experiment_type=movinet_kinetics600 # Achieves 68.40% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/configs/yaml/movinet_t0_stream_k600_8x8.yaml b/official/vision/beta/projects/movinet/configs/yaml/movinet_t0_stream_k600_8x8.yaml index ebf0cc93111..320a51ea681 100644 --- a/official/vision/beta/projects/movinet/configs/yaml/movinet_t0_stream_k600_8x8.yaml +++ b/official/vision/beta/projects/movinet/configs/yaml/movinet_t0_stream_k600_8x8.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Video classification on Kinetics-600 using MoViNet-T0-Stream backbone. # --experiment_type=movinet_kinetics600 # Achieves 67.17% Top-1 accuracy. diff --git a/official/vision/beta/projects/movinet/movinet_tutorial.ipynb b/official/vision/beta/projects/movinet/movinet_tutorial.ipynb index 7ad97fb722e..319489d7a78 100644 --- a/official/vision/beta/projects/movinet/movinet_tutorial.ipynb +++ b/official/vision/beta/projects/movinet/movinet_tutorial.ipynb @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - { "cells": [ { diff --git a/official/vision/beta/projects/movinet/requirements.txt b/official/vision/beta/projects/movinet/requirements.txt index 190cec2dcb7..55b985f2258 100644 --- a/official/vision/beta/projects/movinet/requirements.txt +++ b/official/vision/beta/projects/movinet/requirements.txt @@ -1,15 +1 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - mediapy diff --git a/official/vision/beta/projects/simclr/README.md b/official/vision/beta/projects/simclr/README.md index b381c5c4718..91b4375bd60 100644 --- a/official/vision/beta/projects/simclr/README.md +++ b/official/vision/beta/projects/simclr/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Simple Framework for Contrastive Learning [![Paper](http://img.shields.io/badge/Paper-arXiv.2002.05709-B3181B?logo=arXiv)](https://arxiv.org/abs/2002.05709) diff --git a/official/vision/beta/projects/simclr/configs/experiments/cifar_simclr_pretrain.yaml b/official/vision/beta/projects/simclr/configs/experiments/cifar_simclr_pretrain.yaml index f4b5d006701..07d319a6929 100644 --- a/official/vision/beta/projects/simclr/configs/experiments/cifar_simclr_pretrain.yaml +++ b/official/vision/beta/projects/simclr/configs/experiments/cifar_simclr_pretrain.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Cifar classification. runtime: distribution_strategy: 'mirrored' diff --git a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_gpu.yaml b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_gpu.yaml index 49db1edb081..13b02cdf113 100644 --- a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_gpu.yaml +++ b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_gpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # SimCLR Imagenet 10% finetuning. runtime: distribution_strategy: 'mirrored' diff --git a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_tpu.yaml b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_tpu.yaml index 933736fd509..45cceb5fcd4 100644 --- a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_tpu.yaml +++ b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # SimCLR Imagenet 10% finetuning. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_gpu.yaml b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_gpu.yaml index 31ce98d7ee5..f2fa25ef8e7 100644 --- a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_gpu.yaml +++ b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_gpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # SimCLR Imagenet pretraining. runtime: distribution_strategy: 'mirrored' diff --git a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_tpu.yaml b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_tpu.yaml index 8d7f48e9374..f5c8045483b 100644 --- a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_tpu.yaml +++ b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # SimCLR Imagenet pretraining. runtime: distribution_strategy: 'tpu' diff --git a/official/vision/beta/projects/yolo/README.md b/official/vision/beta/projects/yolo/README.md index 707ad2f0b2a..0a1e27fbe90 100644 --- a/official/vision/beta/projects/yolo/README.md +++ b/official/vision/beta/projects/yolo/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # YOLO Object Detectors, You Only Look Once [![Paper](http://img.shields.io/badge/Paper-arXiv.1804.02767-B3181B?logo=arXiv)](https://arxiv.org/abs/1804.02767) diff --git a/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53.yaml b/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53.yaml index 60feb5fc0f2..10dbdc56855 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - runtime: distribution_strategy: 'mirrored' mixed_precision_dtype: 'float32' diff --git a/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53_tfds.yaml b/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53_tfds.yaml index 778bca2e408..b27ff015708 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53_tfds.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53_tfds.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - runtime: distribution_strategy: 'mirrored' mixed_precision_dtype: 'float16' diff --git a/official/vision/beta/projects/yolo/configs/experiments/darknet53.yaml b/official/vision/beta/projects/yolo/configs/experiments/darknet53.yaml index bf2f37a984c..a3333b599e3 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/darknet53.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/darknet53.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - runtime: distribution_strategy: 'mirrored' mixed_precision_dtype: 'float32' diff --git a/official/vision/beta/projects/yolo/configs/experiments/darknet53_tfds.yaml b/official/vision/beta/projects/yolo/configs/experiments/darknet53_tfds.yaml index 32d935a7cb0..8f9fb2dfc6b 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/darknet53_tfds.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/darknet53_tfds.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - runtime: distribution_strategy: 'mirrored' mixed_precision_dtype: 'float16' diff --git a/official/vision/beta/projects/yt8m/README.md b/official/vision/beta/projects/yt8m/README.md index 2acaef2b906..2f4cf2ab4b0 100644 --- a/official/vision/beta/projects/yt8m/README.md +++ b/official/vision/beta/projects/yt8m/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # YouTube-8M Tensorflow Starter Code DISCLAIMER: This project is still under development. diff --git a/official/vision/beta/projects/yt8m/experiments/yt8m.yaml b/official/vision/beta/projects/yt8m/experiments/yt8m.yaml index 40e7c784587..c099f23f90b 100644 --- a/official/vision/beta/projects/yt8m/experiments/yt8m.yaml +++ b/official/vision/beta/projects/yt8m/experiments/yt8m.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # yt8m config file task: model: diff --git a/official/vision/beta/projects/yt8m/experiments/yt8m_test.yaml b/official/vision/beta/projects/yt8m/experiments/yt8m_test.yaml index 343629aa710..9a7ef94cc73 100644 --- a/official/vision/beta/projects/yt8m/experiments/yt8m_test.yaml +++ b/official/vision/beta/projects/yt8m/experiments/yt8m_test.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # yt8m test config file task: model: diff --git a/official/vision/detection/README.md b/official/vision/detection/README.md index 6985f3a79c7..2633f86d5dc 100644 --- a/official/vision/detection/README.md +++ b/official/vision/detection/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Object Detection Models on TensorFlow 2 **WARNING**: This repository will be deprecated and replaced by the solid diff --git a/official/vision/image_classification/README.md b/official/vision/image_classification/README.md index 6137e9ccce4..78bfe1f27e6 100644 --- a/official/vision/image_classification/README.md +++ b/official/vision/image_classification/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Image Classification This folder contains TF 2.0 model examples for image classification: diff --git a/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-gpu.yaml b/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-gpu.yaml index 756b489fef5..6f40ffb1e30 100644 --- a/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-gpu.yaml +++ b/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-gpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Training configuration for EfficientNet-b0 trained on ImageNet on GPUs. # Takes ~32 minutes per epoch for 8 V100s. # Reaches ~76.1% within 350 epochs. diff --git a/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-tpu.yaml b/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-tpu.yaml index b406fbe0f6c..c5be7e9ba32 100644 --- a/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-tpu.yaml +++ b/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b0-tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Training configuration for EfficientNet-b0 trained on ImageNet on TPUs. # Takes ~2 minutes, 50 seconds per epoch for v3-32. # Reaches ~76.1% within 350 epochs. diff --git a/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-gpu.yaml b/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-gpu.yaml index f4ecba89b8e..2f3dce01a46 100644 --- a/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-gpu.yaml +++ b/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-gpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Note: This configuration uses a scaled per-replica batch size based on the number of devices. runtime: distribution_strategy: 'mirrored' diff --git a/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-tpu.yaml b/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-tpu.yaml index 4a96ce232e8..0bb6a9fe6f0 100644 --- a/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-tpu.yaml +++ b/official/vision/image_classification/configs/examples/efficientnet/imagenet/efficientnet-b1-tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Training configuration for EfficientNet-b1 trained on ImageNet on TPUs. # Takes ~3 minutes, 15 seconds per epoch for v3-32. # Note: This configuration uses a scaled per-replica batch size based on the number of devices. diff --git a/official/vision/image_classification/configs/examples/resnet/imagenet/gpu.yaml b/official/vision/image_classification/configs/examples/resnet/imagenet/gpu.yaml index 5fc88552caa..2037d6b5d1c 100644 --- a/official/vision/image_classification/configs/examples/resnet/imagenet/gpu.yaml +++ b/official/vision/image_classification/configs/examples/resnet/imagenet/gpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Training configuration for ResNet trained on ImageNet on GPUs. # Reaches > 76.1% within 90 epochs. # Note: This configuration uses a scaled per-replica batch size based on the number of devices. diff --git a/official/vision/image_classification/configs/examples/resnet/imagenet/tpu.yaml b/official/vision/image_classification/configs/examples/resnet/imagenet/tpu.yaml index cda4cd6df9d..0a3030333bb 100644 --- a/official/vision/image_classification/configs/examples/resnet/imagenet/tpu.yaml +++ b/official/vision/image_classification/configs/examples/resnet/imagenet/tpu.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Training configuration for ResNet trained on ImageNet on TPUs. # Takes ~4 minutes, 30 seconds seconds per epoch for a v3-32. # Reaches > 76.1% within 90 epochs. diff --git a/official/vision/image_classification/resnet/README.md b/official/vision/image_classification/resnet/README.md index a5243db2e01..5064523fbdc 100644 --- a/official/vision/image_classification/resnet/README.md +++ b/official/vision/image_classification/resnet/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - This folder contains a [custom training loop (CTL)](#resnet-custom-training-loop) implementation for ResNet50. diff --git a/official/vision/keras_cv/LICENSE b/official/vision/keras_cv/LICENSE index 945e036f578..0b1ba442980 100644 --- a/official/vision/keras_cv/LICENSE +++ b/official/vision/keras_cv/LICENSE @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - Copyright 2020 The TensorFlow Authors. All rights reserved. Apache License diff --git a/official/vision/keras_cv/README.md b/official/vision/keras_cv/README.md index 4c9917964f5..1132d521cd7 100644 --- a/official/vision/keras_cv/README.md +++ b/official/vision/keras_cv/README.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # keras-cv ## Losses diff --git a/official/vision/keras_cv/contributing.md b/official/vision/keras_cv/contributing.md index 729528ae38a..d9efe9b0691 100644 --- a/official/vision/keras_cv/contributing.md +++ b/official/vision/keras_cv/contributing.md @@ -1,17 +1,3 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ## Contributing to KerasCV Patches to KerasCV are welcome! diff --git a/official/vision/keras_cv/requirements.txt b/official/vision/keras_cv/requirements.txt index 6d50ed5b314..6bad10388ec 100644 --- a/official/vision/keras_cv/requirements.txt +++ b/official/vision/keras_cv/requirements.txt @@ -1,16 +1,2 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - numpy scipy From a1fd33c5a34c5c24dd6804217f1a0ec0d7ed9856 Mon Sep 17 00:00:00 2001 From: Hongkun Yu Date: Mon, 7 Jun 2021 10:13:01 -0700 Subject: [PATCH 30/50] Internal change PiperOrigin-RevId: 377944468 --- official/nlp/modeling/ops/sampling_module.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/official/nlp/modeling/ops/sampling_module.py b/official/nlp/modeling/ops/sampling_module.py index 5bd758fd911..a9270ba4bba 100644 --- a/official/nlp/modeling/ops/sampling_module.py +++ b/official/nlp/modeling/ops/sampling_module.py @@ -431,17 +431,17 @@ def _process_finished_state( def _continue_search(self, state) -> tf.Tensor: i = state[decoding_module.StateKeys.CUR_INDEX] - return tf.less(i, self.max_decode_length) + # Have we reached max decoding length? + not_at_end = tf.less(i, self.max_decode_length) + # Have all sampled sequences reached an EOS? + all_has_eos = tf.reduce_all( + state[decoding_module.StateKeys.FINISHED_FLAGS], + axis=None, + name="search_finish_cond") + return tf.logical_and(not_at_end, tf.logical_not(all_has_eos)) def _finished_flags(self, topk_ids, state) -> tf.Tensor: new_finished_flags = tf.equal(topk_ids, self.eos_id) new_finished_flags = tf.logical_or( new_finished_flags, state[decoding_module.StateKeys.FINISHED_FLAGS]) return new_finished_flags - - - - - - - From b09e75828e2c65ead9e624a5c7afed8d214247aa Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 7 Jun 2021 11:02:44 -0700 Subject: [PATCH 31/50] Internal change PiperOrigin-RevId: 377957521 --- .../configs/optimization_config.py | 2 ++ .../optimization/configs/optimizer_config.py | 21 +++++++++++++++++++ .../optimization/optimizer_factory.py | 2 ++ .../modeling/optimization/slide_optimizer.py | 20 ++++++++++++++++++ 4 files changed, 45 insertions(+) create mode 100644 official/modeling/optimization/slide_optimizer.py diff --git a/official/modeling/optimization/configs/optimization_config.py b/official/modeling/optimization/configs/optimization_config.py index 4b6e400b61e..49a4db624d9 100644 --- a/official/modeling/optimization/configs/optimization_config.py +++ b/official/modeling/optimization/configs/optimization_config.py @@ -41,6 +41,7 @@ class OptimizerConfig(oneof.OneOfConfig): rmsprop: rmsprop optimizer. lars: lars optimizer. adagrad: adagrad optimizer. + slide: slide optimizer. """ type: Optional[str] = None sgd: opt_cfg.SGDConfig = opt_cfg.SGDConfig() @@ -50,6 +51,7 @@ class OptimizerConfig(oneof.OneOfConfig): rmsprop: opt_cfg.RMSPropConfig = opt_cfg.RMSPropConfig() lars: opt_cfg.LARSConfig = opt_cfg.LARSConfig() adagrad: opt_cfg.AdagradConfig = opt_cfg.AdagradConfig() + slide: opt_cfg.SLIDEConfig = opt_cfg.SLIDEConfig() @dataclasses.dataclass diff --git a/official/modeling/optimization/configs/optimizer_config.py b/official/modeling/optimization/configs/optimizer_config.py index 7b4de948248..1d9570e21a5 100644 --- a/official/modeling/optimization/configs/optimizer_config.py +++ b/official/modeling/optimization/configs/optimizer_config.py @@ -226,3 +226,24 @@ class LARSConfig(BaseOptimizerConfig): classic_momentum: bool = True exclude_from_weight_decay: Optional[List[str]] = None exclude_from_layer_adaptation: Optional[List[str]] = None + + +@dataclasses.dataclass +class SLIDEConfig(BaseOptimizerConfig): + """Configuration for SLIDE optimizer. + + Details coming soon. + """ + name: str = "SLIDE" + beta_1: float = 0.9 + beta_2: float = 0.999 + epsilon: float = 1e-6 + weight_decay_rate: float = 0.0 + weight_decay_type: str = "inner" + exclude_from_weight_decay: Optional[List[str]] = None + exclude_from_layer_adaptation: Optional[List[str]] = None + include_in_sparse_layer_adaptation: Optional[List[str]] = None + sparse_layer_learning_rate: float = 0.1 + do_gradient_rescaling: bool = True + norm_type: str = "layer" + ratio_clip_norm: float = 1e5 diff --git a/official/modeling/optimization/optimizer_factory.py b/official/modeling/optimization/optimizer_factory.py index 09bb5deda6f..c41d98fb607 100644 --- a/official/modeling/optimization/optimizer_factory.py +++ b/official/modeling/optimization/optimizer_factory.py @@ -19,6 +19,7 @@ import tensorflow as tf import tensorflow_addons.optimizers as tfa_optimizers +from official.modeling.optimization import slide_optimizer from official.modeling.optimization import ema_optimizer from official.modeling.optimization import lars_optimizer from official.modeling.optimization import lr_schedule @@ -33,6 +34,7 @@ 'rmsprop': tf.keras.optimizers.RMSprop, 'lars': lars_optimizer.LARS, 'adagrad': tf.keras.optimizers.Adagrad, + 'slide': slide_optimizer.SLIDE } LR_CLS = { diff --git a/official/modeling/optimization/slide_optimizer.py b/official/modeling/optimization/slide_optimizer.py new file mode 100644 index 00000000000..c1975a3111e --- /dev/null +++ b/official/modeling/optimization/slide_optimizer.py @@ -0,0 +1,20 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SLIDE optimizer. + +A new optimizer that will be open sourced soon. +""" + +SLIDE = "Unimplemented" From acdb71d6057e2b70a73a654d22b837a9cfbcfca0 Mon Sep 17 00:00:00 2001 From: Vighnesh Birodkar Date: Tue, 8 Jun 2021 08:05:40 -0700 Subject: [PATCH 32/50] Silence warnings about unmatched checkpoint objects. PiperOrigin-RevId: 378154667 --- research/object_detection/model_lib_v2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/research/object_detection/model_lib_v2.py b/research/object_detection/model_lib_v2.py index 012cae74e36..798c52532e9 100644 --- a/research/object_detection/model_lib_v2.py +++ b/research/object_detection/model_lib_v2.py @@ -397,7 +397,8 @@ def load_fine_tune_checkpoint(model, checkpoint_path, checkpoint_type, fine_tune_checkpoint_type=checkpoint_type) validate_tf_v2_checkpoint_restore_map(restore_from_objects_dict) ckpt = tf.train.Checkpoint(**restore_from_objects_dict) - ckpt.restore(checkpoint_path).assert_existing_objects_matched() + ckpt.restore( + checkpoint_path).expect_partial().assert_existing_objects_matched() def get_filepath(strategy, filepath): From aa94accd98a324e70ddea134e4c6fd467ed43e5b Mon Sep 17 00:00:00 2001 From: Yu-hui Chen Date: Tue, 8 Jun 2021 10:13:42 -0700 Subject: [PATCH 33/50] Updated the keypoint target assigner such that it blacks out the instance bbox region for the keypoint heatmap if the instance's keypoint visibility is 0. PiperOrigin-RevId: 378183175 --- .../object_detection/core/target_assigner.py | 24 ++++++++++++------- .../core/target_assigner_test.py | 19 +++++++++------ .../center_net_meta_arch.py | 3 +-- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/research/object_detection/core/target_assigner.py b/research/object_detection/core/target_assigner.py index 43b54e6327d..e2c1707179c 100644 --- a/research/object_detection/core/target_assigner.py +++ b/research/object_detection/core/target_assigner.py @@ -1409,8 +1409,10 @@ def assign_keypoint_heatmap_targets(self, [batch_size, num_keypoints] representing number of instances for each keypoint type. valid_mask: A float tensor with shape [batch_size, output_height, - output_width] where all values within the regions of the blackout boxes - are 0.0 and 1.0 else where. + output_width, num_keypoints] where all values within the regions of the + blackout boxes are 0.0 and 1.0 else where. Note that the blackout boxes + are per keypoint type and are blacked out if the keypoint + visibility/weight (of the corresponding keypoint type) is zero. """ out_width = tf.cast(tf.maximum(width // self._stride, 1), tf.float32) out_height = tf.cast(tf.maximum(height // self._stride, 1), tf.float32) @@ -1480,13 +1482,17 @@ def assign_keypoint_heatmap_targets(self, keypoint_std_dev = keypoint_std_dev * tf.stack( [sigma] * num_keypoints, axis=1) - # Generate the valid region mask to ignore regions with target class but - # no corresponding keypoints. - # Shape: [num_instances]. - blackout = tf.logical_and(classes[:, self._class_id] > 0, - tf.reduce_max(kp_weights, axis=1) < 1e-3) - valid_mask = ta_utils.blackout_pixel_weights_by_box_regions( - out_height, out_width, boxes.get(), blackout) + # Generate the per-keypoint type valid region mask to ignore regions + # with keypoint weights equal to zeros (e.g. visibility is 0). + # shape of valid_mask: [out_height, out_width, num_keypoints] + kp_weight_list = tf.unstack(kp_weights, axis=1) + valid_mask_channel_list = [] + for kp_weight in kp_weight_list: + blackout = kp_weight < 1e-3 + valid_mask_channel_list.append( + ta_utils.blackout_pixel_weights_by_box_regions( + out_height, out_width, boxes.get(), blackout)) + valid_mask = tf.stack(valid_mask_channel_list, axis=2) valid_mask_list.append(valid_mask) # Apply the Gaussian kernel to the keypoint coordinates. Returned heatmap diff --git a/research/object_detection/core/target_assigner_test.py b/research/object_detection/core/target_assigner_test.py index 510ebd9da99..e9ac80f6ccb 100644 --- a/research/object_detection/core/target_assigner_test.py +++ b/research/object_detection/core/target_assigner_test.py @@ -1699,7 +1699,7 @@ def graph_fn(): np.array([[0.0, 0.0, 0.3, 0.3], [0.0, 0.0, 0.5, 0.5], [0.0, 0.0, 0.5, 0.5], - [0.0, 0.0, 1.0, 1.0]]), + [0.5, 0.5, 1.0, 1.0]]), dtype=tf.float32) ] @@ -1728,15 +1728,20 @@ def graph_fn(): # Verify the number of instances is correct. np.testing.assert_array_almost_equal([[0, 1]], num_instances_batch) + self.assertAllEqual([1, 30, 20, 2], valid_mask.shape) # When calling the function, we specify the class id to be 1 (1th and 3rd) # instance and the keypoint indices to be [0, 2], meaning that the 1st # instance is the target class with no valid keypoints in it. As a result, - # the region of the 1st instance boxing box should be blacked out - # (0.0, 0.0, 0.5, 0.5), transfering to (0, 0, 15, 10) in absolute output - # space. - self.assertAlmostEqual(np.sum(valid_mask[:, 0:15, 0:10]), 0.0) - # All other values are 1.0 so the sum is: 30 * 20 - 15 * 10 = 450. - self.assertAlmostEqual(np.sum(valid_mask), 450.0) + # the region of both keypoint types of the 1st instance boxing box should be + # blacked out (0.0, 0.0, 0.5, 0.5), transfering to (0, 0, 15, 10) in + # absolute output space. + self.assertAlmostEqual(np.sum(valid_mask[:, 0:15, 0:10, 0:2]), 0.0) + # For the 2nd instance, only the 1st keypoint has visibility of 0 so only + # the corresponding valid mask contains zeros. + self.assertAlmostEqual(np.sum(valid_mask[:, 15:30, 10:20, 0]), 0.0) + # All other values are 1.0 so the sum is: + # 30 * 20 * 2 - 15 * 10 * 2 - 15 * 10 * 1 = 750. + self.assertAlmostEqual(np.sum(valid_mask), 750.0) def test_assign_keypoints_offset_targets(self): def graph_fn(): diff --git a/research/object_detection/meta_architectures/center_net_meta_arch.py b/research/object_detection/meta_architectures/center_net_meta_arch.py index 7c6014059e2..a523a559b3d 100644 --- a/research/object_detection/meta_architectures/center_net_meta_arch.py +++ b/research/object_detection/meta_architectures/center_net_meta_arch.py @@ -2755,8 +2755,7 @@ def _compute_kp_heatmap_loss(self, input_height, input_width, task_name, gt_weights_list=gt_weights_list, gt_classes_list=gt_classes_list, gt_boxes_list=gt_boxes_list) - flattened_valid_mask = _flatten_spatial_dimensions( - tf.expand_dims(valid_mask_batch, axis=-1)) + flattened_valid_mask = _flatten_spatial_dimensions(valid_mask_batch) flattened_heapmap_targets = _flatten_spatial_dimensions(keypoint_heatmap) # Sum over the number of instances per keypoint types to get the total # number of keypoints. Note that this is used to normalized the loss and we From 06f74216a048b402278384476f204b0acb924a6a Mon Sep 17 00:00:00 2001 From: Hongkun Yu Date: Tue, 8 Jun 2021 10:42:43 -0700 Subject: [PATCH 34/50] Add outputs_as_dict to SentencePredictionDataLoader. PiperOrigin-RevId: 378190887 --- .../data/sentence_prediction_dataloader.py | 5 +++++ .../sentence_prediction_dataloader_test.py | 22 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/official/nlp/data/sentence_prediction_dataloader.py b/official/nlp/data/sentence_prediction_dataloader.py index 766595bfe84..ddb5f8b8f94 100644 --- a/official/nlp/data/sentence_prediction_dataloader.py +++ b/official/nlp/data/sentence_prediction_dataloader.py @@ -40,6 +40,7 @@ class SentencePredictionDataConfig(cfg.DataConfig): label_type: str = 'int' # Whether to include the example id number. include_example_id: bool = False + outputs_as_dict: bool = False @data_loader_factory.register_data_loader_cls(SentencePredictionDataConfig) @@ -85,6 +86,10 @@ def _parse(self, record: Mapping[str, tf.Tensor]): if self._include_example_id: x['example_id'] = record['example_id'] + if self._params.outputs_as_dict: + x['next_sentence_labels'] = record['label_ids'] + return x + y = record['label_ids'] return (x, y) diff --git a/official/nlp/data/sentence_prediction_dataloader_test.py b/official/nlp/data/sentence_prediction_dataloader_test.py index cbced2ad2c3..6e3172a94b0 100644 --- a/official/nlp/data/sentence_prediction_dataloader_test.py +++ b/official/nlp/data/sentence_prediction_dataloader_test.py @@ -141,6 +141,28 @@ def test_load_dataset(self, label_type, expected_label_type): self.assertEqual(labels.shape, (batch_size,)) self.assertEqual(labels.dtype, expected_label_type) + def test_load_dataset_as_dict(self): + input_path = os.path.join(self.get_temp_dir(), 'train.tf_record') + batch_size = 10 + seq_length = 128 + _create_fake_preprocessed_dataset(input_path, seq_length, 'int') + data_config = loader.SentencePredictionDataConfig( + input_path=input_path, + seq_length=seq_length, + global_batch_size=batch_size, + label_type='int', + outputs_as_dict=True) + dataset = loader.SentencePredictionDataLoader(data_config).load() + features = next(iter(dataset)) + self.assertCountEqual([ + 'input_word_ids', 'input_mask', 'input_type_ids', 'next_sentence_labels' + ], features.keys()) + self.assertEqual(features['input_word_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['input_mask'].shape, (batch_size, seq_length)) + self.assertEqual(features['input_type_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['next_sentence_labels'].shape, (batch_size,)) + self.assertEqual(features['next_sentence_labels'].dtype, tf.int32) + class SentencePredictionTfdsDataLoaderTest(tf.test.TestCase, parameterized.TestCase): From 19738a077851cb6309f417d1c2d43ef4ee6124d3 Mon Sep 17 00:00:00 2001 From: Dan Ellis Date: Tue, 8 Jun 2021 14:51:39 -0400 Subject: [PATCH 35/50] Update link to Embedding Colab (#10048) The original Colab by malcolmslaney didn't work with the current VGGish/tensorflow. Changed the link to an updated version. --- research/audioset/vggish/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/research/audioset/vggish/README.md b/research/audioset/vggish/README.md index d20e5587af4..ec5bf4bd0c4 100644 --- a/research/audioset/vggish/README.md +++ b/research/audioset/vggish/README.md @@ -170,8 +170,7 @@ the postprocessor can be run after inference. If you don't need to use the released embeddings or YouTube-8M, then you could skip postprocessing and use raw embeddings. -A [Colab](https://colab.research.google.com/) -showing how to download the model and calculate the embeddings on your +A Colab showing how to download the model and calculate the embeddings on your own sound data is available here: -[AudioSet Embedding Colab](https://colab.research.google.com/drive/1TbX92UL9sYWbdwdGE0rJ9owmezB-Rl1C). +[VGGish Embedding Colab](https://colab.research.google.com/drive/1E3CaPAqCai9P9QhJ3WYPNCVmrJU4lAhF). From 728818715ec474c97541540f6d3c7d9e5176af24 Mon Sep 17 00:00:00 2001 From: Khanh LeViet Date: Tue, 8 Jun 2021 21:00:32 -0700 Subject: [PATCH 36/50] Add notebook and update ODT TFLite conversion doc to show how to make the model compatible with TFLite Task Library. PiperOrigin-RevId: 378313314 --- .../convert_odt_model_to_TFLite.ipynb | 413 ++++++++++++++++++ .../g3doc/running_on_mobile_tf2.md | 78 ++-- 2 files changed, 466 insertions(+), 25 deletions(-) create mode 100644 research/object_detection/colab_tutorials/convert_odt_model_to_TFLite.ipynb diff --git a/research/object_detection/colab_tutorials/convert_odt_model_to_TFLite.ipynb b/research/object_detection/colab_tutorials/convert_odt_model_to_TFLite.ipynb new file mode 100644 index 00000000000..37f0ab841e4 --- /dev/null +++ b/research/object_detection/colab_tutorials/convert_odt_model_to_TFLite.ipynb @@ -0,0 +1,413 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "RD3uxzaJweYr" + }, + "source": [ + "##### Copyright 2021 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "C-vBUz5IhJs8" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pHTibyMehTvH" + }, + "source": [ + "# Tutorial: Convert models trained using TensorFlow Object Detection API to TensorFlow Lite\n", + "\n", + "This tutorial demonstrate these steps:\n", + "* Convert TensorFlow models trained using the TensorFlow Object Detection API to [TensorFlow Lite](https://www.tensorflow.org/lite).\n", + "* Add the required metadata using [TFLite Metadata Writer API](https://www.tensorflow.org/lite/convert/metadata_writer_tutorial#object_detectors). This will make the TFLite model compatible with [TFLite Task Library](https://www.tensorflow.org/lite/inference_with_metadata/task_library/object_detector), so that the model can be integrated in mobile apps in 3 lines of code." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QIR1IFpnLJJA" + }, + "source": [ + "\u003ctable align=\"left\"\u003e\u003ctd\u003e\n", + " \u003ca target=\"_blank\" href=\"https://colab.sandbox.google.com/github/tensorflow/models/blob/master/research/object_detection/colab_tutorials/convert_odt_model_to_TFLite.ipynb\"\u003e\n", + " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\n", + " \u003c/a\u003e\n", + "\u003c/td\u003e\u003ctd\u003e\n", + " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/models/blob/master/research/object_detection/colab_tutorials/convert_odt_model_to_TFLite.ipynb\"\u003e\n", + " \u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", + "\u003c/td\u003e\u003c/table\u003e" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ok_Rpv7XNaFJ" + }, + "source": [ + "## Preparation" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "t7CAW5C1cmel" + }, + "source": [ + "### Install the TFLite Support Library" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DwtFa0jSnNU4" + }, + "outputs": [], + "source": [ + "!pip install -q tflite_support" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XRfJR9QXctAR" + }, + "source": [ + "### Install the TensorFlow Object Detection API\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7PP2P5XAqeI5" + }, + "outputs": [], + "source": [ + "import os\n", + "import pathlib\n", + "\n", + "# Clone the tensorflow models repository if it doesn't already exist\n", + "if \"models\" in pathlib.Path.cwd().parts:\n", + " while \"models\" in pathlib.Path.cwd().parts:\n", + " os.chdir('..')\n", + "elif not pathlib.Path('models').exists():\n", + " !git clone --depth 1 https://github.com/tensorflow/models" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bP6SSh6zqi07" + }, + "outputs": [], + "source": [ + "%%bash\n", + "cd models/research/\n", + "protoc object_detection/protos/*.proto --python_out=.\n", + "cp object_detection/packages/tf2/setup.py .\n", + "pip install -q ." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "i0to7aXKc0O9" + }, + "source": [ + "### Import the necessary libraries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4M8CC1PgqnSf" + }, + "outputs": [], + "source": [ + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import os\n", + "import random\n", + "import io\n", + "import imageio\n", + "import glob\n", + "import scipy.misc\n", + "import numpy as np\n", + "from six import BytesIO\n", + "from PIL import Image, ImageDraw, ImageFont\n", + "from IPython.display import display, Javascript\n", + "from IPython.display import Image as IPyImage\n", + "\n", + "import tensorflow as tf\n", + "\n", + "from object_detection.utils import label_map_util\n", + "from object_detection.utils import config_util\n", + "from object_detection.utils import visualization_utils as viz_utils\n", + "from object_detection.utils import colab_utils\n", + "from object_detection.utils import config_util\n", + "from object_detection.builders import model_builder\n", + "\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "s9WIOOMTNti5" + }, + "source": [ + "## Download a pretrained model from Model Zoo\n", + "\n", + "In this tutorial, we demonstrate converting a pretrained model `SSD MobileNet V2 FPNLite 640x640` in the [TensorFlow 2 Model Zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md). You can replace the model with your own model and the rest will work the same." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TIY3cxDgsxuZ" + }, + "outputs": [], + "source": [ + "!wget http://download.tensorflow.org/models/object_detection/tf2/20200711/ssd_mobilenet_v2_fpnlite_640x640_coco17_tpu-8.tar.gz\n", + "!tar -xf ssd_mobilenet_v2_fpnlite_640x640_coco17_tpu-8.tar.gz\n", + "!rm ssd_mobilenet_v2_fpnlite_640x640_coco17_tpu-8.tar.gz" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0gV8vr6nN-z9" + }, + "source": [ + "## Generate TensorFlow Lite Model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Z8FjeSmmxpXz" + }, + "source": [ + "### Step 1: Export TFLite inference graph\n", + "\n", + "First, we invoke `export_tflite_graph_tf2.py` to generate a TFLite-friendly intermediate SavedModel. This will then be passed to the TensorFlow Lite Converter for generating the final model.\n", + "\n", + "Use `--help` with the above script to get the full list of supported parameters.\n", + "These can fine-tune accuracy and speed for your model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ChfN-tzBXqko" + }, + "outputs": [], + "source": [ + "!python models/research/object_detection/export_tflite_graph_tf2.py \\\n", + " --trained_checkpoint_dir {'ssd_mobilenet_v2_fpnlite_640x640_coco17_tpu-8/checkpoint'} \\\n", + " --output_directory {'ssd_mobilenet_v2_fpnlite_640x640_coco17_tpu-8/tflite'} \\\n", + " --pipeline_config_path {'ssd_mobilenet_v2_fpnlite_640x640_coco17_tpu-8/pipeline.config'}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IPr06cZ3OY3H" + }, + "source": [ + "### Step 2: Convert to TFLite\n", + "\n", + "Use the [TensorFlow Lite Converter](https://www.tensorflow.org/lite/convert) to\n", + "convert the `SavedModel` to TFLite. Note that you need to use `from_saved_model`\n", + "for TFLite conversion with the Python API.\n", + "\n", + "You can also leverage\n", + "[Post-training Quantization](https://www.tensorflow.org/lite/performance/post_training_quantization)\n", + "to\n", + "[optimize performance](https://www.tensorflow.org/lite/performance/model_optimization)\n", + "and obtain a smaller model. In this tutorial, we use the [dynamic range quantization](https://www.tensorflow.org/lite/performance/post_training_quant)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "JMpy3Rlpq-Yq" + }, + "outputs": [], + "source": [ + "_TFLITE_MODEL_PATH = \"ssd_mobilenet_v2_fpnlite_640x640_coco17_tpu-8/model.tflite\"\n", + "\n", + "converter = tf.lite.TFLiteConverter.from_saved_model('ssd_mobilenet_v2_fpnlite_640x640_coco17_tpu-8/tflite/saved_model')\n", + "converter.optimizations = [tf.lite.Optimize.DEFAULT]\n", + "tflite_model = converter.convert()\n", + "\n", + "with open(_TFLITE_MODEL_PATH, 'wb') as f:\n", + " f.write(tflite_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fyjlnmaEOtKp" + }, + "source": [ + "### Step 3: Add Metadata\n", + "\n", + "The model needs to be packed with [TFLite Metadata](https://www.tensorflow.org/lite/convert/metadata) to enable easy integration into mobile apps using the [TFLite Task Library](https://www.tensorflow.org/lite/inference_with_metadata/task_library/object_detector). This metadata helps the inference code perform the correct pre \u0026 post processing as required by the model. Use the following code to create the metadata." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-ecGLG_Ovjcr" + }, + "outputs": [], + "source": [ + "# Download the COCO dataset label map that was used to trained the SSD MobileNet V2 FPNLite 640x640 model\n", + "!wget https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/mscoco_label_map.pbtxt -q\n", + "\n", + "# We need to convert the Object Detection API's labelmap into what the Task API needs:\n", + "# a txt file with one class name on each line from index 0 to N.\n", + "# The first '0' class indicates the background.\n", + "# This code assumes COCO detection which has 90 classes, you can write a label\n", + "# map file for your model if re-trained.\n", + "_ODT_LABEL_MAP_PATH = 'mscoco_label_map.pbtxt'\n", + "_TFLITE_LABEL_PATH = \"ssd_mobilenet_v2_fpnlite_640x640_coco17_tpu-8/tflite_label_map.txt\"\n", + "\n", + "category_index = label_map_util.create_category_index_from_labelmap(\n", + " _ODT_LABEL_MAP_PATH)\n", + "f = open(_TFLITE_LABEL_PATH, 'w')\n", + "for class_id in range(1, 91):\n", + " if class_id not in category_index:\n", + " f.write('???\\n')\n", + " continue\n", + " name = category_index[class_id]['name']\n", + " f.write(name+'\\n')\n", + "f.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YJSyXq5Qss9X" + }, + "source": [ + "Then we'll add the label map and other necessary metadata (e.g. normalization config) to the TFLite model.\n", + "\n", + "As the `SSD MobileNet V2 FPNLite 640x640` model take input image with pixel value in the range of [-1..1] ([code](https://github.com/tensorflow/models/blob/b09e75828e2c65ead9e624a5c7afed8d214247aa/research/object_detection/models/ssd_mobilenet_v2_keras_feature_extractor.py#L132)), we need to set `norm_mean = 127.5` and `norm_std = 127.5`. See this [documentation](https://www.tensorflow.org/lite/convert/metadata#normalization_and_quantization_parameters) for more details on the normalization parameters." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CRQpfDAWsPeK" + }, + "outputs": [], + "source": [ + "from tflite_support.metadata_writers import object_detector\n", + "from tflite_support.metadata_writers import writer_utils\n", + "\n", + "_TFLITE_MODEL_WITH_METADATA_PATH = \"ssd_mobilenet_v2_fpnlite_640x640_coco17_tpu-8/model_with_metadata.tflite\"\n", + "\n", + "writer = object_detector.MetadataWriter.create_for_inference(\n", + " writer_utils.load_file(_TFLITE_MODEL_PATH), input_norm_mean=[127.5], \n", + " input_norm_std=[127.5], label_file_paths=[_TFLITE_LABEL_PATH])\n", + "writer_utils.save_file(writer.populate(), _TFLITE_MODEL_WITH_METADATA_PATH)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YFEAjRBdPCQb" + }, + "source": [ + "Optional: Print out the metadata added to the TFLite model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FT3-38PJsSOt" + }, + "outputs": [], + "source": [ + "from tflite_support import metadata\n", + "\n", + "displayer = metadata.MetadataDisplayer.with_model_file(_TFLITE_MODEL_WITH_METADATA_PATH)\n", + "print(\"Metadata populated:\")\n", + "print(displayer.get_metadata_json())\n", + "print(\"=============================\")\n", + "print(\"Associated file(s) populated:\")\n", + "print(displayer.get_packed_associated_file_list())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "l7zVslTRnEHX" + }, + "source": [ + "The TFLite model now can be integrated into a mobile app using the TFLite Task Library. See the [documentation](https://www.tensorflow.org/lite/inference_with_metadata/task_library/object_detector) for more details." + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "Convert TF Object Detection API model to TFLite.ipynb", + "private_outputs": true, + "provenance": [ + { + "file_id": "1R4_y-u14YTdvBzhmvC0HQwh3HkcCN2Bd", + "timestamp": 1623114733432 + }, + { + "file_id": "1Rey5kAzNQhJ77tsXGjhcAV0UZ6du0Sla", + "timestamp": 1622897882140 + } + ], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/research/object_detection/g3doc/running_on_mobile_tf2.md b/research/object_detection/g3doc/running_on_mobile_tf2.md index efa335c17b8..fa39bafc5c6 100644 --- a/research/object_detection/g3doc/running_on_mobile_tf2.md +++ b/research/object_detection/g3doc/running_on_mobile_tf2.md @@ -13,17 +13,22 @@ on-device machine learning inference with low latency and a small binary size. TensorFlow Lite uses many techniques for this such as quantized kernels that allow smaller and faster (fixed-point math) models. -This document shows how elgible models from the +This document shows how eligible models from the [TF2 Detection zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md) -can be converted for inference with TFLite. +can be converted for inference with TFLite. See this Colab tutorial for a +runnable tutorial that walks you through the steps explained in this document: + +Run +in Google Colab For an end-to-end Python guide on how to fine-tune an SSD model for mobile inference, look at [this Colab](../colab_tutorials/eager_few_shot_od_training_tflite.ipynb). **NOTE:** TFLite currently only supports **SSD Architectures** (excluding -EfficientDet) for boxes-based detection. Support for EfficientDet is coming -soon. +EfficientDet) for boxes-based detection. Support for EfficientDet is provided +via the [TFLite Model Maker](https://www.tensorflow.org/lite/tutorials/model_maker_object_detection) +library. The output model has the following inputs & outputs: @@ -87,9 +92,46 @@ converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8, converter.representative_dataset = <...> ``` +### Step 3: Add Metadata + +The model needs to be packed with +[TFLite Metadata](https://www.tensorflow.org/lite/convert/metadata) to enable +easy integration into mobile apps using the +[TFLite Task Library](https://www.tensorflow.org/lite/inference_with_metadata/task_library/object_detector). +This metadata helps the inference code perform the correct pre & post processing +as required by the model. Use the following code to create the metadata. + +```python +from tflite_support.metadata_writers import object_detector +from tflite_support.metadata_writers import writer_utils + +writer = object_detector.MetadataWriter.create_for_inference( + writer_utils.load_file(_TFLITE_MODEL_PATH), input_norm_mean=[0], + input_norm_std=[255], label_file_paths=[_TFLITE_LABEL_PATH]) +writer_utils.save_file(writer.populate(), _TFLITE_MODEL_WITH_METADATA_PATH) +``` + +See the TFLite Metadata Writer API [documentation](https://www.tensorflow.org/lite/convert/metadata_writer_tutorial#object_detectors) +for more details. + ## Running our model on Android -To run our TensorFlow Lite model on device, we will use Android Studio to build +### Integrate the model into your app +You can use the TFLite Task Library's [ObjectDetector API](https://www.tensorflow.org/lite/inference_with_metadata/task_library/object_detector) +to integrate the model into your Android app. + +```java +// Initialization +ObjectDetectorOptions options = ObjectDetectorOptions.builder().setMaxResults(1).build(); +ObjectDetector objectDetector = ObjectDetector.createFromFileAndOptions(context, modelFile, options); + +// Run inference +List results = objectDetector.detect(image); +``` + +### Test the model using the TFLite sample app + +To test our TensorFlow Lite model on device, we will use Android Studio to build and run the TensorFlow Lite detection example with the new model. The example is found in the [TensorFlow examples repository](https://github.com/tensorflow/examples) under @@ -102,7 +144,7 @@ that support API >= 21. Additional details are available on the Next we need to point the app to our new detect.tflite file and give it the names of our new labels. Specifically, we will copy our TensorFlow Lite -flatbuffer to the app assets directory with the following command: +model with metadata to the app assets directory with the following command: ```shell mkdir $TF_EXAMPLES/lite/examples/object_detection/android/app/src/main/assets @@ -110,9 +152,6 @@ cp /tmp/tflite/detect.tflite \ $TF_EXAMPLES/lite/examples/object_detection/android/app/src/main/assets ``` -You will also need to copy your new labelmap labelmap.txt to the assets -directory. - We will now edit the gradle build file to use these assets. First, open the `build.gradle` file `$TF_EXAMPLES/lite/examples/object_detection/android/app/build.gradle`. Comment @@ -122,23 +161,12 @@ out the model download script to avoid your assets being overwritten: // apply from:'download_model.gradle' ``` -If your model is named `detect.tflite`, and your labels file `labelmap.txt`, the -example will use them automatically as long as they've been properly copied into -the base assets directory. If you need to use a custom path or filename, open up -the +If your model is named `detect.tflite`, the example will use it automatically as +long as they've been properly copied into the base assets directory. If you need +to use a custom path or filename, open up the $TF_EXAMPLES/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/demo/DetectorActivity.java -file in a text editor and find the definition of TF_OD_API_LABELS_FILE. Update -this path to point to your new label map file: "labels_list.txt". Note that if -your model is quantized, the flag TF_OD_API_IS_QUANTIZED is set to true, and if -your model is floating point, the flag TF_OD_API_IS_QUANTIZED is set to false. -This new section of DetectorActivity.java should now look as follows for a -quantized model: - -```java - private static final boolean TF_OD_API_IS_QUANTIZED = true; - private static final String TF_OD_API_MODEL_FILE = "detect.tflite"; - private static final String TF_OD_API_LABELS_FILE = "labels_list.txt"; -``` +file in a text editor and find the definition of TF_OD_API_MODEL_FILE. Update +this path to point to your new model file. Once you’ve copied the TensorFlow Lite model and edited the gradle build script to not use the downloaded assets, you can build and deploy the app using the From fcd681d2fb48d3fc9a53a7046e8d07a046b7b547 Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Wed, 9 Jun 2021 00:24:47 -0700 Subject: [PATCH 37/50] Internal change PiperOrigin-RevId: 378339245 --- official/vision/beta/configs/common.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/official/vision/beta/configs/common.py b/official/vision/beta/configs/common.py index fb407db82f8..f1b72121117 100644 --- a/official/vision/beta/configs/common.py +++ b/official/vision/beta/configs/common.py @@ -69,7 +69,15 @@ class PseudoLabelDataConfig(cfg.DataConfig): """Psuedo Label input config for training.""" input_path: str = '' data_ratio: float = 1.0 # Per-batch ratio of pseudo-labeled to labeled data. + is_training: bool = True + dtype: str = 'float32' + shuffle_buffer_size: int = 10000 + cycle_length: int = 10 aug_rand_hflip: bool = True aug_type: Optional[ Augmentation] = None # Choose from AutoAugment and RandAugment. file_type: str = 'tfrecord' + + # Keep for backward compatibility. + aug_policy: Optional[str] = None # None, 'autoaug', or 'randaug'. + randaug_magnitude: Optional[int] = 10 From 7ff3ebcc21ad512c5e6660cc53763e3ab6cb0dce Mon Sep 17 00:00:00 2001 From: Daniel Ron Date: Wed, 9 Jun 2021 12:04:38 -0400 Subject: [PATCH 38/50] Fix attention application in DELG (#9906) * Fix attention application in DELG * Adding DELG unit tests * Formatting for review * Formatting for review * Formatting for review --- .../delf/python/training/model/delf_model.py | 18 ++- .../python/training/model/delg_model_test.py | 151 ++++++++++++++++++ 2 files changed, 164 insertions(+), 5 deletions(-) create mode 100644 research/delf/delf/python/training/model/delg_model_test.py diff --git a/research/delf/delf/python/training/model/delf_model.py b/research/delf/delf/python/training/model/delf_model.py index 5cdad73babb..9d770ba4fd1 100644 --- a/research/delf/delf/python/training/model/delf_model.py +++ b/research/delf/delf/python/training/model/delf_model.py @@ -35,6 +35,8 @@ class AttentionModel(tf.keras.Model): Uses two [kernel_size x kernel_size] convolutions and softplus as activation to compute an attention map with the same resolution as the featuremap. Features l2-normalized and aggregated using attention probabilites as weights. + The features (targets) to be aggregated can be the input featuremap, or a + different one with the same resolution. """ def __init__(self, kernel_size=1, decay=_DECAY, name='attention'): @@ -65,7 +67,7 @@ def __init__(self, kernel_size=1, decay=_DECAY, name='attention'): name='attn_conv2') self.activation_layer = layers.Activation('softplus') - def call(self, inputs, training=True): + def call(self, inputs, targets=None, training=True): x = self.conv1(inputs) x = self.bn_conv1(x, training=training) x = tf.nn.relu(x) @@ -73,9 +75,13 @@ def call(self, inputs, training=True): score = self.conv2(x) prob = self.activation_layer(score) + # Aggregate inputs if targets is None. + if targets is None: + targets = inputs + # L2-normalize the featuremap before pooling. - inputs = tf.nn.l2_normalize(inputs, axis=-1) - feat = tf.reduce_mean(tf.multiply(inputs, prob), [1, 2], keepdims=False) + targets = tf.nn.l2_normalize(targets, axis=-1) + feat = tf.reduce_mean(tf.multiply(targets, prob), [1, 2], keepdims=False) return feat, prob, score @@ -208,8 +214,10 @@ def global_and_local_forward_pass(self, images, training=True): block3 = tf.stop_gradient(block3) if self._use_dim_reduction: (dim_expanded_features, dim_reduced_features) = self.autoencoder(block3) - attn_prelogits, attn_scores, _ = self.attention(dim_expanded_features, - training=training) + attn_prelogits, attn_scores, _ = self.attention( + block3, + targets=dim_expanded_features, + training=training) else: attn_prelogits, attn_scores, _ = self.attention(block3, training=training) dim_expanded_features = None diff --git a/research/delf/delf/python/training/model/delg_model_test.py b/research/delf/delf/python/training/model/delg_model_test.py new file mode 100644 index 00000000000..6271ecec698 --- /dev/null +++ b/research/delf/delf/python/training/model/delg_model_test.py @@ -0,0 +1,151 @@ +# Lint as: python3 +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for the DELG model.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import parameterized +import tensorflow as tf + +from delf.python.training.model import delg_model + + +class DelgTest(tf.test.TestCase, parameterized.TestCase): + + @parameterized.named_parameters( + ('block3_stridesTrue', True), + ('block3_stridesFalse', False), + ) + def test_forward_pass(self, block3_strides): + image_size = 321 + num_classes = 1000 + batch_size = 2 + input_shape = (batch_size, image_size, image_size, 3) + local_feature_dim = 64 + feature_map_size = image_size // 16 # reduction factor for resnet50. + if block3_strides: + feature_map_size //= 2 + + model = delg_model.Delg(block3_strides=block3_strides, + use_dim_reduction=True, + reduced_dimension=local_feature_dim) + model.init_classifiers(num_classes) + + images = tf.random.uniform(input_shape, minval=-1.0, maxval=1.0, seed=0) + + # Run a complete forward pass of the model. + global_feature, attn_scores, local_features = model.build_call(images) + + self.assertAllEqual(global_feature.shape, (batch_size, 2048)) + self.assertAllEqual( + attn_scores.shape, + (batch_size, feature_map_size, feature_map_size, 1)) + self.assertAllEqual( + local_features.shape, + (batch_size, feature_map_size, feature_map_size, local_feature_dim)) + + @parameterized.named_parameters( + ('block3_stridesTrue', True), + ('block3_stridesFalse', False), + ) + def test_build_model(self, block3_strides): + image_size = 321 + num_classes = 1000 + batch_size = 2 + input_shape = (batch_size, image_size, image_size, 3) + + model = delg_model.Delg( + block3_strides=block3_strides, + use_dim_reduction=True) + model.init_classifiers(num_classes) + + images = tf.random.uniform(input_shape, minval=-1.0, maxval=1.0, seed=0) + labels = tf.random.uniform((batch_size,), + minval=0, + maxval=model.num_classes - 1, + dtype=tf.int64) + blocks = {} + + desc_prelogits = model.backbone( + images, intermediates_dict=blocks, training=False) + desc_logits = model.desc_classification(desc_prelogits, labels) + self.assertAllEqual(desc_prelogits.shape, (batch_size, 2048)) + self.assertAllEqual(desc_logits.shape, (batch_size, num_classes)) + + features = blocks['block3'] + attn_prelogits, _, _ = model.attention(features) + attn_logits = model.attn_classification(attn_prelogits) + self.assertAllEqual(attn_prelogits.shape, (batch_size, 1024)) + self.assertAllEqual(attn_logits.shape, (batch_size, num_classes)) + + @parameterized.named_parameters( + ('block3_stridesTrue', True), + ('block3_stridesFalse', False), + ) + def test_train_step(self, block3_strides): + image_size = 321 + num_classes = 1000 + batch_size = 2 + clip_val = 10.0 + input_shape = (batch_size, image_size, image_size, 3) + + model = delg_model.Delg( + block3_strides=block3_strides, + use_dim_reduction=True) + model.init_classifiers(num_classes) + + optimizer = tf.keras.optimizers.SGD(learning_rate=0.001, momentum=0.9) + + images = tf.random.uniform(input_shape, minval=0.0, maxval=1.0, seed=0) + labels = tf.random.uniform((batch_size,), + minval=0, + maxval=model.num_classes - 1, + dtype=tf.int64) + + loss_object = tf.keras.losses.SparseCategoricalCrossentropy( + from_logits=True, reduction=tf.keras.losses.Reduction.NONE) + + def compute_loss(labels, predictions): + per_example_loss = loss_object(labels, predictions) + return tf.nn.compute_average_loss( + per_example_loss, global_batch_size=batch_size) + + with tf.GradientTape() as gradient_tape: + (desc_prelogits, attn_prelogits, _, backbone_blocks, + dim_expanded_features, _) = model.global_and_local_forward_pass(images) + # Calculate global loss by applying the descriptor classifier. + desc_logits = model.desc_classification(desc_prelogits, labels) + desc_loss = compute_loss(labels, desc_logits) + # Calculate attention loss by applying the attention block classifier. + attn_logits = model.attn_classification(attn_prelogits) + attn_loss = compute_loss(labels, attn_logits) + # Calculate reconstruction loss between the attention prelogits and the + # backbone. + block3 = tf.stop_gradient(backbone_blocks['block3']) + reconstruction_loss = tf.math.reduce_mean( + tf.keras.losses.MSE(block3, dim_expanded_features)) + # Cumulate global loss and attention loss and backpropagate through the + # descriptor layer and attention layer together. + total_loss = desc_loss + attn_loss + reconstruction_loss + gradients = gradient_tape.gradient(total_loss, model.trainable_weights) + clipped, _ = tf.clip_by_global_norm(gradients, clip_norm=clip_val) + optimizer.apply_gradients(zip(clipped, model.trainable_weights)) + + +if __name__ == '__main__': + tf.test.main() From 520d7271a19302755ad38c44876b10da58177d3b Mon Sep 17 00:00:00 2001 From: Dan Kondratyuk Date: Wed, 9 Jun 2021 09:23:59 -0700 Subject: [PATCH 39/50] Internal change PiperOrigin-RevId: 378423112 --- .../projects/movinet/export_saved_model.py | 150 +++++++++--------- .../movinet/export_saved_model_test.py | 102 ++++++++++++ 2 files changed, 175 insertions(+), 77 deletions(-) create mode 100644 official/vision/beta/projects/movinet/export_saved_model_test.py diff --git a/official/vision/beta/projects/movinet/export_saved_model.py b/official/vision/beta/projects/movinet/export_saved_model.py index 37ce66b2904..25c466727fb 100644 --- a/official/vision/beta/projects/movinet/export_saved_model.py +++ b/official/vision/beta/projects/movinet/export_saved_model.py @@ -19,38 +19,18 @@ ```shell python3 export_saved_model.py \ - --output_path=/tmp/movinet/ \ + --export_path=/tmp/movinet/ \ --model_id=a0 \ --causal=True \ --conv_type="3d" \ --num_classes=600 \ + --use_positional_encoding=False \ --checkpoint_path="" ``` -To use an exported saved_model in various applications: - -```python -import tensorflow as tf -import tensorflow_hub as hub - -saved_model_path = ... - -inputs = tf.keras.layers.Input( - shape=[None, None, None, 3], - dtype=tf.float32) - -encoder = hub.KerasLayer(saved_model_path, trainable=True) -outputs = encoder(inputs) - -model = tf.keras.Model(inputs, outputs) - -example_input = tf.ones([1, 8, 172, 172, 3]) -outputs = model(example_input, states) -``` +To use an exported saved_model, refer to export_saved_model_test.py. """ -from typing import Sequence - from absl import app from absl import flags import tensorflow as tf @@ -59,8 +39,8 @@ from official.vision.beta.projects.movinet.modeling import movinet_model flags.DEFINE_string( - 'output_path', '/tmp/movinet/', - 'Path to saved exported saved_model file.') + 'export_path', '/tmp/movinet/', + 'Export path to save the saved_model file.') flags.DEFINE_string( 'model_id', 'a0', 'MoViNet model name.') flags.DEFINE_bool( @@ -73,8 +53,20 @@ '3x3 followed by 5x1 conv). 3d_2plus1d uses (2+1)D convolution with ' 'Conv3D and no 2D reshaping (e.g., a 5x3x3 kernel becomes 1x3x3 ' 'followed by 5x1x1 conv).') +flags.DEFINE_bool( + 'use_positional_encoding', False, + 'Whether to use positional encoding (only applied when causal=True).') flags.DEFINE_integer( 'num_classes', 600, 'The number of classes for prediction.') +flags.DEFINE_integer( + 'batch_size', None, + 'The batch size of the input. Set to None for dynamic input.') +flags.DEFINE_integer( + 'num_frames', None, + 'The number of frames of the input. Set to None for dynamic input.') +flags.DEFINE_integer( + 'image_size', None, + 'The resolution of the input. Set to None for dynamic input.') flags.DEFINE_string( 'checkpoint_path', '', 'Checkpoint path to load. Leave blank for default initialization.') @@ -82,75 +74,79 @@ FLAGS = flags.FLAGS -def main(argv: Sequence[str]) -> None: - if len(argv) > 1: - raise app.UsageError('Too many command-line arguments.') +def main(_) -> None: + input_specs = tf.keras.layers.InputSpec(shape=[ + FLAGS.batch_size, + FLAGS.num_frames, + FLAGS.image_size, + FLAGS.image_size, + 3, + ]) # Use dimensions of 1 except the channels to export faster, # since we only really need the last dimension to build and get the output # states. These dimensions will be set to `None` once the model is built. - input_shape = [1, 1, 1, 1, 3] + input_shape = [1 if s is None else s for s in input_specs.shape] backbone = movinet.Movinet( - FLAGS.model_id, causal=FLAGS.causal, conv_type=FLAGS.conv_type) + FLAGS.model_id, + causal=FLAGS.causal, + conv_type=FLAGS.conv_type, + use_external_states=FLAGS.causal, + input_specs=input_specs, + use_positional_encoding=FLAGS.use_positional_encoding) model = movinet_model.MovinetClassifier( - backbone, num_classes=FLAGS.num_classes, output_states=FLAGS.causal) + backbone, + num_classes=FLAGS.num_classes, + output_states=FLAGS.causal, + input_specs=dict(image=input_specs)) model.build(input_shape) + # Compile model to generate some internal Keras variables. + model.compile() + if FLAGS.checkpoint_path: - model.load_weights(FLAGS.checkpoint_path) + checkpoint = tf.train.Checkpoint(model=model) + status = checkpoint.restore(FLAGS.checkpoint_path) + status.assert_existing_objects_matched() if FLAGS.causal: # Call the model once to get the output states. Call again with `states` # input to ensure that the inputs with the `states` argument is built - _, states = model(dict(image=tf.ones(input_shape), states={})) - _, states = model(dict(image=tf.ones(input_shape), states=states)) - - input_spec = tf.TensorSpec( - shape=[None, None, None, None, 3], - dtype=tf.float32, - name='inputs') - - state_specs = {} - for name, state in states.items(): - shape = state.shape - if len(state.shape) == 5: - shape = [None, state.shape[1], None, None, state.shape[-1]] - new_spec = tf.TensorSpec(shape=shape, dtype=state.dtype, name=name) - state_specs[name] = new_spec - - specs = (input_spec, state_specs) - - # Define a tf.keras.Model with custom signatures to allow it to accept - # a state dict as an argument. We define it inline here because - # we first need to determine the shape of the state tensors before - # applying the `input_signature` argument to `tf.function`. - class ExportStateModule(tf.Module): - """Module with state for exporting to saved_model.""" - - def __init__(self, model): - self.model = model - - @tf.function(input_signature=[input_spec]) - def __call__(self, inputs): - return self.model(dict(image=inputs, states={})) - - @tf.function(input_signature=[input_spec]) - def base(self, inputs): - return self.model(dict(image=inputs, states={})) - - @tf.function(input_signature=specs) - def stream(self, inputs, states): - return self.model(dict(image=inputs, states=states)) - - module = ExportStateModule(model) - - tf.saved_model.save(module, FLAGS.output_path) + # with the full output state shapes. + input_image = tf.ones(input_shape) + _, states = model({**model.init_states(input_shape), 'image': input_image}) + _, states = model({**states, 'image': input_image}) + + # Create a function to explicitly set the names of the outputs + def predict(inputs): + outputs, states = model(inputs) + return {**states, 'logits': outputs} + + specs = { + name: tf.TensorSpec(spec.shape, name=name, dtype=spec.dtype) + for name, spec in model.initial_state_specs( + input_specs.shape).items() + } + specs['image'] = tf.TensorSpec( + input_specs.shape, dtype=model.dtype, name='image') + + predict_fn = tf.function(predict, jit_compile=True) + predict_fn = predict_fn.get_concrete_function(specs) + + init_states_fn = tf.function(model.init_states, jit_compile=True) + init_states_fn = init_states_fn.get_concrete_function( + tf.TensorSpec([5], dtype=tf.int32)) + + signatures = {'call': predict_fn, 'init_states': init_states_fn} + + tf.keras.models.save_model( + model, FLAGS.export_path, signatures=signatures) else: _ = model(tf.ones(input_shape)) - tf.keras.models.save_model(model, FLAGS.output_path) + tf.keras.models.save_model(model, FLAGS.export_path) - print(' ----- Done. Saved Model is saved at {}'.format(FLAGS.output_path)) + print(' ----- Done. Saved Model is saved at {}'.format(FLAGS.export_path)) if __name__ == '__main__': diff --git a/official/vision/beta/projects/movinet/export_saved_model_test.py b/official/vision/beta/projects/movinet/export_saved_model_test.py new file mode 100644 index 00000000000..0f364fb697f --- /dev/null +++ b/official/vision/beta/projects/movinet/export_saved_model_test.py @@ -0,0 +1,102 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for export_saved_model.""" + +from absl import flags +import tensorflow as tf +import tensorflow_hub as hub + +from official.vision.beta.projects.movinet import export_saved_model + +FLAGS = flags.FLAGS + + +class ExportSavedModelTest(tf.test.TestCase): + + def test_movinet_export_a0_base_with_tfhub(self): + saved_model_path = self.get_temp_dir() + + FLAGS.export_path = saved_model_path + FLAGS.model_id = 'a0' + FLAGS.causal = False + FLAGS.num_classes = 600 + + export_saved_model.main('unused_args') + + encoder = hub.KerasLayer(saved_model_path, trainable=True) + + inputs = tf.keras.layers.Input( + shape=[None, None, None, 3], + dtype=tf.float32) + + outputs = encoder(dict(image=inputs)) + + model = tf.keras.Model(inputs, outputs) + + example_input = tf.ones([1, 8, 172, 172, 3]) + outputs = model(example_input) + + self.assertEqual(outputs.shape, [1, 600]) + + def test_movinet_export_a0_stream_with_tfhub(self): + saved_model_path = self.get_temp_dir() + + FLAGS.export_path = saved_model_path + FLAGS.model_id = 'a0' + FLAGS.causal = True + FLAGS.num_classes = 600 + + export_saved_model.main('unused_args') + + encoder = hub.KerasLayer(saved_model_path, trainable=True) + + image_input = tf.keras.layers.Input( + shape=[None, None, None, 3], + dtype=tf.float32, + name='image') + + init_states_fn = encoder.resolved_object.signatures['init_states'] + state_shapes = { + name: ([s if s > 0 else None for s in state.shape], state.dtype) + for name, state in init_states_fn(tf.constant([0, 0, 0, 0, 3])).items() + } + states_input = { + name: tf.keras.Input(shape[1:], dtype=dtype, name=name) + for name, (shape, dtype) in state_shapes.items() + } + + inputs = {**states_input, 'image': image_input} + + outputs = encoder(inputs) + + model = tf.keras.Model(inputs, outputs) + + example_input = tf.ones([1, 8, 172, 172, 3]) + frames = tf.split(example_input, example_input.shape[1], axis=1) + + init_states = init_states_fn(tf.shape(example_input)) + + expected_outputs, _ = model({**init_states, 'image': example_input}) + + states = init_states + for frame in frames: + outputs, states = model({**states, 'image': frame}) + + self.assertEqual(outputs.shape, [1, 600]) + self.assertNotEmpty(states) + self.assertAllClose(outputs, expected_outputs, 1e-5, 1e-5) + +if __name__ == '__main__': + tf.test.main() From 927e31aa1de2d23fd62b7b2644b67b29d658b944 Mon Sep 17 00:00:00 2001 From: Arjun Karpur Date: Wed, 9 Jun 2021 17:35:06 -0500 Subject: [PATCH 40/50] Merged commit includes the following changes: (#10058) 378492389 by Andre Araujo: Internal change -- 369671840 by Andre Araujo: Cleanup after recent PRs in DELF github repo. -- 360692342 by Andre Araujo: Replace direct TF app import with absl::app. (1) TF::app is deprecated (2) direct TF import is discouraged. go/gpylint-faq#g-direct-tensorflow-import -- 359157762 by Andre Araujo: Internal change -- PiperOrigin-RevId: 378492389 Co-authored-by: Andre Araujo --- research/delf/delf/python/training/model/delg_model_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/research/delf/delf/python/training/model/delg_model_test.py b/research/delf/delf/python/training/model/delg_model_test.py index 6271ecec698..3ac2ec5ad24 100644 --- a/research/delf/delf/python/training/model/delg_model_test.py +++ b/research/delf/delf/python/training/model/delg_model_test.py @@ -37,7 +37,7 @@ def test_forward_pass(self, block3_strides): batch_size = 2 input_shape = (batch_size, image_size, image_size, 3) local_feature_dim = 64 - feature_map_size = image_size // 16 # reduction factor for resnet50. + feature_map_size = image_size // 16 # reduction factor for resnet50. if block3_strides: feature_map_size //= 2 From dcdd2e4015b1c2cccc8e1345f9dc3f42df65d667 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 10 Jun 2021 15:45:08 -0700 Subject: [PATCH 41/50] Make the labels in sentence_prediction DataLoader/Task a dict. PiperOrigin-RevId: 378751789 --- .../data/sentence_prediction_dataloader.py | 24 ++++++---- .../sentence_prediction_dataloader_test.py | 47 +++++++++++-------- official/nlp/tasks/sentence_prediction.py | 18 +++---- 3 files changed, 52 insertions(+), 37 deletions(-) diff --git a/official/nlp/data/sentence_prediction_dataloader.py b/official/nlp/data/sentence_prediction_dataloader.py index ddb5f8b8f94..45f60f0226c 100644 --- a/official/nlp/data/sentence_prediction_dataloader.py +++ b/official/nlp/data/sentence_prediction_dataloader.py @@ -14,7 +14,7 @@ """Loads dataset for the sentence prediction (classification) task.""" import functools -from typing import List, Mapping, Optional +from typing import List, Mapping, Optional, Tuple import dataclasses import tensorflow as tf @@ -40,7 +40,9 @@ class SentencePredictionDataConfig(cfg.DataConfig): label_type: str = 'int' # Whether to include the example id number. include_example_id: bool = False - outputs_as_dict: bool = False + # Maps the key in TfExample to feature name. + # E.g 'label_ids' to 'next_sentence_labels' + label_name: Optional[Tuple[str, str]] = None @data_loader_factory.register_data_loader_cls(SentencePredictionDataConfig) @@ -51,6 +53,10 @@ def __init__(self, params): self._params = params self._seq_length = params.seq_length self._include_example_id = params.include_example_id + if params.label_name: + self._label_name_mapping = dict([params.label_name]) + else: + self._label_name_mapping = dict() def _decode(self, record: tf.Tensor): """Decodes a serialized tf.Example.""" @@ -86,12 +92,12 @@ def _parse(self, record: Mapping[str, tf.Tensor]): if self._include_example_id: x['example_id'] = record['example_id'] - if self._params.outputs_as_dict: - x['next_sentence_labels'] = record['label_ids'] - return x + x['label_ids'] = record['label_ids'] + + if 'label_ids' in self._label_name_mapping: + x[self._label_name_mapping['label_ids']] = record['label_ids'] - y = record['label_ids'] - return (x, y) + return x def load(self, input_context: Optional[tf.distribute.InputContext] = None): """Returns a tf.dataset.Dataset.""" @@ -209,8 +215,8 @@ def _bert_preprocess(self, record: Mapping[str, tf.Tensor]): model_inputs = self._text_processor(segments) if self._include_example_id: model_inputs['example_id'] = record['example_id'] - y = record[self._label_field] - return model_inputs, y + model_inputs['label_ids'] = record[self._label_field] + return model_inputs def _decode(self, record: tf.Tensor): """Decodes a serialized tf.Example.""" diff --git a/official/nlp/data/sentence_prediction_dataloader_test.py b/official/nlp/data/sentence_prediction_dataloader_test.py index 6e3172a94b0..11a64e6b405 100644 --- a/official/nlp/data/sentence_prediction_dataloader_test.py +++ b/official/nlp/data/sentence_prediction_dataloader_test.py @@ -132,16 +132,17 @@ def test_load_dataset(self, label_type, expected_label_type): global_batch_size=batch_size, label_type=label_type) dataset = loader.SentencePredictionDataLoader(data_config).load() - features, labels = next(iter(dataset)) - self.assertCountEqual(['input_word_ids', 'input_mask', 'input_type_ids'], - features.keys()) + features = next(iter(dataset)) + self.assertCountEqual( + ['input_word_ids', 'input_type_ids', 'input_mask', 'label_ids'], + features.keys()) self.assertEqual(features['input_word_ids'].shape, (batch_size, seq_length)) self.assertEqual(features['input_mask'].shape, (batch_size, seq_length)) self.assertEqual(features['input_type_ids'].shape, (batch_size, seq_length)) - self.assertEqual(labels.shape, (batch_size,)) - self.assertEqual(labels.dtype, expected_label_type) + self.assertEqual(features['label_ids'].shape, (batch_size,)) + self.assertEqual(features['label_ids'].dtype, expected_label_type) - def test_load_dataset_as_dict(self): + def test_load_dataset_with_label_mapping(self): input_path = os.path.join(self.get_temp_dir(), 'train.tf_record') batch_size = 10 seq_length = 128 @@ -151,15 +152,18 @@ def test_load_dataset_as_dict(self): seq_length=seq_length, global_batch_size=batch_size, label_type='int', - outputs_as_dict=True) + label_name=('label_ids', 'next_sentence_labels')) dataset = loader.SentencePredictionDataLoader(data_config).load() features = next(iter(dataset)) self.assertCountEqual([ - 'input_word_ids', 'input_mask', 'input_type_ids', 'next_sentence_labels' + 'input_word_ids', 'input_mask', 'input_type_ids', + 'next_sentence_labels', 'label_ids' ], features.keys()) self.assertEqual(features['input_word_ids'].shape, (batch_size, seq_length)) self.assertEqual(features['input_mask'].shape, (batch_size, seq_length)) self.assertEqual(features['input_type_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['label_ids'].shape, (batch_size,)) + self.assertEqual(features['label_ids'].dtype, tf.int32) self.assertEqual(features['next_sentence_labels'].shape, (batch_size,)) self.assertEqual(features['next_sentence_labels'].dtype, tf.int32) @@ -192,13 +196,14 @@ def test_python_wordpiece_preprocessing(self, use_tfds): lower_case=lower_case, vocab_file=vocab_file_path) dataset = loader.SentencePredictionTextDataLoader(data_config).load() - features, labels = next(iter(dataset)) - self.assertCountEqual(['input_word_ids', 'input_type_ids', 'input_mask'], - features.keys()) + features = next(iter(dataset)) + self.assertCountEqual( + ['input_word_ids', 'input_type_ids', 'input_mask', 'label_ids'], + features.keys()) self.assertEqual(features['input_word_ids'].shape, (batch_size, seq_length)) self.assertEqual(features['input_mask'].shape, (batch_size, seq_length)) self.assertEqual(features['input_type_ids'].shape, (batch_size, seq_length)) - self.assertEqual(labels.shape, (batch_size,)) + self.assertEqual(features['label_ids'].shape, (batch_size,)) @parameterized.parameters(True, False) def test_python_sentencepiece_preprocessing(self, use_tfds): @@ -225,13 +230,14 @@ def test_python_sentencepiece_preprocessing(self, use_tfds): vocab_file=sp_model_file_path, ) dataset = loader.SentencePredictionTextDataLoader(data_config).load() - features, labels = next(iter(dataset)) - self.assertCountEqual(['input_word_ids', 'input_type_ids', 'input_mask'], - features.keys()) + features = next(iter(dataset)) + self.assertCountEqual( + ['input_word_ids', 'input_type_ids', 'input_mask', 'label_ids'], + features.keys()) self.assertEqual(features['input_word_ids'].shape, (batch_size, seq_length)) self.assertEqual(features['input_mask'].shape, (batch_size, seq_length)) self.assertEqual(features['input_type_ids'].shape, (batch_size, seq_length)) - self.assertEqual(labels.shape, (batch_size,)) + self.assertEqual(features['label_ids'].shape, (batch_size,)) @parameterized.parameters(True, False) def test_saved_model_preprocessing(self, use_tfds): @@ -258,13 +264,14 @@ def test_saved_model_preprocessing(self, use_tfds): label_type='int' if use_tfds else 'float', ) dataset = loader.SentencePredictionTextDataLoader(data_config).load() - features, labels = next(iter(dataset)) - self.assertCountEqual(['input_word_ids', 'input_type_ids', 'input_mask'], - features.keys()) + features = next(iter(dataset)) + self.assertCountEqual( + ['input_word_ids', 'input_type_ids', 'input_mask', 'label_ids'], + features.keys()) self.assertEqual(features['input_word_ids'].shape, (batch_size, seq_length)) self.assertEqual(features['input_mask'].shape, (batch_size, seq_length)) self.assertEqual(features['input_type_ids'].shape, (batch_size, seq_length)) - self.assertEqual(labels.shape, (batch_size,)) + self.assertEqual(features['label_ids'].shape, (batch_size,)) if __name__ == '__main__': diff --git a/official/nlp/tasks/sentence_prediction.py b/official/nlp/tasks/sentence_prediction.py index e79651e8e80..176173caccf 100644 --- a/official/nlp/tasks/sentence_prediction.py +++ b/official/nlp/tasks/sentence_prediction.py @@ -95,11 +95,12 @@ def build_model(self): use_encoder_pooler=self.task_config.model.use_encoder_pooler) def build_losses(self, labels, model_outputs, aux_losses=None) -> tf.Tensor: + label_ids = labels['label_ids'] if self.task_config.model.num_classes == 1: - loss = tf.keras.losses.mean_squared_error(labels, model_outputs) + loss = tf.keras.losses.mean_squared_error(label_ids, model_outputs) else: loss = tf.keras.losses.sparse_categorical_crossentropy( - labels, tf.cast(model_outputs, tf.float32), from_logits=True) + label_ids, tf.cast(model_outputs, tf.float32), from_logits=True) if aux_losses: loss += tf.add_n(aux_losses) @@ -120,7 +121,8 @@ def dummy_data(_): y = tf.zeros((1,), dtype=tf.float32) else: y = tf.zeros((1, 1), dtype=tf.int32) - return x, y + x['label_ids'] = y + return x dataset = tf.data.Dataset.range(1) dataset = dataset.repeat() @@ -142,7 +144,7 @@ def build_metrics(self, training=None): def process_metrics(self, metrics, labels, model_outputs): for metric in metrics: - metric.update_state(labels, model_outputs) + metric.update_state(labels['label_ids'], model_outputs) def process_compiled_metrics(self, compiled_metrics, labels, model_outputs): compiled_metrics.update_state(labels, model_outputs) @@ -151,7 +153,7 @@ def validation_step(self, inputs, model: tf.keras.Model, metrics=None): if self.metric_type == 'accuracy': return super(SentencePredictionTask, self).validation_step(inputs, model, metrics) - features, labels = inputs + features, labels = inputs, inputs outputs = self.inference_step(features, model) loss = self.build_losses( labels=labels, model_outputs=outputs, aux_losses=model.losses) @@ -161,12 +163,12 @@ def validation_step(self, inputs, model: tf.keras.Model, metrics=None): 'sentence_prediction': # Ensure one prediction along batch dimension. tf.expand_dims(tf.math.argmax(outputs, axis=1), axis=1), 'labels': - labels, + labels['label_ids'], }) if self.metric_type == 'pearson_spearman_corr': logs.update({ 'sentence_prediction': outputs, - 'labels': labels, + 'labels': labels['label_ids'], }) return logs @@ -250,7 +252,7 @@ def predict(task: SentencePredictionTask, def predict_step(inputs): """Replicated prediction calculation.""" - x, _ = inputs + x = inputs example_id = x.pop('example_id') outputs = task.inference_step(x, model) return dict(example_id=example_id, predictions=outputs) From 7af2ff16ac46f9821db1de74b057bdaab4f3b873 Mon Sep 17 00:00:00 2001 From: Vincent Dumoulin Date: Fri, 11 Jun 2021 07:37:27 -0700 Subject: [PATCH 42/50] Internal change PiperOrigin-RevId: 378869744 --- .../vision/beta/data/create_coco_tf_record.py | 37 ++++-- .../vision/beta/data/process_coco_few_shot.sh | 48 +++++++ .../data/process_coco_few_shot_json_files.py | 124 ++++++++++++++++++ 3 files changed, 199 insertions(+), 10 deletions(-) create mode 100644 official/vision/beta/data/process_coco_few_shot.sh create mode 100644 official/vision/beta/data/process_coco_few_shot_json_files.py diff --git a/official/vision/beta/data/create_coco_tf_record.py b/official/vision/beta/data/create_coco_tf_record.py index 27102446142..2e389f02a2c 100644 --- a/official/vision/beta/data/create_coco_tf_record.py +++ b/official/vision/beta/data/create_coco_tf_record.py @@ -46,7 +46,7 @@ flags.DEFINE_boolean( 'include_masks', False, 'Whether to include instance segmentations masks ' '(PNG encoded) in the result. default: False.') -flags.DEFINE_string('image_dir', '', 'Directory containing images.') +flags.DEFINE_multi_string('image_dir', '', 'Directory containing images.') flags.DEFINE_string( 'image_info_file', '', 'File containing image information. ' 'Tf Examples in the output files correspond to the image ' @@ -159,7 +159,7 @@ def encode_caption_annotations(caption_annotations): def create_tf_example(image, - image_dir, + image_dirs, bbox_annotations=None, id_to_name_map=None, caption_annotations=None, @@ -169,7 +169,7 @@ def create_tf_example(image, Args: image: dict with keys: [u'license', u'file_name', u'coco_url', u'height', u'width', u'date_captured', u'flickr_url', u'id'] - image_dir: directory containing the image files. + image_dirs: list of directories containing the image files. bbox_annotations: list of dicts with keys: [u'segmentation', u'area', u'iscrowd', u'image_id', u'bbox', u'category_id', u'id'] Notice that bounding box @@ -190,14 +190,31 @@ def create_tf_example(image, num_annotations_skipped: Number of (invalid) annotations that were ignored. Raises: - ValueError: if the image pointed to by data['filename'] is not a valid JPEG + ValueError: if the image pointed to by data['filename'] is not a valid JPEG, + does not exist, or is not unique across image directories. """ image_height = image['height'] image_width = image['width'] filename = image['file_name'] image_id = image['id'] - full_path = os.path.join(image_dir, filename) + if len(image_dirs) > 1: + full_paths = [os.path.join(image_dir, filename) for image_dir in image_dirs] + full_existing_paths = [p for p in full_paths if tf.io.gfile.exists(p)] + if not full_existing_paths: + raise ValueError( + '{} does not exist across image directories.'.format(filename)) + if len(full_existing_paths) > 1: + raise ValueError( + '{} is not unique across image directories'.format(filename)) + full_path, = full_existing_paths + # If there is only one image directory, it's not worth checking for existence, + # since trying to open the file will raise an informative error message if it + # does not exist. + else: + image_dir, = image_dirs + full_path = os.path.join(image_dir, filename) + with tf.io.gfile.GFile(full_path, 'rb') as fid: encoded_jpg = fid.read() @@ -276,7 +293,7 @@ def _load_images_info(images_info_file): return info_dict['images'] -def generate_annotations(images, image_dir, +def generate_annotations(images, image_dirs, img_to_obj_annotation=None, img_to_caption_annotation=None, id_to_name_map=None, include_masks=False): @@ -289,12 +306,12 @@ def generate_annotations(images, image_dir, caption_annotaion = (img_to_caption_annotation.get(image['id'], None) if img_to_caption_annotation else None) - yield (image, image_dir, object_annotation, id_to_name_map, + yield (image, image_dirs, object_annotation, id_to_name_map, caption_annotaion, include_masks) def _create_tf_record_from_coco_annotations(images_info_file, - image_dir, + image_dirs, output_path, num_shards, object_annotations_file=None, @@ -309,7 +326,7 @@ def _create_tf_record_from_coco_annotations(images_info_file, files Eg. 'image_info_test-dev2017.json', 'instance_annotations_train2017.json', 'caption_annotations_train2017.json', etc. - image_dir: Directory containing the image files. + image_dirs: List of directories containing the image files. output_path: Path to output tf.Record file. num_shards: Number of output files to create. object_annotations_file: JSON file containing bounding box annotations. @@ -333,7 +350,7 @@ def _create_tf_record_from_coco_annotations(images_info_file, _load_caption_annotations(caption_annotations_file)) coco_annotations_iter = generate_annotations( - images, image_dir, img_to_obj_annotation, img_to_caption_annotation, + images, image_dirs, img_to_obj_annotation, img_to_caption_annotation, id_to_name_map=id_to_name_map, include_masks=include_masks) num_skipped = tfrecord_lib.write_tf_record_dataset( diff --git a/official/vision/beta/data/process_coco_few_shot.sh b/official/vision/beta/data/process_coco_few_shot.sh new file mode 100644 index 00000000000..686a31df164 --- /dev/null +++ b/official/vision/beta/data/process_coco_few_shot.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Processes the COCO few-shot benchmark into TFRecord files. Requires `wget`. + +tmp_dir=$(mktemp -d -t coco-XXXXXXXXXX) +output_dir="/tmp/coco_few_shot" +while getopts "o:" o; do + case "${o}" in + o) output_dir=${OPTARG} ;; + *) echo "Usage: ${0} [-o ]" 1>&2; exit 1 ;; + esac +done + +cocosplit_url="dl.yf.io/fs-det/datasets/cocosplit" +wget --recursive --no-parent -q --show-progress --progress=bar:force:noscroll \ + -P "${tmp_dir}" -A "5k.json,*10shot*.json,*30shot*.json" \ + "http://${cocosplit_url}/" +mv "${tmp_dir}/${cocosplit_url}/"* "${tmp_dir}" +rm -rf "${tmp_dir}/${cocosplit_url}/" + +python process_coco_few_shot_json_files.py \ + --logtostderr --workdir="${tmp_dir}" + +for seed in {0..9}; do + for shots in 10 30; do + python create_coco_tf_record.py \ + --logtostderr \ + --image_dir=/namespace/vale-project/datasets/mscoco_raw/images/train2014 \ + --image_dir=/namespace/vale-project/datasets/mscoco_raw/images/val2014 \ + --image_info_file="${tmp_dir}/${shots}shot_seed${seed}.json" \ + --object_annotations_file="${tmp_dir}/${shots}shot_seed${seed}.json" \ + --caption_annotations_file="" \ + --output_file_prefix="${output_dir}/${shots}shot_seed${seed}" \ + --num_shards=4 + done +done + +python create_coco_tf_record.py \ + --logtostderr \ + --image_dir=/namespace/vale-project/datasets/mscoco_raw/images/train2014 \ + --image_dir=/namespace/vale-project/datasets/mscoco_raw/images/val2014 \ + --image_info_file="${tmp_dir}/datasplit/5k.json" \ + --object_annotations_file="${tmp_dir}/datasplit/5k.json" \ + --caption_annotations_file="" \ + --output_file_prefix="${output_dir}/5k" \ + --num_shards=10 + +rm -rf "${tmp_dir}" diff --git a/official/vision/beta/data/process_coco_few_shot_json_files.py b/official/vision/beta/data/process_coco_few_shot_json_files.py new file mode 100644 index 00000000000..7a04cdd2c02 --- /dev/null +++ b/official/vision/beta/data/process_coco_few_shot_json_files.py @@ -0,0 +1,124 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Processes the JSON files for COCO few-shot. + +We assume that `workdir` mirrors the contents of +http://dl.yf.io/fs-det/datasets/cocosplit/, which contains the official JSON +files for the few-shot COCO evaluation procedure that Wang et al. (2020)'s +"Frustratingly Simple Few-Shot Object Detection" paper uses. +""" + +import collections +import itertools +import json +import logging +import os + +from absl import app +from absl import flags + +import tensorflow as tf + +logger = tf.get_logger() +logger.setLevel(logging.INFO) + +flags.DEFINE_string('workdir', None, 'Working directory.') + +FLAGS = flags.FLAGS +CATEGORIES = ['airplane', 'apple', 'backpack', 'banana', 'baseball bat', + 'baseball glove', 'bear', 'bed', 'bench', 'bicycle', 'bird', + 'boat', 'book', 'bottle', 'bowl', 'broccoli', 'bus', 'cake', + 'car', 'carrot', 'cat', 'cell phone', 'chair', 'clock', 'couch', + 'cow', 'cup', 'dining table', 'dog', 'donut', 'elephant', + 'fire hydrant', 'fork', 'frisbee', 'giraffe', 'hair drier', + 'handbag', 'horse', 'hot dog', 'keyboard', 'kite', 'knife', + 'laptop', 'microwave', 'motorcycle', 'mouse', 'orange', 'oven', + 'parking meter', 'person', 'pizza', 'potted plant', + 'refrigerator', 'remote', 'sandwich', 'scissors', 'sheep', + 'sink', 'skateboard', 'skis', 'snowboard', 'spoon', 'sports ball', + 'stop sign', 'suitcase', 'surfboard', 'teddy bear', + 'tennis racket', 'tie', 'toaster', 'toilet', 'toothbrush', + 'traffic light', 'train', 'truck', 'tv', 'umbrella', 'vase', + 'wine glass', 'zebra'] +SEEDS = list(range(10)) +SHOTS = [10, 30] + +FILE_SUFFIXES = collections.defaultdict(list) +for _seed, _shots in itertools.product(SEEDS, SHOTS): + for _category in CATEGORIES: + FILE_SUFFIXES[(_seed, _shots)].append( + '{}full_box_{}shot_{}_trainval.json'.format( + # http://dl.yf.io/fs-det/datasets/cocosplit/ is organized like so: + # + # datasplit/ + # trainvalno5k.json + # 5k.json + # full_box_{1,2,3,5,10,30}shot_{category}_trainval.json + # seed{1-9}/ + # full_box_{1,2,3,5,10,30}shot_{category}_trainval.json + # + # This means that the JSON files for seed0 are located in the root + # directory rather than in a `seed?/` subdirectory, hence the + # conditional expression below. + '' if _seed == 0 else 'seed{}/'.format(_seed), + _shots, + _category)) + + +def main(unused_argv): + workdir = FLAGS.workdir + + for seed, shots in itertools.product(SEEDS, SHOTS): + # Retrieve all examples for a given seed and shots setting. + file_paths = [os.path.join(workdir, suffix) + for suffix in FILE_SUFFIXES[(seed, shots)]] + json_dicts = [] + for file_path in file_paths: + with tf.io.gfile.GFile(file_path, 'r') as f: + json_dicts.append(json.load(f)) + + # Make sure that all JSON files for a given seed and shots setting have the + # same metadata. We count on this to fuse them later on. + metadata_dicts = [{'info': d['info'], 'licenses': d['licenses'], + 'categories': d['categories']} for d in json_dicts] + if not all(d == metadata_dicts[0] for d in metadata_dicts[1:]): + raise RuntimeError( + 'JSON files for {} shots (seed {}) '.format(shots, seed) + + 'have different info, licences, or categories fields') + + # Retrieve images across all JSON files. + images = sum((d['images'] for d in json_dicts), []) + # Remove duplicate image entries. + images = list({image['id']: image for image in images}.values()) + + output_dict = { + 'info': json_dicts[0]['info'], + 'licenses': json_dicts[0]['licenses'], + 'categories': json_dicts[0]['categories'], + 'images': images, + 'annotations': sum((d['annotations'] for d in json_dicts), []) + } + + output_path = os.path.join(workdir, + '{}shot_seed{}.json'.format(shots, seed)) + with tf.io.gfile.GFile(output_path, 'w') as f: + json.dump(output_dict, f) + logger.info('Processed %d shots (seed %d) and saved to %s', + shots, seed, output_path) + + +if __name__ == '__main__': + flags.mark_flag_as_required('workdir') + app.run(main) From 0ef1cce3dc092ea55e60845e70d17a66aec3512b Mon Sep 17 00:00:00 2001 From: Hongkun Yu Date: Fri, 11 Jun 2021 10:27:42 -0700 Subject: [PATCH 43/50] Internal change PiperOrigin-RevId: 378899878 --- official/core/base_trainer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/official/core/base_trainer.py b/official/core/base_trainer.py index 61c74c5b65d..30340b4bbb0 100644 --- a/official/core/base_trainer.py +++ b/official/core/base_trainer.py @@ -246,10 +246,11 @@ def __init__( self._train_loss = tf.keras.metrics.Mean("training_loss", dtype=tf.float32) self._validation_loss = tf.keras.metrics.Mean( "validation_loss", dtype=tf.float32) + model_metrics = model.metrics if hasattr(model, "metrics") else [] self._train_metrics = self.task.build_metrics( - training=True) + self.model.metrics + training=True) + model_metrics self._validation_metrics = self.task.build_metrics( - training=False) + self.model.metrics + training=False) + model_metrics self.init_async() From a9d5da287f2d8ad25ab19aa1674f89b39d5a119d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 11 Jun 2021 11:17:16 -0700 Subject: [PATCH 44/50] Internal change PiperOrigin-RevId: 378911698 --- .../nlp/data/sentence_prediction_dataloader.py | 12 +++++++----- .../data/sentence_prediction_dataloader_test.py | 15 +++++++++------ official/nlp/tasks/sentence_prediction.py | 16 ++++++++++------ 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/official/nlp/data/sentence_prediction_dataloader.py b/official/nlp/data/sentence_prediction_dataloader.py index 45f60f0226c..3d7c38d765e 100644 --- a/official/nlp/data/sentence_prediction_dataloader.py +++ b/official/nlp/data/sentence_prediction_dataloader.py @@ -40,6 +40,7 @@ class SentencePredictionDataConfig(cfg.DataConfig): label_type: str = 'int' # Whether to include the example id number. include_example_id: bool = False + label_field: str = 'label_ids' # Maps the key in TfExample to feature name. # E.g 'label_ids' to 'next_sentence_labels' label_name: Optional[Tuple[str, str]] = None @@ -53,6 +54,7 @@ def __init__(self, params): self._params = params self._seq_length = params.seq_length self._include_example_id = params.include_example_id + self._label_field = params.label_field if params.label_name: self._label_name_mapping = dict([params.label_name]) else: @@ -65,7 +67,7 @@ def _decode(self, record: tf.Tensor): 'input_ids': tf.io.FixedLenFeature([self._seq_length], tf.int64), 'input_mask': tf.io.FixedLenFeature([self._seq_length], tf.int64), 'segment_ids': tf.io.FixedLenFeature([self._seq_length], tf.int64), - 'label_ids': tf.io.FixedLenFeature([], label_type), + self._label_field: tf.io.FixedLenFeature([], label_type), } if self._include_example_id: name_to_features['example_id'] = tf.io.FixedLenFeature([], tf.int64) @@ -92,10 +94,10 @@ def _parse(self, record: Mapping[str, tf.Tensor]): if self._include_example_id: x['example_id'] = record['example_id'] - x['label_ids'] = record['label_ids'] + x[self._label_field] = record[self._label_field] - if 'label_ids' in self._label_name_mapping: - x[self._label_name_mapping['label_ids']] = record['label_ids'] + if self._label_field in self._label_name_mapping: + x[self._label_name_mapping[self._label_field]] = record[self._label_field] return x @@ -215,7 +217,7 @@ def _bert_preprocess(self, record: Mapping[str, tf.Tensor]): model_inputs = self._text_processor(segments) if self._include_example_id: model_inputs['example_id'] = record['example_id'] - model_inputs['label_ids'] = record[self._label_field] + model_inputs[self._label_field] = record[self._label_field] return model_inputs def _decode(self, record: tf.Tensor): diff --git a/official/nlp/data/sentence_prediction_dataloader_test.py b/official/nlp/data/sentence_prediction_dataloader_test.py index 11a64e6b405..85b1531716f 100644 --- a/official/nlp/data/sentence_prediction_dataloader_test.py +++ b/official/nlp/data/sentence_prediction_dataloader_test.py @@ -197,13 +197,14 @@ def test_python_wordpiece_preprocessing(self, use_tfds): vocab_file=vocab_file_path) dataset = loader.SentencePredictionTextDataLoader(data_config).load() features = next(iter(dataset)) + label_field = data_config.label_field self.assertCountEqual( - ['input_word_ids', 'input_type_ids', 'input_mask', 'label_ids'], + ['input_word_ids', 'input_type_ids', 'input_mask', label_field], features.keys()) self.assertEqual(features['input_word_ids'].shape, (batch_size, seq_length)) self.assertEqual(features['input_mask'].shape, (batch_size, seq_length)) self.assertEqual(features['input_type_ids'].shape, (batch_size, seq_length)) - self.assertEqual(features['label_ids'].shape, (batch_size,)) + self.assertEqual(features[label_field].shape, (batch_size,)) @parameterized.parameters(True, False) def test_python_sentencepiece_preprocessing(self, use_tfds): @@ -231,13 +232,14 @@ def test_python_sentencepiece_preprocessing(self, use_tfds): ) dataset = loader.SentencePredictionTextDataLoader(data_config).load() features = next(iter(dataset)) + label_field = data_config.label_field self.assertCountEqual( - ['input_word_ids', 'input_type_ids', 'input_mask', 'label_ids'], + ['input_word_ids', 'input_type_ids', 'input_mask', label_field], features.keys()) self.assertEqual(features['input_word_ids'].shape, (batch_size, seq_length)) self.assertEqual(features['input_mask'].shape, (batch_size, seq_length)) self.assertEqual(features['input_type_ids'].shape, (batch_size, seq_length)) - self.assertEqual(features['label_ids'].shape, (batch_size,)) + self.assertEqual(features[label_field].shape, (batch_size,)) @parameterized.parameters(True, False) def test_saved_model_preprocessing(self, use_tfds): @@ -265,13 +267,14 @@ def test_saved_model_preprocessing(self, use_tfds): ) dataset = loader.SentencePredictionTextDataLoader(data_config).load() features = next(iter(dataset)) + label_field = data_config.label_field self.assertCountEqual( - ['input_word_ids', 'input_type_ids', 'input_mask', 'label_ids'], + ['input_word_ids', 'input_type_ids', 'input_mask', label_field], features.keys()) self.assertEqual(features['input_word_ids'].shape, (batch_size, seq_length)) self.assertEqual(features['input_mask'].shape, (batch_size, seq_length)) self.assertEqual(features['input_type_ids'].shape, (batch_size, seq_length)) - self.assertEqual(features['label_ids'].shape, (batch_size,)) + self.assertEqual(features[label_field].shape, (batch_size,)) if __name__ == '__main__': diff --git a/official/nlp/tasks/sentence_prediction.py b/official/nlp/tasks/sentence_prediction.py index 176173caccf..2f6b80361a8 100644 --- a/official/nlp/tasks/sentence_prediction.py +++ b/official/nlp/tasks/sentence_prediction.py @@ -69,6 +69,10 @@ def __init__(self, params: cfg.TaskConfig, logging_dir=None, name=None): if params.metric_type not in METRIC_TYPES: raise ValueError('Invalid metric_type: {}'.format(params.metric_type)) self.metric_type = params.metric_type + if hasattr(params.train_data, 'label_field'): + self.label_field = params.train_data.label_field + else: + self.label_field = 'label_ids' def build_model(self): if self.task_config.hub_module_url and self.task_config.init_checkpoint: @@ -95,7 +99,7 @@ def build_model(self): use_encoder_pooler=self.task_config.model.use_encoder_pooler) def build_losses(self, labels, model_outputs, aux_losses=None) -> tf.Tensor: - label_ids = labels['label_ids'] + label_ids = labels[self.label_field] if self.task_config.model.num_classes == 1: loss = tf.keras.losses.mean_squared_error(label_ids, model_outputs) else: @@ -121,7 +125,7 @@ def dummy_data(_): y = tf.zeros((1,), dtype=tf.float32) else: y = tf.zeros((1, 1), dtype=tf.int32) - x['label_ids'] = y + x[self.label_field] = y return x dataset = tf.data.Dataset.range(1) @@ -144,10 +148,10 @@ def build_metrics(self, training=None): def process_metrics(self, metrics, labels, model_outputs): for metric in metrics: - metric.update_state(labels['label_ids'], model_outputs) + metric.update_state(labels[self.label_field], model_outputs) def process_compiled_metrics(self, compiled_metrics, labels, model_outputs): - compiled_metrics.update_state(labels, model_outputs) + compiled_metrics.update_state(labels[self.label_field], model_outputs) def validation_step(self, inputs, model: tf.keras.Model, metrics=None): if self.metric_type == 'accuracy': @@ -163,12 +167,12 @@ def validation_step(self, inputs, model: tf.keras.Model, metrics=None): 'sentence_prediction': # Ensure one prediction along batch dimension. tf.expand_dims(tf.math.argmax(outputs, axis=1), axis=1), 'labels': - labels['label_ids'], + labels[self.label_field], }) if self.metric_type == 'pearson_spearman_corr': logs.update({ 'sentence_prediction': outputs, - 'labels': labels['label_ids'], + 'labels': labels[self.label_field], }) return logs From 9d1fe06934243c6cafc0995d66fed96b57af3532 Mon Sep 17 00:00:00 2001 From: Dan Kondratyuk Date: Fri, 11 Jun 2021 12:17:57 -0700 Subject: [PATCH 45/50] Apply stream buffer after the spatial convolution in (2+1)D mode. PiperOrigin-RevId: 378923791 --- .../beta/projects/movinet/modeling/movinet.py | 13 +++++++-- .../movinet/modeling/movinet_layers.py | 23 +++++++++++++-- .../movinet/modeling/movinet_model.py | 28 +++++++++++++++---- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/official/vision/beta/projects/movinet/modeling/movinet.py b/official/vision/beta/projects/movinet/modeling/movinet.py index ddde7c1a416..131cb3455f9 100644 --- a/official/vision/beta/projects/movinet/modeling/movinet.py +++ b/official/vision/beta/projects/movinet/modeling/movinet.py @@ -525,7 +525,6 @@ def _get_initial_state_shapes( Returns: A dict mapping state names to state shapes. """ - def divide_resolution(shape, num_downsamples): """Downsamples the dimension to calculate strided convolution shape.""" if shape is None: @@ -564,6 +563,12 @@ def divide_resolution(shape, num_downsamples): for layer_idx, layer in enumerate(params): expand_filters, kernel_size, strides = layer + # If we use a 2D kernel, we apply spatial downsampling + # before the buffer. + if (tuple(strides[1:3]) != (1, 1) and + self._conv_type in ['2plus1d', '3d_2plus1d']): + num_downsamples += 1 + if kernel_size[0] > 1: states[f'state/b{block_idx}/l{layer_idx}/stream_buffer'] = ( input_shape[0], @@ -585,7 +590,11 @@ def divide_resolution(shape, num_downsamples): if strides[1] != strides[2]: raise ValueError('Strides must match in the spatial dimensions, ' 'got {}'.format(strides)) - if strides[1] != 1 or strides[2] != 1: + + # If we use a 3D kernel, we apply spatial downsampling + # after the buffer. + if (tuple(strides[1:3]) != (1, 1) and + self._conv_type not in ['2plus1d', '3d_2plus1d']): num_downsamples += 1 elif isinstance(block, HeadSpec): states['state/head/pool_buffer'] = ( diff --git a/official/vision/beta/projects/movinet/modeling/movinet_layers.py b/official/vision/beta/projects/movinet/modeling/movinet_layers.py index acde9bff559..369655ce290 100644 --- a/official/vision/beta/projects/movinet/modeling/movinet_layers.py +++ b/official/vision/beta/projects/movinet/modeling/movinet_layers.py @@ -633,9 +633,28 @@ def call(self, states = dict(states) if states is not None else {} x = inputs - if self._stream_buffer is not None: + + # If we have no separate temporal conv, use the buffer before the 3D conv. + if self._conv_temporal is None and self._stream_buffer is not None: x, states = self._stream_buffer(x, states=states) - x = super(StreamConvBlock, self).call(x) + + x = self._conv(x) + if self._batch_norm is not None: + x = self._batch_norm(x) + if self._activation_layer is not None: + x = self._activation_layer(x) + + if self._conv_temporal is not None: + if self._stream_buffer is not None: + # If we have a separate temporal conv, use the buffer before the + # 1D conv instead (otherwise, we may waste computation on the 2D conv). + x, states = self._stream_buffer(x, states=states) + + x = self._conv_temporal(x) + if self._batch_norm_temporal is not None: + x = self._batch_norm_temporal(x) + if self._activation_layer is not None: + x = self._activation_layer(x) return x, states diff --git a/official/vision/beta/projects/movinet/modeling/movinet_model.py b/official/vision/beta/projects/movinet/modeling/movinet_model.py index f95b690e8f7..a1970e67b5a 100644 --- a/official/vision/beta/projects/movinet/modeling/movinet_model.py +++ b/official/vision/beta/projects/movinet/modeling/movinet_model.py @@ -115,15 +115,31 @@ def _build_network( inputs = {**states, 'image': image} if backbone.use_external_states: - before_states = set(states) + before_states = states endpoints, states = backbone(inputs) - after_states = set(states) + after_states = states - new_states = after_states - before_states + new_states = set(after_states) - set(before_states) if new_states: - raise AttributeError('Expected input and output states to be the same. ' - 'Got extra states {}, expected {}'.format( - new_states, before_states)) + raise ValueError( + 'Expected input and output states to be the same. Got extra states ' + '{}, expected {}'.format(new_states, set(before_states))) + + mismatched_shapes = {} + for name in after_states: + before_shape = before_states[name].shape + after_shape = after_states[name].shape + if len(before_shape) != len(after_shape): + mismatched_shapes[name] = (before_shape, after_shape) + continue + for before, after in zip(before_shape, after_shape): + if before is not None and after is not None and before != after: + mismatched_shapes[name] = (before_shape, after_shape) + break + if mismatched_shapes: + raise ValueError( + 'Got mismatched input and output state shapes: {}'.format( + mismatched_shapes)) else: endpoints, states = backbone(inputs) From 86e0e0f93ec3f200da175e461b846b51109dc222 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 11 Jun 2021 17:05:42 -0700 Subject: [PATCH 46/50] Remove num anchors assertion from AnchorGenerator PiperOrigin-RevId: 378977617 --- .../object_detection/core/anchor_generator.py | 32 ++----------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/research/object_detection/core/anchor_generator.py b/research/object_detection/core/anchor_generator.py index 69e29d84db8..e896550a7e9 100644 --- a/research/object_detection/core/anchor_generator.py +++ b/research/object_detection/core/anchor_generator.py @@ -37,7 +37,6 @@ from abc import abstractmethod import six -from six.moves import zip import tensorflow.compat.v1 as tf @@ -107,11 +106,9 @@ def generate(self, feature_map_shape_list, **params): with tf.name_scope(self.name_scope()): anchors_list = self._generate(feature_map_shape_list, **params) if self.check_num_anchors: - with tf.control_dependencies([ - self._assert_correct_number_of_anchors( - anchors_list, feature_map_shape_list)]): - for item in anchors_list: - item.set(tf.identity(item.get())) + for item in anchors_list: + item.set(tf.identity(item.get())) + return anchors_list @abstractmethod @@ -146,26 +143,3 @@ def anchor_index_to_feature_map_index(self, boxlist_list): feature_map_indices_list.append( i * tf.ones([boxes.num_boxes()], dtype=tf.int32)) return tf.concat(feature_map_indices_list, axis=0) - - def _assert_correct_number_of_anchors(self, anchors_list, - feature_map_shape_list): - """Assert that correct number of anchors was generated. - - Args: - anchors_list: A list of box_list.BoxList object holding anchors generated. - feature_map_shape_list: list of (height, width) pairs in the format - [(height_0, width_0), (height_1, width_1), ...] that the generated - anchors must align with. - Returns: - Op that raises InvalidArgumentError if the number of anchors does not - match the number of expected anchors. - """ - expected_num_anchors = 0 - actual_num_anchors = 0 - for num_anchors_per_location, feature_map_shape, anchors in zip( - self.num_anchors_per_location(), feature_map_shape_list, anchors_list): - expected_num_anchors += (num_anchors_per_location - * feature_map_shape[0] - * feature_map_shape[1]) - actual_num_anchors += anchors.num_boxes() - return tf.assert_equal(expected_num_anchors, actual_num_anchors) From 546a33b022d710ec22c53c3957481526341208e0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sat, 12 Jun 2021 22:26:23 -0700 Subject: [PATCH 47/50] Internal change PiperOrigin-RevId: 379108917 --- official/nlp/data/classifier_data_lib.py | 34 +++++++++++------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/official/nlp/data/classifier_data_lib.py b/official/nlp/data/classifier_data_lib.py index 2498c327094..e2b46aa043b 100644 --- a/official/nlp/data/classifier_data_lib.py +++ b/official/nlp/data/classifier_data_lib.py @@ -181,20 +181,21 @@ def _create_examples(self, lines, set_type): class ColaProcessor(DataProcessor): """Processor for the CoLA data set (GLUE version).""" + def __init__(self, process_text_fn=tokenization.convert_to_unicode): + super(ColaProcessor, self).__init__(process_text_fn) + self.dataset = tfds.load("glue/cola", try_gcs=True) + def get_train_examples(self, data_dir): """See base class.""" - return self._create_examples( - self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + return self._create_examples_tfds("train") def get_dev_examples(self, data_dir): """See base class.""" - return self._create_examples( - self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + return self._create_examples_tfds("validation") def get_test_examples(self, data_dir): """See base class.""" - return self._create_examples( - self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + return self._create_examples_tfds("test") def get_labels(self): """See base class.""" @@ -205,22 +206,19 @@ def get_processor_name(): """See base class.""" return "COLA" - def _create_examples(self, lines, set_type): + def _create_examples_tfds(self, set_type): """Creates examples for the training/dev/test sets.""" + dataset = self.dataset[set_type].as_numpy_iterator() examples = [] - for i, line in enumerate(lines): - # Only the test set has a header. - if set_type == "test" and i == 0: - continue + for i, example in enumerate(dataset): guid = "%s-%s" % (set_type, i) - if set_type == "test": - text_a = self.process_text_fn(line[1]) - label = "0" - else: - text_a = self.process_text_fn(line[3]) - label = self.process_text_fn(line[1]) + label = "0" + text_a = self.process_text_fn(example["sentence"]) + if set_type != "test": + label = str(example["label"]) examples.append( - InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + InputExample( + guid=guid, text_a=text_a, text_b=None, label=label, weight=None)) return examples From decf72c8a8ecdd77b1da098ca34657eced11901c Mon Sep 17 00:00:00 2001 From: Yeqing Li Date: Mon, 14 Jun 2021 13:13:59 -0700 Subject: [PATCH 48/50] Adds the Model Garden porjects/ folder. PiperOrigin-RevId: 379338309 --- official/projects/README.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 official/projects/README.md diff --git a/official/projects/README.md b/official/projects/README.md new file mode 100644 index 00000000000..743baae887f --- /dev/null +++ b/official/projects/README.md @@ -0,0 +1,2 @@ +This directory contains projects using TensorFlow Model Garden Modeling +libraries. From 045f34b232830e50a7990a5ba3814d8de5b9cde3 Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Mon, 14 Jun 2021 14:41:51 -0700 Subject: [PATCH 49/50] Switch TF to OSS keras (1/N). 1. Make api_template always use third_party/py/keras (or keras PIP package in OSS). 2. API tests are updated to ignore tensorflow/python/keras end points. 3. All Keras-related code will be removed in a follow up change. Keras-specific APIs are tracked in google3/third_party/py/keras/api/golden/ 4. There are a few APIs (optimizer/initializer) that are exposed as both tf and keras APIs. Their pbtxt are updated to reflect the latest class path change. Once this change is submitted, the current code in tensorflow/python/keras will become a stale copy, and will be removed soon. No more changes should be made to tensorflow/python/keras. PiperOrigin-RevId: 379353790 --- research/object_detection/models/keras_models/resnet_v1.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/research/object_detection/models/keras_models/resnet_v1.py b/research/object_detection/models/keras_models/resnet_v1.py index 62660d4a70d..f57b8bd3468 100644 --- a/research/object_detection/models/keras_models/resnet_v1.py +++ b/research/object_detection/models/keras_models/resnet_v1.py @@ -19,9 +19,10 @@ from __future__ import division from __future__ import print_function +from keras.applications import resnet + import tensorflow.compat.v1 as tf -from tensorflow.python.keras.applications import resnet from object_detection.core import freezable_batch_norm from object_detection.models.keras_models import model_utils From bcbce005922c44efd3c5bda5e8c6e811f0fd419e Mon Sep 17 00:00:00 2001 From: Jaeyoun Kim Date: Mon, 14 Jun 2021 16:32:06 -0700 Subject: [PATCH 50/50] Copybara import of the project: -- 63719f08ae3073dede98d0722a096f77891aa965 by Anirudh Vegesana : YOLO Family: Updated model (#9923) * Update YOLO model * Fix some docstrings * Fix docstrings * Address some of Dr. Davis' changes * Give descriptive names to the test cases * Fix bugs * Fix YOLO head imports * docstring and variable name updates * docstring and variable name updates * docstring and variable name updates Co-authored-by: vishnubanna Co-authored-by: Vishnu Banna <43182884+vishnubanna@users.noreply.github.com> -- 725b8c8c85d36fbee58f0025364f4e1acdcb0c94 by Anirudh Vegesana : disclaimer (#10020) Co-authored-by: Vishnu Banna <43182884+vishnubanna@users.noreply.github.com> -- 404d24b01c68d2ecdf93347814fb4da3ee636f47 by Anirudh Vegesana : YOLO Family: Linting (#10027) * YOLO Family: Updated model (#9923) * Update YOLO model * Fix some docstrings * Fix docstrings * Address some of Dr. Davis' changes * Give descriptive names to the test cases * Fix bugs * Fix YOLO head imports * docstring and variable name updates * docstring and variable name updates * docstring and variable name updates Co-authored-by: vishnubanna Co-authored-by: Vishnu Banna <43182884+vishnubanna@users.noreply.github.com> * disclaimer * Fix some PyLint errors Co-authored-by: vishnubanna Co-authored-by: Vishnu Banna <43182884+vishnubanna@users.noreply.github.com> COPYBARA_INTEGRATE_REVIEW=https://github.com/tensorflow/models/pull/10021 from tensorflow:purdue-yolo 404d24b01c68d2ecdf93347814fb4da3ee636f47 PiperOrigin-RevId: 379372162 --- official/vision/beta/projects/yolo/README.md | 5 + .../beta/projects/yolo/configs/backbones.py | 11 +- .../yolo/configs/darknet_classification.py | 2 +- .../yolo/modeling/backbones/darknet.py | 606 +++++-- .../yolo/modeling/backbones/darknet_test.py | 69 +- .../yolo/modeling/decoders/__init__.py | 14 + .../yolo/modeling/decoders/yolo_decoder.py | 478 +++++ .../modeling/decoders/yolo_decoder_test.py | 153 ++ .../projects/yolo/modeling/heads/__init__.py | 14 + .../projects/yolo/modeling/heads/yolo_head.py | 122 ++ .../yolo/modeling/heads/yolo_head_test.py | 74 + .../yolo/modeling/layers/nn_blocks.py | 1543 +++++++++++++---- .../yolo/modeling/layers/nn_blocks_test.py | 197 ++- 13 files changed, 2659 insertions(+), 629 deletions(-) create mode 100644 official/vision/beta/projects/yolo/modeling/decoders/__init__.py create mode 100644 official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py create mode 100644 official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder_test.py create mode 100644 official/vision/beta/projects/yolo/modeling/heads/__init__.py create mode 100644 official/vision/beta/projects/yolo/modeling/heads/yolo_head.py create mode 100644 official/vision/beta/projects/yolo/modeling/heads/yolo_head_test.py diff --git a/official/vision/beta/projects/yolo/README.md b/official/vision/beta/projects/yolo/README.md index 0a1e27fbe90..5cd4d1f2e59 100644 --- a/official/vision/beta/projects/yolo/README.md +++ b/official/vision/beta/projects/yolo/README.md @@ -1,3 +1,6 @@ +DISCLAIMER: this YOLO implementation is still under development. No support will +be provided during the development phase. + # YOLO Object Detectors, You Only Look Once [![Paper](http://img.shields.io/badge/Paper-arXiv.1804.02767-B3181B?logo=arXiv)](https://arxiv.org/abs/1804.02767) @@ -74,3 +77,5 @@ head could be connected to a new, more powerful backbone if a person chose to. [![TensorFlow 2.2](https://img.shields.io/badge/TensorFlow-2.2-FF6F00?logo=tensorflow)](https://github.com/tensorflow/tensorflow/releases/tag/v2.2.0) [![Python 3.8](https://img.shields.io/badge/Python-3.8-3776AB)](https://www.python.org/downloads/release/python-380/) + + diff --git a/official/vision/beta/projects/yolo/configs/backbones.py b/official/vision/beta/projects/yolo/configs/backbones.py index a79cb09e17e..46e378317c3 100644 --- a/official/vision/beta/projects/yolo/configs/backbones.py +++ b/official/vision/beta/projects/yolo/configs/backbones.py @@ -24,11 +24,14 @@ @dataclasses.dataclass -class DarkNet(hyperparams.Config): - """DarkNet config.""" - model_id: str = "darknet53" +class Darknet(hyperparams.Config): + """Darknet config.""" + model_id: str = 'darknet53' + width_scale: float = 1.0 + depth_scale: float = 1.0 + dilate: bool = False @dataclasses.dataclass class Backbone(backbones.Backbone): - darknet: DarkNet = DarkNet() + darknet: Darknet = Darknet() diff --git a/official/vision/beta/projects/yolo/configs/darknet_classification.py b/official/vision/beta/projects/yolo/configs/darknet_classification.py index b33e149d484..ffaf387fac0 100644 --- a/official/vision/beta/projects/yolo/configs/darknet_classification.py +++ b/official/vision/beta/projects/yolo/configs/darknet_classification.py @@ -32,7 +32,7 @@ class ImageClassificationModel(hyperparams.Config): num_classes: int = 0 input_size: List[int] = dataclasses.field(default_factory=list) backbone: backbones.Backbone = backbones.Backbone( - type='darknet', resnet=backbones.DarkNet()) + type='darknet', darknet=backbones.Darknet()) dropout_rate: float = 0.0 norm_activation: common.NormActivation = common.NormActivation() # Adds a BatchNormalization layer pre-GlobalAveragePooling in classification diff --git a/official/vision/beta/projects/yolo/modeling/backbones/darknet.py b/official/vision/beta/projects/yolo/modeling/backbones/darknet.py index 170c6bb7680..783b46b8c57 100644 --- a/official/vision/beta/projects/yolo/modeling/backbones/darknet.py +++ b/official/vision/beta/projects/yolo/modeling/backbones/darknet.py @@ -13,7 +13,6 @@ # limitations under the License. # Lint as: python3 - """Contains definitions of Darknet Backbone Networks. The models are inspired by ResNet, and CSPNet @@ -29,15 +28,15 @@ arXiv:1911.11929 -DarkNets Are used mainly for Object detection in: +Darknets are used mainly for object detection in: [1] Joseph Redmon, Ali Farhadi YOLOv3: An Incremental Improvement. arXiv:1804.02767 [2] Alexey Bochkovskiy, Chien-Yao Wang, Hong-Yuan Mark Liao YOLOv4: Optimal Speed and Accuracy of Object Detection. arXiv:2004.10934 """ -import collections +import collections import tensorflow as tf from official.modeling import hyperparams @@ -45,28 +44,32 @@ from official.vision.beta.projects.yolo.modeling.layers import nn_blocks -class BlockConfig(object): - """Get layer config to make code more readable. - - Args: - layer: string layer name - stack: the type of layer ordering to use for this specific level - repetitions: integer for the number of times to repeat block - bottelneck: boolean for does this stack have a bottle neck layer - filters: integer for the output depth of the level - pool_size: integer the pool_size of max pool layers - kernel_size: optional integer, for convolution kernel size - strides: integer or tuple to indicate convolution strides - padding: the padding to apply to layers in this stack - activation: string for the activation to use for this stack - route: integer for what level to route from to get the next input - output_name: the name to use for this output - is_output: is this layer an output in the default model - """ +class BlockConfig: + """Class to store layer config to make code more readable.""" def __init__(self, layer, stack, reps, bottleneck, filters, pool_size, - kernel_size, strides, padding, activation, route, output_name, - is_output): + kernel_size, strides, padding, activation, route, dilation_rate, + output_name, is_output): + """Initializing method for BlockConfig. + + Args: + layer: A `str` for layer name. + stack: A `str` for the type of layer ordering to use for this specific + level. + reps: An `int` for the number of times to repeat block. + bottleneck: A `bool` for whether this stack has a bottle neck layer. + filters: An `int` for the output depth of the level. + pool_size: An `int` for the pool_size of max pool layers. + kernel_size: An `int` for convolution kernel size. + strides: A `Union[int, tuple]` that indicates convolution strides. + padding: An `int` for the padding to apply to layers in this stack. + activation: A `str` for the activation to use for this stack. + route: An `int` for the level to route from to get the next input. + dilation_rate: An `int` for the scale used in dialated Darknet. + output_name: A `str` for the name to use for this output. + is_output: A `bool` for whether this layer is an output in the default + model. + """ self.layer = layer self.stack = stack self.repetitions = reps @@ -78,6 +81,7 @@ def __init__(self, layer, stack, reps, bottleneck, filters, pool_size, self.padding = padding self.activation = activation self.route = route + self.dilation_rate = dilation_rate self.output_name = output_name self.is_output = is_output @@ -89,41 +93,41 @@ def build_block_specs(config): return specs -class LayerFactory(object): - """Class for quick look up of default layers. +class LayerBuilder: + """Layer builder class. - Used by darknet to connect, introduce or exit a level. Used in place of an if - condition or switch to make adding new layers easier and to reduce redundant - code. + Class for quick look up of default layers used by darknet to + connect, introduce or exit a level. Used in place of an if condition + or switch to make adding new layers easier and to reduce redundant code. """ def __init__(self): self._layer_dict = { - "ConvBN": (nn_blocks.ConvBN, self.conv_bn_config_todict), - "MaxPool": (tf.keras.layers.MaxPool2D, self.maxpool_config_todict) + 'ConvBN': (nn_blocks.ConvBN, self.conv_bn_config_todict), + 'MaxPool': (tf.keras.layers.MaxPool2D, self.maxpool_config_todict) } def conv_bn_config_todict(self, config, kwargs): dictvals = { - "filters": config.filters, - "kernel_size": config.kernel_size, - "strides": config.strides, - "padding": config.padding + 'filters': config.filters, + 'kernel_size': config.kernel_size, + 'strides': config.strides, + 'padding': config.padding } dictvals.update(kwargs) return dictvals def darktiny_config_todict(self, config, kwargs): - dictvals = {"filters": config.filters, "strides": config.strides} + dictvals = {'filters': config.filters, 'strides': config.strides} dictvals.update(kwargs) return dictvals def maxpool_config_todict(self, config, kwargs): return { - "pool_size": config.pool_size, - "strides": config.strides, - "padding": config.padding, - "name": kwargs["name"] + 'pool_size': config.pool_size, + 'strides': config.strides, + 'padding': config.padding, + 'name': kwargs['name'] } def __call__(self, config, kwargs): @@ -134,90 +138,259 @@ def __call__(self, config, kwargs): # model configs LISTNAMES = [ - "default_layer_name", "level_type", "number_of_layers_in_level", - "bottleneck", "filters", "kernal_size", "pool_size", "strides", "padding", - "default_activation", "route", "level/name", "is_output" + 'default_layer_name', 'level_type', 'number_of_layers_in_level', + 'bottleneck', 'filters', 'kernal_size', 'pool_size', 'strides', 'padding', + 'default_activation', 'route', 'dilation', 'level/name', 'is_output' ] -# pylint: disable=line-too-long CSPDARKNET53 = { - "list_names": LISTNAMES, - "splits": {"backbone_split": 106, - "neck_split": 138}, - "backbone": [ - ["ConvBN", None, 1, False, 32, None, 3, 1, "same", "mish", -1, 0, False], - ["DarkRes", "csp", 1, True, 64, None, None, None, None, "mish", -1, 1, False], - ["DarkRes", "csp", 2, False, 128, None, None, None, None, "mish", -1, 2, False], - ["DarkRes", "csp", 8, False, 256, None, None, None, None, "mish", -1, 3, True], - ["DarkRes", "csp", 8, False, 512, None, None, None, None, "mish", -1, 4, True], - ["DarkRes", "csp", 4, False, 1024, None, None, None, None, "mish", -1, 5, True], + 'list_names': + LISTNAMES, + 'splits': { + 'backbone_split': 106, + 'neck_split': 132 + }, + 'backbone': [ + [ + 'ConvBN', None, 1, False, 32, None, 3, 1, 'same', 'mish', -1, 1, 0, + False + ], + [ + 'DarkRes', 'csp', 1, True, 64, None, None, None, None, 'mish', -1, + 1, 1, False + ], + [ + 'DarkRes', 'csp', 2, False, 128, None, None, None, None, 'mish', -1, + 1, 2, False + ], + [ + 'DarkRes', 'csp', 8, False, 256, None, None, None, None, 'mish', -1, + 1, 3, True + ], + [ + 'DarkRes', 'csp', 8, False, 512, None, None, None, None, 'mish', -1, + 2, 4, True + ], + [ + 'DarkRes', 'csp', 4, False, 1024, None, None, None, None, 'mish', + -1, 4, 5, True + ], + ] +} + +CSPADARKNET53 = { + 'list_names': + LISTNAMES, + 'splits': { + 'backbone_split': 100, + 'neck_split': 135 + }, + 'backbone': [ + [ + 'ConvBN', None, 1, False, 32, None, 3, 1, 'same', 'mish', -1, 1, 0, + False + ], + [ + 'DarkRes', 'residual', 1, True, 64, None, None, None, None, 'mish', + -1, 1, 1, False + ], + [ + 'DarkRes', 'csp', 2, False, 128, None, None, None, None, 'mish', -1, + 1, 2, False + ], + [ + 'DarkRes', 'csp', 8, False, 256, None, None, None, None, 'mish', -1, + 1, 3, True + ], + [ + 'DarkRes', 'csp', 8, False, 512, None, None, None, None, 'mish', -1, + 2, 4, True + ], + [ + 'DarkRes', 'csp', 4, False, 1024, None, None, None, None, 'mish', + -1, 4, 5, True + ], + ] +} + +LARGECSP53 = { + 'list_names': + LISTNAMES, + 'splits': { + 'backbone_split': 100, + 'neck_split': 135 + }, + 'backbone': [ + [ + 'ConvBN', None, 1, False, 32, None, 3, 1, 'same', 'mish', -1, 1, 0, + False + ], + [ + 'DarkRes', 'csp', 1, True, 64, None, None, None, None, 'mish', -1, + 1, 1, False + ], + [ + 'DarkRes', 'csp', 3, False, 128, None, None, None, None, 'mish', -1, + 1, 2, False + ], + [ + 'DarkRes', 'csp', 15, False, 256, None, None, None, None, 'mish', + -1, 1, 3, True + ], + [ + 'DarkRes', 'csp', 15, False, 512, None, None, None, None, 'mish', + -1, 2, 4, True + ], + [ + 'DarkRes', 'csp', 7, False, 1024, None, None, None, None, 'mish', + -1, 4, 5, True + ], + [ + 'DarkRes', 'csp', 7, False, 1024, None, None, None, None, 'mish', + -1, 8, 6, True + ], + [ + 'DarkRes', 'csp', 7, False, 1024, None, None, None, None, 'mish', + -1, 16, 7, True + ], ] } DARKNET53 = { - "list_names": LISTNAMES, - "splits": {"backbone_split": 76}, - "backbone": [ - ["ConvBN", None, 1, False, 32, None, 3, 1, "same", "leaky", -1, 0, False], - ["DarkRes", "residual", 1, True, 64, None, None, None, None, "leaky", -1, 1, False], - ["DarkRes", "residual", 2, False, 128, None, None, None, None, "leaky", -1, 2, False], - ["DarkRes", "residual", 8, False, 256, None, None, None, None, "leaky", -1, 3, True], - ["DarkRes", "residual", 8, False, 512, None, None, None, None, "leaky", -1, 4, True], - ["DarkRes", "residual", 4, False, 1024, None, None, None, None, "leaky", -1, 5, True], + 'list_names': + LISTNAMES, + 'splits': { + 'backbone_split': 76 + }, + 'backbone': [ + [ + 'ConvBN', None, 1, False, 32, None, 3, 1, 'same', 'leaky', -1, 1, 0, + False + ], + [ + 'DarkRes', 'residual', 1, True, 64, None, None, None, None, 'leaky', + -1, 1, 1, False + ], + [ + 'DarkRes', 'residual', 2, False, 128, None, None, None, None, + 'leaky', -1, 1, 2, False + ], + [ + 'DarkRes', 'residual', 8, False, 256, None, None, None, None, + 'leaky', -1, 1, 3, True + ], + [ + 'DarkRes', 'residual', 8, False, 512, None, None, None, None, + 'leaky', -1, 2, 4, True + ], + [ + 'DarkRes', 'residual', 4, False, 1024, None, None, None, None, + 'leaky', -1, 4, 5, True + ], ] } CSPDARKNETTINY = { - "list_names": LISTNAMES, - "splits": {"backbone_split": 28}, - "backbone": [ - ["ConvBN", None, 1, False, 32, None, 3, 2, "same", "leaky", -1, 0, False], - ["ConvBN", None, 1, False, 64, None, 3, 2, "same", "leaky", -1, 1, False], - ["CSPTiny", "csp_tiny", 1, False, 64, None, 3, 2, "same", "leaky", -1, 2, False], - ["CSPTiny", "csp_tiny", 1, False, 128, None, 3, 2, "same", "leaky", -1, 3, False], - ["CSPTiny", "csp_tiny", 1, False, 256, None, 3, 2, "same", "leaky", -1, 4, True], - ["ConvBN", None, 1, False, 512, None, 3, 1, "same", "leaky", -1, 5, True], + 'list_names': + LISTNAMES, + 'splits': { + 'backbone_split': 28 + }, + 'backbone': [ + [ + 'ConvBN', None, 1, False, 32, None, 3, 2, 'same', 'leaky', -1, 1, 0, + False + ], + [ + 'ConvBN', None, 1, False, 64, None, 3, 2, 'same', 'leaky', -1, 1, 1, + False + ], + [ + 'CSPTiny', 'csp_tiny', 1, False, 64, None, 3, 2, 'same', 'leaky', + -1, 1, 2, False + ], + [ + 'CSPTiny', 'csp_tiny', 1, False, 128, None, 3, 2, 'same', 'leaky', + -1, 1, 3, False + ], + [ + 'CSPTiny', 'csp_tiny', 1, False, 256, None, 3, 2, 'same', 'leaky', + -1, 1, 4, True + ], + [ + 'ConvBN', None, 1, False, 512, None, 3, 1, 'same', 'leaky', -1, 1, + 5, True + ], ] } DARKNETTINY = { - "list_names": LISTNAMES, - "splits": {"backbone_split": 14}, - "backbone": [ - ["ConvBN", None, 1, False, 16, None, 3, 1, "same", "leaky", -1, 0, False], - ["DarkTiny", "tiny", 1, True, 32, None, 3, 2, "same", "leaky", -1, 1, False], - ["DarkTiny", "tiny", 1, True, 64, None, 3, 2, "same", "leaky", -1, 2, False], - ["DarkTiny", "tiny", 1, False, 128, None, 3, 2, "same", "leaky", -1, 3, False], - ["DarkTiny", "tiny", 1, False, 256, None, 3, 2, "same", "leaky", -1, 4, True], - ["DarkTiny", "tiny", 1, False, 512, None, 3, 2, "same", "leaky", -1, 5, False], - ["DarkTiny", "tiny", 1, False, 1024, None, 3, 1, "same", "leaky", -1, 5, True], + 'list_names': + LISTNAMES, + 'splits': { + 'backbone_split': 14 + }, + 'backbone': [ + [ + 'ConvBN', None, 1, False, 16, None, 3, 1, 'same', 'leaky', -1, 1, 0, + False + ], + [ + 'DarkTiny', 'tiny', 1, True, 32, None, 3, 2, 'same', 'leaky', -1, 1, + 1, False + ], + [ + 'DarkTiny', 'tiny', 1, True, 64, None, 3, 2, 'same', 'leaky', -1, 1, + 2, False + ], + [ + 'DarkTiny', 'tiny', 1, False, 128, None, 3, 2, 'same', 'leaky', -1, + 1, 3, False + ], + [ + 'DarkTiny', 'tiny', 1, False, 256, None, 3, 2, 'same', 'leaky', -1, + 1, 4, True + ], + [ + 'DarkTiny', 'tiny', 1, False, 512, None, 3, 2, 'same', 'leaky', -1, + 1, 5, False + ], + [ + 'DarkTiny', 'tiny', 1, False, 1024, None, 3, 1, 'same', 'leaky', -1, + 1, 5, True + ], ] } -# pylint: enable=line-too-long BACKBONES = { - "darknettiny": DARKNETTINY, - "darknet53": DARKNET53, - "cspdarknet53": CSPDARKNET53, - "cspdarknettiny": CSPDARKNETTINY + 'darknettiny': DARKNETTINY, + 'darknet53': DARKNET53, + 'cspdarknet53': CSPDARKNET53, + 'altered_cspdarknet53': CSPADARKNET53, + 'cspdarknettiny': CSPDARKNETTINY, + 'csp-large': LARGECSP53, } -@tf.keras.utils.register_keras_serializable(package="yolo") +@tf.keras.utils.register_keras_serializable(package='yolo') class Darknet(tf.keras.Model): - """Darknet backbone.""" + """The Darknet backbone architecture.""" def __init__( self, - model_id="darknet53", + model_id='darknet53', input_specs=tf.keras.layers.InputSpec(shape=[None, None, None, 3]), min_level=None, max_level=5, + width_scale=1.0, + depth_scale=1.0, + csp_level_mod=(), activation=None, use_sync_bn=False, norm_momentum=0.99, norm_epsilon=0.001, - kernel_initializer="glorot_uniform", + dilate=False, + kernel_initializer='glorot_uniform', kernel_regularizer=None, bias_regularizer=None, **kwargs): @@ -227,12 +400,13 @@ def __init__( self._model_name = model_id self._splits = splits self._input_shape = input_specs - self._registry = LayerFactory() + self._registry = LayerBuilder() # default layer look up self._min_size = min_level self._max_size = max_level self._output_specs = None + self._csp_level_mod = set(csp_level_mod) self._kernel_initializer = kernel_initializer self._bias_regularizer = bias_regularizer @@ -241,16 +415,20 @@ def __init__( self._use_sync_bn = use_sync_bn self._activation = activation self._kernel_regularizer = kernel_regularizer + self._dilate = dilate + self._width_scale = width_scale + self._depth_scale = depth_scale self._default_dict = { - "kernel_initializer": self._kernel_initializer, - "kernel_regularizer": self._kernel_regularizer, - "bias_regularizer": self._bias_regularizer, - "norm_momentum": self._norm_momentum, - "norm_epsilon": self._norm_epislon, - "use_sync_bn": self._use_sync_bn, - "activation": self._activation, - "name": None + 'kernel_initializer': self._kernel_initializer, + 'kernel_regularizer': self._kernel_regularizer, + 'bias_regularizer': self._bias_regularizer, + 'norm_momentum': self._norm_momentum, + 'norm_epsilon': self._norm_epislon, + 'use_sync_bn': self._use_sync_bn, + 'activation': self._activation, + 'dilation_rate': 1, + 'name': None } inputs = tf.keras.layers.Input(shape=self._input_shape.shape[1:]) @@ -273,33 +451,39 @@ def _build_struct(self, net, inputs): endpoints = collections.OrderedDict() stack_outputs = [inputs] for i, config in enumerate(net): + if config.output_name > self._max_size: + break + if config.output_name in self._csp_level_mod: + config.stack = 'residual' + + config.filters = int(config.filters * self._width_scale) + config.repetitions = int(config.repetitions * self._depth_scale) + if config.stack is None: - x = self._build_block(stack_outputs[config.route], - config, - name=f"{config.layer}_{i}") + x = self._build_block( + stack_outputs[config.route], config, name=f'{config.layer}_{i}') stack_outputs.append(x) - elif config.stack == "residual": - x = self._residual_stack(stack_outputs[config.route], - config, - name=f"{config.layer}_{i}") + elif config.stack == 'residual': + x = self._residual_stack( + stack_outputs[config.route], config, name=f'{config.layer}_{i}') stack_outputs.append(x) - elif config.stack == "csp": - x = self._csp_stack(stack_outputs[config.route], - config, - name=f"{config.layer}_{i}") + elif config.stack == 'csp': + x = self._csp_stack( + stack_outputs[config.route], config, name=f'{config.layer}_{i}') stack_outputs.append(x) - elif config.stack == "csp_tiny": - x_pass, x = self._csp_tiny_stack(stack_outputs[config.route], - config, name=f"{config.layer}_{i}") + elif config.stack == 'csp_tiny': + x_pass, x = self._csp_tiny_stack( + stack_outputs[config.route], config, name=f'{config.layer}_{i}') stack_outputs.append(x_pass) - elif config.stack == "tiny": - x = self._tiny_stack(stack_outputs[config.route], - config, - name=f"{config.layer}_{i}") + elif config.stack == 'tiny': + x = self._tiny_stack( + stack_outputs[config.route], config, name=f'{config.layer}_{i}') stack_outputs.append(x) if (config.is_output and self._min_size is None): endpoints[str(config.output_name)] = x - elif self._min_size is not None and config.output_name >= self._min_size and config.output_name <= self._max_size: + elif (self._min_size is not None and + config.output_name >= self._min_size and + config.output_name <= self._max_size): endpoints[str(config.output_name)] = x self._output_specs = {l: endpoints[l].get_shape() for l in endpoints.keys()} @@ -308,8 +492,7 @@ def _build_struct(self, net, inputs): def _get_activation(self, activation): if self._activation is None: return activation - else: - return self._activation + return self._activation def _csp_stack(self, inputs, config, name): if config.bottleneck: @@ -320,86 +503,135 @@ def _csp_stack(self, inputs, config, name): csp_filter_scale = 2 residual_filter_scale = 1 scale_filters = 2 - self._default_dict["activation"] = self._get_activation(config.activation) - self._default_dict["name"] = f"{name}_csp_down" - x, x_route = nn_blocks.CSPRoute(filters=config.filters, - filter_scale=csp_filter_scale, - downsample=True, - **self._default_dict)(inputs) - for i in range(config.repetitions): - self._default_dict["name"] = f"{name}_{i}" - x = nn_blocks.DarkResidual(filters=config.filters // scale_filters, - filter_scale=residual_filter_scale, - **self._default_dict)(x) - - self._default_dict["name"] = f"{name}_csp_connect" - output = nn_blocks.CSPConnect(filters=config.filters, - filter_scale=csp_filter_scale, - **self._default_dict)([x, x_route]) - self._default_dict["activation"] = self._activation - self._default_dict["name"] = None + self._default_dict['activation'] = self._get_activation(config.activation) + self._default_dict['name'] = f'{name}_csp_down' + if self._dilate: + self._default_dict['dilation_rate'] = config.dilation_rate + else: + self._default_dict['dilation_rate'] = 1 + + # swap/add dilation + x, x_route = nn_blocks.CSPRoute( + filters=config.filters, + filter_scale=csp_filter_scale, + downsample=True, + **self._default_dict)( + inputs) + + dilated_reps = config.repetitions - self._default_dict['dilation_rate'] // 2 + for i in range(dilated_reps): + self._default_dict['name'] = f'{name}_{i}' + x = nn_blocks.DarkResidual( + filters=config.filters // scale_filters, + filter_scale=residual_filter_scale, + **self._default_dict)( + x) + + for i in range(dilated_reps, config.repetitions): + self._default_dict[ + 'dilation_rate'] = self._default_dict['dilation_rate'] // 2 + self._default_dict[ + 'name'] = f"{name}_{i}_degridded_{self._default_dict['dilation_rate']}" + x = nn_blocks.DarkResidual( + filters=config.filters // scale_filters, + filter_scale=residual_filter_scale, + **self._default_dict)( + x) + + self._default_dict['name'] = f'{name}_csp_connect' + output = nn_blocks.CSPConnect( + filters=config.filters, + filter_scale=csp_filter_scale, + **self._default_dict)([x, x_route]) + self._default_dict['activation'] = self._activation + self._default_dict['name'] = None return output def _csp_tiny_stack(self, inputs, config, name): - self._default_dict["activation"] = self._get_activation(config.activation) - self._default_dict["name"] = f"{name}_csp_tiny" - x, x_route = nn_blocks.CSPTiny(filters=config.filters, - **self._default_dict)(inputs) - self._default_dict["activation"] = self._activation - self._default_dict["name"] = None + self._default_dict['activation'] = self._get_activation(config.activation) + self._default_dict['name'] = f'{name}_csp_tiny' + x, x_route = nn_blocks.CSPTiny( + filters=config.filters, **self._default_dict)( + inputs) + self._default_dict['activation'] = self._activation + self._default_dict['name'] = None return x, x_route def _tiny_stack(self, inputs, config, name): - x = tf.keras.layers.MaxPool2D(pool_size=2, - strides=config.strides, - padding="same", - data_format=None, - name=f"{name}_tiny/pool")(inputs) - self._default_dict["activation"] = self._get_activation(config.activation) - self._default_dict["name"] = f"{name}_tiny/conv" + x = tf.keras.layers.MaxPool2D( + pool_size=2, + strides=config.strides, + padding='same', + data_format=None, + name=f'{name}_tiny/pool')( + inputs) + self._default_dict['activation'] = self._get_activation(config.activation) + self._default_dict['name'] = f'{name}_tiny/conv' x = nn_blocks.ConvBN( filters=config.filters, kernel_size=(3, 3), strides=(1, 1), - padding="same", + padding='same', **self._default_dict)( x) - self._default_dict["activation"] = self._activation - self._default_dict["name"] = None + self._default_dict['activation'] = self._activation + self._default_dict['name'] = None return x def _residual_stack(self, inputs, config, name): - self._default_dict["activation"] = self._get_activation(config.activation) - self._default_dict["name"] = f"{name}_residual_down" - x = nn_blocks.DarkResidual(filters=config.filters, - downsample=True, - **self._default_dict)(inputs) - for i in range(config.repetitions - 1): - self._default_dict["name"] = f"{name}_{i}" - x = nn_blocks.DarkResidual(filters=config.filters, - **self._default_dict)(x) - self._default_dict["activation"] = self._activation - self._default_dict["name"] = None + self._default_dict['activation'] = self._get_activation(config.activation) + self._default_dict['name'] = f'{name}_residual_down' + if self._dilate: + self._default_dict['dilation_rate'] = config.dilation_rate + if config.repetitions < 8: + config.repetitions += 2 + else: + self._default_dict['dilation_rate'] = 1 + + x = nn_blocks.DarkResidual( + filters=config.filters, downsample=True, **self._default_dict)( + inputs) + + dilated_reps = config.repetitions - ( + self._default_dict['dilation_rate'] // 2) - 1 + for i in range(dilated_reps): + self._default_dict['name'] = f'{name}_{i}' + x = nn_blocks.DarkResidual( + filters=config.filters, **self._default_dict)( + x) + + for i in range(dilated_reps, config.repetitions - 1): + self._default_dict[ + 'dilation_rate'] = self._default_dict['dilation_rate'] // 2 + self._default_dict[ + 'name'] = f"{name}_{i}_degridded_{self._default_dict['dilation_rate']}" + x = nn_blocks.DarkResidual( + filters=config.filters, **self._default_dict)( + x) + + self._default_dict['activation'] = self._activation + self._default_dict['name'] = None + self._default_dict['dilation_rate'] = 1 return x def _build_block(self, inputs, config, name): x = inputs i = 0 - self._default_dict["activation"] = self._get_activation(config.activation) + self._default_dict['activation'] = self._get_activation(config.activation) while i < config.repetitions: - self._default_dict["name"] = f"{name}_{i}" + self._default_dict['name'] = f'{name}_{i}' layer = self._registry(config, self._default_dict) x = layer(x) i += 1 - self._default_dict["activation"] = self._activation - self._default_dict["name"] = None + self._default_dict['activation'] = self._activation + self._default_dict['name'] = None return x @staticmethod def get_model_config(name): name = name.lower() - backbone = BACKBONES[name]["backbone"] - splits = BACKBONES[name]["splits"] + backbone = BACKBONES[name]['backbone'] + splits = BACKBONES[name]['splits'] return build_block_specs(backbone), splits @property @@ -412,35 +644,41 @@ def from_config(cls, config, custom_objects=None): def get_config(self): layer_config = { - "model_id": self._model_name, - "min_level": self._min_size, - "max_level": self._max_size, - "kernel_initializer": self._kernel_initializer, - "kernel_regularizer": self._kernel_regularizer, - "bias_regularizer": self._bias_regularizer, - "norm_momentum": self._norm_momentum, - "norm_epsilon": self._norm_epislon, - "use_sync_bn": self._use_sync_bn, - "activation": self._activation + 'model_id': self._model_name, + 'min_level': self._min_size, + 'max_level': self._max_size, + 'kernel_initializer': self._kernel_initializer, + 'kernel_regularizer': self._kernel_regularizer, + 'bias_regularizer': self._bias_regularizer, + 'norm_momentum': self._norm_momentum, + 'norm_epsilon': self._norm_epislon, + 'use_sync_bn': self._use_sync_bn, + 'activation': self._activation, } return layer_config -@factory.register_backbone_builder("darknet") +@factory.register_backbone_builder('darknet') def build_darknet( input_specs: tf.keras.layers.InputSpec, backbone_config: hyperparams.Config, norm_activation_config: hyperparams.Config, l2_regularizer: tf.keras.regularizers.Regularizer = None) -> tf.keras.Model: - """Builds darknet backbone.""" + """Builds darknet.""" backbone_cfg = backbone_config.get() model = Darknet( model_id=backbone_cfg.model_id, - input_shape=input_specs, + min_level=backbone_cfg.min_level, + max_level=backbone_cfg.max_level, + input_specs=input_specs, + dilate=backbone_cfg.dilate, + width_scale=backbone_cfg.width_scale, + depth_scale=backbone_cfg.depth_scale, activation=norm_activation_config.activation, use_sync_bn=norm_activation_config.use_sync_bn, norm_momentum=norm_activation_config.norm_momentum, norm_epsilon=norm_activation_config.norm_epsilon, kernel_regularizer=l2_regularizer) + model.summary() return model diff --git a/official/vision/beta/projects/yolo/modeling/backbones/darknet_test.py b/official/vision/beta/projects/yolo/modeling/backbones/darknet_test.py index 76c595f2dd7..9441b06a311 100644 --- a/official/vision/beta/projects/yolo/modeling/backbones/darknet_test.py +++ b/official/vision/beta/projects/yolo/modeling/backbones/darknet_test.py @@ -13,7 +13,7 @@ # limitations under the License. # Lint as: python3 -"""Tests for resnet.""" +"""Tests for yolo.""" from absl.testing import parameterized import numpy as np @@ -24,35 +24,48 @@ from official.vision.beta.projects.yolo.modeling.backbones import darknet -class DarkNetTest(parameterized.TestCase, tf.test.TestCase): +class DarknetTest(parameterized.TestCase, tf.test.TestCase): @parameterized.parameters( - (224, "darknet53", 2, 1), - (224, "darknettiny", 1, 2), - (224, "cspdarknettiny", 1, 1), - (224, "cspdarknet53", 2, 1), + (224, 'darknet53', 2, 1, True), + (224, 'darknettiny', 1, 2, False), + (224, 'cspdarknettiny', 1, 1, False), + (224, 'cspdarknet53', 2, 1, True), ) - def test_network_creation(self, input_size, model_id, - endpoint_filter_scale, scale_final): + def test_network_creation(self, input_size, model_id, endpoint_filter_scale, + scale_final, dilate): """Test creation of ResNet family models.""" - tf.keras.backend.set_image_data_format("channels_last") + tf.keras.backend.set_image_data_format('channels_last') - network = darknet.Darknet(model_id=model_id, min_level=3, max_level=5) + network = darknet.Darknet( + model_id=model_id, min_level=3, max_level=5, dilate=dilate) self.assertEqual(network.model_id, model_id) inputs = tf.keras.Input(shape=(input_size, input_size, 3), batch_size=1) endpoints = network(inputs) - self.assertAllEqual( - [1, input_size / 2**3, input_size / 2**3, 128 * endpoint_filter_scale], - endpoints["3"].shape.as_list()) - self.assertAllEqual( - [1, input_size / 2**4, input_size / 2**4, 256 * endpoint_filter_scale], - endpoints["4"].shape.as_list()) - self.assertAllEqual([ - 1, input_size / 2**5, input_size / 2**5, - 512 * endpoint_filter_scale * scale_final - ], endpoints["5"].shape.as_list()) + if dilate: + self.assertAllEqual([ + 1, input_size / 2**3, input_size / 2**3, 128 * endpoint_filter_scale + ], endpoints['3'].shape.as_list()) + self.assertAllEqual([ + 1, input_size / 2**3, input_size / 2**3, 256 * endpoint_filter_scale + ], endpoints['4'].shape.as_list()) + self.assertAllEqual([ + 1, input_size / 2**3, input_size / 2**3, + 512 * endpoint_filter_scale * scale_final + ], endpoints['5'].shape.as_list()) + else: + self.assertAllEqual([ + 1, input_size / 2**3, input_size / 2**3, 128 * endpoint_filter_scale + ], endpoints['3'].shape.as_list()) + self.assertAllEqual([ + 1, input_size / 2**4, input_size / 2**4, 256 * endpoint_filter_scale + ], endpoints['4'].shape.as_list()) + self.assertAllEqual([ + 1, input_size / 2**5, input_size / 2**5, + 512 * endpoint_filter_scale * scale_final + ], endpoints['5'].shape.as_list()) @combinations.generate( combinations.combine( @@ -66,20 +79,20 @@ def test_sync_bn_multiple_devices(self, strategy, use_sync_bn): """Test for sync bn on TPU and GPU devices.""" inputs = np.random.rand(1, 224, 224, 3) - tf.keras.backend.set_image_data_format("channels_last") + tf.keras.backend.set_image_data_format('channels_last') with strategy.scope(): - network = darknet.Darknet(model_id="darknet53", min_size=3, max_size=5) + network = darknet.Darknet(model_id='darknet53', min_size=3, max_size=5) _ = network(inputs) @parameterized.parameters(1, 3, 4) def test_input_specs(self, input_dim): """Test different input feature dimensions.""" - tf.keras.backend.set_image_data_format("channels_last") + tf.keras.backend.set_image_data_format('channels_last') input_specs = tf.keras.layers.InputSpec(shape=[None, None, None, input_dim]) network = darknet.Darknet( - model_id="darknet53", min_level=3, max_level=5, input_specs=input_specs) + model_id='darknet53', min_level=3, max_level=5, input_specs=input_specs) inputs = tf.keras.Input(shape=(224, 224, input_dim), batch_size=1) _ = network(inputs) @@ -87,14 +100,14 @@ def test_input_specs(self, input_dim): def test_serialize_deserialize(self): # Create a network object that sets all of its config options. kwargs = dict( - model_id="darknet53", + model_id='darknet53', min_level=3, max_level=5, use_sync_bn=False, - activation="relu", + activation='relu', norm_momentum=0.99, norm_epsilon=0.001, - kernel_initializer="VarianceScaling", + kernel_initializer='VarianceScaling', kernel_regularizer=None, bias_regularizer=None, ) @@ -113,5 +126,5 @@ def test_serialize_deserialize(self): self.assertAllEqual(network.get_config(), new_network.get_config()) -if __name__ == "__main__": +if __name__ == '__main__': tf.test.main() diff --git a/official/vision/beta/projects/yolo/modeling/decoders/__init__.py b/official/vision/beta/projects/yolo/modeling/decoders/__init__.py new file mode 100644 index 00000000000..e419af524b5 --- /dev/null +++ b/official/vision/beta/projects/yolo/modeling/decoders/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py new file mode 100644 index 00000000000..40f71009f67 --- /dev/null +++ b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py @@ -0,0 +1,478 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Feature Pyramid Network and Path Aggregation variants used in YOLO.""" + +import tensorflow as tf +from official.vision.beta.projects.yolo.modeling.layers import nn_blocks + + +@tf.keras.utils.register_keras_serializable(package='yolo') +class _IdentityRoute(tf.keras.layers.Layer): + + def call(self, inputs): + return None, inputs + + +@tf.keras.utils.register_keras_serializable(package='yolo') +class YoloFPN(tf.keras.layers.Layer): + """YOLO Feature pyramid network.""" + + def __init__(self, + fpn_depth=4, + use_spatial_attention=False, + csp_stack=False, + activation='leaky', + fpn_filter_scale=1, + use_sync_bn=False, + norm_momentum=0.99, + norm_epsilon=0.001, + kernel_initializer='glorot_uniform', + kernel_regularizer=None, + bias_regularizer=None, + **kwargs): + """Yolo FPN initialization function (Yolo V4). + + Args: + fpn_depth: `int`, number of layers to use in each FPN path + if you choose to use an FPN. + use_spatial_attention: `bool`, use the spatial attention module. + csp_stack: `bool`, CSPize the FPN. + activation: `str`, the activation function to use typically leaky or mish. + fpn_filter_scale: `int`, scaling factor for the FPN filters. + use_sync_bn: if True, use synchronized batch normalization. + norm_momentum: `float`, normalization momentum for the moving average. + norm_epsilon: `float`, small float added to variance to avoid dividing by + zero. + kernel_initializer: kernel_initializer for convolutional layers. + kernel_regularizer: tf.keras.regularizers.Regularizer object for Conv2D. + bias_regularizer: tf.keras.regularizers.Regularizer object for Conv2D. + **kwargs: keyword arguments to be passed. + """ + + super().__init__(**kwargs) + self._fpn_depth = fpn_depth + + self._activation = activation + self._use_sync_bn = use_sync_bn + self._norm_momentum = norm_momentum + self._norm_epsilon = norm_epsilon + self._kernel_initializer = kernel_initializer + self._kernel_regularizer = kernel_regularizer + self._bias_regularizer = bias_regularizer + self._use_spatial_attention = use_spatial_attention + self._filter_scale = fpn_filter_scale + self._csp_stack = csp_stack + + self._base_config = dict( + activation=self._activation, + use_sync_bn=self._use_sync_bn, + kernel_regularizer=self._kernel_regularizer, + kernel_initializer=self._kernel_initializer, + bias_regularizer=self._bias_regularizer, + norm_epsilon=self._norm_epsilon, + norm_momentum=self._norm_momentum) + + def get_raw_depths(self, minimum_depth, inputs): + """Calculates the unscaled depths of the FPN branches. + + Args: + minimum_depth (int): depth of the smallest branch of the FPN. + inputs (dict): dictionary of the shape of input args as a dictionary of + lists. + + Returns: + The unscaled depths of the FPN branches. + """ + + depths = [] + for i in range(self._min_level, self._max_level + 1): + depths.append(inputs[str(i)][-1] / self._filter_scale) + return list(reversed(depths)) + + def build(self, inputs): + """Use config dictionary to generate all important attributes for head. + + Args: + inputs: dictionary of the shape of input args as a dictionary of lists. + """ + + keys = [int(key) for key in inputs.keys()] + self._min_level = min(keys) + self._max_level = max(keys) + self._min_depth = inputs[str(self._min_level)][-1] + self._depths = self.get_raw_depths(self._min_depth, inputs) + + # directly connect to an input path and process it + self.preprocessors = dict() + # resample an input and merge it with the output of another path + # inorder to aggregate backbone outputs + self.resamples = dict() + # set of convoltion layers and upsample layers that are used to + # prepare the FPN processors for output + + for level, depth in zip( + reversed(range(self._min_level, self._max_level + 1)), self._depths): + if level == self._min_level: + self.resamples[str(level)] = nn_blocks.PathAggregationBlock( + filters=depth // 2, + inverted=True, + upsample=True, + drop_final=self._csp_stack == 0, + upsample_size=2, + **self._base_config) + self.preprocessors[str(level)] = _IdentityRoute() + elif level != self._max_level: + self.resamples[str(level)] = nn_blocks.PathAggregationBlock( + filters=depth // 2, + inverted=True, + upsample=True, + drop_final=False, + upsample_size=2, + **self._base_config) + self.preprocessors[str(level)] = nn_blocks.DarkRouteProcess( + filters=depth, + repetitions=self._fpn_depth - int(level == self._min_level), + block_invert=True, + insert_spp=False, + csp_stack=self._csp_stack, + **self._base_config) + else: + self.preprocessors[str(level)] = nn_blocks.DarkRouteProcess( + filters=depth, + repetitions=self._fpn_depth + 1 * int(self._csp_stack == 0), + insert_spp=True, + block_invert=False, + csp_stack=self._csp_stack, + **self._base_config) + + def call(self, inputs): + outputs = dict() + layer_in = inputs[str(self._max_level)] + for level in reversed(range(self._min_level, self._max_level + 1)): + _, x = self.preprocessors[str(level)](layer_in) + outputs[str(level)] = x + if level > self._min_level: + x_next = inputs[str(level - 1)] + _, layer_in = self.resamples[str(level - 1)]([x_next, x]) + return outputs + + +@tf.keras.utils.register_keras_serializable(package='yolo') +class YoloPAN(tf.keras.layers.Layer): + """YOLO Path Aggregation Network.""" + + def __init__(self, + path_process_len=6, + max_level_process_len=None, + embed_spp=False, + use_spatial_attention=False, + csp_stack=False, + activation='leaky', + use_sync_bn=False, + norm_momentum=0.99, + norm_epsilon=0.001, + kernel_initializer='glorot_uniform', + kernel_regularizer=None, + bias_regularizer=None, + fpn_input=True, + fpn_filter_scale=1.0, + **kwargs): + """Yolo Path Aggregation Network initialization function (Yolo V3 and V4). + + Args: + path_process_len: `int`, number of layers ot use in each Decoder path. + max_level_process_len: `int`, number of layers ot use in the largest + processing path, or the backbones largest output if it is different. + embed_spp: `bool`, use the SPP found in the YoloV3 and V4 model. + use_spatial_attention: `bool`, use the spatial attention module. + csp_stack: `bool`, CSPize the FPN. + activation: `str`, the activation function to use typically leaky or mish. + use_sync_bn: if True, use synchronized batch normalization. + norm_momentum: `float`, normalization omentum for the moving average. + norm_epsilon: `float`, small float added to variance to avoid dividing + by zero. + kernel_initializer: kernel_initializer for convolutional layers. + kernel_regularizer: tf.keras.regularizers.Regularizer object for Conv2D. + bias_regularizer: tf.keras.regularizers.Regularizer object for Conv2d. + fpn_input: `bool`, for whether the input into this fucntion is an FPN or + a backbone. + fpn_filter_scale: `int`, scaling factor for the FPN filters. + **kwargs: keyword arguments to be passed. + """ + + super().__init__(**kwargs) + + self._path_process_len = path_process_len + self._embed_spp = embed_spp + self._use_spatial_attention = use_spatial_attention + + self._activation = activation + self._use_sync_bn = use_sync_bn + self._norm_momentum = norm_momentum + self._norm_epsilon = norm_epsilon + self._kernel_initializer = kernel_initializer + self._kernel_regularizer = kernel_regularizer + self._bias_regularizer = bias_regularizer + self._fpn_input = fpn_input + self._max_level_process_len = max_level_process_len + self._csp_stack = csp_stack + self._fpn_filter_scale = fpn_filter_scale + + if max_level_process_len is None: + self._max_level_process_len = path_process_len + + self._base_config = dict( + activation=self._activation, + use_sync_bn=self._use_sync_bn, + kernel_regularizer=self._kernel_regularizer, + kernel_initializer=self._kernel_initializer, + bias_regularizer=self._bias_regularizer, + norm_epsilon=self._norm_epsilon, + norm_momentum=self._norm_momentum) + + def build(self, inputs): + """Use config dictionary to generate all important attributes for head. + + Args: + inputs: dictionary of the shape of input args as a dictionary of lists. + """ + + # define the key order + keys = [int(key) for key in inputs.keys()] + self._min_level = min(keys) + self._max_level = max(keys) + self._min_depth = inputs[str(self._min_level)][-1] + self._depths = self.get_raw_depths(self._min_depth, inputs) + + # directly connect to an input path and process it + self.preprocessors = dict() + # resample an input and merge it with the output of another path + # inorder to aggregate backbone outputs + self.resamples = dict() + + # FPN will reverse the key process order for the backbone, so we need + # adjust the order that objects are created and processed to adjust for + # this. not using an FPN will directly connect the decoder to the backbone + # therefore the object creation order needs to be done from the largest + # to smallest level. + if self._fpn_input: + # process order {... 3, 4, 5} + self._iterator = range(self._min_level, self._max_level + 1) + self._check = lambda x: x < self._max_level + self._key_shift = lambda x: x + 1 + self._input = self._min_level + downsample = True + upsample = False + else: + # process order {5, 4, 3, ...} + self._iterator = list( + reversed(range(self._min_level, self._max_level + 1))) + self._check = lambda x: x > self._min_level + self._key_shift = lambda x: x - 1 + self._input = self._max_level + downsample = False + upsample = True + + if self._csp_stack == 0: + proc_filters = lambda x: x + resample_filters = lambda x: x // 2 + else: + proc_filters = lambda x: x * 2 + resample_filters = lambda x: x + for level, depth in zip(self._iterator, self._depths): + if level == self._input: + self.preprocessors[str(level)] = nn_blocks.DarkRouteProcess( + filters=proc_filters(depth), + repetitions=self._max_level_process_len, + insert_spp=self._embed_spp, + block_invert=False, + insert_sam=self._use_spatial_attention, + csp_stack=self._csp_stack, + **self._base_config) + else: + self.resamples[str(level)] = nn_blocks.PathAggregationBlock( + filters=resample_filters(depth), + upsample=upsample, + downsample=downsample, + inverted=False, + drop_final=self._csp_stack == 0, + **self._base_config) + self.preprocessors[str(level)] = nn_blocks.DarkRouteProcess( + filters=proc_filters(depth), + repetitions=self._path_process_len, + insert_spp=False, + insert_sam=self._use_spatial_attention, + csp_stack=self._csp_stack, + **self._base_config) + + def get_raw_depths(self, minimum_depth, inputs): + """Calculates the unscaled depths of the FPN branches. + + Args: + minimum_depth: `int` depth of the smallest branch of the FPN. + inputs: `dict[str, tf.InputSpec]` of the shape of input args as a + dictionary of lists. + + Returns: + The unscaled depths of the FPN branches. + """ + + depths = [] + if len(inputs.keys()) > 3 or self._fpn_filter_scale > 1: + for i in range(self._min_level, self._max_level + 1): + depths.append(inputs[str(i)][-1] * 2) + else: + for _ in range(self._min_level, self._max_level + 1): + depths.append(minimum_depth) + minimum_depth *= 2 + if self._fpn_input: + return depths + return list(reversed(depths)) + + def call(self, inputs): + outputs = dict() + layer_in = inputs[str(self._input)] + + for level in self._iterator: + x_route, x = self.preprocessors[str(level)](layer_in) + outputs[str(level)] = x + if self._check(level): + x_next = inputs[str(self._key_shift(level))] + _, layer_in = self.resamples[str( + self._key_shift(level))]([x_route, x_next]) + return outputs + + +@tf.keras.utils.register_keras_serializable(package='yolo') +class YoloDecoder(tf.keras.Model): + """Darknet Backbone Decoder.""" + + def __init__(self, + input_specs, + use_fpn=False, + use_spatial_attention=False, + csp_stack=False, + fpn_depth=4, + fpn_filter_scale=1, + path_process_len=6, + max_level_process_len=None, + embed_spp=False, + activation='leaky', + use_sync_bn=False, + norm_momentum=0.99, + norm_epsilon=0.001, + kernel_initializer='glorot_uniform', + kernel_regularizer=None, + bias_regularizer=None, + **kwargs): + """Yolo Decoder initialization function. + + A unified model that ties all decoder components into a conditionally build + YOLO decoder. + + Args: + input_specs: `dict[str, tf.InputSpec]`: input specs of each of the inputs + to the heads. + use_fpn: `bool`, use the FPN found in the YoloV4 model. + use_spatial_attention: `bool`, use the spatial attention module. + csp_stack: `bool`, CSPize the FPN. + fpn_depth: `int`, number of layers ot use in each FPN path + if you choose to use an FPN. + fpn_filter_scale: `int`, scaling factor for the FPN filters. + path_process_len: `int`, number of layers ot use in each Decoder path. + max_level_process_len: `int`, number of layers ot use in the largest + processing path, or the backbones largest output if it is different. + embed_spp: `bool`, use the SPP found in the YoloV3 and V4 model. + activation: `str`, the activation function to use typically leaky or mish. + use_sync_bn: if True, use synchronized batch normalization. + norm_momentum: `float`, normalization omentum for the moving average. + norm_epsilon: `float`, small float added to variance to avoid dividing by + zero. + kernel_initializer: kernel_initializer for convolutional layers. + kernel_regularizer: tf.keras.regularizers.Regularizer object for Conv2D. + bias_regularizer: tf.keras.regularizers.Regularizer object for Conv2D. + **kwargs: keyword arguments to be passed. + """ + + self._input_specs = input_specs + self._use_fpn = use_fpn + self._fpn_depth = fpn_depth + self._path_process_len = path_process_len + self._max_level_process_len = max_level_process_len + self._embed_spp = embed_spp + + self._activation = activation + self._use_sync_bn = use_sync_bn + self._norm_momentum = norm_momentum + self._norm_epsilon = norm_epsilon + self._kernel_initializer = kernel_initializer + self._kernel_regularizer = kernel_regularizer + self._bias_regularizer = bias_regularizer + + self._base_config = dict( + use_spatial_attention=use_spatial_attention, + csp_stack=csp_stack, + activation=self._activation, + use_sync_bn=self._use_sync_bn, + fpn_filter_scale=fpn_filter_scale, + norm_momentum=self._norm_momentum, + norm_epsilon=self._norm_epsilon, + kernel_initializer=self._kernel_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer) + + self._decoder_config = dict( + path_process_len=self._path_process_len, + max_level_process_len=self._max_level_process_len, + embed_spp=self._embed_spp, + fpn_input=self._use_fpn, + **self._base_config) + + inputs = { + key: tf.keras.layers.Input(shape=value[1:]) + for key, value in input_specs.items() + } + if self._use_fpn: + inter_outs = YoloFPN( + fpn_depth=self._fpn_depth, **self._base_config)( + inputs) + outputs = YoloPAN(**self._decoder_config)(inter_outs) + else: + inter_outs = None + outputs = YoloPAN(**self._decoder_config)(inputs) + + self._output_specs = {key: value.shape for key, value in outputs.items()} + super().__init__(inputs=inputs, outputs=outputs, name='YoloDecoder') + + @property + def use_fpn(self): + return self._use_fpn + + @property + def output_specs(self): + return self._output_specs + + def get_config(self): + config = dict( + input_specs=self._input_specs, + use_fpn=self._use_fpn, + fpn_depth=self._fpn_depth, + **self._decoder_config) + return config + + @classmethod + def from_config(cls, config, custom_objects=None): + return cls(**config) diff --git a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder_test.py b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder_test.py new file mode 100644 index 00000000000..611c4585945 --- /dev/null +++ b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder_test.py @@ -0,0 +1,153 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Tests for YOLO.""" + +# Import libraries +from absl.testing import parameterized +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.vision.beta.projects.yolo.modeling.decoders import yolo_decoder as decoders + + +class YoloDecoderTest(parameterized.TestCase, tf.test.TestCase): + + def _build_yolo_decoder(self, input_specs, name='1'): + # Builds 4 different arbitrary decoders. + if name == '1': + model = decoders.YoloDecoder( + input_specs=input_specs, + embed_spp=False, + use_fpn=False, + max_level_process_len=2, + path_process_len=1, + activation='mish') + elif name == '6spp': + model = decoders.YoloDecoder( + input_specs=input_specs, + embed_spp=True, + use_fpn=False, + max_level_process_len=None, + path_process_len=6, + activation='mish') + elif name == '6sppfpn': + model = decoders.YoloDecoder( + input_specs=input_specs, + embed_spp=True, + use_fpn=True, + max_level_process_len=None, + path_process_len=6, + activation='mish') + elif name == '6': + model = decoders.YoloDecoder( + input_specs=input_specs, + embed_spp=False, + use_fpn=False, + max_level_process_len=None, + path_process_len=6, + activation='mish') + else: + raise NotImplementedError(f'YOLO decoder test {type} not implemented.') + return model + + @parameterized.parameters('1', '6spp', '6sppfpn', '6') + def test_network_creation(self, version): + """Test creation of ResNet family models.""" + tf.keras.backend.set_image_data_format('channels_last') + input_shape = { + '3': [1, 52, 52, 256], + '4': [1, 26, 26, 512], + '5': [1, 13, 13, 1024] + } + decoder = self._build_yolo_decoder(input_shape, version) + + inputs = {} + for key in input_shape: + inputs[key] = tf.ones(input_shape[key], dtype=tf.float32) + + endpoints = decoder.call(inputs) + + for key in endpoints.keys(): + self.assertAllEqual(endpoints[key].shape.as_list(), input_shape[key]) + + @combinations.generate( + combinations.combine( + strategy=[ + strategy_combinations.cloud_tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + ], + use_sync_bn=[False, True], + )) + def test_sync_bn_multiple_devices(self, strategy, use_sync_bn): + """Test for sync bn on TPU and GPU devices.""" + + tf.keras.backend.set_image_data_format('channels_last') + + with strategy.scope(): + input_shape = { + '3': [1, 52, 52, 256], + '4': [1, 26, 26, 512], + '5': [1, 13, 13, 1024] + } + decoder = self._build_yolo_decoder(input_shape, '6') + + inputs = {} + for key in input_shape: + inputs[key] = tf.ones(input_shape[key], dtype=tf.float32) + + _ = decoder.call(inputs) + + @parameterized.parameters(1, 3, 4) + def test_input_specs(self, input_dim): + """Test different input feature dimensions.""" + tf.keras.backend.set_image_data_format('channels_last') + + input_shape = { + '3': [1, 52, 52, 256], + '4': [1, 26, 26, 512], + '5': [1, 13, 13, 1024] + } + decoder = self._build_yolo_decoder(input_shape, '6') + + inputs = {} + for key in input_shape: + inputs[key] = tf.ones(input_shape[key], dtype=tf.float32) + _ = decoder(inputs) + + def test_serialize_deserialize(self): + """Create a network object that sets all of its config options.""" + tf.keras.backend.set_image_data_format('channels_last') + + input_shape = { + '3': [1, 52, 52, 256], + '4': [1, 26, 26, 512], + '5': [1, 13, 13, 1024] + } + decoder = self._build_yolo_decoder(input_shape, '6') + + inputs = {} + for key in input_shape: + inputs[key] = tf.ones(input_shape[key], dtype=tf.float32) + + _ = decoder(inputs) + config = decoder.get_config() + decoder_from_config = decoders.YoloDecoder.from_config(config) + self.assertAllEqual(decoder.get_config(), decoder_from_config.get_config()) + + +if __name__ == '__main__': + tf.test.main() diff --git a/official/vision/beta/projects/yolo/modeling/heads/__init__.py b/official/vision/beta/projects/yolo/modeling/heads/__init__.py new file mode 100644 index 00000000000..e419af524b5 --- /dev/null +++ b/official/vision/beta/projects/yolo/modeling/heads/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolo_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolo_head.py new file mode 100644 index 00000000000..57c46c28ba1 --- /dev/null +++ b/official/vision/beta/projects/yolo/modeling/heads/yolo_head.py @@ -0,0 +1,122 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Yolo heads.""" + +import tensorflow as tf +from official.vision.beta.projects.yolo.modeling.layers import nn_blocks + + +class YoloHead(tf.keras.layers.Layer): + """YOLO Prediction Head.""" + + def __init__(self, + min_level, + max_level, + classes=80, + boxes_per_level=3, + output_extras=0, + norm_momentum=0.99, + norm_epsilon=0.001, + kernel_initializer='glorot_uniform', + kernel_regularizer=None, + bias_regularizer=None, + activation=None, + **kwargs): + """Yolo Prediction Head initialization function. + + Args: + min_level: `int`, the minimum backbone output level. + max_level: `int`, the maximum backbone output level. + classes: `int`, number of classes per category. + boxes_per_level: `int`, number of boxes to predict per level. + output_extras: `int`, number of additional output channels that the head. + should predict for non-object detection and non-image classification + tasks. + norm_momentum: `float`, normalization momentum for the moving average. + norm_epsilon: `float`, small float added to variance to avoid dividing by + zero. + kernel_initializer: kernel_initializer for convolutional layers. + kernel_regularizer: tf.keras.regularizers.Regularizer object for Conv2D. + bias_regularizer: tf.keras.regularizers.Regularizer object for Conv2d. + activation: `str`, the activation function to use typically leaky or mish. + **kwargs: keyword arguments to be passed. + """ + + super().__init__(**kwargs) + self._min_level = min_level + self._max_level = max_level + + self._key_list = [ + str(key) for key in range(self._min_level, self._max_level + 1) + ] + + self._classes = classes + self._boxes_per_level = boxes_per_level + self._output_extras = output_extras + + self._output_conv = (classes + output_extras + 5) * boxes_per_level + + self._base_config = dict( + activation=activation, + norm_momentum=norm_momentum, + norm_epsilon=norm_epsilon, + kernel_initializer=kernel_initializer, + kernel_regularizer=kernel_regularizer, + bias_regularizer=bias_regularizer) + + self._conv_config = dict( + filters=self._output_conv, + kernel_size=(1, 1), + strides=(1, 1), + padding='same', + use_bn=False, + **self._base_config) + + def build(self, input_shape): + self._head = dict() + for key in self._key_list: + self._head[key] = nn_blocks.ConvBN(**self._conv_config) + + def call(self, inputs): + outputs = dict() + for key in self._key_list: + outputs[key] = self._head[key](inputs[key]) + return outputs + + @property + def output_depth(self): + return (self._classes + self._output_extras + 5) * self._boxes_per_level + + @property + def num_boxes(self): + if self._min_level is None or self._max_level is None: + raise Exception( + 'Model has to be built before number of boxes can be determined.') + return (self._max_level - self._min_level + 1) * self._boxes_per_level + + def get_config(self): + config = dict( + min_level=self._min_level, + max_level=self._max_level, + classes=self._classes, + boxes_per_level=self._boxes_per_level, + output_extras=self._output_extras, + **self._base_config) + return config + + @classmethod + def from_config(cls, config, custom_objects=None): + return cls(**config) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolo_head_test.py b/official/vision/beta/projects/yolo/modeling/heads/yolo_head_test.py new file mode 100644 index 00000000000..8c5414e5d84 --- /dev/null +++ b/official/vision/beta/projects/yolo/modeling/heads/yolo_head_test.py @@ -0,0 +1,74 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Tests for yolo heads.""" + +# Import libraries +from absl.testing import parameterized +import tensorflow as tf + +from official.vision.beta.projects.yolo.modeling.heads import yolo_head as heads + + +class YoloDecoderTest(parameterized.TestCase, tf.test.TestCase): + + def test_network_creation(self): + """Test creation of YOLO family models.""" + tf.keras.backend.set_image_data_format('channels_last') + input_shape = { + '3': [1, 52, 52, 256], + '4': [1, 26, 26, 512], + '5': [1, 13, 13, 1024] + } + classes = 100 + bps = 3 + head = heads.YoloHead(3, 5, classes=classes, boxes_per_level=bps) + + inputs = {} + for key in input_shape: + inputs[key] = tf.ones(input_shape[key], dtype=tf.float32) + + endpoints = head(inputs) + # print(endpoints) + + for key in endpoints.keys(): + expected_input_shape = input_shape[key] + expected_input_shape[-1] = (classes + 5) * bps + self.assertAllEqual(endpoints[key].shape.as_list(), expected_input_shape) + + def test_serialize_deserialize(self): + # Create a network object that sets all of its config options. + tf.keras.backend.set_image_data_format('channels_last') + input_shape = { + '3': [1, 52, 52, 256], + '4': [1, 26, 26, 512], + '5': [1, 13, 13, 1024] + } + classes = 100 + bps = 3 + head = heads.YoloHead(3, 5, classes=classes, boxes_per_level=bps) + + inputs = {} + for key in input_shape: + inputs[key] = tf.ones(input_shape[key], dtype=tf.float32) + + _ = head(inputs) + configs = head.get_config() + head_from_config = heads.YoloHead.from_config(configs) + self.assertAllEqual(head.get_config(), head_from_config.get_config()) + + +if __name__ == '__main__': + tf.test.main() diff --git a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py index 8bc6a78078a..119ddd1c22c 100644 --- a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py +++ b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py @@ -13,81 +13,85 @@ # limitations under the License. # Lint as: python3 - """Contains common building blocks for yolo neural networks.""" from typing import Callable, List import tensorflow as tf from official.modeling import tf_utils +from official.vision.beta.ops import spatial_transform_ops -@tf.keras.utils.register_keras_serializable(package="yolo") +@tf.keras.utils.register_keras_serializable(package='yolo') class Identity(tf.keras.layers.Layer): def call(self, inputs): return inputs -@tf.keras.utils.register_keras_serializable(package="yolo") +@tf.keras.utils.register_keras_serializable(package='yolo') class ConvBN(tf.keras.layers.Layer): - """Modified Convolution layer to match that of the DarkNet Library. + """ConvBN block. + Modified Convolution layer to match that of the Darknet Library. The Layer is a standards combination of Conv BatchNorm Activation, - however, the use of bias in the conv is determined by the use of batch norm. - + however, the use of bias in the conv is determined by the use of batch + normalization. Cross Stage Partial networks (CSPNets) were proposed in: - [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang - Chen, Jun-Wei Hsieh. - CSPNet: A New Backbone that can Enhance Learning Capability of CNN. - arXiv:1911.11929 + [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, + Ping-Yang Chen, Jun-Wei Hsieh + CSPNet: A New Backbone that can Enhance Learning Capability of CNN. + arXiv:1911.11929 """ def __init__(self, filters=1, kernel_size=(1, 1), strides=(1, 1), - padding="same", + padding='same', dilation_rate=(1, 1), - kernel_initializer="glorot_uniform", - bias_initializer="zeros", - kernel_regularizer=None, + kernel_initializer='glorot_uniform', + bias_initializer='zeros', bias_regularizer=None, + kernel_regularizer=None, use_bn=True, use_sync_bn=False, norm_momentum=0.99, norm_epsilon=0.001, - activation="leaky", + activation='leaky', leaky_alpha=0.1, **kwargs): - """Initializes ConvBN layer. + """ConvBN initializer. Args: - filters: integer for output depth, or the number of features to learn + filters: integer for output depth, or the number of features to learn. kernel_size: integer or tuple for the shape of the weight matrix or kernel to learn. strides: integer of tuple how much to move the kernel after each kernel - use padding: string 'valid' or 'same', if same, then pad the image, else - do not. - padding: `str`, padding method for conv layers. + use. + padding: string 'valid' or 'same', if same, then pad the image, else do + not. dilation_rate: tuple to indicate how much to modulate kernel weights and - how many pixels in a feature map to skip. + how many pixels in a feature map to skip. kernel_initializer: string to indicate which function to use to initialize weights. bias_initializer: string to indicate which function to use to initialize bias. - kernel_regularizer: string to indicate which function to use to - regularizer weights. bias_regularizer: string to indicate which function to use to regularizer bias. + kernel_regularizer: string to indicate which function to use to + regularizer weights. use_bn: boolean for whether to use batch normalization. - use_sync_bn: boolean for whether sync batch normalization. - norm_momentum: float for moment to use for batch normalization - norm_epsilon: float for batch normalization epsilon + use_sync_bn: boolean for whether sync batch normalization statistics + of all batch norm layers to the models global statistics + (across all input batches). + norm_momentum: float for moment to use for batch normalization. + norm_epsilon: float for batch normalization epsilon. activation: string or None for activation function to use in layer, - if None activation is replaced by linear. + if None activation is replaced by linear. leaky_alpha: float to use as alpha if activation function is leaky. - **kwargs: Keyword Arguments + **kwargs: Keyword Arguments. """ + # convolution params self._filters = filters self._kernel_size = kernel_size @@ -97,15 +101,16 @@ def __init__(self, self._kernel_initializer = kernel_initializer self._bias_initializer = bias_initializer self._kernel_regularizer = kernel_regularizer + self._bias_regularizer = bias_regularizer # batch normalization params self._use_bn = use_bn self._use_sync_bn = use_sync_bn - self._norm_moment = norm_momentum + self._norm_momentum = norm_momentum self._norm_epsilon = norm_epsilon - if tf.keras.backend.image_data_format() == "channels_last": + if tf.keras.backend.image_data_format() == 'channels_last': # format: (batch_size, height, width, channels) self._bn_axis = -1 else: @@ -116,7 +121,7 @@ def __init__(self, self._activation = activation self._leaky_alpha = leaky_alpha - super(ConvBN, self).__init__(**kwargs) + super().__init__(**kwargs) def build(self, input_shape): use_bias = not self._use_bn @@ -136,101 +141,103 @@ def build(self, input_shape): if self._use_bn: if self._use_sync_bn: self.bn = tf.keras.layers.experimental.SyncBatchNormalization( - momentum=self._norm_moment, + momentum=self._norm_momentum, epsilon=self._norm_epsilon, axis=self._bn_axis) else: self.bn = tf.keras.layers.BatchNormalization( - momentum=self._norm_moment, + momentum=self._norm_momentum, epsilon=self._norm_epsilon, axis=self._bn_axis) - else: - self.bn = Identity() - if self._activation == "leaky": + if self._activation == 'leaky': self._activation_fn = tf.keras.layers.LeakyReLU(alpha=self._leaky_alpha) - elif self._activation == "mish": + elif self._activation == 'mish': self._activation_fn = lambda x: x * tf.math.tanh(tf.math.softplus(x)) else: self._activation_fn = tf_utils.get_activation(self._activation) def call(self, x): x = self.conv(x) - x = self.bn(x) + if self._use_bn: + x = self.bn(x) x = self._activation_fn(x) return x def get_config(self): # used to store/share parameters to reconstruct the model layer_config = { - "filters": self._filters, - "kernel_size": self._kernel_size, - "strides": self._strides, - "padding": self._padding, - "dilation_rate": self._dilation_rate, - "kernel_initializer": self._kernel_initializer, - "bias_initializer": self._bias_initializer, - "bias_regularizer": self._bias_regularizer, - "kernel_regularizer": self._kernel_regularizer, - "use_bn": self._use_bn, - "use_sync_bn": self._use_sync_bn, - "norm_moment": self._norm_moment, - "norm_epsilon": self._norm_epsilon, - "activation": self._activation, - "leaky_alpha": self._leaky_alpha + 'filters': self._filters, + 'kernel_size': self._kernel_size, + 'strides': self._strides, + 'padding': self._padding, + 'dilation_rate': self._dilation_rate, + 'kernel_initializer': self._kernel_initializer, + 'bias_initializer': self._bias_initializer, + 'bias_regularizer': self._bias_regularizer, + 'kernel_regularizer': self._kernel_regularizer, + 'use_bn': self._use_bn, + 'use_sync_bn': self._use_sync_bn, + 'norm_momentum': self._norm_momentum, + 'norm_epsilon': self._norm_epsilon, + 'activation': self._activation, + 'leaky_alpha': self._leaky_alpha } - layer_config.update(super(ConvBN, self).get_config()) + layer_config.update(super().get_config()) return layer_config - def __repr__(self): - return repr(self.get_config()) - -@tf.keras.utils.register_keras_serializable(package="yolo") +@tf.keras.utils.register_keras_serializable(package='yolo') class DarkResidual(tf.keras.layers.Layer): - """DarkNet block with Residual connection for Yolo v3 Backbone. - """ + """Darknet block with Residual connection for Yolo v3 Backbone.""" def __init__(self, filters=1, filter_scale=2, - kernel_initializer="glorot_uniform", - bias_initializer="zeros", + dilation_rate=1, + kernel_initializer='glorot_uniform', + bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, use_bn=True, use_sync_bn=False, norm_momentum=0.99, norm_epsilon=0.001, - activation="leaky", + activation='leaky', leaky_alpha=0.1, - sc_activation="linear", + sc_activation='linear', downsample=False, **kwargs): - """Initializes DarkResidual. + """Dark Residual initializer. Args: filters: integer for output depth, or the number of features to learn. - filter_scale: `int`, scale factor for number of filters. + filter_scale: `int` for filter scale. + dilation_rate: tuple to indicate how much to modulate kernel weights and + how many pixels in a feature map to skip. kernel_initializer: string to indicate which function to use to initialize - weights + weights. bias_initializer: string to indicate which function to use to initialize - bias + bias. kernel_regularizer: string to indicate which function to use to - regularizer weights + regularizer weights. bias_regularizer: string to indicate which function to use to regularizer - bias - use_bn: boolean for whether to use batch normalization - use_sync_bn: boolean for whether sync batch normalization. - norm_momentum: float for moment to use for batch normalization - norm_epsilon: float for batch normalization epsilon - activation: string for activation function to use in conv layers. - leaky_alpha: float to use as alpha if activation function is leaky - sc_activation: string for activation function to use in layer + bias. + use_bn: boolean for whether to use batch normalization. + use_sync_bn: boolean for whether sync batch normalization statistics. + of all batch norm layers to the models global statistics + (across all input batches). + norm_momentum: float for moment to use for batch normalization. + norm_epsilon: float for batch normalization epsilon. + activation: string or None for activation function to use in layer, + if None activation is replaced by linear. + leaky_alpha: float to use as alpha if activation function is leaky. + sc_activation: string for activation function to use in layer. downsample: boolean for if image input is larger than layer output, set - downsample to True so the dimensions are forced to match - **kwargs: Keyword Arguments + downsample to True so the dimensions are forced to match. + **kwargs: Keyword Arguments. """ + # downsample self._downsample = downsample @@ -245,8 +252,10 @@ def __init__(self, self._kernel_regularizer = kernel_regularizer # normal params - self._norm_moment = norm_momentum + self._norm_momentum = norm_momentum self._norm_epsilon = norm_epsilon + self._dilation_rate = dilation_rate if isinstance(dilation_rate, + int) else dilation_rate[0] # activation params self._conv_activation = activation @@ -256,138 +265,152 @@ def __init__(self, super().__init__(**kwargs) def build(self, input_shape): - self._dark_conv_args = { - "kernel_initializer": self._kernel_initializer, - "bias_initializer": self._bias_initializer, - "bias_regularizer": self._bias_regularizer, - "use_bn": self._use_bn, - "use_sync_bn": self._use_sync_bn, - "norm_momentum": self._norm_moment, - "norm_epsilon": self._norm_epsilon, - "activation": self._conv_activation, - "kernel_regularizer": self._kernel_regularizer, - "leaky_alpha": self._leaky_alpha + dark_conv_args = { + 'kernel_initializer': self._kernel_initializer, + 'bias_initializer': self._bias_initializer, + 'bias_regularizer': self._bias_regularizer, + 'use_bn': self._use_bn, + 'use_sync_bn': self._use_sync_bn, + 'norm_momentum': self._norm_momentum, + 'norm_epsilon': self._norm_epsilon, + 'activation': self._conv_activation, + 'kernel_regularizer': self._kernel_regularizer, + 'leaky_alpha': self._leaky_alpha } if self._downsample: + if self._dilation_rate > 1: + dilation_rate = 1 + if self._dilation_rate // 2 > 0: + dilation_rate = self._dilation_rate // 2 + down_stride = 1 + else: + dilation_rate = 1 + down_stride = 2 + self._dconv = ConvBN( filters=self._filters, kernel_size=(3, 3), - strides=(2, 2), - padding="same", - **self._dark_conv_args) - else: - self._dconv = Identity() + strides=down_stride, + dilation_rate=dilation_rate, + padding='same', + **dark_conv_args) self._conv1 = ConvBN( filters=self._filters // self._filter_scale, kernel_size=(1, 1), strides=(1, 1), - padding="same", - **self._dark_conv_args) + padding='same', + **dark_conv_args) self._conv2 = ConvBN( filters=self._filters, kernel_size=(3, 3), strides=(1, 1), - padding="same", - **self._dark_conv_args) + dilation_rate=self._dilation_rate, + padding='same', + **dark_conv_args) self._shortcut = tf.keras.layers.Add() - if self._sc_activation == "leaky": - self._activation_fn = tf.keras.layers.LeakyReLU( - alpha=self._leaky_alpha) - elif self._sc_activation == "mish": + if self._sc_activation == 'leaky': + self._activation_fn = tf.keras.layers.LeakyReLU(alpha=self._leaky_alpha) + elif self._sc_activation == 'mish': self._activation_fn = lambda x: x * tf.math.tanh(tf.math.softplus(x)) else: - self._activation_fn = tf_utils.get_activation(self._sc_activation) + self._activation_fn = tf_utils.get_activation( + self._sc_activation + ) super().build(input_shape) - def call(self, inputs): - shortcut = self._dconv(inputs) - x = self._conv1(shortcut) + def call(self, inputs, training=None): + if self._downsample: + inputs = self._dconv(inputs) + x = self._conv1(inputs) x = self._conv2(x) - x = self._shortcut([x, shortcut]) + x = self._shortcut([x, inputs]) return self._activation_fn(x) def get_config(self): # used to store/share parameters to reconstruct the model layer_config = { - "filters": self._filters, - "kernel_initializer": self._kernel_initializer, - "bias_initializer": self._bias_initializer, - "kernel_regularizer": self._kernel_regularizer, - "use_bn": self._use_bn, - "use_sync_bn": self._use_sync_bn, - "norm_moment": self._norm_moment, - "norm_epsilon": self._norm_epsilon, - "activation": self._conv_activation, - "leaky_alpha": self._leaky_alpha, - "sc_activation": self._sc_activation, - "downsample": self._downsample + 'filters': self._filters, + 'kernel_initializer': self._kernel_initializer, + 'bias_initializer': self._bias_initializer, + 'kernel_regularizer': self._kernel_regularizer, + 'dilation_rate': self._dilation_rate, + 'use_bn': self._use_bn, + 'use_sync_bn': self._use_sync_bn, + 'norm_momentum': self._norm_momentum, + 'norm_epsilon': self._norm_epsilon, + 'activation': self._conv_activation, + 'leaky_alpha': self._leaky_alpha, + 'sc_activation': self._sc_activation, + 'downsample': self._downsample, } layer_config.update(super().get_config()) return layer_config -@tf.keras.utils.register_keras_serializable(package="yolo") +@tf.keras.utils.register_keras_serializable(package='yolo') class CSPTiny(tf.keras.layers.Layer): - """A Small size convolution block proposed in the CSPNet. - - The layer uses shortcuts, routing(concatnation), and feature grouping - in order to improve gradient variablity and allow for high efficency, low - power residual learning for small networtf.keras. + """CSP Tiny layer. + A Small size convolution block proposed in the CSPNet. The layer uses + shortcuts, routing(concatnation), and feature grouping in order to improve + gradient variablity and allow for high efficency, low power residual learning + for small networtf.keras. Cross Stage Partial networks (CSPNets) were proposed in: - [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang - Chen, Jun-Wei Hsieh + [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, + Ping-Yang Chen, Jun-Wei Hsieh CSPNet: A New Backbone that can Enhance Learning Capability of CNN. - arXiv:1911.11929 + arXiv:1911.11929 """ def __init__(self, filters=1, - kernel_initializer="glorot_uniform", - bias_initializer="zeros", - kernel_regularizer=None, + kernel_initializer='glorot_uniform', + bias_initializer='zeros', bias_regularizer=None, + kernel_regularizer=None, use_bn=True, + dilation_rate=1, use_sync_bn=False, group_id=1, groups=2, norm_momentum=0.99, norm_epsilon=0.001, - activation="leaky", + activation='leaky', downsample=True, leaky_alpha=0.1, **kwargs): - """Initializes CSPTiny. + """Initializer for CSPTiny block. Args: - filters: integer for output depth, or the number of features to learn + filters: integer for output depth, or the number of features to learn. kernel_initializer: string to indicate which function to use to initialize - weights + weights. bias_initializer: string to indicate which function to use to initialize - bias - kernel_regularizer: string to indicate which function to use to - regularizer weights + bias. bias_regularizer: string to indicate which function to use to regularizer - bias - use_bn: boolean for whether to use batch normalization - use_sync_bn: boolean for whether sync batch normalization statistics of - all batch norm layers to the models global statistics (across all input - batches) - group_id: integer for which group of features to pass through the csp tiny - stack. + bias. + kernel_regularizer: string to indicate which function to use to + regularizer weights. + use_bn: boolean for whether to use batch normalization. + dilation_rate: `int`, dilation rate for conv layers. + use_sync_bn: boolean for whether sync batch normalization statistics + of all batch norm layers to the models global statistics + (across all input batches). + group_id: integer for which group of features to pass through the csp + tiny stack. groups: integer for how many splits there should be in the convolution - feature stack output - norm_momentum: float for moment to use for batch normalization - norm_epsilon: float for batch normalization epsilon + feature stack output. + norm_momentum: float for moment to use for batch normalization. + norm_epsilon: float for batch normalization epsilon. activation: string or None for activation function to use in layer, - if None activation is replaced by linear + if None activation is replaced by linear. downsample: boolean for if image input is larger than layer output, set - downsample to True so the dimensions are forced to match - leaky_alpha: float to use as alpha if activation function is leaky - **kwargs: Keyword Arguments + downsample to True so the dimensions are forced to match. + leaky_alpha: float to use as alpha if activation function is leaky. + **kwargs: Keyword Arguments. """ # ConvBN params @@ -396,6 +419,7 @@ def __init__(self, self._bias_initializer = bias_initializer self._bias_regularizer = bias_regularizer self._use_bn = use_bn + self._dilation_rate = dilation_rate self._use_sync_bn = use_sync_bn self._kernel_regularizer = kernel_regularizer self._groups = groups @@ -403,7 +427,7 @@ def __init__(self, self._downsample = downsample # normal params - self._norm_moment = norm_momentum + self._norm_momentum = norm_momentum self._norm_epsilon = norm_epsilon # activation params @@ -413,37 +437,37 @@ def __init__(self, super().__init__(**kwargs) def build(self, input_shape): - self._dark_conv_args = { - "kernel_initializer": self._kernel_initializer, - "bias_initializer": self._bias_initializer, - "bias_regularizer": self._bias_regularizer, - "use_bn": self._use_bn, - "use_sync_bn": self._use_sync_bn, - "norm_momentum": self._norm_moment, - "norm_epsilon": self._norm_epsilon, - "activation": self._conv_activation, - "kernel_regularizer": self._kernel_regularizer, - "leaky_alpha": self._leaky_alpha + dark_conv_args = { + 'kernel_initializer': self._kernel_initializer, + 'bias_initializer': self._bias_initializer, + 'bias_regularizer': self._bias_regularizer, + 'use_bn': self._use_bn, + 'use_sync_bn': self._use_sync_bn, + 'norm_momentum': self._norm_momentum, + 'norm_epsilon': self._norm_epsilon, + 'activation': self._conv_activation, + 'kernel_regularizer': self._kernel_regularizer, + 'leaky_alpha': self._leaky_alpha } self._convlayer1 = ConvBN( filters=self._filters, kernel_size=(3, 3), strides=(1, 1), - padding="same", - **self._dark_conv_args) + padding='same', + **dark_conv_args) self._convlayer2 = ConvBN( filters=self._filters // 2, kernel_size=(3, 3), strides=(1, 1), - padding="same", + padding='same', kernel_initializer=self._kernel_initializer, bias_initializer=self._bias_initializer, bias_regularizer=self._bias_regularizer, kernel_regularizer=self._kernel_regularizer, use_bn=self._use_bn, use_sync_bn=self._use_sync_bn, - norm_momentum=self._norm_moment, + norm_momentum=self._norm_momentum, norm_epsilon=self._norm_epsilon, activation=self._conv_activation, leaky_alpha=self._leaky_alpha) @@ -452,22 +476,23 @@ def build(self, input_shape): filters=self._filters // 2, kernel_size=(3, 3), strides=(1, 1), - padding="same", - **self._dark_conv_args) + padding='same', + **dark_conv_args) self._convlayer4 = ConvBN( filters=self._filters, kernel_size=(1, 1), strides=(1, 1), - padding="same", - **self._dark_conv_args) + padding='same', + **dark_conv_args) - self._maxpool = tf.keras.layers.MaxPool2D( - pool_size=2, strides=2, padding="same", data_format=None) + if self._downsample: + self._maxpool = tf.keras.layers.MaxPool2D( + pool_size=2, strides=2, padding='same', data_format=None) super().build(input_shape) - def call(self, inputs): + def call(self, inputs, training=None): x1 = self._convlayer1(inputs) x1_group = tf.split(x1, self._groups, axis=-1)[self._group_id] x2 = self._convlayer2(x1_group) # grouping @@ -479,276 +504,303 @@ def call(self, inputs): x = self._maxpool(x) return x, x5 - def get_config(self): - # used to store/share parameters to reconsturct the model - layer_config = { - "filters": self._filters, - "strides": self._strides, - "kernel_initializer": self._kernel_initializer, - "bias_initializer": self._bias_initializer, - "kernel_regularizer": self._kernel_regularizer, - "use_bn": self._use_bn, - "use_sync_bn": self._use_sync_bn, - "norm_moment": self._norm_moment, - "norm_epsilon": self._norm_epsilon, - "activation": self._conv_activation, - "leaky_alpha": self._leaky_alpha, - "sc_activation": self._sc_activation, - } - layer_config.update(super().get_config()) - return layer_config - -@tf.keras.utils.register_keras_serializable(package="yolo") +@tf.keras.utils.register_keras_serializable(package='yolo') class CSPRoute(tf.keras.layers.Layer): - """Down sampling layer to take the place of down sampleing. - - It is applied in Residual networks. This is the first of 2 layers needed to - convert any Residual Network model to a CSPNet. At the start of a new level - change, this CSPRoute layer creates a learned identity that will act as a - cross stage connection, that is used to inform the inputs to the next stage. - It is called cross stage partial because the number of filters required in - every intermitent Residual layer is reduced by half. The sister layer will - take the partial generated by this layer and concatnate it with the output of - the final residual layer in the stack to create a fully feature level output. - This concatnation merges the partial blocks of 2 levels as input to the next - allowing the gradients of each level to be more unique, and reducing the - number of parameters required by each level by 50% while keeping accuracy - consistent. + """CSPRoute block. + + Down sampling layer to take the place of down sampleing done in Residual + networks. This is the first of 2 layers needed to convert any Residual Network + model to a CSPNet. At the start of a new level change, this CSPRoute layer + creates a learned identity that will act as a cross stage connection, + that is used to inform the inputs to the next stage. It is called cross stage + partial because the number of filters required in every intermitent Residual + layer is reduced by half. The sister layer will take the partial generated by + this layer and concatnate it with the output of the final residual layer in + the stack to create a fully feature level output. This concatnation merges the + partial blocks of 2 levels as input to the next allowing the gradients of each + level to be more unique, and reducing the number of parameters required by + each level by 50% while keeping accuracy consistent. Cross Stage Partial networks (CSPNets) were proposed in: - [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang - Chen, Jun-Wei Hsieh. + [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, + Ping-Yang Chen, Jun-Wei Hsieh CSPNet: A New Backbone that can Enhance Learning Capability of CNN. - arXiv:1911.11929 + arXiv:1911.11929 """ def __init__(self, filters, filter_scale=2, - activation="mish", - downsample=True, - kernel_initializer="glorot_uniform", - bias_initializer="zeros", - kernel_regularizer=None, + activation='mish', + kernel_initializer='glorot_uniform', + bias_initializer='zeros', bias_regularizer=None, + kernel_regularizer=None, + dilation_rate=1, use_bn=True, use_sync_bn=False, norm_momentum=0.99, norm_epsilon=0.001, + downsample=True, + leaky_alpha=0.1, **kwargs): - """Initializes CSPRoute. + """CSPRoute layer initializer. Args: filters: integer for output depth, or the number of features to learn filter_scale: integer dicating (filters//2) or the number of filters in the partial feature stack. - activation: string for activation function to use in layer - downsample: down_sample the input. - kernel_initializer: string to indicate which function to use to initialize - weights. + activation: string for activation function to use in layer. + kernel_initializer: string to indicate which function to use to + initialize weights. bias_initializer: string to indicate which function to use to initialize bias. - kernel_regularizer: string to indicate which function to use to - regularizer weights. bias_regularizer: string to indicate which function to use to regularizer bias. + kernel_regularizer: string to indicate which function to use to + regularizer weights. + dilation_rate: dilation rate for conv layers. use_bn: boolean for whether to use batch normalization. - use_sync_bn: boolean for whether sync batch normalization. - norm_momentum: float for moment to use for batch normalization - norm_epsilon: float for batch normalization epsilon - **kwargs: Keyword Arguments + use_sync_bn: boolean for whether sync batch normalization statistics + of all batch norm layers to the models global statistics + (across all input batches). + norm_momentum: float for moment to use for batch normalization. + norm_epsilon: float for batch normalization epsilon. + downsample: down_sample the input. + leaky_alpha: `float`, for leaky alpha value. + **kwargs: Keyword Arguments. """ super().__init__(**kwargs) - # Layer params. + # layer params self._filters = filters self._filter_scale = filter_scale self._activation = activation - # Convoultion params. + # convoultion params self._kernel_initializer = kernel_initializer self._bias_initializer = bias_initializer self._kernel_regularizer = kernel_regularizer self._bias_regularizer = bias_regularizer + self._dilation_rate = dilation_rate self._use_bn = use_bn self._use_sync_bn = use_sync_bn - self._norm_moment = norm_momentum + self._norm_momentum = norm_momentum self._norm_epsilon = norm_epsilon self._downsample = downsample + self._leaky_alpha = leaky_alpha def build(self, input_shape): - self._dark_conv_args = { - "kernel_initializer": self._kernel_initializer, - "bias_initializer": self._bias_initializer, - "bias_regularizer": self._bias_regularizer, - "use_bn": self._use_bn, - "use_sync_bn": self._use_sync_bn, - "norm_momentum": self._norm_moment, - "norm_epsilon": self._norm_epsilon, - "activation": self._activation, - "kernel_regularizer": self._kernel_regularizer, + dark_conv_args = { + 'kernel_initializer': self._kernel_initializer, + 'bias_initializer': self._bias_initializer, + 'bias_regularizer': self._bias_regularizer, + 'use_bn': self._use_bn, + 'use_sync_bn': self._use_sync_bn, + 'norm_momentum': self._norm_momentum, + 'norm_epsilon': self._norm_epsilon, + 'activation': self._activation, + 'kernel_regularizer': self._kernel_regularizer, + 'leaky_alpha': self._leaky_alpha, } if self._downsample: - self._conv1 = ConvBN(filters=self._filters, - kernel_size=(3, 3), - strides=(2, 2), - **self._dark_conv_args) - else: - self._conv1 = ConvBN(filters=self._filters, - kernel_size=(3, 3), - strides=(1, 1), - **self._dark_conv_args) - self._conv2 = ConvBN(filters=self._filters // self._filter_scale, - kernel_size=(1, 1), - strides=(1, 1), - **self._dark_conv_args) - - self._conv3 = ConvBN(filters=self._filters // self._filter_scale, - kernel_size=(1, 1), - strides=(1, 1), - **self._dark_conv_args) + if self._dilation_rate > 1: + dilation_rate = 1 + if self._dilation_rate // 2 > 0: + dilation_rate = self._dilation_rate // 2 + down_stride = 1 + else: + dilation_rate = 1 + down_stride = 2 - def call(self, inputs): - x = self._conv1(inputs) - y = self._conv2(x) - x = self._conv3(x) + self._conv1 = ConvBN( + filters=self._filters, + kernel_size=(3, 3), + strides=down_stride, + dilation_rate=dilation_rate, + **dark_conv_args) + + self._conv2 = ConvBN( + filters=self._filters // self._filter_scale, + kernel_size=(1, 1), + strides=(1, 1), + **dark_conv_args) + + self._conv3 = ConvBN( + filters=self._filters // self._filter_scale, + kernel_size=(1, 1), + strides=(1, 1), + **dark_conv_args) + + def call(self, inputs, training=None): + if self._downsample: + inputs = self._conv1(inputs) + y = self._conv2(inputs) + x = self._conv3(inputs) return (x, y) -@tf.keras.utils.register_keras_serializable(package="yolo") +@tf.keras.utils.register_keras_serializable(package='yolo') class CSPConnect(tf.keras.layers.Layer): - """Sister Layer to the CSPRoute layer. - - Merges the partial feature stacks generated by the CSPDownsampling layer, - and the finaly output of the residual stack. Suggested in the CSPNet paper. + """CSPConnect block. + Sister Layer to the CSPRoute layer. Merges the partial feature stacks + generated by the CSPDownsampling layer, and the finaly output of the + residual stack. Suggested in the CSPNet paper. Cross Stage Partial networks (CSPNets) were proposed in: - [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang - Chen, Jun-Wei Hsieh. + [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, + Ping-Yang Chen, Jun-Wei Hsieh CSPNet: A New Backbone that can Enhance Learning Capability of CNN. - arXiv:1911.11929 + arXiv:1911.11929 """ def __init__(self, filters, filter_scale=2, - activation="mish", - kernel_initializer="glorot_uniform", - bias_initializer="zeros", - kernel_regularizer=None, + drop_final=False, + drop_first=False, + activation='mish', + kernel_size=(1, 1), + kernel_initializer='glorot_uniform', + bias_initializer='zeros', bias_regularizer=None, + kernel_regularizer=None, + dilation_rate=1, use_bn=True, use_sync_bn=False, norm_momentum=0.99, norm_epsilon=0.001, + leaky_alpha=0.1, **kwargs): - """Initializes CSPConnect. + """Initializer for CSPConnect block. Args: - filters: integer for output depth, or the number of features to learn. + filters: integer for output depth, or the number of features to learn filter_scale: integer dicating (filters//2) or the number of filters in the partial feature stack. + drop_final: `bool`, whether to drop final conv layer. + drop_first: `bool`, whether to drop first conv layer. activation: string for activation function to use in layer. + kernel_size: `Tuple`, kernel size for conv layers. kernel_initializer: string to indicate which function to use to initialize weights. bias_initializer: string to indicate which function to use to initialize bias. - kernel_regularizer: string to indicate which function to use to - regularizer weights. bias_regularizer: string to indicate which function to use to regularizer bias. + kernel_regularizer: string to indicate which function to use to + regularizer weights. + dilation_rate: `int`, dilation rate for conv layers. use_bn: boolean for whether to use batch normalization. - use_sync_bn: boolean for whether sync batch normalization. - norm_momentum: float for moment to use for batch normalization - norm_epsilon: float for batch normalization epsilon - **kwargs: Keyword Arguments + use_sync_bn: boolean for whether sync batch normalization statistics + of all batch norm layers to the models global + statistics (across all input batches). + norm_momentum: float for moment to use for batch normalization. + norm_epsilon: float for batch normalization epsilon. + leaky_alpha: `float`, for leaky alpha value. + **kwargs: Keyword Arguments. """ + super().__init__(**kwargs) - # layer params. + # layer params self._filters = filters self._filter_scale = filter_scale self._activation = activation - # Convoultion params. + # convoultion params + self._kernel_size = kernel_size self._kernel_initializer = kernel_initializer self._bias_initializer = bias_initializer self._kernel_regularizer = kernel_regularizer self._bias_regularizer = bias_regularizer self._use_bn = use_bn self._use_sync_bn = use_sync_bn - self._norm_moment = norm_momentum + self._norm_momentum = norm_momentum self._norm_epsilon = norm_epsilon + self._drop_final = drop_final + self._drop_first = drop_first + self._leaky_alpha = leaky_alpha def build(self, input_shape): - self._dark_conv_args = { - "kernel_initializer": self._kernel_initializer, - "bias_initializer": self._bias_initializer, - "bias_regularizer": self._bias_regularizer, - "use_bn": self._use_bn, - "use_sync_bn": self._use_sync_bn, - "norm_momentum": self._norm_moment, - "norm_epsilon": self._norm_epsilon, - "activation": self._activation, - "kernel_regularizer": self._kernel_regularizer, + dark_conv_args = { + 'kernel_initializer': self._kernel_initializer, + 'bias_initializer': self._bias_initializer, + 'bias_regularizer': self._bias_regularizer, + 'use_bn': self._use_bn, + 'use_sync_bn': self._use_sync_bn, + 'norm_momentum': self._norm_momentum, + 'norm_epsilon': self._norm_epsilon, + 'activation': self._activation, + 'kernel_regularizer': self._kernel_regularizer, + 'leaky_alpha': self._leaky_alpha, } - self._conv1 = ConvBN(filters=self._filters // self._filter_scale, - kernel_size=(1, 1), - strides=(1, 1), - **self._dark_conv_args) + if not self._drop_first: + self._conv1 = ConvBN( + filters=self._filters // self._filter_scale, + kernel_size=self._kernel_size, + strides=(1, 1), + **dark_conv_args) self._concat = tf.keras.layers.Concatenate(axis=-1) - self._conv2 = ConvBN(filters=self._filters, - kernel_size=(1, 1), - strides=(1, 1), - **self._dark_conv_args) - def call(self, inputs): + if not self._drop_final: + self._conv2 = ConvBN( + filters=self._filters, + kernel_size=(1, 1), + strides=(1, 1), + **dark_conv_args) + + def call(self, inputs, training=None): x_prev, x_csp = inputs - x = self._conv1(x_prev) - x = self._concat([x, x_csp]) - x = self._conv2(x) + if not self._drop_first: + x_prev = self._conv1(x_prev) + x = self._concat([x_prev, x_csp]) + + # skipped if drop final is true + if not self._drop_final: + x = self._conv2(x) return x class CSPStack(tf.keras.layers.Layer): - """CSP full stack. - - Combines the route and the connect in case you dont want to just quickly wrap - an existing callable or list of layers to make it a cross stage partial. - Added for ease of use. you should be able to wrap any layer stack with a CSP - independent of wether it belongs to the Darknet family. if filter_scale = 2, - then the blocks in the stack passed into the the CSP stack should also have - filters = filters/filter_scale. - + """CSP Stack layer. + + CSP full stack, combines the route and the connect in case you dont want to + jsut quickly wrap an existing callable or list of layers to + make it a cross stage partial. Added for ease of use. you should be able + to wrap any layer stack with a CSP independent of wether it belongs + to the Darknet family. if filter_scale = 2, then the blocks in the stack + passed into the the CSP stack should also have filters = filters/filter_scale Cross Stage Partial networks (CSPNets) were proposed in: - [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang - Chen, Jun-Wei Hsieh + + [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, + Ping-Yang Chen, Jun-Wei Hsieh CSPNet: A New Backbone that can Enhance Learning Capability of CNN. - arXiv:1911.11929 + arXiv:1911.11929 """ def __init__(self, filters, model_to_wrap=None, filter_scale=2, - activation="mish", - kernel_initializer="glorot_uniform", - bias_initializer="zeros", - kernel_regularizer=None, + activation='mish', + kernel_initializer='glorot_uniform', + bias_initializer='zeros', bias_regularizer=None, + kernel_regularizer=None, downsample=True, use_bn=True, use_sync_bn=False, norm_momentum=0.99, norm_epsilon=0.001, **kwargs): - """Initializes CSPStack. + """CSPStack layer initializer. Args: filters: integer for output depth, or the number of features to learn. model_to_wrap: callable Model or a list of callable objects that will - process the output of CSPRoute, and be input into CSPConnect. List will - be called sequentially. + process the output of CSPRoute, and be input into CSPConnect. + list will be called sequentially. filter_scale: integer dicating (filters//2) or the number of filters in the partial feature stack. activation: string for activation function to use in layer. @@ -756,66 +808,829 @@ def __init__(self, weights. bias_initializer: string to indicate which function to use to initialize bias. - kernel_regularizer: string to indicate which function to use to - regularizer weights. bias_regularizer: string to indicate which function to use to regularizer bias. + kernel_regularizer: string to indicate which function to use to + regularizer weights. downsample: down_sample the input. - use_bn: boolean for whether to use batch normalization - use_sync_bn: boolean for whether sync batch normalization. - norm_momentum: float for moment to use for batch normalization - norm_epsilon: float for batch normalization epsilon - **kwargs: Keyword Arguments + use_bn: boolean for whether to use batch normalization. + use_sync_bn: boolean for whether sync batch normalization statistics + of all batch norm layers to the models global statistics + (across all input batches). + norm_momentum: float for moment to use for batch normalization. + norm_epsilon: float for batch normalization epsilon. + **kwargs: Keyword Arguments. + + Raises: + TypeError: model_to_wrap is not a layer or a list of layers """ + super().__init__(**kwargs) - # Layer params. + # layer params self._filters = filters self._filter_scale = filter_scale self._activation = activation self._downsample = downsample - # Convoultion params. + # convoultion params self._kernel_initializer = kernel_initializer self._bias_initializer = bias_initializer self._kernel_regularizer = kernel_regularizer self._bias_regularizer = bias_regularizer self._use_bn = use_bn self._use_sync_bn = use_sync_bn - self._norm_moment = norm_momentum + self._norm_momentum = norm_momentum self._norm_epsilon = norm_epsilon - if model_to_wrap is not None: - if isinstance(model_to_wrap, Callable): - self._model_to_wrap = [model_to_wrap] - elif isinstance(model_to_wrap, List): - self._model_to_wrap = model_to_wrap - else: - raise ValueError("The input to the CSPStack must be a list of layers" - "that we can iterate through, or \n a callable") - else: + if model_to_wrap is None: self._model_to_wrap = [] + elif isinstance(model_to_wrap, Callable): + self._model_to_wrap = [model_to_wrap] + elif isinstance(model_to_wrap, List): + self._model_to_wrap = model_to_wrap + else: + raise TypeError( + 'the input to the CSPStack must be a list of layers that we can' + + 'iterate through, or \n a callable') def build(self, input_shape): - self._dark_conv_args = { - "filters": self._filters, - "filter_scale": self._filter_scale, - "activation": self._activation, - "kernel_initializer": self._kernel_initializer, - "bias_initializer": self._bias_initializer, - "bias_regularizer": self._bias_regularizer, - "use_bn": self._use_bn, - "use_sync_bn": self._use_sync_bn, - "norm_momentum": self._norm_moment, - "norm_epsilon": self._norm_epsilon, - "kernel_regularizer": self._kernel_regularizer, + dark_conv_args = { + 'filters': self._filters, + 'filter_scale': self._filter_scale, + 'activation': self._activation, + 'kernel_initializer': self._kernel_initializer, + 'bias_initializer': self._bias_initializer, + 'bias_regularizer': self._bias_regularizer, + 'use_bn': self._use_bn, + 'use_sync_bn': self._use_sync_bn, + 'norm_momentum': self._norm_momentum, + 'norm_epsilon': self._norm_epsilon, + 'kernel_regularizer': self._kernel_regularizer, } - self._route = CSPRoute(downsample=self._downsample, **self._dark_conv_args) - self._connect = CSPConnect(**self._dark_conv_args) - return + self._route = CSPRoute(downsample=self._downsample, **dark_conv_args) + self._connect = CSPConnect(**dark_conv_args) - def call(self, inputs): + def call(self, inputs, training=None): x, x_route = self._route(inputs) for layer in self._model_to_wrap: x = layer(x) x = self._connect([x, x_route]) return x + + +@tf.keras.utils.register_keras_serializable(package='yolo') +class PathAggregationBlock(tf.keras.layers.Layer): + """Path Aggregation block.""" + + def __init__(self, + filters=1, + drop_final=True, + kernel_initializer='glorot_uniform', + bias_initializer='zeros', + bias_regularizer=None, + kernel_regularizer=None, + use_bn=True, + use_sync_bn=False, + inverted=False, + norm_momentum=0.99, + norm_epsilon=0.001, + activation='leaky', + leaky_alpha=0.1, + downsample=False, + upsample=False, + upsample_size=2, + **kwargs): + """Initializer for path aggregation block. + + Args: + filters: integer for output depth, or the number of features to learn. + drop_final: do not create the last convolution block. + kernel_initializer: string to indicate which function to use to initialize + weights. + bias_initializer: string to indicate which function to use to initialize + bias. + bias_regularizer: string to indicate which function to use to regularizer + bias. + kernel_regularizer: string to indicate which function to use to + regularizer weights. + use_bn: boolean for whether to use batch normalization. + use_sync_bn: boolean for whether sync batch normalization statistics + of all batch norm layers to the models global statistics + (across all input batches). + inverted: boolean for inverting the order of the convolutions. + norm_momentum: float for moment to use for batch normalization. + norm_epsilon: float for batch normalization epsilon. + activation: string or None for activation function to use in layer, + if None activation is replaced by linear. + leaky_alpha: float to use as alpha if activation function is leaky. + downsample: `bool` for whehter to downwample and merge. + upsample: `bool` for whehter to upsample and merge. + upsample_size: `int` how much to upsample in order to match shapes. + **kwargs: Keyword Arguments. + """ + + # Darkconv params + self._filters = filters + self._kernel_initializer = kernel_initializer + self._bias_initializer = bias_initializer + self._bias_regularizer = bias_regularizer + self._kernel_regularizer = kernel_regularizer + self._use_bn = use_bn + self._use_sync_bn = use_sync_bn + + # Normal params + self._norm_momentum = norm_momentum + self._norm_epsilon = norm_epsilon + + # Activation params + self._conv_activation = activation + self._leaky_alpha = leaky_alpha + self._downsample = downsample + self._upsample = upsample + self._upsample_size = upsample_size + self._drop_final = drop_final + + # Block params + self._inverted = inverted + + super().__init__(**kwargs) + + def _build_regular(self, input_shape, kwargs): + if self._downsample: + self._conv = ConvBN( + filters=self._filters, + kernel_size=(3, 3), + strides=(2, 2), + padding='same', + **kwargs) + else: + self._conv = ConvBN( + filters=self._filters, + kernel_size=(1, 1), + strides=(1, 1), + padding='same', + **kwargs) + + if not self._drop_final: + self._conv_concat = ConvBN( + filters=self._filters, + kernel_size=(1, 1), + strides=(1, 1), + padding='same', + **kwargs) + + def _build_reversed(self, input_shape, kwargs): + if self._downsample: + self._conv_prev = ConvBN( + filters=self._filters, + kernel_size=(3, 3), + strides=(2, 2), + padding='same', + **kwargs) + else: + self._conv_prev = ConvBN( + filters=self._filters, + kernel_size=(1, 1), + strides=(1, 1), + padding='same', + **kwargs) + + self._conv_route = ConvBN( + filters=self._filters, + kernel_size=(1, 1), + strides=(1, 1), + padding='same', + **kwargs) + + if not self._drop_final: + self._conv_sync = ConvBN( + filters=self._filters, + kernel_size=(1, 1), + strides=(1, 1), + padding='same', + **kwargs) + + def build(self, input_shape): + dark_conv_args = { + 'kernel_initializer': self._kernel_initializer, + 'bias_initializer': self._bias_initializer, + 'bias_regularizer': self._bias_regularizer, + 'use_bn': self._use_bn, + 'use_sync_bn': self._use_sync_bn, + 'norm_momentum': self._norm_momentum, + 'norm_epsilon': self._norm_epsilon, + 'activation': self._conv_activation, + 'kernel_regularizer': self._kernel_regularizer, + 'leaky_alpha': self._leaky_alpha, + } + + if self._inverted: + self._build_reversed(input_shape, dark_conv_args) + else: + self._build_regular(input_shape, dark_conv_args) + + self._concat = tf.keras.layers.Concatenate() + super().build(input_shape) + + def _call_regular(self, inputs, training=None): + input_to_convolve, input_to_concat = inputs + x_prev = self._conv(input_to_convolve) + if self._upsample: + x_prev = spatial_transform_ops.nearest_upsampling(x_prev, + self._upsample_size) + x = self._concat([x_prev, input_to_concat]) + + # used in csp conversion + if not self._drop_final: + x = self._conv_concat(x) + return x_prev, x + + def _call_reversed(self, inputs, training=None): + x_route, x_prev = inputs + x_prev = self._conv_prev(x_prev) + if self._upsample: + x_prev = spatial_transform_ops.nearest_upsampling(x_prev, + self._upsample_size) + x_route = self._conv_route(x_route) + x = self._concat([x_route, x_prev]) + if not self._drop_final: + x = self._conv_sync(x) + return x_prev, x + + def call(self, inputs, training=None): + # done this way to prevent confusion in the auto graph + if self._inverted: + return self._call_reversed(inputs, training=training) + else: + return self._call_regular(inputs, training=training) + + +@tf.keras.utils.register_keras_serializable(package='yolo') +class SPP(tf.keras.layers.Layer): + """Spatial Pyramid Pooling. + + A non-agregated SPP layer that uses Pooling. + """ + + def __init__(self, sizes, **kwargs): + self._sizes = list(reversed(sizes)) + if not sizes: + raise ValueError('More than one maxpool should be specified in SSP block') + super().__init__(**kwargs) + + def build(self, input_shape): + maxpools = [] + for size in self._sizes: + maxpools.append( + tf.keras.layers.MaxPool2D( + pool_size=(size, size), + strides=(1, 1), + padding='same', + data_format=None)) + self._maxpools = maxpools + super().build(input_shape) + + def call(self, inputs, training=None): + outputs = [] + for maxpool in self._maxpools: + outputs.append(maxpool(inputs)) + outputs.append(inputs) + concat_output = tf.keras.layers.concatenate(outputs) + return concat_output + + def get_config(self): + layer_config = {'sizes': self._sizes} + layer_config.update(super().get_config()) + return layer_config + + +class SAM(tf.keras.layers.Layer): + """Spatial Attention Model. + + [1] Sanghyun Woo, Jongchan Park, Joon-Young Lee, In So Kweon + CBAM: Convolutional Block Attention Module. arXiv:1807.06521 + + implementation of the Spatial Attention Model (SAM) + """ + + def __init__(self, + use_pooling=False, + filter_match=False, + filters=1, + kernel_size=(1, 1), + strides=(1, 1), + padding='same', + dilation_rate=(1, 1), + kernel_initializer='glorot_uniform', + bias_initializer='zeros', + bias_regularizer=None, + kernel_regularizer=None, + use_bn=True, + use_sync_bn=True, + norm_momentum=0.99, + norm_epsilon=0.001, + activation='sigmoid', + output_activation=None, + leaky_alpha=0.1, + **kwargs): + + # use_pooling + self._use_pooling = use_pooling + self._filters = filters + self._output_activation = output_activation + self._leaky_alpha = leaky_alpha + + self.dark_conv_args = { + 'kernel_size': kernel_size, + 'strides': strides, + 'padding': padding, + 'dilation_rate': dilation_rate, + 'kernel_initializer': kernel_initializer, + 'bias_initializer': bias_initializer, + 'bias_regularizer': bias_regularizer, + 'use_bn': use_bn, + 'use_sync_bn': use_sync_bn, + 'norm_momentum': norm_momentum, + 'norm_epsilon': norm_epsilon, + 'activation': activation, + 'kernel_regularizer': kernel_regularizer, + 'leaky_alpha': leaky_alpha + } + + super().__init__(**kwargs) + + def build(self, input_shape): + if self._filters == -1: + self._filters = input_shape[-1] + self._conv = ConvBN(filters=self._filters, **self.dark_conv_args) + if self._output_activation == 'leaky': + self._activation_fn = tf.keras.layers.LeakyReLU(alpha=self._leaky_alpha) + elif self._output_activation == 'mish': + self._activation_fn = lambda x: x * tf.math.tanh(tf.math.softplus(x)) + else: + self._activation_fn = tf_utils.get_activation(self._output_activation) + + def call(self, inputs, training=None): + if self._use_pooling: + depth_max = tf.reduce_max(inputs, axis=-1, keepdims=True) + depth_avg = tf.reduce_mean(inputs, axis=-1, keepdims=True) + input_maps = tf.concat([depth_avg, depth_max], axis=-1) + else: + input_maps = inputs + + attention_mask = self._conv(input_maps) + return self._activation_fn(inputs * attention_mask) + + +class CAM(tf.keras.layers.Layer): + """Channel Attention Model. + + [1] Sanghyun Woo, Jongchan Park, Joon-Young Lee, In So Kweon + CBAM: Convolutional Block Attention Module. arXiv:1807.06521 + + Implementation of the Channel Attention Model (CAM) + """ + + def __init__(self, + reduction_ratio=1.0, + kernel_initializer='glorot_uniform', + bias_initializer='zeros', + bias_regularizer=None, + kernel_regularizer=None, + use_bn=False, + use_sync_bn=False, + use_bias=False, + norm_momentum=0.99, + norm_epsilon=0.001, + mlp_activation='linear', + activation='sigmoid', + leaky_alpha=0.1, + **kwargs): + + self._reduction_ratio = reduction_ratio + + # use_pooling + if use_sync_bn: + self._bn = tf.keras.layers.experimental.SyncBatchNormalization + else: + self._bn = tf.keras.layers.BatchNormalization + + if not use_bn: + self._bn = Identity + self._bn_args = {} + else: + self._bn_args = { + 'momentum': norm_momentum, + 'epsilon': norm_epsilon, + } + + self._mlp_args = { + 'use_bias': use_bias, + 'kernel_initializer': kernel_initializer, + 'bias_initializer': bias_initializer, + 'bias_regularizer': bias_regularizer, + 'activation': mlp_activation, + 'kernel_regularizer': kernel_regularizer, + } + + self._leaky_alpha = leaky_alpha + self._activation = activation + + super().__init__(**kwargs) + + def build(self, input_shape): + self._filters = input_shape[-1] + + self._mlp = tf.keras.Sequential([ + tf.keras.layers.Dense(self._filters, **self._mlp_args), + self._bn(**self._bn_args), + tf.keras.layers.Dense( + int(self._filters * self._reduction_ratio), **self._mlp_args), + self._bn(**self._bn_args), + tf.keras.layers.Dense(self._filters, **self._mlp_args), + self._bn(**self._bn_args), + ]) + + if self._activation == 'leaky': + self._activation_fn = tf.keras.layers.LeakyReLU(alpha=self._leaky_alpha) + elif self._activation == 'mish': + self._activation_fn = lambda x: x * tf.math.tanh(tf.math.softplus(x)) + else: + self._activation_fn = tf_utils.get_activation(self._activation) + + def call(self, inputs, training=None): + depth_max = self._mlp(tf.reduce_max(inputs, axis=(1, 2))) + depth_avg = self._mlp(tf.reduce_mean(inputs, axis=(1, 2))) + channel_mask = self._activation_fn(depth_avg + depth_max) + + channel_mask = tf.expand_dims(channel_mask, axis=1) + attention_mask = tf.expand_dims(channel_mask, axis=1) + + return inputs * attention_mask + + +class CBAM(tf.keras.layers.Layer): + """Convolutional Block Attention Module. + + [1] Sanghyun Woo, Jongchan Park, Joon-Young Lee, In So Kweon + CBAM: Convolutional Block Attention Module. arXiv:1807.06521 + + implementation of the Convolution Block Attention Module (CBAM) + """ + + def __init__(self, + use_pooling=False, + filters=1, + reduction_ratio=1.0, + kernel_size=(1, 1), + strides=(1, 1), + padding='same', + dilation_rate=(1, 1), + kernel_initializer='glorot_uniform', + bias_initializer='zeros', + bias_regularizer=None, + kernel_regularizer=None, + use_bn=True, + use_sync_bn=False, + norm_momentum=0.99, + norm_epsilon=0.001, + mlp_activation=None, + activation='sigmoid', + leaky_alpha=0.1, + **kwargs): + + # use_pooling + + self._sam_args = { + 'use_pooling': use_pooling, + 'filters': filters, + 'kernel_size': kernel_size, + 'strides': strides, + 'padding': padding, + 'dilation_rate': dilation_rate, + } + + self._cam_args = { + 'reduction_ratio': reduction_ratio, + 'mlp_activation': mlp_activation + } + + self._common_args = { + 'kernel_initializer': kernel_initializer, + 'bias_initializer': bias_initializer, + 'bias_regularizer': bias_regularizer, + 'use_bn': use_bn, + 'use_sync_bn': use_sync_bn, + 'norm_momentum': norm_momentum, + 'norm_epsilon': norm_epsilon, + 'activation': activation, + 'kernel_regularizer': kernel_regularizer, + 'leaky_alpha': leaky_alpha + } + + self._cam_args.update(self._common_args) + self._sam_args.update(self._common_args) + super().__init__(**kwargs) + + def build(self, input_shape): + self._cam = CAM(**self._cam_args) + self._sam = SAM(**self._sam_args) + + def call(self, inputs, training=None): + return self._sam(self._cam(inputs)) + + +@tf.keras.utils.register_keras_serializable(package='yolo') +class DarkRouteProcess(tf.keras.layers.Layer): + """Dark Route Process block. + + Process darknet outputs and connect back bone to head more generalizably + Abstracts repetition of DarkConv objects that is common in YOLO. + + It is used like the following: + + x = ConvBN(1024, (3, 3), (1, 1))(x) + proc = DarkRouteProcess(filters = 1024, + repetitions = 3, + insert_spp = False)(x) + """ + + def __init__( + self, + filters=2, + repetitions=2, + insert_spp=False, + insert_sam=False, + insert_cbam=False, + csp_stack=0, + csp_scale=2, + kernel_initializer='glorot_uniform', + bias_initializer='zeros', + bias_regularizer=None, + kernel_regularizer=None, + use_sync_bn=False, + norm_momentum=0.99, + norm_epsilon=0.001, + block_invert=False, + activation='leaky', + leaky_alpha=0.1, + spp_keys=None, + **kwargs): + """DarkRouteProcess initializer. + + Args: + filters: the number of filters to be used in all subsequent layers + filters should be the depth of the tensor input into this layer, + as no downsampling can be done within this layer object. + repetitions: number of times to repeat the processign nodes. + for tiny: 1 repition, no spp allowed. + for spp: insert_spp = True, and allow for 6 repetitions. + for regular: insert_spp = False, and allow for 6 repetitions. + insert_spp: bool if true add the spatial pyramid pooling layer. + insert_sam: bool if true add spatial attention module to path. + insert_cbam: bool if true add convolutional block attention + module to path. + csp_stack: int for the number of sequential layers from 0 + to you would like to convert into a Cross Stage + Partial(csp) type. + csp_scale: int for how much to down scale the number of filters + only for the csp layers in the csp section of the processing + path. A value 2 indicates that each layer that is int eh CSP + stack will have filters = filters/2. + kernel_initializer: method to use to initialize kernel weights. + bias_initializer: method to use to initialize the bias of the conv + layers. + bias_regularizer: string to indicate which function to use to regularizer + bias. + kernel_regularizer: string to indicate which function to use to + regularizer weights. + use_sync_bn: bool if true use the sync batch normalization. + norm_momentum: batch norm parameter see Tensorflow documentation. + norm_epsilon: batch norm parameter see Tensorflow documentation. + block_invert: bool use for switching between the even and odd + repretions of layers. usually the repetition is based on a + 3x3 conv with filters, followed by a 1x1 with filters/2 with + an even number of repetitions to ensure each 3x3 gets a 1x1 + sqeeze. block invert swaps the 3x3/1 1x1/2 to a 1x1/2 3x3/1 + ordering typically used when the model requires an odd number + of repetiitions. All other peramters maintain their affects + activation: activation function to use in processing. + leaky_alpha: if leaky acitivation function, the alpha to use in + processing the relu input. + spp_keys: List[int] of the sampling levels to be applied by + the Spatial Pyramid Pooling Layer. By default it is + [5, 9, 13] inidicating a 5x5 pooling followed by 9x9 + followed by 13x13 then followed by the standard concatnation + and convolution. + **kwargs: Keyword Arguments. + """ + + super().__init__(**kwargs) + # darkconv params + self._filters = filters + self._use_sync_bn = use_sync_bn + self._kernel_initializer = kernel_initializer + self._bias_initializer = bias_initializer + self._bias_regularizer = bias_regularizer + self._kernel_regularizer = kernel_regularizer + + # normal params + self._norm_momentum = norm_momentum + self._norm_epsilon = norm_epsilon + + # activation params + self._activation = activation + self._leaky_alpha = leaky_alpha + + repetitions += (2 * int(insert_spp)) + if repetitions == 1: + block_invert = True + + self._repetitions = repetitions + self.layer_list, self.outputs = self._get_base_layers() + + if csp_stack > 0: + self._csp_scale = csp_scale + csp_stack += (2 * int(insert_spp)) + self._csp_filters = lambda x: x // csp_scale + self._convert_csp(self.layer_list, self.outputs, csp_stack) + block_invert = False + + self._csp_stack = csp_stack + + if block_invert: + self._conv1_filters = lambda x: x + self._conv2_filters = lambda x: x // 2 + self._conv1_kernel = (3, 3) + self._conv2_kernel = (1, 1) + else: + self._conv1_filters = lambda x: x // 2 + self._conv2_filters = lambda x: x + self._conv1_kernel = (1, 1) + self._conv2_kernel = (3, 3) + + # insert SPP will always add to the total nuber of layer, never replace + if insert_spp: + self._spp_keys = spp_keys if spp_keys is not None else [5, 9, 13] + self.layer_list = self._insert_spp(self.layer_list) + + if repetitions > 1: + self.outputs[-2] = True + + if insert_sam: + self.layer_list = self._insert_sam(self.layer_list, self.outputs) + self._repetitions += 1 + self.outputs[-1] = True + + def _get_base_layers(self): + layer_list = [] + outputs = [] + for i in range(self._repetitions): + layers = ['conv1'] * ((i + 1) % 2) + ['conv2'] * (i % 2) + layer_list.extend(layers) + outputs = [False] + outputs + return layer_list, outputs + + def _insert_spp(self, layer_list): + if len(layer_list) <= 3: + layer_list[1] = 'spp' + else: + layer_list[3] = 'spp' + return layer_list + + def _convert_csp(self, layer_list, outputs, csp_stack_size): + layer_list[0] = 'csp_route' + layer_list.insert(csp_stack_size - 1, 'csp_connect') + outputs.insert(csp_stack_size - 1, False) + return layer_list, outputs + + def _insert_sam(self, layer_list, outputs): + if len(layer_list) >= 2 and layer_list[-2] != 'spp': + layer_list.insert(-2, 'sam') + outputs.insert(-1, True) + else: + layer_list.insert(-1, 'sam') + outputs.insert(-1, False) + return layer_list + + def _conv1(self, filters, kwargs, csp=False): + if csp: + filters_ = self._csp_filters + else: + filters_ = self._conv1_filters + + x1 = ConvBN( + filters=filters_(filters), + kernel_size=self._conv1_kernel, + strides=(1, 1), + padding='same', + use_bn=True, + **kwargs) + return x1 + + def _conv2(self, filters, kwargs, csp=False): + if csp: + filters_ = self._csp_filters + else: + filters_ = self._conv2_filters + + x1 = ConvBN( + filters=filters_(filters), + kernel_size=self._conv2_kernel, + strides=(1, 1), + padding='same', + use_bn=True, + **kwargs) + return x1 + + def _csp_route(self, filters, kwargs): + x1 = CSPRoute( + filters=filters, + filter_scale=self._csp_scale, + downsample=False, + **kwargs) + return x1 + + def _csp_connect(self, filters, kwargs): + x1 = CSPConnect(filters=filters, drop_final=True, drop_first=True, **kwargs) + return x1 + + def _spp(self, filters, kwargs): + x1 = SPP(self._spp_keys) + return x1 + + def _sam(self, filters, kwargs): + x1 = SAM(filters=-1, use_pooling=False, use_bn=True, **kwargs) + return x1 + + def build(self, input_shape): + dark_conv_args = { + 'activation': self._activation, + 'kernel_initializer': self._kernel_initializer, + 'bias_initializer': self._bias_initializer, + 'bias_regularizer': self._bias_regularizer, + 'use_sync_bn': self._use_sync_bn, + 'norm_momentum': self._norm_momentum, + 'norm_epsilon': self._norm_epsilon, + 'kernel_regularizer': self._kernel_regularizer, + 'leaky_alpha': self._leaky_alpha, + } + + csp = False + self.layers = [] + for layer in self.layer_list: + if layer == 'csp_route': + self.layers.append(self._csp_route(self._filters, dark_conv_args)) + csp = True + elif layer == 'csp_connect': + self.layers.append(self._csp_connect(self._filters, dark_conv_args)) + csp = False + elif layer == 'conv1': + self.layers.append(self._conv1(self._filters, dark_conv_args, csp=csp)) + elif layer == 'conv2': + self.layers.append(self._conv2(self._filters, dark_conv_args, csp=csp)) + elif layer == 'spp': + self.layers.append(self._spp(self._filters, dark_conv_args)) + elif layer == 'sam': + self.layers.append(self._sam(-1, dark_conv_args)) + + self._lim = len(self.layers) + super().build(input_shape) + + def _call_regular(self, inputs, training=None): + # check efficiency + x = inputs + x_prev = x + output_prev = True + + for (layer, output) in zip(self.layers, self.outputs): + if output_prev: + x_prev = x + x = layer(x) + output_prev = output + return x_prev, x + + def _call_csp(self, inputs, training=None): + # check efficiency + x = inputs + x_prev = x + output_prev = True + x_route = None + + for i, (layer, output) in enumerate(zip(self.layers, self.outputs)): + if output_prev: + x_prev = x + if i == 0: + x, x_route = layer(x) + elif i == self._csp_stack - 1: + x = layer([x, x_route]) + else: + x = layer(x) + output_prev = output + return x_prev, x + + def call(self, inputs, training=None): + if self._csp_stack > 0: + return self._call_csp(inputs, training=training) + else: + return self._call_regular(inputs) diff --git a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks_test.py b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks_test.py index 5df28a4f3fb..455f5ce199f 100644 --- a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks_test.py +++ b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks_test.py @@ -13,7 +13,6 @@ # limitations under the License. # Lint as: python3 - from absl.testing import parameterized import numpy as np import tensorflow as tf @@ -23,8 +22,8 @@ class CSPConnectTest(tf.test.TestCase, parameterized.TestCase): - @parameterized.named_parameters(("same", 224, 224, 64, 1), - ("downsample", 224, 224, 64, 2)) + @parameterized.named_parameters(('same', 224, 224, 64, 1), + ('downsample', 224, 224, 64, 2)) def test_pass_through(self, width, height, filters, mod): x = tf.keras.Input(shape=(width, height, filters)) test_layer = nn_blocks.CSPRoute(filters=filters, filter_scale=mod) @@ -38,8 +37,8 @@ def test_pass_through(self, width, height, filters, mod): [None, np.ceil(width // 2), np.ceil(height // 2), (filters)]) - @parameterized.named_parameters(("same", 224, 224, 64, 1), - ("downsample", 224, 224, 128, 2)) + @parameterized.named_parameters(('same', 224, 224, 64, 1), + ('downsample', 224, 224, 128, 2)) def test_gradient_pass_though(self, filters, width, height, mod): loss = tf.keras.losses.MeanSquaredError() optimizer = tf.keras.optimizers.SGD() @@ -49,10 +48,11 @@ def test_gradient_pass_though(self, filters, width, height, mod): init = tf.random_normal_initializer() x = tf.Variable( initial_value=init(shape=(1, width, height, filters), dtype=tf.float32)) - y = tf.Variable(initial_value=init(shape=(1, int(np.ceil(width // 2)), - int(np.ceil(height // 2)), - filters), - dtype=tf.float32)) + y = tf.Variable( + initial_value=init( + shape=(1, int(np.ceil(width // 2)), int(np.ceil(height // 2)), + filters), + dtype=tf.float32)) with tf.GradientTape() as tape: x_hat, x_prev = test_layer(x) @@ -66,8 +66,8 @@ def test_gradient_pass_though(self, filters, width, height, mod): class CSPRouteTest(tf.test.TestCase, parameterized.TestCase): - @parameterized.named_parameters(("same", 224, 224, 64, 1), - ("downsample", 224, 224, 64, 2)) + @parameterized.named_parameters(('same', 224, 224, 64, 1), + ('downsample', 224, 224, 64, 2)) def test_pass_through(self, width, height, filters, mod): x = tf.keras.Input(shape=(width, height, filters)) test_layer = nn_blocks.CSPRoute(filters=filters, filter_scale=mod) @@ -79,8 +79,8 @@ def test_pass_through(self, width, height, filters, mod): [None, np.ceil(width // 2), np.ceil(height // 2), (filters / mod)]) - @parameterized.named_parameters(("same", 224, 224, 64, 1), - ("downsample", 224, 224, 128, 2)) + @parameterized.named_parameters(('same', 224, 224, 64, 1), + ('downsample', 224, 224, 128, 2)) def test_gradient_pass_though(self, filters, width, height, mod): loss = tf.keras.losses.MeanSquaredError() optimizer = tf.keras.optimizers.SGD() @@ -90,10 +90,11 @@ def test_gradient_pass_though(self, filters, width, height, mod): init = tf.random_normal_initializer() x = tf.Variable( initial_value=init(shape=(1, width, height, filters), dtype=tf.float32)) - y = tf.Variable(initial_value=init(shape=(1, int(np.ceil(width // 2)), - int(np.ceil(height // 2)), - filters), - dtype=tf.float32)) + y = tf.Variable( + initial_value=init( + shape=(1, int(np.ceil(width // 2)), int(np.ceil(height // 2)), + filters), + dtype=tf.float32)) with tf.GradientTape() as tape: x_hat, x_prev = test_layer(x) @@ -107,11 +108,11 @@ def test_gradient_pass_though(self, filters, width, height, mod): class CSPStackTest(tf.test.TestCase, parameterized.TestCase): - def build_layer( - self, layer_type, filters, filter_scale, count, stack_type, downsample): + def build_layer(self, layer_type, filters, filter_scale, count, stack_type, + downsample): if stack_type is not None: layers = [] - if layer_type == "residual": + if layer_type == 'residual': for _ in range(count): layers.append( nn_blocks.DarkResidual( @@ -120,7 +121,7 @@ def build_layer( for _ in range(count): layers.append(nn_blocks.ConvBN(filters=filters)) - if stack_type == "model": + if stack_type == 'model': layers = tf.keras.Sequential(layers=layers) else: layers = None @@ -133,10 +134,10 @@ def build_layer( return stack @parameterized.named_parameters( - ("no_stack", 224, 224, 64, 2, "residual", None, 0, True), - ("residual_stack", 224, 224, 64, 2, "residual", "list", 2, True), - ("conv_stack", 224, 224, 64, 2, "conv", "list", 3, False), - ("callable_no_scale", 224, 224, 64, 1, "residual", "model", 5, False)) + ('no_stack', 224, 224, 64, 2, 'residual', None, 0, True), + ('residual_stack', 224, 224, 64, 2, 'residual', 'list', 2, True), + ('conv_stack', 224, 224, 64, 2, 'conv', 'list', 3, False), + ('callable_no_scale', 224, 224, 64, 1, 'residual', 'model', 5, False)) def test_pass_through(self, width, height, filters, mod, layer_type, stack_type, count, downsample): x = tf.keras.Input(shape=(width, height, filters)) @@ -152,10 +153,10 @@ def test_pass_through(self, width, height, filters, mod, layer_type, self.assertAllEqual(outx.shape.as_list(), [None, width, height, filters]) @parameterized.named_parameters( - ("no_stack", 224, 224, 64, 2, "residual", None, 0, True), - ("residual_stack", 224, 224, 64, 2, "residual", "list", 2, True), - ("conv_stack", 224, 224, 64, 2, "conv", "list", 3, False), - ("callable_no_scale", 224, 224, 64, 1, "residual", "model", 5, False)) + ('no_stack', 224, 224, 64, 2, 'residual', None, 0, True), + ('residual_stack', 224, 224, 64, 2, 'residual', 'list', 2, True), + ('conv_stack', 224, 224, 64, 2, 'conv', 'list', 3, False), + ('callable_no_scale', 224, 224, 64, 1, 'residual', 'model', 5, False)) def test_gradient_pass_though(self, width, height, filters, mod, layer_type, stack_type, count, downsample): loss = tf.keras.losses.MeanSquaredError() @@ -188,10 +189,10 @@ def test_gradient_pass_though(self, width, height, filters, mod, layer_type, class ConvBNTest(tf.test.TestCase, parameterized.TestCase): @parameterized.named_parameters( - ("valid", (3, 3), "valid", (1, 1)), ("same", (3, 3), "same", (1, 1)), - ("downsample", (3, 3), "same", (2, 2)), ("test", (1, 1), "valid", (1, 1))) + ('valid', (3, 3), 'valid', (1, 1)), ('same', (3, 3), 'same', (1, 1)), + ('downsample', (3, 3), 'same', (2, 2)), ('test', (1, 1), 'valid', (1, 1))) def test_pass_through(self, kernel_size, padding, strides): - if padding == "same": + if padding == 'same': pad_const = 1 else: pad_const = 0 @@ -212,16 +213,16 @@ def test_pass_through(self, kernel_size, padding, strides): print(test) self.assertAllEqual(outx.shape.as_list(), test) - @parameterized.named_parameters(("filters", 3)) + @parameterized.named_parameters(('filters', 3)) def test_gradient_pass_though(self, filters): loss = tf.keras.losses.MeanSquaredError() optimizer = tf.keras.optimizers.SGD() - with tf.device("/CPU:0"): - test_layer = nn_blocks.ConvBN(filters, kernel_size=(3, 3), padding="same") + with tf.device('/CPU:0'): + test_layer = nn_blocks.ConvBN(filters, kernel_size=(3, 3), padding='same') init = tf.random_normal_initializer() - x = tf.Variable(initial_value=init(shape=(1, 224, 224, - 3), dtype=tf.float32)) + x = tf.Variable( + initial_value=init(shape=(1, 224, 224, 3), dtype=tf.float32)) y = tf.Variable( initial_value=init(shape=(1, 224, 224, filters), dtype=tf.float32)) @@ -235,9 +236,9 @@ def test_gradient_pass_though(self, filters): class DarkResidualTest(tf.test.TestCase, parameterized.TestCase): - @parameterized.named_parameters(("same", 224, 224, 64, False), - ("downsample", 223, 223, 32, True), - ("oddball", 223, 223, 32, False)) + @parameterized.named_parameters(('same', 224, 224, 64, False), + ('downsample', 223, 223, 32, True), + ('oddball', 223, 223, 32, False)) def test_pass_through(self, width, height, filters, downsample): mod = 1 if downsample: @@ -252,9 +253,9 @@ def test_pass_through(self, width, height, filters, downsample): [None, np.ceil(width / mod), np.ceil(height / mod), filters]) - @parameterized.named_parameters(("same", 64, 224, 224, False), - ("downsample", 32, 223, 223, True), - ("oddball", 32, 223, 223, False)) + @parameterized.named_parameters(('same', 64, 224, 224, False), + ('downsample', 32, 223, 223, True), + ('oddball', 32, 223, 223, False)) def test_gradient_pass_though(self, filters, width, height, downsample): loss = tf.keras.losses.MeanSquaredError() optimizer = tf.keras.optimizers.SGD() @@ -268,10 +269,11 @@ def test_gradient_pass_though(self, filters, width, height, downsample): init = tf.random_normal_initializer() x = tf.Variable( initial_value=init(shape=(1, width, height, filters), dtype=tf.float32)) - y = tf.Variable(initial_value=init(shape=(1, int(np.ceil(width / mod)), - int(np.ceil(height / mod)), - filters), - dtype=tf.float32)) + y = tf.Variable( + initial_value=init( + shape=(1, int(np.ceil(width / mod)), int(np.ceil(height / mod)), + filters), + dtype=tf.float32)) with tf.GradientTape() as tape: x_hat = test_layer(x) @@ -281,5 +283,104 @@ def test_gradient_pass_though(self, filters, width, height, downsample): self.assertNotIn(None, grad) -if __name__ == "__main__": + +class DarkSppTest(tf.test.TestCase, parameterized.TestCase): + + @parameterized.named_parameters(('RouteProcessSpp', 224, 224, 3, [5, 9, 13]), + ('test1', 300, 300, 10, [2, 3, 4, 5]), + ('test2', 256, 256, 5, [10])) + def test_pass_through(self, width, height, channels, sizes): + x = tf.keras.Input(shape=(width, height, channels)) + test_layer = nn_blocks.SPP(sizes=sizes) + outx = test_layer(x) + self.assertAllEqual(outx.shape.as_list(), + [None, width, height, channels * (len(sizes) + 1)]) + return + + @parameterized.named_parameters(('RouteProcessSpp', 224, 224, 3, [5, 9, 13]), + ('test1', 300, 300, 10, [2, 3, 4, 5]), + ('test2', 256, 256, 5, [10])) + def test_gradient_pass_though(self, width, height, channels, sizes): + loss = tf.keras.losses.MeanSquaredError() + optimizer = tf.keras.optimizers.SGD() + test_layer = nn_blocks.SPP(sizes=sizes) + + init = tf.random_normal_initializer() + x = tf.Variable( + initial_value=init( + shape=(1, width, height, channels), dtype=tf.float32)) + y = tf.Variable( + initial_value=init( + shape=(1, width, height, channels * (len(sizes) + 1)), + dtype=tf.float32)) + + with tf.GradientTape() as tape: + x_hat = test_layer(x) + grad_loss = loss(x_hat, y) + grad = tape.gradient(grad_loss, test_layer.trainable_variables) + optimizer.apply_gradients(zip(grad, test_layer.trainable_variables)) + + self.assertNotIn(None, grad) + return + + +class DarkRouteProcessTest(tf.test.TestCase, parameterized.TestCase): + + @parameterized.named_parameters( + ('test1', 224, 224, 64, 7, False), ('test2', 223, 223, 32, 3, False), + ('tiny', 223, 223, 16, 1, False), ('spp', 224, 224, 64, 7, False)) + def test_pass_through(self, width, height, filters, repetitions, spp): + x = tf.keras.Input(shape=(width, height, filters)) + test_layer = nn_blocks.DarkRouteProcess( + filters=filters, repetitions=repetitions, insert_spp=spp) + outx = test_layer(x) + self.assertLen(outx, 2, msg='len(outx) != 2') + if repetitions == 1: + filter_y1 = filters + else: + filter_y1 = filters // 2 + self.assertAllEqual( + outx[1].shape.as_list(), [None, width, height, filter_y1]) + self.assertAllEqual( + filters % 2, + 0, + msg='Output of a DarkRouteProcess layer has an odd number of filters') + self.assertAllEqual(outx[0].shape.as_list(), [None, width, height, filters]) + + @parameterized.named_parameters( + ('test1', 224, 224, 64, 7, False), ('test2', 223, 223, 32, 3, False), + ('tiny', 223, 223, 16, 1, False), ('spp', 224, 224, 64, 7, False)) + def test_gradient_pass_though(self, width, height, filters, repetitions, spp): + loss = tf.keras.losses.MeanSquaredError() + optimizer = tf.keras.optimizers.SGD() + test_layer = nn_blocks.DarkRouteProcess( + filters=filters, repetitions=repetitions, insert_spp=spp) + + if repetitions == 1: + filter_y1 = filters + else: + filter_y1 = filters // 2 + + init = tf.random_normal_initializer() + x = tf.Variable( + initial_value=init(shape=(1, width, height, filters), dtype=tf.float32)) + y_0 = tf.Variable( + initial_value=init(shape=(1, width, height, filters), dtype=tf.float32)) + y_1 = tf.Variable( + initial_value=init( + shape=(1, width, height, filter_y1), dtype=tf.float32)) + + with tf.GradientTape() as tape: + x_hat_0, x_hat_1 = test_layer(x) + grad_loss_0 = loss(x_hat_0, y_0) + grad_loss_1 = loss(x_hat_1, y_1) + grad = tape.gradient([grad_loss_0, grad_loss_1], + test_layer.trainable_variables) + optimizer.apply_gradients(zip(grad, test_layer.trainable_variables)) + + self.assertNotIn(None, grad) + return + + +if __name__ == '__main__': tf.test.main()