From 49e179cd7268133c24fcdadde20a195b745fe674 Mon Sep 17 00:00:00 2001 From: Rostan Tabet Date: Tue, 28 Jul 2026 09:30:38 +0000 Subject: [PATCH] Improve documentation for framework iterators and readers Signed-off-by: Rostan Tabet --- dali/python/nvidia/dali/ops/_docs.py | 14 ++- .../nvidia/dali/plugin/base_iterator.py | 35 +++--- dali/python/nvidia/dali/plugin/jax/clu.py | 72 +++++------- .../python/nvidia/dali/plugin/jax/iterator.py | 72 +++++------- dali/python/nvidia/dali/plugin/paddle.py | 72 +++++------- .../nvidia/dali/plugin/pytorch/__init__.py | 110 ++++++++---------- docs/advanced_topics_sharding.rst | 70 +++++++---- 7 files changed, 219 insertions(+), 226 deletions(-) diff --git a/dali/python/nvidia/dali/ops/_docs.py b/dali/python/nvidia/dali/ops/_docs.py index b82ae872645..130c5312767 100644 --- a/dali/python/nvidia/dali/ops/_docs.py +++ b/dali/python/nvidia/dali/ops/_docs.py @@ -147,6 +147,18 @@ def _get_rng_doc(): ) +def _get_name_doc(): + """Return documentation for the operator name used in pipeline mode.""" + return _numpydoc_formatter( + "name", + "str, optional, default = None", + "Name of the operator instance. Names must be unique within a pipeline.\n\n" + "Set an explicit name when another API needs to refer to the operator, for\n" + "example through an iterator's ``reader_name`` argument. If omitted, DALI\n" + "generates a name automatically.", + ) + + def _get_kwargs(schema, api="ops", args=None): """ Get the numpydoc-formatted docstring section for keywords arguments. @@ -155,7 +167,7 @@ def _get_kwargs(schema, api="ops", args=None): schema : OpSchema Schema of the operator to be documented """ - ret = "" + ret = _get_name_doc() + "\n" if api != "dynamic" else "" for arg in schema.GetArgumentNames(): if args is not None and arg not in args: continue diff --git a/dali/python/nvidia/dali/plugin/base_iterator.py b/dali/python/nvidia/dali/plugin/base_iterator.py index 3c61db5a662..9fb50fcfe70 100644 --- a/dali/python/nvidia/dali/plugin/base_iterator.py +++ b/dali/python/nvidia/dali/plugin/base_iterator.py @@ -69,17 +69,18 @@ class _DaliBaseIterator(object): for data or label correspondingly. output_names should be distinct. size : int, default = -1 - Number of samples in the shard for the wrapped pipeline (if there is more than one - it is a sum). Providing -1 means that the iterator will work until StopIteration - is raised from the inside of iter_setup(). The options `last_batch_policy`, - `last_batch_padded` and `auto_reset` don't work in such case. It works with only - one pipeline inside the iterator. - Mutually exclusive with `reader_name` argument + Number of samples in the shard. For multiple pipelines, this is the sum of + their shard sizes. + Mutually exclusive with `reader_name`. When left at -1 without + `reader_name`, a single-pipeline iterator reads until the pipeline raises + `StopIteration`, for example when an external source is exhausted; + `last_batch_policy` and `last_batch_padded` do not apply. reader_name : str, default = None - Name of the reader which will be queried for the shard size, number of shards, and - all other properties necessary to count properly the number of relevant and padded - samples that iterator needs to deal with. Sets `last_batch_padded` - accordingly to the reader's configuration (`pad_last_batch` reader argument) + Name of the reader operator that determines the iterator length and + last-batch padding. It must match the reader's `name` argument in every + supplied pipeline. + When set, `size` and `last_batch_padded` are determined automatically and + must not be provided. It does not change `last_batch_policy`. auto_reset : string or bool, optional, default = False Whether the iterator resets itself for the next epoch or it requires reset() to be called explicitly. @@ -104,15 +105,11 @@ class _DaliBaseIterator(object): What to do with the last batch when there are not enough samples in the epoch to fully fill it. See :meth:`nvidia.dali.plugin.base_iterator.LastBatchPolicy` last_batch_padded : bool, optional, default = False - Whether the last batch provided by DALI is padded with the last sample - or it just wraps up. In the conjunction with `last_batch_policy` it tells - if the iterator returning last batch with data only partially filled with - data from the current epoch is dropping padding samples or samples from - the next epoch. If set to False next - epoch will end sooner as data from it was consumed but dropped. If set to - True next epoch would be the same length as the first one. For this to happen, - the option `pad_last_batch` in the reader needs to be set to True as well. - It is overwritten when `reader_name` argument is provided + Whether the reader pads the last batch by repeating its last sample + (`True`) or continues into the next epoch (`False`). + Without `reader_name`, set this to the same value as the reader's + `pad_last_batch` argument. With `reader_name`, it is determined + automatically and must not be provided. prepare_first_batch : bool, optional, default = True Whether DALI should buffer the first batch right after the creation of the iterator, so one batch is already prepared when the iterator is prompted for the data diff --git a/dali/python/nvidia/dali/plugin/jax/clu.py b/dali/python/nvidia/dali/plugin/jax/clu.py index e071c706ca0..c32c4cb7d0d 100644 --- a/dali/python/nvidia/dali/plugin/jax/clu.py +++ b/dali/python/nvidia/dali/plugin/jax/clu.py @@ -58,18 +58,18 @@ class DALIGenericPeekableIterator(DALIGenericIterator): of those names. Each name should be distinct size : int, default = -1 - Number of samples in the shard for the wrapped pipeline (if there is more than - one it is a sum) - Providing -1 means that the iterator will work until StopIteration is raised - from the inside of iter_setup(). The options `last_batch_policy` and - `last_batch_padded` don't work in such case. It works with only one pipeline inside - the iterator. - Mutually exclusive with `reader_name` argument + Number of samples in the shard. For multiple pipelines, this is the sum of + their shard sizes. + Mutually exclusive with `reader_name`. When left at -1 without + `reader_name`, a single-pipeline iterator reads until the pipeline raises + `StopIteration`, for example when an external source is exhausted; + `last_batch_policy` and `last_batch_padded` do not apply. reader_name : str, default = None - Name of the reader which will be queried for the shard size, number of shards and - all other properties necessary to count properly the number of relevant and padded - samples that iterator needs to deal with. It automatically sets `last_batch_padded` - accordingly to match the reader's configuration. + Name of the reader operator that determines the iterator length and + last-batch padding. It must match the reader's `name` argument in every + supplied pipeline. + When set, `size` and `last_batch_padded` are determined automatically and + must not be provided. It does not change `last_batch_policy`. auto_reset : string or bool, optional, default = False Whether the iterator resets itself for the next epoch or it requires reset() to be called explicitly. @@ -85,15 +85,11 @@ class DALIGenericPeekableIterator(DALIGenericIterator): to fully fill it. See :meth:`nvidia.dali.plugin.base_iterator.LastBatchPolicy`. JAX iterator does not support LastBatchPolicy.PARTIAL last_batch_padded : bool, optional, default = False - Whether the last batch provided by DALI is padded with the last sample - or it just wraps up. In the conjunction with `last_batch_policy` it tells - if the iterator returning last batch with data only partially filled with - data from the current epoch is dropping padding samples or samples from - the next epoch. If set to ``False`` next - epoch will end sooner as data from it was consumed but dropped. If set to - True next epoch would be the same length as the first one. For this to happen, - the option `pad_last_batch` in the reader needs to be set to True as well. - It is overwritten when `reader_name` argument is provided + Whether the reader pads the last batch by repeating its last sample + (`True`) or continues into the next epoch (`False`). + Without `reader_name`, set this to the same value as the reader's + `pad_last_batch` argument. With `reader_name`, it is determined + automatically and must not be provided. prepare_first_batch : bool, optional, default = True Whether DALI should buffer the first batch right after the creation of the iterator, so one batch is already prepared when the iterator is prompted for the data @@ -323,18 +319,18 @@ def peekable_data_iterator( of those names. Each name should be distinct size : int, default = -1 - Number of samples in the shard for the wrapped pipeline (if there is more than - one it is a sum) - Providing -1 means that the iterator will work until StopIteration is raised - from the inside of iter_setup(). The options `last_batch_policy` and - `last_batch_padded` don't work in such case. It works with only one pipeline inside - the iterator. - Mutually exclusive with `reader_name` argument + Number of samples in the shard. For multiple pipelines, this is the sum of + their shard sizes. + Mutually exclusive with `reader_name`. When left at -1 without + `reader_name`, a single-pipeline iterator reads until the pipeline raises + `StopIteration`, for example when an external source is exhausted; + `last_batch_policy` and `last_batch_padded` do not apply. reader_name : str, default = None - Name of the reader which will be queried for the shard size, number of shards and - all other properties necessary to count properly the number of relevant and padded - samples that iterator needs to deal with. It automatically sets `last_batch_padded` - accordingly to match the reader's configuration. + Name of the reader operator that determines the iterator length and + last-batch padding. It must match the reader's `name` argument in every + supplied pipeline. + When set, `size` and `last_batch_padded` are determined automatically and + must not be provided. It does not change `last_batch_policy`. auto_reset : string or bool, optional, default = False Whether the iterator resets itself for the next epoch or it requires reset() to be called explicitly. @@ -350,15 +346,11 @@ def peekable_data_iterator( to fully fill it. See :meth:`nvidia.dali.plugin.base_iterator.LastBatchPolicy`. JAX iterator does not support LastBatchPolicy.PARTIAL last_batch_padded : bool, optional, default = False - Whether the last batch provided by DALI is padded with the last sample - or it just wraps up. In the conjunction with `last_batch_policy` it tells - if the iterator returning last batch with data only partially filled with - data from the current epoch is dropping padding samples or samples from - the next epoch. If set to ``False`` next - epoch will end sooner as data from it was consumed but dropped. If set to - True next epoch would be the same length as the first one. For this to happen, - the option `pad_last_batch` in the reader needs to be set to True as well. - It is overwritten when `reader_name` argument is provided + Whether the reader pads the last batch by repeating its last sample + (`True`) or continues into the next epoch (`False`). + Without `reader_name`, set this to the same value as the reader's + `pad_last_batch` argument. With `reader_name`, it is determined + automatically and must not be provided. prepare_first_batch : bool, optional, default = True Whether DALI should buffer the first batch right after the creation of the iterator, so one batch is already prepared when the iterator is prompted for the data diff --git a/dali/python/nvidia/dali/plugin/jax/iterator.py b/dali/python/nvidia/dali/plugin/jax/iterator.py index e1e1e445f03..a982f4456cd 100644 --- a/dali/python/nvidia/dali/plugin/jax/iterator.py +++ b/dali/python/nvidia/dali/plugin/jax/iterator.py @@ -43,18 +43,18 @@ class DALIGenericIterator(_DaliBaseIterator): of those names. Each name should be distinct size : int, default = -1 - Number of samples in the shard for the wrapped pipeline (if there is more than - one it is a sum) - Providing -1 means that the iterator will work until StopIteration is raised - from the inside of iter_setup(). The options `last_batch_policy` and - `last_batch_padded` don't work in such case. It works with only one pipeline inside - the iterator. - Mutually exclusive with `reader_name` argument + Number of samples in the shard. For multiple pipelines, this is the sum of + their shard sizes. + Mutually exclusive with `reader_name`. When left at -1 without + `reader_name`, a single-pipeline iterator reads until the pipeline raises + `StopIteration`, for example when an external source is exhausted; + `last_batch_policy` and `last_batch_padded` do not apply. reader_name : str, default = None - Name of the reader which will be queried for the shard size, number of shards and - all other properties necessary to count properly the number of relevant and padded - samples that iterator needs to deal with. It automatically sets `last_batch_padded` - accordingly to match the reader's configuration. + Name of the reader operator that determines the iterator length and + last-batch padding. It must match the reader's `name` argument in every + supplied pipeline. + When set, `size` and `last_batch_padded` are determined automatically and + must not be provided. It does not change `last_batch_policy`. auto_reset : string or bool, optional, default = False Whether the iterator resets itself for the next epoch or it requires reset() to be called explicitly. @@ -70,15 +70,11 @@ class DALIGenericIterator(_DaliBaseIterator): to fully fill it. See :meth:`nvidia.dali.plugin.base_iterator.LastBatchPolicy`. JAX iterator does not support LastBatchPolicy.PARTIAL last_batch_padded : bool, optional, default = False - Whether the last batch provided by DALI is padded with the last sample - or it just wraps up. In the conjunction with `last_batch_policy` it tells - if the iterator returning last batch with data only partially filled with - data from the current epoch is dropping padding samples or samples from - the next epoch. If set to ``False`` next - epoch will end sooner as data from it was consumed but dropped. If set to - True next epoch would be the same length as the first one. For this to happen, - the option `pad_last_batch` in the reader needs to be set to True as well. - It is overwritten when `reader_name` argument is provided + Whether the reader pads the last batch by repeating its last sample + (`True`) or continues into the next epoch (`False`). + Without `reader_name`, set this to the same value as the reader's + `pad_last_batch` argument. With `reader_name`, it is determined + automatically and must not be provided. prepare_first_batch : bool, optional, default = True Whether DALI should buffer the first batch right after the creation of the iterator, so one batch is already prepared when the iterator is prompted for the data @@ -444,18 +440,18 @@ def data_iterator( of those names. Each name should be distinct size : int, default = -1 - Number of samples in the shard for the wrapped pipeline (if there is more than - one it is a sum) - Providing -1 means that the iterator will work until StopIteration is raised - from the inside of iter_setup(). The options `last_batch_policy` and - `last_batch_padded` don't work in such case. It works with only one pipeline inside - the iterator. - Mutually exclusive with `reader_name` argument + Number of samples in the shard. For multiple pipelines, this is the sum of + their shard sizes. + Mutually exclusive with `reader_name`. When left at -1 without + `reader_name`, a single-pipeline iterator reads until the pipeline raises + `StopIteration`, for example when an external source is exhausted; + `last_batch_policy` and `last_batch_padded` do not apply. reader_name : str, default = None - Name of the reader which will be queried for the shard size, number of shards and - all other properties necessary to count properly the number of relevant and padded - samples that iterator needs to deal with. It automatically sets `last_batch_padded` - accordingly to match the reader's configuration. + Name of the reader operator that determines the iterator length and + last-batch padding. It must match the reader's `name` argument in every + supplied pipeline. + When set, `size` and `last_batch_padded` are determined automatically and + must not be provided. It does not change `last_batch_policy`. auto_reset : string or bool, optional, default = False Whether the iterator resets itself for the next epoch or it requires reset() to be called explicitly. @@ -471,15 +467,11 @@ def data_iterator( to fully fill it. See :meth:`nvidia.dali.plugin.base_iterator.LastBatchPolicy`. JAX iterator does not support LastBatchPolicy.PARTIAL last_batch_padded : bool, optional, default = False - Whether the last batch provided by DALI is padded with the last sample - or it just wraps up. In the conjunction with `last_batch_policy` it tells - if the iterator returning last batch with data only partially filled with - data from the current epoch is dropping padding samples or samples from - the next epoch. If set to ``False`` next - epoch will end sooner as data from it was consumed but dropped. If set to - True next epoch would be the same length as the first one. For this to happen, - the option `pad_last_batch` in the reader needs to be set to True as well. - It is overwritten when `reader_name` argument is provided + Whether the reader pads the last batch by repeating its last sample + (`True`) or continues into the next epoch (`False`). + Without `reader_name`, set this to the same value as the reader's + `pad_last_batch` argument. With `reader_name`, it is determined + automatically and must not be provided. prepare_first_batch : bool, optional, default = True Whether DALI should buffer the first batch right after the creation of the iterator, so one batch is already prepared when the iterator is prompted for the data diff --git a/dali/python/nvidia/dali/plugin/paddle.py b/dali/python/nvidia/dali/plugin/paddle.py index 2da56cc59b1..b1cba37400a 100644 --- a/dali/python/nvidia/dali/plugin/paddle.py +++ b/dali/python/nvidia/dali/plugin/paddle.py @@ -147,18 +147,18 @@ class DALIGenericIterator(_DaliBaseIterator): Item can also be a pair of (str, int), where the int value specifies the LoD level of the resulting LoDTensor. size : int, default = -1 - Number of samples in the shard for the wrapped pipeline (if there is more than - one it is a sum) - Providing -1 means that the iterator will work until StopIteration is raised - from the inside of iter_setup(). The options `last_batch_policy` and - `last_batch_padded` don't work in such case. It works with only one pipeline inside - the iterator. - Mutually exclusive with `reader_name` argument + Number of samples in the shard. For multiple pipelines, this is the sum of + their shard sizes. + Mutually exclusive with `reader_name`. When left at -1 without + `reader_name`, a single-pipeline iterator reads until the pipeline raises + `StopIteration`, for example when an external source is exhausted; + `last_batch_policy` and `last_batch_padded` do not apply. reader_name : str, default = None - Name of the reader which will be queried for the shard size, number of shards and - all other properties necessary to count properly the number of relevant and padded - samples that iterator needs to deal with. It automatically sets - `last_batch_padded` accordingly to match the reader's configuration. + Name of the reader operator that determines the iterator length and + last-batch padding. It must match the reader's `name` argument in every + supplied pipeline. + When set, `size` and `last_batch_padded` are determined automatically and + must not be provided. It does not change `last_batch_policy`. auto_reset : string or bool, optional, default = False Whether the iterator resets itself for the next epoch or it requires reset() to be called explicitly. @@ -184,15 +184,11 @@ class DALIGenericIterator(_DaliBaseIterator): What to do with the last batch when there are not enough samples in the epoch to fully fill it. See :meth:`nvidia.dali.plugin.base_iterator.LastBatchPolicy` last_batch_padded : bool, optional, default = False - Whether the last batch provided by DALI is padded with the last sample - or it just wraps up. In the conjunction with `last_batch_policy` it tells - if the iterator returning last batch with data only partially filled with - data from the current epoch is dropping padding samples or samples from - the next epoch. If set to ``False`` next - epoch will end sooner as data from it was consumed but dropped. If set to - True next epoch would be the same length as the first one. For this to happen, - the option `pad_last_batch` in the reader needs to be set to True as well. - It is overwritten when `reader_name` argument is provided + Whether the reader pads the last batch by repeating its last sample + (`True`) or continues into the next epoch (`False`). + Without `reader_name`, set this to the same value as the reader's + `pad_last_batch` argument. With `reader_name`, it is determined + automatically and must not be provided. prepare_first_batch : bool, optional, default = True Whether DALI should buffer the first batch right after the creation of the iterator, so one batch is already prepared when the iterator is prompted for the data @@ -413,18 +409,18 @@ class DALIClassificationIterator(DALIGenericIterator): pipelines : list of nvidia.dali.Pipeline List of pipelines to use size : int, default = -1 - Number of samples in the shard for the wrapped pipeline (if there is more than - one it is a sum) - Providing -1 means that the iterator will work until StopIteration is raised - from the inside of iter_setup(). The options `last_batch_policy` and - `last_batch_padded` don't work in such case. It works with only one pipeline inside - the iterator. - Mutually exclusive with `reader_name` argument + Number of samples in the shard. For multiple pipelines, this is the sum of + their shard sizes. + Mutually exclusive with `reader_name`. When left at -1 without + `reader_name`, a single-pipeline iterator reads until the pipeline raises + `StopIteration`, for example when an external source is exhausted; + `last_batch_policy` and `last_batch_padded` do not apply. reader_name : str, default = None - Name of the reader which will be queried for the shard size, number of shards and - all other properties necessary to count properly the number of relevant and padded - samples that iterator needs to deal with. It automatically sets - `last_batch_padded` accordingly to match the reader's configuration. + Name of the reader operator that determines the iterator length and + last-batch padding. It must match the reader's `name` argument in every + supplied pipeline. + When set, `size` and `last_batch_padded` are determined automatically and + must not be provided. It does not change `last_batch_policy`. auto_reset : string or bool, optional, default = False Whether the iterator resets itself for the next epoch or it requires reset() to be called explicitly. @@ -450,15 +446,11 @@ class DALIClassificationIterator(DALIGenericIterator): What to do with the last batch when there are not enough samples in the epoch to fully fill it. See :meth:`nvidia.dali.plugin.base_iterator.LastBatchPolicy` last_batch_padded : bool, optional, default = False - Whether the last batch provided by DALI is padded with the last sample - or it just wraps up. In the conjunction with `last_batch_policy` it tells - if the iterator returning last batch with data only partially filled with - data from the current epoch is dropping padding samples or samples from - the next epoch. If set to ``False`` next - epoch will end sooner as data from it was consumed but dropped. If set to - True next epoch would be the same length as the first one. For this to happen, - the option `pad_last_batch` in the reader needs to be set to True as well. - It is overwritten when `reader_name` argument is provided + Whether the reader pads the last batch by repeating its last sample + (`True`) or continues into the next epoch (`False`). + Without `reader_name`, set this to the same value as the reader's + `pad_last_batch` argument. With `reader_name`, it is determined + automatically and must not be provided. prepare_first_batch : bool, optional, default = True Whether DALI should buffer the first batch right after the creation of the iterator, so one batch is already prepared when the iterator is prompted for the data diff --git a/dali/python/nvidia/dali/plugin/pytorch/__init__.py b/dali/python/nvidia/dali/plugin/pytorch/__init__.py index 786035cafac..2afe0e90078 100644 --- a/dali/python/nvidia/dali/plugin/pytorch/__init__.py +++ b/dali/python/nvidia/dali/plugin/pytorch/__init__.py @@ -56,18 +56,18 @@ class DALIGenericIterator(_DaliBaseIterator): of those names. Each name should be distinct size : int, default = -1 - Number of samples in the shard for the wrapped pipeline (if there is more than - one it is a sum) - Providing -1 means that the iterator will work until StopIteration is raised - from the inside of iter_setup(). The options `last_batch_policy` and - `last_batch_padded` don't work in such case. It works with only one pipeline inside - the iterator. - Mutually exclusive with `reader_name` argument + Number of samples in the shard. For multiple pipelines, this is the sum of + their shard sizes. + Mutually exclusive with `reader_name`. When left at -1 without + `reader_name`, a single-pipeline iterator reads until the pipeline raises + `StopIteration`, for example when an external source is exhausted; + `last_batch_policy` and `last_batch_padded` do not apply. reader_name : str, default = None - Name of the reader which will be queried for the shard size, number of shards and - all other properties necessary to count properly the number of relevant and padded - samples that iterator needs to deal with. It automatically sets - `last_batch_padded` accordingly to match the reader's configuration. + Name of the reader operator that determines the iterator length and + last-batch padding. It must match the reader's `name` argument in every + supplied pipeline. + When set, `size` and `last_batch_padded` are determined automatically and + must not be provided. It does not change `last_batch_policy`. auto_reset : string or bool, optional, default = False Whether the iterator resets itself for the next epoch or it requires reset() to be called explicitly. @@ -93,15 +93,11 @@ class DALIGenericIterator(_DaliBaseIterator): What to do with the last batch when there are not enough samples in the epoch to fully fill it. See :meth:`nvidia.dali.plugin.base_iterator.LastBatchPolicy` last_batch_padded : bool, optional, default = False - Whether the last batch provided by DALI is padded with the last sample - or it just wraps up. In the conjunction with `last_batch_policy` it tells - if the iterator returning last batch with data only partially filled with - data from the current epoch is dropping padding samples or samples from - the next epoch. If set to ``False`` next - epoch will end sooner as data from it was consumed but dropped. If set to - True next epoch would be the same length as the first one. For this to happen, - the option `pad_last_batch` in the reader needs to be set to True as well. - It is overwritten when `reader_name` argument is provided + Whether the reader pads the last batch by repeating its last sample + (`True`) or continues into the next epoch (`False`). + Without `reader_name`, set this to the same value as the reader's + `pad_last_batch` argument. With `reader_name`, it is determined + automatically and must not be provided. prepare_first_batch : bool, optional, default = True Whether DALI should buffer the first batch right after the creation of the iterator, so one batch is already prepared when the iterator is prompted for the data @@ -308,18 +304,18 @@ class DALIClassificationIterator(DALIGenericIterator): pipelines : list of nvidia.dali.Pipeline List of pipelines to use size : int, default = -1 - Number of samples in the shard for the wrapped pipeline (if there is more than - one it is a sum) - Providing -1 means that the iterator will work until StopIteration is raised - from the inside of iter_setup(). The options `last_batch_policy` and - `last_batch_padded` don't work in such case. It works with only one pipeline inside - the iterator. - Mutually exclusive with `reader_name` argument + Number of samples in the shard. For multiple pipelines, this is the sum of + their shard sizes. + Mutually exclusive with `reader_name`. When left at -1 without + `reader_name`, a single-pipeline iterator reads until the pipeline raises + `StopIteration`, for example when an external source is exhausted; + `last_batch_policy` and `last_batch_padded` do not apply. reader_name : str, default = None - Name of the reader which will be queried for the shard size, number of shards and - all other properties necessary to count properly the number of relevant and padded - samples that iterator needs to deal with. It automatically sets - `last_batch_padded` accordingly to match the reader's configuration. + Name of the reader operator that determines the iterator length and + last-batch padding. It must match the reader's `name` argument in every + supplied pipeline. + When set, `size` and `last_batch_padded` are determined automatically and + must not be provided. It does not change `last_batch_policy`. auto_reset : string or bool, optional, default = False Whether the iterator resets itself for the next epoch or it requires reset() to be called explicitly. @@ -345,15 +341,11 @@ class DALIClassificationIterator(DALIGenericIterator): What to do with the last batch when there are not enough samples in the epoch to fully fill it. See :meth:`nvidia.dali.plugin.base_iterator.LastBatchPolicy` last_batch_padded : bool, optional, default = False - Whether the last batch provided by DALI is padded with the last sample - or it just wraps up. In the conjunction with `last_batch_policy` it tells - if the iterator returning last batch with data only partially filled with - data from the current epoch is dropping padding samples or samples from - the next epoch. If set to ``False`` next - epoch will end sooner as data from it was consumed but dropped. If set to - True next epoch would be the same length as the first one. For this to happen, - the option `pad_last_batch` in the reader needs to be set to True as well. - It is overwritten when `reader_name` argument is provided + Whether the reader pads the last batch by repeating its last sample + (`True`) or continues into the next epoch (`False`). + Without `reader_name`, set this to the same value as the reader's + `pad_last_batch` argument. With `reader_name`, it is determined + automatically and must not be provided. prepare_first_batch : bool, optional, default = True Whether DALI should buffer the first batch right after the creation of the iterator, so one batch is already prepared when the iterator is prompted for the data @@ -424,19 +416,18 @@ class DALIRaggedIterator(_DaliBaseIterator): pipelines : list of nvidia.dali.Pipeline List of pipelines to use size : int, default = -1 - Number of samples in the shard for the wrapped pipeline (if there is more than - one it is a sum) - Providing -1 means that the iterator will work until StopIteration is raised - from the inside of iter_setup(). The options `last_batch_policy` and - `last_batch_padded` don't work in such case. It works with only one pipeline inside - the iterator. - Mutually exclusive with `reader_name` argument + Number of samples in the shard. For multiple pipelines, this is the sum of + their shard sizes. + Mutually exclusive with `reader_name`. When left at -1 without + `reader_name`, a single-pipeline iterator reads until the pipeline raises + `StopIteration`, for example when an external source is exhausted; + `last_batch_policy` and `last_batch_padded` do not apply. reader_name : str, default = None - Name of the reader which will be queried for the shard size, number of shards and - all other properties necessary to count properly the number of relevant and padded - samples that iterator needs to deal with. It automatically sets `last_batch_policy` - to PARTIAL when the FILL is used, and `last_batch_padded` accordingly to match - the reader's configuration + Name of the reader operator that determines the iterator length and + last-batch padding. It must match the reader's `name` argument in every + supplied pipeline. + When set, `size` and `last_batch_padded` are determined automatically and + must not be provided. It does not change `last_batch_policy`. output_types : list of str, optional, default = None List of tags indicating whether the pipeline(s) output batch is uniform (all the samples have the same size) or not. Batch output marked @@ -472,16 +463,11 @@ class DALIRaggedIterator(_DaliBaseIterator): What to do with the last batch when there are not enough samples in the epoch to fully fill it. See :meth:`nvidia.dali.plugin.base_iterator.LastBatchPolicy` last_batch_padded : bool, optional, default = False - Whether the last batch provided by DALI is padded with the last sample - or it just wraps up. In the conjunction with `last_batch_policy` it tells - if the iterator returning last batch with data only partially filled with - data from the current epoch is dropping padding samples or samples from - the next epoch (it doesn't literally drop but sets ``pad`` field of ndarray - so the following code could use it to drop the data). If set to ``False`` next - epoch will end sooner as data from it was consumed but dropped. If set to - True next epoch would be the same length as the first one. For this to happen, - the option `pad_last_batch` in the reader needs to be set to True as well. - It is overwritten when `reader_name` argument is provided + Whether the reader pads the last batch by repeating its last sample + (`True`) or continues into the next epoch (`False`). + Without `reader_name`, set this to the same value as the reader's + `pad_last_batch` argument. With `reader_name`, it is determined + automatically and must not be provided. prepare_first_batch : bool, optional, default = True Whether DALI should buffer the first batch right after the creation of the iterator, so one batch is already prepared when the iterator is prompted for the data diff --git a/docs/advanced_topics_sharding.rst b/docs/advanced_topics_sharding.rst index c2b26cb668a..4dfeea1cd9b 100644 --- a/docs/advanced_topics_sharding.rst +++ b/docs/advanced_topics_sharding.rst @@ -25,29 +25,52 @@ to be aware of this padding and other reader properties. Here are the iterator options: -- | ``reader_name`` - Allows you to provide the name of the reader that drives the iterator and - provides the necessary parameters. - - .. note:: - We recommend that you use this option, so that the next two options - (``size`` and ``last_batch_padded``) are obtained automatically from the pipeline configuration. - If it is used, the ``size`` and ``last_batch_padded`` should not be provided explicitly to - the iterator. - - | This option is more flexible and accurate and takes into account that shard size for a pipeline - can differ between epochs when the shards are rotated. -- ``size``: Provides the size of the shard for an iterator or, if there is more than one shard, - the sum of all shard sizes for all wrapped pipelines. -- | ``last_batch_padded``: Determines whether the tail of the data consists of data from the next - shard (``False``) or is duplicated dummy data (``True``). - | It is applicable when the shard size is not a multiple of the batch size, -- | ``last_batch_policy`` - Determines the handling of the last batch when the shard size is not - divisible by the batch size. - | It affects batches only partially filled with the data. See - :meth:`~nvidia.dali.plugin.base_iterator.LastBatchPolicy` enum for possible values.. - -- ``fill_last_batch`` – (Deprecated in favour of ``last_batch_policy``) Determines whether the last - batch should be full, regardless of whether the shard size is divisible by the batch size. +``reader_name`` + Name of the reader operator that provides the iterator's size and last-batch + padding. It must match the reader's ``name`` argument in every supplied + pipeline. + + For example, use matching names: + + .. code-block:: python + + @pipeline_def(batch_size=64, num_threads=4, device_id=0) + def pipeline(): + return fn.readers.file(file_root="/path/to/images", name="train_reader") + + iterator = DALIGenericIterator( + pipeline(), ["images", "labels"], reader_name="train_reader" + ) + + If a matching, compatible reader is not present in every pipeline, iterator + construction fails. Providing ``reader_name`` does not change + ``last_batch_policy``. + + .. tip:: + + Prefer ``reader_name`` to setting ``size`` and ``last_batch_padded`` + manually. DALI then keeps the iterator length and padding aligned with the + reader configuration, including when shards rotate between epochs. + +``size`` + Provides the size of the shard for an iterator or, if there is more than one + shard, the sum of all shard sizes for all wrapped pipelines. + +``last_batch_padded`` + Whether the reader pads the last batch by repeating its last sample + (``True``) or continues into the next epoch (``False``). It applies when + the shard size is not a multiple of the batch size. + +``last_batch_policy`` + Determines the handling of the last batch when the shard size is not divisible + by the batch size. It affects batches only partially filled with the data. See + :meth:`~nvidia.dali.plugin.base_iterator.LastBatchPolicy` enum for possible + values. + +``fill_last_batch`` + Deprecated in favour of ``last_batch_policy``. Determines whether the last + batch should be full, regardless of whether the shard size is divisible by the + batch size. Enums ~~~~~ @@ -77,4 +100,3 @@ When this occurs, use the first formula. To address these challenges, use the ``reader_name`` parameter and allow the iterator to handle the configuration automatically. -