class GenerationConfig(PushToHubMixin):
    extra_output_flags = ('output_attentions', 'output_hidden_states', 'output_scores', 'output_logits')

    def __init__(self, **kwargs):
        self.max_length = kwargs.pop('max_length', 20)
        self.max_new_tokens = kwargs.pop('max_new_tokens', None)
        self.min_length = kwargs.pop('min_length', 0)
        self.min_new_tokens = kwargs.pop('min_new_tokens', None)
        self.early_stopping = kwargs.pop('early_stopping', False)
        self.max_time = kwargs.pop('max_time', None)
        self.stop_strings = kwargs.pop('stop_strings', None)
        self.do_sample = kwargs.pop('do_sample', False)
        self.num_beams = kwargs.pop('num_beams', 1)
        self.num_beam_groups = kwargs.pop('num_beam_groups', 1)
        self.penalty_alpha = kwargs.pop('penalty_alpha', None)
        self.dola_layers = kwargs.pop('dola_layers', None)
        self.use_cache = kwargs.pop('use_cache', True)
        self.cache_implementation = kwargs.pop('cache_implementation', None)
        self.cache_config = kwargs.pop('cache_config', None)
        if self.cache_implementation is not None and self.cache_implementation in CACHE_CONFIG_MAPPING:
            cache_config_class = CACHE_CONFIG_MAPPING[self.cache_implementation]
            if isinstance(self.cache_config, dict):
                self.cache_config = cache_config_class.from_dict(self.cache_config)
        self.return_legacy_cache = kwargs.pop('return_legacy_cache', None)
        self.prefill_chunk_size = kwargs.pop('prefill_chunk_size', None)
        self.temperature = kwargs.pop('temperature', 1.0)
        self.top_k = kwargs.pop('top_k', 50)
        self.top_p = kwargs.pop('top_p', 1.0)
        self.min_p = kwargs.pop('min_p', None)
        self.typical_p = kwargs.pop('typical_p', 1.0)
        self.epsilon_cutoff = kwargs.pop('epsilon_cutoff', 0.0)
        self.eta_cutoff = kwargs.pop('eta_cutoff', 0.0)
        self.diversity_penalty = kwargs.pop('diversity_penalty', 0.0)
        self.repetition_penalty = kwargs.pop('repetition_penalty', 1.0)
        self.encoder_repetition_penalty = kwargs.pop('encoder_repetition_penalty', 1.0)
        self.length_penalty = kwargs.pop('length_penalty', 1.0)
        self.no_repeat_ngram_size = kwargs.pop('no_repeat_ngram_size', 0)
        self.bad_words_ids = kwargs.pop('bad_words_ids', None)
        self.force_words_ids = kwargs.pop('force_words_ids', None)
        self.renormalize_logits = kwargs.pop('renormalize_logits', False)
        self.constraints = kwargs.pop('constraints', None)
        self.forced_bos_token_id = kwargs.pop('forced_bos_token_id', None)
        self.forced_eos_token_id = kwargs.pop('forced_eos_token_id', None)
        self.remove_invalid_values = kwargs.pop('remove_invalid_values', False)
        self.exponential_decay_length_penalty = kwargs.pop('exponential_decay_length_penalty', None)
        self.suppress_tokens = kwargs.pop('suppress_tokens', None)
        self.begin_suppress_tokens = kwargs.pop('begin_suppress_tokens', None)
        self.sequence_bias = kwargs.pop('sequence_bias', None)
        self.token_healing = kwargs.pop('token_healing', False)
        self.guidance_scale = kwargs.pop('guidance_scale', None)
        self.low_memory = kwargs.pop('low_memory', None)
        watermarking_config = kwargs.pop('watermarking_config', None)
        if watermarking_config is None:
            self.watermarking_config = None
        elif isinstance(watermarking_config, BaseWatermarkingConfig):
            self.watermarking_config = watermarking_config
        else:
            self.watermarking_config = WatermarkingConfig.from_dict(watermarking_config)
        self.num_return_sequences = kwargs.pop('num_return_sequences', 1)
        self.output_attentions = kwargs.pop('output_attentions', False)
        self.output_hidden_states = kwargs.pop('output_hidden_states', False)
        self.output_scores = kwargs.pop('output_scores', False)
        self.output_logits = kwargs.pop('output_logits', None)
        self.return_dict_in_generate = kwargs.pop('return_dict_in_generate', False)
        self.pad_token_id = kwargs.pop('pad_token_id', None)
        self.bos_token_id = kwargs.pop('bos_token_id', None)
        self.eos_token_id = kwargs.pop('eos_token_id', None)
        self.encoder_no_repeat_ngram_size = kwargs.pop('encoder_no_repeat_ngram_size', 0)
        self.decoder_start_token_id = kwargs.pop('decoder_start_token_id', None)
        self.is_assistant = False
        self.num_assistant_tokens = kwargs.pop('num_assistant_tokens', 20)
        self.num_assistant_tokens_schedule = kwargs.pop('num_assistant_tokens_schedule', 'constant')
        self.assistant_confidence_threshold = kwargs.pop('assistant_confidence_threshold', 0.4)
        self.prompt_lookup_num_tokens = kwargs.pop('prompt_lookup_num_tokens', None)
        self.max_matching_ngram_size = kwargs.pop('max_matching_ngram_size', None)
        self.assistant_early_exit = kwargs.pop('assistant_early_exit', None)
        self.assistant_lookbehind = kwargs.pop('assistant_lookbehind', 10)
        self.target_lookbehind = kwargs.pop('target_lookbehind', 10)
        self.compile_config = kwargs.pop('compile_config', None)
        self.disable_compile = kwargs.pop('disable_compile', False)
        self._from_model_config = kwargs.pop('_from_model_config', False)
        self._commit_hash = kwargs.pop('_commit_hash', None)
        self.transformers_version = kwargs.pop('transformers_version', __version__)
        if not self._from_model_config:
            for key, value in kwargs.items():
                try:
                    setattr(self, key, value)
                except AttributeError as err:
                    logger.error(f"Can't set {key} with value {value} for {self}")
                    raise err
        self.validate()

    def __hash__(self):
        return hash(self.to_json_string(ignore_metadata=True))

    def __eq__(self, other):
        if not isinstance(other, GenerationConfig):
            return False
        self_without_metadata = self.to_json_string(use_diff=False, ignore_metadata=True)
        other_without_metadata = other.to_json_string(use_diff=False, ignore_metadata=True)
        return self_without_metadata == other_without_metadata

    def __repr__(self):
        return f'{self.__class__.__name__} {self.to_json_string(ignore_metadata=True)}'

    def get_generation_mode(self, assistant_model: Optional['PreTrainedModel']=None) -> GenerationMode:
        """
        Returns the generation mode triggered by the [`GenerationConfig`] instance.

        Arg:
            assistant_model (`PreTrainedModel`, *optional*):
                The assistant model to be used for assisted generation. If set, the generation mode will be
                assisted generation.

        Returns:
            `GenerationMode`: The generation mode triggered by the instance.
        """
        if self.constraints is not None or self.force_words_ids is not None:
            generation_mode = GenerationMode.CONSTRAINED_BEAM_SEARCH
        elif self.num_beams == 1:
            if self.do_sample is False:
                if self.top_k is not None and self.top_k > 1 and (self.penalty_alpha is not None) and (self.penalty_alpha > 0):
                    generation_mode = GenerationMode.CONTRASTIVE_SEARCH
                else:
                    generation_mode = GenerationMode.GREEDY_SEARCH
            else:
                generation_mode = GenerationMode.SAMPLE
        elif self.num_beam_groups > 1:
            generation_mode = GenerationMode.GROUP_BEAM_SEARCH
        elif self.do_sample is True:
            generation_mode = GenerationMode.BEAM_SAMPLE
        else:
            generation_mode = GenerationMode.BEAM_SEARCH
        if assistant_model is not None or self.prompt_lookup_num_tokens is not None or self.assistant_early_exit is not None:
            if generation_mode in ('greedy_search', 'sample'):
                generation_mode = GenerationMode.ASSISTED_GENERATION
            else:
                logger.warning(f"You've set `assistant_model`, which triggers assisted generate. Currently, assisted generate is only supported with Greedy Search and Sample. However, the base decoding mode (based on current flags) is {generation_mode} -- some of the set flags will be ignored.")
        if self.dola_layers is not None:
            if generation_mode in ('greedy_search', 'sample'):
                generation_mode = GenerationMode.DOLA_GENERATION
            else:
                logger.warning(f"You've set `dola_layers`, which triggers DoLa generate. Currently, DoLa generate is only supported with Greedy Search and Sample.  However, the base decoding mode (based on current flags) is {generation_mode} -- some of the set flags will be ignored.")
        return generation_mode

    @deprecate_kwarg('is_init', version='4.54.0')
    def validate(self, strict=False):
        """
        Validates the values of the attributes of the [`GenerationConfig`] instance. Raises exceptions in the presence
        of parameterization that can be detected as incorrect from the configuration instance alone.

        Note that some parameters not validated here are best validated at generate runtime, as they may depend on
        other inputs and/or the model, such as parameters related to the generation length.

        Args:
            strict (bool): If True, raise an exception for any issues found. If False, only log issues.
        """
        minor_issues = {}
        if self.early_stopping not in {True, False, 'never'}:
            raise ValueError(f"`early_stopping` must be a boolean or 'never', but is {self.early_stopping}.")
        if self.max_new_tokens is not None and self.max_new_tokens <= 0:
            raise ValueError(f'`max_new_tokens` must be greater than 0, but is {self.max_new_tokens}.')
        if self.pad_token_id is not None and self.pad_token_id < 0:
            minor_issues['pad_token_id'] = f'`pad_token_id` should be positive but got {self.pad_token_id}. This will cause errors when batch generating, if there is padding. Please set `pad_token_id` explicitly as `model.generation_config.pad_token_id=PAD_TOKEN_ID` to avoid errors in generation'
        if self.cache_implementation is not None and self.cache_implementation not in ALL_CACHE_IMPLEMENTATIONS:
            raise ValueError(f'Invalid `cache_implementation` ({self.cache_implementation}). Choose one of: {ALL_CACHE_IMPLEMENTATIONS}')
        if self.cache_config is not None:
            cache_class = CACHE_CONFIG_MAPPING.get(self.cache_implementation)
            if cache_class is None:
                raise ValueError(f'You provided a `cache_config` but the cache implementation you are using ({self.cache_implementation}) does not require any config. Make sure to use the correct cache implementation matching your cache config.')
            if not isinstance(self.cache_config, cache_class):
                self.cache_config = cache_class.from_dict(self.cache_config)
            self.cache_config.validate()
        if self.compile_config is not None and (not isinstance(self.compile_config, CompileConfig)):
            raise ValueError(f'You provided `compile_config` as an instance of {type(self.compile_config)}, but it must be an instance of `CompileConfig`.')
        if self.watermarking_config is not None:
            if not (isinstance(self.watermarking_config, WatermarkingConfig) or isinstance(self.watermarking_config, SynthIDTextWatermarkingConfig)):
                minor_issues['watermarking_config'] = '`watermarking_config` as a dict is deprecated and will be removed in v4.54.0. Please construct `watermarking_config` object with `WatermarkingConfig` or `SynthIDTextWatermarkingConfig` class.'
                self.watermarking_config = WatermarkingConfig.from_dict(self.watermarking_config)
            self.watermarking_config.validate()
        if self.do_sample is False:
            greedy_wrong_parameter_msg = '`do_sample` is set to `False`. However, `{flag_name}` is set to `{flag_value}` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `{flag_name}`.'
            if self.temperature is not None and self.temperature != 1.0:
                minor_issues['temperature'] = greedy_wrong_parameter_msg.format(flag_name='temperature', flag_value=self.temperature)
            if self.top_p is not None and self.top_p != 1.0:
                minor_issues['top_p'] = greedy_wrong_parameter_msg.format(flag_name='top_p', flag_value=self.top_p)
            if self.min_p is not None:
                minor_issues['min_p'] = greedy_wrong_parameter_msg.format(flag_name='min_p', flag_value=self.min_p)
            if self.typical_p is not None and self.typical_p != 1.0:
                minor_issues['typical_p'] = greedy_wrong_parameter_msg.format(flag_name='typical_p', flag_value=self.typical_p)
            if self.top_k is not None and self.top_k != 50 and (self.penalty_alpha is None):
                minor_issues['top_k'] = greedy_wrong_parameter_msg.format(flag_name='top_k', flag_value=self.top_k)
            if self.epsilon_cutoff is not None and self.epsilon_cutoff != 0.0:
                minor_issues['epsilon_cutoff'] = greedy_wrong_parameter_msg.format(flag_name='epsilon_cutoff', flag_value=self.epsilon_cutoff)
            if self.eta_cutoff is not None and self.eta_cutoff != 0.0:
                minor_issues['eta_cutoff'] = greedy_wrong_parameter_msg.format(flag_name='eta_cutoff', flag_value=self.eta_cutoff)
        if self.num_beams == 1:
            single_beam_wrong_parameter_msg = '`num_beams` is set to 1. However, `{flag_name}` is set to `{flag_value}` -- this flag is only used in beam-based generation modes. You should set `num_beams>1` or unset `{flag_name}`.'
            if self.early_stopping is not False:
                minor_issues['early_stopping'] = single_beam_wrong_parameter_msg.format(flag_name='early_stopping', flag_value=self.early_stopping)
            if self.num_beam_groups is not None and self.num_beam_groups != 1:
                minor_issues['num_beam_groups'] = single_beam_wrong_parameter_msg.format(flag_name='num_beam_groups', flag_value=self.num_beam_groups)
            if self.diversity_penalty is not None and self.diversity_penalty != 0.0:
                minor_issues['diversity_penalty'] = single_beam_wrong_parameter_msg.format(flag_name='diversity_penalty', flag_value=self.diversity_penalty)
            if self.length_penalty is not None and self.length_penalty != 1.0:
                minor_issues['length_penalty'] = single_beam_wrong_parameter_msg.format(flag_name='length_penalty', flag_value=self.length_penalty)
            if self.constraints is not None:
                minor_issues['constraints'] = single_beam_wrong_parameter_msg.format(flag_name='constraints', flag_value=self.constraints)
            if self.dola_layers is not None and (self.repetition_penalty is None or self.repetition_penalty < 1.2):
                minor_issues['repetition_penalty'] = (f'`dola_layers` is set to trigger DoLa decoding, but `repetition_penalty` is set to a value of {self.repetition_penalty}, which could induce unwanted repetition. The recommended value for DoLa decoding is `repetition_penalty>=1.2`.',)
        elif self.constraints is not None or self.force_words_ids is not None:
            constrained_wrong_parameter_msg = 'one of `constraints`, `force_words_ids` is not `None`, triggering constrained beam search. However, `{flag_name}` is set to `{flag_value}`, which is incompatible with this generation mode. Set `constraints` and `force_words_ids` to `None` or unset `{flag_name}` to continue.'
            if self.do_sample is True:
                raise ValueError(constrained_wrong_parameter_msg.format(flag_name='do_sample', flag_value=self.do_sample))
            if self.num_beam_groups is not None and self.num_beam_groups != 1:
                raise ValueError(constrained_wrong_parameter_msg.format(flag_name='num_beam_groups', flag_value=self.num_beam_groups))
        elif self.diversity_penalty != 0.0 or self.num_beam_groups != 1:
            group_error_prefix = '`diversity_penalty` is not 0.0 or `num_beam_groups` is not 1, triggering group beam search. In this generation mode, '
            if self.do_sample is True:
                raise ValueError(group_error_prefix + '`do_sample` must be set to `False`')
            if self.num_beams % self.num_beam_groups != 0:
                raise ValueError(group_error_prefix + '`num_beams` should be divisible by `num_beam_groups`')
            if self.diversity_penalty == 0.0:
                raise ValueError(group_error_prefix + '`diversity_penalty` should be greater than `0.0`, otherwise your groups will be identical.')
        if self.num_return_sequences != 1:
            if self.num_beams == 1:
                if self.do_sample is False:
                    raise ValueError(f'Greedy methods without beam search do not support `num_return_sequences` different than 1 (got {self.num_return_sequences}).')
            elif self.num_return_sequences > self.num_beams:
                raise ValueError(f'`num_return_sequences` ({self.num_return_sequences}) has to be smaller or equal to `num_beams` ({self.num_beams}).')
        if self.use_cache is False:
            no_cache_warning = 'You have set `use_cache` to `False`, but {cache_arg} is set to {cache_arg_value}. {cache_arg} will have no effect.'
            for arg_name in ('cache_implementation', 'cache_config', 'return_legacy_cache'):
                if getattr(self, arg_name) is not None:
                    minor_issues[arg_name] = no_cache_warning.format(cache_arg=arg_name, cache_arg_value=getattr(self, arg_name))
        if self.return_dict_in_generate is not True:
            for extra_output_flag in self.extra_output_flags:
                if getattr(self, extra_output_flag) is True:
                    minor_issues[extra_output_flag] = f'`return_dict_in_generate` is NOT set to `True`, but `{extra_output_flag}` is. When `return_dict_in_generate` is not `True`, `{extra_output_flag}` is ignored.'
        generate_arguments = ('logits_processor', 'stopping_criteria', 'prefix_allowed_tokens_fn', 'synced_gpus', 'assistant_model', 'streamer', 'negative_prompt_ids', 'negative_prompt_attention_mask', 'use_model_defaults')
        for arg in generate_arguments:
            if hasattr(self, arg):
                raise ValueError(f'Argument `{arg}` is not a valid argument of `GenerationConfig`. It should be passed to `generate()` (or a pipeline) directly.')
        if len(minor_issues) > 0:
            info_message = []
            for attribute_name, issue_description in minor_issues.items():
                info_message.append(f'- `{attribute_name}`: {issue_description}')
            info_message = '\n'.join(info_message)
            info_message += "\nIf you're using a pretrained model, note that some of these attributes may be set through the model's `generation_config.json` file."
            if strict:
                raise ValueError('GenerationConfig is invalid: \n' + info_message)
            else:
                attributes_with_issues = list(minor_issues.keys())
                warning_message = f'The following generation flags are not valid and may be ignored: {attributes_with_issues}.'
                if logging.get_verbosity() >= logging.WARNING:
                    warning_message += ' Set `TRANSFORMERS_VERBOSITY=info` for more details.'
                logger.warning(warning_message)
                logger.info(info_message)

    def save_pretrained(self, save_directory: Union[str, os.PathLike], config_file_name: Optional[Union[str, os.PathLike]]=None, push_to_hub: bool=False, **kwargs):
        """
        Save a generation configuration object to the directory `save_directory`, so that it can be re-loaded using the
        [`~GenerationConfig.from_pretrained`] class method.

        Args:
            save_directory (`str` or `os.PathLike`):
                Directory where the configuration JSON file will be saved (will be created if it does not exist).
            config_file_name (`str` or `os.PathLike`, *optional*, defaults to `"generation_config.json"`):
                Name of the generation configuration JSON file to be saved in `save_directory`.
            push_to_hub (`bool`, *optional*, defaults to `False`):
                Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the
                repository you want to push to with `repo_id` (will default to the name of `save_directory` in your
                namespace).
            kwargs (`Dict[str, Any]`, *optional*):
                Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method.
        """
        try:
            self.validate(strict=True)
        except ValueError as exc:
            raise ValueError(str(exc) + '\n\nFix these issues to save the configuration.')
        use_auth_token = kwargs.pop('use_auth_token', None)
        if use_auth_token is not None:
            warnings.warn('The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.', FutureWarning)
            if kwargs.get('token', None) is not None:
                raise ValueError('`token` and `use_auth_token` are both specified. Please set only the argument `token`.')
            kwargs['token'] = use_auth_token
        config_file_name = config_file_name if config_file_name is not None else GENERATION_CONFIG_NAME
        if os.path.isfile(save_directory):
            raise AssertionError(f'Provided path ({save_directory}) should be a directory, not a file')
        os.makedirs(save_directory, exist_ok=True)
        if push_to_hub:
            commit_message = kwargs.pop('commit_message', None)
            repo_id = kwargs.pop('repo_id', save_directory.split(os.path.sep)[-1])
            repo_id = self._create_repo(repo_id, **kwargs)
            files_timestamps = self._get_files_timestamps(save_directory)
        output_config_file = os.path.join(save_directory, config_file_name)
        self.to_json_file(output_config_file, use_diff=True)
        logger.info(f'Configuration saved in {output_config_file}')
        if push_to_hub:
            self._upload_modified_files(save_directory, repo_id, files_timestamps, commit_message=commit_message, token=kwargs.get('token'))

    @classmethod
    def from_pretrained(cls, pretrained_model_name: Union[str, os.PathLike], config_file_name: Optional[Union[str, os.PathLike]]=None, cache_dir: Optional[Union[str, os.PathLike]]=None, force_download: bool=False, local_files_only: bool=False, token: Optional[Union[str, bool]]=None, revision: str='main', **kwargs) -> 'GenerationConfig':
        """
        Instantiate a [`GenerationConfig`] from a generation configuration file.

        Args:
            pretrained_model_name (`str` or `os.PathLike`):
                This can be either:

                - a string, the *model id* of a pretrained model configuration hosted inside a model repo on
                  huggingface.co.
                - a path to a *directory* containing a configuration file saved using the
                  [`~GenerationConfig.save_pretrained`] method, e.g., `./my_model_directory/`.
            config_file_name (`str` or `os.PathLike`, *optional*, defaults to `"generation_config.json"`):
                Name of the generation configuration JSON file to be loaded from `pretrained_model_name`.
            cache_dir (`str` or `os.PathLike`, *optional*):
                Path to a directory in which a downloaded pretrained model configuration should be cached if the
                standard cache should not be used.
            force_download (`bool`, *optional*, defaults to `False`):
                Whether or not to force to (re-)download the configuration files and override the cached versions if
                they exist.
            resume_download:
                Deprecated and ignored. All downloads are now resumed by default when possible.
                Will be removed in v5 of Transformers.
            proxies (`Dict[str, str]`, *optional*):
                A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
                'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
            token (`str` or `bool`, *optional*):
                The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use
                the token generated when running `huggingface-cli login` (stored in `~/.huggingface`).
            revision (`str`, *optional*, defaults to `"main"`):
                The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
                git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
                identifier allowed by git.

                <Tip>

                To test a pull request you made on the Hub, you can pass `revision="refs/pr/<pr_number>"`.

                </Tip>

            return_unused_kwargs (`bool`, *optional*, defaults to `False`):
                If `False`, then this function returns just the final configuration object.

                If `True`, then this functions returns a `Tuple(config, unused_kwargs)` where *unused_kwargs* is a
                dictionary consisting of the key/value pairs whose keys are not configuration attributes: i.e., the
                part of `kwargs` which has not been used to update `config` and is otherwise ignored.
            subfolder (`str`, *optional*, defaults to `""`):
                In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can
                specify the folder name here.
            kwargs (`Dict[str, Any]`, *optional*):
                The values in kwargs of any keys which are configuration attributes will be used to override the loaded
                values. Behavior concerning key/value pairs whose keys are *not* configuration attributes is controlled
                by the `return_unused_kwargs` keyword parameter.

        Returns:
            [`GenerationConfig`]: The configuration object instantiated from this pretrained model.

        Examples:

        ```python
        >>> from transformers import GenerationConfig

        >>> # Download configuration from huggingface.co and cache.
        >>> generation_config = GenerationConfig.from_pretrained("openai-community/gpt2")

        >>> # E.g. config was saved using *save_pretrained('./test/saved_model/')*
        >>> generation_config.save_pretrained("./test/saved_model/")
        >>> generation_config = GenerationConfig.from_pretrained("./test/saved_model/")

        >>> # You can also specify configuration names to your generation configuration file
        >>> generation_config.save_pretrained("./test/saved_model/", config_file_name="my_configuration.json")
        >>> generation_config = GenerationConfig.from_pretrained("./test/saved_model/", "my_configuration.json")

        >>> # If you'd like to try a minor variation to an existing configuration, you can also pass generation
        >>> # arguments to `.from_pretrained()`. Be mindful that typos and unused arguments will be ignored
        >>> generation_config, unused_kwargs = GenerationConfig.from_pretrained(
        ...     "openai-community/gpt2", top_k=1, foo=False, do_sample=True, return_unused_kwargs=True
        ... )
        >>> generation_config.top_k
        1

        >>> unused_kwargs
        {'foo': False}
        ```"""
        config_file_name = config_file_name if config_file_name is not None else GENERATION_CONFIG_NAME
        resume_download = kwargs.pop('resume_download', None)
        proxies = kwargs.pop('proxies', None)
        use_auth_token = kwargs.pop('use_auth_token', None)
        subfolder = kwargs.pop('subfolder', '')
        from_pipeline = kwargs.pop('_from_pipeline', None)
        from_auto_class = kwargs.pop('_from_auto', False)
        commit_hash = kwargs.pop('_commit_hash', None)
        if use_auth_token is not None:
            warnings.warn('The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.', FutureWarning)
            if token is not None:
                raise ValueError('`token` and `use_auth_token` are both specified. Please set only the argument `token`.')
            token = use_auth_token
        user_agent = {'file_type': 'config', 'from_auto_class': from_auto_class}
        if from_pipeline is not None:
            user_agent['using_pipeline'] = from_pipeline
        config_path = os.path.join(pretrained_model_name, config_file_name)
        config_path = str(config_path)
        is_local = os.path.exists(config_path)
        if os.path.isfile(os.path.join(subfolder, config_path)):
            resolved_config_file = config_path
            is_local = True
        elif is_remote_url(config_path):
            configuration_file = config_path
            resolved_config_file = download_url(config_path)
        else:
            configuration_file = config_file_name
            try:
                resolved_config_file = cached_file(pretrained_model_name, configuration_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies, resume_download=resume_download, local_files_only=local_files_only, token=token, user_agent=user_agent, revision=revision, subfolder=subfolder, _commit_hash=commit_hash)
                commit_hash = extract_commit_hash(resolved_config_file, commit_hash)
            except OSError:
                raise
            except Exception:
                raise OSError(f"Can't load the configuration of '{pretrained_model_name}'. If you were trying to load it from 'https://huggingface.co/models', make sure you don't have a local directory with the same name. Otherwise, make sure '{pretrained_model_name}' is the correct path to a directory containing a {configuration_file} file")
        try:
            config_dict = cls._dict_from_json_file(resolved_config_file)
            config_dict['_commit_hash'] = commit_hash
        except (json.JSONDecodeError, UnicodeDecodeError):
            raise OSError(f"It looks like the config file at '{resolved_config_file}' is not a valid JSON file.")
        if is_local:
            logger.info(f'loading configuration file {resolved_config_file}')
        else:
            logger.info(f'loading configuration file {configuration_file} from cache at {resolved_config_file}')
        if kwargs.get('return_unused_kwargs') is True:
            config, unused_kwargs = cls.from_dict(config_dict, **kwargs)
            config._original_object_hash = hash(config)
            return (config, unused_kwargs)
        else:
            config = cls.from_dict(config_dict, **kwargs)
            config._original_object_hash = hash(config)
            return config

    @classmethod
    def _dict_from_json_file(cls, json_file: Union[str, os.PathLike]):
        with open(json_file, 'r', encoding='utf-8') as reader:
            text = reader.read()
        return json.loads(text)

    @classmethod
    def from_dict(cls, config_dict: Dict[str, Any], **kwargs) -> 'GenerationConfig':
        """
        Instantiates a [`GenerationConfig`] from a Python dictionary of parameters.

        Args:
            config_dict (`Dict[str, Any]`):
                Dictionary that will be used to instantiate the configuration object.
            kwargs (`Dict[str, Any]`):
                Additional parameters from which to initialize the configuration object.

        Returns:
            [`GenerationConfig`]: The configuration object instantiated from those parameters.
        """
        return_unused_kwargs = kwargs.pop('return_unused_kwargs', False)
        kwargs.pop('_from_auto', None)
        kwargs.pop('_from_pipeline', None)
        if '_commit_hash' in kwargs and '_commit_hash' in config_dict:
            kwargs['_commit_hash'] = config_dict['_commit_hash']
        config = cls(**{**config_dict, **kwargs})
        unused_kwargs = config.update(**kwargs)
        logger.info(f'Generate config {config}')
        if return_unused_kwargs:
            return (config, unused_kwargs)
        else:
            return config

    def dict_torch_dtype_to_str(self, d: Dict[str, Any]) -> None:
        """
        Checks whether the passed dictionary and its nested dicts have a *torch_dtype* key and if it's not None,
        converts torch.dtype to a string of just the type. For example, `torch.float32` get converted into *"float32"*
        string, which can then be stored in the json format.
        """
        if d.get('torch_dtype', None) is not None and (not isinstance(d['torch_dtype'], str)):
            d['torch_dtype'] = str(d['torch_dtype']).split('.')[1]
        for value in d.values():
            if isinstance(value, dict):
                self.dict_torch_dtype_to_str(value)

    def to_diff_dict(self) -> Dict[str, Any]:
        """
        Removes all attributes from config which correspond to the default config attributes for better readability and
        serializes to a Python dictionary.

        Returns:
            `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance,
        """
        config_dict = self.to_dict()
        default_config_dict = GenerationConfig().to_dict()
        serializable_config_dict = {}
        for key, value in config_dict.items():
            if key not in default_config_dict or key == 'transformers_version' or value != default_config_dict[key]:
                serializable_config_dict[key] = value
        self.dict_torch_dtype_to_str(serializable_config_dict)
        return serializable_config_dict

    def to_dict(self) -> Dict[str, Any]:
        """
        Serializes this instance to a Python dictionary.

        Returns:
            `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.
        """
        output = copy.deepcopy(self.__dict__)
        if '_commit_hash' in output:
            del output['_commit_hash']
        if '_original_object_hash' in output:
            del output['_original_object_hash']
        if 'compile_config' in output:
            del output['compile_config']
        output['transformers_version'] = __version__
        self.dict_torch_dtype_to_str(output)
        return output

    def to_json_string(self, use_diff: bool=True, ignore_metadata: bool=False) -> str:
        """
        Serializes this instance to a JSON string.

        Args:
            use_diff (`bool`, *optional*, defaults to `True`):
                If set to `True`, only the difference between the config instance and the default `GenerationConfig()`
                is serialized to JSON string.
            ignore_metadata (`bool`, *optional*, defaults to `False`):
                Whether to ignore the metadata fields present in the instance

        Returns:
            `str`: String containing all the attributes that make up this configuration instance in JSON format.
        """
        if use_diff is True:
            config_dict = self.to_diff_dict()
        else:
            config_dict = self.to_dict()
        if ignore_metadata:
            for metadata_field in METADATA_FIELDS:
                config_dict.pop(metadata_field, None)

        def convert_keys_to_string(obj):
            if isinstance(obj, dict):
                return {str(key): convert_keys_to_string(value) for key, value in obj.items()}
            elif isinstance(obj, list):
                return [convert_keys_to_string(item) for item in obj]
            else:
                return obj

        def convert_dataclass_to_dict(obj):
            if isinstance(obj, dict):
                return {key: convert_dataclass_to_dict(value) for key, value in obj.items()}
            elif is_dataclass(obj):
                return obj.to_dict()
            else:
                return obj
        config_dict = convert_keys_to_string(config_dict)
        config_dict = convert_dataclass_to_dict(config_dict)
        return json.dumps(config_dict, indent=2, sort_keys=True) + '\n'

    def to_json_file(self, json_file_path: Union[str, os.PathLike], use_diff: bool=True):
        """
        Save this instance to a JSON file.

        Args:
            json_file_path (`str` or `os.PathLike`):
                Path to the JSON file in which this configuration instance's parameters will be saved.
            use_diff (`bool`, *optional*, defaults to `True`):
                If set to `True`, only the difference between the config instance and the default `GenerationConfig()`
                is serialized to JSON file.
        """
        with open(json_file_path, 'w', encoding='utf-8') as writer:
            writer.write(self.to_json_string(use_diff=use_diff))

    @classmethod
    def from_model_config(cls, model_config: PretrainedConfig) -> 'GenerationConfig':
        """
        Instantiates a [`GenerationConfig`] from a [`PretrainedConfig`]. This function is useful to convert legacy
        [`PretrainedConfig`] objects, which may contain generation parameters, into a stand-alone [`GenerationConfig`].

        Args:
            model_config (`PretrainedConfig`):
                The model config that will be used to instantiate the generation config.

        Returns:
            [`GenerationConfig`]: The configuration object instantiated from those parameters.
        """
        config_dict = model_config.to_dict()
        config_dict.pop('_from_model_config', None)
        config_dict = {key: value for key, value in config_dict.items() if value is not None}
        generation_config = cls.from_dict(config_dict, return_unused_kwargs=False, _from_model_config=True)
        decoder_config = model_config.get_text_config(decoder=True)
        if decoder_config is not model_config:
            default_generation_config = GenerationConfig()
            decoder_config_dict = decoder_config.to_dict()
            for attr in generation_config.to_dict().keys():
                is_unset = getattr(generation_config, attr) == getattr(default_generation_config, attr)
                if attr in decoder_config_dict and is_unset:
                    setattr(generation_config, attr, decoder_config_dict[attr])
        if generation_config.return_dict_in_generate is False:
            if any((getattr(generation_config, extra_output_flag, False) for extra_output_flag in generation_config.extra_output_flags)):
                generation_config.return_dict_in_generate = True
        generation_config._original_object_hash = hash(generation_config)
        return generation_config

    def update(self, **kwargs):
        """
        Updates attributes of this class instance with attributes from `kwargs` if they match existing attributes,
        returning all the unused kwargs.

        Args:
            kwargs (`Dict[str, Any]`):
                Dictionary of attributes to tentatively update this class.

        Returns:
            `Dict[str, Any]`: Dictionary containing all the key-value pairs that were not used to update the instance.
        """
        to_remove = []
        for key, value in kwargs.items():
            if hasattr(self, key):
                setattr(self, key, value)
                to_remove.append(key)
        self.validate()
        unused_kwargs = {key: value for key, value in kwargs.items() if key not in to_remove}
        return unused_kwargs

Class that holds a configuration for a generation task. A `generate` call supports the following generation methods
for text-decoder, text-to-text, speech-to-text, and vision-to-text models:

    - *greedy decoding* if `num_beams=1` and `do_sample=False`
    - *contrastive search* if `penalty_alpha>0.` and `top_k>1`
    - *multinomial sampling* if `num_beams=1` and `do_sample=True`
    - *beam-search decoding* if `num_beams>1` and `do_sample=False`
    - *beam-search multinomial sampling* if `num_beams>1` and `do_sample=True`
    - *diverse beam-search decoding* if `num_beams>1` and `num_beam_groups>1`
    - *constrained beam-search decoding* if `constraints!=None` or `force_words_ids!=None`
    - *assisted decoding* if `assistant_model` or `prompt_lookup_num_tokens` is passed to `.generate()`
    - *dola decoding* if `dola_layers` is passed to `.generate()`

To learn more about decoding strategies refer to the [text generation strategies guide](../generation_strategies).

<Tip>

A large number of these flags control the logits or the stopping criteria of the generation. Make sure you check
the [generate-related classes](https://huggingface.co/docs/transformers/internal/generation_utils) for a full
description of the possible manipulations, as well as examples of their usage.

</Tip>

Arg:
    > Parameters that control the length of the output

    max_length (`int`, *optional*, defaults to 20):
        The maximum length the generated tokens can have. Corresponds to the length of the input prompt +
        `max_new_tokens`. Its effect is overridden by `max_new_tokens`, if also set.
    max_new_tokens (`int`, *optional*):
        The maximum numbers of tokens to generate, ignoring the number of tokens in the prompt.
    min_length (`int`, *optional*, defaults to 0):
        The minimum length of the sequence to be generated. Corresponds to the length of the input prompt +
        `min_new_tokens`. Its effect is overridden by `min_new_tokens`, if also set.
    min_new_tokens (`int`, *optional*):
        The minimum numbers of tokens to generate, ignoring the number of tokens in the prompt.
    early_stopping (`bool` or `str`, *optional*, defaults to `False`):
        Controls the stopping condition for beam-based methods, like beam-search. It accepts the following values:
        `True`, where the generation stops as soon as there are `num_beams` complete candidates; `False`, where an
        heuristic is applied and the generation stops when is it very unlikely to find better candidates;
        `"never"`, where the beam search procedure only stops when there cannot be better candidates (canonical
        beam search algorithm).
    max_time (`float`, *optional*):
        The maximum amount of time you allow the computation to run for in seconds. generation will still finish
        the current pass after allocated time has been passed.
    stop_strings (`str or List[str]`, *optional*):
        A string or a list of strings that should terminate generation if the model outputs them.

    > Parameters that control the generation strategy used

    do_sample (`bool`, *optional*, defaults to `False`):
        Whether or not to use sampling ; use greedy decoding otherwise.
    num_beams (`int`, *optional*, defaults to 1):
        Number of beams for beam search. 1 means no beam search.
    num_beam_groups (`int`, *optional*, defaults to 1):
        Number of groups to divide `num_beams` into in order to ensure diversity among different groups of beams.
        [this paper](https://huggingface.co/papers/1610.02424) for more details.
    penalty_alpha (`float`, *optional*):
        The values balance the model confidence and the degeneration penalty in contrastive search decoding.
    dola_layers (`str` or `List[int]`, *optional*):
        The layers to use for DoLa decoding. If `None`, DoLa decoding is not used. If a string, it must
        be one of "low" or "high", which means using the lower part or higher part of the model layers, respectively.
        "low" means the first half of the layers up to the first 20 layers, and "high" means the last half of the
        layers up to the last 20 layers.
        If a list of integers, it must contain the indices of the layers to use for candidate premature layers in DoLa.
        The 0-th layer is the word embedding layer of the model. Set to `'low'` to improve long-answer reasoning tasks,
        `'high'` to improve short-answer tasks. Check the [documentation](https://github.com/huggingface/transformers/blob/main/docs/source/en/generation_strategies.md)
        or [the paper](https://huggingface.co/papers/2309.03883) for more details.

    > Parameters that control the cache

    use_cache (`bool`, *optional*, defaults to `True`):
        Whether or not the model should use the past last key/values attentions (if applicable to the model) to
        speed up decoding.
    cache_implementation (`str`, *optional*, default to `None`):
        Name of the cache class that will be instantiated in `generate`, for faster decoding. Possible values are:

        - `"dynamic"`: [`DynamicCache`]
        - `"static"`: [`StaticCache`]
        - `"offloaded_static"`: [`OffloadedStaticCache`]
        - `"sliding_window"`: [`SlidingWindowCache`]
        - `"hybrid"`: [`HybridCache`]
        - `"mamba"`: [`MambaCache`]
        - `"quantized"`: [`QuantizedCache`]

        If none is specified, we will use the default cache for the model (which is often [`DynamicCache`]). See
        our [cache documentation](https://huggingface.co/docs/transformers/en/kv_cache) for further information.
    cache_config (`CacheConfig` or `dict`, *optional*, default to `None`):
        Arguments used in the key-value cache class can be passed in `cache_config`. Can be passed as a `Dict` and
        it will be converted to its respective `CacheConfig` internally.
        Otherwise can be passed as a `CacheConfig` class matching the indicated `cache_implementation`.
    return_legacy_cache (`bool`, *optional*, default to `True`):
        Whether to return the legacy or new format of the cache when `DynamicCache` is used by default.

    > Parameters for manipulation of the model output logits

    temperature (`float`, *optional*, defaults to 1.0):
        The value used to module the next token probabilities. This value is set in a model's `generation_config.json` file. If it isn't set, the default value is 1.0
    top_k (`int`, *optional*, defaults to 50):
        The number of highest probability vocabulary tokens to keep for top-k-filtering. This value is set in a model's `generation_config.json` file. If it isn't set, the default value is 50.
    top_p (`float`, *optional*, defaults to 1.0):
        If set to float < 1, only the smallest set of most probable tokens with probabilities that add up to
        `top_p` or higher are kept for generation. This value is set in a model's `generation_config.json` file. If it isn't set, the default value is 1.0
    min_p (`float`, *optional*):
        Minimum token probability, which will be scaled by the probability of the most likely token. It must be a
        value between 0 and 1. Typical values are in the 0.01-0.2 range, comparably selective as setting `top_p` in
        the 0.99-0.8 range (use the opposite of normal `top_p` values).
    typical_p (`float`, *optional*, defaults to 1.0):
        Local typicality measures how similar the conditional probability of predicting a target token next is to
        the expected conditional probability of predicting a random token next, given the partial text already
        generated. If set to float < 1, the smallest set of the most locally typical tokens with probabilities that
        add up to `typical_p` or higher are kept for generation. See [this
        paper](https://huggingface.co/papers/2202.00666) for more details.
    epsilon_cutoff (`float`, *optional*, defaults to 0.0):
        If set to float strictly between 0 and 1, only tokens with a conditional probability greater than
        `epsilon_cutoff` will be sampled. In the paper, suggested values range from 3e-4 to 9e-4, depending on the
        size of the model. See [Truncation Sampling as Language Model
        Desmoothing](https://huggingface.co/papers/2210.15191) for more details.
    eta_cutoff (`float`, *optional*, defaults to 0.0):
        Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to float strictly between
        0 and 1, a token is only considered if it is greater than either `eta_cutoff` or `sqrt(eta_cutoff) *
        exp(-entropy(softmax(next_token_logits)))`. The latter term is intuitively the expected next token
        probability, scaled by `sqrt(eta_cutoff)`. In the paper, suggested values range from 3e-4 to 2e-3,
        depending on the size of the model. See [Truncation Sampling as Language Model
        Desmoothing](https://huggingface.co/papers/2210.15191) for more details.
    diversity_penalty (`float`, *optional*, defaults to 0.0):
        This value is subtracted from a beam's score if it generates a token same as any beam from other group at a
        particular time. Note that `diversity_penalty` is only effective if `group beam search` is enabled.
    repetition_penalty (`float`, *optional*, defaults to 1.0):
        The parameter for repetition penalty. 1.0 means no penalty. See [this
        paper](https://huggingface.co/papers/1909.05858) for more details.
    encoder_repetition_penalty (`float`, *optional*, defaults to 1.0):
        The parameter for encoder_repetition_penalty. An exponential penalty on sequences that are not in the
        original input. 1.0 means no penalty.
    length_penalty (`float`, *optional*, defaults to 1.0):
        Exponential penalty to the length that is used with beam-based generation. It is applied as an exponent to
        the sequence length, which in turn is used to divide the score of the sequence. Since the score is the log
        likelihood of the sequence (i.e. negative), `length_penalty` > 0.0 promotes longer sequences, while
        `length_penalty` < 0.0 encourages shorter sequences.
    no_repeat_ngram_size (`int`, *optional*, defaults to 0):
        If set to int > 0, all ngrams of that size can only occur once.
    bad_words_ids (`List[List[int]]`, *optional*):
        List of list of token ids that are not allowed to be generated. Check
        [`~generation.NoBadWordsLogitsProcessor`] for further documentation and examples.
    force_words_ids (`List[List[int]]` or `List[List[List[int]]]`, *optional*):
        List of token ids that must be generated. If given a `List[List[int]]`, this is treated as a simple list of
        words that must be included, the opposite to `bad_words_ids`. If given `List[List[List[int]]]`, this
        triggers a [disjunctive constraint](https://github.com/huggingface/transformers/issues/14081), where one
        can allow different forms of each word.
    renormalize_logits (`bool`, *optional*, defaults to `False`):
        Whether to renormalize the logits after applying all the logits processors (including the custom
        ones). It's highly recommended to set this flag to `True` as the search algorithms suppose the score logits
        are normalized but some logit processors break the normalization.
    constraints (`List[Constraint]`, *optional*):
        Custom constraints that can be added to the generation to ensure that the output will contain the use of
        certain tokens as defined by `Constraint` objects, in the most sensible way possible.
    forced_bos_token_id (`int`, *optional*, defaults to `model.config.forced_bos_token_id`):
        The id of the token to force as the first generated token after the `decoder_start_token_id`. Useful for
        multilingual models like [mBART](../model_doc/mbart) where the first generated token needs to be the target
        language token.
    forced_eos_token_id (`int` or List[int]`, *optional*, defaults to `model.config.forced_eos_token_id`):
        The id of the token to force as the last generated token when `max_length` is reached. Optionally, use a
        list to set multiple *end-of-sequence* tokens.
    remove_invalid_values (`bool`, *optional*, defaults to `model.config.remove_invalid_values`):
        Whether to remove possible *nan* and *inf* outputs of the model to prevent the generation method to crash.
        Note that using `remove_invalid_values` can slow down generation.
    exponential_decay_length_penalty (`tuple(int, float)`, *optional*):
        This Tuple adds an exponentially increasing length penalty, after a certain amount of tokens have been
        generated. The tuple shall consist of: `(start_index, decay_factor)` where `start_index` indicates where
        penalty starts and `decay_factor` represents the factor of exponential decay
    suppress_tokens (`List[int]`, *optional*):
        A list of tokens that will be suppressed at generation. The `SupressTokens` logit processor will set their
        log probs to `-inf` so that they are not sampled.
    begin_suppress_tokens  (`List[int]`, *optional*):
        A list of tokens that will be suppressed at the beginning of the generation. The `SupressBeginTokens` logit
        processor will set their log probs to `-inf` so that they are not sampled.
    sequence_bias (`Dict[Tuple[int], float]`, *optional*)):
        Dictionary that maps a sequence of tokens to its bias term. Positive biases increase the odds of the
        sequence being selected, while negative biases do the opposite. Check
        [`~generation.SequenceBiasLogitsProcessor`] for further documentation and examples.
    token_healing (`bool`, *optional*, defaults to `False`):
        Heal tail tokens of prompts by replacing them with their appropriate extensions.
        This enhances the quality of completions for prompts affected by greedy tokenization bias.
    guidance_scale (`float`, *optional*):
        The guidance scale for classifier free guidance (CFG). CFG is enabled by setting `guidance_scale > 1`.
        Higher guidance scale encourages the model to generate samples that are more closely linked to the input
        prompt, usually at the expense of poorer quality.
    low_memory (`bool`, *optional*):
        Switch to sequential beam search and sequential topk for contrastive search to reduce peak memory.
        Used with beam search and contrastive search.
    watermarking_config (`BaseWatermarkingConfig` or `dict`, *optional*):
        Arguments used to watermark the model outputs by adding a small bias to randomly selected set of "green"
        tokens. See the docs of [`SynthIDTextWatermarkingConfig`] and [`WatermarkingConfig`] for more
        details. If passed as `Dict`, it will be converted to a `WatermarkingConfig` internally.

    > Parameters that define the output variables of generate

    num_return_sequences (`int`, *optional*, defaults to 1):
        The number of independently computed returned sequences for each element in the batch.
    output_attentions (`bool`, *optional*, defaults to `False`):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
        tensors for more details.
    output_hidden_states (`bool`, *optional*, defaults to `False`):
        Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
        more details.
    output_scores (`bool`, *optional*, defaults to `False`):
        Whether or not to return the prediction scores. See `scores` under returned tensors for more details.
    output_logits (`bool`, *optional*):
        Whether or not to return the unprocessed prediction logit scores. See `logits` under returned tensors for
        more details.
    return_dict_in_generate (`bool`, *optional*, defaults to `False`):
        Whether or not to return a [`~utils.ModelOutput`], as opposed to returning exclusively the generated
        sequence. This flag must be set to `True` to return the generation cache (when `use_cache` is `True`)
        or optional outputs (see flags starting with `output_`)

    > Special tokens that can be used at generation time

    pad_token_id (`int`, *optional*):
        The id of the *padding* token.
    bos_token_id (`int`, *optional*):
        The id of the *beginning-of-sequence* token.
    eos_token_id (`Union[int, List[int]]`, *optional*):
        The id of the *end-of-sequence* token. Optionally, use a list to set multiple *end-of-sequence* tokens.

    > Generation parameters exclusive to encoder-decoder models

    encoder_no_repeat_ngram_size (`int`, *optional*, defaults to 0):
        If set to int > 0, all ngrams of that size that occur in the `encoder_input_ids` cannot occur in the
        `decoder_input_ids`.
    decoder_start_token_id (`int` or `List[int]`, *optional*):
        If an encoder-decoder model starts decoding with a different token than *bos*, the id of that token or a list of length
        `batch_size`. Indicating a list enables different start ids for each element in the batch
        (e.g. multilingual models with different target languages in one batch)

    > Generation parameters exclusive to assistant generation
    is_assistant (`bool`, *optional*, defaults to `False`):
        Whether the model is an assistant (draft) model.
    num_assistant_tokens (`int`, *optional*, defaults to 20):
        Defines the number of _speculative tokens_ that shall be generated by the assistant model before being
        checked by the target model at each iteration. Higher values for `num_assistant_tokens` make the generation
        more _speculative_ : If the assistant model is performant larger speed-ups can be reached, if the assistant
        model requires lots of corrections, lower speed-ups are reached.
    num_assistant_tokens_schedule (`str`, *optional*, defaults to `"constant"`):
        Defines the schedule at which max assistant tokens shall be changed during inference.
        - `"heuristic"`: When all speculative tokens are correct, increase `num_assistant_tokens` by 2 else
          reduce by 1. `num_assistant_tokens` value is persistent over multiple generation calls with the same assistant model.
        - `"heuristic_transient"`: Same as `"heuristic"` but `num_assistant_tokens` is reset to its initial value after each generation call.
        - `"constant"`: `num_assistant_tokens` stays unchanged during generation
    assistant_confidence_threshold (`float`, *optional*, defaults to 0.4):
        The confidence threshold for the assistant model. If the assistant model's confidence in its prediction for the current token is lower
        than this threshold, the assistant model stops the current token generation iteration, even if the number of _speculative tokens_
        (defined by `num_assistant_tokens`) is not yet reached. The assistant's confidence threshold is adjusted throughout the speculative iterations to reduce the number of unnecessary draft and target forward passes, biased towards avoiding false negatives.
        `assistant_confidence_threshold` value is persistent over multiple generation calls with the same assistant model.
        It is an unsupervised version of the dynamic speculation lookahead
        from Dynamic Speculation Lookahead Accelerates Speculative Decoding of Large Language Models <https://huggingface.co/papers/2405.04304>.
    prompt_lookup_num_tokens (`int`, *optional*):
        The number of tokens to be output as candidate tokens.
    max_matching_ngram_size (`int`, *optional*):
        The maximum ngram size to be considered for matching in the prompt. Default to 2 if not provided.
    assistant_early_exit(`int`, *optional*):
        If set to a positive integer, early exit of the model will be used as an assistant. Can only be used with
        models that support early exit (i.e. models where logits from intermediate layers can be interpreted by the LM head).
    assistant_lookbehind(`int`, *optional*, defaults to 10):
        If set to a positive integer, the re-encodeing process will additionally consider the last `assistant_lookbehind` assistant tokens
        to correctly align tokens. Can only be used with different tokenizers in speculative decoding.
        See this [blog](https://huggingface.co/blog/universal_assisted_generation) for more details.
    target_lookbehind(`int`, *optional*, defaults to 10):
        If set to a positive integer, the re-encodeing process will additionally consider the last `target_lookbehind` target tokens
        to correctly align tokens. Can only be used with different tokenizers in speculative decoding.
        See this [blog](https://huggingface.co/blog/universal_assisted_generation) for more details.

    > Parameters related to performances and compilation

    compile_config (CompileConfig, *optional*):
        If using a compilable cache, this controls how `generate` will `compile` the forward pass for faster
        inference.
    disable_compile (`bool`, *optional*):
        Whether to disable the automatic compilation of the forward pass. Automatic compilation happens when
        specific criteria are met, including using a compilable cache. Please open an issue if you find the
        need to use this flag.


def internal_convert_to_tensor_or_composite(value, dtype=None, name=None, as_ref=False) -> Union[EagerTensor, SymbolicTensor, composite_tensor.CompositeTensor]:
    if isinstance(value, composite_tensor.CompositeTensor):
        value_dtype = getattr(value, 'dtype', None)
        if dtype and (not dtypes.as_dtype(dtype).is_compatible_with(value_dtype)):
            raise ValueError(f'Tensor conversion dtype mismatch. Requested dtype is {dtypes.as_dtype(dtype).name}, Tensor has dtype {value.dtype.name}: {value!r}')
        return value
    else:
        return convert_to_tensor(value, dtype=dtype, name=name, as_ref=as_ref, accepted_result_types=(tensor_lib.Tensor, composite_tensor.CompositeTensor))

Converts the given object to a `Tensor` or `CompositeTensor`.

If `value` is a `CompositeTensor` it is returned unmodified.  Otherwise, it
is converted to a `Tensor` using `convert_to_tensor()`.

Args:
  value: A `CompositeTensor`, or an object that can be consumed by
    `convert_to_tensor()`.
  dtype: (Optional.) The required `DType` of the returned `Tensor` or
    `CompositeTensor`.
  name: (Optional.) A name to use if a new `Tensor` is created.
  as_ref: True if the caller wants the results as ref tensors.

Returns:
  A `Tensor` or `CompositeTensor`, based on `value`.

Raises:
  ValueError: If `dtype` does not match the element type of `value`.


def tf_shard_checkpoint(weights, max_shard_size='10GB', weights_name: str=TF2_WEIGHTS_NAME):
    max_shard_size = convert_file_size_to_int(max_shard_size)
    sharded_state_dicts = []
    current_block = []
    current_block_size = 0
    total_size = 0
    for item in weights:
        weight_size = item.numpy().size * item.dtype.size
        if current_block_size + weight_size > max_shard_size:
            sharded_state_dicts.append(current_block)
            current_block = []
            current_block_size = 0
        current_block.append(item)
        current_block_size += weight_size
        total_size += weight_size
    sharded_state_dicts.append(current_block)
    if len(sharded_state_dicts) == 1:
        return ({weights_name: sharded_state_dicts[0]}, None)
    weight_map = {}
    shards = {}
    for idx, shard in enumerate(sharded_state_dicts):
        shard_file = weights_name.replace('.h5', f'-{idx + 1:05d}-of-{len(sharded_state_dicts):05d}.h5')
        shard_file = shard_file.replace('.safetensors', f'-{idx + 1:05d}-of-{len(sharded_state_dicts):05d}.safetensors')
        shards[shard_file] = shard
        for weight in shard:
            weight_name = weight.name
            weight_map[weight_name] = shard_file
    metadata = {'total_size': total_size}
    index = {'metadata': metadata, 'weight_map': weight_map}
    return (shards, index)

Splits a model state dictionary in sub-checkpoints so that the final size of each sub-checkpoint does not exceed a
given size.

The sub-checkpoints are determined by iterating through the `state_dict` in the order of its keys, so there is no
optimization made to make each sub-checkpoint as close as possible to the maximum size passed. For example, if the
limit is 10GB and we have weights of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB],
[6+2+2GB] and not [6+2+2GB], [6+2GB], [6GB].

<Tip warning={true}>

If one of the model's weight is bigger that `max_shard_size`, it will end up in its own sub-checkpoint which will
have a size greater than `max_shard_size`.

</Tip>

Args:
    weights (`Dict[str, tf.RessourceVariable]`): The list of tf.RessourceVariable of a model to save.
    max_shard_size (`int` or `str`, *optional*, defaults to `"10GB"`):
        The maximum size of each sub-checkpoint. If expressed as a string, needs to be digits followed by a unit
        (like `"5MB"`).


def __init__(self, value):
    if not (isinstance(value, tensor.Tensor) and value.dtype.is_floating):
        raise ValueError('Regression output value must be a float32 Tensor; got {}'.format(value))
    self._value = value

Constructor for `RegressionOutput`.

Args:
  value: a float `Tensor` giving the predicted values.  Required.

Raises:
  ValueError: if the value is not a `Tensor` with dtype tf.float32.


def _prepare_4d_causal_attention_mask_with_cache_position(attention_mask: torch.Tensor, sequence_length: int, target_length: int, dtype: torch.dtype, cache_position: torch.Tensor, batch_size: int, **kwargs):
    if attention_mask is not None and attention_mask.dim() == 4:
        causal_mask = attention_mask
    else:
        min_dtype = torch.finfo(dtype).min
        causal_mask = torch.full((sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=cache_position.device)
        if sequence_length != 1:
            causal_mask = torch.triu(causal_mask, diagonal=1)
        causal_mask *= torch.arange(target_length, device=cache_position.device) > cache_position.reshape(-1, 1)
        causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)
        if attention_mask is not None:
            causal_mask = causal_mask.clone()
            mask_length = attention_mask.shape[-1]
            padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :].to(causal_mask.device)
            padding_mask = padding_mask == 0
            causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill(padding_mask, min_dtype)
    return causal_mask

Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
`(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

Args:
    attention_mask (`torch.Tensor`):
        A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
        `(batch_size, 1, query_length, key_value_length)`.
    sequence_length (`int`):
        The sequence length being processed.
    target_length (`int`):
        The target length: when generating with static cache, the mask should be as long as the static cache,
        to account for the 0 padding, the part of the cache that is not filled yet.
    dtype (`torch.dtype`):
        The dtype to use for the 4D attention mask.
    cache_position (`torch.Tensor`):
        Indices depicting the position of the input sequence tokens in the sequence.
    batch_size (`torch.Tensor`):
        Batch size.


def _compute_template(val: BaseValue) -> Sequence[BaseValue]:
    if isinstance(val, _abstract.PyTDClass):
        return [val.ctx.convert.constant_to_value(itm.type_param) for itm in val.pytd_cls.template]
    elif not isinstance(val, _abstract.InterpreterClass):
        return ()
    bases = [abstract_utils.get_atomic_value(base, default=val.ctx.convert.unsolvable) for base in val.bases()]
    template = []
    for base in bases:
        if base.full_name == 'typing.Generic':
            if isinstance(base, _abstract.PyTDClass):
                raise abstract_utils.GenericTypeError(val, 'Cannot inherit from plain Generic')
            if template:
                raise abstract_utils.GenericTypeError(val, 'Cannot inherit from Generic[...] multiple times')
            for item in base.template:
                param = base.formal_type_parameters.get(item.name)
                template.append(param.with_scope(val.full_name))
    if template:
        for base in bases:
            if base.full_name != 'typing.Generic':
                if isinstance(base, _abstract.ParameterizedClass):
                    for item in base.template:
                        param = base.formal_type_parameters.get(item.name)
                        if isinstance(base, _abstract.TypeParameter):
                            t = param.with_scope(val.full_name)
                            if t not in template:
                                raise abstract_utils.GenericTypeError(val, 'Generic should contain all the type variables')
    else:
        seqs = []
        for base in bases:
            if isinstance(base, _abstract.ParameterizedClass):
                seq = []
                for item in base.template:
                    param = base.formal_type_parameters.get(item.name)
                    if isinstance(param, _abstract.TypeParameter):
                        seq.append(param.with_scope(val.full_name))
                seqs.append(seq)
        try:
            template.extend(mro.MergeSequences(seqs))
        except ValueError as e:
            raise abstract_utils.GenericTypeError(val, f'Illegal type parameter order in class {val.name}') from e
    return template

Compute the precedence list of template parameters according to C3.

1. For the base class list, if it contains `typing.Generic`, then all the
type parameters should be provided. That means we don't need to parse extra
base class and then we can get all the type parameters.
2. If there is no `typing.Generic`, parse the precedence list according to
C3 based on all the base classes.
3. If `typing.Generic` exists, it must contain at least one type parameters.
And there is at most one `typing.Generic` in the base classes. Report error
if the check fails.

Args:
  val: The abstract.BaseValue to compute a template for.

Returns:
  parsed type parameters

Raises:
  GenericTypeError: if the type annotation for generic type is incorrect


def zero_fraction(value, name=None):
    with ops.name_scope(name, 'zero_fraction', [value]):
        value = ops.convert_to_tensor(value, name='value')
        size = array_ops.size(value, out_type=dtypes.int64)
        num_nonzero = tf_cond.cond(size <= dtypes.int32.max, true_fn=lambda: math_ops.cast(_count_nonzero(value, dtype=dtypes.int32), dtype=dtypes.int64), false_fn=lambda: _count_nonzero(value, dtype=dtypes.int64))
        with ops.name_scope('counts_to_fraction'):
            num_zero = size - num_nonzero
            num_zero_float32 = math_ops.cast(num_zero, dtype=dtypes.float32)
            size_float32 = math_ops.cast(size, dtype=dtypes.float32)
            zero_fraction_float32 = num_zero_float32 / size_float32
        return array_ops.identity(zero_fraction_float32, 'fraction')

Returns the fraction of zeros in `value`.

If `value` is empty, the result is `nan`.

This is useful in summaries to measure and report sparsity.  For example,

```python
    z = tf.nn.relu(...)
    summ = tf.compat.v1.summary.scalar('sparsity', tf.nn.zero_fraction(z))
```

Args:
  value: A tensor of numeric type.
  name: A name for the operation (optional).

Returns:
  The fraction of zeros in `value`, with type `float32`.


def get_num_bytes(self, batch: Sequence[str]) -> int:
    return sum((sys.getsizeof(element) for element in batch))

Returns:
  The number of bytes of input batch elements.


def _evaluate_once(checkpoint_path, master='', scaffold=None, eval_ops=None, feed_dict=None, final_ops=None, final_ops_feed_dict=None, hooks=None, config=None):
    eval_step = _get_or_create_eval_step()
    hooks = list(hooks or [])
    if eval_ops is not None:
        if any((isinstance(h, _MultiStepStopAfterNEvalsHook) for h in hooks)):
            steps_per_run_variable = basic_session_run_hooks.get_or_create_steps_per_run_variable()
            update_eval_step = state_ops.assign_add(eval_step, math_ops.cast(steps_per_run_variable, dtype=eval_step.dtype), use_locking=True)
        else:
            update_eval_step = state_ops.assign_add(eval_step, 1, use_locking=True)
        if isinstance(eval_ops, dict):
            eval_ops['update_eval_step'] = update_eval_step
        elif isinstance(eval_ops, (tuple, list)):
            eval_ops = list(eval_ops) + [update_eval_step]
        else:
            eval_ops = [eval_ops, update_eval_step]
        eval_step_value = _get_latest_eval_step_value(eval_ops)
        for h in hooks:
            if isinstance(h, (_StopAfterNEvalsHook, _MultiStepStopAfterNEvalsHook)):
                h._set_evals_completed_tensor(eval_step_value)
    logging.info('Starting evaluation at ' + time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime()))
    start = time.time()
    session_creator = monitored_session.ChiefSessionCreator(scaffold=scaffold, checkpoint_filename_with_path=checkpoint_path, master=master, config=config)
    final_ops_hook = basic_session_run_hooks.FinalOpsHook(final_ops, final_ops_feed_dict)
    hooks.append(final_ops_hook)
    with monitored_session.MonitoredSession(session_creator=session_creator, hooks=hooks) as session:
        if eval_ops is not None:
            while not session.should_stop():
                session.run(eval_ops, feed_dict)
    logging.info('Inference Time : {:0.5f}s'.format(time.time() - start))
    logging.info('Finished evaluation at ' + time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime()))
    return final_ops_hook.final_ops_values

Evaluates the model at the given checkpoint path.

During a single evaluation, the `eval_ops` is run until the session is
interrupted or requested to finish. This is typically requested via a
`tf.contrib.training.StopAfterNEvalsHook` which results in `eval_ops` running
the requested number of times.

Optionally, a user can pass in `final_ops`, a single `Tensor`, a list of
`Tensors` or a dictionary from names to `Tensors`. The `final_ops` is
evaluated a single time after `eval_ops` has finished running and the fetched
values of `final_ops` are returned. If `final_ops` is left as `None`, then
`None` is returned.

One may also consider using a `tf.contrib.training.SummaryAtEndHook` to record
summaries after the `eval_ops` have run. If `eval_ops` is `None`, the
summaries run immediately after the model checkpoint has been restored.

Note that `evaluate_once` creates a local variable used to track the number of
evaluations run via `tf.contrib.training.get_or_create_eval_step`.
Consequently, if a custom local init op is provided via a `scaffold`, the
caller should ensure that the local init op also initializes the eval step.

Args:
  checkpoint_path: The path to a checkpoint to use for evaluation.
  master: The BNS address of the TensorFlow master.
  scaffold: An tf.compat.v1.train.Scaffold instance for initializing variables
    and restoring variables. Note that `scaffold.init_fn` is used by the
    function to restore the checkpoint. If you supply a custom init_fn, then
    it must also take care of restoring the model from its checkpoint.
  eval_ops: A single `Tensor`, a list of `Tensors` or a dictionary of names to
    `Tensors`, which is run until the session is requested to stop, commonly
    done by a `tf.contrib.training.StopAfterNEvalsHook`.
  feed_dict: The feed dictionary to use when executing the `eval_ops`.
  final_ops: A single `Tensor`, a list of `Tensors` or a dictionary of names
    to `Tensors`.
  final_ops_feed_dict: A feed dictionary to use when evaluating `final_ops`.
  hooks: List of `tf.estimator.SessionRunHook` callbacks which are run inside
    the evaluation loop.
  config: An instance of `tf.compat.v1.ConfigProto` that will be used to
    configure the `Session`. If left as `None`, the default will be used.

Returns:
  The fetched values of `final_ops` or `None` if `final_ops` is `None`.


def get_rows(self, reportId: int=None, timeout: int=60 * 3) -> typing.Iterator[dict]:
    if reportId is None:
        reportId = self.reportId
    while timeout > 0:
        report = API_SearchAds(self.config, self.auth).reports().get(reportId=reportId).execute()
        if report['isReportReady']:
            for fragment in range(len(report['files'])):
                rows = csv_to_rows(API_SearchAds(self.config, self.auth).reports().getFile(reportId=reportId, reportFragment=fragment).execute())
                if fragment > 0:
                    next(rows)
                yield from rows
            break
        else:
            if self.config.verbose:
                print('.', end='')
            sleep(60)
            timeout -= 1

Return each row of data from a report as a generator.

Wait up to 3 hours with 1 minute poll intervals for report to finish.
Handle fragmented downloads.

Args:
 reportId - optional,  if not given uses prior value from request(...) call.
 timeout - optional, number of minutes to wait for report to complete.

Returns:
 Generator with lists of column values.


def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor):
    image_tokens_list = self.vqmodel.encode(pixel_values, image_sizes)
    bpe_tokens_list = [self.vocabulary_mapping.convert_img2bpe(tokens).flatten() for tokens in image_tokens_list]
    bpe_tokens = torch.cat(bpe_tokens_list)
    return bpe_tokens

Tokenizes images into discrete tokens with VQGAN module. Converts
obtained image tokens into BPE tokens and wraps with "boi" and "eoi"
special tokens.

Args:
    pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
        The tensors corresponding to the input images.
    image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`):
        The sizes of the images in the batch, being (height, width) for each image.


def __call__(self, shape, dtype=dtypes.float32, **kwargs):
    self._validate_kwargs(kwargs)
    dtype = _assert_float_dtype(dtype)
    if _PARTITION_SHAPE in kwargs:
        shape = kwargs[_PARTITION_SHAPE]
    return self._random_generator.truncated_normal(shape, self.mean, self.stddev, dtype)

Returns a tensor object initialized as specified by the initializer.

Args:
  shape: Shape of the tensor.
  dtype: Optional dtype of the tensor. Only floating point types are
    supported.
  **kwargs: Additional keyword arguments.

Raises:
  ValueError: If the dtype is not floating point


def do_not_doc_inheritable(obj: T) -> T:
    setattr(obj, _DO_NOT_DOC_INHERITABLE, None)
    return obj

A decorator: Do not generate docs for this method.

This version of the decorator is "inherited" by subclasses. No docs will be
generated for the decorated method in any subclass. Even if the sub-class
overrides the method.

For example, to ensure that `method1` is **never documented** use this
decorator on the base-class:

```
class Parent(object):
  @do_not_doc_inheritable
  def method1(self):
    pass
  def method2(self):
    pass

class Child(Parent):
  def method1(self):
    pass
  def method2(self):
    pass
```
This will produce the following docs:

```
/Parent.md
  # method2
/Child.md
  # method2
```

When generating docs for a class's arributes, the `__mro__` is searched and
the attribute will be skipped if this decorator is detected on the attribute
on any class in the `__mro__`.

Note: This is implemented by adding a hidden attribute on the object, so it
cannot be used on objects which do not allow new attributes to be added. So
this decorator must go *below* `@property`, `@classmethod`,
or `@staticmethod`:

```
class Example(object):
  @property
  @do_not_doc_inheritable
  def x(self):
    return self._x
```

Args:
  obj: The class-attribute to hide from the generated docs.

Returns:
  obj


def assert_input_compatibility(input_spec, inputs, layer_name):
    if not input_spec:
        return
    input_spec = nest.flatten(input_spec)
    if isinstance(inputs, dict):
        names = [spec.name for spec in input_spec]
        if all(names):
            list_inputs = []
            for name in names:
                if name not in inputs:
                    raise ValueError('Missing data for input "%s". You passed a data dictionary with keys %s. Expected the following keys: %s' % (name, list(inputs.keys()), names))
                list_inputs.append(inputs[name])
            inputs = list_inputs
    inputs = nest.flatten(inputs)
    for x in inputs:
        if not hasattr(x, 'shape'):
            raise TypeError('Inputs to a layer should be tensors. Got: %s' % (x,))
    if len(inputs) != len(input_spec):
        raise ValueError('Layer ' + layer_name + ' expects ' + str(len(input_spec)) + ' input(s), but it received ' + str(len(inputs)) + ' input tensors. Inputs received: ' + str(inputs))
    for input_index, (x, spec) in enumerate(zip(inputs, input_spec)):
        if spec is None:
            continue
        shape = tensor_shape.TensorShape(x.shape)
        if shape.rank is None:
            return
        if spec.ndim is not None and (not spec.allow_last_axis_squeeze):
            ndim = shape.rank
            if ndim != spec.ndim:
                raise ValueError('Input ' + str(input_index) + ' of layer ' + layer_name + ' is incompatible with the layer: expected ndim=' + str(spec.ndim) + ', found ndim=' + str(ndim) + '. Full shape received: ' + str(tuple(shape)))
        if spec.max_ndim is not None:
            ndim = x.shape.rank
            if ndim is not None and ndim > spec.max_ndim:
                raise ValueError('Input ' + str(input_index) + ' of layer ' + layer_name + ' is incompatible with the layer: expected max_ndim=' + str(spec.max_ndim) + ', found ndim=' + str(ndim))
        if spec.min_ndim is not None:
            ndim = x.shape.rank
            if ndim is not None and ndim < spec.min_ndim:
                raise ValueError('Input ' + str(input_index) + ' of layer ' + layer_name + ' is incompatible with the layer: : expected min_ndim=' + str(spec.min_ndim) + ', found ndim=' + str(ndim) + '. Full shape received: ' + str(tuple(shape)))
        if spec.dtype is not None:
            if x.dtype.name != spec.dtype:
                raise ValueError('Input ' + str(input_index) + ' of layer ' + layer_name + ' is incompatible with the layer: expected dtype=' + str(spec.dtype) + ', found dtype=' + str(x.dtype))
        shape_as_list = shape.as_list()
        if spec.axes:
            for axis, value in spec.axes.items():
                if hasattr(value, 'value'):
                    value = value.value
                if value is not None and shape_as_list[int(axis)] not in {value, None}:
                    raise ValueError('Input ' + str(input_index) + ' of layer ' + layer_name + ' is incompatible with the layer: expected axis ' + str(axis) + ' of input shape to have value ' + str(value) + ' but received input with shape ' + display_shape(x.shape))
        if spec.shape is not None and shape.rank is not None:
            spec_shape = spec.shape
            if spec.allow_last_axis_squeeze:
                if shape_as_list and shape_as_list[-1] == 1:
                    shape_as_list = shape_as_list[:-1]
                if spec_shape and spec_shape[-1] == 1:
                    spec_shape = spec_shape[:-1]
            for spec_dim, dim in zip(spec_shape, shape_as_list):
                if spec_dim is not None and dim is not None:
                    if spec_dim != dim:
                        raise ValueError('Input ' + str(input_index) + ' is incompatible with layer ' + layer_name + ': expected shape=' + str(spec.shape) + ', found shape=' + display_shape(x.shape))

Checks compatibility between the layer and provided inputs.

This checks that the tensor(s) `inputs` verify the input assumptions
of a layer (if any). If not, a clear and actional exception gets raised.

Args:
    input_spec: An InputSpec instance, list of InputSpec instances, a nested
        structure of InputSpec instances, or None.
    inputs: Input tensor, list of input tensors, or a nested structure of
        input tensors.
    layer_name: String, name of the layer (for error message formatting).

Raises:
    ValueError: in case of mismatch between
        the provided inputs and the expectations of the layer.


def GetTestConfigs():
    test_configs = [('NHWC', False), ('NHWC', True)]
    if test.is_gpu_available(cuda_only=True):
        test_configs += [('NCHW', True)]
    return test_configs

Get all the valid tests configs to run.

Returns:
  all the valid test configs as tuples of data_format and use_gpu.


def load_attributes_from_hdf5_group(group, name):
    if name in group.attrs:
        data = [n.decode('utf8') if hasattr(n, 'decode') else n for n in group.attrs[name]]
    else:
        data = []
        chunk_id = 0
        while '%s%d' % (name, chunk_id) in group.attrs:
            data.extend([n.decode('utf8') if hasattr(n, 'decode') else n for n in group.attrs['%s%d' % (name, chunk_id)]])
            chunk_id += 1
    return data

Loads attributes of the specified name from the HDF5 group.

This method deals with an inherent problem of HDF5 file which is not able to store data larger than
HDF5_OBJECT_HEADER_LIMIT bytes.

Args:
    group: A pointer to a HDF5 group.
    name: A name of the attributes to load.

Returns:
    data: Attributes data.

Copied from Keras to Transformers to avoid versioning issues.


def _cmp_rel(self, state, op_name, x, y):
    ret = self.ctx.program.NewVariable()
    leftover_x = self.ctx.program.NewVariable()
    leftover_y = self.ctx.program.NewVariable()
    op_not_eq = op_name not in ('EQ', 'NE')
    reported = False
    for b1 in x.bindings:
        for b2 in y.bindings:
            op = getattr(slots, op_name)
            try:
                err = False
                val = compare.cmp_rel(self.ctx, op, b1.data, b2.data)
            except compare.CmpTypeError:
                val = None
                if state.node.HasCombination([b1, b2]):
                    err = True
                    reported = True
                    self.ctx.errorlog.unsupported_operands(self.frames, op, x, y)
            if val is None:
                if op_not_eq and isinstance(b1.data, abstract.Class) and err:
                    ret.AddBinding(self.ctx.convert.unsolvable, {b1, b2}, state.node)
                elif isinstance(b1.data, abstract.SequenceLength):
                    ret.AddBinding(self.ctx.convert.bool_values[val], {b1, b2}, state.node)
                else:
                    leftover_x.PasteBinding(b1, state.node)
                    leftover_y.PasteBinding(b2, state.node)
            else:
                ret.AddBinding(self.ctx.convert.bool_values[val], {b1, b2}, state.node)
    if leftover_x.bindings:
        op = f'__{op_name.lower()}__'
        report_errors = op_not_eq and (not bool(ret.bindings)) and (not reported)
        state, leftover_ret = vm_utils.call_binary_operator(state, op, leftover_x, leftover_y, report_errors=report_errors, ctx=self.ctx)
        ret.PasteVariable(leftover_ret, state.node)
    return (state, ret)

Implementation of relational operators CMP_(LT|LE|EQ|NE|GE|GT).

Args:
  state: Initial FrameState.
  op_name: An operator name, e.g., "EQ".
  x: A variable of the lhs value.
  y: A variable of the rhs value.

Returns:
  A tuple of the new FrameState and the return variable.


def build_inputs_with_special_tokens(self, token_ids_0: List[int], token_ids_1: Optional[List[int]]=None) -> List[int]:
    if token_ids_1 is None:
        return self.prefix_tokens + token_ids_0 + self.suffix_tokens
    return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. An MBART-50 sequence has the following format, where `X` represents the sequence:

- `input_ids` (for encoder) `[src_lang_code] X [eos]`
- `labels`: (for decoder) `[tgt_lang_code] X [eos]`

BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
separator.

Args:
    token_ids_0 (`List[int]`):
        List of IDs to which the special tokens will be added.
    token_ids_1 (`List[int]`, *optional*):
        Optional second list of IDs for sequence pairs.

Returns:
    `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.


def device_coordinates(self):
    return self._device_coordinates

Describes the mapping from TPU devices to topology coordinates.

Returns:
  A rank 3 int32 array with shape `[tasks, devices, axis]`.
  `tasks` is the number of tasks in the TPU cluster, `devices` is the number
  of TPU devices per task, and `axis` is the number of axes in the TPU
  cluster topology. Each entry gives the `axis`-th coordinate in the
  topology of a task/device pair. TPU topologies are 4-dimensional, with
  dimensions `(x, y, z, core number)`.


def forward(self, hidden_states: torch.Tensor, attention_mask: Optional[torch.Tensor]=None, past_key_value: Optional[Cache]=None, output_attentions: Optional[bool]=False, use_cache: Optional[bool]=False, cache_position: Optional[torch.LongTensor]=None, output_router_logits: Optional[bool]=False, position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]]=None, **kwargs) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
    residual = hidden_states
    hidden_states = self.input_layernorm(hidden_states)
    if self.mamba is not None:
        hidden_states = self.mamba(hidden_states=hidden_states, cache_position=cache_position, cache_params=past_key_value, attention_mask=attention_mask)
        self_attn_weights = None
    else:
        hidden_states, self_attn_weights, _ = self.self_attn(hidden_states=hidden_states, attention_mask=attention_mask, past_key_value=past_key_value, output_attentions=output_attentions, use_cache=use_cache, cache_position=cache_position, position_embeddings=position_embeddings, **kwargs)
    hidden_states = residual + hidden_states * self.residual_multiplier
    residual = hidden_states
    hidden_states = self.post_attention_layernorm(hidden_states)
    moe_hidden_states, router_logits = self.block_sparse_moe(hidden_states)
    hidden_states = moe_hidden_states + self.shared_mlp(hidden_states)
    hidden_states = residual + hidden_states * self.residual_multiplier
    outputs = (hidden_states,)
    if output_attentions:
        outputs += (self_attn_weights,)
    if use_cache:
        outputs += (past_key_value,)
    if output_router_logits:
        outputs += (router_logits,)
    return outputs

Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
    attention_mask (`torch.FloatTensor`, *optional*):
        attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
        query_sequence_length, key_sequence_length)` if default attention is used.
    past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under
        returned tensors for more detail.
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
    cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
        Indices depicting the position of the input sequence tokens in the sequence
    output_router_logits (`bool`, *optional*):
        Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
        should not be returned during inference.
    position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
        Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
        with `head_dim` being the embedding dimension of each attention head.
    kwargs (`dict`, *optional*):
        Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
        into the model


def post_process_semantic_segmentation(self, outputs, target_sizes: Optional[List[Tuple]]=None):
    logits = outputs.logits
    if target_sizes is not None:
        if len(logits) != len(target_sizes):
            raise ValueError('Make sure that you pass in as many target sizes as the batch dimension of the logits')
        if is_torch_tensor(target_sizes):
            target_sizes = target_sizes.numpy()
        semantic_segmentation = []
        for idx in range(len(logits)):
            resized_logits = torch.nn.functional.interpolate(logits[idx].unsqueeze(dim=0), size=target_sizes[idx], mode='bilinear', align_corners=False)
            semantic_map = resized_logits[0].argmax(dim=0)
            semantic_segmentation.append(semantic_map)
    else:
        semantic_segmentation = logits.argmax(dim=1)
        semantic_segmentation = [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0])]
    return semantic_segmentation

Converts the output of [`MobileNetV2ForSemanticSegmentation`] into semantic segmentation maps. Only supports PyTorch.

Args:
    outputs ([`MobileNetV2ForSemanticSegmentation`]):
        Raw outputs of the model.
    target_sizes (`List[Tuple]` of length `batch_size`, *optional*):
        List of tuples corresponding to the requested final size (height, width) of each prediction. If unset,
        predictions will not be resized.

Returns:
    semantic_segmentation: `List[torch.Tensor]` of length `batch_size`, where each item is a semantic
    segmentation map of shape (height, width) corresponding to the target_sizes entry (if `target_sizes` is
    specified). Each entry of each `torch.Tensor` correspond to a semantic class id.


def searchsorted(sorted_sequence, values, side='left', out_type=dtypes.int32, name=None):
    sequence_size = shape_internal(sorted_sequence)[-1]
    values_size = shape_internal(values)[-1]
    sorted_sequence_2d = reshape(sorted_sequence, [-1, sequence_size])
    values_2d = reshape(values, [-1, values_size])
    if side == 'right':
        output = gen_array_ops.upper_bound(sorted_sequence_2d, values_2d, out_type, name)
    elif side == 'left':
        output = gen_array_ops.lower_bound(sorted_sequence_2d, values_2d, out_type, name)
    else:
        raise ValueError(f"Argument `side` must be either 'right' or 'left'. Received: `side` = '{side}'.")
    return reshape(output, shape_internal(values))

Searches for where a value would go in a sorted sequence.

This is not a method for checking containment (like python `in`).

The typical use case for this operation is "binning", "bucketing", or
"discretizing". The `values` are assigned to bucket-indices based on the
**edges** listed in `sorted_sequence`. This operation
returns the bucket-index for each value.

>>> edges = [-1, 3.3, 9.1, 10.0]
>>> values = [0.0, 4.1, 12.0]
>>> tf.searchsorted(edges, values).numpy()
array([1, 2, 4], dtype=int32)

The `side` argument controls which index is returned if a value lands exactly
on an edge:

>>> seq = [0, 3, 9, 10, 10]
>>> values = [0, 4, 10]
>>> tf.searchsorted(seq, values).numpy()
array([0, 2, 3], dtype=int32)
>>> tf.searchsorted(seq, values, side="right").numpy()
array([1, 2, 5], dtype=int32)

The `axis` is not settable for this operation. It always operates on the
innermost dimension (`axis=-1`). The operation will accept any number of
outer dimensions. Here it is applied to the rows of a matrix:

>>> sorted_sequence = [[0., 3., 8., 9., 10.],
...                    [1., 2., 3., 4., 5.]]
>>> values = [[9.8, 2.1, 4.3],
...           [0.1, 6.6, 4.5, ]]
>>> tf.searchsorted(sorted_sequence, values).numpy()
array([[4, 1, 2],
       [0, 5, 4]], dtype=int32)

Note: This operation assumes that `sorted_sequence` **is sorted** along the
innermost axis, maybe using `tf.sort(..., axis=-1)`. **If the sequence is not
sorted, no error is raised** and the content of the returned tensor is not well
defined.

Args:
  sorted_sequence: N-D `Tensor` containing a sorted sequence.
  values: N-D `Tensor` containing the search values.
  side: 'left' or 'right'; 'left' corresponds to lower_bound and 'right' to
    upper_bound.
  out_type: The output type (`int32` or `int64`).  Default is `tf.int32`.
  name: Optional name for the operation.

Returns:
  An N-D `Tensor` the size of `values` containing the result of applying
  either lower_bound or upper_bound (depending on side) to each value.  The
  result is not a global index to the entire `Tensor`, but the index in the
  last dimension.

Raises:
  ValueError: If the last dimension of `sorted_sequence >= 2^31-1` elements.
              If the total size of `values` exceeds `2^31 - 1` elements.
              If the first `N-1` dimensions of the two tensors don't match.


def to_tensor_7(self) -> torch.Tensor:
    tensor = self._trans.new_zeros((*self.shape, 7))
    tensor[..., :4] = self._rots.get_quats()
    tensor[..., 4:] = self._trans
    return tensor

Converts a transformation to a tensor with 7 final columns, four for the quaternion followed by three for the
translation.

Returns:
    A [*, 7] tensor representation of the transformation