class BridgeTowerConfig(PretrainedConfig):
    model_type = 'bridgetower'
    sub_configs = {'text_config': BridgeTowerTextConfig, 'vision_config': BridgeTowerVisionConfig}

    def __init__(self, share_cross_modal_transformer_layers=True, hidden_act='gelu', hidden_size=768, initializer_factor=1, layer_norm_eps=1e-05, share_link_tower_layers=False, link_tower_type='add', num_attention_heads=12, num_hidden_layers=6, tie_word_embeddings=False, init_layernorm_from_vision_encoder=False, text_config=None, vision_config=None, **kwargs):
        _ = kwargs.pop('text_config_dict', None)
        _ = kwargs.pop('vision_config_dict', None)
        super().__init__(**kwargs)
        self.share_cross_modal_transformer_layers = share_cross_modal_transformer_layers
        self.hidden_act = hidden_act
        self.hidden_size = hidden_size
        self.initializer_factor = initializer_factor
        self.layer_norm_eps = layer_norm_eps
        self.share_link_tower_layers = share_link_tower_layers
        self.link_tower_type = link_tower_type
        self.num_attention_heads = num_attention_heads
        self.num_hidden_layers = num_hidden_layers
        self.tie_word_embeddings = tie_word_embeddings
        self.init_layernorm_from_vision_encoder = init_layernorm_from_vision_encoder
        if text_config is None:
            text_config = {}
            logger.info('`text_config` is `None`. Initializing the `BridgeTowerTextConfig` with default values.')
        if vision_config is None:
            vision_config = {}
            logger.info('`vision_config` is `None`. Initializing the `BridgeTowerVisionConfig` with default values.')
        self.text_config = BridgeTowerTextConfig(**text_config)
        self.vision_config = BridgeTowerVisionConfig(**vision_config)

    @classmethod
    def from_text_vision_configs(cls, text_config: BridgeTowerTextConfig, vision_config: BridgeTowerVisionConfig, **kwargs):
        """
        Instantiate a [`BridgeTowerConfig`] (or a derived class) from BridgeTower text model configuration. Returns:
            [`BridgeTowerConfig`]: An instance of a configuration object
        """
        return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs)

This is the configuration class to store the configuration of a [`BridgeTowerModel`]. It is used to instantiate a
BridgeTower model according to the specified arguments, defining the model architecture. Instantiating a
configuration with the defaults will yield a similar configuration to that of the bridgetower-base
[BridgeTower/bridgetower-base](https://huggingface.co/BridgeTower/bridgetower-base/) architecture.

Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PretrainedConfig`] for more information.

Args:
    share_cross_modal_transformer_layers (`bool`, *optional*, defaults to `True`):
        Whether cross modal transformer layers are shared.
    hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
        The non-linear activation function (function or string) in the encoder and pooler.
    hidden_size (`int`, *optional*, defaults to 768):
        Dimensionality of the encoder layers and the pooler layer.
    initializer_factor (`float`, *optional*, defaults to 1):
        A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
        testing).
    layer_norm_eps (`float`, *optional*, defaults to 1e-05):
        The epsilon used by the layer normalization layers.
    share_link_tower_layers (`bool`, *optional*, defaults to `False`):
        Whether the bride/link tower layers are shared.
    link_tower_type (`str`, *optional*, defaults to `"add"`):
        Type of the bridge/link layer.
    num_attention_heads (`int`, *optional*, defaults to 12):
        Number of attention heads for each attention layer in the Transformer encoder.
    num_hidden_layers (`int`, *optional*, defaults to 6):
        Number of hidden layers in the Transformer encoder.
    tie_word_embeddings (`bool`, *optional*, defaults to `False`):
        Whether to tie input and output embeddings.
    init_layernorm_from_vision_encoder (`bool`, *optional*, defaults to `False`):
        Whether to init LayerNorm from the vision encoder.
    text_config (`dict`, *optional*):
        Dictionary of configuration options used to initialize [`BridgeTowerTextConfig`].
    vision_config (`dict`, *optional*):
        Dictionary of configuration options used to initialize [`BridgeTowerVisionConfig`].

Example:

```python
>>> from transformers import BridgeTowerModel, BridgeTowerConfig

>>> # Initializing a BridgeTower BridgeTower/bridgetower-base style configuration
>>> configuration = BridgeTowerConfig()

>>> # Initializing a model from the BridgeTower/bridgetower-base style configuration
>>> model = BridgeTowerModel(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```


def enqueue(self, features, weights=None, training: bool=True, name: Optional[Text]=None, device: Optional[Text]=None):
    if not self._using_tpu:
        raise RuntimeError('enqueue is not valid when TPUEmbedding object is not created under a TPUStrategy.')
    in_tpu_context = self._raise_error_for_incorrect_control_flow_context()
    nest.assert_same_structure(self._feature_config, features)
    if not self._verify_output_shapes_on_enqueue:
        if not self._output_shapes or not self._built:
            raise ValueError('Configured not to check output shapes on each enqueue() call; please ensure build() was called with output shapes to initialize the TPU for embeddings.')
    else:
        per_replica = device is None
        input_shapes = self._get_input_shapes(features, per_replica, in_tpu_context)
        self._maybe_build(input_shapes)
        self._check_output_shapes(self._get_output_shapes_from_input_shapes(input_shapes))
    flat_inputs = nest.flatten(features)
    flat_weights = [None] * len(flat_inputs)
    if weights is not None:
        nest.assert_same_structure(self._feature_config, weights)
        flat_weights = nest.flatten(weights)
    flat_features = nest.flatten_with_joined_string_paths(self._feature_config)
    flat_paths, _ = zip(*flat_features)
    self._raise_error_for_inputs_not_on_cpu(flat_inputs, flat_paths)
    if in_tpu_context:
        self._raise_error_for_non_direct_inputs(features)

        def generate_enqueue_ops():
            """Generate enqueue ops for outside compilation."""
            mode_override = array_ops.where_v2(training, constant_op.constant('train'), constant_op.constant('inference'))
            enqueue_op = self._generate_enqueue_op(flat_inputs, flat_weights, flat_features, device_ordinal=-1, mode_override=mode_override)
            if name is not None:
                _add_key_attr(enqueue_op, name)
        tpu_replication.outside_compilation(generate_enqueue_ops)
    elif device is None:
        mode_override = 'train' if training else 'inference'
        enqueue_ops = []

        def _split_fn(ts, idx):
            if ts is None:
                return None
            elif isinstance(ts, tensor_lib.Tensor):
                return array_ops.split(ts, num_or_size_splits=self._num_cores_per_replica, axis=0)[idx]
            elif isinstance(ts, sparse_tensor.SparseTensor):
                return sparse_ops.sparse_split_v2(sp_input=ts, num_split=self._num_cores_per_replica, axis=0)[idx]
            else:
                raise ValueError('SPMD does not support raggedTensor yet.')

        def _maybe_split(ts_inputs, core_id):
            if self._num_cores_per_replica is None:
                return ts_inputs
            else:
                splitter = functools.partial(_split_fn, idx=core_id)
                return nest.map_structure(splitter, ts_inputs)
        for replica_id in range(self._strategy.num_replicas_in_sync):
            replica_inputs = distribute_utils.select_replica(replica_id, flat_inputs)
            replica_weights = distribute_utils.select_replica(replica_id, flat_weights)
            if self._num_cores_per_replica:
                tpu_devices = self._strategy.extended._tpu_devices[replica_id]
            else:
                tpu_devices = [self._strategy.extended.worker_devices[replica_id]]
            for core_id in range(self._num_cores_per_replica or 1):
                tpu_device = tpu_devices[core_id]
                device_ordinal = tf_device.DeviceSpec.from_string(tpu_device).device_index
                with ops.device(device_util.get_host_for_device(tpu_device)):
                    enqueue_op = self._generate_enqueue_op(_maybe_split(replica_inputs, core_id), _maybe_split(replica_weights, core_id), flat_features, device_ordinal=device_ordinal, mode_override=mode_override)
                    if name is not None:
                        _add_key_attr(enqueue_op, name)
                    enqueue_ops.append(enqueue_op)
    else:
        mode_override = 'train' if training else 'inference'
        device_spec = tf_device.DeviceSpec.from_string(device)
        if device_spec.device_type != 'TPU':
            raise ValueError('Non-TPU device {} passed to enqueue.'.format(device))
        with ops.device(device_util.get_host_for_device(device)):
            enqueue_op = self._generate_enqueue_op(flat_inputs, flat_weights, flat_features, device_ordinal=device_spec.device_index, mode_override=mode_override)
            if name is not None:
                _add_key_attr(enqueue_op, name)

Enqueues id tensors for embedding lookup.

This function enqueues a structure of features to be looked up in the
embedding tables. We expect that the input shapes of each of the tensors in
features matches the output shapes set via FeatureConfig or build method
(if any). the output shapes will be auto detected based on the input shapes
with the max_sequence_length or output shape setting in the FeatureConfig.
Note that the output shapes is based on per replica batch size.
If your input dataset is batched to the global batch size and you use
`tf.distribute.TPUStrategy`'s `experimental_distribute_dataset`
or if you use `distribute_datasets_from_function` and batch
to the per core batch size computed by the context passed to your input
function, the output shapes should match automatically.

The auto detected the output shapes:
  1. For dense tensor, if rank 2 or above, make sure the tensor has last
     dimension as 1. The output shape will be the input shape excluding
     the last dimension.
  2. For sparse tensor, make sure the tensor has rank 2 and above.
       a. If feature config has max_sequence_length equals 0 or output shape
          set (the max_sequence_length setting will be ignored), the
          output shape will be the input shape excluding the last dimension.
       b. Otherwise, if the tensor is rank 2, the output shape will be input
          shape  with last dimension set as max_sequence_length. If the
          tensor is above rank 2, the output shape will be the input shape
          excluding the last dimension and the last dimension of the output
          shape will be set to max_sequence_length.
  3. For ragged tensor, make sure the tensor has rank 2.
       a. If feature config has max_sequence_length equals 0 or output shape
          set (the max_sequence_length setting will be ignored), the
          output shape will be the input shape excluding the last dimension.
       b. Otherwise, the output shape will be the input shape excluding the
          last dimension and the last dimension of the output shape will be
          set to max_sequence_length.

```python
strategy = tf.distribute.TPUStrategy(...)
with strategy.scope():
  embedding = tf.tpu.experimental.embedding.TPUEmbedding(...)

distributed_dataset = (
    strategy.distribute_datasets_from_function(
        dataset_fn=...,
        options=tf.distribute.InputOptions(
            experimental_fetch_to_device=False))
dataset_iterator = iter(distributed_dataset)

@tf.function
def training_step():
  def tpu_step(tpu_features):
    with tf.GradientTape() as tape:
      activations = embedding.dequeue()
      tape.watch(activations)

      loss = ... #  some computation involving activations

    embedding_gradients = tape.gradient(loss, activations)
    embedding.apply_gradients(embedding_gradients)

  embedding_features, tpu_features = next(dataset_iterator)
  embedding.enqueue(embedding_features, training=True)
  strategy.run(tpu_step, args=(tpu_features,))

training_step()
```

NOTE: You should specify `training=True` when using
`embedding.apply_gradients` as above and `training=False` when not using
`embedding.apply_gradients` (e.g. for frozen embeddings or when doing
evaluation).

For finer grained control, in the above example the line

```
  embedding.enqueue(embedding_features, training=True)
```

may be replaced with

```
  per_core_embedding_features = self.strategy.experimental_local_results(
      embedding_features)

  def per_core_enqueue(ctx):
    core_id = ctx.replica_id_in_sync_group
    device = strategy.extended.worker_devices[core_id]
    embedding.enqueue(per_core_embedding_features[core_id],
                      device=device)

  strategy.experimental_distribute_values_from_function(
      per_core_queue_inputs)
```

Args:
  features: A nested structure of `tf.Tensor`s, `tf.SparseTensor`s or
    `tf.RaggedTensor`s, with the same structure as `feature_config`. Inputs
    will be downcast to `tf.int32`. Only one type out of `tf.SparseTensor`
    or `tf.RaggedTensor` is supported per call.
  weights: If not `None`, a nested structure of `tf.Tensor`s,
    `tf.SparseTensor`s or `tf.RaggedTensor`s, matching the above, except
    that the tensors should be of float type (and they will be downcast to
    `tf.float32`). For `tf.SparseTensor`s we assume the `indices` are the
    same for the parallel entries from `features` and similarly for
    `tf.RaggedTensor`s we assume the row_splits are the same.
  training: Defaults to `True`. If `False`, enqueue the batch as inference
    batch (forward pass only). Do not call `apply_gradients` when this is
    `False` as this may lead to a deadlock.
   name: A name for the underlying op.
   device: The device name (e.g. '/task:0/device:TPU:2') where this batch
     should be enqueued. This should be set if and only if features is not a
     `tf.distribute.DistributedValues` and enqueue is not being called
     inside a TPU context (e.g. inside `TPUStrategy.run`).

Raises:
  ValueError: When called inside a strategy.run call and input is not
    directly taken from the args of the `strategy.run` call. Also if
    the size of any sequence in `features` does not match corresponding
    sequence in `feature_config`. Similarly for `weights`, if not `None`.
    If input shapes of features is unequal or different from a previous
    call.
  RuntimeError: When called inside a strategy.run call and inside XLA
    control flow. If batch_size is not able to be determined and build was
    not called.
  TypeError: If the type of any sequence in `features` does not match
    corresponding sequence in `feature_config`. Similarly for `weights`, if
    not `None`.


def update_hash_with_primitive_value(hash_value, value):
    hash_const = np.uint64(11400714819323197440)
    hash_value = np.uint64(hash_value)
    value = np.uint64(value)
    hash_value = np.array([hash_value])
    value = np.array([value])
    hash_value = np.bitwise_xor(hash_value, value + hash_const + np.left_shift(hash_value, 10) + np.right_shift(hash_value, 4))[0]
    return hash_value

Update the hash value using a primitive value.

Args:
    hash_value (uint64): The current hash value.
    value: The primitive value to incorporate into the hash.

Returns:
    int: The updated hash value.


class AttenuatorDevice:

    def __init__(self, path_count=1):
        self.path_count = path_count
        self._telnet_client = telnet_scpi_client.TelnetScpiClient(tx_cmd_separator='\r\n', rx_cmd_separator='\r\n', prompt='')

    @property
    def is_open(self):
        """This function returns the state of the telnet connection to the
    underlying AttenuatorDevice.

    Returns:
        True if there is a successfully open connection to the
        AttenuatorDevice.
    """
        return bool(self._telnet_client.is_open)

    def open(self, host, port=23):
        """Opens a telnet connection to the desired AttenuatorDevice and
    queries basic information.

    Args:
        host: A valid hostname (IP address or DNS-resolvable name) to an
            MC-DAT attenuator instrument.
        port: An optional port number (defaults to telnet default 23)
    """
        self._telnet_client.open(host, port)
        config_str = self._telnet_client.cmd('MN?')
        if config_str.startswith('MN='):
            config_str = config_str[len('MN='):]
        self.properties = dict(zip(['model', 'max_freq', 'max_atten'], config_str.split('-', 2)))
        self.max_atten = float(self.properties['max_atten'])

    def close(self):
        """Closes a telnet connection to the desired attenuator device.

    This should be called as part of any teardown procedure prior to the
    attenuator instrument leaving scope.
    """
        if self.is_open:
            self._telnet_client.close()

    def set_atten(self, idx, value):
        """Sets the attenuation value for a particular signal path.

    Args:
        idx: Zero-based index int which is the identifier for a particular
            signal path in an instrument. For instruments that only has one
            channel, this is ignored by the device.
        value: A float that is the attenuation value to set.

    Raises:
        Error: The underlying telnet connection to the instrument is not
            open.
        IndexError: The index of the attenuator is greater than the maximum
            index of the underlying instrument.
        ValueError: The requested set value is greater than the maximum
            attenuation value.
    """
        if not self.is_open:
            raise attenuator.Error('Connection to attenuator at %s is not open!' % self._telnet_client.host)
        if idx + 1 > self.path_count:
            raise IndexError('Attenuator index out of range!', self.path_count, idx)
        if value > self.max_atten:
            raise ValueError('Attenuator value out of range!', self.max_atten, value)
        self._telnet_client.cmd('CHAN:%s:SETATT:%s' % (idx + 1, value))

    def get_atten(self, idx=0):
        """This function returns the current attenuation from an attenuator at a
    given index in the instrument.

    Args:
        idx: This zero-based index is the identifier for a particular
            attenuator in an instrument.

    Raises:
        Error: The underlying telnet connection to the instrument is not
            open.

    Returns:
        A float that is the current attenuation value.
    """
        if not self.is_open:
            raise attenuator.Error('Connection to attenuator at %s is not open!' % self._telnet_client.host)
        if idx + 1 > self.path_count or idx < 0:
            raise IndexError('Attenuator index out of range!', self.path_count, idx)
        telnet_cmd = ':ATT?' if self.path_count == 1 else 'CHAN:%s:ATT?' % (idx + 1)
        atten_val_str = self._telnet_client.cmd(telnet_cmd)
        atten_val = float(atten_val_str)
        return atten_val

This provides a specific telnet-controlled implementation of
AttenuatorDevice for Mini-Circuits RC-DAT attenuators.

Attributes:
    path_count: The number of signal attenuation path this device has.


def generate_inputs_outputs(tflite_model_binary, min_value=0, max_value=255):
    interpreter = lite.Interpreter(model_content=tflite_model_binary)
    interpreter.allocate_tensors()
    input_details = interpreter.get_input_details()
    input_values = {}
    for input_detail in input_details:
        input_value = create_tensor_data(input_detail['dtype'], input_detail['shape'], min_value=min_value, max_value=max_value)
        interpreter.set_tensor(input_detail['index'], input_value)
        input_values.update({_normalize_input_name(input_detail['name']): input_value})
    interpreter.invoke()
    output_details = interpreter.get_output_details()
    output_values = {}
    for output_detail in output_details:
        output_values.update({_normalize_output_name(output_detail['name']): interpreter.get_tensor(output_detail['index'])})
    return (input_values, output_values)

Generate input values and output values of the given tflite model.

Args:
  tflite_model_binary: A serialized flatbuffer as a string.
  min_value: min value for the input tensor.
  max_value: max value for the input tensor.

Returns:
  (input_values, output_values): Maps of input values and output values
  built.


def huber(y_true, y_pred, delta=1.0):
    y_pred = math_ops.cast(y_pred, dtype=backend.floatx())
    y_true = math_ops.cast(y_true, dtype=backend.floatx())
    delta = math_ops.cast(delta, dtype=backend.floatx())
    error = math_ops.subtract(y_pred, y_true)
    abs_error = math_ops.abs(error)
    half = tensor_conversion.convert_to_tensor_v2_with_dispatch(0.5, dtype=abs_error.dtype)
    return backend.mean(array_ops.where_v2(abs_error <= delta, half * math_ops.square(error), delta * abs_error - half * math_ops.square(delta)), axis=-1)

Computes Huber loss value.

For each value x in `error = y_true - y_pred`:

```
loss = 0.5 * x^2                  if |x| <= d
loss = d * |x| - 0.5 * d^2        if |x| > d
```
where d is `delta`. See: https://en.wikipedia.org/wiki/Huber_loss

Args:
  y_true: tensor of true targets.
  y_pred: tensor of predicted targets.
  delta: A float, the point where the Huber loss function changes from a
    quadratic to linear.

Returns:
  Tensor with one scalar loss entry per sample.


def class_predictor(self, image_feats: torch.FloatTensor, query_embeds: Optional[torch.FloatTensor]=None, query_mask: Optional[torch.Tensor]=None) -> Tuple[torch.FloatTensor]:
    pred_logits, image_class_embeds = self.class_head(image_feats, query_embeds, query_mask)
    return (pred_logits, image_class_embeds)

Args:
    image_feats:
        Features extracted from the `image_text_embedder`.
    query_embeds:
        Text query embeddings.
    query_mask:
        Must be provided with query_embeddings. A mask indicating which query embeddings are valid.


def _enter_scope_uncached(self):
    if self._auxiliary_name_scope:
        current_name_scope = None
    else:
        name_scope = ops.get_name_scope()
        if name_scope:
            name_scope += '/'
            current_name_scope = ops.name_scope(name_scope, skip_on_eager=False)
        else:
            current_name_scope = ops.name_scope(name_scope, skip_on_eager=False)
    if self._name_or_scope is not None:
        if not isinstance(self._name_or_scope, (VariableScope, str)):
            raise TypeError('VariableScope: name_or_scope must be a string or VariableScope.')
        if isinstance(self._name_or_scope, str):
            name_scope = self._name_or_scope
        else:
            name_scope = self._name_or_scope.name.split('/')[-1]
        if name_scope or current_name_scope:
            current_name_scope = current_name_scope or ops.name_scope(name_scope, skip_on_eager=False)
            try:
                current_name_scope_name = current_name_scope.__enter__()
            except:
                current_name_scope.__exit__(*sys.exc_info())
                raise
            self._current_name_scope = current_name_scope
            if isinstance(self._name_or_scope, str):
                old_name_scope = current_name_scope_name
            else:
                old_name_scope = self._name_or_scope.original_name_scope
            pure_variable_scope = _pure_variable_scope(self._name_or_scope, reuse=self._reuse, initializer=self._initializer, regularizer=self._regularizer, caching_device=self._caching_device, partitioner=self._partitioner, custom_getter=self._custom_getter, old_name_scope=old_name_scope, dtype=self._dtype, use_resource=self._use_resource, constraint=self._constraint)
            try:
                entered_pure_variable_scope = pure_variable_scope.__enter__()
            except:
                pure_variable_scope.__exit__(*sys.exc_info())
                raise
            self._cached_pure_variable_scope = pure_variable_scope
            return entered_pure_variable_scope
        else:
            self._current_name_scope = None
            pure_variable_scope = _pure_variable_scope(self._name_or_scope, reuse=self._reuse, initializer=self._initializer, regularizer=self._regularizer, caching_device=self._caching_device, partitioner=self._partitioner, custom_getter=self._custom_getter, dtype=self._dtype, use_resource=self._use_resource, constraint=self._constraint)
            try:
                entered_pure_variable_scope = pure_variable_scope.__enter__()
            except:
                pure_variable_scope.__exit__(*sys.exc_info())
                raise
            self._cached_pure_variable_scope = pure_variable_scope
            return entered_pure_variable_scope
    else:
        if self._reuse:
            raise ValueError('reuse=True cannot be used without a name_or_scope')
        current_name_scope = current_name_scope or ops.name_scope(self._default_name, skip_on_eager=False)
        try:
            current_name_scope_name = current_name_scope.__enter__()
        except:
            current_name_scope.__exit__(*sys.exc_info())
            raise
        self._current_name_scope = current_name_scope
        unique_default_name = _get_unique_variable_scope(self._default_name)
        pure_variable_scope = _pure_variable_scope(unique_default_name, initializer=self._initializer, regularizer=self._regularizer, caching_device=self._caching_device, partitioner=self._partitioner, custom_getter=self._custom_getter, old_name_scope=current_name_scope_name, dtype=self._dtype, use_resource=self._use_resource, constraint=self._constraint)
        try:
            entered_pure_variable_scope = pure_variable_scope.__enter__()
        except:
            pure_variable_scope.__exit__(*sys.exc_info())
            raise
        self._cached_pure_variable_scope = pure_variable_scope
        return entered_pure_variable_scope

Enters the context manager when there is no cached scope yet.

Returns:
  The entered variable scope.

Raises:
  TypeError: A wrong type is passed as `scope` at __init__().
  ValueError: `reuse` is incorrectly set at __init__().


def get_special_tokens_mask(self, token_ids_0: List[int], token_ids_1: Optional[List[int]]=None, already_has_special_tokens: bool=False) -> List[int]:
    if already_has_special_tokens:
        return super().get_special_tokens_mask(token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True)
    if token_ids_1 is None:
        return [1] + [0] * len(token_ids_0) + [1]
    return [1] + [0] * len(token_ids_0) + [1, 1] + [0] * len(token_ids_1) + [1]

Args:
Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer `prepare_for_model` method.
    token_ids_0 (`List[int]`):
        List of IDs.
    token_ids_1 (`List[int]`, *optional*):
        Optional second list of IDs for sequence pairs.
    already_has_special_tokens (`bool`, *optional*, defaults to `False`):
        Whether or not the token list is already formatted with special tokens for the model.
Returns:
    `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.


def as_tensor(self):
    with ops.control_dependencies(None):
        return self._concat()

Returns the overall concatenated value as a `Tensor`.

The returned tensor will not inherit the control dependencies from the scope
where the value is used, which is similar to getting the value of
`Variable`.

Returns:
  `Tensor` containing the concatenated value.


class Mamba2Config(PretrainedConfig):
    model_type = 'mamba2'

    def __init__(self, num_heads=128, head_dim=64, vocab_size=32768, hidden_size=4096, state_size=128, num_hidden_layers=64, layer_norm_epsilon=1e-05, pad_token_id=1, bos_token_id=0, eos_token_id=2, expand=2, conv_kernel=4, n_groups=8, use_bias=False, use_conv_bias=True, hidden_act='silu', initializer_range=0.1, residual_in_fp32=True, time_step_rank='auto', time_step_min=0.001, time_step_max=0.1, time_step_floor=0.0001, time_step_limit=(0.0, float('inf')), rescale_prenorm_residual=False, use_cache=True, rms_norm=True, chunk_size=256, tie_word_embeddings=False, **kwargs):
        if hidden_size * expand != num_heads * head_dim:
            raise ValueError(f'Inconsistent configuration: hidden_size * expand ({hidden_size * expand}) must equal num_heads * head_dim ({num_heads * head_dim}).')
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.state_size = state_size
        self.num_hidden_layers = num_hidden_layers
        self.layer_norm_epsilon = layer_norm_epsilon
        self.conv_kernel = conv_kernel
        self.expand = expand
        self.bos_token_id = bos_token_id
        self.eos_token_id = eos_token_id
        self.pad_token_id = pad_token_id
        self.use_bias = use_bias
        self.use_conv_bias = use_conv_bias
        self.hidden_act = hidden_act
        self.initializer_range = initializer_range
        self.time_step_rank = math.ceil(self.hidden_size / 16) if time_step_rank == 'auto' else time_step_rank
        self.time_step_min = time_step_min
        self.time_step_max = time_step_max
        self.time_step_floor = time_step_floor
        self.rescale_prenorm_residual = rescale_prenorm_residual
        self.residual_in_fp32 = residual_in_fp32
        self.use_cache = use_cache
        self.n_groups = n_groups
        self.num_heads = num_heads
        self.head_dim = head_dim
        self.rms_norm = rms_norm
        self.state_size = state_size
        self.chunk_size = chunk_size
        self.time_step_limit = time_step_limit
        self.tie_word_embeddings = tie_word_embeddings
        super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, pad_token_id=pad_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs)

This is the configuration class to store the configuration of a [`Mamba2Model`]. It is used to instantiate a MAMBA2
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
defaults will yield a similar configuration to that of the MAMBA2
[state-spaces/mamba2-2.8b](https://huggingface.co/state-spaces/mamba2-2.8b) architecture.

Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PretrainedConfig`] for more information.

Args:
    num_heads (`int`, *optional*, defaults to 128):
        Number of heads for the evolution matrices of mamba 2.
    head_dim (`int`, *optional*, defaults to 64):
        Dimension of each head.
    vocab_size (`int`, *optional*, defaults to 32768):
        Vocabulary size of the MAMBA2 model. Defines the number of different tokens that can be represented by the
        `inputs_ids` passed when calling [`Mamba2Model`].
    hidden_size (`int`, *optional*, defaults to 4096):
        Dimensionality of the embeddings and hidden states.
    state_size (`int`, *optional*, defaults to 128): shape of the state space latents.
    num_hidden_layers (`int`, *optional*, defaults to 64):
        Number of hidden layers in the model.
    layer_norm_epsilon (`float`, *optional*, defaults to 1e-05):
        The epsilon to use in the layer normalization layers.
    pad_token_id (`int`, *optional*, defaults to 1):
        Padding token id.
    bos_token_id (`int`, *optional*, defaults to 0):
        The id of the beginning of sentence token in the vocabulary.
    eos_token_id (`int`, *optional*, defaults to 2):
        The id of the end of sentence token in the vocabulary.
    expand (`int`, *optional*, defaults to 2): Expanding factor used to determine the intermediate size.
    conv_kernel (`int`, *optional*, defaults to 4): Size of the convolution kernel.
    n_groups (`int`, *optional*, defaults to 8):
        Number of groups for the evolution matrices of mamba 2.
    use_bias (`bool`, *optional*, defaults to `False`):
        Whether or not to use bias in ["in_proj", "out_proj"] of the mixer block
    use_conv_bias (`bool`, *optional*, defaults to `True`):
        Whether or not to use bias in the convolution layer of the mixer block.
    hidden_act (`str`, *optional*, defaults to `"silu"`):
        The non-linear activation function (function or string) in the decoder.
    initializer_range (`float`, *optional*, defaults to 0.1):
        The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
    residual_in_fp32 (`bool`, *optional*, defaults to `True`):
        Whether or not residuals should be in `float32`. If set to `False` residuals will keep the same `dtype` as the rest of the model
    time_step_rank (`Union[int,str]`, *optional*, defaults to `"auto"`):
        Rank of the discretization projection matrix. `"auto"` means that it will default to `math.ceil(self.hidden_size / 16)`
    time_step_min (`float`, *optional*, defaults to 0.001):
        Minimum `time_step` used to bound `dt_proj.bias`.
    time_step_max (`float`, *optional*, defaults to 0.1):
        Maximum `time_step` used to bound `dt_proj.bias`.
    time_step_floor (`float`, *optional*, defaults to 0.0001):
        Minimum clamping value of the `dt_proj.bias` layer initialization.
    time_step_limit (`tuple`, *optional*, defaults to `(0.0, inf)`):
        Accepted range of time step values.
    rescale_prenorm_residual (`bool`, *optional*, defaults to `False`):
        Whether or not to rescale `out_proj` weights when initializing.
    use_cache (`bool`, *optional*, defaults to `True`):
        Whether or not the cache should be used.
    rms_norm (`bool`, *optional*, defaults to `True`):
        Whether to use RMS norm or not.
    chunk_size (`int`, *optional*, defaults to 256):
        Size of the chunks that will comprise the sequence.
    tie_word_embeddings (`bool`, *optional*, defaults to `False`):
        Whether to tie word embeddings or not.

Example:

```python
>>> from transformers import Mamba2Config, Mamba2Model

>>> # Initializing a Mamba2 configuration
>>> configuration = Mamba2Config()

>>> # Initializing a model (with random weights) from the configuration
>>> model = Mamba2Model(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```


def _get_profile_data_generator(self):
    node_to_file_path = {}
    node_to_line_number = {}
    node_to_func_name = {}
    node_to_op_type = {}
    for op in self._graph.get_operations():
        for trace_entry in reversed(op.traceback):
            file_path = trace_entry[0]
            line_num = trace_entry[1]
            func_name = trace_entry[2]
            if not source_utils.guess_is_tensorflow_py_library(file_path):
                break
        node_to_file_path[op.name] = file_path
        node_to_line_number[op.name] = line_num
        node_to_func_name[op.name] = func_name
        node_to_op_type[op.name] = op.type

    def profile_data_generator(device_step_stats):
        for node_stats in device_step_stats.node_stats:
            if node_stats.node_name == '_SOURCE' or node_stats.node_name == '_SINK':
                continue
            yield profiling.ProfileDatum(device_step_stats.device, node_stats, node_to_file_path.get(node_stats.node_name, ''), node_to_line_number.get(node_stats.node_name, 0), node_to_func_name.get(node_stats.node_name, ''), node_to_op_type.get(node_stats.node_name, ''))
    return profile_data_generator

Get function that generates `ProfileDatum` objects.

Returns:
  A function that generates `ProfileDatum` objects.


def build_relative_position(query_size, key_size, bucket_size=-1, max_position=-1):
    q_ids = tf.range(query_size, dtype=tf.int32)
    k_ids = tf.range(key_size, dtype=tf.int32)
    rel_pos_ids = q_ids[:, None] - tf.tile(tf.expand_dims(k_ids, axis=0), [shape_list(q_ids)[0], 1])
    if bucket_size > 0 and max_position > 0:
        rel_pos_ids = make_log_bucket_position(rel_pos_ids, bucket_size, max_position)
    rel_pos_ids = rel_pos_ids[:query_size, :]
    rel_pos_ids = tf.expand_dims(rel_pos_ids, axis=0)
    return tf.cast(rel_pos_ids, tf.int64)

Build relative position according to the query and key

We assume the absolute position of query \(P_q\) is range from (0, query_size) and the absolute position of key
\(P_k\) is range from (0, key_size), The relative positions from query to key is \(R_{q \rightarrow k} = P_q -
P_k\)

Args:
    query_size (int): the length of query
    key_size (int): the length of key
    bucket_size (int): the size of position bucket
    max_position (int): the maximum allowed absolute position

Return:
    `tf.Tensor`: A tensor with shape [1, query_size, key_size]


def mel_filter_bank(num_frequency_bins: int, num_mel_filters: int, min_frequency: float, max_frequency: float, sampling_rate: int, norm: Optional[str]=None, mel_scale: str='htk', triangularize_in_mel_space: bool=False) -> np.ndarray:
    if norm is not None and norm != 'slaney':
        raise ValueError('norm must be one of None or "slaney"')
    if num_frequency_bins < 2:
        raise ValueError(f'Require num_frequency_bins: {num_frequency_bins} >= 2')
    if min_frequency > max_frequency:
        raise ValueError(f'Require min_frequency: {min_frequency} <= max_frequency: {max_frequency}')
    mel_min = hertz_to_mel(min_frequency, mel_scale=mel_scale)
    mel_max = hertz_to_mel(max_frequency, mel_scale=mel_scale)
    mel_freqs = np.linspace(mel_min, mel_max, num_mel_filters + 2)
    filter_freqs = mel_to_hertz(mel_freqs, mel_scale=mel_scale)
    if triangularize_in_mel_space:
        fft_bin_width = sampling_rate / ((num_frequency_bins - 1) * 2)
        fft_freqs = hertz_to_mel(fft_bin_width * np.arange(num_frequency_bins), mel_scale=mel_scale)
        filter_freqs = mel_freqs
    else:
        fft_freqs = np.linspace(0, sampling_rate // 2, num_frequency_bins)
    mel_filters = _create_triangular_filter_bank(fft_freqs, filter_freqs)
    if norm is not None and norm == 'slaney':
        enorm = 2.0 / (filter_freqs[2:num_mel_filters + 2] - filter_freqs[:num_mel_filters])
        mel_filters *= np.expand_dims(enorm, 0)
    if (mel_filters.max(axis=0) == 0.0).any():
        warnings.warn(f'At least one mel filter has all zero values. The value for `num_mel_filters` ({num_mel_filters}) may be set too high. Or, the value for `num_frequency_bins` ({num_frequency_bins}) may be set too low.')
    return mel_filters

Creates a frequency bin conversion matrix used to obtain a mel spectrogram. This is called a *mel filter bank*, and
various implementation exist, which differ in the number of filters, the shape of the filters, the way the filters
are spaced, the bandwidth of the filters, and the manner in which the spectrum is warped. The goal of these
features is to approximate the non-linear human perception of the variation in pitch with respect to the frequency.

Different banks of mel filters were introduced in the literature. The following variations are supported:

- MFCC FB-20: introduced in 1980 by Davis and Mermelstein, it assumes a sampling frequency of 10 kHz and a speech
  bandwidth of `[0, 4600]` Hz.
- MFCC FB-24 HTK: from the Cambridge HMM Toolkit (HTK) (1995) uses a filter bank of 24 filters for a speech
  bandwidth of `[0, 8000]` Hz. This assumes sampling rate ≥ 16 kHz.
- MFCC FB-40: from the Auditory Toolbox for MATLAB written by Slaney in 1998, assumes a sampling rate of 16 kHz and
  speech bandwidth of `[133, 6854]` Hz. This version also includes area normalization.
- HFCC-E FB-29 (Human Factor Cepstral Coefficients) of Skowronski and Harris (2004), assumes a sampling rate of
  12.5 kHz and speech bandwidth of `[0, 6250]` Hz.

This code is adapted from *torchaudio* and *librosa*. Note that the default parameters of torchaudio's
`melscale_fbanks` implement the `"htk"` filters while librosa uses the `"slaney"` implementation.

Args:
    num_frequency_bins (`int`):
        Number of frequency bins (should be the same as `n_fft // 2 + 1` where `n_fft` is the size of the Fourier Transform used to compute the spectrogram).
    num_mel_filters (`int`):
        Number of mel filters to generate.
    min_frequency (`float`):
        Lowest frequency of interest in Hz.
    max_frequency (`float`):
        Highest frequency of interest in Hz. This should not exceed `sampling_rate / 2`.
    sampling_rate (`int`):
        Sample rate of the audio waveform.
    norm (`str`, *optional*):
        If `"slaney"`, divide the triangular mel weights by the width of the mel band (area normalization).
    mel_scale (`str`, *optional*, defaults to `"htk"`):
        The mel frequency scale to use, `"htk"`, `"kaldi"` or `"slaney"`.
    triangularize_in_mel_space (`bool`, *optional*, defaults to `False`):
        If this option is enabled, the triangular filter is applied in mel space rather than frequency space. This
        should be set to `true` in order to get the same results as `torchaudio` when computing mel filters.

Returns:
    `np.ndarray` of shape (`num_frequency_bins`, `num_mel_filters`): Triangular filter bank matrix. This is a
    projection matrix to go from a spectrogram to a mel spectrogram.


def process(self, element, *args, **kwargs):
    yield {'text': element.data.decode('utf-8'), 'id': element.attributes['id']}

For each element in the input PCollection, retrieve the id and decode the bytes into string

Args:
  element: The element that is being processed.


def __init__(self, num_units, use_peepholes=False, cell_clip=None, initializer=None, num_proj=None, proj_clip=None, num_unit_shards=None, num_proj_shards=None, forget_bias=1.0, state_is_tuple=True, activation=None, reuse=None, name=None, dtype=None, **kwargs):
    warnings.warn('`tf.nn.rnn_cell.LSTMCell` is deprecated and will be removed in a future version. This class is equivalent as `tf.keras.layers.LSTMCell`, and will be replaced by that in Tensorflow 2.0.')
    super(LSTMCell, self).__init__(_reuse=reuse, name=name, dtype=dtype, **kwargs)
    _check_supported_dtypes(self.dtype)
    if not state_is_tuple:
        logging.warning('%s: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.', self)
    if num_unit_shards is not None or num_proj_shards is not None:
        logging.warning('%s: The num_unit_shards and proj_unit_shards parameters are deprecated and will be removed in Jan 2017.  Use a variable scope with a partitioner instead.', self)
    if context.executing_eagerly() and tf_config.list_logical_devices('GPU'):
        logging.warning('%s: Note that this cell is not optimized for performance. Please use tf.contrib.cudnn_rnn.CudnnLSTM for better performance on GPU.', self)
    self.input_spec = input_spec.InputSpec(ndim=2)
    self._num_units = num_units
    self._use_peepholes = use_peepholes
    self._cell_clip = cell_clip
    self._initializer = initializers.get(initializer)
    self._num_proj = num_proj
    self._proj_clip = proj_clip
    self._num_unit_shards = num_unit_shards
    self._num_proj_shards = num_proj_shards
    self._forget_bias = forget_bias
    self._state_is_tuple = state_is_tuple
    if activation:
        self._activation = activations.get(activation)
    else:
        self._activation = math_ops.tanh
    if num_proj:
        self._state_size = LSTMStateTuple(num_units, num_proj) if state_is_tuple else num_units + num_proj
        self._output_size = num_proj
    else:
        self._state_size = LSTMStateTuple(num_units, num_units) if state_is_tuple else 2 * num_units
        self._output_size = num_units

Initialize the parameters for an LSTM cell.

Args:
  num_units: int, The number of units in the LSTM cell.
  use_peepholes: bool, set True to enable diagonal/peephole connections.
  cell_clip: (optional) A float value, if provided the cell state is clipped
    by this value prior to the cell output activation.
  initializer: (optional) The initializer to use for the weight and
    projection matrices.
  num_proj: (optional) int, The output dimensionality for the projection
    matrices.  If None, no projection is performed.
  proj_clip: (optional) A float value.  If `num_proj > 0` and `proj_clip` is
    provided, then the projected values are clipped elementwise to within
    `[-proj_clip, proj_clip]`.
  num_unit_shards: Deprecated, will be removed by Jan. 2017. Use a
    variable_scope partitioner instead.
  num_proj_shards: Deprecated, will be removed by Jan. 2017. Use a
    variable_scope partitioner instead.
  forget_bias: Biases of the forget gate are initialized by default to 1 in
    order to reduce the scale of forgetting at the beginning of the
    training. Must set it manually to `0.0` when restoring from CudnnLSTM
    trained checkpoints.
  state_is_tuple: If True, accepted and returned states are 2-tuples of the
    `c_state` and `m_state`.  If False, they are concatenated along the
    column axis.  This latter behavior will soon be deprecated.
  activation: Activation function of the inner states.  Default: `tanh`. It
    could also be string that is within Keras activation function names.
  reuse: (optional) Python boolean describing whether to reuse variables in
    an existing scope.  If not `True`, and the existing scope already has
    the given variables, an error is raised.
  name: String, the name of the layer. Layers with the same name will share
    weights, but to avoid mistakes we require reuse=True in such cases.
  dtype: Default dtype of the layer (default of `None` means use the type of
    the first input). Required when `build` is called before `call`.
  **kwargs: Dict, keyword named properties for common layer attributes, like
    `trainable` etc when constructing the cell from configs of get_config().
    When restoring from CudnnLSTM-trained checkpoints, use
    `CudnnCompatibleLSTMCell` instead.


def from_keras_model(cls, model):
    TFLiteConverterBase._set_original_model_type(conversion_metadata_fb.ModelType.KERAS_MODEL)
    return TFLiteKerasModelConverterV2(model)

Creates a TFLiteConverter object from a Keras model.

Args:
  model: tf.Keras.Model

Returns:
  TFLiteConverter object.


def get_index(uid, i):
    return _SHARED_SEQUENCES[uid][i]

Get the value from the Sequence `uid` at index `i`.

To allow multiple Sequences to be used at the same time, we use `uid` to
get a specific one. A single Sequence would cause the validation to
overwrite the training Sequence.

Args:
    uid: int, Sequence identifier
    i: index

Returns:
    The value at index `i`.


def checksum(self, path):
    try:
        return self._blobstorageIO().checksum(path)
    except Exception as e:
        raise BeamIOError('Checksum operation failed', {path, e})

Fetch checksum metadata of a file on the
:class:`~apache_beam.io.filesystem.FileSystem`.

Args:
  path: string path of a file.

Returns: string containing checksum

Raises:
  ``BeamIOError``: if path isn't a file or doesn't exist.


def read_gbq(table, dataset=None, project_id=None, use_bqstorage_api=False, **kwargs):
    if table is None:
        raise ValueError('Please specify a BigQuery table to read from.')
    elif len(kwargs) > 0:
        raise ValueError(f'Encountered unsupported parameter(s) in read_gbq: {kwargs.keys()!r}')
    return _ReadGbq(table, dataset, project_id, use_bqstorage_api)

This function reads data from a BigQuery table and produces a
:class:`~apache_beam.dataframe.frames.DeferredDataFrame.

Args:
  table (str): Please specify a table. This can be done in the format
    'PROJECT:dataset.table' if one would not wish to utilize
    the parameters below.
  dataset (str): Please specify the dataset
    (can omit if table was specified as 'PROJECT:dataset.table').
  project_id (str): Please specify the project ID
    (can omit if table was specified as 'PROJECT:dataset.table').
  use_bqstorage_api (bool): If you would like to utilize
    the BigQuery Storage API in ReadFromBigQuery, please set
    this flag to true. Otherwise, please set flag
    to false or leave it unspecified.


def wait_for_boot_completion(self, timeout=DEFAULT_TIMEOUT_BOOT_COMPLETION_SECOND):
    deadline = time.perf_counter() + timeout
    self.adb.wait_for_device(timeout=timeout)
    while time.perf_counter() < deadline:
        try:
            if self.is_boot_completed():
                return
        except (adb.AdbError, adb.AdbTimeoutError):
            pass
        time.sleep(5)
    raise DeviceError(self, 'Booting process timed out')

Waits for Android framework to broadcast ACTION_BOOT_COMPLETED.

This function times out after 15 minutes.

Args:
  timeout: float, the number of seconds to wait before timing out.
    If not specified, no timeout takes effect.


def hours(value: Union[int, float]) -> Duration:
    return float(value * 60 * 60)

Converts input value from hours to a `Duration` in seconds.

Example:
    ```python
    >>> timestamps = [tp.duration.hours(i) for i in [1, 2, 10]]
    >>> timestamps
    [3600.0, 7200.0, 36000.0]

    >>> # Usage in a window operation
    >>> a = tp.event_set(timestamps=timestamps, features={"f1": [1, 5, -5]})
    >>> a.moving_sum(window_length=tp.duration.hours(2))
    indexes: ...
            timestamps: [ 3600. 7200. 36000.]
            'f1': [ 1 6 -5]
    ...

    ```

Args:
    value: Number of hours.

Returns:
    Equivalent number of seconds.


def dynamic_update_slice(operand, update, start_indices):
    operand = tf_np.asarray(operand).data
    update = tf_np.asarray(update).data
    start_indices = tf_np.asarray(start_indices, np.int32).data
    if not update.shape.is_fully_defined():
        raise ValueError("update's shape must be fully defined")
    slice_sizes = update.shape
    idx = _get_dynamic_indices(operand, start_indices, slice_sizes)
    if idx is None:
        return tf_np.asarray(update)
    operand = array_ops.tensor_scatter_nd_update(operand, idx, update)
    return tf_np.asarray(operand)

Updates a dynamic slice.

See the docstring of `jax.lax.dynamic_update_slice`
(https://jax.readthedocs.io/en/latest/_autosummary/jax.lax.dynamic_update_slice.html)
for details.

Args:
  operand: an array to slice.
  update: an array containing the new values to write onto `operand`.
  start_indices: a vector of integers, one per dimension. The starts of the
    slice. The vector can be dynamic.

Returns:
  The updated version of `operand`.


def decode(self, token_ids, **kwargs):
    return super().decode(token_ids, **kwargs)

Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special
tokens and clean up tokenization spaces.

Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`.

Args:
    token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
        List of tokenized input ids. Can be obtained using the `__call__` method.
    skip_special_tokens (`bool`, *optional*, defaults to `False`):
        Whether or not to remove special tokens in the decoding.
    clean_up_tokenization_spaces (`bool`, *optional*):
        Whether or not to clean up the tokenization spaces. If `None`, will default to
        `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`).
    use_source_tokenizer (`bool`, *optional*, defaults to `False`):
        Whether or not to use the source tokenizer to decode sequences (only applicable in sequence-to-sequence
        problems).
    kwargs (additional keyword arguments, *optional*):
        Will be passed to the underlying model specific decode method.

Returns:
    `str`: The decoded sentence.


def get_prefix_bias_name(self) -> Union[None, str]:
    warnings.warn('The method get_prefix_bias_name is deprecated. Please use `get_bias` instead.', FutureWarning)
    return None

Get the concatenated _prefix name of the bias from the model name to the parent layer

Return:
    `str`: The _prefix name of the bias.