def string_split_v2(source, sep=None, maxsplit=-1):
    if sep is None:
        sep = ''
    sep = ops.convert_to_tensor(sep, dtype=dtypes.string)
    source = ops.convert_to_tensor(source, dtype=dtypes.string)
    indices, values, shape = gen_string_ops.string_split_v2(source, sep=sep, maxsplit=maxsplit)
    indices.set_shape([None, 2])
    values.set_shape([None])
    shape.set_shape([2])
    return sparse_tensor.SparseTensor(indices, values, shape)

Split elements of `source` based on `sep` into a `SparseTensor`.

    Let N be the size of source (typically N will be the batch size). Split each
    element of `source` based on `sep` and return a `SparseTensor`
    containing the split tokens. Empty tokens are ignored.

    For example, N = 2, source[0] is 'hello world' and source[1] is 'a b c',
    then the output will be

    st.indices = [0, 0;
                0, 1;
                1, 0;
                1, 1;
                1, 2]
    st.shape = [2, 3]
    st.values = ['hello', 'world', 'a', 'b', 'c']

    If `sep` is given, consecutive delimiters are not grouped together and are
    deemed to delimit empty strings. For example, source of `"1<>2<><>3"` and
    sep of `"<>"` returns `["1", "2", "", "3"]`. If `sep` is None or an empty
    string, consecutive whitespace are regarded as a single separator, and the
    result will contain no empty strings at the start or end if the string has
    leading or trailing whitespace.

    Note that the above mentioned behavior matches python's str.split.

Args:
        source: `1-D` string `Tensor`, the strings to split.
        sep: `0-D` string `Tensor`, the delimiter character.
        maxsplit: An `int`. If `maxsplit > 0`, limit of the split of the result.

Raises:
        ValueError: If sep is not a string.

Returns:
        A `SparseTensor` of rank `2`, the strings split according to the delimiter.
        The first column of the indices corresponds to the row in `source` and the
        second column corresponds to the index of the split component in this row.


def build_inputs_with_special_tokens(self, token_ids_0: List[int], token_ids_1: Optional[List[int]]=None) -> List[int]:
    if token_ids_1 is None:
        return self.bos_token_id + token_ids_0 + self.eos_token_id
    return self.bos_token_id + token_ids_0 + token_ids_1 + self.eos_token_id

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
    adding special tokens. The special tokens depend on calling set_lang.

    An NLLB sequence has the following format, where `X` represents the sequence:

    - `input_ids` (for encoder) `X [eos, src_lang_code]`
    - `decoder_input_ids`: (for decoder) `X [eos, tgt_lang_code]`

    BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
    separator.

Args:
        token_ids_0 (`List[int]`):
            List of IDs to which the special tokens will be added.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

Returns:
        `List[int]`: list of [input IDs](../glossary#input-ids) with the appropriate special tokens.


def __init__(self, path_elements: List[Union[str, int]], parent: Optional['Key']=None, project: Optional[str]=None, namespace: Optional[str]=None):
    self.path_elements = tuple(path_elements)
    self.parent = parent
    self.namespace = namespace
    self.project = project

Represents a Datastore key.

    The partition ID is represented by its components: namespace and project.
    If key has a parent, project and namespace should either be unset or match
    the parent's.

Args:
        path_elements: (list of str and int) Key path: an alternating sequence of
        kind and identifier. The kind must be of type ``str`` and identifier may
        be a ``str`` or an ``int``.
        If the last identifier is omitted this is an incomplete key, which is
        unsupported in ``WriteToDatastore`` and ``DeleteFromDatastore``.
        See :class:`google.cloud.datastore.key.Key` for more details.
        parent: (:class:`~apache_beam.io.gcp.datastore.v1new.types.Key`)
        (optional) Parent for this key.
        project: (str) Project ID. Required unless set by parent.
        namespace: (str) (optional) Namespace ID


def custom_gradient(fun):

    def __init__(self, fun):
        warnings.warn('`custom_gradient` for the openvino backend acts as a pass-through to support the forward pass. No gradient computation or modification takes place.')
        self.fun = fun

    def __call__(self, *args, **kwargs):
        outputs, _ = self.fun(*args, **kwargs)
        return outputs

Decorator for custom gradients.

Args:
        fun: Forward pass function.


def write(self, index, value, name=None):
    return self._implementation.write(index, value, name=name)

Write `value` into index `index` of the TensorArray.

Args:
        index: 0-D.  int32 scalar with the index to write to.
        value: N-D.  Tensor of type `dtype`.  The Tensor to write to this index.
        name: A name for the operation (optional).

Returns:
        A new TensorArray object with flow that ensures the write occurs.
        Use this object for all subsequent operations.

Raises:
        ValueError: if there are more writers than specified.


def _process_update(self, item, feed_item):
    item['name'] = feed_item.get(FieldMap.CAMPAIGN_LANDING_PAGE_NAME, None)
    item['url'] = feed_item.get(FieldMap.CAMPAIGN_LANDING_PAGE_URL, None)

Updates an landing page based on the values from the feed.

Args:
        item: Object representing the landing page to be updated, this object is
        updated directly.
        feed_item: Feed item representing landing page values from the Bulkdozer
        feed.


def minimum(inputs, **kwargs):
    return Minimum(**kwargs)(inputs)

Functional interface to the `Minimum` layer.

Args:
        inputs: A list of input tensors (at least 2).
        **kwargs: Standard layer keyword arguments.

Returns:
        A tensor, the element-wise minimum of the inputs.


def dice(y_true, y_pred, axis=None):
    y_pred = ops.convert_to_tensor(y_pred)
    y_true = ops.cast(y_true, y_pred.dtype)
    inputs = y_true
    targets = y_pred
    intersection = ops.sum(inputs * targets, axis=axis)
    dice = ops.divide(2.0 * intersection, ops.sum(y_true, axis=axis) + ops.sum(y_pred, axis=axis) + backend.epsilon())
    return 1 - dice

Computes the Dice loss value between `y_true` and `y_pred`.

    Formula:
    ```python
    loss = 1 - (2 * sum(y_true * y_pred)) / (sum(y_true) + sum(y_pred))
    ```

Args:
        y_true: tensor of true targets.
        y_pred: tensor of predicted targets.
        axis: tuple for which dimensions the loss is calculated

Returns:
        Dice loss value.

Example:
    >>> y_true = [[[[1.0], [1.0]], [[0.0], [0.0]]],
    ...           [[[1.0], [1.0]], [[0.0], [0.0]]]]
    >>> y_pred = [[[[0.0], [1.0]], [[0.0], [1.0]]],
    ...           [[[0.4], [0.0]], [[0.0], [0.9]]]]
    >>> axis = (1, 2, 3)
    >>> loss = keras.losses.dice(y_true, y_pred, axis=axis)
    >>> assert loss.shape == (2,)
    >>> loss
    array([0.5, 0.75757575], shape=(2,), dtype=float32)

    >>> loss = keras.losses.dice(y_true, y_pred)
    >>> assert loss.shape == ()
    >>> loss
    array(0.6164384, shape=(), dtype=float32)


def _compose_output_rep(lhs_rep, rhs_rep, lhs_contraction, rhs_contraction, lhs_batch, rhs_batch):
    output_rep = []
    for dim in lhs_batch:
        output_rep.append(lhs_rep[dim])
    for i in _minus(range(len(lhs_rep)), lhs_batch + lhs_contraction):
        output_rep.append(lhs_rep[i])
    for i in _minus(range(len(rhs_rep)), rhs_batch + rhs_contraction):
        output_rep.append(rhs_rep[i])
    return ''.join(output_rep)

Compose the output string representation.

    e.g., ij, jk, (((1,), (0,)), ((), ())) -> ik
        aij, ajk, (((2,), (1,)), ((0,), (0,))) -> aik

Args:
        lhs_rep: A string representation for the left-hand side input array
        rhs_rep: A string representation for the right-hand side input array
        lhs_contraction: Sequence[int] (the contraction dimensions of lhs)
        rhs_contraction: Sequence[int] (the contraction dimensions of rhs)
        lhs_batch: Sequence[int] (the batch dimensions of lhs)
        rhs_batch: Sequence[int] (the batch dimensions of rhs)

Returns:
        A string representation of the result array.


def BuildParams(self, graph_fn, dtype, input_shapes, output_shapes):
    input_mask = [[False] + [True] * (len(shape) - 1) for shape in input_shapes]
    output_mask = [[False] + [True] * (len(shape) - 1) if shape else [] for shape in output_shapes]
    return self.BuildParamsWithMask(graph_fn, dtype, input_shapes, output_shapes, input_mask, output_mask, [], [])

Build test parameters.

    The input_shapes and output_shapes arguments are known (static) shapes that
    can be used to generate test data. To define the model, we also specify
    corresponding input/output TensorSpecs. These are defined using the shape
    arguments. For each input tensor we define:

    input_spec = [None] + input_shape[1:]

    and similarly for output shapes. This means that we leave the first (batch)
    dimension unknown, the rest is just copied from the shapes arg.

Args:
        graph_fn: The function to build the graph.
        dtype: The element type.
        input_shapes: The input shapes.
        output_shapes: The output shapes.

Returns:
        The test parameters.


def __init__(self, path, file_naming=None, destination=None, temp_directory=None, sink=None, shards=None, output_fn=None, max_writers_per_bundle=MAX_NUM_WRITERS_PER_BUNDLE):
    self.path = path if isinstance(path, ValueProvider) else StaticValueProvider(str, path)
    self.file_naming_fn = file_naming or default_file_naming('output')
    self.destination_fn = self._get_destination_fn(destination)
    self._temp_directory = temp_directory
    self.sink_fn = self._get_sink_fn(sink)
    self.shards = shards or WriteToFiles.DEFAULT_SHARDING
    self.output_fn = output_fn or (lambda x: x)
    self._max_num_writers_per_bundle = max_writers_per_bundle

Initializes a WriteToFiles transform.

Args:
        path (str, ValueProvider): The directory to write files into.
        file_naming (callable): A callable that takes in a window, pane,
        shard_index, total_shards and compression; and returns a file name.
        destination (callable): If this argument is provided, the sink parameter
        must also be a callable.
        temp_directory (str, ValueProvider): To ensure atomicity in the transform,
        the output is written into temporary files, which are written to a
        directory that is meant to be temporary as well. Once the whole output
        has been written, the files are moved into their final destination, and
        given their final names. By default, the temporary directory will be
        within the temp_location of your pipeline.
        sink (callable, ~apache_beam.io.fileio.FileSink): The sink to use to write
        into a file. It should implement the methods of a ``FileSink``. Pass a
        class signature or an instance of FileSink to this parameter. If none is
        provided, a ``TextSink`` is used.
        shards (int): The number of shards per destination and trigger firing.
        output_fn (callable, optional): A callable to process the output. This
        parameter is currently unused and retained for backward compatibility.
        max_writers_per_bundle (int): The number of writers that can be open
        concurrently in a single worker that's processing one bundle.


def random_mixed_density_matrix(num_qubits, num_mixtures=5):
    pre_probs = tf.random.uniform([num_mixtures], 1e-09)
    mixture_probabilities = pre_probs / tf.reduce_sum(pre_probs)
    random_unitary = random_unitary_matrix(num_qubits)
    dim = 2 ** num_qubits
    final_state = tf.zeros([dim, dim], tf.complex128)
    for i in range(num_mixtures):
        pure_state = tf.one_hot(i, dim, 1.0, 0.0, 0, tf.complex128)
        evolved_pure_state = tf.linalg.matvec(random_unitary, pure_state)
        adjoint_evolved_pure_state = tf.squeeze(tf.linalg.adjoint(tf.expand_dims(evolved_pure_state, 0)))
        final_state = final_state + tf.cast(mixture_probabilities[i], tf.complex128) * tf.einsum('i,j->ij', evolved_pure_state, adjoint_evolved_pure_state)
    return (final_state, mixture_probabilities)

Returns a random pure density matrix.

    Applies a common random unitary to `num_mixtures` orthogonal states, then
    mixes them with random weights.

Args:
        num_qubits: Number of qubits on which the matrix acts.
        num_mixtures: The number of orthogonal pure states to mix.

Returns:
        final_state: The mixed density matrix.
        mixture_probabilities: The probability of each state in the mixture.


def wrap_cached_variables(concrete_function):
    outer_graph = func_graph_module.FuncGraph('{}_no_cache'.format(concrete_function.graph.name))
    mapped_captures = None
    remapped_captures = {}
    with outer_graph.as_default():
        for capture, placeholder in concrete_function.graph.captures:
            cached_variable = getattr(capture, '_cached_variable', None)
            if cached_variable is None:
                continue
            cached_variable = cached_variable()
            new_cached_value = cached_variable.read_value()
            key = id(capture)
            external = concrete_function.graph.function_captures.by_val_external[key]
            internal = concrete_function.graph.function_captures.by_val_internal[key]
            remapped_captures[key] = [external, internal]
            concrete_function.graph.function_captures.add_or_replace(key=key, external=new_cached_value, internal=placeholder, is_by_ref=False)
            mapped_captures = True
    if not mapped_captures:
        return concrete_function
    inner_concrete = defun.ConcreteFunction.from_func_graph(concrete_function.graph, concrete_function.function_type, {})

    def wrap_function(*args):
        return inner_concrete._call_flat(list(args), inner_concrete.captured_inputs)
    args = nest.flatten(concrete_function.structured_input_signature, expand_composites=True)
    func_graph_module.func_graph_from_py_func(None, wrap_function, args=tuple(args), kwargs={}, func_graph=outer_graph)
    fn = defun.ConcreteFunction.from_func_graph(outer_graph, concrete_function.function_type, {})
    fn._arg_keywords = concrete_function._arg_keywords
    fn._num_positional_args = concrete_function._num_positional_args
    for key, capture in remapped_captures.items():
        external, internal = capture
        concrete_function.graph._function_captures.add_or_replace(key=key, external=external, internal=internal, is_by_ref=False)
    return fn

Wraps the concrete function if it uses cached read tensors.

    This function creates a new concrete function that captures variables
    instead of the cached read tensors.

Args:
        concrete_function: A Concrete function that maybe captures cached read
        tensors.

Returns:
        A concrete function that wraps the original concrete function, which
        captures variables instead. If the original function did not capture any
        cached values, then the function is not wrapped and the original object is
        returned.


def _build_system_message(self, error: str) -> LogMessage:
    return self._base_log.copy() | LogMessage(log_type=LogType.SYSTEM.value, error=error)

Adds system error information to base log message.

Args:
        * error: error that occurred

Returns:
        * Log: dictionary containing log data


def post_process_grounded_object_detection(self, outputs: 'OwlViTObjectDetectionOutput', threshold: float=0.1, target_sizes: Optional[Union[TensorType, List[Tuple]]]=None, text_labels: Optional[List[List[str]]]=None):
    output = self.image_processor.post_process_object_detection(outputs=outputs, threshold=threshold, target_sizes=target_sizes)
    if text_labels is not None and len(text_labels) != len(output):
        raise ValueError('Make sure that you pass in as many lists of text labels as images')
    if text_labels is not None:
        for image_output, image_text_labels in zip(output, text_labels):
            object_text_labels = [image_text_labels[i] for i in image_output['labels']]
            image_output['text_labels'] = object_text_labels
    else:
        for image_output in output:
            image_output['text_labels'] = None
    return output

Converts the raw output of [`OwlViTForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y,
    bottom_right_x, bottom_right_y) format.

Args:
        outputs ([`OwlViTObjectDetectionOutput`]):
            Raw outputs of the model.
        threshold (`float`, *optional*, defaults to 0.1):
            Score threshold to keep object detection predictions.
        target_sizes (`torch.Tensor` or `List[Tuple[int, int]]`, *optional*):
            Tensor of shape `(batch_size, 2)` or list of tuples (`Tuple[int, int]`) containing the target size
            `(height, width)` of each image in the batch. If unset, predictions will not be resized.
        text_labels (`List[List[str]]`, *optional*):
            List of lists of text labels for each image in the batch. If unset, "text_labels" in output will be
            set to `None`.

Returns:
        `List[Dict]`: A list of dictionaries, each dictionary containing the following keys:
        - "scores": The confidence scores for each predicted box on the image.
        - "labels": Indexes of the classes predicted by the model on the image.
        - "boxes": Image bounding boxes in (top_left_x, top_left_y, bottom_right_x, bottom_right_y) format.
        - "text_labels": The text labels for each predicted bounding box on the image.


def assert_non_singular(self, name='assert_non_singular'):
    with self._name_scope(name):
        return self._assert_non_singular()

Returns an `Op` that asserts this operator is non singular.

    This operator is considered non-singular if

    ```
    ConditionNumber < max{100, range_dimension, domain_dimension} * eps,
    eps := np.finfo(self.dtype.as_numpy_dtype).eps
    ```

Args:
        name:  A string name to prepend to created ops.

Returns:
        An `Assert` `Op`, that, when run, will raise an `InvalidArgumentError` if
        the operator is singular.


def __init__(self, market_data_dict: Dict[str, Any], dtype: Optional[tf.DType]=None):
    self._valuation_date = dateslib.convert_to_date_tensor(market_data_dict['reference_date'])
    self._market_data_dict = market_data_dict
    self._dtype = dtype or tf.float64

Market data constructor.

    The dictionary must follow a format outlined in the doc:
    https://github.com/google/tf-quant-finance/tree/master/tf_quant_finance/experimental/pricing_platform/framework/market_data/market_data.pdf

Args:
        market_data_dict: Market data dictionary.
        dtype: A `dtype` to use for float-like `Tensor`s.
        Default value: `tf.float64`.


def max_pool(x, pool_size, strides, padding):
    x = tf_np.asarray(x)
    return tf_np.asarray(nn_ops.pool(input=x, window_shape=pool_size, pooling_type='MAX', strides=strides, padding=padding))

Performs an N-D max pooling.

Args:
        x: ndarray of rank N+2, of shape `[batch_size] + input_spatial_shape +
        [num_channels]`. Pooling happens over the spatial dimensions only.
        pool_size: sequence of N ints.
        strides: sequence of N ints.
        padding: a string, the padding algorithm. Must be "SAME" or "VALID".

Returns:
        An (N+2)-D array,  of shape
        [batch_size] + output_spatial_shape + [num_channels],
        where `output_spatial_shape` depends on the value of padding:
        If padding = "SAME":
        output_spatial_shape[i] = ceil(input_spatial_shape[i] / strides[i])
        If padding = "VALID":
        output_spatial_shape[i] =
        ceil((input_spatial_shape[i] - (pool_size[i] - 1)) / strides[i]).


def fail(msg, extras=None):
    raise signals.TestFailure(msg, extras)

Explicitly fail a test.

Args:
        msg: A string explaining the details of the failure.
        extras: An optional field for extra information to be included in
        test result.

Raises:
        signals.TestFailure: Mark a test as failed.


def __init__(self, grpc_debug_server_addresses, thread_name_filter=None, send_traceback_and_source_code=True):

    def _gated_grpc_watch_fn(fetches, feeds):
        del fetches, feeds
        return framework.WatchOptions(debug_ops=['DebugIdentity(gated_grpc=true)'])
    super(TensorBoardDebugHook, self).__init__(grpc_debug_server_addresses, watch_fn=_gated_grpc_watch_fn, thread_name_filter=thread_name_filter)
    self._grpc_debug_server_addresses = grpc_debug_server_addresses
    self._send_traceback_and_source_code = send_traceback_and_source_code
    self._sent_graph_version = -1
    grpc_wrapper.register_signal_handler()

Constructor of TensorBoardDebugHook.

Args:
        grpc_debug_server_addresses: gRPC address(es) of debug server(s), as a
        `str` or a `list` of `str`s. E.g., "localhost:2333",
        "grpc://localhost:2333", ["192.168.0.7:2333", "192.168.0.8:2333"].
        thread_name_filter: Optional filter for thread names.
        send_traceback_and_source_code: Whether traceback of graph elements and
        the source code are to be sent to the debug server(s).


def add(self, word: str):
    if not word:
        return
    self._tokens.add(word)
    ref = self.data
    for char in word:
        ref[char] = ref.setdefault(char, {})
        ref = ref[char]
    ref[self._termination_char] = 1

Passes over every char (utf-8 char) on word and recursively adds it to the internal `data` trie representation.
    The special key `""` in `self._termination_char` is used to represent termination.

    This function is idempotent, adding twice the same word will leave the trie unchanged

Example:
    ```python
    >>> trie = Trie()
    >>> trie.add("Hello 友達")
    >>> trie.data
    {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}

    >>> trie.add("Hello")
    >>> trie.data
    {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}
    ```


def __lt__(self: EventSetOrNode, other: Any) -> EventSetOrNode:
    if isinstance(other, self.__class__):
        from temporian.core.operators.binary import less
        return less(input_left=self, input_right=other)
    if isinstance(other, T_SCALAR):
        from temporian.core.operators.scalar import less_scalar
        return less_scalar(input=self, value=other)
    self._raise_error('compare', other, '(int,float)')
    assert False

Computes less (`self < other`) element-wise with another
    [`EventSet`][temporian.EventSet] or a scalar value.

    If an EventSet, each feature in `self` is compared element-wise to the
    feature in `other` in the same position. `self` and `other` must have
    the same sampling and the same number of features.

    If a scalar value, each item in each feature in `input` is compared to
    `value`.

    Note that it will always return False on NaN elements.

    Example with EventSet:
        ```python
        >>> a = tp.event_set(
        ...     timestamps=[1, 2, 3],
        ...     features={"f1": [0, 100, 200]}
        ... )
        >>> b = tp.event_set(
        ...     timestamps=[1, 2, 3],
        ...     features={"f2": [-10, 100, 5]},
        ...     same_sampling_as=a
        ... )

        >>> c = a < b
        >>> c
        indexes: []
        features: [('f1', bool_)]
        events:
            (3 events):
                timestamps: [1. 2. 3.]
                'f1': [False False False]
        ...

        ```

    Example with scalar:
        ```python
        >>> a = tp.event_set(
        ...     timestamps=[1, 2, 3],
        ...     features={"f1": [0, 100, 200], "f2": [-10, 100, 5]}
        ... )

        >>> b = a < 100
        >>> b
        indexes: []
        features: [('f1', bool_), ('f2', bool_)]
        events:
            (3 events):
                timestamps: [1. 2. 3.]
                'f1': [ True False False]
                'f2': [ True False True]
        ...

        ```

Args:
        other: EventSet or scalar value.

Returns:
        Result of the comparison.


def empty(self) -> 'Builder':
    return self._to_builder(_evaluation.EmptyFunction(self.node.context, self.node, []))

The FHIRPath empty() function.

Returns:
        An expression that evaluates to True if the parent evaluates to empty.


def as_default(self) -> ContextManager['Graph']:
    return _default_graph_stack.get_controller(self)

Returns a context manager that makes this `Graph` the default graph.

    This method should be used if you want to create multiple graphs
    in the same process. For convenience, a global default graph is
    provided, and all ops will be added to this graph if you do not
    create a new graph explicitly.

    Use this method with the `with` keyword to specify that ops created within
    the scope of a block should be added to this graph. In this case, once
    the scope of the `with` is exited, the previous default graph is set again
    as default. There is a stack, so it's ok to have multiple nested levels
    of `as_default` calls.

    The default graph is a property of the current thread. If you
    create a new thread, and wish to use the default graph in that
    thread, you must explicitly add a `with g.as_default():` in that
    thread's function.

    The following code examples are equivalent:

    ```python
    # 1. Using Graph.as_default():
    g = tf.Graph()
    with g.as_default():
        c = tf.constant(5.0)
        assert c.graph is g

    # 2. Constructing and making default:
    with tf.Graph().as_default() as g:
        c = tf.constant(5.0)
        assert c.graph is g
    ```

    If eager execution is enabled ops created under this context manager will be
    added to the graph instead of executed eagerly.

Returns:
        A context manager for using this graph as the default graph.


def experimental_tpu_fit_loop(model, dataset, epochs=100, verbose=1, callbacks=None, initial_epoch=0, steps_per_epoch=None, val_dataset=None, validation_steps=None, validation_freq=1):
    mode = ModeKeys.TRAIN
    current_strategy = model._distribution_strategy
    iteration_value = min(steps_per_epoch, current_strategy.extended.steps_per_run)
    steps_per_run = backend.variable(value=iteration_value, dtype='int32', name='steps_per_run')
    iterator = dist_utils.get_iterator(dataset, current_strategy)
    scope = dist_utils.distributed_scope(strategy=current_strategy, learning_phase=1)
    scope.__enter__()
    out_labels = model.metrics_names or []
    step_fn = _make_train_step_fn(model, ModeKeys.TRAIN, current_strategy, out_labels)
    initial_loop_values = {}
    initial_loop_values['loss'] = constant_op.constant(10000000.0)
    for m in model._get_training_eval_metrics():
        tensor = m.result()
        initial_loop_values[m.name] = array_ops.zeros(tensor.shape, tensor.dtype)
    ctx = current_strategy.extended.experimental_run_steps_on_iterator(step_fn, iterator, iterations=steps_per_run, initial_loop_values=initial_loop_values)
    train_op = ctx.run_op
    output_tensors = ctx.last_step_outputs
    do_validation = bool(validation_steps)
    if model._compile_distribution:
        dist_utils._copy_weights_to_distributed_model(model, mode)
    callbacks = cbks.configure_callbacks(callbacks, model, do_validation=do_validation, epochs=epochs, steps_per_epoch=steps_per_epoch, verbose=verbose, count_mode='steps', mode=mode)
    steps_to_run = [current_strategy.extended.steps_per_run] * (steps_per_epoch // current_strategy.extended.steps_per_run)
    if steps_per_epoch % current_strategy.extended.steps_per_run:
        steps_to_run.append(steps_per_epoch % current_strategy.extended.steps_per_run)
    target_steps = len(steps_to_run)
    callbacks._call_begin_hook(mode)
    initial_epoch = model._maybe_load_initial_epoch_from_ckpt(initial_epoch, mode)
    for epoch in range(initial_epoch, epochs):
        dist_utils._reset_metrics(model)
        callbacks.on_epoch_begin(epoch)
        epoch_logs = {}
        step_index = 0
        prev_step_count = None
        current_step = 0
        while current_step < target_steps:
            step_count = steps_to_run[current_step]
            batch_logs = {'batch': step_index, 'size': 1, 'num_steps': step_count}
            callbacks._call_batch_hook(mode, 'begin', step_index, batch_logs)
            if prev_step_count is None or step_count != prev_step_count:
                backend.get_session().run(steps_per_run.assign(step_count))
                prev_step_count = step_count
            try:
                _, outputs = backend.batch_get_value([train_op, output_tensors])
            except errors.OutOfRangeError:
                logging.warning('Your dataset iterator ran out of data; interrupting training. Make sure that your dataset can generate at least `steps_per_epoch * epochs` batches (in this case, %d batches).' % steps_per_epoch * epochs)
                break
            batch_logs.update(outputs)
            callbacks._call_batch_hook(mode, 'end', step_index, batch_logs)
            step_index = step_index + step_count
            current_step += 1
            if callbacks.model.stop_training:
                break
        if do_validation and training_utils_v1.should_run_validation(validation_freq, epoch):
            logging.info('Running validation at fit epoch: %s', epoch)
            if model._compile_distribution:
                dist_utils._copy_weights_to_original_model(model, ModeKeys.TRAIN)
            val_outs = experimental_tpu_test_loop(model, val_dataset, steps=validation_steps, verbose=verbose, callbacks=callbacks)
            if not isinstance(val_outs, list):
                val_outs = [val_outs]
            for label, val_out in zip(out_labels, val_outs):
                epoch_logs['val_' + label] = val_out
        callbacks.on_epoch_end(epoch, epoch_logs)
        if callbacks.model.stop_training:
            break
    model._successful_loop_finish = True
    callbacks._call_end_hook(mode)
    if model._compile_distribution:
        dist_utils._copy_weights_to_original_model(model, ModeKeys.TRAIN)
    scope.__exit__(None, None, None)
    return model.history

Fit loop for training with TPU tf.distribute.Strategy.

Args:
        model: Keras Model instance.
        dataset: Dataset that returns inputs and targets
        epochs: Number of times to iterate over the data
        verbose: Integer, Verbosity mode, 0, 1 or 2
        callbacks: List of callbacks to be called during training
        initial_epoch: Epoch at which to start training
            (useful for resuming a previous training run)
        steps_per_epoch: Total number of steps (batches of samples)
            before declaring one epoch finished and starting the
            next epoch. Ignored with the default value of `None`.
        val_dataset: Dataset for validation data.
        validation_steps: Number of steps to run validation for
            (only if doing validation from data tensors).
            Ignored with the default value of `None`.
        validation_freq: Only relevant if validation data is provided. Integer or
            `collections.abc.Container` instance (e.g. list, tuple, etc.). If an
            integer, specifies how many training epochs to run before a new
            validation run is performed, e.g. `validation_freq=2` runs
            validation every 2 epochs. If a Container, specifies the epochs on
            which to run validation, e.g. `validation_freq=[1, 2, 10]` runs
            validation at the end of the 1st, 2nd, and 10th epochs.

Returns:
        Returns `None`.

Raises:
        ValueError: in case of invalid arguments.


def fn_args(fn):
    if isinstance(fn, functools.partial):
        args = fn_args(fn.func)
        args = [a for a in args[len(fn.args):] if a not in (fn.keywords or [])]
    else:
        if _is_callable_object(fn):
            fn = fn.__call__
        args = tf_inspect.getfullargspec(fn).args
        if _is_bound_method(fn) and args:
            args.pop(0)
    return tuple(args)

Get argument names for function-like object.

Args:
        fn: Function, or function-like object (e.g., result of `functools.partial`).

Returns:
        `tuple` of string argument names.

Raises:
        ValueError: if partial function has positionally bound arguments


def GetBullets(self):
    return self._bullets

Returns the bullet characters list.

    Use the list elements in order for best appearance in nested bullet lists,
    wrapping back to the first element for deep nesting. The list size depends
    on the console implementation.

Returns:
        A tuple of bullet characters.


def collect(self):
    if self.is_external:
        raise ValueError(f'`collect` cannot be called on a dynamic evaluation context that is using an external DNASpec: {self._dna_spec}.')
    _dynamic_evaluation_stack.ensure_thread_safety(self)
    self._hyper_dict = {}
    with dynamic_evaluate(self.add_decision_point, per_thread=self._per_thread):
        try:
            _dynamic_evaluation_stack.push(self)
            yield self._hyper_dict
        finally:
            self._dna_spec = None
            _dynamic_evaluation_stack.pop(self)

A context manager for collecting hyper primitives within this context.

    Example::

        context = DynamicEvaluationContext()
        with context.collect():
        x = pg.oneof([1, 2, 3]) + pg.oneof([4, 5, 6])

        # Will print 1 + 4 = 5. Meanwhile 2 hyper primitives will be registered
        # in the search space represented by the context.
        print(x)

Yields:
        The hyper dict representing the search space.


def list_summaries(logdir):
    result = _SummaryFile()
    for dirpath, _, filenames in os.walk(logdir):
        for filename in filenames:
            if not filename.startswith('events.out.'):
                continue
            path = os.path.join(dirpath, filename)
            for event in _SummaryIterator(path):
                if event.graph_def:
                    result.graph_defs.append(event.graph_def)
                if not event.summary:
                    continue
                for value in event.summary.value:
                    tag = value.tag
                    kind = value.WhichOneof('value')
                    container = {'simple_value': result.scalars, 'image': result.images, 'histo': result.histograms, 'tensor': result.tensors}.get(kind)
                    if container is None:
                        raise ValueError('Unexpected summary kind %r in event file %s:\n%r' % (kind, path, event))
                    elif kind == 'tensor' and tag != 'keras':
                        plugin_name = value.metadata.plugin_data.plugin_name
                        container = {'images': result.images, 'histograms': result.histograms, 'scalars': result.scalars}.get(plugin_name)
                        if container is not None:
                            result.convert_from_v2_summary_proto = True
                        else:
                            container = result.tensors
                    container.add(_ObservedSummary(logdir=dirpath, tag=tag))
    return result

Read all summaries under the logdir into a `_SummaryFile`.

Args:
        logdir: A path to a directory that contains zero or more event
        files, either as direct children or in transitive subdirectories.
        Summaries in these events must only contain old-style scalars,
        images, and histograms. Non-summary events, like `graph_def`s, are
        ignored.

Returns:
        A `_SummaryFile` object reflecting all summaries written to any
        event files in the logdir or any of its descendant directories.

Raises:
        ValueError: If an event file contains an summary of unexpected kind.


def get_image_features(self, pixel_values: Optional[torch.FloatTensor]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, return_dict: Optional[bool]=None, interpolate_pos_encoding: bool=False):
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
    vision_outputs = self.vision_model(pixel_values=pixel_values, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, interpolate_pos_encoding=interpolate_pos_encoding)
    return vision_outputs

Returns:
    vision_outputs (`BaseModelOutputWithPooling` or tuple of `torch.FloatTensor`):
        The vision model outputs. If `return_dict=True`, the output is a [`BaseModelOutputWithPooling`] that
        contains the image features, the pooled image features and the hidden states if
        `output_hidden_states=True`.

Example:
    ```python
    >>> import torch
    >>> from PIL import Image
    >>> import requests
    >>> from transformers import AutoProcessor, Blip2Model

    >>> model = Blip2Model.from_pretrained("Salesforce/blip2-opt-2.7b")

    >>> processor = AutoProcessor.from_pretrained("Salesforce/blip2-opt-2.7b")
    >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
    >>> image = Image.open(requests.get(url, stream=True).raw)
    >>> inputs = processor(images=image, return_tensors="pt")
    >>> image_outputs = model.get_image_features(**inputs)
    ```


def _parse_line(self, instrumentation_block, line):
    if instrumentation_block.state == _InstrumentationBlockStates.METHOD:
        return self._parse_method_block_line(instrumentation_block, line)
    elif instrumentation_block.state == _InstrumentationBlockStates.RESULT:
        return self._parse_result_block_line(instrumentation_block, line)
    else:
        return self._parse_unknown_block_line(instrumentation_block, line)

Parses an arbitrary line from the instrumentation output based upon
    the current parser state.

Args:
        instrumentation_block: _InstrumentationBlock, an instrumentation
        block with any of the possible parser states.
        line: string, the raw instrumentation output line to parse
        appropriately.

Returns:
        The next instrumenation block to continue parsing with.


def stripped_op_list_for_graph(graph_def):
    used_ops = ops_used_by_graph_def(graph_def)
    op_defs = []
    for op in sorted(used_ops):
        op_def = op_def_registry.get(op)
        if op_def is not None:
            op_defs.append(op_def)
    return op_def_pb2.OpList(op=op_defs)

Collect the stripped OpDefs for ops used by a graph.

    This function computes the `stripped_op_list` field of `MetaGraphDef` and
    similar protos.  The result can be communicated from the producer to the
    consumer, which can then use the C++ function
    `RemoveNewDefaultAttrsFromGraphDef` to improve forwards compatibility.

Args:
        graph_def: A `GraphDef` proto, as from `graph.as_graph_def()`.

Returns:
        An `OpList` of ops used by the graph.


def _setup_rotation_strategy(self, creative_rotation, feed_item):
    option = feed_item.get(FieldMap.CREATIVE_ROTATION, 'Even').upper()
    if option == 'EVEN':
        creative_rotation['type'] = 'CREATIVE_ROTATION_TYPE_RANDOM'
        creative_rotation['weightCalculationStrategy'] = 'WEIGHT_STRATEGY_EQUAL'
    elif option == 'SEQUENTIAL':
        creative_rotation['type'] = 'CREATIVE_ROTATION_TYPE_SEQUENTIAL'
        creative_rotation['weightCalculationStrategy'] = None
    elif option == 'CUSTOM':
        creative_rotation['type'] = 'CREATIVE_ROTATION_TYPE_RANDOM'
        creative_rotation['weightCalculationStrategy'] = 'WEIGHT_STRATEGY_CUSTOM'
    elif option == 'CLICK-THROUGH RATE':
        creative_rotation['type'] = 'CREATIVE_ROTATION_TYPE_RANDOM'
        creative_rotation['weightCalculationStrategy'] = 'WEIGHT_STRATEGY_HIGHEST_CTR'
    elif option == 'OPTIMIZED':
        creative_rotation['type'] = 'CREATIVE_ROTATION_TYPE_RANDOM'
        creative_rotation['weightCalculationStrategy'] = 'WEIGHT_STRATEGY_OPTIMIZED'

Analyzes the feed and sets up rotation strategy for the ad.

    For better user experience, the creative rotaion values that come from the
    feed map directly to values in the UI, this function is responsible for
    translating that to API specific values for the creative rotation objects
    under the ad.

Args:
        creative_rotation: Feed item representing the creative rotation setup from
        the feed.
        feed_item: Feed item representing the ad.


def _convert_type(self, t, as_instance=False):
    src = textwrap.dedent(f'\n      from typing import Any, Callable, Iterator, Tuple, Type, Union\n      from protocols import Sequence, SupportsLower\n      x = ...  # type: {t}\n    ')
    filename = str(hash((t, as_instance)))
    x = self._parse_and_lookup(src, 'x', filename).type
    if as_instance:
        x = abstract_utils.AsInstance(x)
    return self.ctx.convert.constant_to_value(x, {}, self.ctx.root_node)

Convenience function for turning a string into an abstract value.

    Note that this function cannot be called more than once per test with
    the same arguments, since we hash the arguments to get a filename for
    the temporary pyi.

Args:
        t: The string representation of a type.
        as_instance: Whether to convert as an instance.

Returns:
        A BaseValue.


def __init__(self, thresholds=None, name=None, dtype=None):
    super().__init__(confusion_matrix_cond=metrics_utils.ConfusionMatrix.FALSE_NEGATIVES, thresholds=thresholds, name=name, dtype=dtype)

Calculates the number of false negatives.

    If `sample_weight` is given, calculates the sum of the weights of
    false negatives. This metric creates one local variable, `accumulator`
    that is used to keep track of the number of false negatives.

    If `sample_weight` is `None`, weights default to 1.
    Use `sample_weight` of 0 to mask values.

Args:
        thresholds: (Optional) Defaults to `0.5`. A float value, or a Python
            list/tuple of float threshold values in `[0, 1]`. A threshold is
            compared with prediction values to determine the truth value of
            predictions (i.e., above the threshold is `True`, below is `False`).
            If used with a loss function that sets `from_logits=True` (i.e. no
            sigmoid applied to predictions), `thresholds` should be set to 0.
            One metric value is generated for each threshold value.
        name: (Optional) string name of the metric instance.
        dtype: (Optional) data type of the metric result.

Example:
    >>> m = keras.metrics.FalseNegatives()
    >>> m.update_state([0, 1, 1, 1], [0, 1, 0, 0])
    >>> m.result()
    2.0

    >>> m.reset_state()
    >>> m.update_state([0, 1, 1, 1], [0, 1, 0, 0], sample_weight=[0, 0, 1, 0])
    >>> m.result()
    1.0


def _from_definition(fdef, grad_func=None):
    func = None
    argnames = [arg.name for arg in fdef.signature.input_arg]
    input_types = tuple((dtypes.as_dtype(arg.type) for arg in fdef.signature.input_arg))
    func_name = fdef.signature.name
    python_grad_func = None
    out_names = [arg.name for arg in fdef.signature.output_arg]
    result = _DefinedFunction(func, argnames, input_types, func_name, grad_func, python_grad_func, out_names)
    if is_oss:
        serialized = fdef.SerializeToString()
        c_func = c_api.TF_FunctionImportFunctionDef(serialized)
    else:
        c_func = c_api.TF_FunctionImportFunctionDefNoSerialization(fdef)
    result._c_func = c_api_util.ScopedTFFunction(c_func, func_name)
    result._extra_inputs = []
    result._op_def = fdef.signature
    return result

Creates a _DefinedFunction initialized from a FunctionDef proto.

Args:
        fdef: a FunctionDef
        grad_func: a _DefinedFunction or None

Returns:
        A _DefinedFunction representing fdef


def get_model_files(model_type: str, frameworks: Optional[List[str]]=None) -> Dict[str, Union[Path, List[Path]]]:
    module_name = model_type_to_module_name(model_type)
    model_module = TRANSFORMERS_PATH / 'models' / module_name
    model_files = list(model_module.glob('*.py'))
    model_files = filter_framework_files(model_files, frameworks=frameworks)
    doc_file = REPO_PATH / 'docs' / 'source' / 'en' / 'model_doc' / f'{model_type}.md'
    test_files = [f'test_modeling_{module_name}.py', f'test_modeling_tf_{module_name}.py', f'test_modeling_flax_{module_name}.py', f'test_tokenization_{module_name}.py', f'test_image_processing_{module_name}.py', f'test_feature_extraction_{module_name}.py', f'test_processor_{module_name}.py']
    test_files = filter_framework_files(test_files, frameworks=frameworks)
    test_files = [REPO_PATH / 'tests' / 'models' / module_name / f for f in test_files]
    test_files = [f for f in test_files if f.exists()]
    return {'doc_file': doc_file, 'model_files': model_files, 'module_name': module_name, 'test_files': test_files}

Retrieves all the files associated to a model.

Args:
        model_type (`str`): A valid model type (like "bert" or "gpt2")
        frameworks (`List[str]`, *optional*):
            If passed, will only keep the model files corresponding to the passed frameworks.

Returns:
        `Dict[str, Union[Path, List[Path]]]`: A dictionary with the following keys:
        - **doc_file** -- The documentation file for the model.
        - **model_files** -- All the files in the model module.
        - **test_files** -- The test files for the model.


def _has_kwargs(fn):
    if isinstance(fn, functools.partial):
        fn = fn.func
    elif _is_callable_object(fn):
        fn = fn.__call__
    elif not callable(fn):
        raise TypeError('fn should be a function-like object, but is of type {}.'.format(type(fn)))
    return tf_inspect.getfullargspec(fn).varkw is not None

Returns whether the passed callable has **kwargs in its signature.

Args:
        fn: Function, or function-like object (e.g., result of `functools.partial`).

Returns:
        `bool`: if `fn` has **kwargs in its signature.

Raises:
        `TypeError`: If fn is not a Function, or function-like object.


def call(self, inputs: List[Any], global_state: pg.geno.AttributeDict, step: int=0) -> List[Any]:
    raise NotImplementedError()

Subclasses should override this method.

    The `global_state` and `step` are optional for the subclasses' call
    signature.

Args:
        inputs: A list of values as inputs.
        global_state: An `AttributeDict` object as the global state container,
        which is readable/writable during the operation.
        step: Number of examples historically proposed, which can be used for
        determining a cross over schedule.

Returns:
        A list of values as output of current operation.


def read(self, size):
    data_list = []
    bytes_read = 0
    last_block_position = self.position
    while bytes_read < size:
        bytes_from_remaining = min(size - bytes_read, len(self.remaining))
        data_list.append(self.remaining[0:bytes_from_remaining])
        self.remaining = self.remaining[bytes_from_remaining:]
        self.position += bytes_from_remaining
        bytes_read += bytes_from_remaining
        if not self.remaining:
            try:
                self.remaining = self.conn.recv_bytes()
            except EOFError:
                break
    last_block = b''.join(data_list)
    if last_block:
        self.last_block_position = last_block_position
        self.last_block = last_block
    return last_block

Read data from the wrapped pipe connection.

Args:
        size: Number of bytes to read. Actual number of bytes read is always
            equal to size unless EOF is reached.

Returns:
        data read as str.


def run_ninja(targets, fail_collector=None, fail_fast=False, verbose=False):
    cmd = ['ninja', '-k', '1' if fail_fast else '100000'] + targets
    with subprocess.Popen(cmd, cwd=OUT_DIR, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) as process:
        failed_targets = []
        print_if_verbose = False
        with open(NINJA_LOG, 'w') as ninja_log:
            while True:
                line = process.stdout.readline()
                if not line:
                    break
                ninja_log.write(line)
                msg_type, modname, logfile = parse_ninja_output_line(line)
                if msg_type == _NINJA_FAILURE_MSG:
                    failed_targets.append(line[len(NINJA_FAILURE_PREFIX):].strip())
                    print_if_verbose = True
                if msg_type == _TEST_MODULE_PASS_MSG or msg_type == _TEST_MODULE_FAIL_MSG:
                    print(line)
                    if msg_type == _TEST_MODULE_FAIL_MSG:
                        fail_collector.add_failure(modname, logfile)
                    print_if_verbose = False
                if verbose and print_if_verbose:
                    print(line.rstrip())
            if failed_targets:
                summary_hdr = '>>> Found Ninja target failures (includes test failures):'
                print('\n' + summary_hdr)
                ninja_log.write('\n' + summary_hdr + '\n')
                for t in failed_targets:
                    target = f'    - {t}'
                    print(target)
                    ninja_log.write(target + '\n')
        process.wait()
        if process.returncode == 0:
            return True
        else:
            print(f">>> FAILED: Ninja command '{shlex.join(cmd)}'.")
            print(">>>         Run it in the 'out' directory to reproduce.")
            print(f">>>         Full Ninja output is available in '{NINJA_LOG}'.")
            print('>>>         Failing test modules (if any) will be reported below.')
            return False

Run ninja over the list of specified targets.

Args:
        targets: The list of targets to run.
        fail_collector: A FailCollector object to collect failures.
        fail_fast: If True, abort at the first target failure.
        verbose: If True, print verbose output.

Returns:
        True if no target fails. False, otherwise.


def forward(self, input_ids: Optional[torch.LongTensor]=None, attention_mask: Optional[torch.Tensor]=None, position_ids: Optional[torch.LongTensor]=None, past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]]=None, inputs_embeds: Optional[torch.FloatTensor]=None, labels: Optional[torch.LongTensor]=None, use_cache: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, cache_position: Optional[torch.LongTensor]=None, logits_to_keep: Union[int, torch.Tensor]=0, **loss_kwargs) -> CausalLMOutputWithPast:
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    outputs: BaseModelOutputWithPast = self.model(input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, cache_position=cache_position)
    hidden_states = outputs.last_hidden_state
    slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
    logits = self.lm_head(hidden_states[:, slice_indices, :])
    loss = None
    if labels is not None:
        loss = self.loss_function(logits, labels, self.vocab_size, **loss_kwargs)
    return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions)

labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:
    ```python
    >>> from transformers import AutoTokenizer, NemotronForCausalLM

    >>> model = NemotronForCausalLM.from_pretrained("nvidia/nemotron-3-8b-base-4k-hf")
    >>> tokenizer = AutoTokenizer.from_pretrained("nvidia/nemotron-3-8b-base-4k-hf")

    >>> prompt = "Hey, are you conscious? Can you talk to me?"
    >>> inputs = tokenizer(prompt, return_tensors="pt")

    >>> # Generate
    >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
    >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
    "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
    ```


def get_models_in_diff():
    fork_point_sha = subprocess.check_output('git merge-base main HEAD'.split()).decode('utf-8')
    modified_files = subprocess.check_output(f'git diff --diff-filter=d --name-only {fork_point_sha}'.split()).decode('utf-8').split()
    relevant_modified_files = [x for x in modified_files if '/models/' in x and x.endswith('.py')]
    model_names = set()
    for file_path in relevant_modified_files:
        model_name = file_path.split('/')[-2]
        model_names.add(model_name)
    return model_names

Finds all models that have been modified in the diff.

Returns:
        A set containing the names of the models that have been modified (e.g. {'llama', 'whisper'}).


def __init__(self, config: RoFormerConfig):
    super().__init__()
    self.summary_type = getattr(config, 'summary_type', 'last')
    if self.summary_type == 'attn':
        raise NotImplementedError
    self.summary = nn.Identity()
    if hasattr(config, 'summary_use_proj') and config.summary_use_proj:
        if hasattr(config, 'summary_proj_to_labels') and config.summary_proj_to_labels and (config.num_labels > 0):
            num_classes = config.num_labels
        else:
            num_classes = config.hidden_size
        self.summary = nn.Linear(config.hidden_size, num_classes)
    activation_string = getattr(config, 'summary_activation', None)
    self.activation: Callable = get_activation(activation_string) if activation_string else nn.Identity()
    self.first_dropout = nn.Identity()
    if hasattr(config, 'summary_first_dropout') and config.summary_first_dropout > 0:
        self.first_dropout = nn.Dropout(config.summary_first_dropout)
    self.last_dropout = nn.Identity()
    if hasattr(config, 'summary_last_dropout') and config.summary_last_dropout > 0:
        self.last_dropout = nn.Dropout(config.summary_last_dropout)

Compute a single vector summary of a sequence hidden states.

Args:
        config ([`RoFormerConfig`]):
            The config used by the model. Relevant arguments in the config class of the model are (refer to the actual
            config class of your model for the default values it uses):

            - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are:

                - `"last"` -- Take the last token hidden state (like XLNet)
                - `"first"` -- Take the first token hidden state (like Bert)
                - `"mean"` -- Take the mean of all tokens hidden states
                - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2)
                - `"attn"` -- Not implemented now, use multi-head attention

            - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
            - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
            (otherwise to `config.hidden_size`).
            - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
            another string or `None` will add no activation.
            - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
            - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.


def __init__(self, dataset: str, level: int=logging.INFO) -> None:
    """Clients are initialized with the general CLI configuration."""
    self.dataset = dataset
    self.logger = logging.getLogger(f'{__name__}.{type(self).__name__}')
    self.logger.setLevel(level)

Weather data provider client interface.

    Defines methods and properties required to efficiently interact with weather
    data providers.

Attributes:
        config: A config that contains pipeline parameters, such as API keys.
        level: Default log level for the client.


def expand_dims(x, axis=-1):
    return array_ops.expand_dims(x, axis)

Adds a 1-sized dimension at index "axis".

Args:
        x: A tensor or variable.
        axis: Position where to add a new axis.

Returns:
        A tensor with expanded dimensions.


def AsyncPopenArgs(self):
    args = {}
    if self.operating_system == OperatingSystem.WINDOWS:
        args['close_fds'] = True
        detached_process = 8
        create_new_process_group = 512
        args['creationflags'] = detached_process | create_new_process_group
    else:
        args['preexec_fn'] = os.setsid
        args['close_fds'] = True
        args['stdin'] = subprocess.PIPE
        args['stdout'] = subprocess.PIPE
        args['stderr'] = subprocess.PIPE
    return args

Returns the args for spawning an async process using Popen on this OS.

    Make sure the main process does not wait for the new process. On windows
    this means setting the 0x8 creation flag to detach the process.

    Killing a group leader kills the whole group. Setting creation flag 0x200 on
    Windows or running setsid on *nix makes sure the new process is in a new
    session with the new process the group leader. This means it can't be killed
    if the parent is killed.

    Finally, all file descriptors (FD) need to be closed so that waiting for the
    output of the main process does not inadvertently wait for the output of the
    new process, which means waiting for the termination of the new process.
    If the new process wants to write to a file, it can open new FDs.

Returns:
        {str:}, The args for spawning an async process using Popen on this OS.


def __eq__(self, other):
    return isinstance(other, ArgumentPlaceholder)

Tests for equality of two placeholder objects.

Args:
        other: Another placeholder object to compare to.

    This method is used only for test code. All placeholder objects are
    equal to each other.


def forward(self, pixel_values: Optional[torch.Tensor]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, labels: Optional[torch.Tensor]=None, return_dict: Optional[bool]=None) -> Union[tuple, SemanticSegmenterOutput]:
    if labels is not None and self.config.num_labels == 1:
        raise ValueError('The number of labels should be greater than one')
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
    output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    outputs = self.backbone.forward_with_filtered_kwargs(pixel_values, output_hidden_states=output_hidden_states, output_attentions=output_attentions)
    features = outputs.feature_maps
    logits = self.decode_head(features)
    logits = nn.functional.interpolate(logits, size=pixel_values.shape[2:], mode='bilinear', align_corners=False)
    auxiliary_logits = None
    if self.auxiliary_head is not None:
        auxiliary_logits = self.auxiliary_head(features)
        auxiliary_logits = nn.functional.interpolate(auxiliary_logits, size=pixel_values.shape[2:], mode='bilinear', align_corners=False)
    loss = None
    if labels is not None:
        loss_fct = CrossEntropyLoss(ignore_index=self.config.loss_ignore_index)
        loss = loss_fct(logits, labels)
        if auxiliary_logits is not None:
            auxiliary_loss = loss_fct(auxiliary_logits, labels)
            loss += self.config.auxiliary_loss_weight * auxiliary_loss
    if not return_dict:
        if output_hidden_states:
            output = (logits,) + outputs[1:]
        else:
            output = (logits,) + outputs[2:]
        return (loss,) + output if loss is not None else output
    return SemanticSegmenterOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states, attentions=outputs.attentions)

labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):
    Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy).

Example:
    ```python
    >>> from transformers import AutoImageProcessor, UperNetForSemanticSegmentation
    >>> from PIL import Image
    >>> from huggingface_hub import hf_hub_download

    >>> image_processor = AutoImageProcessor.from_pretrained("openmmlab/upernet-convnext-tiny")
    >>> model = UperNetForSemanticSegmentation.from_pretrained("openmmlab/upernet-convnext-tiny")

    >>> filepath = hf_hub_download(
    ...     repo_id="hf-internal-testing/fixtures_ade20k", filename="ADE_val_00000001.jpg", repo_type="dataset"
    ... )
    >>> image = Image.open(filepath).convert("RGB")

    >>> inputs = image_processor(images=image, return_tensors="pt")

    >>> outputs = model(**inputs)

    >>> logits = outputs.logits  # shape (batch_size, num_labels, height, width)
    >>> list(logits.shape)
    [1, 150, 512, 512]
    ```


def _benchmarkFetch(self, name, target, size, iters):
    times = []
    with ops.Graph().as_default():
        v = variables.Variable(random_ops.random_normal([size]))
        with session.Session(target) as sess:
            sess.run(v.initializer)
            sess.run(v)
            for _ in range(iters):
                start_time = time.time()
                sess.run(v)
                end_time = time.time()
                times.append(end_time - start_time)
    print('%s %d %f' % (name, size, np.median(times)))
    self.report_benchmark(iters=1, wall_time=np.median(times), name=name)

Runs a microbenchmark to measure the cost of fetching a tensor.

    Reports the median cost of fetching a tensor of `size` * `sizeof(float)`
    bytes.

Args:
        name: A human-readable name for logging the output.
        target: The session target to use for the benchmark.
        size: The number of floating-point numbers to be fetched.
        iters: The number of iterations to perform.


def __init__(self, argv_or_options, command_line=False):
    argument_parser = make_parser()
    if command_line:
        assert isinstance(argv_or_options, list)
        options = argument_parser.parse_args(argv_or_options)
    else:
        if isinstance(argv_or_options, list):
            raise TypeError('Do not construct an Options object directly; call Options.create() instead.')
        options = argv_or_options
    for name, default in _LIBRARY_ONLY_OPTIONS.items():
        if not hasattr(options, name):
            setattr(options, name, default)
    names = set(vars(options))
    opt_map = {k: v.option_strings[-1] for k, v in argument_parser.actions.items() if v.option_strings}
    try:
        Postprocessor(names, opt_map, options, self).process()
    except PostprocessingError as e:
        if command_line:
            argument_parser.error(str(e))
        else:
            raise

Parse and encapsulate the configuration options.

    Also sets up some basic logger configuration.

    IMPORTANT: If creating an Options object from code, do not construct it
    directly! Call Options.create() instead.

Args:
        argv_or_options: Either sys.argv[1:] (sys.argv[0] is the main script), or
        already parsed options object returned by ArgumentParser.parse_args.
        command_line: Set this to true when argv_or_options == sys.argv[1:].

Raises:
        sys.exit(2): bad option or input filenames.


def _on_fail(self, record):
    self.on_fail(record)

Proxy function to guarantee the base implementation of on_fail is
    called.

Args:
        record: records.TestResultRecord, a copy of the test record for
        this test, containing all information of the test execution
        including exception objects.


def _get_test_methods(self, test_names):
    test_methods = []
    for test_name in test_names:
        if test_name.startswith(TEST_SELECTOR_REGEX_PREFIX):
            regex_matching_methods = self._get_regex_matching_test_methods(test_name.removeprefix(TEST_SELECTOR_REGEX_PREFIX))
            test_methods += regex_matching_methods
            continue
        self._assert_valid_test_name(test_name)
        if test_name not in self.get_existing_test_names():
            raise Error(f'{self.TAG} does not have test method {test_name}.')
        if hasattr(self, test_name):
            test_method = getattr(self, test_name)
        elif test_name in self._generated_test_table:
            test_method = self._generated_test_table[test_name]
        test_methods.append((test_name, test_method))
    return test_methods

Resolves test method names to bound test methods.

Args:
        test_names: A list of strings, each string is a test method name or a
        regex for matching test names.

Returns:
        A list of tuples of (string, function). String is the test method
        name, function is the actual python method implementing its logic.

Raises:
        Error: The test name does not follow naming convention 'test_*'.
        This can only be caused by user input.


def resize(self, image: np.ndarray, size: Dict[str, int], resample: PILImageResampling=PILImageResampling.BILINEAR, data_format: Optional[Union[str, ChannelDimension]]=None, input_data_format: Optional[Union[str, ChannelDimension]]=None, **kwargs) -> np.ndarray:
    image_height, image_width = get_image_size(image, input_data_format)
    target_height, target_width = (size['height'], size['width'])
    if image_width <= target_width and image_height <= target_height:
        return image
    height_scale_factor = target_height / image_height
    width_scale_factor = target_width / image_width
    optimal_scale_factor = min(height_scale_factor, width_scale_factor)
    new_height = int(image_height * optimal_scale_factor)
    new_width = int(image_width * optimal_scale_factor)
    scaled_image = resize(image=image, size=(new_height, new_width), resample=resample, data_format=data_format, input_data_format=input_data_format, **kwargs)
    return scaled_image

Resize an image to `(size["height"], size["width"])`.

Args:
        image (`np.ndarray`):
            Image to resize.
        size (`Dict[str, int]`):
            Dictionary in the format `{"height": int, "width": int}` specifying the size of the output image.
        resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BILINEAR`):
            `PILImageResampling` filter to use when resizing the image e.g. `PILImageResampling.BILINEAR`.
        data_format (`ChannelDimension` or `str`, *optional*):
            The channel dimension format for the output image. If unset, the channel dimension format of the input
            image is used. Can be one of:
            - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
            - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
            - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
        input_data_format (`ChannelDimension` or `str`, *optional*):
            The channel dimension format for the input image. If unset, the channel dimension format is inferred
            from the input image. Can be one of:
            - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
            - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
            - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.

Returns:
        `np.ndarray`: The resized image.


def response_utf8_stream(response, chunksize):
    leftovers = b''
    while True:
        chunk = leftovers + response.read(chunksize)
        position = find_utf8_split(chunk)
        if position < len(chunk):
            leftovers = chunk[position:]
            chunk = chunk[:position]
        else:
            leftovers = b''
        if chunk:
            yield chunk.decode('UTF-8')
        else:
            break

Re-aligns a streaming buffer with UTF-8 boundraries.

    Buffers incomplete utf-8 characters to next chunk. Chunks are always returned as <= chunksize.
    In the future may add bytes as input type in addition to io using type detect.

    Parameters:
        reponse (io) - buffer containing bytes data.

Returns:
        (bytes generator) - the reponse buffer in chunksize aligned to utf-8 boundries.


def max(self):
    if self.is_quantized or self.base_dtype in (bool, string, complex64, complex128):
        raise TypeError(f'Cannot find maximum value of {self} with {('quantized type' if self.is_quantized else 'type')} {self.base_dtype}.')
    try:
        return ml_dtypes.finfo(self.as_numpy_dtype).max
    except:
        try:
            return ml_dtypes.iinfo(self.as_numpy_dtype).max
        except:
            raise TypeError(f'Cannot find maximum value of {self}.')

Returns the maximum representable value in this data type.

Raises:
        TypeError: if this is a non-numeric, unordered, or quantized type.


def serialize_state(self, name=None):
    if self._reader_ref.dtype == dtypes.resource:
        return gen_io_ops.reader_serialize_state_v2(self._reader_ref, name=name)
    else:
        return gen_io_ops.reader_serialize_state(self._reader_ref, name=name)

Produce a string tensor that encodes the state of a reader.

    Not all Readers support being serialized, so this can produce an
    Unimplemented error.

Args:
        name: A name for the operation (optional).

Returns:
        A string Tensor.


def _flatten_and_filter_composite(maybe_composite, non_composite_output, composite_output=None):
    if isinstance(maybe_composite, composite_tensor.CompositeTensor):
        num_components = len(nest.flatten(maybe_composite, expand_composites=True))
        return (composite_output,) * num_components
    return non_composite_output

For an input, replaced the input by a tuple if the input is composite.

    If `maybe_composite` is not composite, return the parameter
    `non_composite_output` otherwise return a tuple which consists of the value of
    the parameter `composite_output` the same number of times as there are
    components of the composite tensor.

    This is useful for computing a mask when flattening nested data with
    `expand_composites=True`. For example

    ```python
    nest.flatten(data, expand_composites=True)
    ```

    and

    ```python
    nest.flatten(nest.map(
        data, lambda x: _flatten_and_filter_composite(x, False, True)))
    ```

    will have the same length and second will be True if the tensor in the first
    is derived from a expanding a composite tensor.

Args:
        maybe_composite: A value to test for being a composite tensor.
        non_composite_output: The value to return when `maybe_composite` is not a
        composite.
        composite_output: the value to fill the output tuple with if
        `maybe_composite` is a composite.

Returns:
        `non_composite_output` or a tuple with multiple copies of
        `composite_output`.


def run_task_tests(self, task, torch_dtype='float32'):
    if task not in self.pipeline_model_mapping:
        self.skipTest(f'{self.__class__.__name__}::test_pipeline_{task.replace('-', '_')}_{torch_dtype} is skipped: `{task}` is not in `self.pipeline_model_mapping` for `{self.__class__.__name__}`.')
    model_architectures = self.pipeline_model_mapping[task]
    if not isinstance(model_architectures, tuple):
        model_architectures = (model_architectures,)
    at_least_one_model_is_tested = False
    for model_architecture in model_architectures:
        model_arch_name = model_architecture.__name__
        model_type = model_architecture.config_class.model_type
        for _prefix in ['Flax', 'TF']:
            if model_arch_name.startswith(_prefix):
                model_arch_name = model_arch_name[len(_prefix):]
                break
        if model_arch_name not in tiny_model_summary:
            continue
        tokenizer_names = tiny_model_summary[model_arch_name]['tokenizer_classes']
        image_processor_names = []
        feature_extractor_names = []
        processor_classes = tiny_model_summary[model_arch_name]['processor_classes']
        for cls_name in processor_classes:
            if 'ImageProcessor' in cls_name:
                image_processor_names.append(cls_name)
            elif 'FeatureExtractor' in cls_name:
                feature_extractor_names.append(cls_name)
        processor_names = PROCESSOR_MAPPING_NAMES.get(model_type, None)
        if not isinstance(processor_names, (list, tuple)):
            processor_names = [processor_names]
        commit = None
        if model_arch_name in tiny_model_summary and 'sha' in tiny_model_summary[model_arch_name]:
            commit = tiny_model_summary[model_arch_name]['sha']
        repo_name = f'tiny-random-{model_arch_name}'
        if TRANSFORMERS_TINY_MODEL_PATH != 'hf-internal-testing':
            repo_name = model_arch_name
        self.run_model_pipeline_tests(task, repo_name, model_architecture, tokenizer_names=tokenizer_names, image_processor_names=image_processor_names, feature_extractor_names=feature_extractor_names, processor_names=processor_names, commit=commit, torch_dtype=torch_dtype)
        at_least_one_model_is_tested = True
    if task in task_to_pipeline_and_spec_mapping:
        pipeline, hub_spec = task_to_pipeline_and_spec_mapping[task]
        compare_pipeline_args_to_hub_spec(pipeline, hub_spec)
    if not at_least_one_model_is_tested:
        self.skipTest(f'{self.__class__.__name__}::test_pipeline_{task.replace('-', '_')}_{torch_dtype} is skipped: Could not find any model architecture in the tiny models JSON file for `{task}`.')

Run pipeline tests for a specific `task`

Args:
        task (`str`):
            A task name. This should be a key in the mapping `pipeline_test_mapping`.
        torch_dtype (`str`, `optional`, defaults to `'float32'`):
            The torch dtype to use for the model. Can be used for FP16/other precision inference.


def replace_batch_norm(model):
    for name, module in model.named_children():
        if isinstance(module, nn.BatchNorm2d):
            new_module = DFineFrozenBatchNorm2d(module.num_features)
            if not module.weight.device == torch.device('meta'):
                new_module.weight.data.copy_(module.weight)
                new_module.bias.data.copy_(module.bias)
                new_module.running_mean.data.copy_(module.running_mean)
                new_module.running_var.data.copy_(module.running_var)
            model._modules[name] = new_module
        if len(list(module.children())) > 0:
            replace_batch_norm(module)

Recursively replace all `torch.nn.BatchNorm2d` with `DFineFrozenBatchNorm2d`.

Args:
        model (torch.nn.Module):
            input model


def __init__(self, columns: list[str]) -> None:
    self.columns = columns

Base Opertation class data processing transformations.

Args:
        columns: List of column names to apply the transformation.


def recover_session(self, master: str, saver: saver_lib.Saver=None, checkpoint_dir: str=None, checkpoint_filename_with_path: str=None, wait_for_checkpoint=False, max_wait_secs=7200, config=None) -> Tuple[session.Session, bool]:
    sess, is_loaded_from_checkpoint = self._restore_checkpoint(master, saver, checkpoint_dir=checkpoint_dir, checkpoint_filename_with_path=checkpoint_filename_with_path, wait_for_checkpoint=wait_for_checkpoint, max_wait_secs=max_wait_secs, config=config)
    local_init_success, msg = self._try_run_local_init_op(sess)
    if not is_loaded_from_checkpoint:
        return (sess, False)
    restoring_file = checkpoint_dir or checkpoint_filename_with_path
    if not local_init_success:
        logging.info('Restoring model from %s did not make model ready for local init: %s', restoring_file, msg)
        return (sess, False)
    is_ready, msg = self._model_ready(sess)
    if not is_ready:
        logging.info('Restoring model from %s did not make model ready: %s', restoring_file, msg)
        return (sess, False)
    logging.info('Restored model from %s', restoring_file)
    return (sess, is_loaded_from_checkpoint)

Creates a `Session`, recovering if possible.

    Creates a new session on 'master'.  If the session is not initialized
    and can be recovered from a checkpoint, recover it.

Args:
        master: `String` representation of the TensorFlow master to use.
        saver: A `Saver` object used to restore a model.
        checkpoint_dir: Path to the checkpoint files. The latest checkpoint in the
        dir will be used to restore.
        checkpoint_filename_with_path: Full file name path to the checkpoint file.
        wait_for_checkpoint: Whether to wait for checkpoint to become available.
        max_wait_secs: Maximum time to wait for checkpoints to become available.
        config: Optional `ConfigProto` proto used to configure the session.

Returns:
        A pair (sess, initialized) where 'initialized' is `True` if
        the session could be recovered and initialized, `False` otherwise.

Raises:
        ValueError: If both checkpoint_dir and checkpoint_filename_with_path are
        set.


def _strip_layer_names(self, summaries, model_type):
    result = set()
    for s in summaries:
        if '/' not in s.tag:
            raise ValueError(f'tag has no layer name: {s.tag!r}')
        start_from = 2 if 'subclass' in model_type else 1
        new_tag = '/'.join(s.tag.split('/')[start_from:])
        result.add(s._replace(tag=new_tag))
    return result

Deduplicate summary names modulo layer prefix.

    This removes the first slash-component of each tag name: for
    instance, "foo/bar/baz" becomes "bar/baz".

Args:
        summaries: A `set` of `_ObservedSummary` values.
        model_type: The model type currently being tested.

Returns:
        A new `set` of `_ObservedSummary` values with layer prefixes
        removed.


def _broadcast_half(ac_0: _LayerBroadcaster, a_1: RowPartition) -> Tuple[_LayerBroadcaster, RowPartition]:
    c_1 = ac_0.broadcast_row_partition(a_1)
    old_value_rowids = array_ops.gather(ac_0.gather_index, c_1.value_rowids())
    old_row_starts = array_ops.gather(a_1.row_splits(), old_value_rowids)
    gather_index = old_row_starts + c_1.offsets_in_rows()
    return [_LayerBroadcaster.from_gather_index(gather_index), c_1]

Does a NOOP broadcast of a_1.

    *-ac_0-->*
    |        |
    a_1      c_1
    |        |
    V        V
    *-ac_1-->*

    Note that by definition this cannot fail: there is always a well-defined
    NOOP broadcast. This is usually intended as half of broadcasting two shapes
    together.

Args:
    ac_0: previous LayerBroadcaster
    a_1: previous RowPartition

Returns:
    [ac_1, c_1] where ac_1 is the next LayerBroadcaster, and c_1 is the
    broadcast RowPartition


def gauge(namespace: Union[Type, str], name: str) -> 'Metrics.DelegatingGauge':
    namespace = Metrics.get_namespace(namespace)
    return Metrics.DelegatingGauge(MetricName(namespace, name))

Obtains or creates a Gauge metric.

    Gauge metrics are restricted to integer-only values.

Args:
        namespace: A class or string that gives the namespace to a metric
        name: A string that gives a unique name to a metric

Returns:
        A Distribution object.


def __le__(self: EventSetOrNode, other: Any) -> EventSetOrNode:
    if isinstance(other, self.__class__):
        from temporian.core.operators.binary import less_equal
        return less_equal(input_left=self, input_right=other)
    if isinstance(other, T_SCALAR):
        from temporian.core.operators.scalar import less_equal_scalar
        return less_equal_scalar(input=self, value=other)
    self._raise_error('compare', other, '(int,float)')
    assert False

Computes less equal (`self <= other`) element-wise with another
    [`EventSet`][temporian.EventSet] or a scalar value.

    If an EventSet, each feature in `self` is compared element-wise to the
    feature in `other` in the same position. `self` and `other` must have
    the same sampling and the same number of features.

    If a scalar value, each item in each feature in `input` is compared to
    `value`.

    Note that it will always return False on NaN elements.

    Example with EventSet:
        ```python
        >>> a = tp.event_set(
        ...     timestamps=[1, 2, 3],
        ...     features={"f1": [0, 100, 200]}
        ... )
        >>> b = tp.event_set(
        ...     timestamps=[1, 2, 3],
        ...     features={"f2": [-10, 100, 5]},
        ...     same_sampling_as=a
        ... )

        >>> c = a <= b
        >>> c
        indexes: []
        features: [('f1', bool_)]
        events:
            (3 events):
                timestamps: [1. 2. 3.]
                'f1': [False True False]
        ...

        ```

    Example with scalar:
        ```python
        >>> a = tp.event_set(
        ...     timestamps=[1, 2, 3],
        ...     features={"f1": [0, 100, 200], "f2": [-10, 100, 5]}
        ... )

        >>> b = a <= 100
        >>> b
        indexes: []
        features: [('f1', bool_), ('f2', bool_)]
        events:
            (3 events):
                timestamps: [1. 2. 3.]
                'f1': [ True True False]
                'f2': [ True True True]
        ...

        ```

Args:
        other: EventSet or scalar value.

Returns:
        Result of the comparison.


def load_checkpoint(ckpt_dir_or_file):
    filename = _get_checkpoint_filename(ckpt_dir_or_file)
    if filename is None:
        raise ValueError("Couldn't find 'checkpoint' file or checkpoints in given directory %s" % ckpt_dir_or_file)
    return py_checkpoint_reader.NewCheckpointReader(filename)

Returns `CheckpointReader` for checkpoint found in `ckpt_dir_or_file`.

    If `ckpt_dir_or_file` resolves to a directory with multiple checkpoints,
    reader for the latest checkpoint is returned.

    Example usage:

    ```python
    import tensorflow as tf
    a = tf.Variable(1.0)
    b = tf.Variable(2.0)
    ckpt = tf.train.Checkpoint(var_list={'a': a, 'b': b})
    ckpt_path = ckpt.save('tmp-ckpt')
    reader= tf.train.load_checkpoint(ckpt_path)
    print(reader.get_tensor('var_list/a/.ATTRIBUTES/VARIABLE_VALUE'))  # 1.0
    ```

Args:
        ckpt_dir_or_file: Directory with checkpoints file or path to checkpoint
        file.

Returns:
        `CheckpointReader` object.

Raises:
        ValueError: If `ckpt_dir_or_file` resolves to a directory with no
        checkpoints.


def _assert_weights_created(self):
    if self.dynamic:
        return
    if 'build' in self.__class__.__dict__ and self.__class__ != Model and (not self.built):
        raise ValueError('Weights for model %s have not yet been created. Weights are created when the Model is first called on inputs or `build()` is called with an `input_shape`.' % self.name)

Asserts that all the weights for the model have been created.

    For a non-dynamic model, the weights must already be created after the
    layer has been called. For a dynamic model, the exact list of weights can
    never be known for certain since it may change at any time during execution.

    We run this check right before accessing weights or getting the Numpy value
    for the current weights. Otherwise, if the layer has never been called,
    the user would just get an empty list, which is misleading.

Raises:
        ValueError: if the weights of the network has not yet been created.


def rot90(array, k=1, axes=(0, 1)):
    array = convert_to_tensor(array)
    if array.shape.rank < 2:
        raise ValueError(f'Input array must have at least 2 dimensions. Received: array.ndim={array.shape.rank}')
    if len(axes) != 2 or axes[0] == axes[1]:
        raise ValueError(f'Invalid axes: {axes}. Axes must be a tuple of two different dimensions.')
    k = k % 4
    if k == 0:
        return array
    axes = tuple((axis if axis >= 0 else array.shape.rank + axis for axis in axes))
    perm = [i for i in range(array.shape.rank) if i not in axes]
    perm.extend(axes)
    array = tf.transpose(array, perm)
    shape = tf.shape(array)
    non_rot_shape = shape[:-2]
    h, w = (shape[-2], shape[-1])
    array = tf.reshape(array, tf.concat([[-1], [h, w]], axis=0))
    array = tf.reverse(array, axis=[2])
    array = tf.transpose(array, [0, 2, 1])
    if k % 2 == 1:
        final_h, final_w = (w, h)
    else:
        final_h, final_w = (h, w)
    if k > 1:
        array = tf.reshape(array, tf.concat([[-1], [final_h, final_w]], axis=0))
        for _ in range(k - 1):
            array = tf.reverse(array, axis=[2])
            array = tf.transpose(array, [0, 2, 1])
    final_shape = tf.concat([non_rot_shape, [final_h, final_w]], axis=0)
    array = tf.reshape(array, final_shape)
    inv_perm = [0] * len(perm)
    for i, p in enumerate(perm):
        inv_perm[p] = i
    array = tf.transpose(array, inv_perm)
    return array

Rotate an array by 90 degrees in the specified plane.

Args:
        array: Input tensor
        k: Number of 90-degree rotations (default=1)
        axes: Tuple of two axes that define the plane of rotation.
        Defaults to (0, 1).

Returns:
        Rotated tensor with correct shape transformation


def __len__(self):
    raise NotImplementedError

Number of batch in the Sequence.

Returns:
        The number of batches in the Sequence.


def copy(self, source_file_names, destination_file_names):
    raise NotImplementedError

Recursively copy the file tree from the source to the destination

Args:
        source_file_names: list of source file objects that needs to be copied
        destination_file_names: list of destination of the new object

Raises:
        ``BeamIOError``: if any of the copy operations fail


def _merge_choice_field(self, json_value: Any, choice_field: descriptor.FieldDescriptor, field_name: str, parent: message.Message) -> None:
    choice_field_name = _get_choice_field_name(choice_field, field_name)
    choice_field_map = _get_field_map(choice_field.message_type)
    choice_value_field = choice_field_map.get(choice_field_name)
    if choice_value_field is None:
        raise ValueError(f'Cannot find {choice_field_name!r} on {choice_field.full_name}')
    choice_message = proto_utils.set_in_parent_or_add(parent, choice_field)
    self._merge_field(json_value, choice_value_field, choice_message)

Creates a Message based on the choice_field Descriptor and json_value.

    The resulting message is merged into parent.

Args:
        json_value: The JSON value to merge into a message of the type described
        by choice_field.
        choice_field: The field descriptor of the FHIR choice type on parent.
        field_name: The nested field name of the choice type, e.g.: _valueBoolean.
        parent: The parent Message to merge into.


def print_layer_summary_with_connections(layer):
    try:
        output_shape = layer.output_shape
    except AttributeError:
        output_shape = 'multiple'
    connections = []
    for node in layer._inbound_nodes:
        if relevant_nodes and node not in relevant_nodes:
            continue
        for inbound_layer, node_index, tensor_index, _ in node.iterate_inbound():
            connections.append('{}[{}][{}]'.format(inbound_layer.name, node_index, tensor_index))
    name = layer.name
    cls_name = layer.__class__.__name__
    if not connections:
        first_connection = ''
    else:
        first_connection = connections[0]
    fields = [name + ' (' + cls_name + ')', output_shape, layer.count_params(), first_connection]
    print_row(fields, positions)
    if len(connections) > 1:
        for i in range(1, len(connections)):
            fields = ['', '', '', connections[i]]
            print_row(fields, positions)

Prints a summary for a single layer (including topological connections).

Args:
        layer: target layer.


def as_default(self, step=None):
    return _SummaryContextManager(self, step)

Returns a context manager that enables summary writing.

    For convenience, if `step` is not None, this function also sets a default
    value for the `step` parameter used in summary-writing functions elsewhere
    in the API so that it need not be explicitly passed in every such
    invocation. The value can be a constant or a variable.

    Note: when setting `step` in a @tf.function, the step value will be
    captured at the time the function is traced, so changes to the step outside
    the function will not be reflected inside the function unless using
    a `tf.Variable` step.

    For example, `step` can be used as:

    ```python
    with writer_a.as_default(step=10):
        tf.summary.scalar(tag, value)   # Logged to writer_a with step 10
        with writer_b.as_default(step=20):
        tf.summary.scalar(tag, value) # Logged to writer_b with step 20
        tf.summary.scalar(tag, value)   # Logged to writer_a with step 10
    ```

Args:
        step: An `int64`-castable default step value, or `None`. When not `None`,
        the current step is captured, replaced by a given one, and the original
        one is restored when the context manager exits. When `None`, the current
        step is not modified (and not restored when the context manager exits).

Returns:
        The context manager.


def __call__(self, text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]], text_pair: Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]]=None, boxes: Optional[Union[List[List[int]], List[List[List[int]]]]]=None, word_labels: Optional[Union[List[int], List[List[int]]]]=None, add_special_tokens: bool=True, padding: Union[bool, str, PaddingStrategy]=False, truncation: Union[bool, str, TruncationStrategy]=None, max_length: Optional[int]=None, stride: int=0, pad_to_multiple_of: Optional[int]=None, padding_side: Optional[str]=None, return_tensors: Optional[Union[str, TensorType]]=None, return_token_type_ids: Optional[bool]=None, return_attention_mask: Optional[bool]=None, return_overflowing_tokens: bool=False, return_special_tokens_mask: bool=False, return_offsets_mapping: bool=False, return_length: bool=False, verbose: bool=True, **kwargs) -> BatchEncoding:

    def _is_valid_text_input(t):
        if isinstance(t, str):
            return True
        elif isinstance(t, (list, tuple)):
            if len(t) == 0:
                return True
            elif isinstance(t[0], str):
                return True
            elif isinstance(t[0], (list, tuple)):
                return len(t[0]) == 0 or isinstance(t[0][0], str)
            else:
                return False
        else:
            return False
    if text_pair is not None:
        if not _is_valid_text_input(text):
            raise ValueError('text input must of type `str` (single example) or `List[str]` (batch of examples). ')
        if not isinstance(text_pair, (list, tuple)):
            raise ValueError('Words must be of type `List[str]` (single pretokenized example), or `List[List[str]]` (batch of pretokenized examples).')
    elif not isinstance(text, (list, tuple)):
        raise ValueError('Words must be of type `List[str]` (single pretokenized example), or `List[List[str]]` (batch of pretokenized examples).')
    if text_pair is not None:
        is_batched = isinstance(text, (list, tuple))
    else:
        is_batched = isinstance(text, (list, tuple)) and text and isinstance(text[0], (list, tuple))
    words = text if text_pair is None else text_pair
    if boxes is None:
        raise ValueError('You must provide corresponding bounding boxes')
    if is_batched:
        if len(words) != len(boxes):
            raise ValueError('You must provide words and boxes for an equal amount of examples')
        for words_example, boxes_example in zip(words, boxes):
            if len(words_example) != len(boxes_example):
                raise ValueError('You must provide as many words as there are bounding boxes')
    elif len(words) != len(boxes):
        raise ValueError('You must provide as many words as there are bounding boxes')
    if is_batched:
        if text_pair is not None and len(text) != len(text_pair):
            raise ValueError(f'batch length of `text`: {len(text)} does not match batch length of `text_pair`: {len(text_pair)}.')
        batch_text_or_text_pairs = list(zip(text, text_pair)) if text_pair is not None else text
        is_pair = bool(text_pair is not None)
        return self.batch_encode_plus(batch_text_or_text_pairs=batch_text_or_text_pairs, is_pair=is_pair, boxes=boxes, word_labels=word_labels, add_special_tokens=add_special_tokens, padding=padding, truncation=truncation, max_length=max_length, stride=stride, pad_to_multiple_of=pad_to_multiple_of, padding_side=padding_side, return_tensors=return_tensors, return_token_type_ids=return_token_type_ids, return_attention_mask=return_attention_mask, return_overflowing_tokens=return_overflowing_tokens, return_special_tokens_mask=return_special_tokens_mask, return_offsets_mapping=return_offsets_mapping, return_length=return_length, verbose=verbose, **kwargs)
    else:
        return self.encode_plus(text=text, text_pair=text_pair, boxes=boxes, word_labels=word_labels, add_special_tokens=add_special_tokens, padding=padding, truncation=truncation, max_length=max_length, stride=stride, pad_to_multiple_of=pad_to_multiple_of, padding_side=padding_side, return_tensors=return_tensors, return_token_type_ids=return_token_type_ids, return_attention_mask=return_attention_mask, return_overflowing_tokens=return_overflowing_tokens, return_special_tokens_mask=return_special_tokens_mask, return_offsets_mapping=return_offsets_mapping, return_length=return_length, verbose=verbose, **kwargs)

Main method to tokenize and prepare for the model one or several sequence(s) or one or several pair(s) of
    sequences with word-level normalized bounding boxes and optional labels.

Args:
        text (`str`, `List[str]`, `List[List[str]]`):
            The sequence or batch of sequences to be encoded. Each sequence can be a string, a list of strings
            (words of a single example or questions of a batch of examples) or a list of list of strings (batch of
            words).
        text_pair (`List[str]`, `List[List[str]]`):
            The sequence or batch of sequences to be encoded. Each sequence should be a list of strings
            (pretokenized string).
        boxes (`List[List[int]]`, `List[List[List[int]]]`):
            Word-level bounding boxes. Each bounding box should be normalized to be on a 0-1000 scale.
        word_labels (`List[int]`, `List[List[int]]`, *optional*):
            Word-level integer labels (for token classification tasks such as FUNSD, CORD).


def __init__(self, custom_op_registerers=None, **kwargs):
    self._custom_op_registerers = custom_op_registerers or []
    super(InterpreterWithCustomOps, self).__init__(**kwargs)

Constructor.

Args:
        custom_op_registerers: List of str (symbol names) or functions that take a
        pointer to a MutableOpResolver and register a custom op. When passing
        functions, use a pybind function that takes a uintptr_t that can be
        recast as a pointer to a MutableOpResolver.
        **kwargs: Additional arguments passed to Interpreter.

Raises:
        ValueError: If the interpreter was unable to create.


def forward(self, input_ids: torch.LongTensor=None, pixel_values: torch.FloatTensor=None, attention_mask: Optional[torch.Tensor]=None, position_ids: Optional[torch.LongTensor]=None, past_key_values: Optional[List[torch.FloatTensor]]=None, inputs_embeds: Optional[torch.FloatTensor]=None, labels: Optional[torch.LongTensor]=None, use_cache: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, return_dict: Optional[bool]=None, cache_position: Optional[torch.LongTensor]=None, logits_to_keep: Union[int, torch.Tensor]=0, **kwargs: Unpack[KwargsForCausalLM]) -> Union[Tuple, GotOcr2CausalLMOutputWithPast]:
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
    outputs = self.model(input_ids=input_ids, pixel_values=pixel_values, attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=True, cache_position=cache_position, logits_to_keep=logits_to_keep, **kwargs)
    hidden_states = outputs[0]
    slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
    logits = self.lm_head(hidden_states[:, slice_indices, :])
    loss = None
    if labels is not None:
        loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size, **kwargs)
    return GotOcr2CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions, image_hidden_states=outputs.image_hidden_states)

labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:
    ```python
    >>> from PIL import Image
    >>> import requests
    >>> from transformers import AutoProcessor, GotOcr2ForConditionalGeneration, TextStreamer

    >>> model = GotOcr2ForConditionalGeneration.from_pretrained("stepfun-ai/GOT-OCR-2.0-hf").to("cuda")
    >>> processor = AutoProcessor.from_pretrained("stepfun-ai/GOT-OCR-2.0-hf")

    >>> url = "https://huggingface.co/datasets/hf-internal-testing/fixtures_got_ocr/resolve/main/multi_box.png"
    >>> image = Image.open(requests.get(url, stream=True).raw)

    >>> inputs = processor(image, return_tensors="pt", color="green").to("cuda")

    >>> # Generate
    >>> streamer = TextStreamer(processor.tokenizer, skip_prompt=True, skip_special_tokens=True)
    >>> generate_ids = model.generate(
    ...     **inputs,
    ...     do_sample=False,
    ...     tokenizer = processor.tokenizer,
    ...     stop_strings='<|im_end|>',
    ...     streamer=streamer,
    ...     max_new_tokens=4096,
    ... )
    "You should keep in mind what features from the module should be used, especially
    when you're planning to sell a template."
    ```