def replace_model_patterns(text: str, old_model_patterns: ModelPatterns, new_model_patterns: ModelPatterns) -> Tuple[str, str]:
    attributes_to_check = ['config_class']
    for attr in ['tokenizer_class', 'image_processor_class', 'image_processor_fast_class', 'feature_extractor_class', 'processor_class']:
        if getattr(old_model_patterns, attr) is not None and getattr(new_model_patterns, attr) is not None:
            attributes_to_check.append(attr)
    if old_model_patterns.checkpoint not in [old_model_patterns.model_type, old_model_patterns.model_lower_cased]:
        attributes_to_check.append('checkpoint')
    if old_model_patterns.model_type != old_model_patterns.model_lower_cased:
        attributes_to_check.append('model_type')
    else:
        text = re.sub(f'(\\s*)model_type = "{old_model_patterns.model_type}"', '\\1model_type = "[MODEL_TYPE]"', text)
    if old_model_patterns.model_upper_cased == old_model_patterns.model_camel_cased:
        old_model_value = old_model_patterns.model_upper_cased
        if re.search(f'{old_model_value}_[A-Z_]*[^A-Z_]', text) is not None:
            text = re.sub(f'{old_model_value}([A-Z_]*)([^a-zA-Z_])', '[MODEL_UPPER_CASED]\\1\\2', text)
    else:
        attributes_to_check.append('model_upper_cased')
    attributes_to_check.extend(['model_camel_cased', 'model_lower_cased', 'model_name'])
    for attr in attributes_to_check:
        text = text.replace(getattr(old_model_patterns, attr), ATTRIBUTE_TO_PLACEHOLDER[attr])
    replacements = []
    for attr, placeholder in ATTRIBUTE_TO_PLACEHOLDER.items():
        if placeholder in text:
            replacements.append((getattr(old_model_patterns, attr), getattr(new_model_patterns, attr)))
            text = text.replace(placeholder, getattr(new_model_patterns, attr))
    old_replacement_values = [old for old, new in replacements]
    if len(set(old_replacement_values)) != len(old_replacement_values):
        return (text, '')
    replacements = simplify_replacements(replacements)
    replacements = [f'{old}->{new}' for old, new in replacements]
    return (text, ','.join(replacements))

Replace all patterns present in a given text.

Args:
        text (`str`): The text to treat.
        old_model_patterns (`ModelPatterns`): The patterns for the old model.
        new_model_patterns (`ModelPatterns`): The patterns for the new model.

Returns:
        `Tuple(str, str)`: A tuple of with the treated text and the replacement actually done in it.


def sort_all_auto_mappings(overwrite: bool=False):
    fnames = [os.path.join(PATH_TO_AUTO_MODULE, f) for f in os.listdir(PATH_TO_AUTO_MODULE) if f.endswith('.py')]
    diffs = [sort_auto_mapping(fname, overwrite=overwrite) for fname in fnames]
    if not overwrite and any(diffs):
        failures = [f for f, d in zip(fnames, diffs) if d]
        raise ValueError(f'The following files have auto mappings that need sorting: {', '.join(failures)}. Run `make style` to fix this.')

Sort all auto mappings in the library.

Args:
        overwrite (`bool`, *optional*, defaults to `False`): Whether or not to fix and overwrite the file.


def _load_metadata_files(self):
    metadata_paths = file_io.get_matching_files(os.path.join(self._dump_root, '*%s' % self._METADATA_SUFFIX))
    if not metadata_paths:
        raise ValueError('Cannot find any tfdbg metadata file in directory: %s' % self._dump_root)
    wall_times = []
    run_ids = []
    tensorflow_versions = []
    file_versions = []
    for metadata_path in metadata_paths:
        reader = tf_record.tf_record_random_reader(metadata_path)
        try:
            record = reader.read(0)[0]
            debug_event = debug_event_pb2.DebugEvent.FromString(record)
            wall_times.append(debug_event.wall_time)
            run_ids.append(debug_event.debug_metadata.tfdbg_run_id)
            tensorflow_versions.append(debug_event.debug_metadata.tensorflow_version)
            file_versions.append(debug_event.debug_metadata.file_version)
        except Exception as e:
            raise errors.DataLossError(None, None, 'Error reading tfdbg metadata from paths %s' % metadata_paths) from e
        finally:
            reader.close()
    self._starting_wall_time = wall_times[0]
    self._tfdbg_run_id = run_ids[0]
    self._tensorflow_version = tensorflow_versions[0]
    self._file_version = file_versions[0]
    if len(metadata_paths) == 1:
        return metadata_paths
    num_no_id = len([run_id for run_id in run_ids if not run_id])
    if num_no_id:
        paths_without_run_id = [metadata_path for metadata_path, run_id in zip(metadata_paths, run_ids) if not run_id]
        raise ValueError('Found %d tfdbg metadata files and %d of them do not have tfdbg run ids. The metadata files without run ids are: %s' % (len(run_ids), num_no_id, paths_without_run_id))
    elif len(set(run_ids)) != 1:
        raise ValueError('Unexpected: Found multiple (%d) tfdbg2 runs in directory %s' % (len(set(run_ids)), self._dump_root))
    paths_and_timestamps = sorted(zip(metadata_paths, wall_times), key=lambda t: t[1])
    self._starting_wall_time = paths_and_timestamps[0][1]
    return [path[0] for path in paths_and_timestamps]

Load and parse metadata files in the dump root.

    Check that all metadata files have a common tfdbg_run_id, and raise
    a ValueError if their tfdbg_run_ids differ.

Returns:
        A list of metadata file paths in ascending order of their starting
        wall_time timestamp.


def operator_and_matrix(self, shapes_info, dtype, use_placeholder, ensure_self_adjoint_and_pd=False):
    raise NotImplementedError('Not implemented yet.')

Build a batch matrix and an Operator that should have similar behavior.

    Every operator acts like a (batch) matrix.  This method returns both
    together, and is used by tests.

Args:
        shapes_info: `OperatorShapesInfo`, encoding shape information about the
        operator.
        dtype:  Numpy dtype.  Data type of returned array/operator.
        use_placeholder:  Python bool.  If True, initialize the operator with a
        placeholder of undefined shape and correct dtype.
        ensure_self_adjoint_and_pd: If `True`,
        construct this operator to be Hermitian Positive Definite, as well
        as ensuring the hints `is_positive_definite` and `is_self_adjoint`
        are set.
        This is useful for testing methods such as `cholesky`.

Returns:
        operator:  `LinearOperator` subclass instance.
        mat:  `Tensor` representing operator.


def configure_tpu_version(self, version, restart_type='always'):

    def configure_worker(worker):
        """Configure individual TPU worker.

      Args:
        worker: A dict with the field ipAddress where the configure request will
          be sent.
      """
        ip_address = worker['ipAddress']
        url = (_VERSION_SWITCHER_ENDPOINT + '/{}?restartType={}').format(ip_address, version, restart_type)
        req = urllib.request.Request(url, data=b'')
        try:
            urllib.request.urlopen(req)
        except urllib.error.HTTPError as e:
            status_code = e.code
            if status_code == 404:
                raise Exception('Tensorflow version {} is not available on Cloud TPU, try a previous nightly version or refer to https://cloud.google.com/tpu/docs/release-notes for the latest official version.'.format(version))
            else:
                raise Exception('Failed to configure worker {}'.format(ip_address))
    workers = self.network_endpoints()
    with futures.ThreadPoolExecutor(max_workers=len(workers)) as executor:
        results = executor.map(configure_worker, workers)
        for result in results:
            if result:
                result.result()

Configure TPU software version.

Args:
        version (string): Version of software to configure the TPU with.
        restart_type (string): Restart behaviour when switching versions,
        defaults to always restart. Options are 'always', 'ifNeeded'.


def control_status_ctx():
    ret = _control_ctx()[-1]
    return ret

Returns the current control context for autograph.

    This method is useful when calling `tf.__internal__.autograph.tf_convert`,
    The context will be used by tf_convert to determine whether it should convert
    the input function. See the sample usage like below:

    ```
    def foo(func):
        return tf.__internal__.autograph.tf_convert(
        input_fn, ctx=tf.__internal__.autograph.control_status_ctx())()
    ```

Returns:
        The current control context of autograph.


def verify_error_on_save(self, ds_fn, num_outputs, error, break_point=None, sparse_tensors=False, assert_items_equal=False):
    del assert_items_equal
    break_point = num_outputs // 2 if not break_point else break_point
    if context.executing_eagerly():
        iterator = iter(ds_fn())
        ckpt = tracking_util.Checkpoint(iterator=iterator)
        for _ in range(break_point):
            next(iterator)
        with self.assertRaises(error):
            ckpt.save(self._ckpt_path())
    else:
        with ops.Graph().as_default() as g:
            init_op, get_next_op, saver = self._build_graph(ds_fn, sparse_tensors=sparse_tensors)
            get_next_op = remove_variants(get_next_op)
            with self.session(graph=g) as sess:
                self._initialize(init_op, sess)
                for _ in range(break_point):
                    sess.run(get_next_op)
                with self.assertRaises(error):
                    self._save(sess, saver)

Attempts to save a non-saveable iterator.

Args:
        ds_fn: 0-argument function that returns a Dataset.
        num_outputs: Total number of outputs expected from this Dataset.
        error: Declared error when trying to save iterator.
        break_point: Break point. Optional. Defaults to num_outputs/2.
        sparse_tensors: Whether dataset is built from SparseTensor(s).
        assert_items_equal: Tests the output has the expected elements regardless
        of order.

Raises:
        AssertionError if any test fails.


def parse_single_example(serialized, features, name=None, example_names=None):
    return parse_single_example_v2(serialized, features, example_names, name)

Parses a single `Example` proto.

    Similar to `parse_example`, except:

    For dense tensors, the returned `Tensor` is identical to the output of
    `parse_example`, except there is no batch dimension, the output shape is the
    same as the shape given in `dense_shape`.

    For `SparseTensor`s, the first (batch) column of the indices matrix is removed
    (the indices matrix is a column vector), the values vector is unchanged, and
    the first (`batch_size`) entry of the shape vector is removed (it is now a
    single element vector).

    One might see performance advantages by batching `Example` protos with
    `parse_example` instead of using this function directly.

Args:
        serialized: A scalar string Tensor, a single serialized Example.
        features: A mapping of feature keys to `FixedLenFeature` or
        `VarLenFeature` values.
        name: A name for this operation (optional).
        example_names: (Optional) A scalar string Tensor, the associated name.

Returns:
        A `dict` mapping feature keys to `Tensor` and `SparseTensor` values.

Raises:
        ValueError: if any feature is invalid.


def to_dict(self):
    output = asdict(self)
    output['structure_module'] = self.structure_module.to_dict()
    return output

Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].

Returns:
        `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,


def until_next(self: EventSetOrNode, sampling: EventSetOrNode, timeout: Duration) -> EventSetOrNode:
    from temporian.core.operators.until_next import until_next
    return until_next(self, timeout=timeout, sampling=sampling)

Gets the duration until the next sampling event for each input event.

    If no sampling event is observed before `timeout` time-units, returns
    NaN.

    `until_next` is different from `since_last` in that `since_last` returns
    one value for each sampling (sampling events are after input events),
    while `until_next` returns one value for each input value (here again,
    sampling events are after input events).

    The output [`EventSet`][temporian.EventSet] has one event for each event
    in input, but with its timestamp moved forward to the nearest future
    event in `sampling`. If no timestamp in sampling is closer than timeout,
    it is moved by `timeout` into the future instead.

    `until_next` is useful to measure the time it takes for an issue
    (`input`) to be detected by an alert (`sampling`).

    Basic example with 1 and 2 steps:
        ```python
        >>> a = tp.event_set(timestamps=[0, 10, 11, 20, 30])
        >>> b = tp.event_set(timestamps=[1, 12, 21, 22, 42])
        >>> c = a.until_next(sampling=b, timeout=5)
        >>> c
        indexes: []
        features: [('until_next', float64)]
        events:
            (5 events):
                timestamps: [ 1. 12. 12. 21. 35.]
                'until_next': [ 1.  2.  1.  1. nan]
        ...

        ```

Args:
        sampling: EventSet to use the sampling of.
        timeout: Maximum amount of time to wait. If no sampling is observed
            before the timeout expires, the output feature value is NaN.

Returns:
        Resulting EventSet.


def coder_benchmark_factory(coder, generate_fn):

    class CoderBenchmark(object):

        def __init__(self, num_elements_per_benchmark):
            self._coder = coders.IterableCoder(coder)
            self._list = [generate_fn() for _ in range(num_elements_per_benchmark)]

        def __call__(self):
            _ = self._coder.decode(self._coder.encode(self._list))
    CoderBenchmark.__name__ = '%s, %s' % (generate_fn.__name__, str(coder))
    return CoderBenchmark

Creates a benchmark that encodes and decodes a list of elements.

Args:
        coder: coder to use to encode an element.
        generate_fn: a callable that generates an element.


def __init__(self, config: MT5Config):
    super().__init__(config)
    self.shared = nn.Embedding(config.vocab_size, config.d_model)
    encoder_config = copy.deepcopy(config)
    encoder_config.use_cache = False
    encoder_config.is_encoder_decoder = False
    self.encoder = MT5Stack(encoder_config, self.shared)
    self.post_init()
    self.model_parallel = False
    self.device_map = None

Examples:

    ```python
    >>> from transformers import MT5EncoderModel, AutoTokenizer

    >>> model = MT5EncoderModel.from_pretrained("google/mt5-small")
    >>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
    >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien."
    >>> input_ids = tokenizer(article, return_tensors="pt").input_ids
    >>> outputs = model(input_ids)
    >>> hidden_state = outputs.last_hidden_state
    ```


def check_accessible(value_provider_list):
    assert isinstance(value_provider_list, list)

    def _check_accessible(fnc):

        @wraps(fnc)
        def _f(self, *args, **kwargs):
            for obj in [getattr(self, vp) for vp in value_provider_list]:
                if not obj.is_accessible():
                    raise error.RuntimeValueProviderError('%s not accessible' % obj)
            return fnc(self, *args, **kwargs)
        return _f
    return _check_accessible

A decorator that checks accessibility of a list of ValueProvider objects.

Args:
        value_provider_list: list of ValueProvider objects

Raises:
        ``RuntimeValueProviderError``: if any of the provided objects are not
        accessible.


def __init__(self, queue, ev_writer, flush_secs, flush_complete, flush_sentinel, close_sentinel):
    threading.Thread.__init__(self, name='EventLoggerThread')
    self.daemon = True
    self._queue = queue
    self._ev_writer = ev_writer
    self._flush_secs = flush_secs
    self._next_event_flush_time = 0
    self._flush_complete = flush_complete
    self._flush_sentinel = flush_sentinel
    self._close_sentinel = close_sentinel
    self.failure_exc_info = ()

Creates an _EventLoggerThread.

Args:
        queue: A CloseableQueue from which to dequeue events. The queue will be
        closed just before the thread exits, whether due to `close_sentinel` or
        any exception raised in the writing loop.
        ev_writer: An event writer. Used to log brain events for
        the visualizer.
        flush_secs: How often, in seconds, to flush the
        pending file to disk.
        flush_complete: A threading.Event that will be set whenever a flush
        operation requested via `flush_sentinel` has been completed.
        flush_sentinel: A sentinel element in queue that tells this thread to
        flush the writer and mark the current flush operation complete.
        close_sentinel: A sentinel element in queue that tells this thread to
        terminate and close the queue.


def soft_shrink(x, threshold=0.5):
    return ops.soft_shrink(x, threshold=threshold)

Soft Shrink activation function.

    It is defined as:

    `soft_shrink(x) = x - threshold` if `x > threshold`,
    `soft_shrink(x) = x + threshold` if `x < -threshold`,
    `soft_shrink(x) = 0` otherwise.

Args:
        x: Input tensor.
        threshold: Threshold value. Defaults to 0.5.


def variables(self):
    return self._opt.variables()

Fetches a list of optimizer variables in the default graph.

    This wraps `variables()` from the actual optimizer. It does not include
    the `SyncReplicasOptimizer`'s local step.

Returns:
        A list of variables.


def get_numpy_to_framework_fn(arr) -> Callable:
    if isinstance(arr, np.ndarray):
        return np.array
    if is_tf_available() and is_tf_tensor(arr):
        import tensorflow as tf
        return tf.convert_to_tensor
    if is_torch_available() and is_torch_tensor(arr):
        import torch
        return torch.tensor
    if is_flax_available() and is_jax_tensor(arr):
        import jax.numpy as jnp
        return jnp.array
    raise ValueError(f'Cannot convert arrays of type {type(arr)}')

Returns a function that converts a numpy array to the framework of the input array.

Args:
        arr (`np.ndarray`): The array to convert.


def alias_inplace_add(x, i, v):
    return _inplace_helper(x, i, v, gen_array_ops.inplace_add)

Applies an inplace add on input x at index i with value v. Aliases x.

    If i is None, x and v must be the same shape. Computes
        x += v;
    If i is a scalar, x has a rank 1 higher than v's. Computes
        x[i, :] += v;
    Otherwise, x and v must have the same rank. Computes
        x[i, :] += v;

Args:
        x: A Tensor.
        i: None, a scalar or a vector.
        v: A Tensor.

Returns:
        Returns x.


def assertNotAllClose(self, a, b, rtol=1e-06, atol=1e-06, msg=None):
    try:
        self.assertAllClose(a, b, rtol=rtol, atol=atol, msg=msg)
    except AssertionError:
        return
    msg = msg or ''
    raise AssertionError('The two values are close at all elements. %s' % msg)

Assert that two numpy arrays, or Tensors, do not have near values.

Args:
        a: The expected numpy `ndarray`, or anything that can be converted into a
        numpy `ndarray` (including Tensor), or any arbitrarily nested of
        structure of these.
        b: The actual numpy `ndarray`, or anything that can be converted into a
        numpy `ndarray` (including Tensor), or any arbitrarily nested of
        structure of these.
        rtol: relative tolerance.
        atol: absolute tolerance.
        msg: Optional message to report on failure.

Raises:
        AssertionError: If `a` and `b` are unexpectedly close at all elements.


def name_scope_only_in_function_or_graph(name):
    if not context.executing_eagerly():
        return ops.name_scope_v1(name)
    else:
        return NullContextmanager()

Internal-only entry point for `name_scope*`.

    Enters a compat.v1.name_scope only when in a function or graph,
    not when running fully eagerly.

Args:
        name: The name argument that is passed to the op function.

Returns:
        `name_scope*` context manager.


def _infer_num_gpus_per_worker(devices):
    if _is_device_list_single_worker(devices):
        return sum((1 for d in devices if _is_gpu_device(d)))
    else:
        device_dict = _group_device_list(devices)
        num_gpus = None
        for _, devices_in_task in device_dict.items():
            for device_in_task in devices_in_task:
                if num_gpus is None:
                    num_gpus = sum((1 for d in device_in_task if _is_gpu_device(d)))
                elif num_gpus != sum((1 for d in device_in_task if _is_gpu_device(d))):
                    raise ValueError('All workers should have the same number of GPUs.')
                for d in device_in_task:
                    d_spec = tf_device.DeviceSpec.from_string(d)
                    if d_spec.device_type == 'GPU' and d_spec.device_index >= num_gpus:
                        raise ValueError('GPU `device_index` on a worker should be consecutive and start from 0.')
        return num_gpus

Infers the number of GPUs on each worker.

    Currently to make multi-worker cross device ops work, we need all workers to
    have the same number of GPUs.

Args:
        devices: a list of device strings, can be either local devices or remote
        devices.

Returns:
        number of GPUs per worker.

Raises:
        ValueError if workers have different number of GPUs or GPU indices are not
        consecutive and starting from 0.


def ragged_binary_elementwise_assert_op_impl(op, x, y):
    x_is_ragged = ragged_tensor.is_ragged(x)
    y_is_ragged = ragged_tensor.is_ragged(y)
    x = ragged_tensor.convert_to_tensor_or_ragged_tensor(x, preferred_dtype=y.dtype if y_is_ragged else None)
    y = ragged_tensor.convert_to_tensor_or_ragged_tensor(y, preferred_dtype=x.dtype)
    if x_is_ragged and y_is_ragged:
        x, y = ragged_tensor.match_row_splits_dtypes(x, y)
    if x_is_ragged and y_is_ragged or (x_is_ragged and x.flat_values.shape.ndims <= y.shape.ndims) or (y_is_ragged and y.flat_values.shape.ndims <= x.shape.ndims):
        shape_x = DynamicRaggedShape.from_tensor(x)
        shape_y = DynamicRaggedShape.from_tensor(y)
        if shape_x.dtype != shape_y.dtype:
            if not x_is_ragged:
                shape_x = shape_x.with_dtype(shape_y.dtype)
            elif not y_is_ragged:
                shape_y = shape_y.with_dtype(shape_x.dtype)
        if _row_partitions_identical(shape_x, shape_y):
            return op(x.flat_values, y.flat_values)
        _, bcast_xz, bcast_yz = broadcast_dynamic_shape_extended(shape_x, shape_y)
        x_new_flat = bcast_xz.broadcast_flat_values(x, inner_dimensions=False)
        y_new_flat = bcast_yz.broadcast_flat_values(y, inner_dimensions=False)
        return op(x_new_flat, y_new_flat)
    x_values = x.flat_values if ragged_tensor.is_ragged(x) else x
    y_values = y.flat_values if ragged_tensor.is_ragged(y) else y
    return op(x_values, y_values)

Binary elementwise assert api handler for RaggedTensors.

    This handles binary assert operations for ragged tensors. Compared with
    `ragged_binary_elementwise_op_impl`, this handler does not compute a ragged
    tensor as output. Instead, it applies the assert operation `op` to input
    tensors based on their ragged shapes and flat_values, and returns the result
    of the assertion operation.

Args:
        op: a binary assert operation on Tensors.
        x: something that can be coerced to a Tensor or RaggedTensor.
        y: something that can be coerced to a Tensor or RaggedTensor.

Returns:
        the result of the assertion operation.


def __init__(self, initializer: tf.keras.initializers.Initializer=tf.keras.initializers.RandomUniform()):
    super().__init__()
    self._initializer = initializer

Initializes a VariableDot layer.

Args:
        initializer: A `tf.keras.initializers.Initializer` which specifies how to
        initialize the values of the parameters.


def _select_forward_and_backward_functions(self, args, possible_gradient_type, executing_eagerly):
    if executing_eagerly:
        input_tangents = forwardprop_util.pack_tangents(args)
    else:
        input_tangents = forwardprop_util.TangentInfo()
    need_gradients_for_jvps = record.should_record_backprop(input_tangents.tangents)
    cache_key = (need_gradients_for_jvps, input_tangents.indices)
    if possible_gradient_type == gradients_util.POSSIBLE_GRADIENT_TYPES_FIRST_ORDER:
        if input_tangents.indices or executing_eagerly:
            functions = self._first_order_tape_functions.get(cache_key, None)
            if functions is None:
                functions = _FirstOrderTapeGradientFunctions(self._func_graph, self._attrs, self._garbage_collector, forwardprop_input_indices=input_tangents.indices, delayed_rewrite_functions=self._delayed_rewrite_functions, need_gradients_for_jvps=need_gradients_for_jvps)
                self._first_order_tape_functions[cache_key] = functions
            return _ForwardBackwardCall(functions, args, input_tangents.tangents, tape_watching=True)
        else:
            return _ForwardBackwardCall(self._delayed_rewrite_functions, args, input_tangents.tangents, tape_watching=True)
    elif possible_gradient_type == gradients_util.POSSIBLE_GRADIENT_TYPES_HIGHER_ORDER:
        functions = self._higher_order_tape_functions.get(cache_key, None)
        if functions is None:
            functions = _HigherOrderTapeGradientFunctions(self._func_graph, self._attrs, self._garbage_collector, forwardprop_input_indices=input_tangents.indices, delayed_rewrite_functions=self._delayed_rewrite_functions, need_gradients_for_jvps=need_gradients_for_jvps)
            self._higher_order_tape_functions[cache_key] = functions
        return _ForwardBackwardCall(functions, args, input_tangents.tangents, tape_watching=True)
    return _ForwardBackwardCall(self._delayed_rewrite_functions, args, input_tangents.tangents, tape_watching=False)

Selects forward and backward functions based on the calling context.

    The forward function computes the "real" function outputs, `self._outputs`,
    and any extra values needed by the corresponding backward function.

Args:
        args: A flat list of Tensors with all of the inputs to the forward
        function (including user-specified and captured inputs).
        possible_gradient_type: One of gradients_util.POSSIBLE_GRADIENT_TYPES_*.
        executing_eagerly: Boolean, the value of context.executing_eagerly().

Returns:
        An object with a `forward` method returning a tuple of (forward_function :
        AtomicFunction, augmented_arguments : List), and a corresponding
        `record` method which takes outputs from the forward function and records
        the operation. forward_function should be called with augmented_arguments.


def get_config(self):
    raise NotImplementedError(f'{self} does not implement get_config()')

Returns the config of the quantizer.

    A quantizer config is a Python dictionary (serializable)
    containing all configuration parameters of the quantizer.
    The same quantizer can be reinstantiated later
    (without any saved state) from this configuration.

    This method is optional if you are just training and executing models,
    exporting to and from SavedModels, or using weight checkpoints.

    This method is required for Keras `model_to_estimator`, saving and
    loading models to HDF5 formats, Keras model cloning, some visualization
    utilities, and exporting models to and from JSON.

Returns:
        Python dictionary.


def output_all_intermediates():
    if _EXPERIMENTAL_OUTPUT_ALL_INTERMEDIATES_OVERRIDE is not None:
        return _EXPERIMENTAL_OUTPUT_ALL_INTERMEDIATES_OVERRIDE
    if in_defun():
        return False
    if control_flow_util.GraphOrParentsInXlaContext(ops.get_default_graph()):
        return False
    if context.context().function_call_options.executor_type == 'SINGLE_THREADED_EXECUTOR':
        return False
    return _is_building_keras_layer()

Whether to output all intermediates of a functional control flow op.

    The default behavior is to output intermediates only when building a Keras
    Layer in graph mode and that too when certain other conditions are met:
    1. We do not output intermediates if the functional control flow op
        is being built inside a FuncGraph which is not a If/While graph. This
        guards against outputting intermediates in eager mode since keras adds
        tensors to a FuncGraph named "keras_graph" in that case. Also because we
        do not output intermediates of tf.function (since this feature is only for
        backwards compatibility) outputting intermediates of functional control
        flow ops built inside tf.function is of no value.
    2. We do not output intermediates when the compilation is using XLA or for a
        TPU.
    3. We do not output intermediates when a single threaded executor is used
        since that does not perform inlining and pruning.

Returns:
        A bool telling whether to output all intermediates.


def __init__(self, reduction=losses_utils.ReductionV2.AUTO, name='hinge'):
    super().__init__(hinge, name=name, reduction=reduction)

Initializes `Hinge` instance.

Args:
        reduction: Type of `tf.keras.losses.Reduction` to apply to
        loss. Default value is `AUTO`. `AUTO` indicates that the reduction
        option will be determined by the usage context. For almost all cases
        this defaults to `SUM_OVER_BATCH_SIZE`. When used with
        `tf.distribute.Strategy`, outside of built-in training loops such as
        `tf.keras` `compile` and `fit`, using `AUTO` or `SUM_OVER_BATCH_SIZE`
        will raise an error. Please see this custom training [tutorial](
        https://www.tensorflow.org/tutorials/distribute/custom_training) for
            more details.
        name: Optional name for the instance. Defaults to 'hinge'.


def __init__(self, options=None):
    if options is not None:
        self._options = copy.deepcopy(options)
    else:
        self._options = {'max_depth': 100, 'min_bytes': 0, 'min_micros': 0, 'min_params': 0, 'min_float_ops': 0, 'min_occurrence': 0, 'order_by': 'name', 'account_type_regexes': ['.*'], 'start_name_regexes': ['.*'], 'trim_name_regexes': [], 'show_name_regexes': ['.*'], 'hide_name_regexes': [], 'account_displayed_op_only': False, 'select': ['micros'], 'step': -1, 'output': 'stdout'}

Constructor.

Args:
        options: Optional initial option dict to start with.


def enable_collective_ops(self, server_def):
    if not server_def:
        raise ValueError('server_def is None.')
    self._collective_ops_server_def = server_def
    if self._context_handle is not None:
        logging.warning('Enabling collective ops after program startup may cause error when accessing previously created tensors.')
        with self._initialize_lock:
            assert self._initialized
            server_def_str = self._collective_ops_server_def.SerializeToString()
            pywrap_tfe.TFE_EnableCollectiveOps(self._context_handle, server_def_str)
            self._initialize_logical_devices()
            self._clear_caches()

Enable distributed collective ops with an appropriate server_def.

Args:
        server_def: A tensorflow::ServerDef proto. Enables execution on remote
        devices.

Raises:
        ValueError: if server_def is None.
        RuntimeError: if this method is not called at program startup.


def create_position_ids_from_inputs_embeds(self, inputs_embeds):
    input_shape = shape_list(inputs_embeds)[:-1]
    sequence_length = input_shape[1]
    position_ids = tf.range(start=self.padding_idx + 1, limit=sequence_length + self.padding_idx + 1, dtype=tf.int64)
    return tf.broadcast_to(tf.expand_dims(position_ids, 0), input_shape)

We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

Args:
        inputs_embeds: tf.Tensor

    Returns: tf.Tensor


def get_graph_debug_info(self, name):
    with c_api_util.tf_buffer() as buffer_:
        pywrap_tfe.TFE_ContextGetGraphDebugInfo(self._handle, name, buffer_)
        proto_data = pywrap_tf_session.TF_GetBuffer(buffer_)
    graph_debug_info = graph_debug_info_pb2.GraphDebugInfo()
    graph_debug_info.ParseFromString(proto_data)
    return graph_debug_info

Get GraphDebugInfo associated with a function from the context.

Args:
        name: function signature name.

Returns:
        The requested GraphDebugInfo.

Raises:
        tf.errors.NotFoundError: if name is not the name of a registered function.


def __call__(self, text_inputs: Union[str, List[str]], **forward_params) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
    return super().__call__(text_inputs, **forward_params)

Generates speech/audio from the inputs. See the [`TextToAudioPipeline`] documentation for more information.

Args:
        text_inputs (`str` or `List[str]`):
            The text(s) to generate.
        forward_params (`dict`, *optional*):
            Parameters passed to the model generation/forward method. `forward_params` are always passed to the
            underlying model.
        generate_kwargs (`dict`, *optional*):
            The dictionary of ad-hoc parametrization of `generate_config` to be used for the generation call. For a
            complete overview of generate, check the [following
            guide](https://huggingface.co/docs/transformers/en/main_classes/text_generation). `generate_kwargs` are
            only passed to the underlying model if the latter is a generative model.

Returns:
        A `dict` or a list of `dict`: The dictionaries have two keys:

        - **audio** (`np.ndarray` of shape `(nb_channels, audio_length)`) -- The generated audio waveform.
        - **sampling_rate** (`int`) -- The sampling rate of the generated audio waveform.


def moving_sum(self: EventSetOrNode, window_length: WindowLength, sampling: Optional[EventSetOrNode]=None) -> EventSetOrNode:
    from temporian.core.operators.window.moving_sum import moving_sum
    return moving_sum(self, window_length=window_length, sampling=sampling)

Computes the sum of values in a sliding window over an
    [`EventSet`][temporian.EventSet].

    For each t in sampling, and for each feature independently, returns at
    time t the sum of values for the feature in the window
    (t - window_length, t].

    `sampling` can't be  specified if a variable `window_length` is
    specified (i.e. if `window_length` is an EventSet).

    If `sampling` is specified or `window_length` is an EventSet, the moving
    window is sampled at each timestamp in them, else it is sampled on the
    input's.

    Missing values (such as NaNs) are ignored.

    If the window does not contain any values (e.g., all the values are
    missing, or the window does not contain any sampling), outputs missing
    values.

Example:
        ```python
        >>> a = tp.event_set(
        ...     timestamps=[0, 1, 2, 5, 6, 7],
        ...     features={"value": [np.nan, 1, 5, 10, 15, 20]},
        ... )

        >>> b = a.moving_sum(tp.duration.seconds(4))
        >>> b
        indexes: ...
            (6 events):
                timestamps: [0. 1. 2. 5. 6. 7.]
                'value': [ 0. 1.  6.  15.  25.  45.]
        ...

        ```

    See [`EventSet.moving_count()`][temporian.EventSet.moving_count] for
    examples of moving window operations with external sampling and indices.

Args:
        window_length: Sliding window's length.
        sampling: Timestamps to sample the sliding window's value at. If not
            provided, timestamps in the input are used.

Returns:
        EventSet containing the moving sum of each feature in the input.


def _clone_sequential_model(model, clone_function, input_tensors=None):
    if not isinstance(model, Sequential):
        raise ValueError(f'Expected `model` argument to be a `Sequential` model instance. Received: model={model}')
    if not callable(clone_function):
        raise ValueError(f'Expected `clone_function` argument to be a callable. Received: clone_function={clone_function}')
    new_layers = [clone_function(layer) for layer in model.layers]
    if isinstance(model._layers[0], InputLayer):
        ref_input_layer = model._layers[0]
        input_name = ref_input_layer.name
        input_batch_shape = ref_input_layer.batch_shape
        input_dtype = ref_input_layer._dtype
    else:
        input_name = None
        input_dtype = None
        input_batch_shape = None
    if input_tensors is not None:
        if isinstance(input_tensors, (list, tuple)):
            if len(input_tensors) != 1:
                raise ValueError('Argument `input_tensors` must contain a single tensor.')
            input_tensors = input_tensors[0]
        if not isinstance(input_tensors, backend.KerasTensor):
            raise ValueError(f'Argument `input_tensors` must be a KerasTensor. Received invalid value: input_tensors={input_tensors}')
        inputs = Input(tensor=input_tensors, name=input_name)
        new_layers = [inputs] + new_layers
    elif input_batch_shape is not None:
        inputs = Input(batch_shape=input_batch_shape, dtype=input_dtype, name=input_name)
        new_layers = [inputs] + new_layers
    cloned_model = Sequential(new_layers, name=model.name, trainable=model.trainable)
    if model.compiled:
        compiled_config = model.get_compile_config()
        cloned_model.compile_from_config(compiled_config)
    return cloned_model

Clone a `Sequential` model instance.

    Model cloning is similar to calling a model on new inputs,
    except that it creates new layers (and thus new weights) instead
    of sharing the weights of the existing layers.

Args:
        model: Instance of `Sequential`.
        input_tensors: optional list of input tensors
            to build the model upon. If not provided,
            placeholders will be created.
        clone_function: callable to be applied on non-input layers in the model.
            By default, it clones the layer (without copying the weights).

Returns:
        An instance of `Sequential` reproducing the behavior
        of the original model, on top of new inputs tensors,
        using newly instantiated weights.


def get(self, key=None, indices=None, name=None):
    if key is None:
        return self._popitem(indices=indices, name=name)
    else:
        return self._pop(key, indices=indices, name=name)

If the key is provided, the associated (key, value) is returned from the staging area.

    If the key is not in the staging area, this method will block until
    the associated (key, value) is inserted.
    If no key is provided and the staging area is ordered,
    the (key, value) with the smallest key will be returned.
    Otherwise, a random (key, value) will be returned.

    If the staging area is empty when this operation executes,
    it will block until there is an element to dequeue.

Args:
        key: Key associated with the required data (Optional)
        indices: Partial list of tensors to retrieve (optional).
                A list of integer or string indices.
                String indices are only valid if the Staging Area
                has names associated with it.
        name: A name for the operation (optional)

Returns:
        The created op


def objects_ids_and_slot_variables_and_paths(graph_view, skip_slot_variables=False):
    trackable_objects, node_paths = graph_view.breadth_first_traversal()
    object_names = object_identity.ObjectIdentityDictionary()
    for obj, path in node_paths.items():
        object_names[obj] = trackable_utils.object_path_to_string(path)
    node_ids = object_identity.ObjectIdentityDictionary()
    for node_id, node in enumerate(trackable_objects):
        node_ids[node] = node_id
    if skip_slot_variables:
        slot_variables = object_identity.ObjectIdentityDictionary()
    else:
        slot_variables = serialize_slot_variables(trackable_objects=trackable_objects, node_ids=node_ids, object_names=object_names)
    return (trackable_objects, node_paths, node_ids, slot_variables, object_names)

Traverse the object graph and list all accessible objects.

    Looks for `Trackable` objects which are dependencies of
    `root_trackable`. Includes slot variables only if the variable they are
    slotting for and the optimizer are dependencies of `root_trackable`
    (i.e. if they would be saved with a checkpoint).

Args:
        graph_view: A GraphView object.
        skip_slot_variables: If True does not return trackables for slot variable.
        Default False.

Returns:
        A tuple of (trackable objects, paths from root for each object,
                object -> node id, slot variables, object_names)


def __init__(self, num_packs=1):
    if num_packs < 0:
        raise ValueError('HierarchicalCopy requires num_packs >= 0, but {} is specified'.format(num_packs))
    super(HierarchicalCopyAllReduce, self).__init__(all_reduce_alg='hierarchical_copy', num_packs=num_packs)

Initializes the object.

Args:
        num_packs: a non-negative integer. The number of packs to split values
        into. If zero, no packing will be done.

Raises:
        ValueError if `num_packs` is negative.


def _annotate_variable_ops(func, graph_def):
    ph_shape_map = {}
    for ph, var in zip(func.graph.internal_captures, func.variables):
        ph_shape_map[ph.name] = var.shape
    name_to_node = {node.name: node for node in graph_def.node}
    for node in graph_def.node:
        if node.op == 'ReadVariableOp' or node.op == 'ResourceGather':
            node_ = node
            while name_to_node[node_.input[0]].op == 'Identity':
                node_ = name_to_node[node_.input[0]]
            ph_name = node_.input[0] + ':0'
            if ph_name in ph_shape_map:
                shape = ph_shape_map[ph_name]
                node.attr['_shape'].shape.CopyFrom(shape.as_proto())
            else:
                raise RuntimeError('Not found in the function captures: {}'.format(ph_name))

Annotates variable operations with custom `_shape` attribute.

    This is required for the converters and shape inference. The graph
    definition is modified in-place.

Args:
        func: Function represented by the graph definition.
        graph_def: Graph definition to be annotated in-place.

Raises:
        RuntimeError: if some shapes cannot be annotated.


def forward(self, input_ids: Optional[torch.Tensor]=None, attention_mask: Optional[torch.Tensor]=None, token_type_ids: Optional[torch.Tensor]=None, head_mask: Optional[torch.Tensor]=None, inputs_embeds: Optional[torch.Tensor]=None, labels: Optional[torch.Tensor]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, return_dict: Optional[bool]=None, **kwargs) -> Union[Tuple[torch.Tensor], NextSentencePredictorOutput]:
    if 'next_sentence_label' in kwargs:
        warnings.warn('The `next_sentence_label` argument is deprecated and will be removed in a future version, use `labels` instead.', FutureWarning)
        labels = kwargs.pop('next_sentence_label')
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
    outputs = self.nezha(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, head_mask=head_mask, inputs_embeds=inputs_embeds, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict)
    pooled_output = outputs[1]
    seq_relationship_scores = self.cls(pooled_output)
    next_sentence_loss = None
    if labels is not None:
        loss_fct = CrossEntropyLoss()
        next_sentence_loss = loss_fct(seq_relationship_scores.view(-1, 2), labels.view(-1))
    if not return_dict:
        output = (seq_relationship_scores,) + outputs[2:]
        return (next_sentence_loss,) + output if next_sentence_loss is not None else output
    return NextSentencePredictorOutput(loss=next_sentence_loss, logits=seq_relationship_scores, hidden_states=outputs.hidden_states, attentions=outputs.attentions)

labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
    (see `input_ids` docstring). Indices should be in `[0, 1]`:

    - 0 indicates sequence B is a continuation of sequence A,
    - 1 indicates sequence B is a random sequence.

Returns:

Example:
    ```python
    >>> from transformers import AutoTokenizer, NezhaForNextSentencePrediction
    >>> import torch

    >>> tokenizer = AutoTokenizer.from_pretrained("sijunhe/nezha-cn-base")
    >>> model = NezhaForNextSentencePrediction.from_pretrained("sijunhe/nezha-cn-base")

    >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
    >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
    >>> encoding = tokenizer(prompt, next_sentence, return_tensors="pt")

    >>> outputs = model(**encoding, labels=torch.LongTensor([1]))
    >>> logits = outputs.logits
    >>> assert logits[0, 0] < logits[0, 1]  # next sentence was random
    ```


def GetIamPolicy(self, request, global_params=None):
    config = self.GetMethodConfig('GetIamPolicy')
    return self._RunMethod(config, request, global_params=global_params)

Gets the access control policy for a resource. Returns an empty policy if the resource exists and does not have a policy set.

Args:
        request: (BigqueryTablesGetIamPolicyRequest) input message
        global_params: (StandardQueryParameters, default: None) global arguments

Returns:
        (Policy) The response message.


def __init__(self, debug_dump, config):
    self._debug_dump = debug_dump
    self._evaluator = evaluator.ExpressionEvaluator(self._debug_dump)
    self._tensor_filters = {}
    self._build_argument_parsers(config)
    config.set_callback('graph_recursion_depth', self._build_argument_parsers)

DebugAnalyzer constructor.

Args:
        debug_dump: A DebugDumpDir object.
        config: A `cli_config.CLIConfig` object that carries user-facing
        configurations.


def preface_inference(f):

    @functools.wraps(f)
    def wrapper(self, *args, **kwargs):
        self._preface_inference()
        return f(self, *args, **kwargs)
    return wrapper

Wraps given function with things to run before every inference call.

Args:
        f: The method of `EnergyInference` to wrap.

Returns:
        wrapper: The wrapped function.


def set_clang_compiler_path(environ_cp):
    default_clang_path = '/usr/lib/llvm-18/bin/clang'
    if not os.path.exists(default_clang_path):
        default_clang_path = '/usr/lib/llvm-17/bin/clang'
        if not os.path.exists(default_clang_path):
            default_clang_path = '/usr/lib/llvm-16/bin/clang'
        if not os.path.exists(default_clang_path):
            default_clang_path = shutil.which('clang') or ''
    clang_compiler_path = prompt_loop_or_load_from_env(environ_cp, var_name='CLANG_COMPILER_PATH', var_default=default_clang_path, ask_for_var='Please specify the path to clang executable.', check_success=os.path.exists, resolve_symlinks=True, error_msg='Invalid clang path. %s cannot be found. Note that TensorFlow now requires clang to compile. You may override this behavior by setting TF_NEED_CLANG=0')
    write_action_env_to_bazelrc('CLANG_COMPILER_PATH', clang_compiler_path)
    write_to_bazelrc('build --repo_env=CC=%s' % clang_compiler_path)
    write_to_bazelrc('build --repo_env=BAZEL_COMPILER=%s' % clang_compiler_path)
    return clang_compiler_path

Set CLANG_COMPILER_PATH and environment variables.

    Loop over user prompts for clang path until receiving a valid response.
    Default is used if no input is given. Set CLANG_COMPILER_PATH and write
    environment variables CC and BAZEL_COMPILER to .bazelrc.

Args:
        environ_cp: (Dict) copy of the os.environ.

Returns:
        string value for clang_compiler_path.


def post_process_video_grounding(self, logits, video_durations):
    start, end = (round(logits.tolist()[0][0] * video_durations, 1), round(logits.tolist()[0][1] * video_durations, 1))
    return (start, end)

Compute the time of the video.

Args:
        logits (`torch.Tensor`):
            The logits output of TvpForVideoGrounding.
        video_durations (`float`):
            The video's duration.

Returns:
        start (`float`):
            The start time of the video.
        end (`float`):
            The end time of the video.


def period_from_dict(period: Dict[int, List[int]]) -> dateslib.PeriodTensor:
    amount = period['frequency']
    period_type = period_pb2.PeriodType.Name(period['type'])
    return dateslib.PeriodTensor(amount, dateslib.PeriodType[period_type])

Utility to convert a dictionary of periods to a PeriodTensor.

Args:
        period: A dictionary with keys "type" (which corresponds to the proto type
        of the period (see `period_pb2.Period`)) and "frequency".

Returns:
        An instance of the `PeriodTensor`.


def convert_padding(padding, expected_length=4):
    explicit_paddings = []
    if padding == 'EXPLICIT':
        raise ValueError("'EXPLICIT' is not a valid value for `padding`. To use explicit padding, `padding` must be a list.")
    if isinstance(padding, (list, tuple)):
        for i, dim_paddings in enumerate(padding):
            if not isinstance(dim_paddings, (list, tuple)):
                raise ValueError(f'When `padding` is a list, each element of `padding` must be a list/tuple of size 2. Received: padding={padding} with element at index {i} of type {type(dim_paddings)}')
            if len(dim_paddings) != 2:
                raise ValueError(f'When `padding` is a list, each element of `padding` must be a list/tuple of size 2. Received: padding={padding} with element at index {i} of size {len(dim_paddings)}')
            explicit_paddings.extend(dim_paddings)
        if len(padding) != expected_length:
            raise ValueError(f'When padding is a list, it must be of size {expected_length}. Received: padding={padding} of size {len(padding)}')
        padding = 'EXPLICIT'
    return (padding, explicit_paddings)

Converts Python padding to C++ padding for ops which take EXPLICIT padding.

Args:
        padding: the `padding` argument for a Python op which supports EXPLICIT
        padding.
        expected_length: Expected number of entries in the padding list when
        explicit padding is used.

Returns:
        (padding, explicit_paddings) pair, which should be passed as attributes to a
        C++ op.

Raises:
        ValueError: If padding is invalid.


def _create_saveables_by_attribute_name(self, saveable_factories):
    named_saveables = {}
    existing_restore_ops = []
    created_compat_names = set()
    for serialized_tensor in self.object_proto.attributes:
        if context.executing_eagerly():
            existing_op = None
        else:
            existing_op = self._checkpoint.restore_ops_by_name.get(serialized_tensor.checkpoint_key, None)
        if existing_op is not None:
            existing_restore_ops.append(existing_op)
            continue
        if any((serialized_tensor.name.startswith(name) for name in created_compat_names)):
            continue
        saveables_cache = self._checkpoint.saveables_cache
        if saveables_cache is None:
            saveable = None
        else:
            saveable_list = saveables_cache.get(self.trackable, {}).get(serialized_tensor.name, (None,))
            if len(saveable_list) == 1:
                saveable, = saveable_list
            else:
                saveable = None
        if saveable is not None:
            if serialized_tensor.checkpoint_key not in saveable.name:
                saveable = None
                del saveables_cache[self.trackable]
        if saveable is None:
            saveable = _get_saveable_from_factory(saveable_factories, serialized_tensor, created_compat_names)
            if saveable is None:
                self._checkpoint.unused_attributes.setdefault(self._proto_id, []).append(serialized_tensor.name)
                continue
            if saveables_cache is not None:
                saveables_cache.setdefault(self.trackable, {})[serialized_tensor.name] = [saveable]
        named_saveables[serialized_tensor.checkpoint_key] = saveable
    return (existing_restore_ops, named_saveables)

Creates or caches SaveableObjects by matching the attribute names.

    The attribute name keys in the `saveable_factories` is used to find the
    corresponding attribute in the object proto. Attributes contain checkpoint
    keys which are passed to the factory function to generate the
    SaveableObject.

Args:
        saveable_factories: a dict mapping attribute name to a callable factory
        function that produces a SaveableObject.

Returns:
        A tuple of (
        existing_restore_ops: list,
        named_saveables: dict)


def decode_csv(records, record_defaults, field_delim=',', use_quote_delim=True, name=None, na_value='', select_cols=None):
    return decode_csv_v2(records, record_defaults, field_delim, use_quote_delim, na_value, select_cols, name)

Convert CSV records to tensors. Each column maps to one tensor.

    RFC 4180 format is expected for the CSV records.
    (https://tools.ietf.org/html/rfc4180)
    Note that we allow leading and trailing spaces with int or float field.

Args:
        records: A `Tensor` of type `string`.
        Each string is a record/row in the csv and all records should have
        the same format.
        record_defaults: A list of `Tensor` objects with specific types.
        Acceptable types are `float32`, `float64`, `int32`, `int64`, `string`.
        One tensor per column of the input record, with either a
        scalar default value for that column or an empty vector if the column is
        required.
        field_delim: An optional `string`. Defaults to `","`.
        char delimiter to separate fields in a record.
        use_quote_delim: An optional `bool`. Defaults to `True`.
        If false, treats double quotation marks as regular
        characters inside of the string fields (ignoring RFC 4180, Section 2,
        Bullet 5).
        name: A name for the operation (optional).
        na_value: Additional string to recognize as NA/NaN.
        select_cols: Optional sorted list of column indices to select. If specified,
        only this subset of columns will be parsed and returned.

Returns:
        A list of `Tensor` objects. Has the same type as `record_defaults`.
        Each tensor will have the same shape as records.

Raises:
        ValueError: If any of the arguments is malformed.


def ragged_tensor_to_string(rt, summarize=None):
    if summarize is not None and summarize != -1 and (not (isinstance(summarize, int) and summarize > 0)):
        raise ValueError('Expected summarize to be -1 or a positive int, got %r' % summarize)
    with ops.name_scope(None, 'AsString', [rt]):
        rt = ragged_tensor.convert_to_tensor_or_ragged_tensor(rt)
        if rt.shape.rank is None:
            raise ValueError('RaggedTensor to_string requires that rt.shape.rank is not None.')
        if rt.dtype == dtypes.string:
            escaped = string_ops.regex_replace(rt.flat_values, "(['\\\\])", '\\\\\\1')
            str_t = rt.with_flat_values("'" + escaped + "'")
        else:
            str_t = rt.with_flat_values(string_ops.as_string(rt.flat_values))
        return _ragged_tensor_to_string(str_t, summarize)

Returns a scalar string tensor with the contents of a RaggedTensor.

    Requires that `rt.shape.rank` is not `None`.

    Note: this converts the entire `RaggedTensor` into a single string scalar.
    If you want to convert individual elements, use `tf.strings.as_string(rt)`.

    >>> rt1 = tf.ragged.constant([[1, 2, 3], [4, 5]])
    >>> ragged_tensor_to_string(rt1).numpy()
    b'[[1, 2, 3], [4, 5]]'

    >>> rt2 = tf.ragged.constant([[['a'], ['b', 'c']], [['d', 'e', 'f'], []]])
    >>> ragged_tensor_to_string(rt2).numpy()
    b"[[['a'], ['b', 'c']], [['d', 'e', 'f'], []]]"

    >>> rt3 = tf.ragged.constant([[1], [2, 3, 4, 5, 6], [], [], [7], [8, 9]])
    >>> ragged_tensor_to_string(rt3, summarize=2).numpy()
    b'[[1], [2, 3, ..., 5, 6], ..., [7], [8, 9]]'

Args:
        rt: The RaggedTensor that should be converted to a string.
        summarize: If specified, then only the first and last `summarize` elements
        within each dimension are included in the string. If `-1` or `None`, then
        all elements are included.


def _GetMaxSizeFromNestedMaximumIterations(value, while_ctxt):
    value_name = value.name
    curr_ctxt = ops.get_default_graph()._get_control_flow_context()
    curr_ctxt_name = curr_ctxt.name if curr_ctxt is not None else ''
    max_size = constant_op.constant(1)
    while while_ctxt not in (None, curr_ctxt):
        max_iter = while_ctxt.maximum_iterations
        if max_iter is None:
            raise ValueError("Cannot create a gradient accumulator for tensor '%s' inside XLA while_loop because maximum_iterations was not passed to the tf.while_loop call ('%s')." % (value_name, while_ctxt.name))
        max_iter_ctxt = max_iter.op._get_control_flow_context()
        if util.IsContainingContext(curr_ctxt, max_iter_ctxt):
            max_size *= max_iter
        else:
            const_max_iter = tensor_util.constant_value(max_iter)
            if const_max_iter is None:
                raise ValueError("Cannot create a gradient accumulator for tensor '%s' inside XLA while_loop. maximum_iterations tensor '%s' for while_loop context '%s' must be statically known (e.g. a constant value or known shape dimension), or be defined at or outside the while loop context '%s' (currently defined in '%s')." % (value_name, max_iter.name, while_ctxt.name, curr_ctxt_name, max_iter_ctxt.name))
            max_size *= const_max_iter
        while_ctxt = util.GetContainingWhileContext(while_ctxt.outer_context, stop_ctxt=curr_ctxt)
    return max_size

Calculate a max_size for use by stack ops inside an XLA while_loop.

Args:
        value: The value inside the while_loop forward context.  Used for printing
        error messages.
        while_ctxt: The forward context inside which value resides.  This does not
        always match the value's immediate context, as `value` may be inside e.g.
        a cond context inside the while_loop.

Returns:
        A tensor containing the `max_size` to feed to a Stack initializer.

Raises:
        ValueError: If `value` is nested inside a `while_loop` that either
        lacks a `maximum_iterations` parameter, or the `maximum_iterations`
        parameter:

        - is inside a `while_loop` that is a parent of the calling context, and
        - cannot be evaluated at graph build time to a constant.


def _IsFlag(argument):
    return _IsSingleCharFlag(argument) or _IsMultiCharFlag(argument)

Determines if the argument is a flag argument.

    If it starts with a hyphen and isn't a negative number, it's a flag.

Args:
        argument: A command line argument that may or may not be a flag.

Returns:
        A boolean indicating whether the argument is a flag.


def get_resource(self, uri: str) -> Optional[message.Message]:
    for collection in (self.structure_definitions, self.search_parameters, self.code_systems, self.value_sets):
        resource = collection.get(uri)
        if resource is not None:
            return resource
    return None

Retrieves a protocol buffer representation of the given resource.

Args:
        uri: The URI of the resource to retrieve.

Returns:
        Protocol buffer for the resource or `None` if the `uri` can not be found.


def __init__(self, name, shape, dtype, variable_list, partitions):
    if not isinstance(variable_list, (list, tuple)):
        raise TypeError('variable_list is not a list or tuple: %s' % variable_list)
    if not isinstance(partitions, (list, tuple)):
        raise TypeError('partitions is not a list or tuple: %s' % partitions)
    if not all((p >= 1 for p in partitions)):
        raise ValueError('partition values must be positive: %s' % partitions)
    if not variable_list:
        raise ValueError('variable_list may not be empty')
    for v in variable_list:
        if not all((v._get_save_slice_info() is not None for v in variable_list)):
            raise ValueError('All variables must have a save_slice_info available: %s' % [v.name for v in variable_list])
        if len(shape) != len(partitions):
            raise ValueError('len(shape) != len(partitions): %s vs. %s' % (shape, partitions))
        if v._get_save_slice_info().full_shape != shape:
            raise ValueError("All variables' full shapes must match shape: %s; but full shapes were: %s" % (shape, str([v._get_save_slice_info().full_shape])))
    self._variable_list = sorted(variable_list, key=lambda v: v._get_save_slice_info().var_offset)
    self._name = name
    self._shape = shape
    self._dtype = dtype
    self._partitions = partitions
    self._as_tensor = None

Creates a new partitioned variable wrapper.

    Variables passed via the variable_list must contain a save_slice_info
    field.  Concatenation and iteration is in lexicographic order according
    to the var_offset property of the save_slice_info.

Args:
        name: String. Overall name of the variables.
        shape: List of integers.  Overall shape of the variables.
        dtype: Type of the variables.
        variable_list: List of `Variable` that comprise this partitioned variable.
        partitions: List of integers.  Number of partitions for each dimension.

Raises:
        TypeError: If `variable_list` is not a list of `Variable` objects, or
        `partitions` is not a list.
        ValueError: If `variable_list` is empty, or the `Variable` shape
        information does not match `shape`, or `partitions` has invalid values.


def serialized_tensors_to_saveable_cache(serialized_tensors):
    saveables_cache = object_identity.ObjectIdentityWeakKeyDictionary()
    for obj, tensor_dict in serialized_tensors.items():
        if not tensor_dict:
            continue
        if isinstance(obj, SaveableCompatibilityConverter):
            trackable_obj = obj.obj
            saveables_cache[trackable_obj] = {}
            for saveable in obj.saveables:
                local_name = trackable_utils.extract_local_name(saveable.name)
                saveables_cache[trackable_obj][local_name] = [saveable]
            continue
        specs = []
        local_names = []
        prefix = saveable_compat.get_saveable_name(obj) or ''
        for checkpoint_key, maybe_tensor in tensor_dict.items():
            if not isinstance(maybe_tensor, dict):
                maybe_tensor = {'': maybe_tensor}
            for slice_spec, tensor in maybe_tensor.items():
                if isinstance(tensor, saveable_object.SaveSpec):
                    specs.append(tensor)
                else:
                    specs.append(saveable_object.SaveSpec(tensor, slice_spec, checkpoint_key))
            local_names.append(trackable_utils.extract_local_name(checkpoint_key, prefix))
        object_name = trackable_utils.extract_object_name(next(iter(tensor_dict.keys())))
        saveables_cache[obj] = {trackable_utils.SERIALIZE_TO_TENSORS_NAME: [TrackableSaveable(obj, specs, object_name, local_names=local_names, prefix=prefix)]}
    return saveables_cache

Converts a tensor dict to a SaveableObject cache.

Args:
        serialized_tensors: Map from Trackable to a tensor dict. The tensor dict
        maps checkpoint key (-> slice_spec) -> Tensor

Returns:
        A dict mapping Trackable objects to a map from local savable name to
        SaveableObject.


def sequence_like(instance, args):
    if _is_mutable_mapping(instance):
        result = dict(zip(_tf_core_sorted(instance), args))
        instance_type = type(instance)
        if instance_type == _collections.defaultdict:
            d = _collections.defaultdict(instance.default_factory)
        else:
            d = instance_type()
        for key in instance:
            d[key] = result[key]
        return d
    elif _is_mapping(instance):
        result = dict(zip(_tf_core_sorted(instance), args))
        instance_type = type(instance)
        if not getattr(instance_type, '__supported_by_tf_nest__', False):
            tf_logging.log_first_n(tf_logging.WARN, 'Mapping types may not work well with tf.nest. Prefer using MutableMapping for {}'.format(instance_type), 1)
        try:
            return instance_type(((key, result[key]) for key in instance))
        except TypeError as err:
            raise TypeError('Error creating an object of type {} like {}. Note that it must accept a single positional argument representing an iterable of key-value pairs, in addition to self. Cause: {}'.format(type(instance), instance, err))
    elif _is_mapping_view(instance):
        return list(args)
    elif is_namedtuple(instance) or _is_attrs(instance):
        if isinstance(instance, _wrapt.ObjectProxy):
            instance_type = type(instance.__wrapped__)
        else:
            instance_type = type(instance)
        return instance_type(*args)
    elif _is_composite_tensor(instance):
        assert len(args) == 1
        spec = instance._type_spec
        return spec._from_components(args[0])
    elif _is_type_spec(instance):
        assert len(args) == 1
        return instance._from_components(args[0])
    elif isinstance(instance, range):
        return sequence_like(list(instance), args)
    elif isinstance(instance, _wrapt.ObjectProxy):
        return type(instance)(sequence_like(instance.__wrapped__, args))
    elif isinstance(instance, CustomNestProtocol):
        metadata = instance.__tf_flatten__()[0]
        return instance.__tf_unflatten__(metadata, tuple(args))
    else:
        return type(instance)(args)

Converts the sequence `args` to the same type as `instance`.

Args:
        instance: an instance of `tuple`, `list`, `namedtuple`, `dict`,
        `collections.OrderedDict`, or `composite_tensor.Composite_Tensor` or
        `type_spec.TypeSpec`.
        args: items to be converted to the `instance` type.

Returns:
        `args` with the type of `instance`.


def seek(self, offset=None, whence=0, position=None):
    self._preread_check()
    if offset is None and position is None:
        raise TypeError('seek(): offset argument required')
    if offset is not None and position is not None:
        raise TypeError('seek(): offset and position may not be set simultaneously.')
    if position is not None:
        offset = position
    if whence == 0:
        pass
    elif whence == 1:
        offset += self.tell()
    elif whence == 2:
        offset += self.size()
    else:
        raise errors.InvalidArgumentError(None, None, 'Invalid whence argument: {}. Valid values are 0, 1, or 2.'.format(whence))
    self._read_buf.seek(offset)

Seeks to the offset in the file.

Args:
        offset: The byte count relative to the whence argument.
        whence: Valid values for whence are:
        0: start of the file (default)
        1: relative to the current position of the file
        2: relative to the end of file. `offset` is usually negative.


def __init__(self, do_lower_case=False, never_split=None, normalize_text=True, trim_whitespace=False):
    self.do_lower_case = do_lower_case
    self.never_split = never_split if never_split is not None else []
    self.normalize_text = normalize_text
    self.trim_whitespace = trim_whitespace
    try:
        import rhoknp
    except ImportError:
        raise ImportError('You need to install rhoknp to use JumanppTokenizer. See https://github.com/ku-nlp/rhoknp for installation.')
    self.juman = rhoknp.Jumanpp()

Constructs a JumanppTokenizer.

Args:
        **do_lower_case**: (*optional*) boolean (default True)
            Whether to lowercase the input.
        **never_split**: (*optional*) list of str
            Kept for backward compatibility purposes. Now implemented directly at the base class level (see
            [`PreTrainedTokenizer.tokenize`]) List of tokens not to split.
        **normalize_text**: (*optional*) boolean (default True)
            Whether to apply unicode normalization to text before tokenization.
        **trim_whitespace**: (*optional*) boolean (default False)
            Whether to trim all whitespace, tab, newline from tokens.


def Decrement(self, key):
    with self._lock:
        if _IsHashable(key):
            if key in self._d:
                if self._d[key] > 1:
                    self._d[key] -= 1
                else:
                    del self._d[key]
        else:
            try:
                i = self._unhashable_items.index(key)
                if self._unhashable_counts[i] > 1:
                    self._unhashable_counts[i] -= 1
                else:
                    del self._unhashable_counts[i]
                    del self._unhashable_items[i]
            except ValueError:
                pass

Atomically decrement a count by 1. Expunge the item if the count is 0.

    If the item is not present, has no effect.

Args:
        key: the key being counted.


def _compute_linear_scaling_rope_parameters(config: Optional[PretrainedConfig]=None, device: Optional['torch.device']=None, seq_len: Optional[int]=None, **rope_kwargs) -> tuple['torch.Tensor', float]:
    if config is not None and len(rope_kwargs) > 0:
        raise ValueError(f'Unexpected arguments: `**rope_kwargs` and `config` are mutually exclusive in `_compute_linear_scaling_rope_parameters`, got `rope_kwargs`={rope_kwargs} and `config`={config}')
    if len(rope_kwargs) > 0:
        factor = rope_kwargs['factor']
    elif config is not None:
        factor = config.rope_scaling['factor']
    inv_freq, attention_factor = _compute_default_rope_parameters(config, device, seq_len, **rope_kwargs)
    inv_freq /= factor
    return (inv_freq, attention_factor)

Computes the inverse frequencies with linear scaling. Credits to the Reddit user /u/kaiokendev

Args:
        config ([`~transformers.PretrainedConfig`]):
            The model configuration.
        device (`torch.device`):
            The device to use for initialization of the inverse frequencies.
        seq_len (`int`, *optional*):
            The current sequence length. Unused for this type of RoPE.
        rope_kwargs (`Dict`, *optional*):
            BC compatibility with the previous RoPE class instantiation, will be removed in v4.45.

Returns:
        Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
        post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).


def nl_to_sql_query(input_statement: str) -> str:
    api_key = os.getenv('GOOGLE_API_KEY')
    if api_key is None:
        raise RuntimeError('Environment variable GOOGLE_API_KEY is not set.')
    table_map_prompt, table_map = get_table_map_prompt()
    model = ChatGoogleGenerativeAI(model='gemini-pro')
    select_dataset_model = get_invocation_steps(DEFINED_PROMPTS['select_dataset'], model)
    generate_sql_model = get_invocation_steps(DEFINED_PROMPTS['generate_sql'], model)
    select_dataset_res = select_dataset_model.invoke({'question': input_statement, 'table_map': table_map_prompt})
    dataset_key = select_dataset_res.split(':')[-1].strip()
    dataset_metadata = table_map[dataset_key]
    generate_sql_res = generate_sql_model.invoke({'question': input_statement, 'table': dataset_metadata['uri'], 'columns': dataset_metadata['columns'], 'few_shot_examples': few_shots, 'dims': dataset_metadata['dims'], 'latitude_dim': dataset_metadata['latitude_dim'], 'latitude_range': dataset_metadata['latitude_range'], 'longitude_dim': dataset_metadata['longitude_dim'], 'longitude_range': dataset_metadata['longitude_range']})
    sql_query = generate_sql_res[11:-1]
    return sql_query

Convert a natural language query to SQL.

    Parameters:
    - input_statement (str): The natural language query.

Returns:
    - str: The generated SQL query.


def _pyval_find_struct_keys_and_depth(pyval, keys):
    if isinstance(pyval, dict):
        keys.update(pyval.keys())
        return 0
    elif isinstance(pyval, (list, tuple)):
        depth = None
        for child in pyval:
            child_depth = _pyval_find_struct_keys_and_depth(child, keys)
            if child_depth is not None:
                if depth is None:
                    depth = child_depth + 1
                elif depth != child_depth + 1:
                    raise ValueError('Inconsistent depth of dictionaries')
        return depth
    else:
        return None

Finds the keys & depth of nested dictionaries in `pyval`.

Args:
        pyval: A nested structure of lists, tuples, and dictionaries.
        keys: (output parameter) A set, which will be updated with any keys that are
        found in the nested dictionaries.

Returns:
        The nesting depth of dictionaries in `pyval`, or `None` if `pyval` does
        not contain any dictionaries.

Raises:
        ValueError: If dictionaries have inconsistent depth.


def _maybe_stacked(self, cache, inp):
    if inp in cache:
        return cache[inp]
    if not self.op_is_inside_loop(inp.op):
        return False
    op = inp.op
    output = False
    if op.type in ['OnesLike', 'Shape', 'Rank', 'ShapeN', 'ZerosLike', 'TensorArrayV3', 'TensorArraySizeV3']:
        output = False
    elif _is_stateful_pfor_op(op):
        output = True
    elif op.type == 'Exit':
        output = True
    else:
        for t in op.inputs:
            if self._maybe_stacked(cache, t):
                output = True
                break
    cache[inp] = output
    return output

Heuristic to figure out if the converting inp leads to a stacked value.

Args:
        cache: map from Tensor to boolean indicating stacked/unstacked.
        inp: input Tensor.

Returns:
        True if `inp` could get stacked. If the function returns False, the
        converted value should be guaranteed to be unstacked. If returning True,
        it may or may not be stacked.


def _get_scopes(state, names: Sequence[str], ctx) -> Sequence[abstract.InterpreterClass | abstract.InterpreterFunction]:
    scopes = []
    for name in names:
        prev = scopes[-1] if scopes else None
        if not prev:
            try:
                _, var = ctx.vm.load_global(state, name)
            except KeyError:
                break
        elif isinstance(prev, abstract.InterpreterClass):
            if name in prev.members:
                var = prev.members[name]
            else:
                break
        else:
            assert isinstance(prev, abstract.InterpreterFunction)
            if prev.last_frame and name in prev.last_frame.f_locals.pyval:
                var = prev.last_frame.f_locals.pyval[name]
            else:
                break
        try:
            scopes.append(abstract_utils.get_atomic_value(var, (abstract.InterpreterClass, abstract.InterpreterFunction)))
        except abstract_utils.ConversionError:
            break
    return scopes

Gets the class or function objects for a sequence of nested scope names.

    For example, if the code under analysis is:
        class Foo:
        def f(self):
        def g(): ...
    then when called with ['Foo', 'f', 'g'], this method returns
    [InterpreterClass(Foo), InterpreterFunction(f), InterpreterFunction(g)].

Args:
        state: The current state.
        names: A sequence of names for consecutive nested scopes in the module under
        analysis. Must start with a module-level name.
        ctx: The current context.

Returns:
        The class or function object corresponding to each name in 'names'.


def forward(self, hidden_states: torch.Tensor, original_hidden_states: Optional[torch.Tensor]=None, layer_idx: Optional[int]=None, attention_mask: Optional[torch.Tensor]=None, causal_mask: Optional[torch.Tensor]=None, past_key_value: Optional[ZambaHybridDynamicCache]=None, output_attentions: Optional[bool]=False, use_cache: Optional[bool]=False, cache_position: Optional[torch.LongTensor]=None, transformer_hidden_states: Optional[torch.Tensor]=None, **kwargs) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
    residual = hidden_states
    hidden_states = hidden_states + transformer_hidden_states if transformer_hidden_states is not None else hidden_states
    hidden_states = self.input_layernorm(hidden_states)
    hidden_states = self.mamba(hidden_states=hidden_states, cache_params=past_key_value, attention_mask=attention_mask)
    self_attn_weights = None
    hidden_states = residual + hidden_states
    outputs = (hidden_states,)
    if output_attentions:
        outputs += (self_attn_weights,)
    if use_cache:
        outputs += (past_key_value,)
    return outputs

Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
    attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
        `(batch, sequence_length)` where padding elements are indicated by 0.
    past_key_value (`ZambaHybridDynamicCache`, *optional*): cached past key and value projection states
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under
        returned tensors for more detail.
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
    cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
        Indices depicting the position of the input sequence tokens in the sequence.


def _process_new(self, feed_item):
    creative = {'advertiserId': feed_item.get(FieldMap.ADVERTISER_ID, None), 'name': feed_item.get(FieldMap.CREATIVE_NAME, None), 'active': True}
    self._associate_third_party_urls(feed_item, creative)
    self._associate_click_tags(feed_item, creative)
    if feed_item.get(FieldMap.CREATIVE_TYPE, None) == 'VIDEO':
        creative['type'] = 'INSTREAM_VIDEO'
        for association in feed_item.get('associations', []):
            identifier = self.creative_asset_dao.get_identifier(association, self._creative_asset_feed)
            creative['creativeAssets'] = [{'assetIdentifier': identifier, 'role': 'PARENT_VIDEO'}]
        del creative['active']
    elif feed_item.get(FieldMap.CREATIVE_TYPE, None) == 'DISPLAY':
        creative['type'] = 'DISPLAY'
        if feed_item.get(FieldMap.CREATIVE_WIDTH, None) and feed_item.get(FieldMap.CREATIVE_HEIGHT, None):
            creative['size'] = {'kind': 'dfareporting#size', 'width': feed_item.get(FieldMap.CREATIVE_WIDTH, None), 'height': feed_item.get(FieldMap.CREATIVE_HEIGHT, None)}
        for association in feed_item.get('associations', []):
            identifier = self.creative_asset_dao.get_identifier(association, self._creative_asset_feed)
            creative['creativeAssets'] = [{'assetIdentifier': identifier, 'role': 'PRIMARY'}]
            if feed_item.get(FieldMap.CREATIVE_BACKUP_ASSET_ID, None) and feed_item.get(FieldMap.CREATIVE_BACKUP_ASSET_ID, None) != '':
                backup_identifier = self.creative_asset_dao.get_backup_identifier(association, self._creative_asset_feed)
                creative['backupImageReportingLabel'] = feed_item.get(FieldMap.CREATIVE_BACKUP_NAME, None)
                backup_features = feed_item.get(FieldMap.BACKUP_IMAGE_FEATURES, None)
                if backup_features != None or backup_features != '':
                    features = backup_features.split(',')
                    creative['backupImageFeatures'] = features
                creative['backupImageTargetWindow'] = {'targetWindowOption': feed_item.get(FieldMap.BACKUP_IMAGE_TARGET_WINDOW_OPTION, None), 'customHtml': feed_item.get(FieldMap.BACKUP_IMAGE_CUSTOM_HTML, None)}
                lp = self.landing_page_dao.get(feed_item, column_name=FieldMap.BACKUP_IMAGE_CLICK_THROUGH_LANDING_PAGE_ID)
                creative['backupImageClickThroughUrl'] = {'landingPageId': feed_item.get(FieldMap.BACKUP_IMAGE_CLICK_THROUGH_LANDING_PAGE_ID) if not lp else lp['id']}
                creative['creativeAssets'].append({'assetIdentifier': backup_identifier, 'role': 'BACKUP_IMAGE'})
        del creative['active']
    else:
        raise Exception('Only video and display are supported at the moment!')
    return creative

Creates a new creative DCM object from a feed item representing an creative from the Bulkdozer feed.

    This function simply creates the object to be inserted later by the BaseDAO
    object.

Args:
        feed_item: Feed item representing the creative from the Bulkdozer feed.

Returns:
        A creative object ready to be inserted in DCM through the API.


def _event_set_dict_to_event_set(input: Dict[str, Any], schema: Schema, timestamp_key: str) -> Iterator[FeatureItemWithIdx]:
    timestamps = input[timestamp_key]
    if not isinstance(timestamps, np.ndarray) or timestamps.dtype.type != np.float64:
        raise ValueError(f'Timestamp with value {timestamps} is expected to be np.float64 numpy array, but has dtype {type(timestamps)} instead.')
    index = []
    for index_schema in schema.indexes:
        expected_type = tp_dtype_to_py_type(index_schema.dtype)
        src_value = input[index_schema.name]
        if not isinstance(src_value, expected_type):
            raise ValueError(f'Index "{index_schema.name}" with value "{src_value}" is expected to be of type {expected_type} (since the Temporian  dtype is {index_schema.dtype}) but type {type(src_value)} was found.')
        index.append(src_value)
    index_tuple = tuple(index)
    for feature_idx, feature_schema in enumerate(schema.features):
        expected_dtype = tp_dtype_to_np_dtype(feature_schema.dtype)
        src_value = input[feature_schema.name]
        if not isinstance(src_value, np.ndarray) or src_value.dtype.type != expected_dtype:
            if isinstance(src_value, np.ndarray):
                effective_type = src_value.dtype.type
            else:
                effective_type = type(src_value)
            raise ValueError(f'Feature "{feature_schema.name}" with value "{src_value}" is expected to be a numpy array with dtype {expected_dtype} (since the Temporian dtype is {feature_schema.dtype}) but numpy dtype {effective_type} was found.')
        yield (index_tuple, (timestamps, src_value, feature_idx))


def GetContainingWhileContext(ctxt, stop_ctxt=None):
    while ctxt:
        if ctxt.IsWhileContext() or ctxt == stop_ctxt:
            return ctxt
        ctxt = ctxt.outer_context
    return None

Returns the first ancestor WhileContext of `ctxt`.

    Returns `ctxt` if `ctxt` is a WhileContext, or None if `ctxt` is not in a
    while loop.

Args:
        ctxt: ControlFlowContext
        stop_ctxt: ControlFlowContext, optional. If provided, the search will end
        if it sees stop_ctxt.

Returns:
        `ctxt` if `ctxt` is a WhileContext, the most nested WhileContext containing
        `ctxt`, or None if `ctxt` is not in a while loop.  If `stop_ctxt` is not
        `None`, this returns `ctxt` if it matches `stop_ctxt` in its traversal.


def on_train_batch_end(self, batch, logs=None):
    if self._should_call_train_batch_hooks:
        self._call_batch_hook(ModeKeys.TRAIN, 'end', batch, logs=logs)

Calls the `on_train_batch_end` methods of its callbacks.

Args:
        batch: Integer, index of batch within the current epoch.
        logs: Dict. Aggregated metric results up until this batch.


Return dist-belief momentum values.

    Return values been generated from the dist-belief momentum unittest,
    running with a learning rate of 0.1 and a momentum of 0.1.

    These values record how a parameter vector of size 10, initialized with 0.0,
    gets updated with 10 consecutive momentum steps.  It uses random gradients.

Returns:
        db_grad: The gradients to apply
        db_out: The parameters after the momentum update.


def _create_simple_tf1_gather_model(self, input_type: dtypes.DType, use_variable_for_filter=False) -> Tuple[core.Tensor, core.Tensor]:
    in_placeholder = array_ops.placeholder(input_type, shape=6)
    filters = np.random.randn(128, 32).astype(np.float32)
    if use_variable_for_filter:
        filters = variables.Variable(filters)
    output_tensor = array_ops.gather_v2(filters, in_placeholder)
    return (in_placeholder, output_tensor)

Creates a basic gather model.

    This is intended to be used for TF1 (graph mode) tests.

Args:
        input_type: type of the input index tensor for gather operation.
        use_variable_for_filter: Setting this to `True` makes the filter for the
        gather operation a `tf.Variable`.

Returns:
        in_placeholder: Input tensor placeholder.
        output_tensor: The resulting tensor of the gather operation.


def temp_pubsub_emulator(project_id='apache-beam-testing'):
    with PubSubContainer(project=project_id) as pubsub_container:
        publisher = pubsub_v1.PublisherClient()
        random_front_charactor = random.choice(string.ascii_lowercase)
        topic_id = f'{random_front_charactor}{uuid.uuid4().hex[:8]}'
        topic_name_to_create = f'projects/{pubsub_container.project}/topics/{topic_id}'
        created_topic_object = publisher.create_topic(name=topic_name_to_create)
        yield created_topic_object.name

Context manager to provide a temporary Pub/Sub emulator for testing.

    This function uses 'testcontainers' to spin up a Google Cloud SDK
    container running the Pub/Sub emulator. It yields the emulator host
    string (e.g., "localhost:xxxxx") that can be used to configure Pub/Sub
    clients.

    The Docker container is automatically managed and torn down when the
    context manager exits.

Args:
        project_id (str): The GCP project ID to be used by the emulator.
                        This doesn't need to be a real project for the emulator.

Yields:
        str: The host and port for the Pub/Sub emulator (e.g., "localhost:xxxx").
            This will be the address to point your Pub/Sub client to.

Raises:
        Exception: If the container fails to start or the emulator isn't ready.


def load_file_to_base64_str(f_path):
    path = abs_path(f_path)
    with io.open(path, 'rb') as f:
        f_bytes = f.read()
        base64_str = base64.b64encode(f_bytes).decode('utf-8')
        return base64_str

Loads the content of a file into a base64 string.

Args:
        f_path: full path to the file including the file name.

Returns:
        A base64 string representing the content of the file in utf-8 encoding.


def read(self, save_path, options=None):
    if options and options.experimental_enable_async_checkpoint:
        self._checkpoint_options = options
    if self._checkpoint_options and self._checkpoint_options.experimental_enable_async_checkpoint:
        if context.executing_eagerly():
            return self._async_checkpointer().read(save_path, options)
        else:
            logging.warning('Saving async checkpoint in graph mode is currently not supported; switching to regular sync checkpoint instead.')
    start_time = time.time()
    if isinstance(save_path, os.PathLike):
        save_path = os.fspath(save_path)
    options = options or checkpoint_options.CheckpointOptions()
    result = self._saver.restore(save_path=save_path, options=options)
    metrics.AddCheckpointReadDuration(api_label=_CHECKPOINT_V2, microseconds=_get_duration_microseconds(start_time, time.time()))
    return result

Reads a training checkpoint written with `write`.

    Reads this `Checkpoint` and any objects it depends on.

    This method is just like `restore()` but does not expect the `save_counter`
    variable in the checkpoint. It only restores the objects that the checkpoint
    already depends on.

    The method is primarily intended for use by higher level checkpoint
    management utilities that use `write()` instead of `save()` and have their
    own mechanisms to number and track checkpoints.

    Example usage:

    ```python
    # Create a checkpoint with write()
    ckpt = tf.train.Checkpoint(v=tf.Variable(1.))
    path = ckpt.write('/tmp/my_checkpoint')

    # Later, load the checkpoint with read()
    # With restore() assert_consumed() would have failed.
    checkpoint.read(path).assert_consumed()

    # You can also pass options to read(). For example this
    # runs the IO ops on the localhost:
    options = tf.train.CheckpointOptions(
        experimental_io_device="/job:localhost")
    checkpoint.read(path, options=options)
    ```

Args:
        save_path: The path to the checkpoint as returned by `write`.
        options: Optional `tf.train.CheckpointOptions` object.

Returns:
        A load status object, which can be used to make assertions about the
        status of a checkpoint restoration.  See `restore` for details.


def _parse_tensor_value(tensor_proto, return_list=False):
    try:
        ndarray = tensor_util.MakeNdarray(tensor_proto)
        return ndarray.tolist() if return_list else ndarray
    except TypeError:
        return None

Helper method for reading a tensor value from a tensor proto.

    The rationale for the distinction between `True` and `False value of
    `return_list` is as follows:
    - `return_list=True` is used for TensorDebugMode values other than
        FULL_TENSOR, e.g., CONCISE_HEALTH, SHAPE and FULL_HEATLH. Under
        those modes, the value is guaranteed (by contract) to be a 1D float64
        tensor.
    - `return_list=False` is used for the FULL_HEALTH TensorDebugMode
        specifically. Instead, we use `numpy.ndarray` to maximally preserve
        the shape, dtype and value information regarding the underlying tensor
        value. Under that mode, we don't use a python list to represent the
        tensor value because that can lead to loss of information (e.g., both
        float16 and float32 dtypes get mapped to Python floats).

Args:
        tensor_proto: The TensorProto instance from which the tensor value will be
        loaded.
        return_list: Whether the return value will be a nested Python list that
        comes out from `numpy.ndarray.tolist()`.

Returns:
        If parsing is successful, the tensor value as a `numpy.ndarray` or the
        nested Python list converted from it.
        If parsing fails, `None`.


def _spectrum_to_circulant_1d(self, spectrum, shape, dtype):
    spectrum = _to_complex(spectrum)
    spectrum_shape = self._shape_to_spectrum_shape(shape)
    domain_dimension = spectrum_shape[-1]
    if not domain_dimension:
        return array_ops.zeros(shape, dtype)
    matrix_rows = []
    for m in range(domain_dimension):
        x = np.zeros([domain_dimension])
        x[m] = 1.0
        fft_x = fft_ops.fft(math_ops.cast(x, spectrum.dtype))
        h_convolve_x = fft_ops.ifft(spectrum * fft_x)
        matrix_rows.append(h_convolve_x)
    matrix = array_ops_stack.stack(matrix_rows, axis=-1)
    return math_ops.cast(matrix, dtype)

Creates a circulant matrix from a spectrum.

    Intentionally done in an explicit yet inefficient way.  This provides a
    cross check to the main code that uses fancy reshapes.

Args:
        spectrum: Float or complex `Tensor`.
        shape:  Python list.  Desired shape of returned matrix.
        dtype:  Type to cast the returned matrix to.

Returns:
        Circulant (batch) matrix of desired `dtype`.


def _orthogonal_kernel(self, ksize, cin, cout):
    if cin > cout:
        raise ValueError(f'The number of input channels (cin={cin}) cannot exceed the number of output channels (cout={cout}).')
    orth = self._orthogonal_matrix(cout)[0:cin, :]
    if ksize == 1:
        return array_ops.expand_dims(array_ops.expand_dims(orth, 0), 0)
    p = self._block_orth(self._symmetric_projection(cout), self._symmetric_projection(cout))
    for _ in range(ksize - 2):
        temp = self._block_orth(self._symmetric_projection(cout), self._symmetric_projection(cout))
        p = self._matrix_conv(p, temp)
    for i in range(ksize):
        for j in range(ksize):
            p[i, j] = math_ops.matmul(orth, p[i, j])
    return self._dict_to_tensor(p, ksize, ksize)

Construct orthogonal kernel for convolution.

Args:
        ksize: Kernel size.
        cin: Number of input channels.
        cout: Number of output channels.

Returns:
        An [ksize, ksize, cin, cout] orthogonal kernel.

Raises:
        ValueError: If cin > cout.


def get(self):
    with warnings.catch_warnings(record=False):
        warnings.simplefilter('ignore')
        return np.nanquantile(self._queue, self._q)

Calculates and returns the specified quantile of the current sliding
    window.

Returns:
        float: The specified quantile of the values in the current sliding window.
            Returns NaN if the window is empty.


def forward(self, input_ids: Optional[torch.LongTensor]=None, attention_mask: Optional[torch.Tensor]=None, position_ids: Optional[torch.LongTensor]=None, past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]]=None, inputs_embeds: Optional[torch.FloatTensor]=None, labels: Optional[torch.LongTensor]=None, use_cache: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, cache_position: Optional[torch.LongTensor]=None, logits_to_keep: Union[int, torch.Tensor]=0, **kwargs: Unpack[KwargsForCausalLM]) -> CausalLMOutputWithPast:
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    outputs: BaseModelOutputWithPast = self.model(input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, cache_position=cache_position, **kwargs)
    hidden_states = outputs.last_hidden_state
    slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
    logits = self.lm_head(hidden_states[:, slice_indices, :])
    logits = logits * self.logit_scale
    loss = None
    if labels is not None:
        loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
    return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions)

labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:
    ```python
    >> from transformers import AutoTokenizer, Cohere2ForCausalLM

    >> model = Cohere2ForCausalLM.from_pretrained("Cohere2ForAI/c4ai-command-r-v01")
    >> tokenizer = AutoTokenizer.from_pretrained("Cohere2ForAI/c4ai-command-r-v01")

    >> prompt = "Hey, are you conscious? Can you talk to me?"
    >> inputs = tokenizer(prompt, return_tensors="pt")

    >> # Generate
    >> generate_ids = model.generate(inputs.input_ids, max_length=30)
    >> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
    "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
    ```


def _get_bounding_box(self, box: 'torch.Tensor') -> Dict[str, int]:
    if self.framework != 'pt':
        raise ValueError('The ZeroShotObjectDetectionPipeline is only available in PyTorch.')
    xmin, ymin, xmax, ymax = box.int().tolist()
    bbox = {'xmin': xmin, 'ymin': ymin, 'xmax': xmax, 'ymax': ymax}
    return bbox

Turns list [xmin, xmax, ymin, ymax] into dict { "xmin": xmin, ... }

Args:
        box (`torch.Tensor`): Tensor containing the coordinates in corners format.

Returns:
        bbox (`Dict[str, int]`): Dict containing the coordinates in corners format.