class MobileViTImageProcessor(BaseImageProcessor):
    model_input_names = ['pixel_values']

    def __init__(self, do_resize: bool=True, size: Optional[Dict[str, int]]=None, resample: PILImageResampling=PILImageResampling.BILINEAR, do_rescale: bool=True, rescale_factor: Union[int, float]=1 / 255, do_center_crop: bool=True, crop_size: Optional[Dict[str, int]]=None, do_flip_channel_order: bool=True, **kwargs) -> None:
        super().__init__(**kwargs)
        size = size if size is not None else {'shortest_edge': 224}
        size = get_size_dict(size, default_to_square=False)
        crop_size = crop_size if crop_size is not None else {'height': 256, 'width': 256}
        crop_size = get_size_dict(crop_size, param_name='crop_size')
        self.do_resize = do_resize
        self.size = size
        self.resample = resample
        self.do_rescale = do_rescale
        self.rescale_factor = rescale_factor
        self.do_center_crop = do_center_crop
        self.crop_size = crop_size
        self.do_flip_channel_order = do_flip_channel_order

    def resize(self, image: np.ndarray, size: Dict[str, int], resample: PILImageResampling=PILImageResampling.BILINEAR, data_format: Optional[Union[str, ChannelDimension]]=None, input_data_format: Optional[Union[str, ChannelDimension]]=None, **kwargs) -> np.ndarray:
        """
        Resize an image. The shortest edge of the image is resized to size["shortest_edge"], with the longest edge
        resized to keep the input aspect ratio.

        Args:
            image (`np.ndarray`):
                Image to resize.
            size (`Dict[str, int]`):
                Size of the output image.
            resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BILINEAR`):
                Resampling filter to use when resiizing the image.
            data_format (`str` or `ChannelDimension`, *optional*):
                The channel dimension format of the image. If not provided, it will be the same as the input image.
            input_data_format (`ChannelDimension` or `str`, *optional*):
                The channel dimension format of the input image. If not provided, it will be inferred.
        """
        default_to_square = True
        if 'shortest_edge' in size:
            size = size['shortest_edge']
            default_to_square = False
        elif 'height' in size and 'width' in size:
            size = (size['height'], size['width'])
        else:
            raise ValueError("Size must contain either 'shortest_edge' or 'height' and 'width'.")
        output_size = get_resize_output_image_size(image, size=size, default_to_square=default_to_square, input_data_format=input_data_format)
        return resize(image, size=output_size, resample=resample, data_format=data_format, input_data_format=input_data_format, **kwargs)

    def flip_channel_order(self, image: np.ndarray, data_format: Optional[Union[str, ChannelDimension]]=None, input_data_format: Optional[Union[str, ChannelDimension]]=None) -> np.ndarray:
        """
        Flip the color channels from RGB to BGR or vice versa.

        Args:
            image (`np.ndarray`):
                The image, represented as a numpy array.
            data_format (`ChannelDimension` or `str`, *optional*):
                The channel dimension format of the image. If not provided, it will be the same as the input image.
            input_data_format (`ChannelDimension` or `str`, *optional*):
                The channel dimension format of the input image. If not provided, it will be inferred.
        """
        return flip_channel_order(image, data_format=data_format, input_data_format=input_data_format)

    def __call__(self, images, segmentation_maps=None, **kwargs):
        """
        Preprocesses a batch of images and optionally segmentation maps.

        Overrides the `__call__` method of the `Preprocessor` class so that both images and segmentation maps can be
        passed in as positional arguments.
        """
        return super().__call__(images, segmentation_maps=segmentation_maps, **kwargs)

    def _preprocess(self, image: ImageInput, do_resize: bool, do_rescale: bool, do_center_crop: bool, do_flip_channel_order: bool, size: Optional[Dict[str, int]]=None, resample: PILImageResampling=None, rescale_factor: Optional[float]=None, crop_size: Optional[Dict[str, int]]=None, input_data_format: Optional[Union[str, ChannelDimension]]=None):
        if do_resize:
            image = self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format)
        if do_rescale:
            image = self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format)
        if do_center_crop:
            image = self.center_crop(image=image, size=crop_size, input_data_format=input_data_format)
        if do_flip_channel_order:
            image = self.flip_channel_order(image, input_data_format=input_data_format)
        return image

    def _preprocess_image(self, image: ImageInput, do_resize: Optional[bool]=None, size: Optional[Dict[str, int]]=None, resample: PILImageResampling=None, do_rescale: Optional[bool]=None, rescale_factor: Optional[float]=None, do_center_crop: Optional[bool]=None, crop_size: Optional[Dict[str, int]]=None, do_flip_channel_order: Optional[bool]=None, data_format: Optional[Union[str, ChannelDimension]]=None, input_data_format: Optional[Union[str, ChannelDimension]]=None) -> np.ndarray:
        """Preprocesses a single image."""
        image = to_numpy_array(image)
        if do_rescale and is_scaled_image(image):
            logger.warning_once('It looks like you are trying to rescale already rescaled images. If the input images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again.')
        if input_data_format is None:
            input_data_format = infer_channel_dimension_format(image)
        image = self._preprocess(image=image, do_resize=do_resize, size=size, resample=resample, do_rescale=do_rescale, rescale_factor=rescale_factor, do_center_crop=do_center_crop, crop_size=crop_size, do_flip_channel_order=do_flip_channel_order, input_data_format=input_data_format)
        image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format)
        return image

    def _preprocess_mask(self, segmentation_map: ImageInput, do_resize: Optional[bool]=None, size: Optional[Dict[str, int]]=None, do_center_crop: Optional[bool]=None, crop_size: Optional[Dict[str, int]]=None, input_data_format: Optional[Union[str, ChannelDimension]]=None) -> np.ndarray:
        """Preprocesses a single mask."""
        segmentation_map = to_numpy_array(segmentation_map)
        if segmentation_map.ndim == 2:
            added_channel_dim = True
            segmentation_map = segmentation_map[None, ...]
            input_data_format = ChannelDimension.FIRST
        else:
            added_channel_dim = False
            if input_data_format is None:
                input_data_format = infer_channel_dimension_format(segmentation_map, num_channels=1)
        segmentation_map = self._preprocess(image=segmentation_map, do_resize=do_resize, size=size, resample=PILImageResampling.NEAREST, do_rescale=False, do_center_crop=do_center_crop, crop_size=crop_size, do_flip_channel_order=False, input_data_format=input_data_format)
        if added_channel_dim:
            segmentation_map = segmentation_map.squeeze(0)
        segmentation_map = segmentation_map.astype(np.int64)
        return segmentation_map

    @filter_out_non_signature_kwargs()
    def preprocess(self, images: ImageInput, segmentation_maps: Optional[ImageInput]=None, do_resize: Optional[bool]=None, size: Optional[Dict[str, int]]=None, resample: PILImageResampling=None, do_rescale: Optional[bool]=None, rescale_factor: Optional[float]=None, do_center_crop: Optional[bool]=None, crop_size: Optional[Dict[str, int]]=None, do_flip_channel_order: Optional[bool]=None, return_tensors: Optional[Union[str, TensorType]]=None, data_format: ChannelDimension=ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]]=None) -> PIL.Image.Image:
        """
        Preprocess an image or batch of images.

        Args:
            images (`ImageInput`):
                Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
                passing in images with pixel values between 0 and 1, set `do_rescale=False`.
            segmentation_maps (`ImageInput`, *optional*):
                Segmentation map to preprocess.
            do_resize (`bool`, *optional*, defaults to `self.do_resize`):
                Whether to resize the image.
            size (`Dict[str, int]`, *optional*, defaults to `self.size`):
                Size of the image after resizing.
            resample (`int`, *optional*, defaults to `self.resample`):
                Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`, Only
                has an effect if `do_resize` is set to `True`.
            do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
                Whether to rescale the image by rescale factor.
            rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
                Rescale factor to rescale the image by if `do_rescale` is set to `True`.
            do_center_crop (`bool`, *optional*, defaults to `self.do_center_crop`):
                Whether to center crop the image.
            crop_size (`Dict[str, int]`, *optional*, defaults to `self.crop_size`):
                Size of the center crop if `do_center_crop` is set to `True`.
            do_flip_channel_order (`bool`, *optional*, defaults to `self.do_flip_channel_order`):
                Whether to flip the channel order of the image.
            return_tensors (`str` or `TensorType`, *optional*):
                The type of tensors to return. Can be one of:
                    - Unset: Return a list of `np.ndarray`.
                    - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.
                    - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
                    - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
                    - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.
            data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):
                The channel dimension format for the output image. Can be one of:
                    - `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                    - `ChannelDimension.LAST`: image in (height, width, num_channels) format.
            input_data_format (`ChannelDimension` or `str`, *optional*):
                The channel dimension format for the input image. If unset, the channel dimension format is inferred
                from the input image. Can be one of:
                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
        """
        do_resize = do_resize if do_resize is not None else self.do_resize
        resample = resample if resample is not None else self.resample
        do_rescale = do_rescale if do_rescale is not None else self.do_rescale
        rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor
        do_center_crop = do_center_crop if do_center_crop is not None else self.do_center_crop
        do_flip_channel_order = do_flip_channel_order if do_flip_channel_order is not None else self.do_flip_channel_order
        size = size if size is not None else self.size
        size = get_size_dict(size, default_to_square=False)
        crop_size = crop_size if crop_size is not None else self.crop_size
        crop_size = get_size_dict(crop_size, param_name='crop_size')
        images = make_list_of_images(images)
        if segmentation_maps is not None:
            segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2)
        images = make_list_of_images(images)
        if not valid_images(images):
            raise ValueError('Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, torch.Tensor, tf.Tensor or jax.ndarray.')
        if segmentation_maps is not None and (not valid_images(segmentation_maps)):
            raise ValueError('Invalid segmentation map type. Must be of type PIL.Image.Image, numpy.ndarray, torch.Tensor, tf.Tensor or jax.ndarray.')
        validate_preprocess_arguments(do_rescale=do_rescale, rescale_factor=rescale_factor, do_center_crop=do_center_crop, crop_size=crop_size, do_resize=do_resize, size=size, resample=resample)
        images = [self._preprocess_image(image=img, do_resize=do_resize, size=size, resample=resample, do_rescale=do_rescale, rescale_factor=rescale_factor, do_center_crop=do_center_crop, crop_size=crop_size, do_flip_channel_order=do_flip_channel_order, data_format=data_format, input_data_format=input_data_format) for img in images]
        data = {'pixel_values': images}
        if segmentation_maps is not None:
            segmentation_maps = [self._preprocess_mask(segmentation_map=segmentation_map, do_resize=do_resize, size=size, do_center_crop=do_center_crop, crop_size=crop_size, input_data_format=input_data_format) for segmentation_map in segmentation_maps]
            data['labels'] = segmentation_maps
        return BatchFeature(data=data, tensor_type=return_tensors)

    def post_process_semantic_segmentation(self, outputs, target_sizes: Optional[List[Tuple]]=None):
        """
        Converts the output of [`MobileViTForSemanticSegmentation`] into semantic segmentation maps. Only supports PyTorch.

        Args:
            outputs ([`MobileViTForSemanticSegmentation`]):
                Raw outputs of the model.
            target_sizes (`List[Tuple]` of length `batch_size`, *optional*):
                List of tuples corresponding to the requested final size (height, width) of each prediction. If unset,
                predictions will not be resized.

        Returns:
            semantic_segmentation: `List[torch.Tensor]` of length `batch_size`, where each item is a semantic
            segmentation map of shape (height, width) corresponding to the target_sizes entry (if `target_sizes` is
            specified). Each entry of each `torch.Tensor` correspond to a semantic class id.
        """
        logits = outputs.logits
        if target_sizes is not None:
            if len(logits) != len(target_sizes):
                raise ValueError('Make sure that you pass in as many target sizes as the batch dimension of the logits')
            if is_torch_tensor(target_sizes):
                target_sizes = target_sizes.numpy()
            semantic_segmentation = []
            for idx in range(len(logits)):
                resized_logits = torch.nn.functional.interpolate(logits[idx].unsqueeze(dim=0), size=target_sizes[idx], mode='bilinear', align_corners=False)
                semantic_map = resized_logits[0].argmax(dim=0)
                semantic_segmentation.append(semantic_map)
        else:
            semantic_segmentation = logits.argmax(dim=1)
            semantic_segmentation = [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0])]
        return semantic_segmentation

Constructs a MobileViT image processor.

Args:
    do_resize (`bool`, *optional*, defaults to `True`):
        Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by the
        `do_resize` parameter in the `preprocess` method.
    size (`Dict[str, int]` *optional*, defaults to `{"shortest_edge": 224}`):
        Controls the size of the output image after resizing. Can be overridden by the `size` parameter in the
        `preprocess` method.
    resample (`PILImageResampling`, *optional*, defaults to `Resampling.BILINEAR`):
        Defines the resampling filter to use if resizing the image. Can be overridden by the `resample` parameter
        in the `preprocess` method.
    do_rescale (`bool`, *optional*, defaults to `True`):
        Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the `do_rescale`
        parameter in the `preprocess` method.
    rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):
        Scale factor to use if rescaling the image. Can be overridden by the `rescale_factor` parameter in the
        `preprocess` method.
    do_center_crop (`bool`, *optional*, defaults to `True`):
        Whether to crop the input at the center. If the input size is smaller than `crop_size` along any edge, the
        image is padded with 0's and then center cropped. Can be overridden by the `do_center_crop` parameter in
        the `preprocess` method.
    crop_size (`Dict[str, int]`, *optional*, defaults to `{"height": 256, "width": 256}`):
        Desired output size `(size["height"], size["width"])` when applying center-cropping. Can be overridden by
        the `crop_size` parameter in the `preprocess` method.
    do_flip_channel_order (`bool`, *optional*, defaults to `True`):
        Whether to flip the color channels from RGB to BGR. Can be overridden by the `do_flip_channel_order`
        parameter in the `preprocess` method.


def run_user_main(wrapped_test_module):
    tree = ast.parse(tf_inspect.getsource(wrapped_test_module))
    target = ast.dump(ast.parse('if __name__ == "__main__": pass').body[0].test)
    for expr in reversed(tree.body):
        if isinstance(expr, ast.If) and ast.dump(expr.test) == target:
            break
    else:
        raise NotImplementedError(f'Could not find `if __name__ == "main":` block in {wrapped_test_module.__name__}.')
    new_ast = ast.Module(body=expr.body, type_ignores=[])
    exec(compile(new_ast, '<ast>', 'exec'), globals(), wrapped_test_module.__dict__)

Runs the "if __name__ == '__main__'" at the bottom of a module.

TensorFlow practice is to have a main if at the bottom of the module which
might call an API compat function before calling test.main().

Since this is a statement, not a function, we can't cleanly reference it, but
we can inspect it from the user module and run it in the context of that
module so all imports and variables are available to it.

Args:
  wrapped_test_module: The user-provided test code to run.

Raises:
  NotImplementedError: If main block was not found in module. This should not
    be caught, as it is likely an error on the user's part -- absltest is all
    too happy to report a successful status (and zero tests executed) if a
    user forgets to end a class with "test.main()".


def summary_writer(self):
    return self._summary_writer

Return the SummaryWriter used by the chief supervisor.

Returns:
  A SummaryWriter.


def _process_new(self, feed_item):
    return {'name': feed_item.get(FieldMap.CAMPAIGN_LANDING_PAGE_NAME, None), 'url': feed_item.get(FieldMap.CAMPAIGN_LANDING_PAGE_URL, None), 'advertiserId': feed_item.get(FieldMap.ADVERTISER_ID, None)}

Creates a new landing page DCM object from a feed item representing a landing page from the Bulkdozer feed.

This function simply creates the object to be inserted later by the BaseDAO
object.

Args:
  feed_item: Feed item representing the landing page from the Bulkdozer
    feed.

Returns:
  An landing page object ready to be inserted in DCM through the API.


def _axis_gather(params, indices, axis):
    if axis > 1:
        if not isinstance(params, ragged_tensor.RaggedTensor):
            params = ragged_tensor.RaggedTensor.from_tensor(params, ragged_rank=1, row_splits_dtype=indices.row_splits.dtype)
        return params.with_values(_gather(params.values, indices, axis - 1, 0))
    if indices.shape.rank is None:
        raise ValueError('rank(indices) must be known statically')
    assert axis == 1
    flat_params = _flatten_dims_0_and_1(params)
    adjustments = _row_starts(params, indices.dtype)
    adjustments = _increase_rank_to(adjustments, indices.shape.ndims + 1)
    adjusted_indices = indices + adjustments
    return _gather(flat_params, adjusted_indices, axis - 1, 0)

Helper that implements ragged gather when axis>0 and batch_dims==0.

Args:
  params: The tensor from which to gather values.
  indices: The indices of values to gather.
  axis: The axis in `params` to gather `indices` from.

Returns:
  A potentially ragged tensor.


def _expand_param_on_rank(self, param, expand_rank, axis):
    param_tensor = tf.convert_to_tensor(param, dtype=self._dtype)
    param_expand = param_tensor
    for _ in range(expand_rank):
        param_expand = tf.expand_dims(param_expand, axis)
    return param_expand

Adds dimensions to `param`, not inplace.

Args:
  param: initial element.
  expand_rank: is amount of dimensions that need to be added.
  axis: is axis where to place these dimensions.

Returns:
  New `Tensor`.


def dispatch_for_binary_elementwise_assert_apis(x_type, y_type):

    def decorator(handler):
        api_handler_key = (x_type, y_type, _ASSERT_API_TAG)
        if api_handler_key in _ELEMENTWISE_API_HANDLERS:
            raise ValueError(f'A binary elementwise assert dispatch handler ({_ELEMENTWISE_API_HANDLERS[api_handler_key]}) has already been registered for ({x_type}, {y_type}).')
        _ELEMENTWISE_API_HANDLERS[api_handler_key] = handler
        for api in _BINARY_ELEMENTWISE_ASSERT_APIS:
            _add_dispatch_for_binary_elementwise_api(api, x_type, y_type, handler)
        return handler
    return decorator

Decorator to override default implementation for binary elementwise assert APIs.

The decorated function (known as the "elementwise assert handler")
overrides the default implementation for any binary elementwise assert API
whenever the value for the first two arguments (typically named `x` and `y`)
match the specified type annotations.  The handler is called with two
arguments:

  `elementwise_assert_handler(assert_func, x, y)`

Where `x` and `y` are the first two arguments to the binary elementwise assert
operation, and `assert_func` is a TensorFlow function that takes two
parameters and performs the elementwise assert operation (e.g.,
`tf.debugging.assert_equal`).

The following example shows how this decorator can be used to update all
binary elementwise assert operations to handle a `MaskedTensor` type:

>>> class MaskedTensor(tf.experimental.ExtensionType):
...   values: tf.Tensor
...   mask: tf.Tensor
>>> @dispatch_for_binary_elementwise_assert_apis(MaskedTensor, MaskedTensor)
... def binary_elementwise_assert_api_handler(assert_func, x, y):
...   merged_mask = tf.logical_and(x.mask, y.mask)
...   selected_x_values = tf.boolean_mask(x.values, merged_mask)
...   selected_y_values = tf.boolean_mask(y.values, merged_mask)
...   assert_func(selected_x_values, selected_y_values)
>>> a = MaskedTensor([1, 1, 0, 1, 1], [False, False, True, True, True])
>>> b = MaskedTensor([2, 2, 0, 2, 2], [True, True, True, False, False])
>>> tf.debugging.assert_equal(a, b) # assert passed; no exception was thrown

>>> a = MaskedTensor([1, 1, 1, 1, 1], [True, True, True, True, True])
>>> b = MaskedTensor([0, 0, 0, 0, 2], [True, True, True, True, True])
>>> tf.debugging.assert_greater(a, b)
Traceback (most recent call last):
...
InvalidArgumentError: Condition x > y did not hold.

Args:
  x_type: A type annotation indicating when the api handler should be called.
  y_type: A type annotation indicating when the api handler should be called.

Returns:
  A decorator.

#### Registered APIs

The binary elementwise assert APIs are:

<<API_LIST>>


def recipe_cm360_segmentology(config, account, auth_read, auth_write, recipe_name, date_range, recipe_slug, advertisers):
    dataset(config, {'description': 'Create a dataset for bigquery tables.', 'hour': [4], 'auth': auth_write, 'dataset': recipe_slug})
    bigquery(config, {'auth': auth_write, 'function': 'Pearson Significance Test', 'to': {'dataset': recipe_slug}})
    google_api(config, {'auth': 'user', 'api': 'dfareporting', 'version': 'v3.4', 'function': 'accounts.get', 'kwargs': {'id': account, 'fields': 'id,name'}, 'results': {'bigquery': {'auth': auth_write, 'dataset': recipe_slug, 'table': 'CM360_Account'}}})
    dcm(config, {'auth': auth_read, 'report': {'filters': {'advertiser': {'values': advertisers}}, 'account': account, 'body': {'name': recipe_name, 'criteria': {'dateRange': {'kind': 'dfareporting#dateRange', 'relativeDateRange': date_range}, 'dimensions': [{'kind': 'dfareporting#sortedDimension', 'name': 'advertiserId'}, {'kind': 'dfareporting#sortedDimension', 'name': 'advertiser'}, {'kind': 'dfareporting#sortedDimension', 'name': 'zipCode'}], 'metricNames': ['impressions', 'clicks', 'totalConversions']}, 'type': 'STANDARD', 'delivery': {'emailOwner': False}, 'format': 'CSV'}}})
    dcm(config, {'auth': auth_read, 'report': {'account': account, 'name': recipe_name}, 'out': {'bigquery': {'auth': auth_write, 'dataset': recipe_slug, 'table': 'CM360_KPI', 'header': True}}})
    bigquery(config, {'auth': auth_write, 'from': {'query': 'SELECT\n          Id AS Partner_Id,\n          Name AS Partner,\n          Advertiser_Id,\n          Advertiser,\n          Zip_Postal_Code AS Zip,\n          SAFE_DIVIDE(Impressions, SUM(Impressions) OVER(PARTITION BY Advertiser_Id)) AS Impression,\n          SAFE_DIVIDE(Clicks, Impressions) AS Click,\n          SAFE_DIVIDE(Total_Conversions, Impressions) AS Conversion,\n          Impressions AS Impressions          FROM `{dataset}.CM360_KPI`          CROSS JOIN `{dataset}.CM360_Account`        ', 'parameters': {'dataset': recipe_slug}, 'legacy': False}, 'to': {'dataset': recipe_slug, 'view': 'CM360_KPI_Normalized'}})
    census(config, {'auth': auth_write, 'normalize': {'census_geography': 'zip_codes', 'census_year': '2018', 'census_span': '5yr'}, 'to': {'dataset': recipe_slug, 'type': 'view'}})
    census(config, {'auth': auth_write, 'correlate': {'join': 'Zip', 'pass': ['Partner_Id', 'Partner', 'Advertiser_Id', 'Advertiser'], 'sum': ['Impressions'], 'correlate': ['Impression', 'Click', 'Conversion'], 'dataset': recipe_slug, 'table': 'CM360_KPI_Normalized', 'significance': 80}, 'to': {'dataset': recipe_slug, 'type': 'view'}})

CM360 funnel analysis using Census data.

Args:
  account (string) - NA
  auth_read (authentication) - Credentials used for reading data.
  auth_write (authentication) - Authorization used for writing data.
  recipe_name (string) - Name of report, not needed if ID used.
  date_range (choice) - Timeframe to run report for.
  recipe_slug (string) - Name of Google BigQuery dataset to create.
  advertisers (integer_list) - Comma delimited list of CM360 advertiser ids.


def cross(inputs, name=None):
    return _cross_internal(inputs=inputs, hashed_output=False, name=name)

Generates feature cross from a list of tensors.

The input tensors must have `rank=2`, and must all have the same number of
rows.  The result is a `RaggedTensor` with the same number of rows as the
inputs, where `result[row]` contains a list of all combinations of values
formed by taking a single value from each input's corresponding row
(`inputs[i][row]`).  Values are combined by joining their strings with '_X_'.
E.g.:

>>> tf.ragged.cross([tf.ragged.constant([['a'], ['b', 'c']]),
...                  tf.ragged.constant([['d'], ['e']]),
...                  tf.ragged.constant([['f'], ['g']])])
<tf.RaggedTensor [[b'a_X_d_X_f'], [b'b_X_e_X_g', b'c_X_e_X_g']]>

Args:
  inputs: A list of `RaggedTensor` or `Tensor` or `SparseTensor`.
  name: Optional name for the op.

Returns:
  A 2D `RaggedTensor` of type `string`.


def update(self, key_states: torch.Tensor, value_states: torch.Tensor, layer_idx: int, cache_kwargs: Optional[Dict[str, Any]]=None) -> Tuple[torch.Tensor, torch.Tensor]:
    key_states = key_states.to(self.key_cache[layer_idx].dtype)
    value_states = value_states.to(self.value_cache[layer_idx].dtype)
    if layer_idx == 0:
        self._seen_tokens += key_states.shape[-2]
        k_out = self.key_cache[0]
        v_out = self.value_cache[0]
    else:
        if self._prefetch_stream is not None:
            torch.cuda.default_stream(self.device).wait_stream(self._prefetch_stream)
        k_out = self._device_key_cache[layer_idx & 1]
        v_out = self._device_value_cache[layer_idx & 1]
    self._prefetch_layer(layer_idx + 1)
    cache_position = cache_kwargs.get('cache_position') if cache_kwargs is not None else None
    if cache_position is None:
        k_out.copy_(key_states)
        v_out.copy_(value_states)
        if layer_idx == 0:
            self.key_cache[layer_idx].copy_(key_states.to(self.offload_device))
            self.value_cache[layer_idx].copy_(value_states.to(self.offload_device))
    else:
        try:
            k_out.index_copy_(2, cache_position, key_states)
            v_out.index_copy_(2, cache_position, value_states)
        except NotImplementedError:
            k_out[:, :, cache_position] = key_states
            v_out[:, :, cache_position] = value_states
        if layer_idx != 0:
            cache_position = cache_position.to(self.offload_device)
            key_states = key_states.to(self.offload_device)
            value_states = value_states.to(self.offload_device)
            try:
                self.key_cache[layer_idx].index_copy_(2, cache_position, key_states)
                self.value_cache[layer_idx].index_copy_(2, cache_position, value_states)
            except NotImplementedError:
                self.key_cache[layer_idx][:, :, cache_position] = key_states
                self.value_cache[layer_idx][:, :, cache_position] = value_states
    return (k_out, v_out)

Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`.
It is VERY important to index using a tensor, otherwise you introduce a copy to the device.

Parameters:
    key_states (`torch.Tensor`):
        The new key states to cache.
    value_states (`torch.Tensor`):
        The new value states to cache.
    layer_idx (`int`):
        The index of the layer to cache the states for.
    cache_kwargs (`Dict[str, Any]`, *optional*):
        Additional arguments for the cache subclass. The `OffloadedStaticCache` needs the
        `cache_position` input to know how where to write in the cache.

Return:
    A tuple containing the updated key and value states.


def download_wmt_dataset(src_lang='ro', tgt_lang='en', dataset='wmt16', save_dir=None) -> None:
    try:
        import datasets
    except (ModuleNotFoundError, ImportError):
        raise ImportError('run pip install datasets')
    pair = f'{src_lang}-{tgt_lang}'
    print(f'Converting {dataset}-{pair}')
    ds = datasets.load_dataset(dataset, pair)
    if save_dir is None:
        save_dir = f'{dataset}-{pair}'
    save_dir = Path(save_dir)
    save_dir.mkdir(exist_ok=True)
    for split in ds.keys():
        print(f'Splitting {split} with {ds[split].num_rows} records')
        fn = 'val' if split == 'validation' else split
        src_path = save_dir.joinpath(f'{fn}.source')
        tgt_path = save_dir.joinpath(f'{fn}.target')
        src_fp = src_path.open('w+')
        tgt_fp = tgt_path.open('w+')
        for x in tqdm(ds[split]):
            ex = x['translation']
            src_fp.write(ex[src_lang] + '\n')
            tgt_fp.write(ex[tgt_lang] + '\n')
    print(f'Saved {dataset} dataset to {save_dir}')

Download a dataset using the datasets package and save it to the format expected by finetune.py
Format of save_dir: train.source, train.target, val.source, val.target, test.source, test.target.

Args:
    src_lang: <str> source language
    tgt_lang: <str> target language
    dataset: <str> wmt16, wmt17, etc. wmt16 is a good start as it's small. To get the full list run `import datasets; print([d.id for d in datasets.list_datasets() if "wmt" in d.id])`
    save_dir: <str>, where to save the datasets, defaults to f'{dataset}-{src_lang}-{tgt_lang}'

Usage:
    >>> download_wmt_dataset('ro', 'en', dataset='wmt16') # saves to wmt16-ro-en


def assign_group_v2(group_assignment, device_index, base_key):
    group_size, group_key = gen_collective_ops.collective_assign_group_v2(group_assignment=group_assignment, device_index=device_index, base_key=base_key)
    return (group_size, group_key)

Assign group key based on group_assignment.

Args:
  group_assignment: a 2 dimensional integer Tensor that encodes which devices
    belong to the same group. The values are indices of the devices within 0
    to number of devices.
  device_index: integer for the index of the current device
  base_key: integer to offset the resulted group_key. The base key shall be
    unique for different values of group_assignment in the same tf.function.
Notes: The device_index argument must be consistent with the index of the
  device of this Op in the device assignment list. The behavior of this Op is
  undefined if they are inconsistent.

Returns:
  group_size, group_key: The group size and group key for the current device.


def psnr(a, b, max_val, name=None):
    with ops.name_scope(name, 'PSNR', [a, b]):
        max_val = math_ops.cast(max_val, a.dtype)
        max_val = convert_image_dtype(max_val, dtypes.float32)
        a = convert_image_dtype(a, dtypes.float32)
        b = convert_image_dtype(b, dtypes.float32)
        mse = math_ops.reduce_mean(math_ops.squared_difference(a, b), [-3, -2, -1])
        psnr_val = math_ops.subtract(20 * math_ops.log(max_val) / math_ops.log(10.0), np.float32(10 / np.log(10)) * math_ops.log(mse), name='psnr')
        _, _, checks = _verify_compatible_image_shapes(a, b)
        with ops.control_dependencies(checks):
            return array_ops.identity(psnr_val)

Returns the Peak Signal-to-Noise Ratio between a and b.

This is intended to be used on signals (or images). Produces a PSNR value for
each image in batch.

The last three dimensions of input are expected to be [height, width, depth].

Example:

```python
    # Read images from file.
    im1 = tf.decode_png('path/to/im1.png')
    im2 = tf.decode_png('path/to/im2.png')
    # Compute PSNR over tf.uint8 Tensors.
    psnr1 = tf.image.psnr(im1, im2, max_val=255)

    # Compute PSNR over tf.float32 Tensors.
    im1 = tf.image.convert_image_dtype(im1, tf.float32)
    im2 = tf.image.convert_image_dtype(im2, tf.float32)
    psnr2 = tf.image.psnr(im1, im2, max_val=1.0)
    # psnr1 and psnr2 both have type tf.float32 and are almost equal.
```

Args:
  a: First set of images.
  b: Second set of images.
  max_val: The dynamic range of the images (i.e., the difference between the
    maximum the and minimum allowed values).
  name: Namespace to embed the computation in.

Returns:
  The scalar PSNR between a and b. The returned tensor has type `tf.float32`
  and shape [batch_size, 1].


class FalsePositives(_ConfusionMatrixConditionCount):

    def __init__(self, thresholds=None, name=None, dtype=None):
        super().__init__(confusion_matrix_cond=metrics_utils.ConfusionMatrix.FALSE_POSITIVES, thresholds=thresholds, name=name, dtype=dtype)

Calculates the number of false positives.

If `sample_weight` is given, calculates the sum of the weights of
false positives. This metric creates one local variable, `accumulator`
that is used to keep track of the number of false positives.

If `sample_weight` is `None`, weights default to 1.
Use `sample_weight` of 0 to mask values.

Args:
    thresholds: (Optional) Defaults to `0.5`. A float value, or a Python
        list/tuple of float threshold values in `[0, 1]`. A threshold is
        compared with prediction values to determine the truth value of
        predictions (i.e., above the threshold is `True`, below is `False`).
        If used with a loss function that sets `from_logits=True` (i.e. no
        sigmoid applied to predictions), `thresholds` should be set to 0.
        One metric value is generated for each threshold value.
    name: (Optional) string name of the metric instance.
    dtype: (Optional) data type of the metric result.

Examples:

>>> m = keras.metrics.FalsePositives()
>>> m.update_state([0, 1, 0, 0], [0, 0, 1, 1])
>>> m.result()
2.0

>>> m.reset_state()
>>> m.update_state([0, 1, 0, 0], [0, 0, 1, 1], sample_weight=[0, 0, 1, 0])
>>> m.result()
1.0


def Lookup(self, name):
    if not self._name2item:
        self._InitCache()
    return self._name2item[name]

Convenience function: Look up a given name in the global namespace.

Tries to find a constant, function or class by this name.

Args:
  name: Name to look up.

Returns:
  A Constant, Function or Class.

Raises:
  KeyError: if this identifier doesn't exist.


def summary(self, line_length=None, positions=None, print_fn=None):
    if not self.built:
        raise ValueError('This model has not yet been built. Build the model first by calling `build()` or calling `fit()` with some data, or specify an `input_shape` argument in the first layer(s) for automatic build.')
    layer_utils.print_summary(self, line_length=line_length, positions=positions, print_fn=print_fn)

Prints a string summary of the network.

Args:
    line_length: Total length of printed lines
        (e.g. set this to adapt the display to different
        terminal window sizes).
    positions: Relative or absolute positions of log elements
        in each line. If not provided,
        defaults to `[.33, .55, .67, 1.]`.
    print_fn: Print function to use. Defaults to `print`.
        It will be called on each line of the summary.
        You can set it to a custom function
        in order to capture the string summary.

Raises:
    ValueError: if `summary()` is called before the model is built.


def parallel_part_functions(fns: Sequence[PartFn], match_fns: Sequence[MatchFn] | None=None, with_default_output: bool=False, with_always_output: bool=False) -> PartFn:
    return functools.partial(_parallel_part_functions, _to_tuple_fns(fns, match_fns), with_default_output=with_default_output, with_always_output=with_always_output)

Combine `fns` to execute on _T in parallel across the `fns`.

Args:
  fns: sequence of part functions to chain.
  match_fns: sequence of functions that return True if the part should be
    processed by the part function. When the part should not be processed, the
    part function will not be called and nothing will be yielded by the part
    function. When match_fns is not provided, all parts are processed by
    default.
  with_default_output: True when the parallel execution should fallback to
    return the input part as is when fns do not return any output part.
  with_always_output: True when the parallel execution should always return
    the input part as is independent of the output of the fns. This is a
    stronger condition than `with_default_output`. When `with_always_output`
    is True, `with_default_output` is basically ignored.

Returns:
  Part function that runs all functions 'fns' in parallel. The output stream
  will keep the order of the input parts:

  f_0(c) = c00, c01
  f_1(c) = c10, c11, c12, c14

  f_0(c) // f_1(c) = c00, c01, c10, c11, c12


def output_types(self):
    return nest.map_structure(lambda component_spec: component_spec._to_legacy_output_types(), self._element_spec)

Returns the type of each component of an element of this iterator.

Returns:
  A (nested) structure of `tf.DType` objects corresponding to each component
  of an element of this dataset.


def get_splits(client, query, num_splits):
    if num_splits <= 1:
        raise SplitNotPossibleError('num_splits must be > 1, got: %d' % num_splits)
    validate_split(query)
    splits = []
    client_scatter_keys = _get_scatter_keys(client, query, num_splits)
    last_client_key = None
    for next_client_key in _get_split_key(client_scatter_keys, num_splits):
        splits.append(_create_split(last_client_key, next_client_key, query))
        last_client_key = next_client_key
    splits.append(_create_split(last_client_key, None, query))
    return splits

Returns a list of sharded queries for the given Cloud Datastore query.

This will create up to the desired number of splits, however it may return
less splits if the desired number of splits is unavailable. This will happen
if the number of split points provided by the underlying Datastore is less
than the desired number, which will occur if the number of results for the
query is too small.

This implementation of the QuerySplitter uses the __scatter__ property to
gather random split points for a query.

Note: This implementation is derived from the java query splitter in
https://github.com/GoogleCloudPlatform/google-cloud-datastore/blob/master/java/datastore/src/main/java/com/google/datastore/v1/client/QuerySplitterImpl.java

Args:
  client: the datastore client.
  query: the query to split.
  num_splits: the desired number of splits.

Returns:
  A list of split queries, of a max length of `num_splits`

Raises:
  QuerySplitterError: if split could not be performed owing to query or split
    parameters.


def __init__(self, sv, sess):
    super(SVTimerCheckpointThread, self).__init__(sv.coord, sv.save_model_secs)
    self._sv = sv
    self._sess = sess

Create a `SVTimerCheckpointThread`.

Args:
  sv: A `Supervisor`.
  sess: A `Session`.


def add_object(self, object_path, weights):
    if not isinstance(weights, dict):
        raise ValueError(f"Argument `weights` should be a dict where keys are weight names (usually '0', '1', etc.) and values are NumPy arrays. Received: type(weights)={type(weights)}")
    if '/' in object_path:
        elements = object_path.split('/')
        partial_path = '/'.join(elements[:-1])
        weights_dict = self.weights_dict
        for e in elements[:-1]:
            if e not in weights_dict:
                raise ValueError(f"Path '{partial_path}' not found in model.")
            weights_dict = weights_dict[e]
        weights_dict[elements[-1]] = weights
    else:
        self.weights_dict[object_path] = weights

Add a new object to the file (e.g. a layer).

Args:
    object_path: String, full path of the
        object to add (e.g. `"layers/dense_2"`).
    weights: Dict mapping weight names to weight
        values (arrays),
        e.g. `{"0": kernel_value, "1": bias_value}`.


def get_html_content(id_: str) -> str:
    try:
        node = nodes.Node.from_id(id_)
        return node.inner_html
    except Exception as e:
        epy.reraise(e, prefix='`ecolab.inspect` internal error. Please report an issue.\n')

Returns the inner content of the block id.

Is called the first time a block is expanded.

Args:
  id_: Id of the block to load

Returns:
  The html to add.


def recipe_dcm_to_sheets(config, auth_read, account, report_id, report_name, sheet, tab):
    dcm(config, {'auth': auth_read, 'report': {'account': account, 'report_id': report_id, 'name': report_name}, 'out': {'sheets': {'sheet': sheet, 'tab': tab, 'range': 'A1'}}})

Move existing CM report into a Sheet tab.

Args:
  auth_read (authentication) - Credentials used for reading data.
  account (integer) - NA
  report_id (integer) - NA
  report_name (string) - NA
  sheet (string) - NA
  tab (string) - NA


def from_text_vision_configs(cls, text_config: CLIPSegTextConfig, vision_config: CLIPSegVisionConfig, **kwargs):
    return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs)

Instantiate a [`CLIPSegConfig`] (or a derived class) from clipseg text model configuration and clipseg vision
model configuration.

Returns:
    [`CLIPSegConfig`]: An instance of a configuration object


def _get_longest_diag_dict(input_matrix, nonzero_idx):
    visited = set()
    diags = {}
    for idx in nonzero_idx:
        start_idx = torch.clone(idx)
        tuple_start_idx = tuple(start_idx.tolist())
        if tuple_start_idx in visited:
            continue
        visited.add(tuple_start_idx)
        cur_diag_len = 1
        start_idx += 1
        while start_idx[0] < input_matrix.shape[0] and start_idx[1] < input_matrix.shape[1]:
            tuple_start_idx = tuple(start_idx.tolist())
            visited.add(tuple_start_idx)
            if input_matrix[start_idx[0], start_idx[1]] == 1:
                cur_diag_len += 1
                start_idx += 1
            else:
                break
        diags[idx] = cur_diag_len
    return diags

Calculates the length of the longest diagonal sequence in a given matrix.
Args:
    input_matrix (torch.Tensor): The input matrix.
    nonzero_idx (torch.Tensor): The indices of the non-zero elements in the matrix.
Returns:
    dict: A dictionary where the keys are the indices of the non-zero elements and the values are the lengths of the longest diagonal sequences starting from those indices.


def model(x):
    hidden_act = dense_layer(hidden_weights, x)
    logits_act = dense_layer(output_weights, hidden_act, tf.identity)
    y = tf.nn.softmax(logits_act)
    return y

Feed forward function of the model.

Args:
  x: a (?, 28*28) tensor consisting of the feature inputs for a batch of
    examples.

Returns:
  A (?, 10) tensor containing the class scores for each example.


def replace_batch_norm(model):
    for name, module in model.named_children():
        if isinstance(module, nn.BatchNorm2d):
            new_module = DeformableDetrFrozenBatchNorm2d(module.num_features)
            if not module.weight.device == torch.device('meta'):
                new_module.weight.data.copy_(module.weight)
                new_module.bias.data.copy_(module.bias)
                new_module.running_mean.data.copy_(module.running_mean)
                new_module.running_var.data.copy_(module.running_var)
            model._modules[name] = new_module
        if len(list(module.children())) > 0:
            replace_batch_norm(module)

Recursively replace all `torch.nn.BatchNorm2d` with `DeformableDetrFrozenBatchNorm2d`.

Args:
    model (torch.nn.Module):
        input model


def assert_integer_v2(x, message=None, name=None):
    assert_integer(x=x, message=message, name=name)

Assert that `x` is of integer dtype.

If `x` has a non-integer type, `message`, as well as the dtype of `x` are
printed, and `InvalidArgumentError` is raised.

This can always be checked statically, so this method returns nothing.

Args:
  x: A `Tensor`.
  message: A string to prefix to the default message.
  name: A name for this operation (optional). Defaults to "assert_integer".

Raises:
  TypeError:  If `x.dtype` is not a non-quantized integer type.


def abs_path(path):
    return os.path.abspath(os.path.expanduser(path))

Resolve the '.' and '~' in a path to get the absolute path.

Args:
  path: The path to expand.

Returns:
  The absolute path of the input path.


def get_cosine_schedule_with_warmup(optimizer: Optimizer, num_warmup_steps: int, num_training_steps: int, num_cycles: float=0.5, last_epoch: int=-1):
    lr_lambda = partial(_get_cosine_schedule_with_warmup_lr_lambda, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps, num_cycles=num_cycles)
    return LambdaLR(optimizer, lr_lambda, last_epoch)

Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the
initial lr set in the optimizer.

Args:
    optimizer ([`~torch.optim.Optimizer`]):
        The optimizer for which to schedule the learning rate.
    num_warmup_steps (`int`):
        The number of steps for the warmup phase.
    num_training_steps (`int`):
        The total number of training steps.
    num_cycles (`float`, *optional*, defaults to 0.5):
        The number of waves in the cosine schedule (the defaults is to just decrease from the max value to 0
        following a half-cosine).
    last_epoch (`int`, *optional*, defaults to -1):
        The index of the last epoch when resuming training.

Return:
    `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule.


def binary_mask_to_rle(mask):
    if is_torch_tensor(mask):
        mask = mask.numpy()
    pixels = mask.flatten()
    pixels = np.concatenate([[0], pixels, [0]])
    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
    runs[1::2] -= runs[::2]
    return list(runs)

Converts given binary mask of shape `(height, width)` to the run-length encoding (RLE) format.

Args:
    mask (`torch.Tensor` or `numpy.array`):
        A binary mask tensor of shape `(height, width)` where 0 denotes background and 1 denotes the target
        segment_id or class_id.
Returns:
    `List`: Run-length encoded list of the binary mask. Refer to COCO API for more information about the RLE
    format.


def preprocess(self, images: Optional[List[ImageInput]]=None, videos: Optional[List[VideoInput]]=None, do_resize: Optional[bool]=None, size: Optional[Dict[str, int]]=None, resample: PILImageResampling=None, do_center_crop: Optional[bool]=None, crop_size: Optional[int]=None, do_rescale: Optional[bool]=None, rescale_factor: Optional[float]=None, do_normalize: Optional[bool]=None, image_mean: Optional[Union[float, List[float]]]=None, image_std: Optional[Union[float, List[float]]]=None, do_convert_rgb: Optional[bool]=None, return_tensors: Optional[Union[str, TensorType]]=None, data_format: Optional[ChannelDimension]=ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]]=None) -> BatchFeature:
    do_resize = do_resize if do_resize is not None else self.do_resize
    size = size if size is not None else self.size
    size = get_size_dict(size, param_name='size', default_to_square=False)
    resample = resample if resample is not None else self.resample
    do_center_crop = do_center_crop if do_center_crop is not None else self.do_center_crop
    crop_size = crop_size if crop_size is not None else self.crop_size
    crop_size = get_size_dict(crop_size, param_name='crop_size', default_to_square=True)
    do_rescale = do_rescale if do_rescale is not None else self.do_rescale
    rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor
    do_normalize = do_normalize if do_normalize is not None else self.do_normalize
    image_mean = image_mean if image_mean is not None else self.image_mean
    image_std = image_std if image_std is not None else self.image_std
    do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
    if images is not None:
        images = make_list_of_images(images)
    if images is not None and (not valid_images(images)):
        raise ValueError('Invalid input type. Must be of type PIL.Image.Image, numpy.ndarray, torch.Tensor, tf.Tensor or jax.ndarray.')
    data = {}
    if videos is not None:
        logger.warning("`VideoLlavaImageProcessor` works only with image inputs and doesn't process videos anymore. This is a deprecated behavior and will be removed in v5.0. Your videos should be forwarded to `VideoLlavaVideoProcessor`. ")
        videos = make_batched_videos(videos)
        pixel_values_videos = [[self._preprocess_image(image=frame, do_resize=do_resize, size=size, resample=resample, do_rescale=do_rescale, rescale_factor=rescale_factor, do_normalize=do_normalize, image_mean=image_mean, image_std=image_std, do_center_crop=do_center_crop, crop_size=crop_size, do_convert_rgb=do_convert_rgb, data_format=data_format, input_data_format=input_data_format) for frame in video] for video in videos]
        data['pixel_values_videos'] = pixel_values_videos
    if images is not None:
        pixel_values_images = [self._preprocess_image(image=image, do_resize=do_resize, size=size, resample=resample, do_rescale=do_rescale, rescale_factor=rescale_factor, do_normalize=do_normalize, image_mean=image_mean, image_std=image_std, do_center_crop=do_center_crop, crop_size=crop_size, do_convert_rgb=do_convert_rgb, data_format=data_format, input_data_format=input_data_format) for image in images]
        data['pixel_values_images'] = pixel_values_images
    encoded_outputs = BatchFeature(data, tensor_type=return_tensors)
    return encoded_outputs

Preprocess an image or batch of images.

Args:
    images (`ImageInput`, *optional*):
        List of images to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
        passing in images with pixel values between 0 and 1, set `do_rescale=False`.
    videos (`VideoInput`, *optional*):
        List of videos to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If
        passing in videos with pixel values between 0 and 1, set `do_rescale=False`.
    do_resize (`bool`, *optional*, defaults to `self.do_resize`):
        Whether to resize the image.
    size (`Dict[str, int]`, *optional*, defaults to `self.size`):
        Size of the image after resizing. Shortest edge of the image is resized to size["shortest_edge"], with
        the longest edge resized to keep the input aspect ratio.
    resample (`int`, *optional*, defaults to `self.resample`):
        Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only
        has an effect if `do_resize` is set to `True`.
    do_center_crop (`bool`, *optional*, defaults to `self.do_center_crop`):
        Whether to center crop the image.
    crop_size (`Dict[str, int]`, *optional*, defaults to `self.crop_size`):
        Size of the center crop. Only has an effect if `do_center_crop` is set to `True`.
    do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
        Whether to rescale the image.
    rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
        Rescale factor to rescale the image by if `do_rescale` is set to `True`.
    do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
        Whether to normalize the image.
    image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
        Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
    image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
        Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
        `True`.
    do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
        Whether to convert the image to RGB.
    return_tensors (`str` or `TensorType`, *optional*):
        The type of tensors to return. Can be one of:
        - Unset: Return a list of `np.ndarray`.
        - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.
        - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
        - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
        - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.
    data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):
        The channel dimension format for the output image. Can be one of:
        - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
        - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
        - Unset: Use the channel dimension format of the input image.
    input_data_format (`ChannelDimension` or `str`, *optional*):
        The channel dimension format for the input image. If unset, the channel dimension format is inferred
        from the input image. Can be one of:
        - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
        - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
        - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.


def get_executor():
    return context().executor

Get the Executor of the current thread.

Returns:
  The Executor of the current thread.


def erfinv(x):
    if any_symbolic_tensors((x,)):
        return Erfinv().symbolic_call(x)
    x = backend.convert_to_tensor(x)
    return backend.math.erfinv(x)

Computes the inverse error function of `x`, element-wise.

Args:
    x: Input tensor.

Returns:
    A tensor with the same dtype as `x`.

Example:

>>> x = np.array([-0.5, -0.2, -0.1, 0.0, 0.3])
>>> keras.ops.erfinv(x)
array([-0.47694, -0.17914, -0.08886,  0. ,  0.27246], dtype=float32)


def price(self, market: pmd.ProcessedMarketData, name: Optional[str]=None) -> types.FloatTensor:
    name = name or self._name + '_price'
    with tf.name_scope(name):
        discount_curve = cashflow_streams.get_discount_curve(self._discount_curve_type, market, self._discount_curve_mask)
        currencies = [cur.currency.value for cur in self._discount_curve_type]
        vol_surface = equity_utils.get_vol_surface(currencies, self._equity, market, self._equity_mask)
        spots = tf.stack(market.spot(currencies, self._equity), axis=0)
        discount_factors = discount_curve.discount_factor(self._expiry_date.expand_dims(axis=-1))
        daycount_convention = discount_curve.daycount_convention
        day_count_fn = market_data_utils.get_daycount_fn(daycount_convention)
        if spots.shape.rank > 0:
            spots = tf.gather(spots, self._equity_mask)
        if self._model == 'BS-LSM':
            vols = vol_surface.volatility(expiry_dates=self._expiry_date.expand_dims(axis=-1), strike=tf.expand_dims(self._strike, axis=-1))
            prices = utils.bs_lsm_price(spots=spots, expiry_times=day_count_fn(start_date=market.date, end_date=self._expiry_date, dtype=self._dtype), strikes=self._strike, volatility=tf.squeeze(vols, axis=-1), discount_factors=tf.squeeze(discount_factors), is_call_option=self._is_call_option, num_samples=self._num_samples, num_exercise_times=self._num_exercise_times, num_calibration_samples=self._num_calibration_samples, seed=self._seed)
            return self._short_position * self._contract_amount * prices
        else:
            raise ValueError('Only BS-LSM model is supported. Supplied {}'.format(self._model))

Returns the present value of the American options.

Args:
  market: An instance of `ProcessedMarketData`.
  name: Python str. The name to give to the ops created by this function.
    Default value: `None` which maps to 'price'.

Returns:
  A `Tensor` of shape `batch_shape`  containing the modeled price of each
  American option contract based on the input market data.


def deprecated_internal_learning_phase_scope(value):
    global _GRAPH_LEARNING_PHASES
    if value not in {0, 1}:
        raise ValueError('Expected learning phase to be 0 or 1.')
    with ops.init_scope():
        if context.executing_eagerly():
            previous_eager_value = _GRAPH_LEARNING_PHASES.get(_DUMMY_EAGER_GRAPH.key, None)
        previous_graph_value = _GRAPH_LEARNING_PHASES.get(get_graph(), None)
    learning_phase_previously_set = _DUMMY_EAGER_GRAPH.learning_phase_is_set
    try:
        deprecated_internal_set_learning_phase(value)
        yield
    finally:
        if not learning_phase_previously_set:
            _DUMMY_EAGER_GRAPH.learning_phase_is_set = False
        with ops.init_scope():
            if context.executing_eagerly():
                if previous_eager_value is not None:
                    _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH.key] = previous_eager_value
                elif _DUMMY_EAGER_GRAPH.key in _GRAPH_LEARNING_PHASES:
                    del _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH.key]
            graph = get_graph()
            if previous_graph_value is not None:
                _GRAPH_LEARNING_PHASES[graph] = previous_graph_value
            elif graph in _GRAPH_LEARNING_PHASES:
                del _GRAPH_LEARNING_PHASES[graph]

An internal-only version of `learning_phase_scope`.

Unlike the public method, this method does not raise a deprecation warning.
This is needed because saved model saving needs to set learning phase
to maintain compatibility
with code that sets/gets the learning phase, but saved model
saving itself shouldn't raise a deprecation warning.

We can get rid of this method and its usages when the public API is
removed.

Args:
   value: Learning phase value, either 0 or 1 (integers). 0 = test, 1 = train

Yields:
  None.

Raises:
   ValueError: if `value` is neither `0` nor `1`.


def _MatchBrackets(line):
    bracket_stack = []
    for token in line.tokens:
        if token.value in _OPENING_BRACKETS:
            bracket_stack.append(token)
        elif token.value in _CLOSING_BRACKETS:
            bracket_stack[-1].matching_bracket = token
            token.matching_bracket = bracket_stack[-1]
            bracket_stack.pop()
        for bracket in bracket_stack:
            if id(pytree_utils.GetOpeningBracket(token.node)) == id(bracket.node):
                bracket.container_elements.append(token)
                token.container_opening = bracket

Visit the node and match the brackets.

For every open bracket ('[', '{', or '('), find the associated closing bracket
and "match" them up. I.e., save in the token a pointer to its associated open
or close bracket.

Arguments:
  line: (LogicalLine) A logical line.


class BarthezTokenizerFast(PreTrainedTokenizerFast):
    vocab_files_names = VOCAB_FILES_NAMES
    model_input_names = ['input_ids', 'attention_mask']
    slow_tokenizer_class = BarthezTokenizer

    def __init__(self, vocab_file=None, tokenizer_file=None, bos_token='<s>', eos_token='</s>', sep_token='</s>', cls_token='<s>', unk_token='<unk>', pad_token='<pad>', mask_token='<mask>', **kwargs):
        mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token
        super().__init__(vocab_file, tokenizer_file=tokenizer_file, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, sep_token=sep_token, cls_token=cls_token, pad_token=pad_token, mask_token=mask_token, **kwargs)
        self.vocab_file = vocab_file

    def build_inputs_with_special_tokens(self, token_ids_0: List[int], token_ids_1: Optional[List[int]]=None) -> List[int]:
        """
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. A BARThez sequence has the following format:

        - single sequence: `<s> X </s>`
        - pair of sequences: `<s> A </s></s> B </s>`

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        """
        if token_ids_1 is None:
            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
        cls = [self.cls_token_id]
        sep = [self.sep_token_id]
        return cls + token_ids_0 + sep + sep + token_ids_1 + sep

    def create_token_type_ids_from_sequences(self, token_ids_0: List[int], token_ids_1: Optional[List[int]]=None) -> List[int]:
        """
        Create a mask from the two sequences passed to be used in a sequence-pair classification task.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of zeros.
        """
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]
        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str]=None) -> Tuple[str]:
        if not self.can_save_slow_tokenizer:
            raise ValueError('Your fast tokenizer does not have the necessary information to save the vocabulary for a slow tokenizer.')
        if not os.path.isdir(save_directory):
            logger.error(f'Vocabulary path ({save_directory}) should be a directory')
            return
        out_vocab_file = os.path.join(save_directory, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
            copyfile(self.vocab_file, out_vocab_file)
        return (out_vocab_file,)

Adapted from [`CamembertTokenizer`] and [`BartTokenizer`]. Construct a "fast" BARThez tokenizer. Based on
[SentencePiece](https://github.com/google/sentencepiece).

This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
refer to this superclass for more information regarding those methods.

Args:
    vocab_file (`str`):
        [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that
        contains the vocabulary necessary to instantiate a tokenizer.
    bos_token (`str`, *optional*, defaults to `"<s>"`):
        The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.

        <Tip>

        When building a sequence using special tokens, this is not the token that is used for the beginning of
        sequence. The token used is the `cls_token`.

        </Tip>

    eos_token (`str`, *optional*, defaults to `"</s>"`):
        The end of sequence token.

        <Tip>

        When building a sequence using special tokens, this is not the token that is used for the end of sequence.
        The token used is the `sep_token`.

        </Tip>

    sep_token (`str`, *optional*, defaults to `"</s>"`):
        The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
        sequence classification or for a text and a question for question answering. It is also used as the last
        token of a sequence built with special tokens.
    cls_token (`str`, *optional*, defaults to `"<s>"`):
        The classifier token which is used when doing sequence classification (classification of the whole sequence
        instead of per-token classification). It is the first token of the sequence when built with special tokens.
    unk_token (`str`, *optional*, defaults to `"<unk>"`):
        The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
        token instead.
    pad_token (`str`, *optional*, defaults to `"<pad>"`):
        The token used for padding, for example when batching sequences of different lengths.
    mask_token (`str`, *optional*, defaults to `"<mask>"`):
        The token used for masking values. This is the token used when training this model with masked language
        modeling. This is the token which the model will try to predict.
    additional_special_tokens (`List[str]`, *optional*, defaults to `["<s>NOTUSED", "</s>NOTUSED"]`):
        Additional special tokens used by the tokenizer.


def __init__(self, name, default_name=None, values=None) -> None:
    self._name_scope = name_scope(name, default_name, values, skip_on_eager=False)
    self._name = default_name if name is None else name

Initialize the context manager.

Args:
  name: The name argument that is passed to the op function.
  default_name: The default name to use if the `name` argument is `None`.
  values: The list of `Tensor` arguments that are passed to the op function.

Raises:
  TypeError: if `default_name` is passed in but not a string.


def compute_gradients(self, loss, var_list=None, gate_gradients=optimizer.Optimizer.GATE_OP, aggregation_method=None, colocate_gradients_with_ops=False, grad_loss=None):
    loss = self._scale_loss(loss)
    grads_and_vars = self._optimizer.compute_gradients(loss=loss, var_list=var_list, gate_gradients=gate_gradients, aggregation_method=aggregation_method, colocate_gradients_with_ops=colocate_gradients_with_ops, grad_loss=grad_loss)
    grads = [g for g, _ in grads_and_vars]
    variables = [v for _, v in grads_and_vars]
    unscaled_grads = self._unscale_grads(grads)
    return list(zip(unscaled_grads, variables))

Compute gradients of `loss` for the variables in `var_list`.

This adjusts the dynamic range of the gradient evaluation by scaling up
the `loss` value. The gradient values are then scaled back down by the
reciprocal of the loss scale. This is useful in reduced precision training
where small gradient values would otherwise underflow the representable
range.

Args:
  loss: A Tensor containing the value to minimize or a callable taking no
    arguments which returns the value to minimize. When eager execution is
    enabled it must be a callable.
  var_list: Optional list or tuple of `tf.Variable` to update to minimize
    `loss`.  Defaults to the list of variables collected in the graph under
    the key `GraphKeys.TRAINABLE_VARIABLES`.
  gate_gradients: How to gate the computation of gradients.  Can be
    `GATE_NONE`, `GATE_OP`, or `GATE_GRAPH`.
  aggregation_method: Specifies the method used to combine gradient terms.
    Valid values are defined in the class `AggregationMethod`.
  colocate_gradients_with_ops: If True, try colocating gradients with the
    corresponding op.
  grad_loss: Optional. A `Tensor` holding the gradient computed for `loss`.

Returns:
  A list of (gradient, variable) pairs. Variable is always present, but
  gradient can be `None`.


def sigmoid(x):
    return nn.sigmoid(x)

Element-wise sigmoid.

Args:
    x: A tensor or variable.

Returns:
    A tensor.


def get_pipeline_options(project: str, job_name: str, mode: str, num_workers: int=cfg.NUM_WORKERS, streaming: bool=True) -> PipelineOptions:
    job_name = f'{job_name}-{datetime.now().strftime('%Y%m%d%H%M%S')}'
    staging_bucket = f'gs://{cfg.PROJECT_ID}-ml-examples'
    dataflow_options = {'runner': 'DirectRunner' if mode == 'local' else 'DataflowRunner', 'job_name': job_name, 'project': project, 'region': 'us-central1', 'staging_location': f'{staging_bucket}/dflow-staging', 'temp_location': f'{staging_bucket}/dflow-temp', 'setup_file': './setup.py', 'streaming': streaming}
    if num_workers:
        dataflow_options.update({'num_workers': num_workers})
    return PipelineOptions(flags=[], **dataflow_options)

Function to retrieve the pipeline options.
Args:
    project: GCP project to run on
    mode: Indicator to run local, cloud or template
    num_workers: Number of Workers for running the job parallely
    max_num_workers: Maximum number of workers running the job parallely
Returns:
    Dataflow pipeline options


def insert_rows(self, project_id, dataset_id, table_id, rows, insert_ids=None, skip_invalid_rows=False, ignore_unknown_values=False):
    insert_ids = [str(self.unique_row_id) if not insert_ids else insert_ids[i] for i, _ in enumerate(rows)]
    rows = [fast_json_loads(fast_json_dumps(r, default=default_encoder)) for r in rows]
    result, errors = self._insert_all_rows(project_id, dataset_id, table_id, rows, insert_ids, skip_invalid_rows=skip_invalid_rows, ignore_unknown_values=ignore_unknown_values)
    return (result, errors)

Inserts rows into the specified table.

Args:
  project_id: The project id owning the table.
  dataset_id: The dataset id owning the table.
  table_id: The table id.
  rows: A list of plain Python dictionaries. Each dictionary is a row and
    each key in it is the name of a field.
  skip_invalid_rows: If there are rows with insertion errors, whether they
    should be skipped, and all others should be inserted successfully.
  ignore_unknown_values: Set this option to true to ignore unknown column
    names. If the input rows contain columns that are not
    part of the existing table's schema, those columns are ignored, and
    the rows are successfully inserted.

Returns:
  A tuple (bool, errors). If first element is False then the second element
  will be a bigquery.InsertErrorsValueListEntry instance containing
  specific errors.


def from_csv(path: str, timestamps: str='timestamp', indexes: Optional[List[str]]=None, sep: str=',') -> EventSet:
    import pandas as pd
    if indexes is None:
        indexes = []
    df = pd.read_csv(path, sep=sep)
    return from_pandas(df, indexes=indexes, timestamps=timestamps)

Reads an [`EventSet`][temporian.EventSet] from a CSV file.

    Example:
        ```python
        >>> # Example CSV
        >>> temp_file = str(tmp_dir / "temporal_data.csv")
        >>> _ = open(temp_file, "w").write(
        ...     "date,feature_1,feature_2
"
        ...     "2023-01-01,10.0,3.0
"
        ...     "2023-01-02,20.0,4.0
"
        ...     "2023-02-01,30.0,5.0"
        ... )
        >>> # Load CSV
        >>> evset = tp.from_csv(temp_file, timestamps="date")
        >>> evset
        indexes: []
        features: [('feature_1', float64), ('feature_2', float64)]
        events:
            (3 events):
                timestamps: ['2023-01-01T00:00:00' '2023-01-02T00:00:00'
                             '2023-02-01T00:00:00']
                'feature_1': [10. 20. 30.]
                'feature_2': [3. 4. 5.]
        ...

        ```

    Args:
        path: Path to the file.
        timestamps: Name of the column to be used as timestamps for the
            EventSet.
        indexes: Names of the columns to be used as indexes for the EventSet.
            If None, a flat EventSet will be created.
        sep: Separator to use.

    Returns:
        EventSet read from file.


def deep_variable_product(variables, limit: int=DEEP_VARIABLE_LIMIT):
    return _deep_values_list_product([v.bindings for v in variables], set(), ComplexityLimit(limit))

Take the deep Cartesian product of a list of Variables.

For example:
  x1.children = {v2, v3}
  v1 = {x1, x2}
  v2 = {x3}
  v3 = {x4, x5}
  v4 = {x6}
then
  deep_variable_product([v1, v4]) will return:
    [[x1, x3, x4, x6],
     [x1, x3, x5, x6],
     [x2, x6]]
.
Args:
  variables: A sequence of Variables.
  limit: How many results we allow before aborting.

Returns:
  A list of lists of Values, where each sublist has one Value from each
  of the corresponding Variables and the Variables of their Values' children.

Raises:
  TooComplexError: If we expanded too many values.


class SamHQVisionConfig(PretrainedConfig):
    base_config_key = 'vision_config'
    model_type = 'sam_hq_vision_model'

    def __init__(self, hidden_size=768, output_channels=256, num_hidden_layers=12, num_attention_heads=12, num_channels=3, image_size=1024, patch_size=16, hidden_act='gelu', layer_norm_eps=1e-06, attention_dropout=0.0, initializer_range=1e-10, qkv_bias=True, mlp_ratio=4.0, use_abs_pos=True, use_rel_pos=True, window_size=14, global_attn_indexes=[2, 5, 8, 11], num_pos_feats=128, mlp_dim=None, **kwargs):
        super().__init__(**kwargs)
        self.hidden_size = hidden_size
        self.output_channels = output_channels
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.num_channels = num_channels
        self.image_size = image_size
        self.patch_size = patch_size
        self.hidden_act = hidden_act
        self.layer_norm_eps = layer_norm_eps
        self.attention_dropout = attention_dropout
        self.initializer_range = initializer_range
        self.qkv_bias = qkv_bias
        self.mlp_ratio = mlp_ratio
        self.use_abs_pos = use_abs_pos
        self.use_rel_pos = use_rel_pos
        self.window_size = window_size
        self.global_attn_indexes = global_attn_indexes
        self.num_pos_feats = num_pos_feats
        self.mlp_dim = int(hidden_size * mlp_ratio) if mlp_dim is None else mlp_dim

This is the configuration class to store the configuration of a [`SamHQVisionModel`]. It is used to instantiate a SAM_HQ
vision encoder according to the specified arguments, defining the model architecture. Instantiating a configuration
defaults will yield a similar configuration to that of the SAM_HQ ViT-h
[facebook/sam_hq-vit-huge](https://huggingface.co/facebook/sam_hq-vit-huge) architecture.

Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PretrainedConfig`] for more information.

Args:
    hidden_size (`int`, *optional*, defaults to 768):
        Dimensionality of the encoder layers and the pooler layer.
    output_channels (`int`, *optional*, defaults to 256):
        Dimensionality of the output channels in the Patch Encoder.
    num_hidden_layers (`int`, *optional*, defaults to 12):
        Number of hidden layers in the Transformer encoder.
    num_attention_heads (`int`, *optional*, defaults to 12):
        Number of attention heads for each attention layer in the Transformer encoder.
    num_channels (`int`, *optional*, defaults to 3):
        Number of channels in the input image.
    image_size (`int`, *optional*, defaults to 1024):
        Expected resolution. Target size of the resized input image.
    patch_size (`int`, *optional*, defaults to 16):
        Size of the patches to be extracted from the input image.
    hidden_act (`str`, *optional*, defaults to `"gelu"`):
        The non-linear activation function (function or string)
    layer_norm_eps (`float`, *optional*, defaults to 1e-06):
        The epsilon used by the layer normalization layers.
    attention_dropout (`float`, *optional*, defaults to 0.0):
        The dropout ratio for the attention probabilities.
    initializer_range (`float`, *optional*, defaults to 1e-10):
        The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
    qkv_bias (`bool`, *optional*, defaults to `True`):
        Whether to add a bias to query, key, value projections.
    mlp_ratio (`float`, *optional*, defaults to 4.0):
        Ratio of mlp hidden dim to embedding dim.
    use_abs_pos (`bool`, *optional*, defaults to `True`):
        Whether to use absolute position embedding.
    use_rel_pos (`bool`, *optional*, defaults to `True`):
        Whether to use relative position embedding.
    window_size (`int`, *optional*, defaults to 14):
        Window size for relative position.
    global_attn_indexes (`List[int]`, *optional*, defaults to `[2, 5, 8, 11]`):
        The indexes of the global attention layers.
    num_pos_feats (`int`, *optional*, defaults to 128):
        The dimensionality of the position embedding.
    mlp_dim (`int`, *optional*):
        The dimensionality of the MLP layer in the Transformer encoder. If `None`, defaults to `mlp_ratio *
        hidden_size`.

Example:

```python
>>> from transformers import (
...     SamHQVisionConfig,
...     SamHQVisionModel,
... )

>>> # Initializing a SamHQVisionConfig with `"facebook/sam_hq-vit-huge"` style configuration
>>> configuration = SamHQVisionConfig()

>>> # Initializing a SamHQVisionModel (with random weights) from the `"facebook/sam_hq-vit-huge"` style configuration
>>> model = SamHQVisionModel(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```


def forward(self, input_ids: torch.Tensor, input_embeds: Optional[torch.Tensor]=None) -> torch.Tensor:
    if input_ids is not None:
        input_embeds = self.word_embeddings(input_ids)
    seq_length = input_embeds.size(1)
    if hasattr(self, 'position_ids'):
        position_ids = self.position_ids[:, :seq_length]
    else:
        position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device)
        position_ids = position_ids.unsqueeze(0).expand_as(input_ids)
    position_embeddings = self.position_embeddings(position_ids)
    embeddings = input_embeds + position_embeddings
    embeddings = self.LayerNorm(embeddings)
    embeddings = self.dropout(embeddings)
    return embeddings

Parameters:
    input_ids (torch.Tensor):
        torch.tensor(bs, max_seq_length) The token ids to embed.
    input_embeds (*optional*, torch.Tensor):
        The pre-computed word embeddings. Can only be passed if the input ids are `None`.

Returns: torch.tensor(bs, max_seq_length, dim) The embedded tokens (plus position embeddings, no token_type
embeddings)


def _v2_get_resized_embeddings(self, old_embeddings: keras.layers.Embedding, new_num_tokens: int) -> keras.layers.Embedding:
    init_range = 0.02
    potential_initialization_variable_names = ['initializer_range', 'initializer_factor', 'init_std']
    for var_name in potential_initialization_variable_names:
        if hasattr(self.config, var_name):
            init_range = getattr(self.config, var_name)
    new_embeddings = keras.layers.Embedding(input_dim=new_num_tokens, output_dim=old_embeddings.output_dim, embeddings_initializer=keras.initializers.TruncatedNormal(stddev=init_range), name=old_embeddings.embeddings.name[:-13])
    new_embeddings(tf.constant([[0]]))
    if old_embeddings.input_dim >= new_num_tokens:
        init_embeddings = old_embeddings.embeddings[:new_num_tokens]
    else:
        init_embeddings = tf.concat([old_embeddings.embeddings, new_embeddings.embeddings[old_embeddings.input_dim:]], axis=0)
    new_embeddings.embeddings.assign(init_embeddings)
    return new_embeddings

Build a resized Embedding layer from a provided Embedding layer. Increasing the size will add newly initialized
vectors at the end. Reducing the size will remove vectors from the end.

Args:
    old_embeddings (`keras.layers.Embedding`):
        Old embeddings to be resized.
    new_num_tokens (`int`, *optional*):
        New number of tokens in the embedding matrix.

Return:
    `keras.layers.Embedding`: Resized Embedding layer.


class RwkvOutput(ModelOutput):
    last_hidden_state: Optional[torch.FloatTensor] = None
    state: Optional[List[torch.FloatTensor]] = None
    hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
    attentions: Optional[Tuple[torch.FloatTensor, ...]] = None

Class for the RWKV model outputs.

Args:
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Sequence of hidden-states at the output of the last layer of the model.
    state (list of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`):
        The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
        avoid providing the old `input_ids`.
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
        one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

        Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
        heads.


def Start(self, port, data_callback, error_callback, websocket_url, index_url, html_url, html_dir):

Starts the web server.

Underlying implementation can be based on http or websocket, or support
SSL or not.

Args:
  port: tcp/ssl port for http or websocket.
  data_callback: a function called on incoming data from GUI clients.
      The arguments are (web_server, client_id, data) where
        web_server: this instance.
        client_id: an opaque ID or object for a GUI client for the incoming
            data or request.
        data: an incoming byte stream data from a GUI client.
  error_callback: a function called on any error from GUI clients which
      made the connection to the GUI client is invalid. The arguments are
      (web_server, client_id, error) where
        web_server: this instance.
        client_id: an opaque ID or object for a GUI client for the error.
        error: an error string.
  websocket_url: the absolute URL path of websocket or long-live http
      connection.
  index_url: the absolute URL path of main index.html.
  html_url: the absolute URL path of a directory serving static html or
      javascript resources.
  html_dir: the absolute file path of a directory corresponding to html_url.

Raises:
  IOError: Cannot start a web server.


class MyT5Tokenizer(PreTrainedTokenizer):
    model_input_names = ['input_ids', 'attention_mask']
    vocab_files_names = VOCAB_FILES_NAMES

    def __init__(self, vocab_file, eos_token='</s>', unk_token='<unk>', pad_token='<pad>', extra_ids=125, additional_special_tokens=None, **kwargs) -> None:
        if extra_ids > 0 and additional_special_tokens is None:
            additional_special_tokens = [f'<extra_id_{i}>' for i in range(extra_ids)]
        elif extra_ids > 0 and additional_special_tokens is not None and (len(additional_special_tokens) > 0):
            extra_tokens = len(set(filter(lambda x: bool('extra_id' in str(x)), additional_special_tokens)))
            if extra_tokens != extra_ids:
                raise ValueError(f'Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are provided to MyT5Tokenizer. In this case the additional_special_tokens must include the extra_ids tokens')
        pad_token = AddedToken(pad_token, lstrip=True, rstrip=True) if isinstance(pad_token, str) else pad_token
        eos_token = AddedToken(eos_token, lstrip=True, rstrip=True) if isinstance(eos_token, str) else eos_token
        unk_token = AddedToken(unk_token, lstrip=True, rstrip=True) if isinstance(unk_token, str) else unk_token
        self._added_tokens_decoder = {0: pad_token, 1: eos_token, 2: unk_token}
        self.offset = len(self._added_tokens_decoder)
        self._utf_vocab_size = 2 ** 8
        self.byte_maps = json.load(open(vocab_file, 'r'))
        self.decompose_rewriter = ByteRewriter(self.byte_maps['decompose_map'])
        self.merge_rewriter = ByteRewriter(self.byte_maps['merge_map'])
        super().__init__(eos_token=eos_token, unk_token=unk_token, pad_token=pad_token, extra_ids=0, additional_special_tokens=additional_special_tokens, **kwargs)

    @property
    def vocab_size(self):
        return self._utf_vocab_size

    def get_vocab(self):
        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size + self.offset)}
        vocab.update(self.added_tokens_encoder)
        return vocab

    def get_special_tokens_mask(self, token_ids_0: List[int], token_ids_1: Optional[List[int]]=None, already_has_special_tokens: bool=False) -> List[int]:
        """
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        """
        if already_has_special_tokens:
            return super().get_special_tokens_mask(token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True)
        if token_ids_1 is None:
            return [0] * len(token_ids_0) + [1]
        return [0] * len(token_ids_0) + [1] + [0] * len(token_ids_1) + [1]

    def _add_eos_if_not_present(self, token_ids: List[int]) -> List[int]:
        """Do not add eos again if user already added it."""
        if len(token_ids) > 0 and token_ids[-1] == self.eos_token_id:
            warnings.warn(f'This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated eos tokens being added.')
            return token_ids
        else:
            return token_ids + [self.eos_token_id]

    def create_token_type_ids_from_sequences(self, token_ids_0: List[int], token_ids_1: Optional[List[int]]=None) -> List[int]:
        """
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. MyT5 does not
        make use of token type ids, therefore a list of zeros is returned.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of zeros.
        """
        eos = [self.eos_token_id]
        if token_ids_1 is None:
            return len(token_ids_0 + eos) * [0]
        return len(token_ids_0 + eos + token_ids_1 + eos) * [0]

    def build_inputs_with_special_tokens(self, token_ids_0: List[int], token_ids_1: Optional[List[int]]=None) -> List[int]:
        """
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. A sequence has the following format:

        - single sequence: `X </s>`
        - pair of sequences: `A </s> B </s>`

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        """
        token_ids_0 = self._add_eos_if_not_present(token_ids_0)
        if token_ids_1 is None:
            return token_ids_0
        else:
            token_ids_1 = self._add_eos_if_not_present(token_ids_1)
            return token_ids_0 + token_ids_1

    def _tokenize(self, text: str, **kwargs) -> List[str]:
        """Take as input a string and return a list of strings (tokens) for words/sub-words.
        Represents tokens in two character hex format"""
        tokens = [f'{i:02x}' for i in text.encode('utf-8')]
        tokens = self.morphological_encode(tokens)
        return tokens

    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""
        if len(token) != 2:
            token_id = None
        else:
            token_id = int(token, 16) + self.offset
        return token_id

    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (str) using the vocab."""
        token = f'{index - self.offset:02x}'
        return token

    def morphological_encode(self, indices: List[str]) -> List[str]:
        indices = self.decompose_rewriter.rewrite_bytes(indices, reverse=False)
        indices = self.merge_rewriter.rewrite_bytes(indices, reverse=False)
        return indices

    def morphological_decode(self, indices: List[str]) -> List[str]:
        indices = self.merge_rewriter.rewrite_bytes(indices, reverse=True)
        indices = self.decompose_rewriter.rewrite_bytes(indices, reverse=True)
        return indices

    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (string) in a single string."""
        bstring = b''
        out_tokens = []
        for token in tokens:
            if token in self.added_tokens_decoder:
                out_tokens.append(self.added_tokens_decoder[token])
            elif token in self.added_tokens_encoder:
                out_tokens.append(token)
            else:
                out_tokens.append(token)
        out_tokens = self.morphological_decode(out_tokens)
        _added_tokens = set(self.added_tokens_decoder.values()) | set(self.added_tokens_encoder)
        for token in out_tokens:
            if token in _added_tokens:
                bstring += bytes(token, 'utf-8')
            else:
                bstring += bytes.fromhex(token)
        string = bstring.decode('utf-8', errors='ignore')
        return string

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str]=None) -> Tuple[str]:
        if os.path.isdir(save_directory):
            vocab_file = os.path.join(save_directory, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
        else:
            vocab_file = (filename_prefix + '-' if filename_prefix else '') + save_directory
        with open(vocab_file, 'w', encoding='utf-8') as writer:
            writer.write(json.dumps(self.byte_maps, indent=2, ensure_ascii=False))
        return (vocab_file,)

Construct a MyT5 tokenizer.

This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.

Args:
    vocab_file (`str`): The file containing the byte rewriting rules.
    eos_token (`str`, *optional*, defaults to `"</s>"`):
        The end of sequence token.

    unk_token (`str`, *optional*, defaults to `"<unk>"`):
        The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
        token instead.
    pad_token (`str`, *optional*, defaults to `"<pad>"`):
        The token used for padding, for example when batching sequences of different lengths.
    extra_ids (`int`, *optional*, defaults to 125):
        Add a number of extra ids added to the end of the vocabulary for use as sentinels. These tokens are
        accessible as "<extra_id_{%d}>" where "{%d}" is a number between 0 and extra_ids-1. Extra tokens are
        indexed from the end of the vocabulary up to beginning ("<extra_id_0>" is the last token in the vocabulary
        like in ByT5 preprocessing see
        [here](https://github.com/google-research/text-to-text-transfer-transformer/blob/9fd7b14a769417be33bc6c850f9598764913c833/t5/data/preprocessors.py#L2117)).
    additional_special_tokens (`List[str]`, *optional*):
        Additional special tokens used by the tokenizer.


def to_json_file(self, json_file_path: Union[str, os.PathLike]):
    with open(json_file_path, 'w', encoding='utf-8') as writer:
        config_dict = self.to_dict()
        json_string = json.dumps(config_dict, indent=2, sort_keys=True) + '\n'
        writer.write(json_string)

Save this instance to a JSON file.

Args:
    json_file_path (`str` or `os.PathLike`):
        Path to the JSON file in which this configuration instance's parameters will be saved.
    use_diff (`bool`, *optional*, defaults to `True`):
        If set to `True`, only the difference between the config instance and the default
        `QuantizationConfig()` is serialized to JSON file.


def serialize_keras_object(instance):
    instance = inspect.unwrap(instance)
    if instance is None:
        return None
    if hasattr(instance, 'get_config'):
        name = object_registration.get_registered_name(instance.__class__)
        try:
            config = instance.get_config()
        except NotImplementedError as e:
            if _SKIP_FAILED_SERIALIZATION:
                return serialize_keras_class_and_config(name, {_LAYER_UNDEFINED_CONFIG_KEY: True})
            raise e
        serialization_config = {}
        for key, item in config.items():
            if isinstance(item, str):
                serialization_config[key] = item
                continue
            try:
                serialized_item = serialize_keras_object(item)
                if isinstance(serialized_item, dict) and (not isinstance(item, dict)):
                    serialized_item['__passive_serialization__'] = True
                serialization_config[key] = serialized_item
            except ValueError:
                serialization_config[key] = item
        name = object_registration.get_registered_name(instance.__class__)
        return serialize_keras_class_and_config(name, serialization_config, instance)
    if hasattr(instance, '__name__'):
        return object_registration.get_registered_name(instance)
    raise ValueError(f"Cannot serialize {instance} because it doesn't implement `get_config()`.")

Serialize a Keras object into a JSON-compatible representation.

Calls to `serialize_keras_object` while underneath the
`SharedObjectSavingScope` context manager will cause any objects re-used
across multiple layers to be saved with a special shared object ID. This
allows the network to be re-created properly during deserialization.

Args:
  instance: The object to serialize.

Returns:
  A dict-like, JSON-compatible representation of the object's config.


def compatible_with(value, logical_value):
    if isinstance(value, abstract.List) and (not value.is_concrete):
        return True
    elif isinstance(value, abstract.Dict) and (not value.is_concrete):
        return not logical_value or bool(value.get_instance_type_parameter(abstract_utils.K).bindings)
    elif isinstance(value, abstract.LazyConcreteDict):
        return value.is_empty() != logical_value
    elif isinstance(value, abstract.PythonConstant):
        return bool(value.pyval) == logical_value
    elif isinstance(value, abstract.Instance):
        name = value.full_name
        if logical_value and name in _CONTAINER_NAMES:
            ret = value.has_instance_type_parameter(abstract_utils.T) and bool(value.get_instance_type_parameter(abstract_utils.T).bindings)
            return ret
        elif name == 'builtins.NoneType':
            return not logical_value
        elif name in NUMERIC:
            return True
        elif isinstance(value.cls, abstract.Class) and (not value.cls.overrides_bool):
            if getattr(value.cls, 'template', None):
                return True
            return logical_value
        return True
    elif isinstance(value, (abstract.Function, abstract.Class)):
        return logical_value
    else:
        return True

Returns the conditions under which the value could be True or False.

Args:
  value: An abstract value.
  logical_value: Either True or False.

Returns:
  False: If the value could not evaluate to logical_value under any
      circumstance (e.g. value is the empty list and logical_value is True).
  True: If it is possible for the value to evaluate to the logical_value,
      and any ambiguity cannot be resolved by additional bindings.