Counting tokens

pip install tokenizers

from tokenizers import Tokenizer
 
tokenizer = Tokenizer.from_pretrained("upstage/solar-pro-preview-tokenizer")
text = "Hi, how are you?"
enc = tokenizer.encode(text)
print("Encoded input:")
print(enc)
 
inv_vocab = {v: k for k, v in tokenizer.get_vocab().items()}
tokens = [inv_vocab[token_id] for token_id in enc.ids]
print("Tokens:")
print(tokens)
 
number_of_tokens = len(enc.ids)
print("Number of tokens:", number_of_tokens)

# =============================================================================
# This code includes parts derived from or inspired by the "mm_utils.py" file
# from the LLaVA project (https://github.com/haotian-liu/LLaVA).
#
# The original code is licensed under the Apache License 2.0. You may obtain
# a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# The code is used here under the terms of the Apache License 2.0.
#
# Copyright 2024 Upstage
# =============================================================================
 
 
import sys, argparse
from transformers.image_processing_utils import select_best_resolution
 
 
def get_anyres_image_grid_shape(image_size, grid_pinpoints, patch_size):
    """
    Calculate the shape of the image patch grid after the preprocessing for images of any resolution.
    Args:
        image_size (`tuple`):
            The size of the input image in the format (width, height).
        grid_pinpoints (`List`):
            A list containing possible resolutions. Each item in the list should be a tuple or list
            of the form `(height, width)`.
        patch_size (`int`):
            The size of each image patch.
    Returns:
        tuple: The shape of the image patch grid in the format (width, height).
    """
    if not isinstance(grid_pinpoints, list):
        raise ValueError("grid_pinpoints should be a list of tuples or lists")
 
    height, width = select_best_resolution(image_size, grid_pinpoints)
    return height // patch_size, width // patch_size
 
def count_image_tokens(width, height):
    input_size = (width, height)
 
    base_size = 784
    pinpoints = [[base_size * 1, base_size * 4], [base_size * 4, base_size * 1], [base_size * 2, base_size *2]]
    patch_size = 14
    downsampling_ratio = 2
    special_tokens = 11
 
    num_patch_height, num_patch_width = get_anyres_image_grid_shape(input_size, pinpoints, base_size)
 
    original_height, original_width = input_size
    current_height, current_width = num_patch_height * (base_size // patch_size // downsampling_ratio), num_patch_width * (base_size // patch_size // downsampling_ratio)
 
    original_aspect_ratio = original_width / original_height
    current_aspect_ratio = current_width / current_height
 
    if original_aspect_ratio > current_aspect_ratio:
        scale_factor = current_width / original_width
        new_height = int(original_height * scale_factor)
        padding = (current_height - new_height) // 2
        tokens = current_width* (current_height - padding * 2 + 1)
    else:
        scale_factor = current_height / original_height
        new_width = int(original_width * scale_factor)
        padding = (current_width - new_width) // 2
        tokens = (current_height + 1) * (current_width - padding * 2)
 
    final_tokens = tokens + (base_size // patch_size // downsampling_ratio) ** 2 + special_tokens
    return final_tokens
 
def main():
    parser = argparse.ArgumentParser(description="Count the number of tokens in the solar-docvision model.")
    parser.add_argument("width", type=int, help="The width of the input image.")
    parser.add_argument("height", type=int, help="The height of the input image.")
    args = parser.parse_args()
 
    if len(sys.argv) != 3:
        raise ValueError("Please provide the width and height of the input image.")
 
    if args.width <= 0 or args.height <= 0:
        raise ValueError("The width and height must be positive integers.")
 
    width = args.width
    height = args.height
 
    tokens = count_image_tokens(width, height)
    print(tokens)
 
if __name__ == "__main__":
    main()
 

# usage: python count_image_tokens.py <width> <height>
python count_image_tokens.py 1920 1080