from abc import ABC, abstractmethod
from typing import List
class AbstractTokenizer(ABC):
"""
An abstract base class for tokenizer implementations.
A tokenizer is responsible for breaking down a string into a sequence of
strings (tokens).
"""
@abstractmethod
def tokenize(self, text: str) -> List[float]:
"""
Tokenizes a given string into a list of strings.
Args:
text: The input string to be tokenized.
Returns:
A list of strings, where each string is a token.
"""
pass
@abstractmethod
def tokenize_multiple(self, text_list: List[str]) -> List[List[str]]:
"""
Tokenizes a given string list by calling the remote tokenization API.
Args:
text_list: The input string to be tokenized.
Returns:
A list of list of floats
"""
