# -*- coding: utf-8 -*-
"""Location: ./plugins/toon_encoder/toon.py
Copyright 2025
SPDX-License-Identifier: Apache-2.0
TOON (Token-Oriented Object Notation) Encoder/Decoder.
Pure Python implementation of TOON format specification v3.0.
https://github.com/toon-format/spec
TOON is a compact, human-readable encoding of the JSON data model designed
specifically for LLM prompts. It provides lossless serialization of JSON
objects, arrays, and primitives in a syntax that minimizes tokens.
Token Reduction Strategies:
1. Eliminates quotation marks around keys and simple string values
2. Uses compact array syntax: key[N]: val1,val2,val3
3. Uses columnar format for homogeneous object arrays
4. Removes colons/commas where context makes them unnecessary
Examples:
>>> from plugins.toon_encoder.toon import encode, decode
>>> data = {"name": "alice", "age": 30}
>>> toon = encode(data)
>>> toon
'name: alice\\nage: 30'
>>> decode(toon) == data
True
>>> arr = [{"id": 1, "name": "a"}, {"id": 2, "name": "b"}]
>>> print(encode(arr))
[2]{id,name}:
1,a
2,b
"""
from __future__ import annotations
import re
from typing import Any, Dict, List, Optional, Tuple
# =============================================================================
# Constants and Patterns
# =============================================================================
# Reserved words that must be quoted if used as string values
_RESERVED_WORDS = frozenset({"null", "true", "false"})
# Characters that require string quoting (excluding control chars handled separately)
_SPECIAL_CHARS_RE = re.compile(r'[\n\r\t,:\[\]{}"\\\-]')
# Pattern to detect if string looks like a number (full match for encoding)
_NUMBER_LIKE_RE = re.compile(r'^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$')
# Pattern for leading zeros (must be quoted per spec)
_LEADING_ZEROS_RE = re.compile(r'^0\d+$')
# Pattern to match number at start of string (partial match for decoding)
_NUMBER_PREFIX_RE = re.compile(r'^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?')
# Pattern for parsing TOON array header: key[N] or [N] or [N]{key1,key2}
# Key can be: unquoted (word chars + dots), or quoted (with escaped quotes inside)
# Don't consume whitespace after colon - preserve indentation for list item parsing
_ARRAY_HEADER_RE = re.compile(r'^(?:([A-Za-z_][A-Za-z0-9_.]*|"(?:[^"\\]|\\.)*"))?\[(\d+)\](?:\{([^}]*)\})?:')
# Pattern for simple key: value lines
_KEY_VALUE_RE = re.compile(r'^([^:\s]+):\s*(.*)$')
# Pattern for valid unquoted keys per TOON spec: ^[A-Za-z_][A-Za-z0-9_.]*$
_VALID_KEY_RE = re.compile(r'^[A-Za-z_][A-Za-z0-9_.]*$')
# Default indent size (2 spaces per spec recommendation)
_INDENT_SIZE = 2
# =============================================================================
# Encoder
# =============================================================================
def encode(obj: Any, *, indent: int = 0, _as_root: bool = True) -> str:
"""Encode a Python object to TOON format.
Args:
obj: Python object to encode (dict, list, str, int, float, bool, None).
indent: Current indentation level (internal use).
_as_root: Whether this is the root object (internal use).
Returns:
TOON-formatted string representation.
Raises:
TypeError: If object type is not JSON-serializable.
Examples:
>>> encode(None)
'null'
>>> encode(True)
'true'
>>> encode(42)
'42'
>>> encode(3.14)
'3.14'
>>> encode("hello")
'hello'
>>> encode("hello world")
'hello world'
>>> encode("with,comma")
'"with,comma"'
>>> encode({"key": "value"})
'key: value'
>>> encode([1, 2, 3])
'[3]: 1,2,3'
"""
if obj is None:
return "null"
if isinstance(obj, bool):
return "true" if obj else "false"
if isinstance(obj, int):
return str(obj)
if isinstance(obj, float):
return _encode_float(obj)
if isinstance(obj, str):
return _encode_string(obj)
if isinstance(obj, (list, tuple)):
return _encode_array(list(obj), indent=indent, _as_root=_as_root)
if isinstance(obj, dict):
return _encode_object(obj, indent=indent, _as_root=_as_root)
# Fallback for other types - convert to string
raise TypeError(f"Object of type {type(obj).__name__} is not TOON serializable")
def _encode_float(obj: float) -> str:
"""Encode a float value per TOON spec.
Args:
obj: Float to encode.
Returns:
TOON float representation.
"""
# Handle special float values - per TOON spec, these must be null
if obj != obj: # NaN
return "null"
if obj == float("inf") or obj == float("-inf"):
return "null"
# Normalize -0 to 0 per spec
if obj == 0.0:
return "0"
# Avoid trailing .0 for whole numbers
if obj.is_integer():
return str(int(obj))
# Per spec: no exponent notation, emit full decimal
formatted = f"{obj:.15g}"
if "e" in formatted.lower():
# Fall back to full decimal representation
formatted = f"{obj:.15f}".rstrip("0").rstrip(".")
return formatted
def _needs_quotes(s: str) -> bool:
"""Determine if a string value needs to be quoted in TOON.
Per TOON spec, strings need quotes if they:
- Are empty
- Match reserved words (null, true, false)
- Contain special characters (newlines, commas, colons, brackets, quotes, backslash)
- Look like numbers
- Have leading zeros (e.g., "05")
- Start with hyphen (e.g., "-a" or "-")
- Start/end with whitespace
Args:
s: String to check.
Returns:
True if string needs quoting.
Examples:
>>> _needs_quotes("")
True
>>> _needs_quotes("null")
True
>>> _needs_quotes("hello")
False
>>> _needs_quotes("hello world")
False
>>> _needs_quotes("has,comma")
True
>>> _needs_quotes("123")
True
>>> _needs_quotes("05")
True
>>> _needs_quotes("-a")
True
>>> _needs_quotes("-")
True
>>> _needs_quotes(" leading")
True
"""
if not s:
return True
if s in _RESERVED_WORDS:
return True
if _SPECIAL_CHARS_RE.search(s):
return True
if _NUMBER_LIKE_RE.match(s):
return True
# Per spec: leading zeros must be quoted
if _LEADING_ZEROS_RE.match(s):
return True
# Per spec: strings starting with hyphen must be quoted
if s[0] == "-":
return True
if s[0].isspace() or s[-1].isspace():
return True
# Check for control characters
if any(ord(c) < 32 for c in s):
return True
return False
def _encode_string(s: str) -> str:
"""Encode a string value, quoting only when necessary.
Args:
s: String to encode.
Returns:
TOON string representation.
Examples:
>>> _encode_string("simple")
'simple'
>>> _encode_string("with space")
'with space'
>>> _encode_string("has\\nnewline")
'"has\\\\nnewline"'
>>> _encode_string("")
'""'
"""
if not _needs_quotes(s):
return s
return _quote_string(s)
def _quote_string(s: str) -> str:
"""Quote and escape a string unconditionally.
Per TOON spec, only these escapes are valid: \\\\ \\\" \\n \\r \\t
Control characters that can't be escaped must cause the string to be skipped.
Args:
s: String to quote.
Returns:
Quoted string with escapes applied.
Raises:
ValueError: If string contains unencodable control characters.
"""
result = ['"']
for char in s:
if char == "\\":
result.append("\\\\")
elif char == '"':
result.append('\\"')
elif char == "\n":
result.append("\\n")
elif char == "\r":
result.append("\\r")
elif char == "\t":
result.append("\\t")
elif ord(char) < 32:
# Per TOON spec, only the above escapes are valid
# Control characters that can't be escaped should cause an error
raise ValueError(f"Cannot encode control character U+{ord(char):04X} in TOON")
else:
result.append(char)
result.append('"')
return "".join(result)
def _encode_key(key: str) -> str:
"""Encode an object key.
Per TOON spec, unquoted keys must match: ^[A-Za-z_][A-Za-z0-9_.]*$
Args:
key: Object key to encode.
Returns:
TOON key representation.
Examples:
>>> _encode_key("simple")
'simple'
>>> _encode_key("has space")
'"has space"'
>>> _encode_key("with:colon")
'"with:colon"'
"""
# Per TOON spec, unquoted keys must match ^[A-Za-z_][A-Za-z0-9_.]*$
if key and _VALID_KEY_RE.match(key) and key not in _RESERVED_WORDS:
return key
# Key needs quoting
return _quote_string(key)
def _encode_array(arr: List[Any], *, indent: int = 0, _as_root: bool = True, key_prefix: str = "") -> str:
"""Encode an array in TOON format.
Per TOON spec:
- Empty arrays: key[0]: or [0]: at root
- Primitive arrays: key[N]: val1,val2,val3 or [N]: val1,val2,val3
- Columnar arrays: key[N]{f1,f2}: followed by rows
- Mixed/complex arrays: Use list item syntax with - prefix
Args:
arr: Array to encode.
indent: Current indentation level.
_as_root: Whether this is the root array (unused, for API consistency).
key_prefix: Key name to prefix (for arrays in objects).
Returns:
TOON array representation.
Examples:
>>> _encode_array([])
'[0]:'
>>> _encode_array([1, 2, 3])
'[3]: 1,2,3'
"""
prefix = key_prefix if key_prefix else ""
if not arr:
return f"{prefix}[0]:"
# Check if all elements are dicts with same keys (columnar opportunity)
if len(arr) >= 1 and all(isinstance(item, dict) for item in arr):
columnar = _try_columnar_encoding(arr, indent=indent, key_prefix=key_prefix)
if columnar is not None:
return columnar
# Check if elements are simple (no nested structures)
if all(_is_simple_value(item) for item in arr):
encoded_items = [encode(item, indent=indent, _as_root=False) for item in arr]
return f"{prefix}[{len(arr)}]: " + ",".join(encoded_items)
# Complex array - use list item syntax with - prefix
lines = [f"{prefix}[{len(arr)}]:"]
child_ind = " " * (_INDENT_SIZE * (indent + 1))
for item in arr:
if _is_simple_value(item):
lines.append(f"{child_ind}- {encode(item, indent=indent+1, _as_root=False)}")
elif isinstance(item, dict):
if not item:
# Empty object as list item is just -
lines.append(f"{child_ind}-")
else:
# Object as list item
obj_lines = _encode_object_as_list_item(item, indent=indent + 1)
lines.extend(obj_lines)
elif isinstance(item, list):
# Nested array as list item
nested = _encode_array(item, indent=indent + 2, _as_root=False, key_prefix="")
nested_lines = nested.split("\n")
lines.append(f"{child_ind}- {nested_lines[0]}")
for nl in nested_lines[1:]:
lines.append(f"{child_ind} {nl}")
else:
lines.append(f"{child_ind}- {encode(item, indent=indent+1, _as_root=False)}")
return "\n".join(lines)
def _encode_object_as_list_item(obj: Dict[str, Any], indent: int) -> List[str]:
"""Encode an object as a list item with - prefix.
Per TOON spec §10:
- First field may appear on hyphen line
- If first field is a columnar array, emit "- key[N]{fields}:" on hyphen line
Args:
obj: Dictionary to encode.
indent: Current indentation level.
Returns:
List of lines.
"""
lines = []
ind = " " * (_INDENT_SIZE * indent)
field_ind = " " * (_INDENT_SIZE * (indent + 1))
items = list(obj.items())
for i, (key, value) in enumerate(items):
encoded_key = _encode_key(key)
if isinstance(value, list) and value:
# Check if this is a columnar array (first field on hyphen line per §10)
if i == 0:
columnar = _try_columnar_encoding(value, indent=0, key_prefix="")
if columnar is not None:
# Emit columnar header on hyphen line: - key[N]{fields}:
# Extract header and rows from columnar encoding
columnar_lines = columnar.split("\n")
header = columnar_lines[0] # [N]{fields}:
rows = columnar_lines[1:]
lines.append(f"{ind}- {encoded_key}{header}")
for row in rows:
lines.append(f"{field_ind} {row.strip()}")
continue
# Non-columnar array or not first field
if i == 0:
lines.append(f"{ind}- {encoded_key}:")
else:
lines.append(f"{field_ind}{encoded_key}:")
encoded_value = encode(value, indent=indent + 2, _as_root=False)
for vline in encoded_value.split("\n"):
lines.append(f"{field_ind} {vline}")
elif isinstance(value, dict) and value:
# Nested object - put on separate lines
if i == 0:
lines.append(f"{ind}- {encoded_key}:")
else:
lines.append(f"{field_ind}{encoded_key}:")
encoded_value = encode(value, indent=indent + 2, _as_root=False)
for vline in encoded_value.split("\n"):
lines.append(f"{field_ind} {vline}")
else:
encoded_value = encode(value, indent=indent + 1, _as_root=False)
if i == 0:
lines.append(f"{ind}- {encoded_key}: {encoded_value}")
else:
lines.append(f"{field_ind}{encoded_key}: {encoded_value}")
return lines
def _is_simple_value(obj: Any) -> bool:
"""Check if value is simple (not nested dict/list).
Args:
obj: Value to check.
Returns:
True if value is a primitive type.
"""
return obj is None or isinstance(obj, (bool, int, float, str))
def _try_columnar_encoding(arr: List[Dict[str, Any]], *, indent: int = 0, key_prefix: str = "") -> Optional[str]:
"""Try to encode array of objects in columnar format.
Per TOON spec:
- All objects must have identical key sets
- All values must be primitives
- Field order follows first object's key encounter order
Args:
arr: Array of dictionaries.
indent: Current indentation level.
key_prefix: Key name prefix for the array.
Returns:
Columnar TOON string, or None if not suitable.
Examples:
>>> result = _try_columnar_encoding([{"a": 1, "b": 2}, {"a": 3, "b": 4}])
>>> print(result)
[2]{a,b}:
1,2
3,4
"""
if not arr:
return None
# Get keys from first object - preserve encounter order per spec
first_keys = list(arr[0].keys())
if not first_keys:
return None
first_keys_set = set(first_keys)
# Check all objects have same keys
for obj in arr[1:]:
if set(obj.keys()) != first_keys_set:
return None
# Check all values are simple (columnar doesn't work well with nested)
for obj in arr:
if not all(_is_simple_value(v) for v in obj.values()):
return None
# Build columnar format with encounter order (not sorted)
prefix = key_prefix if key_prefix else ""
header = f"{prefix}[{len(arr)}]" + "{" + ",".join(first_keys) + "}:"
# Rows indented one level below header (2 spaces)
# Note: Don't include outer indentation here - caller handles that
row_ind = " " * _INDENT_SIZE
rows = []
for obj in arr:
row_values = [encode(obj[k], indent=0, _as_root=False) for k in first_keys]
rows.append(f"{row_ind}{','.join(row_values)}")
return header + "\n" + "\n".join(rows)
def _encode_object(obj: Dict[str, Any], *, indent: int = 0, _as_root: bool = True) -> str:
"""Encode an object (dict) in TOON format.
Per TOON spec:
- Empty object at root: empty document (no output)
- Empty object as field: key: with no value
- Arrays in objects use key[N]: format
Args:
obj: Dictionary to encode.
indent: Current indentation level.
_as_root: Whether this is the root object (unused, for API consistency).
Returns:
TOON object representation.
Examples:
>>> _encode_object({})
''
>>> _encode_object({"a": 1})
'a: 1'
>>> print(_encode_object({"a": 1, "b": 2}))
a: 1
b: 2
"""
# Per spec: empty object at root is empty document
if not obj:
return ""
lines = []
for key, value in obj.items():
encoded_key = _encode_key(key)
if isinstance(value, list):
# Arrays use key[N]: format per spec
arr_encoded = _encode_array(value, indent=indent, _as_root=False, key_prefix=encoded_key)
lines.append(arr_encoded)
elif isinstance(value, dict):
if not value:
# Empty nested object: key: with nothing after
lines.append(f"{encoded_key}:")
else:
# Nested object
lines.append(f"{encoded_key}:")
encoded_value = _encode_object(value, indent=indent + 1, _as_root=False)
for vline in encoded_value.split("\n"):
lines.append(f"{' ' * _INDENT_SIZE}{vline}")
else:
encoded_value = encode(value, indent=indent, _as_root=False)
lines.append(f"{encoded_key}: {encoded_value}")
return "\n".join(lines)
# =============================================================================
# Decoder
# =============================================================================
def decode(toon_str: str) -> Any:
"""Decode a TOON string back to Python objects.
Args:
toon_str: TOON-formatted string.
Returns:
Decoded Python object.
Raises:
ValueError: If TOON string is malformed.
Examples:
>>> decode("null")
>>> decode("true")
True
>>> decode("false")
False
>>> decode("42")
42
>>> decode("3.14")
3.14
>>> decode("hello")
'hello'
>>> decode('"quoted"')
'quoted'
>>> decode("[3]: 1,2,3")
[1, 2, 3]
>>> decode("key: value")
{'key': 'value'}
"""
toon_str = toon_str.strip()
# Per spec: empty document is empty object
if not toon_str:
return {}
# Try primitives first
primitive = _try_decode_primitive(toon_str)
if primitive is not None:
return primitive[0]
# Check if this is an object with multiple fields at root level
# (as opposed to a single array)
if _looks_like_object(toon_str):
return _decode_object(toon_str)
# Try array (with or without key prefix)
if toon_str.startswith("[") or _ARRAY_HEADER_RE.match(toon_str):
return _decode_array(toon_str)
# Try object (key: value format)
if ":" in toon_str:
return _decode_object(toon_str)
# Unquoted string
return toon_str
def _looks_like_object(s: str) -> bool:
"""Determine if string looks like a TOON object with multiple fields.
This helps disambiguate between an array-as-object-field like:
items[5]: 1,2,3
name: test
vs a standalone array like:
[5]: 1,2,3,4,5
Args:
s: TOON string to check.
Returns:
True if string appears to be an object with multiple fields.
"""
lines = s.split("\n")
root_level_keys = 0
for line in lines:
stripped = line.strip()
if not stripped:
continue
# Check if line starts at root level (no leading whitespace in original)
if line and not line[0].isspace():
# Check if it looks like a key: value or key[N]: line
if ":" in stripped:
root_level_keys += 1
if root_level_keys > 1:
return True
# Also consider it an object if it has key[N]: format with key name
# at root level (vs just [N]:)
first_line = lines[0].strip() if lines else ""
arr_match = _ARRAY_HEADER_RE.match(first_line)
if arr_match and arr_match.group(1) and root_level_keys >= 1:
# Has a key prefix like "items[5]:" - this is an object field
return True
return False
def _try_decode_primitive(s: str) -> Optional[Tuple[Any, str]]:
"""Try to decode a primitive value from start of string.
Args:
s: String to parse.
Returns:
Tuple of (decoded_value, remaining_string) or None.
Examples:
>>> _try_decode_primitive("null")
(None, '')
>>> _try_decode_primitive("true")
(True, '')
>>> _try_decode_primitive("42")
(42, '')
>>> _try_decode_primitive("3.14")
(3.14, '')
"""
s = s.strip()
# Null
if s == "null" or s.startswith("null,") or s.startswith("null\n"):
return (None, s[4:].lstrip(",\n "))
# Boolean
if s == "true" or s.startswith("true,") or s.startswith("true\n"):
return (True, s[4:].lstrip(",\n "))
if s == "false" or s.startswith("false,") or s.startswith("false\n"):
return (False, s[5:].lstrip(",\n "))
# Quoted string
if s.startswith('"'):
return _decode_quoted_string(s)
# Number - use partial matching pattern to extract number from start
match = _NUMBER_PREFIX_RE.match(s)
if match:
num_str = match.group()
# Check that what follows is a delimiter or end of string
rest = s[len(num_str):]
if not rest or rest[0] in ",\n :]}":
remaining = rest.lstrip(",\n ")
if "." in num_str or "e" in num_str.lower():
return (float(num_str), remaining)
return (int(num_str), remaining)
return None
def _decode_quoted_string(s: str) -> Tuple[str, str]:
"""Decode a quoted string from TOON.
Per TOON spec, only these escapes are valid: \\\\ \\\" \\n \\r \\t
Any other escape sequence must be rejected.
Args:
s: String starting with quote.
Returns:
Tuple of (decoded_string, remaining_string).
Raises:
ValueError: If string is malformed or contains invalid escapes.
Examples:
>>> _decode_quoted_string('"hello"')
('hello', '')
>>> _decode_quoted_string('"with\\\\nescapes"')
('with\\nescapes', '')
"""
if not s.startswith('"'):
raise ValueError("Expected quoted string")
result = []
i = 1
while i < len(s):
char = s[i]
if char == '"':
return ("".join(result), s[i + 1:].lstrip(",\n "))
if char == "\\":
if i + 1 >= len(s):
raise ValueError("Unterminated escape sequence")
next_char = s[i + 1]
if next_char == "n":
result.append("\n")
elif next_char == "r":
result.append("\r")
elif next_char == "t":
result.append("\t")
elif next_char == "\\":
result.append("\\")
elif next_char == '"':
result.append('"')
else:
# Per spec: reject invalid escape sequences
raise ValueError(f"Invalid escape sequence: \\{next_char}")
i += 2
else:
result.append(char)
i += 1
raise ValueError("Unterminated string")
def _decode_array(s: str) -> List[Any]:
"""Decode a TOON array.
Args:
s: TOON array string starting with '[' or key[.
Returns:
Decoded Python list.
Examples:
>>> _decode_array("[0]:")
[]
>>> _decode_array("[3]: 1,2,3")
[1, 2, 3]
>>> _decode_array("[2]{x,y}:\\n 1,2\\n 3,4")
[{'x': 1, 'y': 2}, {'x': 3, 'y': 4}]
"""
match = _ARRAY_HEADER_RE.match(s)
if not match:
raise ValueError(f"Invalid array format: {s[:50]}")
# key_name = match.group(1) # May be None for root arrays
count = int(match.group(2))
keys_str = match.group(3)
# Get remaining content, preserving indentation
remaining = s[match.end():]
# Strip just the leading space/newline after colon (": " or ":\n")
if remaining.startswith(" "):
remaining = remaining[1:]
elif remaining.startswith("\n"):
remaining = remaining[1:]
remaining_stripped = remaining.strip()
if count == 0:
return []
# Columnar format with keys
if keys_str:
# Detect delimiter from header (comma, pipe, or tab per spec)
delimiter = _detect_delimiter(keys_str)
keys = [k.strip() for k in keys_str.split(delimiter)]
return _decode_columnar_array(remaining_stripped, count, keys, delimiter)
# Check for list item syntax (lines starting with -)
if remaining_stripped.startswith("-") or "\n-" in remaining or "\n -" in remaining:
return _decode_list_item_array(remaining, count)
# Simple array format
return _decode_simple_array(remaining_stripped, count)
def _decode_list_item_array(s: str, count: int) -> List[Any]:
"""Decode array using list item syntax (- prefix).
Per TOON spec §9.4/§10, list items can be:
- Simple values: - value
- Empty objects: -
- Object with first field: - key: value
- Nested arrays: - [N]: values
Args:
s: Array content with list items.
count: Expected number of elements.
Returns:
Decoded list.
"""
result = []
lines = s.split("\n")
i = 0
while i < len(lines) and len(result) < count:
line = lines[i]
stripped = line.strip()
if not stripped:
i += 1
continue
# Find the base indentation of this list item
item_indent = len(line) - len(line.lstrip())
if stripped.startswith("- "):
item_content = stripped[2:]
# Check if this is a standalone array (no key prefix)
if item_content.startswith("["):
result.append(_decode_array(item_content))
i += 1
# Check if this is an object field (key: value or key[N]: per §10)
elif ":" in item_content:
# This is an object list item - gather all lines at item_indent + 2 or more
obj_lines = [item_content]
i += 1
child_indent = item_indent + _INDENT_SIZE
while i < len(lines):
next_line = lines[i]
if not next_line.strip():
i += 1
continue
next_indent = len(next_line) - len(next_line.lstrip())
# Stop if we hit another list item at same level or less indentation
if next_indent <= item_indent:
break
# Add the content with adjusted indentation
if next_indent >= child_indent:
obj_lines.append(next_line[child_indent:] if len(next_line) >= child_indent else next_line.strip())
i += 1
# Decode the object
obj_text = "\n".join(obj_lines)
result.append(_decode_object(obj_text))
else:
# Simple value
primitive = _try_decode_primitive(item_content)
if primitive is not None:
result.append(primitive[0])
else:
result.append(item_content)
i += 1
elif stripped == "-":
# Empty object
result.append({})
i += 1
else:
# Unexpected line, skip
i += 1
return result
def _decode_simple_array(s: str, count: int) -> List[Any]:
"""Decode a simple (non-columnar) TOON array.
Args:
s: Array content after header.
count: Expected number of elements.
Returns:
Decoded list.
"""
if not s:
return []
result = []
remaining = s
for _ in range(count):
if not remaining:
break
remaining = remaining.strip()
# Try to parse a value
primitive = _try_decode_primitive(remaining)
if primitive is not None:
result.append(primitive[0])
remaining = primitive[1]
elif remaining.startswith("["):
# Nested array - find matching bracket
nested, remaining = _extract_nested_structure(remaining, "[", "]")
result.append(_decode_array(nested))
elif remaining.startswith("{"):
# Nested object
nested, remaining = _extract_nested_structure(remaining, "{", "}")
result.append(_decode_object(nested))
else:
# Unquoted string - read until comma or newline
end = len(remaining)
for i, c in enumerate(remaining):
if c in ",\n":
end = i
break
value = remaining[:end].strip()
result.append(value if value else None)
remaining = remaining[end:].lstrip(",\n ")
return result
def _detect_delimiter(keys_str: str) -> str:
"""Detect the delimiter used in columnar array header.
Per TOON spec v3.0, columnar headers can use comma, pipe, or tab as delimiter.
Args:
keys_str: The keys portion of the header (e.g., "a,b" or "a|b" or "a\\tb").
Returns:
The detected delimiter character.
"""
# Check for pipe first (more specific)
if "|" in keys_str:
return "|"
# Check for tab
if "\t" in keys_str:
return "\t"
# Default to comma
return ","
def _decode_columnar_array(s: str, count: int, keys: List[str], delimiter: str = ",") -> List[Dict[str, Any]]:
"""Decode a columnar TOON array.
Args:
s: Array content (rows).
count: Expected number of rows.
keys: Column keys.
delimiter: The delimiter character used to separate values.
Returns:
List of dictionaries.
"""
result = []
lines = s.strip().split("\n")
for line in lines[:count]:
line = line.strip()
if not line:
continue
values = _split_row_values(line, len(keys), delimiter)
obj = {}
for i, key in enumerate(keys):
if i < len(values):
# Decode each value
val_str = values[i].strip()
primitive = _try_decode_primitive(val_str)
if primitive is not None:
obj[key] = primitive[0]
else:
obj[key] = val_str
else:
obj[key] = None
result.append(obj)
return result
def _split_row_values(line: str, _expected_count: int, delimiter: str = ",") -> List[str]:
"""Split a columnar row into values, respecting quotes.
Args:
line: Row string.
_expected_count: Expected number of values (unused, for potential validation).
delimiter: The delimiter character to split on.
Returns:
List of value strings.
"""
values = []
current = []
in_quotes = False
escape = False
for char in line:
if escape:
current.append(char)
escape = False
elif char == "\\":
current.append(char)
escape = True
elif char == '"':
current.append(char)
in_quotes = not in_quotes
elif char == delimiter and not in_quotes:
values.append("".join(current))
current = []
else:
current.append(char)
if current:
values.append("".join(current))
return values
def _decode_object(s: str) -> Dict[str, Any]:
"""Decode a TOON object.
Args:
s: TOON object string.
Returns:
Decoded Python dictionary.
Examples:
>>> _decode_object("")
{}
>>> _decode_object("a: 1")
{'a': 1}
>>> _decode_object("a: 1\\nb: 2")
{'a': 1, 'b': 2}
"""
s = s.strip()
if not s:
return {}
result = {}
lines = s.split("\n")
i = 0
while i < len(lines):
line = lines[i]
stripped = line.strip()
if not stripped:
i += 1
continue
# Check for array header format: key[N]: at base indent
# Format: key[N]: or key[N]{fields}:
arr_match = _ARRAY_HEADER_RE.match(stripped)
if arr_match and arr_match.group(1):
key = arr_match.group(1)
# Unquote key if it's quoted
if key.startswith('"') and key.endswith('"'):
key = _decode_quoted_string(key)[0]
# Collect all lines for this array
arr_lines = [stripped]
i += 1
if i < len(lines) and lines[i].strip():
base_indent = len(lines[i]) - len(lines[i].lstrip())
else:
base_indent = 0
while i < len(lines):
next_line = lines[i]
if not next_line.strip():
i += 1
continue
next_indent = len(next_line) - len(next_line.lstrip())
# Stop if we reach a line at base level that's a new key
if next_indent == 0 and (":" in next_line or _ARRAY_HEADER_RE.match(next_line.strip())):
break
if next_indent < base_indent and next_line.strip():
break
arr_lines.append(next_line)
i += 1
result[key] = _decode_array("\n".join(arr_lines))
continue
# Parse key: value (or key: for empty value / nested object)
if ":" in stripped:
# Find the colon - handle quoted keys
if stripped.startswith('"'):
# Quoted key - find end quote then colon
try:
key, rest = _decode_quoted_string(stripped)
if rest.startswith(":"):
value_str = rest[1:].strip()
else:
i += 1
continue
except ValueError:
i += 1
continue
else:
colon_idx = stripped.index(":")
key = stripped[:colon_idx].strip()
value_str = stripped[colon_idx + 1:].strip()
# Check if value continues on next lines (indented)
if not value_str and i + 1 < len(lines):
# Find base indent from first non-empty line after this one
base_indent = 0
for j in range(i + 1, len(lines)):
if lines[j].strip():
base_indent = len(lines[j]) - len(lines[j].lstrip())
break
# If next non-empty line is at root level, this is an empty value
if base_indent == 0:
result[key] = {}
i += 1
continue
# Multi-line value - gather indented lines
nested_lines = []
i += 1
while i < len(lines):
next_line = lines[i]
if not next_line.strip():
nested_lines.append("")
i += 1
continue
next_indent = len(next_line) - len(next_line.lstrip())
if next_indent < base_indent:
break
# Remove the base indentation for nested content
if len(next_line) >= base_indent:
nested_lines.append(next_line[base_indent:])
else:
nested_lines.append(next_line.strip())
i += 1
value_str = "\n".join(nested_lines)
result[key] = decode(value_str)
else:
# Single line value
primitive = _try_decode_primitive(value_str)
if primitive is not None:
result[key] = primitive[0]
elif value_str.startswith("["):
result[key] = _decode_array(value_str)
else:
result[key] = value_str if value_str else {}
i += 1
else:
i += 1
return result
def _extract_nested_structure(s: str, open_char: str, close_char: str) -> Tuple[str, str]:
"""Extract a nested structure (array or object) from string.
Args:
s: String starting with open_char.
open_char: Opening character ('[' or '{').
close_char: Closing character (']' or '}').
Returns:
Tuple of (nested_content, remaining_string).
"""
depth = 0
in_quotes = False
escape = False
for i, char in enumerate(s):
if escape:
escape = False
continue
if char == "\\":
escape = True
continue
if char == '"':
in_quotes = not in_quotes
continue
if in_quotes:
continue
if char == open_char:
depth += 1
elif char == close_char:
depth -= 1
if depth == 0:
return (s[: i + 1], s[i + 1:].lstrip(",\n "))
return (s, "")
# =============================================================================
# Utility Functions
# =============================================================================
def estimate_token_savings(json_str: str) -> Tuple[int, int, float]:
"""Estimate token savings from JSON to TOON conversion.
This is a rough estimate based on byte count, not actual tokenization.
Actual savings depend on the specific tokenizer used.
Args:
json_str: Original JSON string.
Returns:
Tuple of (json_bytes, toon_bytes, savings_percent).
Examples:
>>> import json
>>> data = {"users": [{"id": 1, "name": "alice"}, {"id": 2, "name": "bob"}]}
>>> json_str = json.dumps(data)
>>> json_len, toon_len, savings = estimate_token_savings(json_str)
>>> savings > 0
True
"""
import json
try:
obj = json.loads(json_str)
toon_str = encode(obj)
# Use byte length for accurate measurement
json_len = len(json_str.encode("utf-8"))
toon_len = len(toon_str.encode("utf-8"))
savings = ((json_len - toon_len) / json_len) * 100 if json_len > 0 else 0
return (json_len, toon_len, savings)
except Exception:
return (len(json_str), len(json_str), 0.0)
