Eccelerators.Tokenization

src/Unigram/MetaspaceDecoder.lvt

namespace Eccelerators.Tokenization.Unigram

using Eccelerators.Tokenization.Storage

/**
 * Array-backed compatibility facade for StoredMetaspaceDecoder.
 * Owns local output storage and the shared decoder; borrows the vocabulary.
 * Reset before the first Append and after failure; output bytes are valid only within Length.
 */
component MetaspaceDecoder<MAX_OUTPUT: int, MAX_PIECE: int, VOCAB: IUnigramVocabulary>
{
	// Owned output storage and shared decoding core; vocabulary is borrowed.
	storage: ArrayStorage<byte, MAX_OUTPUT>
	core: StoredMetaspaceDecoder<MAX_OUTPUT, MAX_PIECE, VOCAB, ArrayStorage<byte, MAX_OUTPUT>>

	/**
	 * Borrows the vocabulary and owns local output storage plus the shared decoding core.
	 * Call Reset before the first Append to initialize first-piece handling.
	 */
	new(vocab: VOCAB)
	{
		storage = new ArrayStorage<byte, MAX_OUTPUT>()
		core = new StoredMetaspaceDecoder<MAX_OUTPUT, MAX_PIECE, VOCAB, ArrayStorage<byte, MAX_OUTPUT>>(vocab, storage)
	}

	/**
	 * Starts a new decoded sequence, enabling first-piece marker removal and clearing sticky failure.
	 * Call before the first Append and each new sequence. Storage and vocabulary state are not cleared.
	 */
	public fn Reset()
	{
		core.Reset()
	}

	/**
	 * Appends one vocabulary piece and waits for its output writes to complete.
	 * Special/unassigned IDs are skipped without consuming first-piece state. Requires valid UTF-8
	 * pieces of at most MAX_PIECE bytes. Bad records, insufficient storage or MAX_OUTPUT overflow
	 * return false and invalidate all output until Reset; no partial result may be consumed.
	 */
	public fn Append(token: int) bool
	{
		return core.Append(token)
	}

	/**
	 * Returns the live decoded UTF-8 byte count, or zero after Reset or failure.
	 */
	public fn Length() int
	{
		return core.Length()
	}

	/**
	 * Returns a live decoded byte at index 0..Length()-1, waiting for storage completion.
	 * Returns zero outside the live range or after failure.
	 */
	public fn Read(index: int) byte
	{
		return core.Read(index)
	}

	/**
	 * Reports sticky vocabulary, storage-capacity or output-overflow failure since Reset.
	 */
	public fn Failed() bool
	{
		return core.Failed()
	}
}