dart_bert_tokenizer

Features

Installation

dependencies:
 dart_bert_tokenizer: ^1.1.0

Quick Start

import 'package:dart_bert_tokenizer/dart_bert_tokenizer.dart';
void main() {
 // Load tokenizer
 final tokenizer = WordPieceTokenizer.fromVocabFileSync('vocab.txt');
 // Encode text
 final encoding = tokenizer.encode('Hello, world!');
 print(encoding.tokens); // [[CLS], hello, ,, world, !, [SEP]]
 print(encoding.ids); // [101, 7592, 1010, 2088, 999, 102]
 // Decode back to text
 final text = tokenizer.decode(encoding.ids, skipSpecialTokens: true);
 print(text); // hello , world !
}

Loading from tokenizer.json

// From file (async)
final tokenizer = await WordPieceTokenizer.fromTokenizerJson('tokenizer.json');
// From file (sync)
final tokenizer = WordPieceTokenizer.fromTokenizerJsonSync('tokenizer.json');
// From JSON string (e.g., embedded asset)
final tokenizer = WordPieceTokenizer.fromTokenizerJsonString(jsonString);
// Override extracted config if needed
final tokenizer = WordPieceTokenizer.fromTokenizerJsonSync(
 'tokenizer.json',
 configOverride: WordPieceConfig(lowercase: false),
);

Usage

Single Text Encoding

final encoding = tokenizer.encode('Hello world');
print(encoding.tokens); // Token strings
print(encoding.ids); // Token IDs (Int32List)
print(encoding.attentionMask); // Attention mask (Uint8List)
print(encoding.typeIds); // Type IDs (Uint8List)
print(encoding.offsets); // Character offsets [(start, end), ...]
print(encoding.wordIds); // Word indices
print(encoding.sequenceIds); // Sequence indices (0, 1, or null)
// Without special tokens
final raw = tokenizer.encode('Hello', addSpecialTokens: false);

Sentence Pair Encoding

// For QA, NLI, sentence similarity tasks
final encoding = tokenizer.encodePair(
 'What is machine learning?',
 'Machine learning is a subset of AI.',
);
print(encoding.typeIds); // [0,0,0,0,0,0, 1,1,1,1,1,1,1]
print(encoding.sequenceIds); // [null,0,0,0,0,null, 1,1,1,1,1,1,null]

Batch Encoding

// Sequential batch
final encodings = tokenizer.encodeBatch(['Hello', 'World', 'Test']);
// Parallel batch (uses Isolates for batches >= 8)
final encodings = await tokenizer.encodeBatchParallel(texts);
// Pair batch
final pairs = [('Q1', 'A1'), ('Q2', 'A2')];
final encodings = tokenizer.encodePairBatch(pairs);

Padding

// Fluent API
final tokenizer = WordPieceTokenizer.fromVocabFileSync('vocab.txt')
 ..enablePadding(length: 512, direction: PaddingDirection.right);
// Or pad to longest in batch
tokenizer.enablePadding(); // Auto-pads to longest
// Manual padding
final padded = encoding.withPadding(
 targetLength: 128,
 padTokenId: tokenizer.vocab.padTokenId,
 padOnRight: true,
);
// Pad to multiple of N
final padded = encoding.withPaddingToMultipleOf(
 multiple: 8,
 padTokenId: tokenizer.vocab.padTokenId,
);

Truncation

// Fluent API
final tokenizer = WordPieceTokenizer.fromVocabFileSync('vocab.txt')
 ..enableTruncation(maxLength: 512, direction: TruncationDirection.right);
// Manual truncation
final truncated = encoding.withTruncation(maxLength: 64);
// Truncation strategies for pairs
tokenizer.encodePair(textA, textB,
 maxLength: 128,
 truncationStrategy: TruncationStrategy.longestFirst,
);

Offset Mapping

final encoding = tokenizer.encode('Hello world');
// Character position -> Token index
final tokenIdx = encoding.charToToken(6); // 'w' -> token index
// Token index -> Character span
final (start, end) = encoding.tokenToChars(1)!; // token -> (0, 5)
// Word index -> Token span
final (startToken, endToken) = encoding.wordToTokens(0)!;
// Token -> Word index
final wordIdx = encoding.tokenToWord(1);
// Token -> Sequence index (0, 1, or null for special tokens)
final seqIdx = encoding.tokenToSequence(1);

Vocabulary Access

print(tokenizer.vocab.size); // 30522
print(tokenizer.vocab.clsTokenId); // 101
print(tokenizer.vocab.sepTokenId); // 102
print(tokenizer.vocab.padTokenId); // 0
print(tokenizer.vocab.unkTokenId); // 100
print(tokenizer.vocab.maskTokenId); // 103
// Token <-> ID conversion
tokenizer.convertTokensToIds(['hello', 'world']); // [7592, 2088]
tokenizer.convertIdsToTokens([7592, 2088]); // ['hello', 'world']
// Check if token exists
tokenizer.vocab.contains('hello'); // true

Decoding

// Decode with special tokens
final text = tokenizer.decode(encoding.ids, skipSpecialTokens: false);
// Decode without special tokens (default: true)
final text = tokenizer.decode(encoding.ids);
// Batch decode
final texts = tokenizer.decodeBatch(idsBatch);

ONNX Runtime Integration

import 'package:dart_bert_tokenizer/dart_bert_tokenizer.dart';
import 'dart:typed_data';
final tokenizer = WordPieceTokenizer.fromVocabFileSync('vocab.txt')
 ..enableTruncation(maxLength: 512);
final encoding = tokenizer.encode('search_query: What is ML?');
// Encoding.ids is already Int32List, convert to Int64List for ONNX
final inputIds = Int64List.fromList(encoding.ids);
final attentionMask = Int64List.fromList(encoding.attentionMask);
// Pass to ONNX session
// final outputs = await session.run({
// 'input_ids': inputIds,
// 'attention_mask': attentionMask,
// });

Configuration

final tokenizer = WordPieceTokenizer(
 vocab: vocab,
 config: WordPieceConfig(
 lowercase: true, // Convert to lowercase (default: true)
 stripAccents: true, // Remove accents (default: true)
 handleChineseChars: true, // Space around Chinese chars (default: true)
 subwordPrefix: '##', // Subword prefix (default: '##')
 maxWordLength: 200, // Max word length before [UNK] (default: 200)
 addClsToken: true, // Add [CLS] token (default: true)
 addSepToken: true, // Add [SEP] token (default: true)
 ),
);

API Reference

WordPieceTokenizer

Encoding

Performance

Vocabulary Files

Testing

# Run all tests (328 tests)
dart test
# Run specific test file
dart test test/hf_compatible_test.dart
# Run benchmarks
dart run benchmark/performance_benchmark.dart
# Run HuggingFace compatibility benchmark
dart run benchmark/hf_compatibility_benchmark.dart

HuggingFace Compatibility Verification

# Run HuggingFace compatibility benchmark (85 tests, 100% accuracy)
dart run benchmark/hf_compatibility_benchmark.dart
# Regenerate benchmark expected values (requires Python + tokenizers)
pip install tokenizers
python scripts/generate_hf_benchmark_data.py

License