WordPiece tokenizer: maximum likelihood subword segmentation

Greedily merges the pair maximising freq(AB)/(freq(A)*freq(B)) (not raw frequency, that is BPE); continuation pieces carry "##"; final segmentation is greedy longest-match-first.

Usage

morie_geron_wordpiece_tokenizer(corpus, vocab_size = 50)

Arguments

corpus

Character string or vector (whitespace-tokenised).

vocab_size

Target vocabulary size (>= alphabet size).

Value

list with vocab, merges, scores, tokenize (function), alphabet, word_counts, estimate, n, method.

Examples

S <- c("a", "b", "c")
morie_geron_wordpiece_tokenizer(S)
#> $vocab
#> [1] "##" "a"  "b"  "c" 
#> 
#> $merges
#> list()
#> 
#> $scores
#> numeric(0)
#> 
#> $tokenize
#> function (word) 
#> {
#>     w <- as.character(word)
#>     chars <- strsplit(w, "")[[1]]
#>     n <- length(chars)
#>     out <- character(0)
#>     start <- 1
#>     while (start <= n) {
#>         end <- n
#>         piece <- NULL
#>         while (start <= end) {
#>             cand <- if (start == 1) 
#>                 paste(chars[start:end], collapse = "")
#>             else paste0("##", paste(chars[start:end], collapse = ""))
#>             if (cand %in% vocab_set) {
#>                 piece <- cand
#>                 break
#>             }
#>             end <- end - 1
#>         }
#>         if (is.null(piece)) {
#>             return("[UNK]")
#>         }
#>         out <- c(out, piece)
#>         start <- end + 1
#>     }
#>     out
#> }
#> <bytecode: 0x5621340ed4b0>
#> <environment: 0x56214874a1e0>
#> 
#> $alphabet
#> [1] "##" "a"  "b"  "c" 
#> 
#> $word_counts
#> $word_counts$a
#> [1] 1
#> 
#> $word_counts$b
#> [1] 1
#> 
#> $word_counts$c
#> [1] 1
#> 
#> 
#> $estimate
#> [1] 4
#> 
#> $n
#> [1] 3
#> 
#> $method
#> [1] "WordPiece: greedy likelihood-scored merges freq(AB)/(freq(A)freq(B)), longest-match segmentation"
#>