Skip to contents

XLNet: permutation-based autoregressive pretraining

Source: R/geron_w4d_native.R
morie_geron_xlnet.Rd

Draws a factorisation order from an LCG Fisher-Yates shuffle, builds two-stream masks (morie_geron_permutation_masks), scores each conditional from the visible (query-stream) context under an embedding-based softmax head.

Usage

morie_geron_xlnet(X, n_layers = 1, vocab_size = NULL, d_model = 8, seed = 0)

Arguments

X

Integer (0-based) token ids, length >= 2.

n_layers, vocab_size, d_model, seed

As in Python original.

Value

list with permutation, content_mask, query_mask, logprobs, total_logprob, perplexity, embeddings, n_layers, estimate, n, method.

Examples

V <- c(1, 2, 3, 4, 5, 6, 7, 8)
morie_geron_xlnet(V)
#> $permutation
#> [1] 2 0 5 6 3 4 7 1
#> 
#> $content_mask
#>      [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8]
#> [1,]    1    0    1    0    0    0    0    0
#> [2,]    1    1    1    1    1    1    1    1
#> [3,]    0    0    1    0    0    0    0    0
#> [4,]    1    0    1    1    0    1    1    0
#> [5,]    1    0    1    1    1    1    1    0
#> [6,]    1    0    1    0    0    1    0    0
#> [7,]    1    0    1    0    0    1    1    0
#> [8,]    1    0    1    1    1    1    1    1
#> 
#> $query_mask
#>      [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8]
#> [1,]    0    0    1    0    0    0    0    0
#> [2,]    1    0    1    1    1    1    1    1
#> [3,]    0    0    0    0    0    0    0    0
#> [4,]    1    0    1    0    0    1    1    0
#> [5,]    1    0    1    1    0    1    1    0
#> [6,]    1    0    1    0    0    0    0    0
#> [7,]    1    0    1    0    0    1    0    0
#> [8,]    1    0    1    1    1    1    1    0
#> 
#> $logprobs
#> [1] -3.784256 -1.710846 -2.197225 -2.581492 -2.054310 -1.952978 -2.202139
#> [8] -1.986245
#> 
#> $total_logprob
#> [1] -18.46949
#> 
#> $perplexity
#> [1] 10.0612
#> 
#> $embeddings
#>              [,1]        [,2]        [,3]        [,4]       [,5]        [,6]
#>  [1,]  0.28007122  0.01555620  0.16359509  0.59345099  0.4751312  0.22765792
#>  [2,]  0.02950148  0.41546576 -0.37664962 -0.23196298  0.2893568  0.04971004
#>  [3,] -0.50565744 -0.97516525 -0.46912933  0.96712020  0.2244745 -0.09381007
#>  [4,] -0.95569651 -0.25832278 -0.24630179  0.98242188  0.2505963  0.21909847
#>  [5,]  0.12978176 -0.54954254 -0.82927218  0.19074640  0.6314130 -0.83553515
#>  [6,] -0.13610543 -0.41150384  0.03548809 -0.72336261 -0.6812740  0.80125287
#>  [7,]  0.22951014 -0.16262328 -0.03557735 -0.91996606 -0.0280219 -0.67473290
#>  [8,] -0.08872113 -0.06064849 -0.44982857 -0.43107305  0.6027852 -0.55194077
#>  [9,]  0.75266467 -0.36125193 -0.40014811 -0.06298471  0.8444670  0.90083424
#>              [,7]       [,8]
#>  [1,]  0.77023207 -0.9865015
#>  [2,]  0.08341078 -0.1993752
#>  [3,] -0.73281494 -0.3138057
#>  [4,] -0.65532420  0.9532003
#>  [5,] -0.67112797 -0.8095779
#>  [6,] -0.10158778 -0.9224183
#>  [7,]  0.69836549 -0.7045004
#>  [8,] -0.73398712  0.5612510
#>  [9,] -0.41837142  0.7883224
#> 
#> $n_layers
#> [1] 1
#> 
#> $estimate
#> [1] -18.46949
#> 
#> $n
#> [1] 8
#> 
#> $method
#> [1] "Permutation LM: two-stream masks over a sampled factorisation order with softmax conditionals"
#>