Encoder-only transformer (BERT-family), resolved to exact param counts (hmencox)

Usage

morie_geron_encoder_only(
  X,
  n_layers = 12,
  n_heads = 12,
  d_model = 768,
  vocab_size = 30522,
  max_len = 512,
  d_ff = NULL,
  n_segments = 2,
  n_classes = NULL
)

Arguments

X: Token ids (raw, before CLS/SEP).
n_layers, n_heads: int.
d_model, vocab_size, max_len: int.
d_ff: FFN width.
n_segments: Segment embedding rows.
n_classes: Optional CLS head.

Value

List with total_params, block_params, embedding_params, head_params, attention_mask, seq_len, cls_index.

Examples

V <- c(1, 2, 3, 4, 5, 6, 7, 8)
morie_geron_encoder_only(V)
#> $total_params
#> [1] 108891648
#> 
#> $block_params
#> [1] 7087872
#> 
#> $per_block
#> $per_block$self_attention
#> [1] 2362368
#> 
#> $per_block$ffn
#> [1] 4722432
#> 
#> $per_block$layer_norms
#> [1] 3072
#> 
#> $per_block$total
#> [1] 7087872
#> 
#> 
#> $embedding_params
#> [1] 23835648
#> 
#> $head_params
#> [1] 0
#> 
#> $attention_mask
#>        [,1]  [,2]  [,3]  [,4]  [,5]  [,6]  [,7]  [,8]  [,9] [,10]
#>  [1,] FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
#>  [2,] FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
#>  [3,] FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
#>  [4,] FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
#>  [5,] FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
#>  [6,] FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
#>  [7,] FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
#>  [8,] FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
#>  [9,] FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
#> [10,] FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
#> 
#> $seq_len
#> [1] 10
#> 
#> $raw_len
#> [1] 8
#> 
#> $cls_index
#> [1] 0
#> 
#> $sep_index
#> [1] 9
#> 
#> $is_bidirectional
#> [1] TRUE
#> 
#> $d_head
#> [1] 64
#> 
#> $n_layers
#> [1] 12
#> 
#> $n_heads
#> [1] 12
#> 
#> $d_model
#> [1] 768
#> 
#> $vocab_size
#> [1] 30522
#> 
#> $max_len
#> [1] 512
#> 
#> $estimate
#> [1] 108891648
#> 
#> $n
#> [1] 12
#> 
#> $method
#> [1] "encoder-only transformer resolved to exact parameter counts; block cost delegated to hmdctr"
#>