related URL: https://github.com/sephiroce/kmlm/blob/master/kmlm/base/utils.py
import numpy as np
# ppl = exp((sum(logprob) * - 1) / words)
# if batch is equal to 1 then,
# 4 words, logprob= -27.016243, ppl= 222.126892
# ppl = exp((-27.016243 * -1) / (4 + 1)) ==> considering eos
# example inputs
length=5
# softmax, it may contain zero probability for the target word
logit=[
[0.0, -5, 10, 0.0],
[15,0, 20,0.0],
[0,25,1,30],
[100,-100.0,35,0],
[0.00001,0,7,0],
[100,200,300,400],
[100,200,300,400]
]
label= [1.0, 0.0, 3.0, 0.0, 0.0, 0.0, 0.0]
logits = [] #KEY_INPUT_SEQ: INPUT_SEQUENCE
labels = [] #KEY_TARGET_SEQ: TARGET_SEQUENCE
seq_len = [] #KEY_SEQ_LEN: SEQUENCE_LENGTH
batch_size = 5
for _ in range(batch_size):
logits.append(logit) #[batch_size, 5, 4]
labels.append(label) #[batch_size, 5]
seq_len.append(length) #[batch_size]
# calculating a ppl by using tensorflow
import tensorflow as tf
logits = tf.constant(logits)
seq_len = tf.constant(seq_len)
"""
logprobs = tf.nn.log_softmax(logits)
filtered_logprob = tf.multiply(logprobs,
tf.one_hot(labels,
tf.shape(logits)[2]))
logprob = tf.reduce_sum(filtered_logprob, axis=2) * -1
"""
# calculates an batch-wise accumulated log probability
full_logprob = \
tf.nn.sparse_softmax_cross_entropy_with_logits(labels=\
tf.cast(labels, tf.int32),
logits=logits)
# generating sequence mask to handle variable length of inputs
# in this case actually squeeze is not needed! but my lm toolkit
# needs it. (I'll figure it out someday later..)
seq_mask = tf.squeeze(tf.sequence_mask(seq_len,
&n bsp; maxlen=\
tf.shape(full_logprob)[1],
dtype=tf.float32))
logprob = tf.reduce_sum(full_logprob * seq_mask)
# calculates ppl
ppl = tf.math.exp(logprob / tf.cast(tf.reduce_sum(seq_len), tf.float32))
expanded_words = tf.reduce_sum(seq_len)
with tf.compat.v1.Session() as sess:
w, lp, p = sess.run([expanded_words, logprob, ppl])
print("%d words, logprob= %.6f, ppl= %.6f"%(w - batch_size, -lp, p))