Bayesian A/B Testing

import warnings warnings.filterwarnings("ignore") import json from js import Bokeh, JSON from bokeh.embed import json_item from bokeh.plotting import figure #import arviz as az #az.rcParams["plot.backend"] = "bokeh" import os, sys import numpy as np from scipy import stats import matplotlib.pyplot as plt import matplotlib.tri as tri sys.stderr = open(os.devnull, "w") def relative_increase(a,b): assert b!=0, "denominator is 0" return (a-b)/b def run_dummy(vA, vB, cA, cB): Element("out-cA").element.innerHTML = cA Element("out-cB").element.innerHTML = cB Element("out-vA").element.innerHTML = vA Element("out-vB").element.innerHTML = vB def run_model(pr_alphaA, pr_betaA, pr_alphaB, pr_betaB, vA, vB, cA, cB): visitors_to_A = int(vA) visitors_to_B = int(vB) conversions_from_A = int(cA) conversions_from_B = int(cB) alpha_priorA = int(pr_alphaA) beta_priorA = int(pr_betaA) alpha_priorB = int(pr_alphaB) beta_priorB = int(pr_betaB) samples = 30000 posterior_A = stats.beta(alpha_priorA+conversions_from_A,beta_priorA+visitors_to_A-conversions_from_A) posterior_B = stats.beta(alpha_priorB+conversions_from_B,beta_priorB+visitors_to_B-conversions_from_B) samples_posterior_A = posterior_A.rvs(samples) samples_posterior_B = posterior_B.rvs(samples) samples_diff = samples_posterior_A - samples_posterior_B posterior_rel_increase = np.divide(samples_diff,samples_posterior_B) posterior_better = samples_diff res_10 = np.round((posterior_rel_increase>0.1).mean(),3) res_20 = np.round((posterior_rel_increase>0.2).mean(),3) res_50 = np.round((posterior_rel_increase>0.5).mean(),3) A_betterthan_B = np.round((posterior_better>0.).mean(),3) #Element("out-vA").element.innerHTML = vA #Element("out-vB").element.innerHTML = vB #Element("out-cA").element.innerHTML = cA #Element("out-cB").element.innerHTML = cB str_10 = "Probability that a relative increase is more than 10%: "+str(res_10) str_20 = "Probability that a relative increase is more than 20%: "+str(res_20) str_50 = "Probability that a relative increase is more than 50%: "+str(res_50) str_better = "Probability that A is better than B: "+str(A_betterthan_B) Element("out-res_10").element.innerHTML = str_10 Element("out-res_20").element.innerHTML = str_20 Element("out-res_50").element.innerHTML = str_50 Element("out-better").element.innerHTML = str_better #---------------- plotting ---------------# """ p = figure(plot_width=400, plot_height=400) # add a circle renderer with x and y coordinates, size, color, and alpha p.circle([1, 2, 3, 4, 5], [6, 7, 2, 4, 5], size=15, line_color="navy", fill_color="orange", fill_alpha=0.5) #p_json = json.dumps(json_item(p, "myplot1")) #Bokeh.embed.embed_item(JSON.parse(p_json)) """ #--- prior f_pr = plt.figure() ax = plt.subplot(121) f_pr.set_figwidth(5) f_pr.set_figheight(2) x = np.linspace(0, 1, 100) y = stats.beta(alpha_priorA, beta_priorA).pdf(x) plt.plot(x, y,label="prior A") #plt.xlabel('probability') #plt.ylabel('density (prior)') plt.legend(loc="upper right") ax = plt.subplot(122) f_pr.set_figwidth(5) f_pr.set_figheight(2) x = np.linspace(0, 1, 100) y = stats.beta(alpha_priorB, beta_priorB).pdf(x) plt.plot(x, y,label="prior B") #plt.xlabel('probability') #plt.ylabel('density (prior)') plt.legend(loc="upper right") pyscript.write("myplot1",f_pr) #--- histogram of posteriors f_post = plt.figure() ax = plt.subplot(311) plt.xlim(0, .5) plt.hist(samples_posterior_A, histtype='stepfilled', bins=25, alpha=0.85, label="posterior of $p_A$", color="#A60628", density=True) #plt.vlines(true_p_A, 0, 80, linestyle="--", label="true $p_A$ (unknown)") plt.legend(loc="upper right") plt.title("Posterior distributions of $p_A$, $p_B$, and delta unknowns") ax = plt.subplot(312) plt.xlim(0, .5) plt.hist(samples_posterior_B, histtype='stepfilled', bins=25, alpha=0.85, label="posterior of $p_B$", color="#467821", density=True) #plt.vlines(true_p_B, 0, 80, linestyle="--", label="true $p_B$ (unknown)") plt.legend(loc="upper right") ax = plt.subplot(313) plt.hist(samples_diff, histtype='stepfilled', bins=30, alpha=0.85, label="posterior of delta", color="#7A68A6", density=True) #plt.vlines(true_p_A - true_p_B, 0, 60, linestyle="--", # label="true delta (unknown)") plt.vlines(0, 0, 60, color="black", alpha=0.2) plt.legend(loc="upper right") pyscript.write("myplot2",f_post) pr_alphaA = Element('input_box_alphaA').element.value pr_betaA = Element('input_box_betaA').element.value vA = Element('input_box_vA').element.value vB = Element('input_box_vB').element.value cA = Element('input_box_cA').element.value cB = Element('input_box_cB').element.value